def test_dag_course(self, mock_logging):
        """
        Test a course whose structure is not a tree.
        """
        store = XMLModuleStore(
            DATA_DIR,
            source_dirs=['xml_dag'],
            xblock_mixins=(XModuleMixin,),
        )
        course_key = store.get_courses()[0].id

        mock_logging.warning.assert_called_with(
            "%s has more than one definition", course_key.make_usage_key('discussion', 'duplicate_def')
        )

        shared_item_loc = course_key.make_usage_key('html', 'toyhtml')
        shared_item = store.get_item(shared_item_loc)
        parent = shared_item.get_parent()
        self.assertIsNotNone(parent, "get_parent failed to return a value")
        parent_loc = course_key.make_usage_key('vertical', 'vertical_test')
        self.assertEqual(parent.location, parent_loc)
        self.assertIn(shared_item, parent.get_children())
        # ensure it's still a child of the other parent even tho it doesn't claim the other parent as its parent
        other_parent_loc = course_key.make_usage_key('vertical', 'zeta')
        other_parent = store.get_item(other_parent_loc)
        # children rather than get_children b/c the instance returned by get_children != shared_item
        self.assertIn(shared_item_loc, other_parent.children)

    def test_cohort_config(self):
        """
        Check that cohort config parsing works right.

        Note: The cohort config on the CourseModule is no longer used.
        See openedx.core.djangoapps.course_groups.models.CourseCohortSettings.
        """
        modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])

        toy_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        course = modulestore.get_course(toy_id)

        # No config -> False
        self.assertFalse(course.is_cohorted)

        # empty config -> False
        course.cohort_config = {}
        self.assertFalse(course.is_cohorted)

        # false config -> False
        course.cohort_config = {'cohorted': False}
        self.assertFalse(course.is_cohorted)

        # and finally...
        course.cohort_config = {'cohorted': True}
        self.assertTrue(course.is_cohorted)

    def test_unicode(self):
        """Check that courses with unicode characters in filenames and in
        org/course/name import properly. Currently, this means: (a) Having
        files with unicode names does not prevent import; (b) if files are not
        loaded because of unicode filenames, there are appropriate
        exceptions/errors to that effect."""

        print("Starting import")
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['test_unicode'])
        courses = modulestore.get_courses()
        self.assertEquals(len(courses), 1)
        course = courses[0]

        print("course errors:")

        # Expect to find an error/exception about characters in "®esources"
        expect = "InvalidKeyError"
        errors = [
            (msg.encode("utf-8"), err.encode("utf-8"))
            for msg, err
            in modulestore.get_course_errors(course.id)
        ]

        self.assertTrue(any(
            expect in msg or expect in err
            for msg, err in errors
        ))
        chapters = course.get_children()
        self.assertEqual(len(chapters), 4)

    def test_poll_and_conditional_import(self):
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['conditional_and_poll'])

        course = modulestore.get_courses()[0]
        chapters = course.get_children()
        ch1 = chapters[0]
        sections = ch1.get_children()

        self.assertEqual(len(sections), 1)

        conditional_location = course.id.make_usage_key('conditional', 'condone')
        module = modulestore.get_item(conditional_location)
        self.assertEqual(len(module.children), 1)

        poll_location = course.id.make_usage_key('poll_question', 'first_poll')
        module = modulestore.get_item(poll_location)
        self.assertEqual(len(module.get_children()), 0)
        self.assertEqual(module.voted, False)
        self.assertEqual(module.poll_answer, '')
        self.assertEqual(module.poll_answers, {})
        self.assertEqual(
            module.answers,
            [
                {'text': u'Yes', 'id': 'Yes'},
                {'text': u'No', 'id': 'No'},
                {'text': u"Don't know", 'id': 'Dont_know'}
            ]
        )

    def test_poll_and_conditional_import(self):
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=["conditional_and_poll"])

        course = modulestore.get_courses()[0]
        chapters = course.get_children()
        ch1 = chapters[0]
        sections = ch1.get_children()

        self.assertEqual(len(sections), 1)

        location = course.location

        conditional_location = Location(location.tag, location.org, location.course, "conditional", "condone")
        module = modulestore.get_instance(course.id, conditional_location)
        self.assertEqual(len(module.children), 1)

        poll_location = Location(location.tag, location.org, location.course, "poll_question", "first_poll")
        module = modulestore.get_instance(course.id, poll_location)
        self.assertEqual(len(module.get_children()), 0)
        self.assertEqual(module.voted, False)
        self.assertEqual(module.poll_answer, "")
        self.assertEqual(module.poll_answers, {})
        self.assertEqual(
            module.answers,
            [{"text": u"Yes", "id": "Yes"}, {"text": u"No", "id": "No"}, {"text": u"Don't know", "id": "Dont_know"}],
        )

    def test_colon_in_url_name(self):
        """Ensure that colons in url_names convert to file paths properly"""

        print("Starting import")
        # Not using get_courses because we need the modulestore object too afterward
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
        courses = modulestore.get_courses()
        self.assertEquals(len(courses), 1)
        course = courses[0]
        course_id = course.id

        print("course errors:")
        for (msg, err) in modulestore.get_item_errors(course.location):
            print(msg)
            print(err)

        chapters = course.get_children()
        self.assertEquals(len(chapters), 2)

        ch2 = chapters[1]
        self.assertEquals(ch2.url_name, "secret:magic")

        print("Ch2 location: ", ch2.location)

        also_ch2 = modulestore.get_instance(course_id, ch2.location)
        self.assertEquals(ch2, also_ch2)

        print("making sure html loaded")
        cloc = course.location
        loc = Location(cloc.tag, cloc.org, cloc.course, 'html', 'secret:toylab')
        html = modulestore.get_instance(course_id, loc)
        self.assertEquals(html.display_name, "Toy lab")

    def test_colon_in_url_name(self):
        """Ensure that colons in url_names convert to file paths properly"""

        print "Starting import"
        # Not using get_courses because we need the modulestore object too afterward
        modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
        courses = modulestore.get_courses()
        self.assertEquals(len(courses), 1)
        course = courses[0]

        print "course errors:"
        for (msg, err) in modulestore.get_course_errors(course.id):
            print msg
            print err

        chapters = course.get_children()
        self.assertEquals(len(chapters), 5)

        ch2 = chapters[1]
        self.assertEquals(ch2.url_name, "secret:magic")

        print "Ch2 location: ", ch2.location

        also_ch2 = modulestore.get_item(ch2.location)
        self.assertEquals(ch2, also_ch2)

        print "making sure html loaded"
        loc = course.id.make_usage_key('html', 'secret:toylab')
        html = modulestore.get_item(loc)
        self.assertEquals(html.display_name, "Toy lab")

    def check_export_roundtrip(self, data_dir, course_dir, mock_get):

        # Patch network calls to retrieve the textbook TOC
        mock_get.return_value.text = dedent("""
            <?xml version="1.0"?><table_of_contents>
            <entry page="5" page_label="ii" name="Table of Contents"/>
            </table_of_contents>
        """).strip()

        root_dir = path(self.temp_dir)
        print("Copying test course to temp dir {0}".format(root_dir))

        data_dir = path(data_dir)
        shutil.copytree(data_dir / course_dir, root_dir / course_dir)

        print("Starting import")
        initial_import = XMLModuleStore(root_dir, course_dirs=[course_dir])

        courses = initial_import.get_courses()
        self.assertEquals(len(courses), 1)
        initial_course = courses[0]

        # export to the same directory--that way things like the custom_tags/ folder
        # will still be there.
        print("Starting export")
        fs = OSFS(root_dir)
        export_fs = fs.makeopendir(course_dir)

        xml = initial_course.export_to_xml(export_fs)
        with export_fs.open('course.xml', 'w') as course_xml:
            course_xml.write(xml)

        print("Starting second import")
        second_import = XMLModuleStore(root_dir, course_dirs=[course_dir])

        courses2 = second_import.get_courses()
        self.assertEquals(len(courses2), 1)
        exported_course = courses2[0]

        print("Checking course equality")

        # HACK: filenames change when changing file formats
        # during imports from old-style courses.  Ignore them.
        strip_filenames(initial_course)
        strip_filenames(exported_course)

        self.assertEquals(initial_course, exported_course)
        self.assertEquals(initial_course.id, exported_course.id)
        course_id = initial_course.id

        print("Checking key equality")
        self.assertEquals(sorted(initial_import.modules[course_id].keys()),
                          sorted(second_import.modules[course_id].keys()))

        print("Checking module equality")
        for location in initial_import.modules[course_id].keys():
            print("Checking", location)
            self.assertEquals(initial_import.modules[course_id][location],
                              second_import.modules[course_id][location])

 def test_tilde_files_ignored(self, _fake_glob):
     modulestore = XMLModuleStore(DATA_DIR, course_dirs=['tilde'], load_error_modules=False)
     about_location = SlashSeparatedCourseKey('edX', 'tilde', '2012_Fall').make_usage_key(
         'about', 'index',
     )
     about_module = modulestore.get_item(about_location)
     self.assertIn("GREEN", about_module.data)
     self.assertNotIn("RED", about_module.data)

    def get_course(self, name):
        """Get a test course by directory name.  If there's more than one, error."""
        print("Importing {0}".format(name))

        modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])
        courses = modulestore.get_courses()
        self.assertEquals(len(courses), 1)
        return courses[0]

 def test_tilde_files_ignored(self, _fake_glob):
     modulestore = XMLModuleStore(DATA_DIR, source_dirs=['course_ignore'], load_error_modules=False)
     about_location = CourseKey.from_string('edX/course_ignore/2014_Fall').make_usage_key(
         'about', 'index',
     )
     about_module = modulestore.get_item(about_location)
     self.assertIn("GREEN", about_module.data)
     self.assertNotIn("RED", about_module.data)

def import_with_checks(course_dir):
    all_ok = True

    print('Attempting to load "{}"'.format(course_dir))

    course_dir = path(course_dir)
    data_dir = course_dir.dirname()
    source_dirs = [course_dir.basename()]

    # No default class--want to complain if it doesn't find plugins for any
    # module.
    modulestore = XMLModuleStore(
        data_dir,
        default_class=None,
        source_dirs=source_dirs
    )

    def str_of_err(tpl):
        (msg, exc_str) = tpl
        return '{msg}\n{exc}'.format(msg=msg, exc=exc_str)

    courses = modulestore.get_courses()

    n = len(courses)
    if n != 1:
        print('ERROR: Expect exactly 1 course.  Loaded {n}: {lst}'.format(n=n, lst=courses))
        return (False, None)

    course = courses[0]
    errors = modulestore.get_course_errors(course.id)
    if len(errors) != 0:
        all_ok = False
        print(
            '\n' +
            '========================================' +
            'ERRORs during import:' +
            '\n'.join(map(str_of_err, errors)) +
            '========================================' +
            '\n'
        )

    # print course
    validators = (
        traverse_tree,
    )

    print('========================================')
    print('Running validators...')

    for validate in validators:
        print('Running {}'.format(validate.__name__))
        all_ok = validate(course) and all_ok

    if all_ok:
        print('Course passes all checks!')
    else:
        print('Course fails some checks.  See above for errors.')
    return all_ok, course

def import_with_checks(course_dir, verbose=True):
    all_ok = True

    print "Attempting to load '{0}'".format(course_dir)

    course_dir = path(course_dir)
    data_dir = course_dir.dirname()
    course_dirs = [course_dir.basename()]

    # No default class--want to complain if it doesn't find plugins for any
    # module.
    modulestore = XMLModuleStore(data_dir,
                   default_class=None,
                   course_dirs=course_dirs)

    def str_of_err(tpl):
        (msg, exc_str) = tpl
        return '{msg}\n{exc}'.format(msg=msg, exc=exc_str)

    courses = modulestore.get_courses()

    n = len(courses)
    if n != 1:
        print 'ERROR: Expect exactly 1 course.  Loaded {n}: {lst}'.format(
            n=n, lst=courses)
        return (False, None)

    course = courses[0]
    errors = modulestore.get_item_errors(course.location)
    if len(errors) != 0:
        all_ok = False
        print '\n'
        print "=" * 40
        print 'ERRORs during import:'
        print '\n'.join(map(str_of_err, errors))
        print "=" * 40
        print '\n'


    #print course
    validators = (
        traverse_tree,
        )

    print "=" * 40
    print "Running validators..."

    for validate in validators:
        print 'Running {0}'.format(validate.__name__)
        all_ok = validate(course) and all_ok


    if all_ok:
        print 'Course passes all checks!'
    else:
        print "Course fails some checks.  See above for errors."
    return all_ok, course

    def check_export_roundtrip(self, data_dir, course_dir):
        root_dir = path(self.temp_dir)
        print("Copying test course to temp dir {0}".format(root_dir))

        data_dir = path(data_dir)
        shutil.copytree(data_dir / course_dir, root_dir / course_dir)

        print("Starting import")
        initial_import = XMLModuleStore(root_dir, course_dirs=[course_dir])

        courses = initial_import.get_courses()
        self.assertEquals(len(courses), 1)
        initial_course = courses[0]

        # export to the same directory--that way things like the custom_tags/ folder
        # will still be there.
        print("Starting export")
        fs = OSFS(root_dir)
        export_fs = fs.makeopendir(course_dir)

        xml = initial_course.export_to_xml(export_fs)
        with export_fs.open('course.xml', 'w') as course_xml:
            course_xml.write(xml)

        print("Starting second import")
        second_import = XMLModuleStore(root_dir, course_dirs=[course_dir])

        courses2 = second_import.get_courses()
        self.assertEquals(len(courses2), 1)
        exported_course = courses2[0]

        print("Checking course equality")

        # HACK: filenames change when changing file formats
        # during imports from old-style courses.  Ignore them.
        strip_filenames(initial_course)
        strip_filenames(exported_course)

        self.assertEquals(initial_course, exported_course)
        self.assertEquals(initial_course.id, exported_course.id)
        course_id = initial_course.id

        print("Checking key equality")
        self.assertEquals(sorted(initial_import.modules[course_id].keys()),
                          sorted(second_import.modules[course_id].keys()))

        print("Checking module equality")
        for location in initial_import.modules[course_id].keys():
            print("Checking", location)
            if location.category == 'html':
                print(
                    "Skipping html modules--they can't import in"
                    " final form without writing files..."
                )
                continue
            self.assertEquals(initial_import.modules[course_id][location],
                              second_import.modules[course_id][location])

    def get_course(self, name):
        """Get a test course by directory name.  If there's more than one, error."""
        print("Importing {0}".format(name))

        modulestore = XMLModuleStore(
            DATA_DIR, course_dirs=[name], xblock_mixins=(InheritanceMixin,), xblock_select=only_xmodules
        )
        courses = modulestore.get_courses()
        self.assertEquals(len(courses), 1)
        return courses[0]

    def test_static_tabs_import(self):
        """Make sure that the static tabs are imported correctly"""

        modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])

        location_tab_syllabus = Location("edX", "toy", "2012_Fall", "static_tab", "syllabus", None)
        toy_tab_syllabus = modulestore.get_item(location_tab_syllabus)
        self.assertEqual(toy_tab_syllabus.display_name, 'Syllabus')
        self.assertEqual(toy_tab_syllabus.course_staff_only, False)

        location_tab_resources = Location("edX", "toy", "2012_Fall", "static_tab", "resources", None)
        toy_tab_resources = modulestore.get_item(location_tab_resources)
        self.assertEqual(toy_tab_resources.display_name, 'Resources')
        self.assertEqual(toy_tab_resources.course_staff_only, True)

    def test_graphicslidertool_import(self):
        '''
        Check to see if definition_from_xml in gst_module.py
        works properly.  Pulls data from the graphic_slider_tool directory
        in the test data directory.
        '''
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['graphic_slider_tool'])

        sa_id = SlashSeparatedCourseKey("edX", "gst_test", "2012_Fall")
        location = sa_id.make_usage_key("graphical_slider_tool", "sample_gst")
        gst_sample = modulestore.get_item(location)
        render_string_from_sample_gst_xml = """
        <slider var="a" style="width:400px;float:left;"/>\
<plot style="margin-top:15px;margin-bottom:15px;"/>""".strip()
        self.assertIn(render_string_from_sample_gst_xml, gst_sample.data)

    def test_graphicslidertool_import(self):
        '''
        Check to see if definition_from_xml in gst_module.py
        works properly.  Pulls data from the graphic_slider_tool directory
        in the test data directory.
        '''
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['graphic_slider_tool'])

        sa_id = "edX/gst_test/2012_Fall"
        location = Location(["i4x", "edX", "gst_test", "graphical_slider_tool", "sample_gst"])
        gst_sample = modulestore.get_instance(sa_id, location)
        render_string_from_sample_gst_xml = """
        <slider var="a" style="width:400px;float:left;"/>\
<plot style="margin-top:15px;margin-bottom:15px;"/>""".strip()
        self.assertEqual(gst_sample.render, render_string_from_sample_gst_xml)

    def test_word_cloud_import(self):
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['word_cloud'])

        course = modulestore.get_courses()[0]
        chapters = course.get_children()
        ch1 = chapters[0]
        sections = ch1.get_children()

        self.assertEqual(len(sections), 1)

        location = course.id.make_usage_key('word_cloud', 'cloud1')
        module = modulestore.get_item(location)
        self.assertEqual(len(module.get_children()), 0)
        self.assertEqual(module.num_inputs, 5)
        self.assertEqual(module.num_top_words, 250)

    def test_word_cloud_import(self):
        modulestore = XMLModuleStore(DATA_DIR, course_dirs=["word_cloud"])

        course = modulestore.get_courses()[0]
        chapters = course.get_children()
        ch1 = chapters[0]
        sections = ch1.get_children()

        self.assertEqual(len(sections), 1)

        location = course.location
        location = Location(location.tag, location.org, location.course, "word_cloud", "cloud1")
        module = modulestore.get_instance(course.id, location)
        self.assertEqual(len(module.get_children()), 0)
        self.assertEqual(module.num_inputs, 5)
        self.assertEqual(module.num_top_words, 250)