def setUp(self):
        """
        Create user and login.
        """
        self.staff_pwd = super(ContentStoreToyCourseTest, self).setUp()
        self.staff_usr = self.user
        self.non_staff_usr, self.non_staff_pwd = self.create_non_staff_user()

        self.client = Client()
        self.contentstore = contentstore()
        store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)  # pylint: disable=protected-access

        self.course_key = store.make_course_key('edX', 'toy', '2012_Fall')

        import_course_from_xml(
            store, self.user.id, TEST_DATA_DIR, ['toy'],
            static_content_store=self.contentstore, verbose=True
        )

        # A locked asset
        self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt')
        self.url_locked = unicode(self.locked_asset)
        self.contentstore.set_attr(self.locked_asset, 'locked', True)

        # An unlocked asset
        self.unlocked_asset = self.course_key.make_asset_key('asset', 'another_static.txt')
        self.url_unlocked = unicode(self.unlocked_asset)
        self.length_unlocked = self.contentstore.get_attr(self.unlocked_asset, 'length')

    def load_courses(cls):
        """
        Load test courses and return list of ids
        """
        store = modulestore()

        unique_org = factory.Sequence(lambda n: 'edX.%d' % n)
        cls.course = CourseFactory.create(
            emit_signals=True,
            org=unique_org,
            course='simple',
            run="run",
            display_name=u'2012_Fáĺĺ',
            modulestore=store
        )

        cls.discussion = ItemFactory.create(
            category='discussion', parent_location=cls.course.location
        )

        courses = store.get_courses()
        # NOTE: if xml store owns these, it won't import them into mongo
        if cls.test_course_key not in [c.id for c in courses]:
            import_course_from_xml(
                store, ModuleStoreEnum.UserID.mgmt_command, DATA_DIR, XML_COURSE_DIRS, create_if_not_present=True
            )

        return [course.id for course in store.get_courses()]

    def setUp(self):
        """
        Set up the tests
        """
        super(StaticTabDateTestCaseXML, self).setUp()

        # The following XML test course (which lives at common/test/data/2014)
        # is closed; we're testing that tabs still appear when
        # the course is already closed
        self.xml_course_key = self.store.make_course_key('edX', 'detached_pages', '2014')
        import_course_from_xml(
            self.store,
            'test_user',
            TEST_DATA_DIR,
            source_dirs=['2014'],
            static_content_store=None,
            target_id=self.xml_course_key,
            raise_on_failure=True,
            create_if_not_present=True,
        )

        # this text appears in the test course's tab
        # common/test/data/2014/tabs/8e4cce2b4aaf4ba28b1220804619e41f.html
        self.xml_data = "static 463139"
        self.xml_url = "8e4cce2b4aaf4ba28b1220804619e41f"

 def test_rewrite_reference_list(self):
     # This test fails with split modulestore (the HTML component is not in "different_course_id" namespace).
     # More investigation needs to be done.
     module_store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
     target_id = module_store.make_course_key('testX', 'conditional_copy', 'copy_run')
     import_course_from_xml(
         module_store,
         self.user.id,
         TEST_DATA_DIR,
         ['conditional'],
         target_id=target_id
     )
     conditional_module = module_store.get_item(
         target_id.make_usage_key('conditional', 'condone')
     )
     self.assertIsNotNone(conditional_module)
     different_course_id = module_store.make_course_key('edX', 'different_course', None)
     self.assertListEqual(
         [
             target_id.make_usage_key('problem', 'choiceprob'),
             different_course_id.make_usage_key('html', 'for_testing_import_rewrites')
         ],
         conditional_module.sources_list
     )
     self.assertListEqual(
         [
             target_id.make_usage_key('html', 'congrats'),
             target_id.make_usage_key('html', 'secret_page')
         ],
         conditional_module.show_tag_list
     )

    def setUpClass(cls):
        super(ContentStoreToyCourseTest, cls).setUpClass()

        cls.contentstore = contentstore()
        cls.modulestore = modulestore()

        cls.course_key = cls.modulestore.make_course_key('edX', 'toy', '2012_Fall')

        import_course_from_xml(
            cls.modulestore, 1, TEST_DATA_DIR, ['toy'],
            static_content_store=cls.contentstore, verbose=True
        )

        # A locked asset
        cls.locked_asset = cls.course_key.make_asset_key('asset', 'sample_static.html')
        cls.url_locked = six.text_type(cls.locked_asset)
        cls.url_locked_versioned = get_versioned_asset_url(cls.url_locked)
        cls.url_locked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_locked)
        cls.contentstore.set_attr(cls.locked_asset, 'locked', True)

        # An unlocked asset
        cls.unlocked_asset = cls.course_key.make_asset_key('asset', 'another_static.txt')
        cls.url_unlocked = six.text_type(cls.unlocked_asset)
        cls.url_unlocked_versioned = get_versioned_asset_url(cls.url_unlocked)
        cls.url_unlocked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_unlocked)
        cls.length_unlocked = cls.contentstore.get_attr(cls.unlocked_asset, 'length')

    def import_test_course(self, solution_attribute=None, solution_element=None):
        """
        Import the test course with the sga unit
        """
        # adapted from edx-platform/cms/djangoapps/contentstore/management/commands/tests/test_cleanup_assets.py
        root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
        input_dir = os.path.join(root, "test_data")

        temp_dir = tempfile.mkdtemp()
        self.addCleanup(lambda: shutil.rmtree(temp_dir))

        xml_dir = os.path.join(temp_dir, "xml")
        shutil.copytree(input_dir, xml_dir)

        with open(os.path.join(xml_dir, "2017_SGA", "vertical", "vertical.xml"), "w") as f:
            f.write(self.make_test_vertical(solution_attribute, solution_element))

        store = modulestore()
        import_course_from_xml(
            store,
            'sga_user',
            xml_dir,
        )

        return store.get_course(CourseLocator.from_string('SGAU/SGA101/course'))

    def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets):
        """
        Generate timings for different amounts of asset metadata and different modulestores.
        """
        if CodeBlockTimer is None:
            raise SkipTest("CodeBlockTimer undefined.")

        desc = "XMLRoundTrip:{}->{}:{}".format(
            SHORT_NAME_MAP[source_ms],
            SHORT_NAME_MAP[dest_ms],
            num_assets
        )

        with CodeBlockTimer(desc):

            with CodeBlockTimer("fake_assets"):
                # First, make the fake asset metadata.
                make_asset_xml(num_assets, ASSET_XML_PATH)
                validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)

            with source_ms.build() as (source_content, source_store):
                with dest_ms.build() as (dest_content, dest_store):
                    source_course_key = source_store.make_course_key('a', 'course', 'course')
                    dest_course_key = dest_store.make_course_key('a', 'course', 'course')

                    with CodeBlockTimer("initial_import"):
                        import_course_from_xml(
                            source_store,
                            'test_user',
                            TEST_DATA_ROOT,
                            source_dirs=TEST_COURSE,
                            static_content_store=source_content,
                            target_id=source_course_key,
                            create_if_not_present=True,
                            raise_on_failure=True,
                        )

                    with CodeBlockTimer("export"):
                        export_course_to_xml(
                            source_store,
                            source_content,
                            source_course_key,
                            self.export_dir,
                            'exported_source_course',
                        )

                    with CodeBlockTimer("second_import"):
                        import_course_from_xml(
                            dest_store,
                            'test_user',
                            self.export_dir,
                            source_dirs=['exported_source_course'],
                            static_content_store=dest_content,
                            target_id=dest_course_key,
                            create_if_not_present=True,
                            raise_on_failure=True,
                        )

    def test_generate_find_timings(self, source_ms, num_assets):
        """
        Generate timings for different amounts of asset metadata and different modulestores.
        """
        if CodeBlockTimer is None:
            raise SkipTest("CodeBlockTimer undefined.")

        desc = "FindAssetTest:{}:{}".format(
            SHORT_NAME_MAP[source_ms],
            num_assets,
        )

        with CodeBlockTimer(desc):

            with CodeBlockTimer("fake_assets"):
                # First, make the fake asset metadata.
                make_asset_xml(num_assets, ASSET_XML_PATH)
                validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)

            with source_ms.build() as (source_content, source_store):
                source_course_key = source_store.make_course_key('a', 'course', 'course')
                asset_key = source_course_key.make_asset_key(
                    AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif'
                )

                with CodeBlockTimer("initial_import"):
                    import_course_from_xml(
                        source_store,
                        'test_user',
                        TEST_DATA_ROOT,
                        source_dirs=TEST_COURSE,
                        static_content_store=source_content,
                        target_id=source_course_key,
                        create_if_not_present=True,
                        raise_on_failure=True,
                    )

                with CodeBlockTimer("find_nonexistent_asset"):
                    # More correct would be using the AssetManager.find() - but since the test
                    # has created its own test modulestore, the AssetManager can't be used.
                    __ = source_store.find_asset_metadata(asset_key)

                # Perform get_all_asset_metadata for each sort.
                for sort in ALL_SORTS:
                    with CodeBlockTimer("get_asset_list:{}-{}".format(
                        sort[0],
                        'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc'
                    )):
                        # Grab two ranges of 50 assets using different sorts.
                        # Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
                        start_middle = num_assets / 2
                        __ = source_store.get_all_asset_metadata(
                            source_course_key, 'asset', start=0, sort=sort, maxresults=50
                        )
                        __ = source_store.get_all_asset_metadata(
                            source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
                        )

 def test_rewrite_reference(self):
     module_store = modulestore()
     target_id = module_store.make_course_key("testX", "peergrading_copy", "copy_run")
     import_course_from_xml(
         module_store, self.user.id, TEST_DATA_DIR, ["open_ended"], target_id=target_id, create_if_not_present=True
     )
     peergrading_module = module_store.get_item(target_id.make_usage_key("peergrading", "PeerGradingLinked"))
     self.assertIsNotNone(peergrading_module)
     self.assertEqual(
         target_id.make_usage_key("combinedopenended", "SampleQuestion"), peergrading_module.link_to_location
     )

    def test_split_course_export_import(self):
        # Construct the contentstore for storing the first import
        with MongoContentstoreBuilder().build() as source_content:
            # Construct the modulestore for storing the first import (using the previously created contentstore)
            with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store:
                # Construct the contentstore for storing the second import
                with MongoContentstoreBuilder().build() as dest_content:
                    # Construct the modulestore for storing the second import (using the second contentstore)
                    with SPLIT_MODULESTORE_SETUP.build(contentstore=dest_content) as dest_store:
                        source_course_key = source_store.make_course_key('a', 'source', '2015_Fall')
                        dest_course_key = dest_store.make_course_key('a', 'dest', '2015_Fall')

                        import_course_from_xml(
                            source_store,
                            'test_user',
                            TEST_DATA_DIR,
                            source_dirs=['split_course_with_static_tabs'],
                            static_content_store=source_content,
                            target_id=source_course_key,
                            raise_on_failure=True,
                            create_if_not_present=True,
                        )

                        export_course_to_xml(
                            source_store,
                            source_content,
                            source_course_key,
                            self.export_dir,
                            EXPORTED_COURSE_DIR_NAME,
                        )

                        source_course = source_store.get_course(source_course_key, depth=None, lazy=False)

                        self.assertEqual(source_course.url_name, 'course')

                        export_dir_path = path(self.export_dir)
                        policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run
                        policy_path = policy_dir / 'policy.json'
                        self.assertTrue(os.path.exists(policy_path))

                        import_course_from_xml(
                            dest_store,
                            'test_user',
                            self.export_dir,
                            source_dirs=[EXPORTED_COURSE_DIR_NAME],
                            static_content_store=dest_content,
                            target_id=dest_course_key,
                            raise_on_failure=True,
                            create_if_not_present=True,
                        )

                        dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False)

                        self.assertEqual(dest_course.url_name, 'course')

 def setUpClass(cls):
     super(CommandExecutionTestCase, cls).setUpClass()
     cls.course_key = cls.store.make_course_key(u'edX', u'lti_provider', u'3000')
     import_course_from_xml(
         cls.store,
         u'test_user',
         TEST_DATA_DIR,
         source_dirs=[u'simple'],
         static_content_store=None,
         target_id=cls.course_key,
         raise_on_failure=True,
         create_if_not_present=True,
     )
     cls.lti_block = u'block-v1:[email protected][email protected]_2'

    def test_asset_sizes(self, source_ms, num_assets):
        """
        Generate timings for different amounts of asset metadata and different modulestores.
        """
        # First, make the fake asset metadata.
        make_asset_xml(num_assets, ASSET_XML_PATH)
        validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)

        with source_ms.build() as (source_content, source_store):
            source_course_key = source_store.make_course_key('a', 'course', 'course')

            import_course_from_xml(
                source_store,
                'test_user',
                TEST_DATA_ROOT,
                source_dirs=TEST_COURSE,
                static_content_store=source_content,
                target_id=source_course_key,
                create_if_not_present=True,
                raise_on_failure=True,
            )

            asset_collection = source_ms.asset_collection()
            # Ensure the asset collection exists.
            if asset_collection.name in asset_collection.database.collection_names():

                # Map gets the size of each structure.
                mapper = Code("""
                    function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
                    """)

                # Reduce finds the largest structure size and returns only it.
                reducer = Code("""
                    function(key, values) {
                        var max_size = 0;
                        for (var i=0; i < values.length; i++) {
                            if (values[i] > max_size) {
                                max_size = values[i];
                            }
                        }
                        return max_size;
                    }
                """)

                results = asset_collection.map_reduce(mapper, reducer, "size_results")
                result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
                    self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
                )
                with open("bson_sizes.txt", "a") as f:
                    f.write(result_str)

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) == 0:
            raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")

        data_dir = args[0]
        do_import_static = not options.get('nostatic', False)
        if len(args) > 1:
            source_dirs = args[1:]
        else:
            source_dirs = None
        self.stdout.write("Importing.  Data_dir={data}, source_dirs={courses}\n".format(
            data=data_dir,
            courses=source_dirs,
        ))
        mstore = modulestore()

        course_items = import_course_from_xml(
            mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, source_dirs, load_error_modules=False,
            static_content_store=contentstore(), verbose=True,
            do_import_static=do_import_static,
            create_if_not_present=True,
        )

        for course in course_items:
            course_id = course.id
            if not are_permissions_roles_seeded(course_id):
                self.stdout.write('Seeding forum roles for course {0}\n'.format(course_id))
                seed_permissions_roles(course_id)

 def setUp(self):
     super(TestPeerGradingFound, self).setUp()
     self.user = factories.UserFactory()
     store = modulestore()
     course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['open_ended_nopath'])  # pylint: disable=maybe-no-member
     self.course = course_items[0]
     self.course_key = self.course.id

 def _import_course(self, content_store, modulestore):
     """
     Imports a course for testing.
     Returns the course key.
     """
     course_key = modulestore.make_course_key('a', 'course', 'course')
     import_course_from_xml(
         modulestore,
         'test_user',
         TEST_DATA_DIR,
         source_dirs=['manual-testing-complete'],
         static_content_store=content_store,
         target_id=course_key,
         create_if_not_present=True,
         raise_on_failure=True,
     )
     return course_key

    def test_unicode_chars_in_course_name_import(self):
        """
        # Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
        """
        # Test with the split modulestore because store.has_course fails in old mongo with unicode characters.
        with modulestore().default_store(ModuleStoreEnum.Type.split):
            module_store = modulestore()
            course_id = module_store.make_course_key(u"Юникода", u"unicode_course", u"échantillon")
            import_course_from_xml(
                module_store, self.user.id, TEST_DATA_DIR, ["2014_Uni"], target_id=course_id, create_if_not_present=True
            )

            course = module_store.get_course(course_id)
            self.assertIsNotNone(course)

            # test that course 'display_name' same as imported course 'display_name'
            self.assertEqual(course.display_name, u"Φυσικά το όνομα Unicode")

    def test_export_all_courses(self):
        """
        This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are
        cleaned up on import
        """
        import_course_from_xml(
            self.module_store,
            '**replace_user**',
            TEST_DATA_DIR,
            ['course_ignore'],
            static_content_store=self.content_store,
            do_import_static=True,
            verbose=True
        )

        course = self.module_store.get_course(CourseKey.from_string('/'.join(['edX', 'course_ignore', '2014_Fall'])))
        self.assertIsNotNone(course)

        # check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 2)
        self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))

        # manually add redundant assets (file ".DS_Store" and filename starts with "._")
        course_filter = course.id.make_asset_key("asset", None)
        query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG)
        query['_id.name'] = all_assets[0]['_id']['name']
        asset_doc = self.content_store.fs_files.find_one(query)
        asset_doc['_id']['name'] = u'._example_test.txt'
        self.content_store.fs_files.insert(asset_doc)
        asset_doc['_id']['name'] = u'.DS_Store'
        self.content_store.fs_files.insert(asset_doc)

        # check that now course has four assets
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 4)
        self.assertEqual(
            set([asset['_id']['name'] for asset in all_assets]),
            set([u'.example.txt', u'example.txt', u'._example_test.txt', u'.DS_Store'])
        )
        # now call asset_cleanup command and check that there is only two proper assets in contentstore for the course
        call_command('cleanup_assets')
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 2)
        self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))

 def test_import_course_into_similar_namespace(self):
     # Checks to make sure that a course with an org/course like
     # edx/course can be imported into a namespace with an org/course
     # like edx/course_name
     module_store, __, course = self.load_test_import_course()
     course_items = import_course_from_xml(
         module_store, self.user.id, TEST_DATA_DIR, ["test_import_course_2"], target_id=course.id, verbose=True
     )
     self.assertEqual(len(course_items), 1)

    def _verify_split_test_import(self, target_course_name, source_course_name, split_test_name, groups_to_verticals):
        module_store = modulestore()
        target_id = module_store.make_course_key("testX", target_course_name, "copy_run")
        import_course_from_xml(
            module_store,
            self.user.id,
            TEST_DATA_DIR,
            [source_course_name],
            target_id=target_id,
            create_if_not_present=True,
        )
        split_test_module = module_store.get_item(target_id.make_usage_key("split_test", split_test_name))
        self.assertIsNotNone(split_test_module)

        remapped_verticals = {
            key: target_id.make_usage_key("vertical", value) for key, value in groups_to_verticals.iteritems()
        }

        self.assertEqual(remapped_verticals, split_test_module.group_id_to_child)

    def test_asset_import_nostatic(self):
        '''
        This test validates that an image asset is NOT imported when do_import_static=False
        '''
        content_store = contentstore()

        module_store = modulestore()
        import_course_from_xml(
            module_store, self.user.id, TEST_DATA_DIR, ['toy'],
            static_content_store=content_store, do_import_static=False,
            create_if_not_present=True, verbose=True
        )

        course = module_store.get_course(module_store.make_course_key('edX', 'toy', '2012_Fall'))

        # make sure we have NO assets in our contentstore
        all_assets, count = content_store.get_all_content_for_course(course.id)
        self.assertEqual(len(all_assets), 0)
        self.assertEqual(count, 0)