def test_non_admin_permissions_failures(self):
        actions.login(STUDENT_EMAIL)
        student_xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(settings.HtmlHookRESTHandler.XSRF_ACTION)

        response = self.get(ADMIN_SETTINGS_URL)
        self.assertEquals(200, response.status_int)
        payload = transforms.loads(response.body)
        self.assertEquals(401, payload["status"])
        self.assertEquals("Access denied.", payload["message"])

        response = self.put(
            ADMIN_SETTINGS_URL,
            {
                "request": transforms.dumps(
                    {"key": "base:after_body_tag_begins", "xsrf_token": cgi.escape(student_xsrf_token), "payload": "{}"}
                )
            },
        )
        payload = transforms.loads(response.body)
        self.assertEquals(401, payload["status"])
        self.assertEquals("Access denied.", payload["message"])

        response = self.delete(ADMIN_SETTINGS_URL + "?xsrf_token=" + cgi.escape(student_xsrf_token))
        self.assertEquals(200, response.status_int)
        payload = transforms.loads(response.body)
        self.assertEquals(401, payload["status"])
        self.assertEquals("Access denied.", payload["message"])

    def put(self):
        """A PUT REST method shared by all unit types."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, 'put-unit', {'key': key}):
            return

        if not CourseOutlineRights.can_edit(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        unit = courses.Course(self).find_unit_by_id(key)
        if not unit:
            transforms.send_json_response(
                self, 404, 'Object not found.', {'key': key})
            return

        payload = request.get('payload')
        updated_unit_dict = transforms.json_to_dict(
            transforms.loads(payload), self.SCHEMA_DICT)

        errors = []
        self.apply_updates(unit, updated_unit_dict, errors)
        if not errors:
            course = courses.Course(self)
            assert course.update_unit(unit)
            course.save()
            common_utils.run_hooks(self.POST_SAVE_HOOKS, unit)
            transforms.send_json_response(self, 200, 'Saved.')
        else:
            transforms.send_json_response(self, 412, '\n'.join(errors))

    def put(self):
        """Store a question in the datastore in response to a PUT."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, self.XSRF_TOKEN, {'key': key}):
            return

        if not CourseOutlineRights.can_edit(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        payload = request.get('payload')
        question_dict = transforms.loads(payload)
        question_dict['description'] = question_dict['description'].strip()

        question_dict, errors = self.import_and_validate(question_dict, key)

        if errors:
            self.validation_error('\n'.join(errors), key=key)
            return

        if key:
            question = QuestionDTO(key, question_dict)
        else:
            question = QuestionDTO(None, question_dict)

        question.type = self.TYPE
        key_after_save = QuestionDAO.save(question)

        transforms.send_json_response(
            self, 200, 'Saved.', payload_dict={'key': key_after_save})

    def put(self):
        """Handles REST PUT verb with JSON payload."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, 'announcement-put', {'key': key}):
            return

        if not AnnouncementsRights.can_edit(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        entity = AnnouncementEntity.get(key)
        if not entity:
            transforms.send_json_response(
                self, 404, 'Object not found.', {'key': key})
            return

        payload = request.get('payload')
        transforms.dict_to_entity(entity, transforms.json_to_dict(
            transforms.loads(payload),
            AnnouncementsItemRESTHandler.SCHEMA_DICT))
        entity.put()

        transforms.send_json_response(self, 200, 'Saved.')

    def put(self):
        """A PUT REST method shared by all unit types."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, 'put-unit', {'key': key}):
            return

        if not CourseOutlineRights.can_edit(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        unit = courses.Course(self).find_unit_by_id(key)
        if not unit:
            transforms.send_json_response(
                self, 404, 'Object not found.', {'key': key})
            return

        payload = request.get('payload')
        errors = []

        try:
            updated_unit_dict = transforms.json_to_dict(
                transforms.loads(payload), self.SCHEMA_DICT)
            self.apply_updates(unit, updated_unit_dict, errors)
        except (TypeError, ValueError), ex:
            errors.append(str(ex))

    def test_oeditor_returns_state(self):
        actions.login('[email protected]', is_admin=True)
        xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(
                oeditor.EditorPrefsRestHandler.XSRF_TOKEN)

        self._post(xsrf_token=xsrf_token)
        response = self.get('dashboard?action=edit_lesson&key=%s' % (
            self.lesson.lesson_id))

        expected = {
          'xsrf_token': xsrf_token,
          'location': self.location,
          'key': str(self.key),
          'prefs': self.EDITOR_STATE
        }
        expected = transforms.loads(transforms.dumps(expected))

        match = re.search(
            r'cb_global.editor_prefs = JSON.parse\((.*)\);', response.body)

        actual = match.group(1)
        actual = transforms.loads('"%s"' % actual)
        actual = transforms.loads(actual[1:-1])

        self.assertEquals(expected, actual)

    def test_parameters_can_be_omitted_if_using_source_context(self):
        email = '[email protected]'
        actions.login(email, is_admin=True)

        response = transforms.loads(self.get(
            '/rest/data/character/items?filter=rank>=5&ordering=rank'
            '&chunk_size=3&page_number=1').body)
        source_context = response['source_context']
        self._verify_data([self.characters[4], self.characters[6]],
                          response['data'])

        # This should load identical items, without having to respecify
        # filters, ordering, chunk_size.
        response = transforms.loads(self.get(
            '/rest/data/character/items?page_number=1'
            '&source_context=%s' % source_context).body)
        self.assertEquals(1, response['page_number'])
        self._verify_data([self.characters[4], self.characters[6]],
                          response['data'])
        self._assert_have_only_logs(response, [
            'Continuing use of existing context',
            'fetch page 1 start cursor present; end cursor missing',
            'fetch page 1 using limit 3',
            'fetch page 1 is partial; not saving end cursor',
            ])

    def test_pii_encoding(self):
        email = '[email protected]'
        actions.login(email, is_admin=True)
        token = data_sources_utils.generate_data_source_token(
            crypto.XsrfTokenManager)

        response = transforms.loads(self.get('/rest/data/character/items').body)
        for d in response['data']:
            # Ensure that field marked as needing transformation is cleared
            # when we don't pass in an XSRF token used for generating a secret
            # for encrypting.
            self.assertEquals('None', d['user_id'])
            self.assertEquals(str(db.Key.from_path(Character.kind(), 'None')),
                              d['key'])

            # Ensure that field marked for blacklist is suppressed.
            self.assertFalse('name' in d)

        response = transforms.loads(self.get(
            '/rest/data/character/items?data_source_token=' + token).body)

        for d in response['data']:
            # Ensure that field marked as needing transformation is cleared
            # when we don't pass in an XSRF token used for generating a secret
            # for encrypting.
            self.assertIsNotNone(d['user_id'])
            self.assertNotEquals('None', d['key'])

            # Ensure that field marked for blacklist is still suppressed.
            self.assertFalse('name' in d)

    def test_nonsequential_pagination(self):
        email = '[email protected]'
        actions.login(email, is_admin=True)

        response = transforms.loads(self.get(
            '/rest/data/character/items?chunk_size=3&page_number=2').body)
        source_context = response['source_context']
        self.assertEquals(2, response['page_number'])
        self._verify_data(self.characters[6:9], response['data'])
        self._assert_have_only_logs(response, [
            'Creating new context for given parameters',
            'fetch page 0 start cursor missing; end cursor missing',
            'fetch page 0 using limit 3',
            'fetch page 0 saving end cursor',
            'fetch page 1 start cursor present; end cursor missing',
            'fetch page 1 using limit 3',
            'fetch page 1 saving end cursor',
            'fetch page 2 start cursor present; end cursor missing',
            'fetch page 2 using limit 3',
            'fetch page 2 saving end cursor',
            ])

        response = transforms.loads(self.get(
            '/rest/data/character/items?chunk_size=3&page_number=1'
            '&source_context=%s' % source_context).body)
        source_context = response['source_context']
        self._verify_data(self.characters[3:6], response['data'])
        self._assert_have_only_logs(response, [
            'Existing context matches parameters; using existing context',
            'fetch page 1 start cursor present; end cursor present',
            ])

    def put(self):
        """A PUT REST method shared by all unit types."""
        request = transforms.loads(self.request.get("request"))
        key = request.get("key")

        if not self.assert_xsrf_token_or_fail(request, "put-unit", {"key": key}):
            return

        if not self.can_edit(self.app_context):
            transforms.send_json_response(self, 401, "Access denied.", {"key": key})
            return

        unit = courses.Course(self).find_unit_by_id(key)
        if not unit:
            transforms.send_json_response(self, 404, "Object not found.", {"key": key})
            return

        payload = request.get("payload")
        errors = []

        course = courses.Course(self)
        try:
            schema = self.get_schema(course, key)
            updated_unit_dict = transforms.json_to_dict(transforms.loads(payload), schema.get_json_schema_dict())
            schema.redact_entity_to_schema(updated_unit_dict)
            self.apply_updates(unit, updated_unit_dict, errors)
        except (TypeError, ValueError), ex:
            errors.append(str(ex))

    def test_filtered_read(self):
        email = '[email protected]'
        actions.login(email, is_admin=True)

        # Single greater-equal filter
        response = transforms.loads(self.get(
            '/rest/data/character/items?filter=rank>=7').body)
        self.assertEquals(3, len(response['data']))
        for character in response['data']:
            self.assertTrue(character['rank'] >= 7)

        # Single less-than filter
        response = transforms.loads(self.get(
            '/rest/data/character/items?filter=rank<7').body)
        self.assertEquals(7, len(response['data']))
        for character in response['data']:
            self.assertTrue(character['rank'] < 7)

        # Multiple filters finding some rows
        response = transforms.loads(self.get(
            '/rest/data/character/items?filter=rank<5&filter=goal=L').body)
        self.assertEquals(2, len(response['data']))
        for character in response['data']:
            self.assertTrue(character['rank'] < 5)
            self.assertTrue(character['goal'] == 'L')

    def setUp(self):
        super(MultipleChoiceTagTests, self).setUp()

        self.base = '/' + COURSE_NAME
        self.app_context = actions.simple_add_course(
            COURSE_NAME, ADMIN_EMAIL, 'Assessment Tags')
        self.namespace = 'ns_%s' % COURSE_NAME

        with utils.Namespace(self.namespace):
            dto = models.QuestionDTO(None, transforms.loads(self.MC_1_JSON))
            self.mc_1_id = models.QuestionDAO.save(dto)
            dto = models.QuestionDTO(None, transforms.loads(self.MC_2_JSON))
            self.mc_2_id = models.QuestionDAO.save(dto)
            dto = models.QuestionGroupDTO(
                None, transforms.loads(
                    self.QG_1_JSON_TEMPLATE % (self.mc_1_id, self.mc_2_id)))
            self.qg_1_id = models.QuestionGroupDAO.save(dto)

        self.course = courses.Course(None, self.app_context)
        self.assessment = self.course.add_assessment()
        self.assessment.now_available = True
        self.assessment.html_content = (
            '<question quid="%s" weight="1" instanceid="q1"></question>'
            '<question-group qgid="%s" instanceid="qg1"></question-group' % (
                self.mc_1_id, self.qg_1_id))
        self.course.save()

    def put(self):
        """Handles PUT REST verb to save lesson and associated activity."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, 'lesson-edit', {'key': key}):
            return

        if not roles.Roles.is_course_admin(self.app_context):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        course = courses.Course(self)
        lesson = course.find_lesson_by_id(None, key)
        if not lesson:
            transforms.send_json_response(
                self, 404, 'Object not found.', {'key': key})
            return

        payload = request.get('payload')
        updates_dict = transforms.json_to_dict(
            transforms.loads(payload),
            self.get_schema(course, key).get_json_schema_dict())

        lesson.title = updates_dict['title']
        lesson.unit_id = updates_dict['unit_id']
        lesson.scored = (updates_dict['scored'] == 'scored')
        lesson.objectives = updates_dict['objectives']
        lesson.video = updates_dict['video']
        lesson.notes = updates_dict['notes']
        lesson.auto_index = updates_dict['auto_index']
        lesson.activity_title = updates_dict['activity_title']
        lesson.activity_listed = updates_dict['activity_listed']
        lesson.manual_progress = updates_dict['manual_progress']

        activity = updates_dict.get('activity', '').strip()
        errors = []
        if activity:
            if lesson.has_activity:
                course.set_activity_content(lesson, activity, errors=errors)
            else:
                errors.append('Old-style activities are not supported.')
        else:
            lesson.has_activity = False
            fs = self.app_context.fs
            path = fs.impl.physical_to_logical(course.get_activity_filename(
                lesson.unit_id, lesson.lesson_id))
            if fs.isfile(path):
                fs.delete(path)

        if not errors:
            common_utils.run_hooks(self.PRE_SAVE_HOOKS, lesson, updates_dict)
            assert course.update_lesson(lesson)
            course.save()
            common_utils.run_hooks(self.POST_SAVE_HOOKS, lesson)
            transforms.send_json_response(self, 200, 'Saved.')
        else:
            transforms.send_json_response(self, 412, '\n'.join(errors))

 def test_get_defaulted_hook_content(self):
     url = "%s?key=%s" % (ADMIN_SETTINGS_URL, cgi.escape("base.after_body_tag_begins"))
     response = transforms.loads(self.get(url).body)
     self.assertEquals(200, response["status"])
     self.assertEquals("Success.", response["message"])
     payload = transforms.loads(response["payload"])
     self.assertEquals("<!-- base.after_body_tag_begins -->", payload["hook_content"])

    def put(self):
        """Store a question group in the datastore in response to a PUT."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, self.XSRF_TOKEN, {'key': key}):
            return

        if not CourseOutlineRights.can_edit(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        payload = request.get('payload')
        question_group_dict = transforms.json_to_dict(
            transforms.loads(payload),
            self.get_schema().get_json_schema_dict())

        validation_errors = self.validate(question_group_dict, key)
        if validation_errors:
            self.validation_error('\n'.join(validation_errors), key=key)
            return

        assert self.SCHEMA_VERSION == question_group_dict.get('version')

        if key:
            question_group = QuestionGroupDTO(key, question_group_dict)
        else:
            question_group = QuestionGroupDTO(None, question_group_dict)

        key_after_save = QuestionGroupDAO.save(question_group)
        transforms.send_json_response(
            self, 200, 'Saved.', payload_dict={'key': key_after_save})

    def test_pagination_filtering_and_ordering(self):
        email = '[email protected]'
        actions.login(email, is_admin=True)

        response = transforms.loads(self.get(
            '/rest/data/character/items?filter=rank>=5&ordering=rank'
            '&chunk_size=3&page_number=1').body)
        source_context = response['source_context']
        self.assertEquals(1, response['page_number'])
        self._verify_data([self.characters[4], self.characters[6]],
                          response['data'])
        self._assert_have_only_logs(response, [
            'Creating new context for given parameters',
            'fetch page 0 start cursor missing; end cursor missing',
            'fetch page 0 using limit 3',
            'fetch page 0 saving end cursor',
            'fetch page 1 start cursor present; end cursor missing',
            'fetch page 1 using limit 3',
            'fetch page 1 is partial; not saving end cursor',
            ])

        response = transforms.loads(self.get(
            '/rest/data/character/items?filter=rank>=5&ordering=rank'
            '&chunk_size=3&page_number=0'
            '&source_context=%s' % source_context).body)
        source_context = response['source_context']
        self.assertEquals(0, response['page_number'])
        self._verify_data([self.characters[7], self.characters[1],
                           self.characters[8]], response['data'])
        self._assert_have_only_logs(response, [
            'Existing context matches parameters; using existing context',
            'fetch page 0 start cursor missing; end cursor present',
            ])

    def test_non_admin_permissions_failures(self):
        actions.login(STUDENT_EMAIL)
        student_xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(
            settings.HtmlHookRESTHandler.XSRF_ACTION)

        response = self.get(ADMIN_SETTINGS_URL)
        self.assertEquals(200, response.status_int)
        payload = transforms.loads(response.body)
        self.assertEquals(401, payload['status'])
        self.assertEquals('Access denied.', payload['message'])

        response = self.put(ADMIN_SETTINGS_URL, {'request': transforms.dumps({
                'key': 'base:after_body_tag_begins',
                'xsrf_token': cgi.escape(student_xsrf_token),
                'payload': '{}'})})
        payload = transforms.loads(response.body)
        self.assertEquals(401, payload['status'])
        self.assertEquals('Access denied.', payload['message'])

        response = self.delete(ADMIN_SETTINGS_URL + '?xsrf_token=' +
                               cgi.escape(student_xsrf_token))
        self.assertEquals(200, response.status_int)
        payload = transforms.loads(response.body)
        self.assertEquals(401, payload['status'])
        self.assertEquals('Access denied.', payload['message'])

    def test_stable_ids(self):
        self._course.add_assessment()
        unit2 = self._course.add_unit()
        self._course.add_assessment()
        self._course.add_unit()
        self._course.add_assessment()
        self._course.add_unit()
        self._course.add_assessment()
        self._course.add_unit()
        self._course.add_assessment()
        self._course.add_unit()
        self._course.add_assessment()
        self._course.add_assessment()
        self._course.add_assessment()
        self._course.add_unit()
        self._course.save()

        response = transforms.loads(self.get(
            '/test/rest/data/units/items').body)
        self.assertListEqual(['2', '4', '6', '8', '10', '14'],
                             [u['unit_id'] for u in response['data']])

        self._course.delete_unit(unit2)
        self._course.save()

        response = transforms.loads(self.get(
            '/test/rest/data/units/items').body)
        self.assertListEqual(['4', '6', '8', '10', '14'],
                             [u['unit_id'] for u in response['data']])

    def test_cron_job(self):
        # ensure chunks don't exist
        response = self.get(
            'modules/drive/item/content?key=3', expect_errors=True)
        self.assertEqual(response.status_code, 404)

        response = self.get(
            'modules/drive/item/content?key=5', expect_errors=True)
        self.assertEqual(response.status_code, 404)

        # sync
        response = self.get('/cron/drive/sync')
        self.assertEqual(response.status_code, 200)
        self.execute_all_deferred_tasks()

        # now chunks should exist
        response = self.get('modules/drive/item/content?key=3')
        content = transforms.loads(response.body)
        payload = transforms.loads(content['payload'])
        self.assertIn('worksheets', payload)
        self.assertEqual(payload['id'], '3')

        response = self.get('modules/drive/item/content?key=5')
        self.assertEqual(
            response.headers['Content-Type'], 'text/html; charset=utf-8')
        self.assertEqual(response.body, '<p>Some HTML</p>')

def record_event_listener(source, user, data):
    # Note the code in this method has similiarities to methods in
    # models.event_transforms, but is (a) more limited in scope, and (b) needs
    # less background information marshalled about the structure of the course

    if source == 'tag-assessment':
        # Sent when the "Check Answer" button is presson in a lesson
        data = transforms.loads(data)
        question_scores = _get_questions_scores_from_single_item(data)

    elif source == 'attempt-lesson':
        # Sent when the "Grade Questions" button is pressed in a lesson
        # or when the "Check Answers" button is pressed in an assessment
        data = transforms.loads(data)
        question_scores = _get_questions_scores_from_many_items(data)

    elif source == 'submit-assessment':
        # Sent when an assignment is submitted.
        data = transforms.loads(data)['values']
        question_scores = _get_questions_scores_from_many_items(data)

    else:
        return

    scores_by_skill = collections.defaultdict(list)
    for question_score in question_scores:
        question = models.QuestionDAO.load(question_score.quid)
        for skill_id in question.dict.get(constants.SKILLS_KEY, []):
            scores_by_skill[skill_id].append(question_score.score)

    for skill_id, scores in scores_by_skill.iteritems():
        updater = Registry.get_updater(user.user_id(), skill_id)
        for score in scores:
            updater.update(score)
        updater.save()