def _attach_filter_data(handler, element):
    course = courses.Course(handler)
    unit_list = []
    assessment_list = []
    for unit in course.get_units():
        if verify.UNIT_TYPE_UNIT == unit.type:
            unit_list.append((unit.unit_id, unit.title))
        if unit.is_assessment():
            assessment_list.append((unit.unit_id, unit.title))

    lessons_map = {}
    for (unit_id, unused_title) in unit_list:
        lessons_map[unit_id] = [
            (l.lesson_id, l.title) for l in course.get_lessons(unit_id)]

    element.add_attribute(
        data_units=transforms.dumps(unit_list + assessment_list),
        data_lessons_map=transforms.dumps(lessons_map),
        data_questions=transforms.dumps(
            [(question.id, question.description) for question in sorted(
                models.QuestionDAO.get_all(), key=lambda q: q.description)]
        ),
        data_groups=transforms.dumps(
            [(group.id, group.description) for group in sorted(
                models.QuestionGroupDAO.get_all(), key=lambda g: g.description)]
        ),
        data_types=transforms.dumps([
            (models.QuestionDTO.MULTIPLE_CHOICE, 'Multiple Choice'),
            (models.QuestionDTO.SHORT_ANSWER, 'Short Answer')])
    )

 def validate(self, value):
     value = super(_SerializedProperty, self).validate(value)
     try:
         transforms.dumps(value)
     except TypeError, e:
         raise db.BadValueError(
             '%s is not JSON-serializable; error was "%s"' % (value, e))

 def test_adding_empty_question_group(self):
     QG_URL = '/%s%s' % (
         self.COURSE_NAME,
         question_group_editor.QuestionGroupRESTHandler.URI)
     xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(
         question_group_editor.QuestionGroupRESTHandler.XSRF_TOKEN)
     description = 'Question Group'
     version = (
         question_group_editor.QuestionGroupRESTHandler.SCHEMA_VERSIONS[0])
     payload = {
         'description': description,
         'version': version,
         'introduction': '',
         'items': []
     }
     response = self.put(QG_URL, {'request': transforms.dumps({
         'xsrf_token': cgi.escape(xsrf_token),
         'payload': transforms.dumps(payload)})})
     self.assertEquals(response.status_int, 200)
     payload = transforms.loads(response.body)
     self.assertEquals(payload['status'], 200)
     self.assertEquals(payload['message'], 'Saved.')
     question_groups_table = self._soup_table()
     self.assertEquals(
         question_groups_table.select('.description-cell a')[0].text.strip(),
         description)

    def test_set_hook_content(self):
        html_text = "<table><tbody><tr><th>;&lt;&gt;</th></tr></tbody></table>"

        response = self.put(
            ADMIN_SETTINGS_URL,
            {
                "request": transforms.dumps(
                    {
                        "xsrf_token": cgi.escape(self.xsrf_token),
                        "key": "base.after_body_tag_begins",
                        "payload": transforms.dumps({"hook_content": html_text}),
                    }
                )
            },
        )
        self.assertEquals(200, response.status_int)
        response = transforms.loads(response.body)
        self.assertEquals(200, response["status"])
        self.assertEquals("Saved.", response["message"])

        # And verify that the changed text appears on course pages.
        # NOTE that text is as-is; no escaping of special HTML
        # characters should have been done.
        response = self.get(BASE_URL)
        self.assertIn(html_text, response.body)

    def setUp(self):
        super(RolesTest, self).setUp()

        actions.login(COURSE_ADMIN_EMAIL, is_admin=True)
        payload_dict = {
            'name': COURSE_NAME,
            'title': 'Roles Test',
            'admin_email': COURSE_ADMIN_EMAIL}
        request = {
            'payload': transforms.dumps(payload_dict),
            'xsrf_token': crypto.XsrfTokenManager.create_xsrf_token(
                'add-course-put')}
        response = self.testapp.put('/rest/courses/item?%s' % urllib.urlencode(
            {'request': transforms.dumps(request)}), {})
        self.assertEquals(response.status_int, 200)
        sites.setup_courses('course:/%s::ns_%s, course:/:/' % (
                COURSE_NAME, COURSE_NAME))
        actions.logout()

        config.Registry.test_overrides[roles.GCB_ADMIN_LIST.name] = (
            '[%s]' % SITE_ADMIN_EMAIL)

        # pylint: disable-msg=protected-access
        self.old_registered_permission = roles.Roles._REGISTERED_PERMISSIONS
        roles.Roles._REGISTERED_PERMISSIONS = {}

        config.Registry.test_overrides[models.CAN_USE_MEMCACHE.name] = True

    def test_update_wont_clobber_status(self):
        # This fixture should already have a sync time
        with utils.Namespace(self.app_context.namespace):
            dto = drive_models.DriveSyncDAO.load('6')
            self.assertIsNotNone(dto.last_synced)

        # update existing record
        self.assertRestStatus(self.put('rest/modules/drive/item', {
            'request': transforms.dumps({
                'xsrf_token':
                    crypto.XsrfTokenManager.create_xsrf_token(
                        'drive-item-rest'),
                'key': '6',
                'payload': transforms.dumps({
                    'sync_interval': 'hour',
                    'version': '1.0',
                    'availability': 'public',
                }),
            }),
        }), 200)

        # The sync time should still exist
        with utils.Namespace(self.app_context.namespace):
            dto = drive_models.DriveSyncDAO.load('6')
            self.assertIsNotNone(dto.last_synced)

    def fill_values(app_context, template_values, job):
        accumulated_question_answers, accumulated_assessment_answers = (
            transforms.loads(job.output))

        template_values['accumulated_question_answers'] = transforms.dumps(
            accumulated_question_answers)
        template_values['accumulated_assessment_answers'] = transforms.dumps(
            accumulated_assessment_answers)

    def test_malformed_requests(self):
        response = self.put(ADMIN_SETTINGS_URL, {})
        payload = transforms.loads(response.body)
        self.assertEquals(400, payload["status"])
        self.assertEquals('Missing "request" parameter.', payload["message"])

        response = self.put(ADMIN_SETTINGS_URL, {"request": "asdfasdf"})
        payload = transforms.loads(response.body)
        self.assertEquals(400, payload["status"])
        self.assertEquals('Malformed "request" parameter.', payload["message"])

        response = self.put(
            ADMIN_SETTINGS_URL, {"request": transforms.dumps({"xsrf_token": cgi.escape(self.xsrf_token)})}
        )
        payload = transforms.loads(response.body)
        self.assertEquals(400, payload["status"])
        self.assertEquals('Request missing "key" parameter.', payload["message"])

        response = self.put(
            ADMIN_SETTINGS_URL,
            {
                "request": transforms.dumps(
                    {"xsrf_token": cgi.escape(self.xsrf_token), "key": "base:after_body_tag_begins"}
                )
            },
        )
        payload = transforms.loads(response.body)
        self.assertEquals(400, payload["status"])
        self.assertEquals('Request missing "payload" parameter.', payload["message"])

        response = self.put(
            ADMIN_SETTINGS_URL,
            {
                "request": transforms.dumps(
                    {
                        "xsrf_token": cgi.escape(self.xsrf_token),
                        "key": "base:after_body_tag_begins",
                        "payload": "asdfsdfasdf",
                    }
                )
            },
        )
        payload = transforms.loads(response.body)
        self.assertEquals(400, payload["status"])
        self.assertEquals('Malformed "payload" parameter.', payload["message"])

        response = self.put(
            ADMIN_SETTINGS_URL,
            {
                "request": transforms.dumps(
                    {"xsrf_token": cgi.escape(self.xsrf_token), "key": "base:after_body_tag_begins", "payload": "{}"}
                )
            },
        )
        payload = transforms.loads(response.body)
        self.assertEquals(400, payload["status"])
        self.assertEquals('Payload missing "hook_content" parameter.', payload["message"])

    def post(self):
        name = COURSE_EXPLORER_SETTINGS.name
        request = transforms.loads(self.request.get('request'))

        if not self.assert_xsrf_token_or_fail(
                request, self.ACTION, {}):
            return

        if not roles.Roles.is_course_admin(self.app_context):
            transforms.send_json_response(
                self, 401, 'Access denied.', {})
            return

        raw_data = transforms.loads(request.get('payload'))
        raw_data.pop('logo', None)
        try:
            data = transforms.json_to_dict(
                raw_data, schema_provider(None).get_json_schema_dict())
        except (TypeError, ValueError) as err:
            self.validation_error(err.replace('\n', ' '))
            return

        logo = self.request.POST.get('logo')
        logo_uploaded = isinstance(logo, cgi.FieldStorage)
        if logo_uploaded:
            data['logo_bytes_base64'] = base64.b64encode(logo.file.read())
            data['logo_mime_type'] = logo.type

        with common_utils.Namespace(appengine_config.DEFAULT_NAMESPACE_NAME):
            entity = config.ConfigPropertyEntity.get_by_key_name(name)

            if entity is None:
                entity = config.ConfigPropertyEntity(key_name=name)
                old_value = None
            else:
                old_value = entity.value

            # Don't delete the logo.
            if not logo_uploaded and old_value:
                old_dict = transforms.loads(old_value)
                if (
                        'logo_bytes_base64' in old_dict and
                        'logo_mime_type' in old_dict):
                    data['logo_bytes_base64'] = old_dict['logo_bytes_base64']
                    data['logo_mime_type'] = old_dict['logo_mime_type']

            entity.value = transforms.dumps(data)
            entity.is_draft = False
            entity.put()

            # is this necessary?
            models.EventEntity.record(
                'put-property', users.get_current_user(), transforms.dumps({
                    'name': name,
                    'before': str(old_value), 'after': str(entity.value)}))

        transforms.send_file_upload_response(self, 200, 'Saved.')

def add_header_diagrams(handler, app_context, unit, lesson, student):
    # Make sure that checkbox in Skills dashboard section is checked
    env = courses.Course.get_environ(app_context)
    if SETTINGS_SCHEMA_SECTION_NAME not in env:
        return None
    if not env[SETTINGS_SCHEMA_SECTION_NAME].get(SETTING_LOCAL_GRAPH_ENABLED):
        return None

    if isinstance(student, models.TransientStudent):
        my_skill_map = skill_map.SkillMap.load(handler.get_course())
    else:
        my_skill_map = skill_map.SkillMap.load(
            handler.get_course(), user_id=student.user_id)
    skills = my_skill_map.get_skills_for_lesson(lesson.lesson_id)
    skills_set = set(skills) # We convert this to a set for O(1) lookup time.

    course = handler.get_course()
    nodes = [get_node_data(course, my_skill_map, skill, student)
             for skill in skills]
    edges = []
    for node in nodes:
        node['highlight'] = True

    for index in xrange(len(skills)):
        skill = skill_map.filter_visible_lessons(handler, student,
                                                 skills[index])

        prerequisites = skill.prerequisites
        for prereq_skill in prerequisites:
            if prereq_skill not in skills_set:
                # Add 0-based link index. The source is the node that's about to
                # be placed at the end, and the target is the one at the current
                # index.
                edges.append({'source': len(nodes), 'target': index})
                nodes.append(get_node_data(course, my_skill_map, prereq_skill,
                                           student))

        successors = my_skill_map.successors(skill)
        for succ_skill in successors:
            if succ_skill not in skills_set:
                # Add 0-based link index. The source is the node at the current
                # index, and the target is the one that's about to be placed at
                # the end.
                edges.append({'source': index, 'target': len(nodes)})
                nodes.append(get_node_data(course, my_skill_map, succ_skill,
                                           student))

    template_values = {'nodes': transforms.dumps(nodes),
                       'edges': transforms.dumps(edges)}

    title = 'Skill graph'
    content = jinja2.Markup(
    handler.get_template('unit_header.html', [TEMPLATES_DIR]
        ).render(template_values))

    return {'title': title, 'content': content}

 def post_settings(self, payload, upload_files=None):
     response = self.post('rest/explorer-settings', {
         'request': transforms.dumps({
             'xsrf_token': crypto.XsrfTokenManager.create_xsrf_token(
                 'explorer-settings-rest'),
             'payload': transforms.dumps(payload),
         })
     }, upload_files=upload_files)
     self.assertEqual(response.status_code, 200)
     return response

 def test_nested_json_flattens_correctly(self):
     dict1 = dict(aaa=111)
     dict2 = dict(aa=11, bb=22, cc=transforms.dumps(dict1))
     dict3 = dict(a=transforms.dumps(dict2), b=2)
     json = transforms.loads(transforms.dumps(dict3))
     flattened_json = mapreduce.CsvGenerator._flatten_json(json)
     result_json = transforms.loads(
         transforms.dumps(
             {'a_aa': '11', 'a_bb': '22', 'b': '2', 'a_cc_aaa': '111'}))
     self.assertEquals(result_json, flattened_json)

    def test_admin_post_change_report_allowed(self):
        xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(
            'config_override')
        response = self.post(
            '/admin?action=config_override&name=%s' %
            config.REPORT_ALLOWED.name,
            {'xsrf_token': xsrf_token})
        response = self.get('/rest/config/item?key=%s' %
                            config.REPORT_ALLOWED.name)
        payload = {
            'name': config.REPORT_ALLOWED.name,
            'value': True,
            'is_draft': False,
            }
        message = {
            'key': config.REPORT_ALLOWED.name,
            'payload': transforms.dumps(payload),
            'xsrf_token': crypto.XsrfTokenManager.create_xsrf_token(
                'config-property-put'),
            }
        response = self.put('/rest/config/item',
                            {'request': transforms.dumps(message)})
        self.assertEqual(200, response.status_int)

        payload = {
            'name': config.REPORT_ALLOWED.name,
            'value': False,
            'is_draft': False,
            }
        message = {
            'key': config.REPORT_ALLOWED.name,
            'payload': transforms.dumps(payload),
            'xsrf_token': crypto.XsrfTokenManager.create_xsrf_token(
                'config-property-put'),
            }
        response = self.put('/rest/config/item',
                            {'request': transforms.dumps(message)})
        self.assertEqual(200, response.status_int)

        expected = [{
            messaging.Message._INSTALLATION: FAKE_INSTALLATION_ID,
            messaging.Message._TIMESTAMP: FAKE_TIMESTAMP,
            messaging.Message._VERSION: os.environ['GCB_PRODUCT_VERSION'],
            messaging.Message._METRIC: messaging.Message.METRIC_REPORT_ALLOWED,
            messaging.Message._VALUE: True,
            messaging.Message._SOURCE: messaging.Message.ADMIN_SOURCE,
        }, {
            messaging.Message._INSTALLATION: FAKE_INSTALLATION_ID,
            messaging.Message._TIMESTAMP: FAKE_TIMESTAMP,
            messaging.Message._VERSION: os.environ['GCB_PRODUCT_VERSION'],
            messaging.Message._METRIC: messaging.Message.METRIC_REPORT_ALLOWED,
            messaging.Message._VALUE: False,
            messaging.Message._SOURCE: messaging.Message.ADMIN_SOURCE,
        }]
        self.assertEquals(expected, MockSender.get_sent())

    def _post(self, xsrf_token=None, payload=None):
        request = {}
        if xsrf_token is None:
            xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(oeditor.EditorPrefsRestHandler.XSRF_TOKEN)
        request["xsrf_token"] = xsrf_token

        if payload is None:
            payload = {"location": self.location, "key": self.key, "state": self.EDITOR_STATE}
        request["payload"] = transforms.dumps(payload)

        data = {"request": transforms.dumps(request)}
        return self.post("oeditor/rest/editor_prefs", data, expect_errors=True)

 def test_hook_rest_edit_removes_from_old_location(self):
     actions.update_course_config(COURSE_NAME,
                                  {'html_hooks': {'foo': {'bar': 'zab'}}})
     actions.update_course_config(COURSE_NAME,
                                  {'foo': {'bar': 'baz'}})
     response = self.put(ADMIN_SETTINGS_URL, {'request': transforms.dumps({
             'xsrf_token': cgi.escape(self.xsrf_token),
             'key': 'foo.bar',
             'payload': transforms.dumps({'hook_content': 'BAZ'})})})
     env = self.course.get_environ(self.app_context)
     self.assertNotIn('bar', env['foo'])
     self.assertEquals('BAZ', env['html_hooks']['foo']['bar'])

    def get_markup(self, job):
        """Returns Jinja markup for question stats analytics."""

        errors = []
        stats_calculated = False
        update_message = safe_dom.Text('')

        accumulated_question_answers = None
        accumulated_assessment_answers = None

        if not job:
            update_message = safe_dom.Text(
                'Multiple-choice question statistics have not been calculated '
                'yet.')
        else:
            if job.status_code == jobs.STATUS_CODE_COMPLETED:
                accumulated_question_answers, accumulated_assessment_answers = (
                    transforms.loads(job.output))
                stats_calculated = True
                update_message = safe_dom.Text("""
                    Multiple-choice question statistics were last updated at
                    %s in about %s second(s).""" % (
                        job.updated_on.strftime(
                            HUMAN_READABLE_TIME_FORMAT),
                        job.execution_time_sec))
            elif job.status_code == jobs.STATUS_CODE_FAILED:
                update_message = safe_dom.NodeList().append(
                    safe_dom.Text("""
                        There was an error updating multiple-choice question
                        statistics. Here is the message:""")
                ).append(
                    safe_dom.Element('br')
                ).append(
                    safe_dom.Element('blockquote').add_child(
                        safe_dom.Element('pre').add_text('\n%s' % job.output)))
            else:
                update_message = safe_dom.Text("""
                    Multiple-choice question statistics update started at %s
                    and is running now. Please come back shortly.""" % (
                        job.updated_on.strftime(HUMAN_READABLE_TIME_FORMAT)))

        return jinja2.utils.Markup(self.get_template(
            'question_stats.html', [os.path.dirname(__file__)]
        ).render({
            'errors': errors,
            'accumulated_question_answers': transforms.dumps(
                accumulated_question_answers),
            'accumulated_assessment_answers': transforms.dumps(
                accumulated_assessment_answers),
            'stats_calculated': stats_calculated,
            'update_message': update_message,
        }, autoescape=True))

 def _put_translation(self, data, locale, title, html):
     resource_key = str(i18n_dashboard.ResourceBundleKey(
         announcements.ResourceHandlerAnnouncement.TYPE,
         db.Key(encoded=data['key']).id(), locale))
     xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(
         i18n_dashboard.TranslationConsoleRestHandler.XSRF_TOKEN_NAME)
     request = {
         'key': resource_key,
         'xsrf_token': xsrf_token,
         'validate': False,
         'payload': transforms.dumps({
             'title': 'Announcements',
             'key': resource_key,
             'source_locale': 'en_US',
             'target_locale': locale,
             'sections': [
                 {
                     'name': 'title',
                     'label': 'Title',
                     'type': 'string',
                     'source_value': '',
                     'data': [{
                         'source_value': data['title'],
                         'target_value': title,
                         'verb': 1,  # verb NEW
                         'old_source_value': '',
                         'changed': True
                     }]
                 },
                 {
                     'name': 'html',
                     'label': 'Body',
                     'type': 'html',
                     'source_value': '',
                     'data': [{
                         'source_value': data['html'],
                         'target_value': html,
                         'verb': 1,  # verb NEW
                         'old_source_value': '',
                         'changed': True
                     }]
                 }
             ]
         })
     }
     response = self.put(
         self.base +
         i18n_dashboard.TranslationConsoleRestHandler.URL.lstrip(),
         {'request': transforms.dumps(request)})
     self.assertEquals(200, response.status_int)
     response = transforms.loads(response.body)
     self.assertEquals(200, response['status'])

    def test_add_file_in_unsupported_dir(self):

        # Upload file via REST POST
        xsrf_token = crypto.XsrfTokenManager.create_xsrf_token(
            filer.TextAssetRESTHandler.XSRF_TOKEN_NAME)
        key = 'assets/unsupported/foo.js'
        contents = 'alert("Hello, world");'
        response = self.put(TEXT_ASSET_URL, {'request': transforms.dumps({
            'xsrf_token': cgi.escape(xsrf_token),
            'key': key,
            'payload': transforms.dumps({'contents': contents})})})
        payload = transforms.loads(response.body)
        self.assertEquals(400, payload['status'])

    def get_data(self, key=None, xsrf_token=None):
        if key is None:
            key = self.key
        if xsrf_token is None:
            xsrf_token = utils.XsrfTokenManager.create_xsrf_token('rating')

        request = {
            'xsrf_token': xsrf_token,
            'payload': transforms.dumps({'key': key})
        }
        return transforms.loads(
            self.get('rest/modules/rating?%s' % urllib.urlencode(
                {'request': transforms.dumps(request)})).body)

 def _initiate_import(self):
     # GET and then POST to the clone-a-course REST handler.
     response = transforms.loads(self.get(self.import_start_url).body)
     payload = transforms.loads(response['payload'])
     content = transforms.dumps({
         'course': payload['course'],
         })
     request = transforms.dumps({
         'xsrf_token': response['xsrf_token'],
         'payload': content,
         })
     response = self.put(self.import_start_url, {'request': request})
     self.assertEquals(transforms.loads(response.body)['status'], 200)