def test_reset_different_student_item(self, changed):
        # Create a submissions for two students
        submission = sub_api.create_submission(self.STUDENT_ITEM, "test answer")
        sub_api.set_score(submission["uuid"], 1, 2)

        other_student = copy.copy(self.STUDENT_ITEM)
        other_student.update(changed)
        submission = sub_api.create_submission(other_student, "other test answer")
        sub_api.set_score(submission["uuid"], 3, 4)

        # Reset the score for the first student
        sub_api.reset_score(
            self.STUDENT_ITEM["student_id"], self.STUDENT_ITEM["course_id"], self.STUDENT_ITEM["item_id"]
        )

        # The first student's scores should be reset
        self.assertIs(sub_api.get_score(self.STUDENT_ITEM), None)
        scores = sub_api.get_scores(self.STUDENT_ITEM["course_id"], self.STUDENT_ITEM["student_id"])
        self.assertNotIn(self.STUDENT_ITEM["item_id"], scores)

        # But the second student should still have a score
        score = sub_api.get_score(other_student)
        self.assertEqual(score["points_earned"], 3)
        self.assertEqual(score["points_possible"], 4)
        scores = sub_api.get_scores(other_student["course_id"], other_student["student_id"])
        self.assertIn(other_student["item_id"], scores)

    def test_override_with_no_score(self):

        sub_api.score_override(
            self.STUDENT_ITEM,
            8,
            10,
        )

        self.assertEqual(sub_api.get_score(self.STUDENT_ITEM)['points_earned'], 8)
        self.assertEqual(sub_api.get_score(self.STUDENT_ITEM)['points_possible'], 10)

    def test_peer_evaluation_workflow(self):
        tim = self._create_student_and_submission("Tim", "Tim's answer")
        bob = self._create_student_and_submission("Bob", "Bob's answer")
        sally = self._create_student_and_submission("Sally", "Sally's answer")
        jim = self._create_student_and_submission("Jim", "Jim's answer")
        buffy = self._create_student_and_submission("Buffy", "Buffy's answer")
        xander = self._create_student_and_submission("Xander", "Xander's answer")

        # Tim should not have a score, because he has not evaluated enough
        # peer submissions.
        scores = sub_api.get_score(STUDENT_ITEM)
        self.assertFalse(scores)

        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
        api.create_evaluation(
            bob["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )
        api.create_evaluation(
            sally["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )
        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
        api.create_evaluation(
            jim["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )
        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
        api.create_evaluation(
            buffy["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )
        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
        api.create_evaluation(
            xander["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )
        self.assertTrue(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))

        # Tim should not have a score, because his submission does not have
        # enough evaluations.
        scores = sub_api.get_score(STUDENT_ITEM)
        self.assertFalse(scores)

        api.create_evaluation(
            tim["uuid"], "Bob", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )
        api.create_evaluation(
            tim["uuid"], "Sally", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )
        api.create_evaluation(
            tim["uuid"], "Jim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
        )

        # Tim has met the critera, and should now have a score.
        scores = sub_api.get_score(STUDENT_ITEM)
        self.assertTrue(scores)
        self.assertEqual(6, scores[0]["points_earned"])
        self.assertEqual(12, scores[0]["points_possible"])

    def test_override_with_one_score(self):
        # Create a submission for the student and score it
        submission = sub_api.create_submission(self.STUDENT_ITEM, 'test answer')
        sub_api.set_score(submission['uuid'], 1, 10)

        sub_api.score_override(
            self.STUDENT_ITEM,
            5,
            10,
        )

        self.assertEqual(sub_api.get_score(self.STUDENT_ITEM)['points_earned'], 5)
        self.assertEqual(sub_api.get_score(self.STUDENT_ITEM)['points_possible'], 10)

    def test_delete_submission_scores(self, _lti_mock):
        user = UserFactory()
        problem_location = self.course_key.make_usage_key("dummy", "module")

        # Create a student module for the user
        StudentModule.objects.create(
            student=user, course_id=self.course_key, module_state_key=problem_location, state=json.dumps({})
        )

        # Create a submission and score for the student using the submissions API
        student_item = {
            "student_id": anonymous_id_for_user(user, self.course_key),
            "course_id": self.course_key.to_deprecated_string(),
            "item_id": problem_location.to_deprecated_string(),
            "item_type": "openassessment",
        }
        submission = sub_api.create_submission(student_item, "test answer")
        sub_api.set_score(submission["uuid"], 1, 2)

        # Delete student state using the instructor dash
        reset_student_attempts(self.course_key, user, problem_location, requesting_user=user, delete_module=True)

        # Verify that the student's scores have been reset in the submissions API
        score = sub_api.get_score(student_item)
        self.assertIs(score, None)

 def get_score(self, id=None):
     """
     Get student's current score.
     """
     score = submissions_api.get_score(self.student_submission_id(id))
     if score:
         return score['points_earned']

def _has_database_updated_with_new_score(
        user_id, scored_block_usage_key, expected_modified_time, score_deleted,
):
    """
    Returns whether the database has been updated with the
    expected new score values for the given problem and user.
    """
    score = get_score(user_id, scored_block_usage_key)

    if score is None:
        # score should be None only if it was deleted.
        # Otherwise, it hasn't yet been saved.
        return score_deleted
    elif score.module_type == 'openassessment':
        anon_id = anonymous_id_for_user(User.objects.get(id=user_id), scored_block_usage_key.course_key)
        course_id = unicode(scored_block_usage_key.course_key)
        item_id = unicode(scored_block_usage_key)

        api_score = sub_api.get_score(
            {
                "student_id": anon_id,
                "course_id": course_id,
                "item_id": item_id,
                "item_type": "openassessment"
            }
        )
        reported_modified_time = api_score.created_at
    else:
        reported_modified_time = score.modified

    return reported_modified_time >= expected_modified_time

    def test_delete_student_state_resets_scores(self):
        problem_location = self.course.id.make_usage_key('dummy', 'module')

        # Create a student module for the user
        StudentModule.objects.create(
            student=self.student,
            course_id=self.course.id,
            module_state_key=problem_location,
            state=json.dumps({})
        )

        # Create a submission and score for the student using the submissions API
        student_item = {
            'student_id': anonymous_id_for_user(self.student, self.course.id),
            'course_id': self.course.id.to_deprecated_string(),
            'item_id': problem_location.to_deprecated_string(),
            'item_type': 'openassessment'
        }
        submission = sub_api.create_submission(student_item, 'test answer')
        sub_api.set_score(submission['uuid'], 1, 2)

        # Delete student state using the instructor dash
        url = reverse('instructor_dashboard_legacy', kwargs={'course_id': self.course.id.to_deprecated_string()})
        response = self.client.post(url, {
            'action': 'Delete student state for module',
            'unique_student_identifier': self.student.email,
            'problem_for_student': problem_location.to_deprecated_string(),
        })

        self.assertEqual(response.status_code, 200)

        # Verify that the student's scores have been reset in the submissions API
        score = sub_api.get_score(student_item)
        self.assertIs(score, None)

    def test_delete_student_state_resets_scores(self):
        item_id = 'i4x://MITx/999/openassessment/b3dce2586c9c4876b73e7f390e42ef8f'

        # Create a student module for the user
        StudentModule.objects.create(
            student=self.student, course_id=self.course.id, module_state_key=item_id, state=json.dumps({})
        )

        # Create a submission and score for the student using the submissions API
        student_item = {
            'student_id': anonymous_id_for_user(self.student, self.course.id),
            'course_id': self.course.id,
            'item_id': item_id,
            'item_type': 'openassessment'
        }
        submission = sub_api.create_submission(student_item, 'test answer')
        sub_api.set_score(submission['uuid'], 1, 2)

        # Delete student state using the instructor dash
        url = reverse('instructor_dashboard_legacy', kwargs={'course_id': self.course.id})
        response = self.client.post(url, {
            'action': 'Delete student state for module',
            'unique_student_identifier': self.student.email,
            'problem_for_student': 'openassessment/b3dce2586c9c4876b73e7f390e42ef8f',
        })

        self.assertEqual(response.status_code, 200)

        # Verify that the student's scores have been reset in the submissions API
        score = sub_api.get_score(student_item)
        self.assertIs(score, None)

    def test_delete_submission_scores(self):
        user = UserFactory()
        course_id = 'ora2/1/1'
        item_id = 'i4x://ora2/1/openassessment/b3dce2586c9c4876b73e7f390e42ef8f'

        # Create a student module for the user
        StudentModule.objects.create(
            student=user, course_id=course_id, module_state_key=item_id, state=json.dumps({})
        )

        # Create a submission and score for the student using the submissions API
        student_item = {
            'student_id': anonymous_id_for_user(user, course_id),
            'course_id': course_id,
            'item_id': item_id,
            'item_type': 'openassessment'
        }
        submission = sub_api.create_submission(student_item, 'test answer')
        sub_api.set_score(submission['uuid'], 1, 2)

        # Delete student state using the instructor dash
        reset_student_attempts(course_id, user, item_id, delete_module=True)

        # Verify that the student's scores have been reset in the submissions API
        score = sub_api.get_score(student_item)
        self.assertIs(score, None)

    def test_delete_submission_scores(self):
        user = UserFactory()
        problem_location = self.course_key.make_usage_key('dummy', 'module')

        # Create a student module for the user
        StudentModule.objects.create(
            student=user,
            course_id=self.course_key,
            module_state_key=problem_location,
            state=json.dumps({})
        )

        # Create a submission and score for the student using the submissions API
        student_item = {
            'student_id': anonymous_id_for_user(user, self.course_key),
            'course_id': self.course_key.to_deprecated_string(),
            'item_id': problem_location.to_deprecated_string(),
            'item_type': 'openassessment'
        }
        submission = sub_api.create_submission(student_item, 'test answer')
        sub_api.set_score(submission['uuid'], 1, 2)

        # Delete student state using the instructor dash
        reset_student_attempts(
            self.course_key, user, problem_location,
            delete_module=True
        )

        # Verify that the student's scores have been reset in the submissions API
        score = sub_api.get_score(student_item)
        self.assertIs(score, None)

    def test_reset_with_no_scores(self):
        sub_api.reset_score(
            self.STUDENT_ITEM["student_id"], self.STUDENT_ITEM["course_id"], self.STUDENT_ITEM["item_id"]
        )
        self.assertIs(sub_api.get_score(self.STUDENT_ITEM), None)

        scores = sub_api.get_scores(self.STUDENT_ITEM["course_id"], self.STUDENT_ITEM["student_id"])
        self.assertEqual(len(scores), 0)

 def get_score(self, student_id=None):
     """
     Return student's current score.
     """
     score = submissions_api.get_score(
         self.get_student_item_dict(student_id)
     )
     if score:
         return score['points_earned']

    def test_override_after_reset_score(self):
        # Create a submission for the student and score it
        submission = sub_api.create_submission(self.STUDENT_ITEM, 'test answer')
        sub_api.set_score(submission['uuid'], 1, 10)

        # Reset score
        sub_api.reset_score(
            self.STUDENT_ITEM['student_id'],
            self.STUDENT_ITEM['course_id'],
            self.STUDENT_ITEM['item_id'],
        )

        sub_api.score_override(
            self.STUDENT_ITEM,
            5,
            10,
        )

        self.assertEqual(sub_api.get_score(self.STUDENT_ITEM)['points_earned'], 5)
        self.assertEqual(sub_api.get_score(self.STUDENT_ITEM)['points_possible'], 10)

    def test_clear_state(self):
        # Create a submission, give it a score, and verify that score exists
        submission = api.create_submission(STUDENT_ITEM, ANSWER_ONE)
        api.set_score(submission["uuid"], 11, 12)
        score = api.get_score(STUDENT_ITEM)
        self._assert_score(score, 11, 12)
        self.assertEqual(score['submission_uuid'], submission['uuid'])

        # Reset the score with clear_state=True
        # This should set the submission's score to None, and make it unavailable to get_submissions
        api.reset_score(
            STUDENT_ITEM["student_id"],
            STUDENT_ITEM["course_id"],
            STUDENT_ITEM["item_id"],
            clear_state=True,
        )
        score = api.get_score(STUDENT_ITEM)
        self.assertIsNone(score)
        subs = api.get_submissions(STUDENT_ITEM)
        self.assertEqual(subs, [])

    def setUp(self):
        super(TestOraAggregateDataIntegration, self).setUp()
        # Create submissions and assessments
        self.submission = self._create_submission(STUDENT_ITEM)
        self.scorer_submission = self._create_submission(SCORER_ITEM)
        self.earned_points = 1
        self.possible_points = 2
        peer_api.get_submission_to_assess(self.scorer_submission['uuid'], 1)
        self.assessment = self._create_assessment(self.scorer_submission['uuid'])

        sub_api.set_score(self.submission['uuid'], self.earned_points, self.possible_points)
        self.score = sub_api.get_score(STUDENT_ITEM)
        peer_api.get_score(self.submission['uuid'], {'must_be_graded_by': 1, 'must_grade': 0})
        self._create_assessment_feedback(self.submission['uuid'])

    def test_reset_with_one_score(self):
        # Create a submission for the student and score it
        submission = sub_api.create_submission(self.STUDENT_ITEM, "test answer")
        sub_api.set_score(submission["uuid"], 1, 2)

        # Reset scores
        sub_api.reset_score(
            self.STUDENT_ITEM["student_id"], self.STUDENT_ITEM["course_id"], self.STUDENT_ITEM["item_id"]
        )

        # Expect that no scores are available for the student
        self.assertIs(sub_api.get_score(self.STUDENT_ITEM), None)
        scores = sub_api.get_scores(self.STUDENT_ITEM["course_id"], self.STUDENT_ITEM["student_id"])
        self.assertEqual(len(scores), 0)

    def test_asynch_generate_score(self, xblock, mock_is_admin):
        # Test that AI grading, which creates assessments asynchronously,
        # updates the workflow so students can receive a score.
        mock_is_admin.return_value = True

        # Train classifiers for the problem
        self.request(xblock, 'schedule_training', json.dumps({}), response_format='json')

        # Submit a response
        self.request(xblock, 'submit', self.SUBMISSION, response_format='json')

        # BEFORE viewing the grade page, check that we get a score
        score = sub_api.get_score(xblock.get_student_item_dict())
        self.assertIsNot(score, None)
        self.assertEqual(score['submission_uuid'], xblock.submission_uuid)

    def test_reset_with_multiple_scores(self):
        # Create a submission for the student and score it
        submission = sub_api.create_submission(self.STUDENT_ITEM, 'test answer')
        sub_api.set_score(submission['uuid'], 1, 2)
        sub_api.set_score(submission['uuid'], 2, 2)

        # Reset scores
        sub_api.reset_score(
            self.STUDENT_ITEM['student_id'],
            self.STUDENT_ITEM['course_id'],
            self.STUDENT_ITEM['item_id'],
        )

        # Expect that no scores are available for the student
        self.assertIs(sub_api.get_score(self.STUDENT_ITEM), None)
        scores = sub_api.get_scores(self.STUDENT_ITEM['course_id'], self.STUDENT_ITEM['student_id'])
        self.assertEqual(len(scores), 0)

def _has_db_updated_with_new_score(self, scored_block_usage_key, **kwargs):
    """
    Returns whether the database has been updated with the
    expected new score values for the given problem and user.
    """
    if kwargs['score_db_table'] == ScoreDatabaseTableEnum.courseware_student_module:
        score = get_score(kwargs['user_id'], scored_block_usage_key)
        found_modified_time = score.modified if score is not None else None

    elif kwargs['score_db_table'] == ScoreDatabaseTableEnum.submissions:
        score = sub_api.get_score(
            {
                "student_id": kwargs['anonymous_user_id'],
                "course_id": unicode(scored_block_usage_key.course_key),
                "item_id": unicode(scored_block_usage_key),
                "item_type": scored_block_usage_key.block_type,
            }
        )
        found_modified_time = score['created_at'] if score is not None else None
    else:
        assert kwargs['score_db_table'] == ScoreDatabaseTableEnum.overrides
        score = GradesService().get_subsection_grade_override(
            user_id=kwargs['user_id'],
            course_key_or_id=kwargs['course_id'],
            usage_key_or_id=kwargs['usage_id']
        )
        found_modified_time = score.modified if score is not None else None

    if score is None:
        # score should be None only if it was deleted.
        # Otherwise, it hasn't yet been saved.
        db_is_updated = kwargs['score_deleted']
    else:
        db_is_updated = found_modified_time >= from_timestamp(kwargs['expected_modified_time'])

    if not db_is_updated:
        log.info(
            u"Grades: tasks._has_database_updated_with_new_score is False. Task ID: {}. Kwargs: {}. Found "
            u"modified time: {}".format(
                self.request.id,
                kwargs,
                found_modified_time,
            )
        )

    return db_is_updated