def test_get_single_peer_grading_item(self):
        for i in xrange(0,settings.MIN_TO_USE_PEER):
            test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
            test_sub.save()
            handle_submission(test_sub)
            test_grader = test_util.get_grader("IN")
            test_grader.submission=test_sub
            test_grader.save()

            test_sub.state = SubmissionState.finished
            test_sub.previous_grader_type = "IN"
            test_sub.posted_results_back_to_queue = True
            test_sub.save()

        test_sub = test_util.get_sub("PE", ALTERNATE_STUDENT, LOCATION, "PE")
        test_sub.save()
        handle_submission(test_sub)
        test_sub.is_duplicate = False
        test_sub.save()

        pl = peer_grading_util.PeerLocation(LOCATION, STUDENT_ID)
        found, grading_item = pl.next_item()
        self.assertEqual(found, True)

        pl = peer_grading_util.PeerLocation(LOCATION,"1")
        subs_graded = pl.graded()

    def test_get_next_submission_true(self):
        test_sub = test_util.get_sub("PE", "1", LOCATION, "PE")
        test_sub.save()
        grader = test_util.get_grader("BC")
        grader.submission = test_sub
        grader.grader_id = "2"
        grader.save()

        for i in xrange(0,settings.MIN_TO_USE_PEER):
            test_sub = test_util.get_sub("PE", "1", LOCATION, "PE")
            test_sub.save()
            grader = test_util.get_grader("IN")
            grader.submission = test_sub
            grader.save()

        test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
        test_sub.save()
        content = self.c.get(
            GET_NEXT,
            data={'grader_id' : STUDENT_ID, "location" : LOCATION},
            )

        body = json.loads(content.content)

        self.assertEqual(body['success'], True)

    def test_get_next_submission_true(self):
        test_sub = test_util.get_sub("PE", "1", LOCATION, "PE")
        test_sub.save()
        grader = test_util.get_grader("BC")
        grader.submission = test_sub
        grader.grader_id = "2"
        grader.save()

        pl = peer_grading_util.PeerLocation(LOCATION, "1")
        control = SubmissionControl(pl.latest_submission())

        for i in xrange(0, control.minimum_to_use_peer):
            test_sub = test_util.get_sub("PE", "1", LOCATION, "PE")
            test_sub.save()
            grader = test_util.get_grader("IN")
            grader.submission = test_sub
            grader.save()

        test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
        test_sub.save()
        content = self.c.get(
            GET_NEXT,
            data={'grader_id' : STUDENT_ID, "location" : LOCATION},
            )

        body = json.loads(content.content)

        self.assertEqual(body['success'], True)

    def test_submission_location(self):
        Submission.objects.all().delete()
        test_sub=test_util.get_sub("IN",STUDENT_ID, LOCATION)
        test_sub.save()

        test_sub2=test_util.get_sub("PE",STUDENT_ID, LOCATION)
        test_sub2.state = SubmissionState.finished
        test_sub2.previous_grader_type = "PE"
        test_sub2.save()

        test_grader_in=test_util.get_grader("IN")
        test_grader_in.submission=test_sub2
        test_grader_in.save()

        test_grader_pe=test_util.get_grader("PE")
        test_grader_pe.submission=test_sub2
        test_grader_pe.save()


        sl = StaffLocation(LOCATION)
        self.assertEqual(sl.location_submissions().count(),2)
        self.assertEqual(sl.all_pending_count(),1)
        self.assertEqual(sl.graded_count(),1)
        self.assertEqual(sl.pending_count(),1)
        self.assertEqual(len(sl.graded_submission_text()),1)
        test_sub2.delete()
        next_item_id = sl.item_to_score()[1]
        self.assertEqual(next_item_id ,test_sub.id)

        test_sub = Submission.objects.get(id=next_item_id)
        self.assertEqual(test_sub.state,SubmissionState.being_graded)
        test_sub.state = SubmissionState.waiting_to_be_graded
        test_sub.save()
        self.assertEqual(sl.next_item()[1],test_sub.id)

 def test_check_name_uniqueness(self):
     success, name_unique = grader_util.check_name_uniqueness("temp", LOCATION, "course_id")
     self.assertTrue(name_unique)
     test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
     test_sub.save()
     test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION + "1", "PE")
     test_sub.save()
     success, name_unique = grader_util.check_name_uniqueness("id", LOCATION, "course_id")
     self.assertFalse(name_unique)

    def test_get_next_get_finished_subs(self):
        Submission.objects.all().delete()
        all_students = [STUDENT_ID, ALTERNATE_STUDENT, STUDENT3]
        # setup 3 submissions from 3 students, passed basic check
        submissions = []
        for student in all_students:
            test_sub = test_util.get_sub("PE", student, LOCATION, preferred_grader_type="PE")
            test_sub.next_grader_type="PE"
            test_sub.is_duplicate=False
            test_sub.save()
            submissions.append(test_sub)
            bc_grader = test_util.get_grader("BC")
            bc_grader.submission = test_sub
            bc_grader.save()

        # have them each grade the other two and call that finished
        for student in all_students:
            for submission in Submission.objects.all().exclude(student_id=student):
                test_grader = test_util.get_grader("PE")
                test_grader.grader_id = student
                test_grader.submission = submission
                test_grader.save()
        for sub in submissions:
            sub.state = SubmissionState.finished
            sub.posted_results_back_to_queue = True
            sub.save()

        pls = []
        for student in all_students:
            pls.append(peer_grading_util.PeerLocation(LOCATION, student))

        # check that each student graded 2, and so no submissions are pending
        for pl in pls:
            self.assertEqual(pl.graded_count(),2)
            self.assertEqual(pl.pending_count(),0)
            # check that next_item() cannot returns a submission because each of these students
            # has graded the submissions by the other 2 students
            found, _ = pl.next_item()
            self.assertFalse(found)

        # now a 4th student comes along and submits.  They should get something to grade despite nothing pending
        student4 = "10"
        test_sub = test_util.get_sub("PE", student4, LOCATION, preferred_grader_type="PE")
        test_sub.next_grader_type="PE"
        test_sub.is_duplicate=False
        test_sub.control_fields=json.dumps({'peer_grade_finished_submissions_when_none_pending': True})
        test_sub.save()

        pl4 = peer_grading_util.PeerLocation(LOCATION, student4)
        self.assertEqual(pl4.pending_count(),0)
        found, next_sub_id = pl4.next_item()
        self.assertTrue(found)
        student4_sub_to_grade = Submission.objects.get(id=next_sub_id)
        self.assertIn(student4_sub_to_grade, submissions)
        self.assertEqual(student4_sub_to_grade.state, SubmissionState.being_graded)

    def test_get_required(self):
        pl = peer_grading_util.PeerLocation(LOCATION, STUDENT_ID)
        student_required = pl.required_count()
        test_sub = test_util.get_sub("PE", ALTERNATE_STUDENT, LOCATION, "PE")
        test_sub.save()

        self.assertEqual(pl.required_count(), student_required)

        test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
        test_sub.save()

        self.assertEqual(pl.required_count(), settings.REQUIRED_PEER_GRADING_PER_STUDENT + student_required)

    def test_check_if_grading_finished_for_duplicates(self):
        test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
        test_sub.save()
        handle_submission(test_sub)

        sl = StaffLocation(LOCATION)
        control = SubmissionControl(sl.latest_submission())

        for i in xrange(0, control.minimum_to_use_peer):
            test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
            test_sub.save()
            handle_submission(test_sub)
            test_grader = test_util.get_grader("IN")
            test_grader.submission=test_sub
            test_grader.save()

            test_sub.state = SubmissionState.finished
            test_sub.previous_grader_type = "IN"
            test_sub.posted_results_back_to_queue = True
            test_sub.save()

        test_sub2 = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
        test_sub2.save()
        handle_submission(test_sub2)
        self.assertTrue(test_sub2.is_duplicate)

        success = expire_submissions.check_if_grading_finished_for_duplicates()
        self.assertEqual(success, True)
        test_sub2.is_duplicate = False
        test_sub2.save()

        test_sub3 = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
        test_sub3.is_duplicate = False
        test_sub3.save()

        self.assertEqual(test_sub3.is_duplicate, False)
        test_sub3.has_been_duplicate_checked = False
        test_sub3.save()
        expire_submissions.mark_student_duplicate_submissions()
        test_sub3 = Submission.objects.get(id=test_sub3.id)
        self.assertEqual(test_sub3.is_duplicate,True)

        test_sub3.duplicate_submission_id = None
        test_sub3.is_plagiarized = False
        test_sub3.save()
        expire_submissions.add_in_duplicate_ids()
        test_sub3 = Submission.objects.get(id=test_sub3.id)
        self.assertTrue(test_sub3.duplicate_submission_id is not None)

    def test_put_result(self):
        sub = test_util.get_sub("IN",STUDENT_ID,LOCATION)
        sub.save()
        post_dict={
            'feedback': "test feedback",
            'submission_id' : 1 ,
            'grader_type' : "ML" ,
            'status' : "S",
            'confidence' : 1,
            'grader_id' : 1,
            'score' : 1,
            'errors' : "test",
            "rubric_scores_complete" : True,
            "rubric_scores" : json.dumps([1,1]),
            }

        content = self.c.post(
            PUT_URL,
            post_dict,
        )

        body=json.loads(content.content)

        return_code=body['success']

        #Male sure that function returns true
        self.assertEqual(return_code,True)

        sub=Submission.objects.get(id=1)
        successful_grader_count=sub.get_successful_graders().count()

        #Make sure that grader object is actually created!
        self.assertEqual(successful_grader_count,1)

    def _message_submission(self, success, score=None, submission_id=None):
        sub = test_util.get_sub("IN",STUDENT_ID,LOCATION)
        sub.save()
        grade=test_util.get_grader("IN")
        grade.submission=sub
        grade.save()
        grader_id = grade.grader_id
        if submission_id is None:
            submission_id = sub.id

        message = {
            'grader_id': grader_id,
            'submission_id': submission_id,
            'feedback': "This is test feedback",
            'student_info': test_util.get_student_info(STUDENT_ID),
        }
        if score is not None:
            message['score'] = score
        
        content = {
            'xqueue_header': test_util.get_xqueue_header(),
            'xqueue_body': json.dumps(message),
        }
        content = self.c.post(
                SUBMIT_MESSAGE_URL,
                content
        )
        body = json.loads(content.content)
        self.assertEqual(body['success'], success)

    def test_get_submissions_that_have_expired(self):
        test_sub = test_util.get_sub("IN", STUDENT_ID, LOCATION)
        test_sub.save()

        expired_submissions = expire_submissions.get_submissions_that_have_expired(Submission.objects.all())

        self.assertEqual(len(expired_submissions),1)

    def test_save_grade_with_no_rubrics_and_submission_unknown(self):
        """
        Test save grade when submission is mark as unknown and rubric score are not provided.
        """

        test_sub = test_util.get_sub("PE", "blah", LOCATION, "PE")
        test_sub.save()

        test_dict = {
            'location': LOCATION,
            'grader_id': STUDENT_ID,
            'submission_id': 1,
            'score': 5,
            'feedback': 'feedback',
            'submission_key': 'string',
            'submission_flagged': False,
            'answer_unknown': True,
            'rubric_scores_complete': False,
            'rubric_scores': [],
            }

        content = self.c.post(
            SAVE_GRADE,
            test_dict,
        )

        body = json.loads(content.content)
        #Should succeed, as we created a submission above that save_grade can use
        self.assertEqual(body["success"], True)

        sub = Submission.objects.get(id=1)

        #Score should be 0.
        self.assertEqual(sub.get_last_grader().score, 0)

 def test_get_problems_student_has_tried(self):
     success, sub_list = grader_util.get_problems_student_has_tried(STUDENT_ID, "course_id")
     self.assertTrue(isinstance(sub_list, list))
     test_sub = test_util.get_sub("PE", STUDENT_ID, LOCATION, "PE")
     test_sub.save()
     success, sub_list = grader_util.get_problems_student_has_tried(STUDENT_ID, "course_id")
     self.assertTrue(len(sub_list)>0)

 def test_skip_problem_next_grader_type_ml(self):
     """Next grader type should be ML."""
     test_sub = test_util.get_sub("IN", LOCATION, STUDENT_ID, course_id=COURSE_ID, preferred_grader_type="ML")
     test_sub.save()
     self.save_grade(True, True)
     test_sub = Submission.objects.get(id=test_sub.id)
     self.assertEqual(test_sub.next_grader_type, "ML")

    def test_is_calibrated_false(self):

        content = self.c.get(
            IS_CALIBRATED,
            data=self.get_data,
        )

        body=json.loads(content.content)

        #No records exist for given problem_id, so calibration check should fail and return an error
        self.assertEqual(body['success'], False)


        sub=test_util.get_sub("IN",STUDENT_ID,LOCATION)
        sub.save()

        content = self.c.get(
            IS_CALIBRATED,
            data=self.get_data,
        )

        body=json.loads(content.content)

        #Now one record exists for given problem_id, so calibration check should return False (student is not calibrated)
        self.assertEqual(body['calibrated'], False)

    def test_save_grade_true(self):
        test_sub=test_util.get_sub("PE", "blah",LOCATION, "PE")
        test_sub.save()

        test_dict={
            'location': LOCATION,
            'grader_id': STUDENT_ID,
            'submission_id': 1,
            'score': 0,
            'feedback': 'feedback',
            'submission_key' : 'string',
            'submission_flagged' : False,
            'rubric_scores_complete' : True,
            'rubric_scores' : [1,1],
            }

        content = self.c.post(
            SAVE_GRADE,
            test_dict,
        )

        body=json.loads(content.content)
        #Should succeed, as we created a submission above that save_grade can use
        self.assertEqual(body['success'], True)

        sub=Submission.objects.get(id=1)

        #Ensure that grader object is created
        self.assertEqual(sub.grader_set.all().count(),1)

    def test_course_data_real(self):
        """
        Test to ensure that generating data for a course results in proper csv output.
        """

        submission_count = 100

        # Generate our submissions and grade them.
        for i in xrange(0, submission_count):
            sub = test_util.get_sub("IN", STUDENT_ID, LOCATION, course_id=COURSE)
            sub.save()

            grade = test_util.get_grader("IN")
            grade.submission = sub
            grade.save()

        # Generate the data file and get the filename.
        data_filename = self.get_course_data_file(COURSE)

        with open(data_filename, 'rb') as data_file:
            # Ensure that we have data.
            data = data_file.read()
            self.assertGreater(len(data), 0)

            data_file.seek(0)
            reader = csv.reader(data_file, delimiter=',', doublequote=True, quoting=csv.QUOTE_MINIMAL)
            rows = []
            for row in reader:
                rows.append(row)

            # The number of rows should equal the number of submissions plus one for the header row.
            self.assertEqual(len(rows), submission_count + 1)

    def test_submission_course(self):
        test_sub=test_util.get_sub("IN",STUDENT_ID,LOCATION)
        test_sub.save()

        sc = StaffCourse(COURSE_ID)
        self.assertEqual(len(sc.locations()),1)
        self.assertEqual(sc.notifications()[1],True)
        self.assertEqual(sc.next_item()[1],test_sub.id)

    def test_get_flagged_submissions(self):
        test_sub = test_util.get_sub("PE", ALTERNATE_STUDENT, LOCATION, "PE")
        test_sub.state = SubmissionState.flagged
        test_sub.save()
        
        success, flagged_submission_list = peer_grading_util.get_flagged_submissions(COURSE_ID)

        self.assertTrue(len(flagged_submission_list)==1)

    def test_save_grade_true(self):
        test_sub=test_util.get_sub("IN",STUDENT_ID, LOCATION)
        test_sub.save()

        #Should work because submission was just created
        self.save_grade(True, False)
        test_sub = Submission.objects.get(id=test_sub.id)
        # make sure the submission isn't skipped
        self.assertNotEqual(test_sub.next_grader_type,"ML")