def test_grade_timestamp(self, gradebook):
        """Is a timestamp correctly read in?"""
        self._copy_file("files/submitted-unchanged.ipynb", "source/ps1/p1.ipynb")
        run_command('nbgrader assign ps1 --db="{}" '.format(gradebook))

        self._copy_file("files/submitted-unchanged.ipynb", "submitted/foo/ps1/p1.ipynb")
        self._make_file('submitted/foo/ps1/timestamp.txt', "2015-02-02 15:58:23.948203 PST")

        self._copy_file("files/submitted-changed.ipynb", "submitted/bar/ps1/p1.ipynb")
        self._make_file('submitted/bar/ps1/timestamp.txt', "2015-02-01 14:58:23.948203 PST")

        run_command('nbgrader autograde ps1 --db="{}"'.format(gradebook))

        assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
        assert os.path.isfile("autograded/foo/ps1/timestamp.txt")
        assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
        assert os.path.isfile("autograded/bar/ps1/timestamp.txt")

        gb = Gradebook(gradebook)
        submission = gb.find_submission('ps1', 'foo')
        assert submission.total_seconds_late > 0
        submission = gb.find_submission('ps1', 'bar')
        assert submission.total_seconds_late == 0

        # make sure it still works to run it a second time
        run_command('nbgrader autograde ps1 --db="{}"'.format(gradebook))

    def test_grade_timestamp(self):
        """Is a timestamp correctly read in?"""
        with self._temp_cwd(["files/submitted-unchanged.ipynb", "files/submitted-changed.ipynb"]):
            dbpath = self._setup_db()

            os.makedirs('source/ps1')
            shutil.copy('submitted-unchanged.ipynb', 'source/ps1/p1.ipynb')
            self._run_command('nbgrader assign ps1 --db="{}" '.format(dbpath))

            os.makedirs('submitted/foo/ps1')
            shutil.move('submitted-unchanged.ipynb', 'submitted/foo/ps1/p1.ipynb')
            with open('submitted/foo/ps1/timestamp.txt', 'w') as fh:
                fh.write("2015-02-02 15:58:23.948203 PST")

            os.makedirs('submitted/bar/ps1')
            shutil.move('submitted-changed.ipynb', 'submitted/bar/ps1/p1.ipynb')
            with open('submitted/bar/ps1/timestamp.txt', 'w') as fh:
                fh.write("2015-02-01 14:58:23.948203 PST")

            self._run_command('nbgrader autograde ps1 --db="{}"'.format(dbpath))

            assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
            assert os.path.isfile("autograded/foo/ps1/timestamp.txt")
            assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
            assert os.path.isfile("autograded/bar/ps1/timestamp.txt")

            gb = Gradebook(dbpath)
            submission = gb.find_submission('ps1', 'foo')
            assert submission.total_seconds_late > 0
            submission = gb.find_submission('ps1', 'bar')
            assert submission.total_seconds_late == 0

            # make sure it still works to run it a second time
            self._run_command('nbgrader autograde ps1 --db="{}"'.format(dbpath))

 def _setup_db(self):
     dbpath = self._init_db()
     gb = Gradebook(dbpath)
     gb.add_assignment("ps1", duedate="2015-02-02 14:58:23.948203 PST")
     gb.add_student("foo")
     gb.add_student("bar")
     return dbpath

    def test_add_remove_extra_notebooks(self, db):
        """Are extra notebooks added and removed?"""
        gb = Gradebook(db)
        assignment = gb.add_assignment("ps1")

        self._copy_file("files/test.ipynb", "source/ps1/test.ipynb")
        run_command('nbgrader assign ps1 --db="{}"'.format(db))

        gb.db.refresh(assignment)
        assert len(assignment.notebooks) == 1
        notebook1 = gb.find_notebook("test", "ps1")

        self._copy_file("files/test.ipynb", "source/ps1/test2.ipynb")
        run_command('nbgrader assign ps1 --db="{}" --force'.format(db))

        gb.db.refresh(assignment)
        assert len(assignment.notebooks) == 2
        gb.db.refresh(notebook1)
        notebook2 = gb.find_notebook("test2", "ps1")

        os.remove("source/ps1/test2.ipynb")
        run_command('nbgrader assign ps1 --db="{}" --force'.format(db))

        gb.db.refresh(assignment)
        assert len(assignment.notebooks) == 1
        gb.db.refresh(notebook1)
        with pytest.raises(InvalidRequestError):
            gb.db.refresh(notebook2)

    def _clean_old_notebooks(self, assignment_id, student_id):
        gb = Gradebook(self.db_url)
        assignment = gb.find_assignment(assignment_id)
        regexp = os.path.join(
            self._format_source("(?P<assignment_id>.*)", "(?P<student_id>.*)"),
            "(?P<notebook_id>.*).ipynb")

        # find a set of notebook ids for new notebooks
        new_notebook_ids = set([])
        for notebook in self.notebooks:
            m = re.match(regexp, notebook)
            if m is None:
                raise RuntimeError("Could not match '%s' with regexp '%s'", notebook, regexp)
            gd = m.groupdict()
            if gd['assignment_id'] == assignment_id and gd['student_id'] == student_id:
                new_notebook_ids.add(gd['notebook_id'])

        # pull out the existing notebook ids
        old_notebook_ids = set(x.name for x in assignment.notebooks)

        # no added or removed notebooks, so nothing to do
        if old_notebook_ids == new_notebook_ids:
            return

        # some notebooks have been removed, but there are submissions associated
        # with the assignment, so we don't want to overwrite stuff
        if len(assignment.submissions) > 0:
            self.fail("Cannot modify existing assignment '%s' because there are submissions associated with it", assignment)

        # remove the old notebooks
        for notebook_id in (old_notebook_ids - new_notebook_ids):
            self.log.warning("Removing notebook '%s' from the gradebook", notebook_id)
            gb.remove_notebook(notebook_id, assignment_id)

    def test_grade(self, gradebook):
        """Can files be graded?"""
        self._copy_file("files/submitted-unchanged.ipynb", "source/ps1/p1.ipynb")
        run_command('nbgrader assign ps1 --db="{}" '.format(gradebook))

        self._copy_file("files/submitted-unchanged.ipynb", "submitted/foo/ps1/p1.ipynb")
        self._copy_file("files/submitted-changed.ipynb", "submitted/bar/ps1/p1.ipynb")
        run_command('nbgrader autograde ps1 --db="{}"'.format(gradebook))

        assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
        assert not os.path.isfile("autograded/foo/ps1/timestamp.txt")
        assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
        assert not os.path.isfile("autograded/bar/ps1/timestamp.txt")

        gb = Gradebook(gradebook)
        notebook = gb.find_submission_notebook("p1", "ps1", "foo")
        assert notebook.score == 1
        assert notebook.max_score == 4
        assert notebook.needs_manual_grade == False

        comment1 = gb.find_comment(0, "p1", "ps1", "foo")
        comment2 = gb.find_comment(1, "p1", "ps1", "foo")
        assert comment1.comment == "No response."
        assert comment2.comment == "No response."

        notebook = gb.find_submission_notebook("p1", "ps1", "bar")
        assert notebook.score == 2
        assert notebook.max_score == 4
        assert notebook.needs_manual_grade == True

        comment1 = gb.find_comment(0, "p1", "ps1", "bar")
        comment2 = gb.find_comment(1, "p1", "ps1", "bar")
        assert comment1.comment == None
        assert comment2.comment == None

    def test_save_cells(self, db):
        """Ensure cells are saved into the database"""
        self._copy_file("files/test.ipynb", "source/ps1/test.ipynb")

        gb = Gradebook(db)
        gb.add_assignment("ps1")

        run_command('nbgrader assign ps1 --db="{}"'.format(db))

        notebook = gb.find_notebook("test", "ps1")
        assert len(notebook.grade_cells) == 8

    def init_assignment(self, assignment_id, student_id):
        super(AutogradeApp, self).init_assignment(assignment_id, student_id)

        # try to get the student from the database, and throw an error if it
        # doesn't exist
        gb = Gradebook(self.db_url)
        try:
            gb.find_student(student_id)
        except MissingEntry:
            if self.create_student:
                self.log.warning("Creating student with ID '%s'", student_id)
                gb.add_student(student_id)
            else:
                self.fail("No student with ID '%s' exists in the database", student_id)

        # try to read in a timestamp from file
        src_path = self._format_source(assignment_id, student_id)
        timestamp = self._get_existing_timestamp(src_path)
        if timestamp:
            submission = gb.update_or_create_submission(
                assignment_id, student_id, timestamp=timestamp)
            self.log.info("%s submitted at %s", submission, timestamp)

            # if the submission is late, print out how many seconds late it is
            if timestamp and submission.total_seconds_late > 0:
                self.log.warning("%s is %s seconds late", submission, submission.total_seconds_late)

        else:
            submission = gb.update_or_create_submission(assignment_id, student_id)

        # copy files over from the source directory
        self.log.info("Overwriting files with master versions from the source directory")
        dest_path = self._format_dest(assignment_id, student_id)
        source_path = self.directory_structure.format(
            nbgrader_step=self.source_directory,
            student_id='.',
            assignment_id=assignment_id)
        source_files = utils.find_all_files(source_path, self.ignore + ["*.ipynb"])

        # copy them to the build directory
        for filename in source_files:
            dest = os.path.join(dest_path, os.path.relpath(filename, source_path))
            ensure_dir_exists(os.path.dirname(dest))
            if not os.path.normpath(dest) == os.path.normpath(filename):
                self.log.info("Linking %s -> %s", filename, dest)
                link_or_copy(filename, dest)

        # ignore notebooks that aren't in the database
        notebooks = []
        for notebook in self.notebooks:
            notebook_id = os.path.splitext(os.path.basename(notebook))[0]
            try:
                gb.find_notebook(notebook_id, assignment_id)
            except MissingEntry:
                self.log.warning("Skipping unknown notebook: %s", notebook)
                continue
            else:
                notebooks.append(notebook)
        self.notebooks = notebooks

    def init_assignment(self, assignment_id, student_id):
        super(AssignApp, self).init_assignment(assignment_id, student_id)

        # try to get the assignment from the database, and throw an error if it
        # doesn't exist
        gb = Gradebook(self.db_url)
        try:
            gb.find_assignment(assignment_id)
        except MissingEntry:
            if self.create_assignment:
                self.log.warning("Creating assignment '%s'", assignment_id)
                gb.add_assignment(assignment_id)
            else:
                self.fail("No assignment called '%s' exists in the database", assignment_id)

    def test_save_cells(self):
        """Ensure cells are saved into the database"""
        with self._temp_cwd(["files/test.ipynb"]):
            os.makedirs('source/ps1')
            shutil.move("test.ipynb", "source/ps1/test.ipynb")

            dbpath = self._init_db()
            gb = Gradebook(dbpath)
            gb.add_assignment("ps1")

            self._run_command('nbgrader assign ps1 --db="{}"'.format(dbpath))

            notebook = gb.find_notebook("test", "ps1")
            assert_equal(len(notebook.grade_cells), 8)

    def preprocess(self, nb, resources):
        # pull information from the resources
        self.notebook_id = resources['nbgrader']['notebook']
        self.assignment_id = resources['nbgrader']['assignment']
        self.db_url = resources['nbgrader']['db_url']

        if self.notebook_id == '':
            raise ValueError("Invalid notebook id: '{}'".format(self.notebook_id))
        if self.assignment_id == '':
            raise ValueError("Invalid assignment id: '{}'".format(self.assignment_id))

        # create a place to put new cell information
        self.new_grade_cells = {}
        self.new_solution_cells = {}
        self.new_source_cells = {}

        # connect to the database
        self.gradebook = Gradebook(self.db_url)

        nb, resources = super(SaveCells, self).preprocess(nb, resources)

        # create the notebook and save it to the database
        self._create_notebook()

        return nb, resources

def gradebook(request, tempdir):
    # create a "class files" directory
    origdir = os.getcwd()
    os.mkdir("class_files")
    os.chdir("class_files")

    # copy files from the user guide
    source_path = os.path.join(os.path.dirname(__file__), "..", "..", "..", "docs", "source", "user_guide", "source")
    submitted_path = os.path.join(os.path.dirname(__file__), "..", "..", "..", "docs", "source", "user_guide", "submitted")

    shutil.copytree(os.path.join(os.path.dirname(__file__), source_path), "source")
    shutil.copytree(os.path.join(os.path.dirname(__file__), submitted_path), "submitted")

    # create the gradebook
    gb = Gradebook("sqlite:///gradebook.db")
    gb.add_assignment("Problem Set 1")
    gb.add_student("Bitdiddle", first_name="Ben", last_name="B")
    gb.add_student("Hacker", first_name="Alyssa", last_name="H")
    gb.add_student("Reasoner", first_name="Louis", last_name="R")

    # run nbgrader assign
    run_command([
        "nbgrader", "assign", "Problem Set 1",
        "--IncludeHeaderFooter.header=source/header.ipynb"
    ])

    # run the autograder
    run_command(["nbgrader", "autograde", "Problem Set 1"])

    def fin():
        os.chdir(origdir)
        shutil.rmtree("class_files")
    request.addfinalizer(fin)

    return gb

    def init_assignment(self, assignment_id, student_id):
        super(AssignApp, self).init_assignment(assignment_id, student_id)

        # try to get the assignment from the database, and throw an error if it
        # doesn't exist
        gb = Gradebook(self.db_url)
        try:
            gb.find_assignment(assignment_id)
        except MissingEntry:
            if self.create_assignment:
                self.log.warning("Creating assignment '%s'", assignment_id)
                gb.add_assignment(assignment_id)
            else:
                self.fail("No assignment called '%s' exists in the database", assignment_id)
        else:
            # check if there are any extra notebooks in the db that are no longer
            # part of the assignment, and if so, remove them
            if self.notebook_id == "*":
                self._clean_old_notebooks(assignment_id, student_id)

    def preprocess(self, nb, resources):
        # pull information from the resources
        self.notebook_id = resources['nbgrader']['notebook']
        self.assignment_id = resources['nbgrader']['assignment']
        self.db_url = resources['nbgrader']['db_url']

        # connect to the database
        self.gradebook = Gradebook(self.db_url)

        nb, resources = super(OverwriteCells, self).preprocess(nb, resources)

        return nb, resources

    def test_grade(self):
        """Can files be graded?"""
        with self._temp_cwd(["files/submitted-unchanged.ipynb", "files/submitted-changed.ipynb"]):
            dbpath = self._setup_db()

            os.makedirs('source/ps1')
            shutil.copy('submitted-unchanged.ipynb', 'source/ps1/p1.ipynb')
            self._run_command('nbgrader assign ps1 --db="{}" '.format(dbpath))

            os.makedirs('submitted/foo/ps1')
            shutil.move('submitted-unchanged.ipynb', 'submitted/foo/ps1/p1.ipynb')
            os.makedirs('submitted/bar/ps1')
            shutil.move('submitted-changed.ipynb', 'submitted/bar/ps1/p1.ipynb')
            self._run_command('nbgrader autograde ps1 --db="{}"'.format(dbpath))

            assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
            assert not os.path.isfile("autograded/foo/ps1/timestamp.txt")
            assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
            assert not os.path.isfile("autograded/bar/ps1/timestamp.txt")

            gb = Gradebook(dbpath)
            notebook = gb.find_submission_notebook("p1", "ps1", "foo")
            assert_equal(notebook.score, 1)
            assert_equal(notebook.max_score, 4)
            assert_equal(notebook.needs_manual_grade, False)

            comment1 = gb.find_comment(0, "p1", "ps1", "foo")
            comment2 = gb.find_comment(1, "p1", "ps1", "foo")
            assert_equal(comment1.comment, "No response.")
            assert_equal(comment2.comment, "No response.")

            notebook = gb.find_submission_notebook("p1", "ps1", "bar")
            assert_equal(notebook.score, 2)
            assert_equal(notebook.max_score, 4)
            assert_equal(notebook.needs_manual_grade, True)

            comment1 = gb.find_comment(0, "p1", "ps1", "bar")
            comment2 = gb.find_comment(1, "p1", "ps1", "bar")
            assert_equal(comment1.comment, None)
            assert_equal(comment2.comment, None)

def class_files(request, tempdir):
    # copy files from the user guide
    source_path = os.path.join(os.path.dirname(__file__), "..", "..", "..", "docs", "source", "user_guide", "source")
    shutil.copytree(os.path.join(os.path.dirname(__file__), source_path), "source")

    # create a fake ps1
    os.mkdir(os.path.join("source", "ps1"))
    with open(os.path.join("source", "ps1", "problem 1.ipynb"), "w") as fh:
        write_nb(new_notebook(), fh, 4)

    # create the gradebook
    gb = Gradebook("sqlite:///gradebook.db")
    gb.add_assignment("Problem Set 1")
    gb.add_assignment("ps1")
    gb.add_student("Bitdiddle", first_name="Ben", last_name="B")
    gb.add_student("Hacker", first_name="Alyssa", last_name="H")
    gb.add_student("Reasoner", first_name="Louis", last_name="R")

    return tempdir