except Exception, e:
            raise ExtensionRunError (str (e))

        write_cursor = cnn.cursor ()
        icursor = ICursor (cursor, self.INTERVAL_SIZE)
        icursor.execute (statement ("SELECT id, rev, composed_rev from scmlog where repository_id = ?",
                                    db.place_holder), (repo_id,))
        rs = icursor.fetchmany ()
        while rs:
            for commit_id, revision, composed_rev in rs:
                if commit_id in commits:
                    continue

                if composed_rev:
                    rev = revision.split ("|")[0]
                else:
                    rev = revision

                p = DBPatch (None, commit_id, self.get_patch_for_commit (rev))
                write_cursor.execute (statement (DBPatch.__insert__, self.db.place_holder),
                                      (p.id, p.commit_id, self.db.to_binary (p.patch)))

            rs = icursor.fetchmany ()

        cnn.commit ()
        write_cursor.close ()
        cursor.close ()
        cnn.close ()

register_extension ("Patches", Patches)

        def patch_generator(repo, repo_uri, repo_id, db, cursor):
            icursor = ICursor(cursor, self.INTERVAL_SIZE)
            icursor.execute(
                statement("SELECT id, rev, composed_rev " + "from scmlog where repository_id = ?", db.place_holder),
                (repo_id,),
            )

            rs = icursor.fetchmany()

            while rs:
                for commit_id, revision, composed_rev in rs:
                    # Get the patch
                    pj = PatchJob(revision, commit_id)

                    path = uri_to_filename(repo_uri)
                    pj.run(repo, path or repo.get_uri())

                    # Yield the patch to hunks
                    yield (pj.commit_id, pj.data, pj.rev)

                rs = icursor.fetchmany()

        profiler_start("Running PatchesAndHunks extension")

        hunks = Hunks()
        hunks.get_patches = patch_generator
        hunks.run(repo, uri, db)


register_extension("PatchesAndHunks", PatchesAndHunks)

        lastMonth = (maxDate.year - minDate.year) * 12 + maxDate.month - minDate.month
        self.cursor.execute("SELECT id FROM branches")
        branches = [row[0] for row in self.cursor.fetchall()]

        # Fill in months, with months considered
        theTableMonths = TableMonths(db, cnn, repo_id)
        for period in range(0, lastMonth):
            month = (minDate.month + period) % 12 + 1
            year = minDate.year + (period + minDate.month) // 12
            date = str(year) + "-" + str(month) + "-01"
            theTableMonths.add_pending_row((None, date))
        theTableMonths.insert_rows(write_cursor)

        # Fill in metrics_evo, with the metrics for monthly snapshots
        theTableMetricsEvo = TableMetricsEvo(db, cnn, repo_id)
        for branch in branches:
            for period in range(0, lastMonth):
                month = (minDate.month + period) % 12 + 1
                year = minDate.year + (period + minDate.month) // 12
                date = str(year) + "-" + str(month) + "-01"
                (loc, sloc, files) = self._metrics_period(branch, date)
                theTableMetricsEvo.add_pending_row((None, branch, date, loc, sloc, files))
            theTableMetricsEvo.insert_rows(write_cursor)
        cnn.commit()
        write_cursor.close()
        self.cursor.close()
        cnn.close()


register_extension("MetricsEvo", MetricsEvo)

            if i >= queuesize:
                printdbg("FileCount queue is now at %d, flushing to database", 
                         (i,))

                processed_jobs = self.__process_finished_jobs(job_pool, 
                                                              write_cursor, db)

                connection.commit()
                i = i - processed_jobs
                
                if processed_jobs < (queuesize / 5):
                    job_pool.join()
        
        job_pool.join()
        self.__process_finished_jobs(job_pool, write_cursor, db)
        read_cursor.close()
        connection.commit()
        connection.close()

        # This turns off the profiler and deletes its timings
        profiler_stop("Running FileCount extension", delete=True)
                
    def backout(self, repo, uri, db):
        update_statement = """update scmlog
                       set file_count = NULL
                       where repository_id = ?"""

        self._do_backout(repo, uri, db, update_statement)

register_extension("FileCount", FileCount)

            cursor.execute(statement(query, self.db.place_holder),
                           (h.rev,))

            fetched_row = cursor.fetchone()

            if fetched_row is not None:
                args.append((job.file_id, job.commit_id, h.start, h.end, fetched_row[0]))
            
        cursor.close()
        cnn.close()
        return args

    def get_max_id(self, db):
        return None

    def get_blames(self, cursor, repoid):
        query = "select distinct b.file_id, b.commit_id from line_blames b, files f " + \
                "where b.file_id = f.id and repository_id = ?"
        cursor.execute(statement(query, self.db.place_holder), (repoid,))
        return [(res[0], res[1]) for res in cursor.fetchall()]

    
    def backout(self, repo, uri, db):
        update_statement = """delete from lineblames where
                              commit_id in (select s.id from scmlog s
                                          where s.repository_id = ?)"""

        self._do_backout(repo, uri, db, update_statement)

register_extension("LineBlame", LineBlame)

        # Cursor for reading from the database
        cursor = cnn.cursor()
        # Cursor for writing to the database
        write_cursor = cnn.cursor()

        cursor.execute("SELECT MIN(date) FROM scmlog")
        minDate = cursor.fetchone()[0]
        cursor.execute("SELECT MAX(date) FROM scmlog")
        maxDate = cursor.fetchone()[0]
        cursor.execute("DROP TABLE IF EXISTS months")

        theMonthsTable = MonthsTable(db, cnn, repo)

        firstMonth = minDate.year * 12 + minDate.month
        lastMonth = maxDate.year * 12 + maxDate.month

        for period in range(firstMonth, lastMonth + 1):
            month = (period - 1) % 12 + 1
            year = (period - 1) // 12
            date = str(year) + "-" + str(month) + "-01"
            theMonthsTable.add_pending_row((period, year, month, date))
        theMonthsTable.insert_rows(write_cursor)
        cnn.commit()
        write_cursor.close()
        cursor.close()
        cnn.close()


# Register in the CVSAnalY extension system
register_extension("Months", Months)

        write_cursor = cnn.cursor()
        rs = cursor.fetchmany()
        while rs:
            commit_list = []

            for commit_id, revision, composed_rev in rs:
                if commit_id in commits:
                    continue

                if composed_rev:
                    rev = revision.split("|")[0]
                else:
                    rev = revision

                (added, removed) = counter.get_lines_for_revision(revision)
                commit_list.append(DBCommitLines(None, commit_id, added, removed))

            if commit_list:
                commits_lines = [(commit.id, commit.commit_id, commit.added, commit.removed) for commit in commit_list]
                write_cursor.executemany(statement(DBCommitLines.__insert__, self.db.place_holder), commits_lines)

            rs = cursor.fetchmany()

        cnn.commit()
        write_cursor.close()
        cursor.close()
        cnn.close()


register_extension("CommitsLOC", CommitsLOC)

        theWeeksTable = WeeksTable(db, cnn, repo)

        # The ISO year consists of 52 or 53 full weeks
        weeks_year_real = 52.1775
        weeks_year = int(weeks_year_real)+1
        
        minDateWeek = minDate.date().isocalendar()[1]
        maxDateWeek = maxDate.date().isocalendar()[1]
        
        firstWeek = minDate.year * int(weeks_year) + minDateWeek
        lastWeek = maxDate.year * int(weeks_year) + maxDateWeek
        
        for period in range (firstWeek, lastWeek+1):
            week = (period -1 ) % weeks_year + 1 
            year = (period - 1)// weeks_year
            # When used with the strptime() method, %U and %W are only used in 
            # calculations when the day of the week and the year are specified.
            date_time = datetime.strptime(str(year) + " " + str(week)+" 0", "%Y %U %w")
            if (date_time.year > year): continue
            date = date_time.strftime("%y-%m-%d")
            theWeeksTable.add_pending_row ((period, year, week, date))
        theWeeksTable.insert_rows (write_cursor)
        cnn.commit ()
        write_cursor.close ()
        cursor.close ()
        cnn.close ()

# Register in the CVSAnalY extension system
register_extension ("Weeks", Weeks)

        cursor.execute("SELECT MIN(date) FROM words_freq")
        minDate = cursor.fetchone()[0]
        cursor.execute("SELECT MAX(date) FROM words_freq")
        maxDate = cursor.fetchone()[0]

        lastMonth = (maxDate.year - minDate.year) * 12 + maxDate.month - minDate.month

        for period in range(0, lastMonth):
            wordsFreq = {}
            month = (minDate.month + period) % 12 + 1
            year = minDate.year + (period + minDate.month) // 12
            date = str(year) + "-" + str(month) + "-01"
            query = """SELECT word, times
               FROM words_freq
               WHERE date = '%s' ORDER BY times DESC"""
            cursor.execute(query % date)
            rows = cursor.fetchall()
            print '*** ' + date + ":",
            count = 0
            for (text, times) in rows:
                if not self._wordToExclude(text):
                    count += 1
                    print text + " (" + str(times) + ")",
                if count > 10:
                    break
            print


register_extension("MessageWordsPrint", MessageWordsPrint)

                job_pool.push (job)
                n_blames += 1
        
                if n_blames >= self.MAX_BLAMES:
                    processed_jobs = self.process_finished_jobs (job_pool, write_cursor)
                    n_blames -= processed_jobs
                    if processed_jobs<=self.MAX_BLAMES/5:
                        profiler_start("Joining unprocessed jobs")
                        job_pool.join()
                        profiler_stop("Joining unprocessed jobs", delete=True)
            except NotValidHunkWarning as e:
                printerr("Not a valid hunk: "+str(e))
            finally:
                file_rev = read_cursor.fetchone()

        job_pool.join ()
        self.process_finished_jobs (job_pool, write_cursor, True)

        try:
            self.__drop_cache(cnn)
        except:
            printdbg("Couldn't drop cache because of " + str(e))

        read_cursor.close ()
        write_cursor.close ()
        cnn.close()

        profiler_stop ("Running HunkBlame extension", delete = True)

register_extension ("HunkBlame", HunkBlame)

            execute_statement(statement(__insert__,
                                            self.db.place_holder),
                                  (patch_id, commit_id_new, file_id, old_class, new_class, old_function, new_function, 0),
                                  write_cursor,
                                  db,
                                  "\nCouldn't insert, duplicate patch?",
                                  exception=ExtensionRunError)
            #clear
            old_cla.clear()
            new_cla.clear()
            old_func.clear()
            new_func.clear()
            
        cnn.commit()
        write_cursor.close()
        cursor.close()
        cnn.close()
        profiler_stop("Running Patches extension", delete=True)
        
        end = time.time()
        print function_name_change_count, 'file change name!'
        print 'num of source file:', num_of_source
        print 'num of exception:', num_of_exception
        print 'num of non_source_file:', non_source_file    
        print 'num of files can not be recovered:', num_of_unrecovered
        print 'num_of_id1:', num_of_id1
        print 'consuming time: %ss' % str(end - start)

register_extension("Analyse_patch", Analyse_patch)
    

                execute_statement(statement(insert, db.place_holder),
                                  (file_id, commit_id,
                                   hunk.old_start_line,
                                   hunk.old_end_line,
                                   hunk.new_start_line,
                                   hunk.new_end_line),
                                   write_cursor,
                                   db,
                                   "Couldn't insert hunk, dup record?",
                                   exception=ExtensionRunError)

            connection.commit()
            progress.finished_one()

        read_cursor.close()
        connection.commit()
        connection.close()
        progress.done()

        # This turns off the profiler and deletes its timings
        profiler_stop("Running hunks extension", delete=True)

    def backout(self, repo, uri, db):
        update_statement = """delete from hunks
                              where commit_id in (select s.id from scmlog s
                                          where s.repository_id = ?)"""

        self._do_backout(repo, uri, db, update_statement)

register_extension("Hunks", Hunks)

                        set is_bug_fix = ?
                        where id = ?"""

            if self.fixes_bug(commit_message):
                is_bug_fix = 1
            else:
                is_bug_fix = 0

            execute_statement(statement(update, db.place_holder),
                              (is_bug_fix, row_id),
                              write_cursor,
                              db,
                              "Couldn't update scmlog",
                              exception=ExtensionRunError)

        read_cursor.close()
        connection.commit()
        connection.close()

        # This turns off the profiler and deletes its timings
        profiler_stop("Running BugFixMessage extension", delete=True)

    def backout(self, repo, uri, db):
        backout_statement = """update scmlog
                       set is_bug_fix = NULL
                       where repository_id = ?"""

        self._do_backout(repo, uri, db, backout_statement)
          
register_extension("BugFixMessage", BugFixMessage)

            wordsFreq = {}
            month = (minDate.month + period) % 12 + 1
            year = minDate.year + (period + minDate.month) // 12
            date = str(year) + "-" + str(month) + "-01"
            query = "SELECT log.message " + \
                "FROM scmlog log " + \
                "WHERE year(log.date) = %s " + \
                "AND month(log.date) = %s "
            cursor.execute (query % (year, month))
            rows = cursor.fetchall()
            for message in rows:
                words = [word.strip(":()[],.#<>*'`~")
                         for word in message[0].lower().split ()]
                for word in words:
                    if len(word) > 2:
                        if word in wordsFreq:
                            wordsFreq[word] += 1
                        else:
                            wordsFreq[word] = 1
            for word in wordsFreq:
                theTableWords.add_pending_row ((None, date,
                                                word, wordsFreq[word]))
            theTableWords.insert_rows (write_cursor)
        cnn.commit ()
        write_cursor.close ()
        cursor.close ()
        cnn.close ()

register_extension ("MessageWords", MessageWords)

            job = ContentJob(commit_id, file_id, rev, relative_path)
            job_pool.push(job)
            i = i + 1
            if i >= queuesize:
                printdbg("Content queue is now at %d, flushing to database", (i,))

                processed_jobs = self.__process_finished_jobs(job_pool, connection, db)
                i = i - processed_jobs
                if processed_jobs < (queuesize / 5):
                    job_pool.join()

        job_pool.join()
        self.__process_finished_jobs(job_pool, connection, db)

        read_cursor.close()
        connection.close()

        # This turns off the profiler and deletes it's timings
        profiler_stop("Running content extension", delete=True)

    def backout(self, repo, uri, db):
        update_statement = """delete from content where
                              commit_id in (select id from scmlog s
                                            where s.repository_id = ?)"""

        self._do_backout(repo, uri, db, update_statement)


register_extension("Content", Content)

                cadded = cremoved = 0
                if not theTableComLines.in_table(id):
                    (cadded, cremoved) = counter.get_lines_for_commit(commit)
                    theTableComLines.add_pending_row((None, id,
                                                      cadded, cremoved))
                tadded = tremoved = 0
                for path in counter.get_paths_for_commit(commit):
                    if not theTableComFilLines.in_table(str(id) + ',' + path):
                        (added, removed) = \
                            counter.get_lines_for_commit_file(commit, path)
                        theTableComFilLines.add_pending_row((None, id, path,
                                                             added, removed))
                        tadded += int(added)
                        tremoved += int(removed)
                # Sanity check
                if (cadded != tadded) or (cremoved != tremoved):
                    printerr("Sanity check failed: %d, %s, %d, %d, %d, %d" %
                             (id, commit, cadded, tadded, cremoved, tremoved))
                    printerr(counter.get_paths_for_commit(commit))
            theTableComLines.insert_rows(write_cursor)
            theTableComFilLines.insert_rows(write_cursor)
            if not rows:
                rows_left = False
        cnn.commit()
        write_cursor.close()
        cursor.close()
        cnn.close()


register_extension("CommitsLOCDet", CommitsLOCDet)

            + "not exists (select id from file_links where parent_id = a.file_id) "
            + "and f.repository_id = ? group by fid, fname"
        )

        cursor.execute(statement(query, db.place_holder), (repo_id,))
        write_cursor = cnn.cursor()
        rs = cursor.fetchmany()
        while rs:
            types = []

            for file_id, file_name in rs:
                if file_id in files:
                    continue

                type = guess_file_type(file_name)
                types.append(DBFileType(None, type, file_id))

            if types:
                file_types = [(type.id, type.file_id, type.type) for type in types]
                write_cursor.executemany(statement(DBFileType.__insert__, self.db.place_holder), file_types)

            rs = cursor.fetchmany()

        cnn.commit()
        write_cursor.close()
        cursor.close()
        cnn.close()


register_extension("FileTypes", FileTypes)

            if composed:
                rev = revision.split ("|")[0]
            else:
                rev = revision

            relative_path = fr.get_path ()
            printdbg ("Path for %d at %s -> %s", (file_id, rev, relative_path))

            if repo.get_type () == 'svn' and relative_path == 'tags':
                printdbg ("Skipping file %s", (relative_path,))
                continue

            job = BlameJob (file_id, commit_id, relative_path, rev)
            job_pool.push (job)
            n_blames += 1

            if n_blames >= self.MAX_BLAMES:
                job_pool.join()
                self.process_finished_jobs (job_pool, write_cursor)
                n_blames = 0
        job_pool.join ()
        self.process_finished_jobs (job_pool, write_cursor, True)

        read_cursor.close ()
        write_cursor.close ()
        cnn.close()

        profiler_stop ("Running Blame extension", delete = True)

register_extension ("Blame", Blame)

                        p.patch is not NULL""",
            db.place_holder), (repo_id,))
        nr_records = cursor.fetchone()[0]
        progress = Progress("[Extension PatchesLOC]", nr_records)

        patches = self.get_patches(repo, path or repo.get_uri(), repo_id, db,
                                   cursor)

        for commit_id, file_id, patch_content, rev in patches:
            (added, removed) = self.count_lines(patch_content)
            insert = """insert into patch_lines(file_id, commit_id,
                        added, removed)
                        values(?,?,?,?)"""
            execute_statement(statement(insert, db.place_holder),
                              (file_id, commit_id, added, removed),
                               cursor,
                               db,
                               "Couldn't insert patch, dup record?",
                               exception=ExtensionRunError)
            connection.commit()
            progress.finished_one()

        cursor.close()
        connection.commit()
        connection.close()
        progress.done()

        profiler_stop("Running PatchLOC extension", delete=True)

register_extension("PatchLOC", PatchLOC)