def __reader(self, templog, queue):
        def commit_cb(item):
            queue.put(item)

        printdbg("DBProxyContentHandler: thread __reader started")
        templog.foreach(commit_cb, self.order)
        printdbg("DBProxyContentHandler: thread __reader finished")

 def __load_caches_from_disk(self):
     printdbg("DBContentHandler: Loading caches from disk (%s)", (self.cache_file,))
     f = open(self.cache_file, 'r')
     (self.file_cache, self.moves_cache, self.deletes_cache,
      self.revision_cache, self.branch_cache, self.tags_cache,
      self.people_cache) = load(f)
     f.close()

        def ensure_person(person):
            profiler_start("Ensuring person %s for repository %d",
                            (person.name, self.repo_id))
            printdbg("DBContentHandler: ensure_person %s <%s>",
                      (person.name, person.email))
            cursor = self.cursor

            name = to_utf8(person.name)
            email = person.email

            if email is not None:
                email = to_utf8(email).decode("utf-8")

            cursor.execute(statement(
                "SELECT id from people where name = ?", self.db.place_holder),
                (to_utf8(name).decode("utf-8"),))
            rs = cursor.fetchone()
            if not rs:
                p = DBPerson(None, person)

                cursor.execute(statement(DBPerson.__insert__,
                                self.db.place_holder),
                                (p.id, to_utf8(p.name).decode("utf-8"),
                                 email))
                person_id = p.id
            else:
                person_id = rs[0]

            profiler_stop("Ensuring person %s for repository %d",
                           (person.name, self.repo_id), True)

            return person_id

def _get_uri_and_repo(path):
    """ Get a URI and repositoryhandler object for a path.

    This function returns a URI as a string, and the repositoryhandler
    object that represents that URI. They are returned together as a tuple.

    Args:
      path: The path to the repository
    """
    # Create repository
    if path is not None:
        try:
            printdbg("Creating repositoryhandler instance")
            repo = create_repository_from_path(path)
            repo.timeout = 120
        except RepositoryUnknownError:
            printerr("Path %s doesn't seem to point to a repository " + \
                     "supported by cvsanaly", (path,))
            sys.exit(1)
        except Exception, e:
            printerr("Unknown error creating repository for path %s (%s)",
                     (path, str(e)))
            sys.exit(1)
        uri = repo.get_uri_for_path(path)
        return (uri, repo)

 def __save_caches_to_disk(self):
     printdbg("DBContentHandler: Saving caches to disk (%s)", (self.cache_file,))
     cache = [self.file_cache, self.moves_cache, self.deletes_cache,
              self.revision_cache, self.branch_cache, self.tags_cache,
              self.people_cache]
     f = open(self.cache_file, 'w')
     dump(cache, f, -1)
     f.close()

def _get_extensions_manager(extensions, hard_order=False):
    try:
        printdbg("Starting ExtensionsManager")
        emg = ExtensionsManager(extensions,
                                hard_order=hard_order)
        return emg
    except InvalidExtension, e:
        printerr("Invalid extension %s", (e.name,))
        sys.exit(1)

    def __execute (self):
        q = "%s LIMIT %d OFFSET %d" % (self.query, self.interval_size, self.i)
        self.i += self.interval_size

        printdbg (q)
        if self.args:
            self.cursor.execute (q, self.args)
        else:
            self.cursor.execute (q)

        self.need_exec = False

    def end(self):
        # flush pending inserts
        printdbg("DBContentHandler: flushing pending inserts")
        self.__insert_many()

        # Save the caches to disk
        profiler_start("Saving caches to disk")
        self.__save_caches_to_disk()
        profiler_stop("Saving caches to disk", delete=True)

        self.cursor.close()
        self.cnn.close()
        self.cnn = None

def statement(str, ph_mark):
    if "?" == ph_mark or "?" not in str:
        printdbg(str)
        return str

    tokens = str.split("'")
    for i in range(0, len(tokens), 2):
        tokens[i] = tokens[i].replace("?", ph_mark)

    retval = "'".join(tokens)
    printdbg(retval)
    
    return retval

        def ensure_tag(tag):
            profiler_start("Ensuring tag %s for repository %d", (tag, self.repo_id))
            printdbg("DBContentHandler: ensure_tag %s", (tag,))
            cursor = self.cursor

            cursor.execute(statement("SELECT id from tags where name = ?", self.db.place_holder), (tag,))
            rs = cursor.fetchone()
            if not rs:
                t = DBTag(None, tag)
                cursor.execute(statement(DBTag.__insert__, self.db.place_holder), (t.id, t.name))
                tag_id = t.id
            else:
                tag_id = rs[0]

            profiler_stop("Ensuring tag %s for repository %d", (tag, self.repo_id), True)

            return tag_id

        def ensure_branch(branch):
            profiler_start("Ensuring branch %s for repository %d", (branch, self.repo_id))
            printdbg("DBContentHandler: ensure_branch %s", (branch,))
            cursor = self.cursor

            cursor.execute(statement("SELECT id from branches where name = ?", self.db.place_holder), (branch,))
            rs = cursor.fetchone()
            if not rs:
                b = DBBranch(None, branch)
                cursor.execute(statement(DBBranch.__insert__, self.db.place_holder), (b.id, b.name))
                branch_id = b.id
            else:
                branch_id = rs[0]

            profiler_stop("Ensuring branch %s for repository %d", (branch, self.repo_id), True)

            return branch_id

    def __get_file_from_moves_cache(self, path):
        # Path is not in the cache, but it should
        # Look if any of its parents was moved
        printdbg("DBContentHandler: looking for path %s in moves cache", (path,))
        current_path = path
        replaces = []
        while current_path not in self.file_cache:
            found = False
            for new_path in self.moves_cache.keys():
                if not current_path.startswith(new_path) or new_path in replaces:
                    continue

                current_path = current_path.replace(new_path, self.moves_cache[new_path], 1)
                replaces.append(new_path)
                found = True

            if not found:
                raise FileNotInCache

        return self.file_cache[current_path]

    def __convert_commit_actions(self, commit):
        # We detect here files that have been moved or
        # copied. Files moved are converted into a
        # single action of type 'V'. For copied files
        # we just change its actions type from 'A' to 'C'

        def find_action(actions, type, path):
            for action in actions:
                if action.type == type and action.f1 == path:
                    return action
            
            return None

        remove_actions = []
        
        for action in commit.actions:
            if action.f2 is not None:
                # Move or copy action
                if action.type == 'A':
                    del_action = find_action(commit.actions, 'D', action.f2)
                    if del_action is not None and del_action \
                    not in remove_actions:
                        # FIXME: See http://goo.gl/eymoH
                        printdbg("SVN Parser: File %s has been renamed to %s", 
                                 (action.f2, action.f1))
                        action.type = 'V'
                        remove_actions.append(del_action)
                    else:
                        action.type = 'C'
                        printdbg("SVN Parser: File %s has been copied to %s", 
                                 (action.f2, action.f1))

                        # Try to guess if it was a tag
                        # Yes, with svn we are always guessing :-/
                        tag = self.__guess_tag_from_path(action.f1)
                        if tag is not None:
                            if commit.tags is None:
                                commit.tags = []

                            commit.tags.append(tag)
                            
                elif action.type == 'R':
                    # TODO
                    printdbg("SVN Parser: File %s replaced to %s", 
                             (action.f2, action.f1))
                    pass

        for action in remove_actions:
            printdbg("SVN Parser: Removing action %s %s", 
                     (action.type, action.f1))
            commit.actions.remove(action)

 def do_delete(self, delete_statement, params=None,
               error_message="Delete failed, data needs manual cleanup"):
     if self.repo_id is None:
         # Repo wasn't found anyway, so continue
         return True
     
     # You can't reference instance variables in default
     # parameters, so I have to do this.
     if params is None:
         params = (self.repo_id,)
     
     try:
         delete_cursor = self.connection.cursor()
         execute_statement(statement(delete_statement, 
                                     self.db.place_holder),
                           params, delete_cursor,
                           self.db, error_message)
     except Exception:
         printdbg("Deletion exception")
     finally:
         delete_cursor.close()

        def ensure_path(path, commit_id):
            profiler_start("Ensuring path %s for repository %d",
                           (path, self.repo_id))
            printdbg("DBContentHandler: ensure_path %s", (path,))

            prefix, lpath = path.split("://", 1)
            prefix += "://"
            tokens = lpath.strip('/').split('/')

            parent = -1
            node_id = None
            for i, token in enumerate(tokens):
                rpath = prefix + '/' + '/'.join(tokens[:i + 1])
                if not ":///" in path:
                    # If the repo paths don't start with /
                    # remove it here
                    rpath = rpath.replace(':///', '://')
                printdbg("DBContentHandler: rpath: %s", (rpath,))
                try:
                    node_id, parent_id = self.file_cache[rpath]
                    parent = node_id
                    continue
                except:
                    pass

                # Rpath not in cache, add it
                node_id = self.__add_new_file_and_link(token, parent,
                                                       commit_id)
                parent_id = parent
                parent = node_id

                # Also add to file_paths
                self.__add_file_path(commit_id, node_id,
                    re.sub('^\d+://', '', rpath))

                self.file_cache[rpath] = (node_id, parent_id)

            assert node_id is not None

            printdbg("DBContentHandler: path ensured %s = %d (%d)",
                     (path, node_id, parent_id))
            profiler_stop("Ensuring path %s for repository %d",
                          (path, self.repo_id), True)

            return node_id, parent_id

    def end(self):
        # The log is now in the temp table
        # Retrieve the data now and pass it to
        # the real content handler

        self.templog.flush()
        printdbg("DBProxyContentHandler: parsing finished, creating thread")

        self.db_handler.begin()
        self.db_handler.repository(self.repo_uri)

        queue = AsyncQueue(50)
        reader_thread = threading.Thread(target=self.__reader,
                                          args=(self.templog, queue))
        reader_thread.setDaemon(True)
        reader_thread.start()

        # Use the queue with mutexes while the
        # thread is alive
        while reader_thread.isAlive():
            try:
                item = queue.get(1)
            except TimeOut:
                continue
            printdbg("DBProxyContentHandler: commit: %s", (item.revision,))
            self.db_handler.commit(item)
            del item

        # No threads now, we don't need locks
        printdbg("DBProxyContentHandler: thread __reader is finished, " + \
                 "continue without locks")
        while not queue.empty_unlocked():
            item = queue.get_unlocked()
            self.db_handler.commit(item)
            del item

        self.db_handler.end()
        self.templog.clear()

    def __get_file_for_path(self, path, commit_id, old=True):
        """Get a pair of (node_id, parent_id) regarding a path.
           First, it looks at file_cache, the at the moves cache,
           then at the deleted cache and finally, when it is not
           found in the cache, it is added and linked in the
           database.
        """

        def ensure_path(path, commit_id):
            profiler_start("Ensuring path %s for repository %d", (path, self.repo_id))
            printdbg("DBContentHandler: ensure_path %s", (path,))

            prefix, lpath = path.split("://", 1)
            prefix += "://"
            tokens = lpath.strip('/').split('/')

            parent = -1
            node_id = None
            for i, token in enumerate(tokens):
                file_path = '/'.join(tokens[:i + 1])
                rpath = prefix + '/' + file_path
                if not ":///" in path:
                    # If the repo paths don't start with /
                    # remove it here
                    rpath = rpath.replace(':///', '://')
                printdbg("DBContentHandler: rpath: %s", (rpath,))
                try:
                    node_id, parent_id = self.file_cache[rpath]
                    parent = node_id
                    continue
                except:
                    pass

                # Rpath not in cache, add it
                node_id = self.__add_new_file_and_link(token, parent, commit_id, file_path)
                parent_id = parent
                parent = node_id

                self.file_cache[rpath] = (node_id, parent_id)

            assert node_id is not None

            printdbg("DBContentHandler: path ensured %s = %d (%d)", (path, node_id, parent_id))
            profiler_stop("Ensuring path %s for repository %d", (path, self.repo_id), True)

            return node_id, parent_id

        printdbg("DBContentHandler: Looking for path %s in cache", (path,))
        # First of all look at the cache
        try:
            return self.file_cache[path]
        except KeyError:
            pass

        # It's not in the cache look now at moves cache
        try:
            retval = self.__get_file_from_moves_cache(path)
            printdbg("DBContentHandler: Found %s in moves cache", (path,))
            self.file_cache[path] = retval
            return retval
        except FileNotInCache:
            pass

        # Due to branching, the file may be deleted in other branches,
        # and thus in deletes_cache. Unless in A action when we are
        # pretty sure that it is a new file, we should always look
        # at the deletes_cache for file_id
        if old and path in self.deletes_cache:
            return self.deletes_cache[path]

        # It hasen't been moved (or any of its parents)
        # so it was copied at some point
        return ensure_path(path, commit_id)

 def __init__ (self):
     options = Config()
     self.backend = create_backend (options.type)
     printdbg ("Bicho object created, options and backend initialized")

    def commit(self, commit):
        if commit.revision in self.revision_cache:
            return

        profiler_start("New commit %s for repository %d", (commit.revision,
                                                           self.repo_id))

        log = DBLog(None, commit)
        log.repository_id = self.repo_id
        self.revision_cache[commit.revision] = log.id

        log.committer = self.__get_person(commit.committer)

        if commit.author == commit.committer:
            log.author = log.committer
        elif commit.author is not None:
            log.author = self.__get_person(commit.author)

        self.commits.append(log)

        printdbg("DBContentHandler: commit: %d rev: %s", (log.id, log.rev))

        # TODO: sort actions? R, A, D, M, V, C
        for action in commit.actions:
            printdbg("DBContentHandler: Action: %s", (action.type,))
            dbaction = DBAction(None, action.type)
            dbaction.commit_id = log.id

            branch = commit.branch or action.branch_f1
            branch_id = self.__get_branch(branch)
            dbaction.branch_id = branch_id

            prefix = "%d://" % (branch_id)
            path = prefix + action.f1

            if action.type == 'A':
                # A file has been added
                file_id = self.__action_add(path, prefix, log)
            elif action.type == 'M':
                # A file has been modified
                file_id = self.__get_file_for_path(path, log.id)[0]
            elif action.type == 'D':
                # A file has been deleted
                file_id = self.__action_delete(path, log)
            elif action.type == 'V':
                # A file has been renamed
                file_id = self.__action_rename(path, prefix, log, action,
                                               dbaction)
            elif action.type == 'C':
                # A file has been copied
                file_id = self.__action_copy(path, prefix, log, action,
                                             dbaction)
            elif action.type == 'R':
                # A file has been replaced
                file_id = self.__action_replace(path, prefix, log, action,
                                                dbaction)
                if file_id is None:
                    continue
            else:
                assert "Unknown action type %s" % (action.type)

            dbaction.file_id = file_id
            self.actions.append(dbaction)

        # Tags
        if commit.tags is not None:
            tag_revs = []
            for tag in commit.tags:
                tag_id = self.__get_tag(tag)
                db_tagrev = DBTagRev(None)
                tag_revs.append((db_tagrev.id, tag_id, log.id))

            self.cursor.executemany(statement(DBTagRev.__insert__,
                                              self.db.place_holder), tag_revs)

        if len(self.actions) >= self.MAX_ACTIONS:
            printdbg("DBContentHandler: %d actions inserting",
                     (len(self.actions),))
            self.__insert_many()

        profiler_stop("New commit %s for repository %d", (commit.revision,
                                                          self.repo_id), True)

    def __get_file_for_path(self, path, commit_id, old=False):
        """Get a pair of (node_id, parent_id) regarding a path.
           First, it looks at file_cache, the at the moves cache,
           then at the deleted cache and finally, when it is not
           found in the cache, it is added and linked in the
           database.
        """
        def ensure_path(path, commit_id):
            profiler_start("Ensuring path %s for repository %d",
                           (path, self.repo_id))
            printdbg("DBContentHandler: ensure_path %s", (path,))

            prefix, lpath = path.split("://", 1)
            prefix += "://"
            tokens = lpath.strip('/').split('/')

            parent = -1
            node_id = None
            for i, token in enumerate(tokens):
                rpath = prefix + '/' + '/'.join(tokens[:i + 1])
                if not ":///" in path:
                    # If the repo paths don't start with /
                    # remove it here
                    rpath = rpath.replace(':///', '://')
                printdbg("DBContentHandler: rpath: %s", (rpath,))
                try:
                    node_id, parent_id = self.file_cache[rpath]
                    parent = node_id
                    continue
                except:
                    pass

                # Rpath not in cache, add it
                node_id = self.__add_new_file_and_link(token, parent,
                                                       commit_id)
                parent_id = parent
                parent = node_id

                # Also add to file_paths
                self.__add_file_path(commit_id, node_id,
                    re.sub('^\d+://', '', rpath))

                self.file_cache[rpath] = (node_id, parent_id)

            assert node_id is not None

            printdbg("DBContentHandler: path ensured %s = %d (%d)",
                     (path, node_id, parent_id))
            profiler_stop("Ensuring path %s for repository %d",
                          (path, self.repo_id), True)

            return node_id, parent_id

        printdbg("DBContentHandler: Looking for path %s in cache", (path,))
        # First of all look at the cache
        try:
            return self.file_cache[path]
        except KeyError:
            pass

        # It's not in the cache look now at moves cache
        try:
            retval = self.__get_file_from_moves_cache(path)
            printdbg("DBContentHandler: Found %s in moves cache", (path,))
            self.file_cache[path] = retval
            return retval
        except FileNotInCache:
            pass

        # If it's an old file (that is, the path has been
        # taken from the "from" part of an action that
        # has two paths) it might be deletes or replaced
        if old:
            try:
                return self.deletes_cache[path]
            except KeyError:
                pass

        # It hasen't been moved (or any of its parents)
        # so it was copied at some point
        return ensure_path(path, commit_id)