def update(self, force_rebuild=False):
        """ Adds/updates all items in repo to index. Note: querying will call this automatically."""

        # if we've already updated the index during this script run, we're done!
        if self.index_updated:
            return False

        # if the index is not based on the current commit, rebuild from scratch
        if not self.index_based_on_current_commit():
            force_rebuild = True

        if force_rebuild:
            # get a new clean/empty index
            index = self.get_index(force_rebuild)
            index_writer = index.writer()

            # index all documents
            documents = self.document_iterator()
            activity_description = 'Rebuilding'
        else:
            # use the current index
            index = self.get_index()
            index_writer = index.writer()

            # delete uncommitted files that are in index already
            for filepath in self.get_indexed_uncommitted_files():
                index_writer.delete_by_term('path', filepath)

            # get list of uncommitted files and persist it
            uncommitted_files = lib_git.get_uncommitted_oval()
            self.set_indexed_uncommitted_files(uncommitted_files)

            # if there are no uncommitted files to index, we're done
            if not uncommitted_files:
                index_writer.commit()
                return False

            # index only uncommitted files
            documents = self.document_iterator(uncommitted_files)
            activity_description = 'Updating'

        # add all definition files to index
        counter = 0
        for document in documents:
            counter = counter + 1
            self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label)
            if 'deleted' in document and document['deleted']:
                index_writer.delete_by_term('path', document['path'])
                #self.message('debug', 'Deleting from index:\n\t{0} '.format(document['path']))
            else:
                index_writer.add_document(**document)
                #self.message('debug', 'Upserting to index:\n\t{0} '.format(document['path']))
        index_writer.commit()
        self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label, True)

        # update indexed commit
        self.set_indexed_commit_hash()
        self.index_updated = True

 def _get_index(self, index_path, index_name):
     if not whoosh.index.exists_in(index_path, index_name):
         print 'creating %s index at %s' % (index_name, index_path)
         if not os.path.exists(index_path):
             os.makedirs(index_path)
         schema = whoosh.fields.Schema(
                 id = whoosh.fields.ID(stored=True, unique=True),
                 artist = whoosh.fields.TEXT(stored=True),
                 title = whoosh.fields.TEXT(stored=True),
                 lyrics = whoosh.fields.TEXT(stored=True),
                 )
         index = whoosh.index.create_in(index_path, schema, index_name)
         index.writer().commit()
     return whoosh.index.open_dir(index_path, index_name)

def create_index_writer(index_path):
    '''
    Constructs a whoosh index writer, which has ID, artist and title fields

    :parameters:
        - index_path : str
            Path to whoosh index to be written

    :returns:
        - index : whoosh.writing.IndexWriter
            Whoosh index writer
    '''
    if not os.path.exists(index_path):
        os.mkdir(index_path)

    A = (whoosh.analysis.StandardAnalyzer(stoplist=None, minsize=1) |
         whoosh.analysis.CharsetFilter(accent_map))

    Schema = whoosh.fields.Schema(
        id=whoosh.fields.ID(stored=True),
        path=whoosh.fields.TEXT(stored=True),
        artist=whoosh.fields.TEXT(stored=True, analyzer=A),
        title=whoosh.fields.TEXT(stored=True, analyzer=A))

    index = whoosh.index.create_in(index_path, Schema)
    return index.writer()

def reindex_snippets():
    from flask_website.database import Snippet
    writer = index.writer()
    for snippet in Snippet.query.all():
        snippet.remove_from_search_index(writer)
        snippet.add_to_search_index(writer)
    writer.commit()

def update_documentation_index():
    from flask_website.docs import DocumentationPage
    writer = index.writer()
    for page in DocumentationPage.iter_pages():
        page.remove_from_search_index(writer)
        page.add_to_search_index(writer)
    writer.commit()

def update_model_based_indexes(session, flush_context):
    """Called by a session event, updates the model based documents."""
    to_delete = []
    to_add = []
    for model in session.new:
        if isinstance(model, Indexable):
            to_add.append(model)

    for model in session.dirty:
        if isinstance(model, Indexable):
            to_delete.append(model)
            to_add.append(model)

    for model in session.dirty:
        if isinstance(model, Indexable):
            to_delete.append(model)

    if not (to_delete or to_add):
        return

    writer = index.writer()
    for model in to_delete:
        model.remove_from_search_index(writer)
    for model in to_add:
        model.add_to_search_index(writer)
    writer.commit()

def _after_flush(app, changes):
    # Any db updates go through here. We check if any of these models have
    # ``__searchable__`` fields, indicating they need to be indexed. With these
    # we update the whoosh index for the model. If no index exists, it will be
    # created here; this could impose a penalty on the initial commit of a
    # model.
    if app.config.get('WHOOSH_DISABLED') is True:
        return
    bytype = {}  # sort changes by type so we can use per-model writer
    for change in changes:
        update = change[1] in ('update', 'insert')

        if hasattr(change[0].__class__, __searchable__):
            bytype.setdefault(change[0].__class__.__name__, []).append(
                (update, change[0]))
    if not bytype:
        return
    try:
        for model, values in bytype.items():
            index = whoosh_index(app, values[0][1].__class__)
            with index.writer() as writer:
                for update, v in values:
                    has_parent = isinstance(
                        v.__class__.__base__, DeclarativeMeta)
                    index_one_record(
                        v, not update, writer, index_parent=has_parent)
    except Exception as ex:
        logging.warning("FAIL updating index of %s msg: %s" % (model, str(ex)))

  def after_commit(self, session):
    """
    Any db updates go through here. We check if any of these models have
    ``__searchable__`` fields, indicating they need to be indexed. With these
    we update the whoosh index for the model. If no index exists, it will be
    created here; this could impose a penalty on the initial commit of a model.
    """

    for typ, values in self.to_update.iteritems():
      model_class = values[0][1].__class__
      index = self.index_for_model_class(model_class)
      with index.writer() as writer:
        primary_field = model_class.search_query.primary
        searchable = model_class.__searchable__

        for change_type, model in values:
          # delete everything. stuff that's updated or inserted will get
          # added as a new doc. Could probably replace this with a whoosh
          # update.

          if change_type == "deleted":
              writer.delete_by_term(primary_field, unicode(getattr(model, primary_field)))
          else:
            attrs = dict((key, getattr(model, key)) for key in searchable)
            attrs[primary_field] = unicode(getattr(model, primary_field))
            if change_type == "new":
                writer.add_document(**attrs)
            elif change_type == "changed":
                writer.update_document(**attrs)

    self.to_update = {}

def index_one_record(record, delete=False, writer=None, index_parent=False):
    index = whoosh_index(current_app, record.__class__)
    close = False
    if not writer:
        writer = index.writer()
        close = True
    if index_parent:
        # index parent class
        parent_writer = whoosh_index(
            current_app, record.__class__.__base__).writer()
    primary_field = record.pure_whoosh.primary_key_name
    searchable = index.schema.names()
    if not delete:
        attrs = {}
        for key in searchable:
            attrs[key] = unicode(getattr(record, key))
        attrs[primary_field] = unicode(
            getattr(record, primary_field))
        writer.update_document(**attrs)
        if index_parent:
            parent_writer.update_document(**attrs)
    else:
        writer.delete_by_term(
            primary_field, unicode(getattr(record, primary_field)))
        if index_parent:
            parent_writer.delete_by_term(
                primary_field, unicode(getattr(record, primary_field)))
    if close:
        writer.commit()

def _after_flush(app, changes):
    # Any db updates go through here. We check if any of these models have
    # ``__searchable__`` fields, indicating they need to be indexed. With these
    # we update the whoosh index for the model. If no index exists, it will be
    # created here; this could impose a penalty on the initial commit of a
    # model.

    bytype = {}  # sort changes by type so we can use per-model writer
    for change in changes:
        update = change[1] in ("update", "insert")

        if hasattr(change[0].__class__, __searchable__):
            bytype.setdefault(change[0].__class__.__name__, []).append((update, change[0]))

    for model, values in bytype.items():
        index = whoosh_index(app, values[0][1].__class__)
        with index.writer() as writer:
            primary_field = values[0][1].pure_whoosh.primary_key_name
            searchable = values[0][1].__searchable__

            for update, v in values:
                if update:
                    attrs = {}
                    for key in searchable:
                        try:
                            attrs[key] = unicode(getattr(v, key))
                        except AttributeError:
                            raise AttributeError("{0} does not have {1} field {2}".format(model, __searchable__, key))

                    attrs[primary_field] = unicode(getattr(v, primary_field))
                    writer.update_document(**attrs)
                else:
                    writer.delete_by_term(primary_field, unicode(getattr(v, primary_field)))

def create_index_writer(index_path):
    '''Create a new whoosh index in the given directory path.
    
    Input: directory in which to create the index
    
    Output: `whoosh.index` writer object
    '''
    

    if not os.path.exists(index_path):
        os.mkdir(index_path)

    analyzer = (whoosh.analysis.StemmingAnalyzer() | 
                whoosh.analysis.CharsetFilter(accent_map))

    schema = whoosh.fields.Schema(track_id=whoosh.fields.STORED,
                                  title=whoosh.fields.TEXT(stored=True, analyzer=analyzer),
                                  artist=whoosh.fields.TEXT(stored=True, analyzer=analyzer),
                                  album=whoosh.fields.TEXT(stored=True, analyzer=analyzer),
                                  collection=whoosh.fields.KEYWORD(stored=True),
                                  collection_id=whoosh.fields.NUMERIC(stored=True))

    index = whoosh.index.create_in(index_path, schema)

    return index.writer()

def _get_writer(index):
    writer = None
    while writer is None:
        try:
            writer = index.writer()
        except whoosh.index.LockError:
            time.sleep(0.25)

    return writer

 def rebuild_index_model(self, model_class, session):
   index = self.index_for_model_class(model_class)
   with index.writer() as writer:
     primary_field = model_class.search_query.primary
     searchable = model_class.__searchable__
     for i in session.query(model_class):
       attrs = dict((key, getattr(i, key)) for key in searchable)
       attrs[primary_field] = unicode(getattr(i, primary_field))
       writer.delete_by_term(primary_field, unicode(getattr(i, primary_field)))
       writer.add_document(**attrs)

def handle_postupdate(item_id): # pragma: no cover
    """Insert item data into indexer.

    """
    item = icecrate.items.by_item_id(item_id)

    writer = index.writer()
    writer.update_document(
        upc=item_id,
        name=item.get("name"),
        tags=list(icecrate.tags._split_tags(item.get("tags", ""))))
    writer.commit()

def createIndexWriter(indexPath):
    if not os.path.exists(indexPath):
        os.mkdir(indexPath)

    A = whoosh.analysis.FancyAnalyzer() | whoosh.analysis.CharsetFilter(accent_map)

    Schema = whoosh.fields.Schema(  song_id     = whoosh.fields.ID(stored=True),
                                    artist      = whoosh.fields.TEXT(stored=True, analyzer=A),
                                    title       = whoosh.fields.TEXT(stored=True, analyzer=A))

    index = whoosh.index.create_in(indexPath, Schema)
    return index.writer()
    pass

def appendtextindex(table, index_or_dirname, indexname=None, merge=True,
                    optimize=False):
    """
    Load all rows from `table` into a Whoosh index, adding them to any existing
    data in the index.

    Keyword arguments:

    table
        A table container with the data to be loaded.
    index_or_dirname
        Either an instance of `whoosh.index.Index` or a string containing the
        directory path where the index is to be stored.
    indexname
        String containing the name of the index, if multiple indexes are stored
        in the same directory.
    merge
        Merge small segments during commit?
    optimize
        Merge all segments together?

    """
    import whoosh.index

    # deal with polymorphic argument
    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.open_dir(dirname, indexname=indexname,
                                      readonly=False)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r'
                            % index_or_dirname)

    writer = index.writer()
    try:

        for d in dicts(table):
            writer.add_document(**d)
        writer.commit(merge=merge, optimize=optimize)

    except Exception:
        writer.cancel()
        raise

    finally:
        if needs_closing:
            index.close()

def write_db():
    index = storage.create_index(schema)
    writer = index.writer()
    
    # read doc from Database by using django
    for dou in Preview.objects.all():
        doc = {}
        doc['id'] = _from_python(str(dou.id))
        text = dou.description
        doc[index_fieldname] = text
        try:
            writer.update_document(**doc)
        except Exception, e:
            raise

def write_db(storage,schema):
    index = storage.create_index(schema)
    writer = index.writer()
        
    #for dou in DoubanMovie.objects.filter(id__lte=4000).annotate(cnt=Count('movielink')).filter(cnt__gt=0).order_by('-cnt'):
    for obj in HtmlContent.objects.filter(~Q(retry=3)).filter(~Q(content='')):
        doc = {}
        doc['id'] = _from_python(str(obj.id))
        doc['title'] = obj.title
        doc['content'] = obj.content
        try:
            writer.update_document(**doc)
        except Exception, e:
            raise

    def reindex(self):
        """Reindex all data

        This method retrieves all the data from the registered models and
        calls the ``update_<model>()`` function for every instance of such
        model.
        """
        for wh in self.whoosheers:
            index = type(self).get_or_create_index(_get_app(self), wh)
            writer = index.writer(timeout=_get_config(self)['writer_timeout'])
            for model in wh.models:
                method_name = "{0}_{1}".format(UPDATE_KWD, model.__name__.lower())
                for item in model.query.all():
                    getattr(wh, method_name)(writer, item)
            writer.commit()

def add_snippet():
    title = request.form['title'].strip()
    content = request.form['content'].strip()
    tag = request.form['tag']
    language = request.form['language']
    snippet_id = unicode(uuid.uuid4())

    if not title or not content:
        raise Exception("Empty title or snippet content")

    writer = index.writer()
    writer.update_document(id=snippet_id, content=content, tag=tag, title=title, language=language)
    writer.commit()

    return '{"success": true, "message": "Snippet added successfully", "snippet_id": "%s"}' % snippet_id