def execute(pagename, request):
    """ set bookmarks (in time) for RecentChanges or delete them """
    _ = request.getText
    if not request.user.valid:
        actname = __name__.split('.')[-1]
        request.theme.add_msg(_("You must login to use this action: %(action)s.") % {"action": actname}, "error")
        return Page(request, pagename).send_page()

    timestamp = request.form.get('time', [None])[0]
    if timestamp is not None:
        if timestamp == 'del':
            tm = None
        else:
            try:
                tm = int(timestamp)
            except StandardError:
                tm = wikiutil.timestamp2version(time.time())
    else:
        tm = wikiutil.timestamp2version(time.time())

    if tm is None:
        request.user.delBookmark()
    else:
        request.user.setBookmark(tm)
    request.page.send_page()

 def _index_file(self, request, writer, filename, update):
     """ index a file as it were a page named pagename
         Assumes that the write lock is acquired
     """
     fs_rootpage = 'FS' # XXX FS hardcoded
     try:
         mtime = os.path.getmtime(filename)
         mtime = wikiutil.timestamp2version(mtime)
         if update:
             query = BooleanQuery()
             query.add(TermQuery(Term("pagename", fs_rootpage)), True, False)
             query.add(TermQuery(Term("attachment", filename)), True, False)
             docs = self._search(query)
             updated = len(docs) == 0 or mtime > int(docs[0].get('mtime'))
         else:
             updated = True
         request.log("%s %r" % (filename, updated))
         if updated:
             file_content = self.contentfilter(filename)
             d = document.Document()
             d.add(document.Keyword('pagename', fs_rootpage))
             d.add(document.Keyword('mtime', str(mtime)))
             d.add(document.Keyword('attachment', filename)) # XXX we should treat files like real pages, not attachments
             pagename = " ".join(os.path.join(fs_rootpage, filename).split("/"))
             d.add(document.Text('title', pagename, store=False))        
             d.add(document.Text('text', file_content, store=False))
             writer.addDocument(d)
     except (OSError, IOError), err:
         pass

def gather_editlog(dir_from, el_from):
    """ this gathers everything that is in edit-log into internal
        data structures, converting to the future format
    """
    if not os.path.exists(el_from): 
        return
    for l in open(el_from):
        data = l.rstrip('\n').split('\t')
        origlen = len(data)
        while len(data) < 7: data.append('')
        (pagename,ip,timestamp,host,id,comment,action) = data
        if origlen == 6:
            action = comment
            comment = ''
        
        extra = ''
        if action == 'SAVE/REVERT': # we missed to convert that in mig4
            ts = long(comment) # must be long for py 2.2.x
            if ts < 4000000000: # UNIX timestamp (secs)
                extra = str(wikiutil.timestamp2version(ts))
            else: # usecs timestamp
                extra = str(ts)
            # later we convert this timestamp to a revision number
            comment = ''
        if action in ['ATTNEW','ATTDRW','ATTDEL',]:
            extra = comment # filename
            comment = '' # so we can use comments on ATT* in future

        timestamp = long(timestamp) # must be long for py 2.2.x
        data = [timestamp,'',action,pagename,ip,host,id,extra,comment]
        
        entry = info.get(pagename, {})
        entry[timestamp] = [None, data]
        info[pagename] = entry

    def history(self, request):
        # files = self._list_files()
        pages = []
        for root, dirs, files in os.walk(self.basepath):
            for name in dirs: pages.append(os.path.join(root, name))
            for name in files: pages.append(os.path.join(root, name))

        # pages = sorted(pages, lambda x,y: os.path.getmtime(x) < os.path.getmtime(y), reverse=True)
        # logging.warning(str(pages))
        pages = sorted(pages, key=lambda(x): os.path.getmtime(x), reverse=True)
        
        _usercache = {}
        for filename in pages:
            result = editlog.EditLogLine(_usercache)
            result.ed_time_usecs = wikiutil.timestamp2version(os.path.getmtime(filename))
            result.rev = 0
            result.action = 'SAVE'
            filename = filename[len(self.basepath)+1:].replace(os.sep, '/')
            if filename.endswith(request.cfg.fs_extension):
                filename = filename[:-len(request.cfg.fs_extension)]
            result.pagename = filename.decode(request.cfg.fs_encoding)
            result.addr = ''
            result.hostname = ''
            result.userid = ''
            result.extra = None
            result.comment = ''
            yield result

    def add(self, request, eventtype, values=None, add_http_info=1,
            mtime_usecs=None):
        """ Write an event of type `eventtype, with optional key/value
            pairs appended (i.e. you have to pass a dict).
        """
        cfg = request.cfg
        if cfg.log_events_format == 0 or request.isSpiderAgent:
            # no event logging enabled or user agent is a bot / spider
            return

        if mtime_usecs is None:
            mtime_usecs = wikiutil.timestamp2version(time.time())

        if values is None:
            values = {}
        if cfg.log_remote_addr and add_http_info:
            # if cfg.log_remote_addr is False (usually for privacy reasons),
            # we likely do not want to log user agent and http referer either.
            for key in ['remote_addr', 'http_user_agent', 'http_referer']:
                value = getattr(request, key, '')
                if value:
                    # Save those http headers in UPPERcase
                    values[key.upper()] = value

        if cfg.log_events_format == 2:
            values['username'] = request.user.name
            values['wikiname'] = cfg.interwikiname
            values['url'] = request.url

        # Encode values in a query string TODO: use more readable format
        values = wikiutil.makeQueryString(values)
        self._add(u"%d\t%s\t%s\n" % (mtime_usecs, eventtype, values))

def do_bookmark(pagename, request):
    if request.form.has_key('time'):
        if request.form['time'][0] == 'del':
            tm = None
        else:
            try:
                tm = long(request.form["time"][0]) # must be long for py 2.2.x
            except StandardError:
                tm = wikiutil.timestamp2version(time.time())
    else:
        tm = wikiutil.timestamp2version(time.time())
  
    if tm is None:
        request.user.delBookmark()
    else:
        request.user.setBookmark(tm)
    Page(request, pagename).send_page(request)

 def _writeLockFile(self):
     """Write new lock file."""
     self._deleteLockFile()
     try:
         editlog.EditLog(self.request, filename=self._filename()).add(
            self.request, wikiutil.timestamp2version(self.now), 0, "LOCK", self.page_name)
     except IOError:
         pass

    def _index_page(self, writer, page, update):
        """ Index a page - assumes that the write lock is acquired
            @arg writer: the index writer object
            @arg page: a page object
            @arg update: False = index in any case, True = index only when changed
        """
        pagename = page.page_name
        request = page.request
        mtime = page.mtime_usecs()
        if update:
            query = BooleanQuery()
            query.add(TermQuery(Term("pagename", pagename)), True, False)
            query.add(TermQuery(Term("attachment", "")), True, False)
            docs = self._search(query)
            updated = len(docs) == 0 or mtime > int(docs[0].get('mtime'))
        else:
            updated = True
        request.log("%s %r" % (pagename, updated))
        if updated:
            d = document.Document()
            d.add(document.Keyword('pagename', pagename))
            d.add(document.Keyword('mtime', str(mtime)))
            d.add(document.Keyword('attachment', '')) # this is a real page, not an attachment
            d.add(document.Text('title', pagename, store=False))        
            d.add(document.Text('text', page.get_raw_body(), store=False))
            
            links = page.getPageLinks(request)
            t = document.Text('links', '', store=False)
            t.stringVal = links
            d.add(t)
            d.add(document.Text('link_text', ' '.join(links), store=False))

            writer.addDocument(d)
        
        from MoinMoin.action import AttachFile

        attachments = AttachFile._get_files(request, pagename)
        for att in attachments:
            filename = AttachFile.getFilename(request, pagename, att)
            mtime = wikiutil.timestamp2version(os.path.getmtime(filename))
            if update:
                query = BooleanQuery()
                query.add(TermQuery(Term("pagename", pagename)), True, False)
                query.add(TermQuery(Term("attachment", att)), True, False)
                docs = self._search(query)
                updated = len(docs) == 0 or mtime > int(docs[0].get('mtime'))
            else:
                updated = True
            request.log("%s %s %r" % (pagename, att, updated))
            if updated:
                att_content = self.contentfilter(filename)
                d = document.Document()
                d.add(document.Keyword('pagename', pagename))
                d.add(document.Keyword('mtime', str(mtime)))
                d.add(document.Keyword('attachment', att)) # this is an attachment, store its filename
                d.add(document.Text('title', att, store=False)) # the filename is the "title" of an attachment
                d.add(document.Text('text', att_content, store=False))
                writer.addDocument(d)

def convert_userdir(dir_from, dir_to):
    os.mkdir(dir_to)
    for fname in listdir(dir_from):
        if fname.endswith('.bookmark'):
            bm = open(opj(dir_from, fname)).read().strip()
            bm = str(wikiutil.timestamp2version(float(bm)))
            f = open(opj(dir_to, fname), 'w')
            f.write(bm)
            f.close()
        else:
            copy_file(opj(dir_from, fname), opj(dir_to, fname))

def edit_logfile_append(self, pagename, pagefile, rev, action, logname='edit-log', comment=u'', author=u"Scripting Subsystem"):
    glog = editlog.EditLog(self.request, uid_override=author)
    pagelog = Page(self.request, pagename).getPagePath(logname, use_underlay=0, isfile=1)
    llog = editlog.EditLog(self.request, filename=pagelog,
                               uid_override=author)
    mtime_usecs = wikiutil.timestamp2version(os.path.getmtime(pagefile))
    host = '::1'
    extra = u''
    glog.add(self.request, mtime_usecs, rev, action, pagename, host, comment)
    llog.add(self.request, mtime_usecs, rev, action, pagename, host, extra, comment)
    event_logfile(self, pagename, pagefile)

    def _index_file(self, request, writer, filename, mode='update'):
        """ index a file as it were a page named pagename
            Assumes that the write lock is acquired
        """
        fs_rootpage = 'FS' # XXX FS hardcoded

        try:
            wikiname = request.cfg.interwikiname or 'Self'
            itemid = "%s:%s" % (wikiname, os.path.join(fs_rootpage, filename))
            mtime = os.path.getmtime(filename)
            mtime = wikiutil.timestamp2version(mtime)
            if mode == 'update':
                query = xapidx.RawQuery(xapdoc.makePairForWrite('itemid', itemid))
                enq, mset, docs = writer.search(query, valuesWanted=['pagename', 'attachment', 'mtime', 'wikiname', ])
                if docs:
                    doc = docs[0] # there should be only one
                    uid = doc['uid']
                    docmtime = long(doc['values']['mtime'])
                    updated = mtime > docmtime
                    logging.debug("uid %r: mtime %r > docmtime %r == updated %r" % (uid, mtime, docmtime, updated))
                else:
                    uid = None
                    updated = True
            elif mode == 'add':
                updated = True
            logging.debug("%s %r" % (filename, updated))
            if updated:
                xitemid = xapdoc.Keyword('itemid', itemid)
                mimetype, file_content = self.contentfilter(filename)
                xwname = xapdoc.SortKey('wikiname', request.cfg.interwikiname or "Self")
                xpname = xapdoc.SortKey('pagename', fs_rootpage)
                xattachment = xapdoc.SortKey('attachment', filename) # XXX we should treat files like real pages, not attachments
                xmtime = xapdoc.SortKey('mtime', mtime)
                xrev = xapdoc.SortKey('revision', '0')
                title = " ".join(os.path.join(fs_rootpage, filename).split("/"))
                xtitle = xapdoc.Keyword('title', title)
                xmimetypes = [xapdoc.Keyword('mimetype', mt) for mt in [mimetype, ] + mimetype.split('/')]
                xcontent = xapdoc.TextField('content', file_content)
                doc = xapdoc.Document(textFields=(xcontent, ),
                                      keywords=xmimetypes + [xtitle, xitemid, ],
                                      sortFields=(xpname, xattachment,
                                          xmtime, xwname, xrev, ),
                                     )
                doc.analyzerFactory = getWikiAnalyzerFactory()
                if mode == 'update':
                    logging.debug("%s (replace %r)" % (filename, uid))
                    doc.uid = uid
                    id = writer.index(doc)
                elif mode == 'add':
                    logging.debug("%s (add)" % (filename, ))
                    id = writer.index(doc)
        except (OSError, IOError):
            pass

def update_page(page, db):
    id_file = page.getPagePath("xapian.id", check_create=0, isfile=1)
    try:
        f = open(id_file, 'w')
        id = f.read()
        f.close()
        docid = int(id)
    except:
        add_page(page, db)
        return
    print "update id: %s <br>" % docid
    if wikiutil.timestamp2version(os.path.getmtime(id_file)) < page.mtime_usecs():
        db.replace_document(docid, _index_page(page))

def convert_textdir(dir_from, dir_to, enc_from, enc_to, is_backupdir=0):
    os.mkdir(dir_to)
    for fname_from in listdir(dir_from):
        if is_backupdir:
            fname, timestamp = fname_from.split('.',1)
            timestamp = str(wikiutil.timestamp2version(float(timestamp)))
        else:
            fname = fname_from
        fname = qf_convert_string(fname, enc_from, enc_to)
        if is_backupdir:
            fname_to = '.'.join([fname, timestamp])
        else:
            fname_to = fname
        convert_file(opj(dir_from, fname_from), opj( dir_to, fname_to),
                     enc_from, enc_to)

    def _index_attachment(self, request, connection, pagename, attachmentname, mode="update"):
        """ Index an attachment

        @param request: request suitable for indexing
        @param connection: the Indexer connection object
        @param pagename: the page name
        @param attachmentname: the attachment's name
        @param mode: 'add' = just add, no checks
                     'update' = check if already in index and update if needed (mtime)
        """
        from MoinMoin.action import AttachFile

        wikiname = request.cfg.interwikiname or u"Self"
        itemid = "%s:%s//%s" % (wikiname, pagename, attachmentname)

        filename = AttachFile.getFilename(request, pagename, attachmentname)
        # check if the file is still there. as we might be doing queued index updates,
        # the file could be gone meanwhile...
        if os.path.exists(filename):
            mtime = wikiutil.timestamp2version(os.path.getmtime(filename))
            doc = self._get_document(connection, itemid, mtime, mode)
            logging.debug("%s %s %r" % (pagename, attachmentname, doc))
            if doc:
                page = Page(request, pagename)
                mimetype, att_content = self.contentfilter(filename)

                fields = {}
                fields["wikiname"] = wikiname
                fields["pagename"] = pagename
                fields["attachment"] = attachmentname
                fields["mtime"] = str(mtime)
                fields["revision"] = "0"
                fields["title"] = "%s/%s" % (pagename, attachmentname)
                fields["content"] = att_content
                fields["lang"], fields["stem_lang"] = self._get_languages(page)

                multivalued_fields = {}
                multivalued_fields["mimetype"] = [mt for mt in [mimetype] + mimetype.split("/")]
                multivalued_fields["domain"] = self._get_domains(page)

                self._add_fields_to_document(request, doc, fields, multivalued_fields)

                connection.replace(doc)
                logging.debug("attachment %s (page %s) updated in index" % (attachmentname, pagename))
        else:
            # attachment file was deleted, remove it from index also
            connection.delete(itemid)
            logging.debug("attachment %s (page %s) removed from index" % (attachmentname, pagename))

def convert_editlog(log_from, log_to, enc_from, enc_to):
        file_from = open(log_from)
        file_to = open(log_to, "w")
        for line in file_from:
            line = line.replace('\r','')
            line = line.replace('\n','')
            if not line.strip(): # skip empty lines
                continue
            fields = line.split('\t')
            fields[0] = qf_convert_string(fields[0], enc_from, enc_to)
            fields[2] = str(wikiutil.timestamp2version(float(fields[2])))
            if len(fields) < 6:
                fields.append('SAVE')
            fields[5] = convert_string(fields[5], enc_from, enc_to)
            line = '\t'.join(fields) + '\n'
            file_to.write(line)

def _addLogEntry(request, action, pagename, filename):
    """ Add an entry to the edit log on uploads and deletes.

        `action` should be "ATTNEW" or "ATTDEL"
    """
    from MoinMoin.logfile import editlog
    t = wikiutil.timestamp2version(time.time())
    fname = wikiutil.url_quote(filename)

    # Write to global log
    log = editlog.EditLog(request)
    log.add(request, t, 99999999, action, pagename, request.remote_addr, fname)

    # Write to local log
    log = editlog.EditLog(request, rootpagename=pagename)
    log.add(request, t, 99999999, action, pagename, request.remote_addr, fname)

def _addLogEntry(request, action, pagename, filename):
    """ Add an entry to the edit log on uploads and deletes.

        `action` should be "ATTNEW" or "ATTDEL"
    """
    from MoinMoin.logfile import editlog
    t = wikiutil.timestamp2version(time.time())
    fname = wikiutil.url_quote(filename, want_unicode=True)

    # TODO: for now we simply write 2 logs, maybe better use some multilog stuff
    # Write to global log
    log = editlog.EditLog(request)
    log.add(request, t, 99999999, action, pagename, request.remote_addr, fname)

    # Write to local log
    log = editlog.EditLog(request, rootpagename=pagename)
    log.add(request, t, 99999999, action, pagename, request.remote_addr, fname)

def addLogEntry(request, action, pagename, msg):
    # Add an entry to the edit log on adding comments.
    from MoinMoin.logfile import editlog
    t = wikiutil.timestamp2version(time.time())
    msg = unicode(msg)

    pg = Page( request, pagename )
    #rev = pg.current_rev()
    rev = 99999999

    # TODO: for now we simply write 2 logs, maybe better use some multilog stuff
    # Write to global log
    log = editlog.EditLog(request)
    log.add(request, t, rev, action, pagename, request.remote_addr, '', msg)

    # Write to local log
    log = editlog.EditLog(request, rootpagename=pagename)
    log.add(request, t, rev, action, pagename, request.remote_addr, '', msg)

 def write(self, fname, deleted=False):
     """ write complete edit-log to disk """
     if self.data:
         editlog = self.data.items()
         editlog.sort()
         f = file(fname, 'wb') # write in binary mode, so it stays exactly as we write it, even on windows.
                               # the code in MoinMoin.logfile also uses binary mode and writes \n only.
         max_rev = 0
         for key, fields in editlog:
             timestamp, rev, action, pagename, ip, hostname, userid, extra, comment = fields
             if action.startswith('ATT'):
                 try:
                     fname = urllib.unquote(extra).decode('utf-8')
                 except UnicodeDecodeError:
                     fname = urllib.unquote(extra).decode('iso-8859-1')
                 if ('FILE', pagename, fname) in self.renames:
                     fname = self.renames[('FILE', pagename, fname)]
                 extra = urllib.quote(fname.encode('utf-8'))
             if ('PAGE', pagename) in self.renames:
                 pagename = self.renames[('PAGE', pagename)]
             timestamp = str(timestamp)
             if rev != 99999999:
                 max_rev = max(rev, max_rev)
             revstr = '%08d' % rev
             pagename = wikiutil.quoteWikinameFS(pagename)
             fields = timestamp, revstr, action, pagename, ip, hostname, userid, extra, comment
             log_str = '\t'.join(fields) + '\n'
             f.write(log_str)
         if create_rev and not deleted:
             timestamp = str(wikiutil.timestamp2version(time.time()))
             revstr = '%08d' % (max_rev + 1)
             action = 'SAVE'
             ip = '127.0.0.1'
             hostname = 'localhost'
             userid = ''
             extra = ''
             comment = "converted to 1.6 markup"
             fields = timestamp, revstr, action, pagename, ip, hostname, userid, extra, comment
             log_str = '\t'.join(fields) + '\n'
             f.write(log_str)
         f.close()

 def history(self, request):
     if not self.user or (request.user.valid and request.user.name == self.user):
         files = self._list_files(request)
         # files = sorted(files, lambda x,y:os.path.getmtime(x) < os.path.getmtime(y), reverse=True)
         files = sorted(files, key=lambda x:os.path.getmtime(x), reverse=True)
         _usercache = {}
         for filename in files:
             result = editlog.EditLogLine(_usercache)
             result.ed_time_usecs = wikiutil.timestamp2version(os.path.getmtime(filename))
             result.rev = 0
             result.action = 'SAVE'
             result.pagename = wikiutil.unquoteWikiname(self.prefix + os.path.splitext(os.path.basename(filename))[0])
             result.addr = ''
             result.hostname = ''
             if self.user:
                 result.userid = request.user.id #restrict_user is self
             else:
                 result.userid = ''
             result.extra = None
             result.comment = ''
             yield result