def command(self, tempFilename, text, jumpIndex=None):
        """Return editor selected in user-config.py."""
        command = config.editor
        if jumpIndex:
            # Some editors make it possible to mark occurences of substrings,
            # or to jump to the line of the first occurence.
            # TODO: Find a better solution than hardcoding these, e.g. a config
            # option.
            line = text[:jumpIndex].count('\n')
            column = jumpIndex - (text[:jumpIndex].rfind('\n') + 1)
        else:
            line = column = 0
        # Linux editors. We use startswith() because some users might use
        # parameters.
        if config.editor.startswith('kate'):
            command += " -l %i -c %i" % (line + 1, column + 1)
        elif config.editor.startswith('gedit'):
            command += " +%i" % (line + 1)  # seems not to support columns
        elif config.editor.startswith('emacs'):
            command += " +%i" % (line + 1)  # seems not to support columns
        elif config.editor.startswith('jedit'):
            command += " +line:%i" % (line + 1)  # seems not to support columns
        elif config.editor.startswith('vim'):
            command += " +%i" % (line + 1)  # seems not to support columns
        elif config.editor.startswith('nano'):
            command += " +%i,%i" % (line + 1, column + 1)
        # Windows editors
        elif config.editor.lower().endswith('notepad++.exe'):
            command += " -n%i" % (line + 1)  # seems not to support columns

        command += ' %s' % tempFilename
        pywikibot.log(u'Running editor: %s' % command)
        return command

    def change(self, text):
        """
        Given a wiki source code text, return the cleaned up version.
        """
        oldText = text
        if self.site.sitename() == u'commons:commons' and self.namespace == 6:
            text = self.commonsfiledesc(text)
        text = self.fixSelfInterwiki(text)
        text = self.standardizePageFooter(text)
        text = self.fixSyntaxSave(text)
        text = self.cleanUpLinks(text)
        text = self.cleanUpSectionHeaders(text)
        text = self.putSpacesInLists(text)
        text = self.translateAndCapitalizeNamespaces(text)
##        text = self.translateMagicWords(text)
        text = self.replaceDeprecatedTemplates(text)
##        text = self.resolveHtmlEntities(text)
        text = self.validXhtml(text)
        text = self.removeUselessSpaces(text)
        text = self.removeNonBreakingSpaceBeforePercent(text)

        text = self.fixHtml(text)
        text = self.fixReferences(text)
        text = self.fixStyle(text)
        text = self.fixTypo(text)
        if self.site.lang in ['ckb', 'fa']:
            text = self.fixArabicLetters(text)
        try:
            text = isbn.hyphenateIsbnNumbers(text)
        except isbn.InvalidIsbnException, error:
            pywikibot.log(u"ISBN error: %s" % error)
            pass

def _call_cmd(args, lib='djvulibre'):
    """
    Tiny wrapper around subprocess.Popen().

    @param args: same as Popen()
    @type args: sequence or string

    @param library: library to be logged in logging messages
    @type library: string

    @param log: log process output; errors are always logged.
    @type library: bool


    @return: returns a tuple (res, stdoutdata), where
        res is True if dp.returncode != 0 else False
    """
    if not isinstance(args, StringTypes):
        # upcast if any param in sequence args is not in StringTypes
        args = [str(a) if not isinstance(a, StringTypes) else a for a in args]
        cmd = ' '.join(args)
    else:
        cmd = args

    dp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdoutdata, stderrdata = dp.communicate()

    if dp.returncode != 0:
        pywikibot.error('{0} error; {1}'.format(lib, cmd))
        pywikibot.error('{0}'.format(stderrdata))
        return (False, stdoutdata)

    pywikibot.log('SUCCESS: {0} (PID: {1})'.format(cmd, dp.pid))

    return (True, stdoutdata)

    def tearDown(self):
        """Tear down test."""
        super(TestLoggingMixin, self).tearDown()

        if hasattr(self, "_outcomeForDoCleanups"):
            # Python 3 unittest & nose
            outcome = self._outcomeForDoCleanups
        elif hasattr(self, "_outcome"):
            # Python 3.4 nose
            outcome = self._outcome
        elif hasattr(self, "_resultForDoCleanups"):
            # Python 2 unittest & nose
            outcome = self._resultForDoCleanups
        else:
            return

        if len(outcome.errors) > self._previous_errors:
            status = " NOT OK: ERROR"
        # nose 3.4 doesn't has failures
        elif hasattr(outcome, "failures") and len(outcome.failures) > self._previous_failures:
            status = " NOT OK: FAILURE"
        else:
            status = " OK"

        log("END " + self._log_prefix + "." + self._testMethodName + status)

 def isbn_execute(text):
     """Hyphenate ISBN numbers and catch 'InvalidIsbnException'."""
     try:
         return isbn.hyphenateIsbnNumbers(text)
     except isbn.InvalidIsbnException as error:
         pywikibot.log(u"ISBN error: %s" % error)
         return None

def validate_options(options, site):
    """
    Validate the options and return bool.

    @param options: options to validate
    @type options: dict

    @rtype: bool
    """
    pywikibot.log('Options:')
    required_keys = ['editnotice_template']
    has_keys = list()
    for key, value in options.items():
        pywikibot.log('-%s = %s' % (key, value))
        if key in required_keys:
            has_keys.append(key)
        if key in ('subject_only', 'talk_only', 'to_subject', 'to_talk'):
            pass
        elif key == 'editnotice_template':
            if isinstance(key, str):
                editnotice_page = pywikibot.Page(site, 'Template:%s' % value)
                if not editnotice_page.exists():
                    return False
            else:
                return False
    if sorted(has_keys) != sorted(required_keys):
        return False
    options['editnotice_page'] = editnotice_page
    options.pop('editnotice_template')
    return True

    def read_file_content(self, file_url=None):
        """Return name of temp file in which remote file is saved."""
        if not file_url:
            file_url = self.url
            pywikibot.warning("file_url is not given. "
                              "Set to self.url by default.")
        pywikibot.output(u'Reading file %s' % file_url)
        resume = False
        rlen = 0
        _contents = None
        dt = 15
        uo = URLopener()
        retrieved = False

        while not retrieved:
            if resume:
                pywikibot.output(u"Resume download...")
                uo.addheader('Range', 'bytes=%s-' % rlen)

            infile = uo.open(file_url)

            if 'text/html' in infile.info().getheader('Content-Type'):
                pywikibot.output(u"Couldn't download the image: "
                                 "the requested URL was not found on server.")
                return

            content_len = infile.info().getheader('Content-Length')
            accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes'

            if resume:
                _contents += infile.read()
            else:
                _contents = infile.read()

            infile.close()
            retrieved = True

            if content_len:
                rlen = len(_contents)
                content_len = int(content_len)
                if rlen < content_len:
                    retrieved = False
                    pywikibot.output(
                        u"Connection closed at byte %s (%s left)"
                        % (rlen, content_len))
                    if accept_ranges and rlen > 0:
                        resume = True
                    pywikibot.output(u"Sleeping for %d seconds..." % dt)
                    time.sleep(dt)
                    if dt <= 60:
                        dt += 15
                    elif dt < 360:
                        dt += 60
            else:
                pywikibot.log(
                    u"WARNING: length check of retrieved data not possible.")
        handle, tempname = tempfile.mkstemp()
        with os.fdopen(handle, "wb") as t:
            t.write(_contents)
        return tempname

    def get_redirects_from_dump(self, alsoGetPageTitles=False):
        """
        Extract redirects from dump.

        Load a local XML dump file, look at all pages which have the
        redirect flag set, and find out where they're pointing at. Return
        a dictionary where the redirect names are the keys and the redirect
        targets are the values.
        """
        xmlFilename = self.xmlFilename
        redict = {}
        # open xml dump and read page titles out of it
        dump = xmlreader.XmlDump(xmlFilename)
        redirR = self.site.redirectRegex()
        readPagesCount = 0
        if alsoGetPageTitles:
            pageTitles = set()
        for entry in dump.parse():
            readPagesCount += 1
            # always print status message after 10000 pages
            if readPagesCount % 10000 == 0:
                pywikibot.output(u'{0:d} pages read...'.format(readPagesCount))
            if len(self.namespaces) > 0:
                if pywikibot.Page(self.site, entry.title).namespace() \
                        not in self.namespaces:
                    continue
            if alsoGetPageTitles:
                pageTitles.add(space_to_underscore(pywikibot.Link(entry.title, self.site)))

            m = redirR.match(entry.text)
            if m:
                target = m.group(1)
                # There might be redirects to another wiki. Ignore these.
                target_link = pywikibot.Link(target, self.site)
                try:
                    target_link.parse()
                except pywikibot.SiteDefinitionError as e:
                    pywikibot.log(e)
                    pywikibot.output(
                        u'NOTE: Ignoring {0} which is a redirect ({1}) to an '
                        u'unknown site.'.format(entry.title, target))
                    target_link = None
                else:
                    if target_link.site != self.site:
                        pywikibot.output(
                            u'NOTE: Ignoring {0} which is a redirect to '
                            u'another site {1}.'.format(entry.title, target_link.site))
                        target_link = None
                # if the redirect does not link to another wiki
                if target_link and target_link.title:
                    source = pywikibot.Link(entry.title, self.site)
                    if target_link.anchor:
                        pywikibot.output(
                            u'HINT: {0!s} is a redirect with a pipelink.'.format(entry.title))
                    redict[space_to_underscore(source)] = (
                        space_to_underscore(target_link))
        if alsoGetPageTitles:
            return redict, pageTitles
        else:
            return redict

def _flush():
    for i in threads:
        http_queue.put(None)
    pywikibot.log(u'Waiting for threads to finish... ')
    for i in threads:
        i.join()
    pywikibot.log(u"All threads finished.")

    def _command(self, file_name, text, jump_index=None):
        """Return editor selected in user-config.py."""
        if jump_index:
            # Some editors make it possible to mark occurrences of substrings,
            # or to jump to the line of the first occurrence.
            # TODO: Find a better solution than hardcoding these, e.g. a config
            # option.
            line = text[:jump_index].count('\n')
            column = jump_index - (text[:jump_index].rfind('\n') + 1)
        else:
            line = column = 0
        # Linux editors. We use startswith() because some users might use
        # parameters.
        if config.editor.startswith('kate'):
            command = ['-l', '%i' % (line + 1), '-c', '%i' % (column + 1)]
        elif config.editor.startswith('gedit'):
            command = ['+%i' % (line + 1)]  # seems not to support columns
        elif config.editor.startswith('emacs'):
            command = ['+%i' % (line + 1)]  # seems not to support columns
        elif config.editor.startswith('jedit'):
            command = ['+line:%i' % (line + 1)]  # seems not to support columns
        elif config.editor.startswith('vim'):
            command = ['+%i' % (line + 1)]  # seems not to support columns
        elif config.editor.startswith('nano'):
            command = ['+%i,%i' % (line + 1, column + 1)]
        # Windows editors
        elif config.editor.lower().endswith('notepad++.exe'):
            command = ['-n%i' % (line + 1)]  # seems not to support columns
        else:
            command = []

        # See T102465 for problems relating to using config.editor unparsed.
        command = [config.editor] + command + [file_name]
        pywikibot.log(u'Running editor: %s' % TextEditor._concat(command))
        return command

 def __iter__(self):
     """Yield pages."""
     # TODO: start yielding before all referring pages have been found
     refs = [
         page for page in self.disambPage.getReferences(
             follow_redirects=False,
             withTemplateInclusion=False,
             namespaces=0 if self.main_only else None
         )
     ]
     pywikibot.output(u"Found %d references." % len(refs))
     # Remove ignorables
     if self.disambPage.site.family.name in ignore_title and \
        self.disambPage.site.lang in ignore_title[self.disambPage.site.family.name]:
         for ig in ignore_title[self.disambPage.site.family.name
                                ][self.disambPage.site.lang]:
             for i in range(len(refs) - 1, -1, -1):
                 if re.match(ig, refs[i].title()):
                     pywikibot.log(u'Ignoring page %s'
                                   % refs[i].title())
                     del refs[i]
                 elif self.primaryIgnoreManager.isIgnored(refs[i]):
                     del refs[i]
     if len(refs) < self.minimum:
         pywikibot.output(u"Found only %d pages to work on; skipping."
                          % len(refs))
         return
     pywikibot.output(u"Will work on %d pages." % len(refs))
     for ref in refs:
         yield ref

    def findCommonscatLink(self, page=None):
        """Find CommonsCat template on interwiki pages.

        In Pywikibot 2.0, page.interwiki() now returns Link objects,
        not Page objects

        @rtype: unicode, name of a valid commons category
        """
        for ipageLink in page.langlinks():
            ipage = pywikibot.page.Page(ipageLink)
            pywikibot.log("Looking for template on %s" % (ipage.title()))
            try:
                if (not ipage.exists() or ipage.isRedirectPage() or
                        ipage.isDisambig()):
                    continue
                commonscatLink = self.getCommonscatLink(ipage)
                if not commonscatLink:
                    continue
                (currentTemplate,
                 possibleCommonscat, linkText, Note) = commonscatLink
                checkedCommonscat = self.checkCommonscatLink(possibleCommonscat)
                if (checkedCommonscat != u''):
                    pywikibot.output(
                        u"Found link for %s at [[%s:%s]] to %s."
                        % (page.title(), ipage.site.code,
                           ipage.title(), checkedCommonscat))
                    return checkedCommonscat
            except pywikibot.BadTitle:
                # The interwiki was incorrect
                return u''
        return u''

    def tearDown(self):
        """Tear down test."""
        super(TestLoggingMixin, self).tearDown()

        if hasattr(self, '_outcomeForDoCleanups'):
            # Python 3 unittest & nose
            outcome = self._outcomeForDoCleanups
        elif hasattr(self, '_outcome'):
            # Python 3.4 nose
            outcome = self._outcome
        elif hasattr(self, '_resultForDoCleanups'):
            # Python 2 unittest & nose
            outcome = self._resultForDoCleanups
        else:
            return

        if len(outcome.errors) > self._previous_errors:
            status = ' NOT OK: ERROR'
        # nose 3.4 doesn't has failures
        elif (hasattr(outcome, 'failures') and
                len(outcome.failures) > self._previous_failures):
            status = ' NOT OK: FAILURE'
        else:
            status = ' OK'

        log('END ' + self._log_prefix + '.' + self._testMethodName + status)

    def lag(self, lagtime):
        """Seize the throttle lock due to server lag.

        This will prevent any thread from accessing this site.

        """
        started = time.time()
        self.lock.acquire()
        try:
            # start at 1/2 the current server lag time
            # wait at least 5 seconds but not more than 120 seconds
            delay = min(max(5, lagtime//2), 120)
            # account for any time we waited while acquiring the lock
            wait = delay - (time.time() - started)
            if wait > 0:
                if wait > config.noisysleep:
                    pywikibot.output(
                        u"Sleeping for %(wait).1f seconds, %(now)s"
                        % {'wait': wait,
                           'now': time.strftime("%Y-%m-%d %H:%M:%S",
                                                time.localtime())
                        } )
                else:
                    pywikibot.log(
                        u"Sleeping for %(wait).1f seconds, %(now)s"
                        % {'wait': wait,
                           'now': time.strftime("%Y-%m-%d %H:%M:%S",
                                                time.localtime())
                        } )
                time.sleep(wait)
        finally:
            self.lock.release()

    def login(self, retry=False):
        if not self.password:
            # As we don't want the password to appear on the screen, we set
            # password = True
            self.password = pywikibot.input(
                u'Password for user %(name)s on %(site)s (no characters will '
                u'be shown):' % {'name': self.username, 'site': self.site},
                password=True)
#        self.password = self.password.encode(self.site.encoding())

        pywikibot.output(u"Logging in to %(site)s as %(name)s"
                         % {'name': self.username, 'site': self.site})
        try:
            cookiedata = self.getCookie()
        except pywikibot.data.api.APIError as e:
            pywikibot.error(u"Login failed (%s)." % e.code)
            if retry:
                self.password = None
                return self.login(retry=True)
            else:
                return False
        self.storecookiedata(cookiedata)
        pywikibot.log(u"Should be logged in now")
##        # Show a warning according to the local bot policy
##   FIXME: disabled due to recursion; need to move this to the Site object after
##   login
##        if not self.botAllowed():
##            logger.error(
##                u"Username '%(name)s' is not listed on [[%(page)s]]."
##                 % {'name': self.username,
##                    'page': botList[self.site.family.name][self.site.code]})
##            logger.error(
##"Please make sure you are allowed to use the robot before actually using it!")
##            return False
        return True

    def sauvegarder(self):
        """
        Sauvegarder dans une base de données
        """
        pywikibot.log(u'# Sauvegarde dans la base pour la langue "%s".' % self.langue)

        for q in self.nouveau:
            self.req_bdd(q, 'insert')

    def __init__(self, generator, **kwargs):
        """- generator : Page generator."""
        self.availableOptions.update({
            'ignorepdf': False,  # boolean
            'limit': None,  # int, stop after n modified pages
            'summary': None,
        })

        super(ReferencesRobot, self).__init__(**kwargs)
        self.generator = generator
        self.site = pywikibot.Site()
        self._user_agent = comms.http.get_fake_user_agent()
        pywikibot.log('Using fake user agent: {0}'.format(self._user_agent))
        # Check
        manual = 'mw:Manual:Pywikibot/refLinks'
        code = None
        for alt in [self.site.code] + i18n._altlang(self.site.code):
            if alt in localized_msg:
                code = alt
                break
        if code:
            manual += '/%s' % code
        if self.getOption('summary') is None:
            self.msg = i18n.twtranslate(self.site, 'reflinks-msg', locals())
        else:
            self.msg = self.getOption('summary')
        self.stopPage = pywikibot.Page(self.site,
                                       i18n.translate(self.site, stopPage))

        local = i18n.translate(self.site, badtitles)
        if local:
            bad = '(' + globalbadtitles + '|' + local + ')'
        else:
            bad = globalbadtitles
        self.titleBlackList = re.compile(bad, re.I | re.S | re.X)
        self.norefbot = noreferences.NoReferencesBot(None, verbose=False)
        self.deduplicator = DuplicateReferences()
        try:
            self.stopPageRevId = self.stopPage.latest_revision_id
        except pywikibot.NoPage:
            pywikibot.output(u'The stop page %s does not exist'
                             % self.stopPage.title(asLink=True))
            raise

        # Regex to grasp content-type meta HTML tag in HTML source
        self.META_CONTENT = re.compile(br'(?i)<meta[^>]*content\-type[^>]*>')
        # Extract the encoding from a charset property (from content-type !)
        self.CHARSET = re.compile(r'(?i)charset\s*=\s*(?P<enc>[^\'",;>/]*)')
        # Extract html title from page
        self.TITLE = re.compile(r'(?is)(?<=<title>).*?(?=</title>)')
        # Matches content inside <script>/<style>/HTML comments
        self.NON_HTML = re.compile(
            br'(?is)<script[^>]*>.*?</script>|<style[^>]*>.*?</style>|'
            br'<!--.*?-->|<!\[CDATA\[.*?\]\]>')

        # Authorized mime types for HTML pages
        self.MIME = re.compile(
            r'application/(?:xhtml\+xml|xml)|text/(?:ht|x)ml')

    def login(self, retry=False):
        """
        Attempt to log into the server.

        @param retry: infinitely retry if the API returns an unknown error
        @type retry: bool

        @raises NoUsername: Username is not recognised by the site.
        """
        if not self.password:
            # First check that the username exists,
            # to avoid asking for a password that will not work.
            self.check_user_exists()

            # As we don't want the password to appear on the screen, we set
            # password = True
            self.password = pywikibot.input(
                u'Password for user %(name)s on %(site)s (no characters will '
                u'be shown):' % {'name': self.login_name, 'site': self.site},
                password=True)

        pywikibot.output(u"Logging in to %(site)s as %(name)s"
                         % {'name': self.login_name, 'site': self.site})
        try:
            cookiedata = self.getCookie()
        except pywikibot.data.api.APIError as e:
            pywikibot.error(u"Login failed (%s)." % e.code)
            if e.code == 'NotExists':
                raise NoUsername(u"Username '%s' does not exist on %s"
                                 % (self.login_name, self.site))
            elif e.code == 'Illegal':
                raise NoUsername(u"Username '%s' is invalid on %s"
                                 % (self.login_name, self.site))
            elif e.code == 'readapidenied':
                raise NoUsername(
                    'Username "{0}" does not have read permissions on '
                    '{1}'.format(self.login_name, self.site))
            # TODO: investigate other unhandled API codes (bug T75539)
            if retry:
                self.password = None
                return self.login(retry=True)
            else:
                return False
        self.storecookiedata(cookiedata)
        pywikibot.log(u"Should be logged in now")
#        # Show a warning according to the local bot policy
#   FIXME: disabled due to recursion; need to move this to the Site object after
#   login
#        if not self.botAllowed():
#            logger.error(
#                u"Username '%(name)s' is not listed on [[%(page)s]]."
#                 % {'name': self.username,
#                    'page': botList[self.site.family.name][self.site.code]})
#            logger.error(
# "Please make sure you are allowed to use the robot before actually using it!")
#            return False
        return True

 def save_wikipage(self, page_text, page_name, summary="Bot: Update der Ergebnisliste"):
     try:
         article = pywikibot.Page(self.site, page_name)
         updater = ArticleUpdater(article)
         if not updater.save_text(page_text, summary):
             pywikibot.log("Result page has not changed, skipping update ...")
     except pywikibot.Error:
         with tempfile.NamedTemporaryFile(delete=False) as dump_file:
             dump_file.write(page_name.encode('utf-8'))
             pywikibot.error("Could not update result page, page dumped to {}".format(dump_file.name), exc_info=True)

 def vider_base(self):
     """
     Vide la base de donnée associée (pour retirer les déchus)
     """
     pywikibot.log(u"## Vidage de l'ancienne base")
     req = u'TRUNCATE TABLE %s' % self.nom_base
     try:
         self.curseur.execute(req)
     except MySQLdb.Error, e:
         pywikibot.warning(u"Truncate error %d: %s" % (e.args[0], e.args[1]))