def treat(self, page):
        """
        Load the given page, make the required changes, and save it.

        @param page: the page to treat
        @type page: pywikibot.Page
        """
        self.current_page = page
        page.get()
        wikicode = mwparserfromhell.parse(page.text)
        for el in wikicode.ifilter_tags():
            if el.tag in self.getOption('tags'):
                try:
                    result = self.process_lines(el.contents)
                except PoemError as e:
                    pywikibot.warning(e)
                    continue
                if result:
                    lines, numbering, lines_count = result
                    if lines != el.contents:
                        scheme = self.detect_numbering(numbering, lines_count)
                        if scheme:
                            el.contents = lines
                            for attr, val in scheme.items():
                                el.add(attr, val)
                            if self.getOption('preferred'):
                                # change the tag name to the preferred form
                                el.tag = self.getOption('preferred')
                        else:
                            pywikibot.warning(u'a reliable line numbering scheme could not be obtained')
        newtext = unicode(wikicode)
        summary = self.getOption('summary')
        if not summary:
            summary = i18n.translate(page.site, self.summary, fallback=True)
        self.userPut(page, page.text, newtext, comment=summary)

    def readFromCache(self, queryStr):
        """
        Load the query result from the cache, if possible.

        Returns None if the data is not there or if it is too old.
        """

        if self.cacheMaxAge <= 0:
            return None

        cacheFile = self.getCacheFilename(queryStr)

        if os.path.isfile(cacheFile):
            mtime = os.path.getmtime(cacheFile)
            now = time.time()

            if ((now - mtime) / 60) < self.cacheMaxAge:
                with open(cacheFile, 'rb') as f:
                    try:
                        data = pickle.load(f)
                    except pickle.UnpicklingError:
                        pywikibot.warning(u"Couldn't read cached data from %s"
                                            % cacheFile)
                        data = None

                return data

        return None

    def __init__(self, site, name, parameters, aliases=None, subst=False):
        self._name = name
        if not aliases:
            aliases = []
        elif not subst:
            aliases = list(aliases) + [name]
        else:
            name = 'subst:' + name
        if parameters:
            name += '|' + parameters
        self.template = '{{' + name + '}}'
        self._names = frozenset(aliases)

        template_ns = site.namespaces[10]
        # TODO: Add redirects to self.names too
        if not pywikibot.Page(site, self._name, template_ns.id).exists():
            raise ValueError('Orphan template "{0}" does not exist on '
                             '"{1}".'.format(self._name, site))
        for name in self._names:
            if not pywikibot.Page(site, name, template_ns.id).exists():
                pywikibot.warning('Orphan template alias "{0}" does not exist '
                                  'on "{1}"'.format(name, site))
        self.regex = re.compile(
            r'\{\{(?:' + ':|'.join(template_ns) + '|)(' +
            '|'.join(re.escape(name) for name in self._names) +
            r')[\|\}]', re.I)

 def setLinkDead(self, url, error, page, day):
     """Add the fact that the link was found dead to the .dat file."""
     self.semaphore.acquire()
     now = time.time()
     if url in self.historyDict:
         timeSinceFirstFound = now - self.historyDict[url][0][1]
         timeSinceLastFound = now - self.historyDict[url][-1][1]
         # if the last time we found this dead link is less than an hour
         # ago, we won't save it in the history this time.
         if timeSinceLastFound > 60 * 60:
             self.historyDict[url].append((page.title(), now, error))
         # if the first time we found this link longer than x day ago
         # (default is a week), it should probably be fixed or removed.
         # We'll list it in a file so that it can be removed manually.
         if timeSinceFirstFound > 60 * 60 * 24 * day:
             # search for archived page
             try:
                 archiveURL = get_archive_url(url)
             except Exception as e:
                 pywikibot.warning(
                     'get_closest_memento_url({0}) failed: {1}'.format(
                         url, e))
                 archiveURL = None
             if archiveURL is None:
                 archiveURL = weblib.getInternetArchiveURL(url)
             if archiveURL is None:
                 archiveURL = weblib.getWebCitationURL(url)
             self.log(url, error, page, archiveURL)
     else:
         self.historyDict[url] = [(page.title(), now, error)]
     self.semaphore.release()

def getOpenStreetMap(latitude, longitude):
    """
    Get the result from https://nominatim.openstreetmap.org/reverse .

    @rtype: list of tuples
    """
    result = []
    gotInfo = False
    parameters = urlencode({'lat': latitude, 'lon': longitude, 'accept-language': 'en'})
    while not gotInfo:
        try:
            page = fetch('https://nominatim.openstreetmap.org/reverse?format=xml&%s' % parameters)
            et = xml.etree.ElementTree.fromstring(page.content)
            gotInfo = True
        except IOError:
            pywikibot.output(u'Got an IOError, let\'s try again')
            time.sleep(30)
        except socket.timeout:
            pywikibot.output(u'Got a timeout, let\'s try again')
            time.sleep(30)
    validParts = [u'hamlet', u'village', u'city', u'county', u'country']
    invalidParts = [u'path', u'road', u'suburb', u'state', u'country_code']
    addressparts = et.find('addressparts')

    for addresspart in addressparts.getchildren():
        if addresspart.tag in validParts:
            result.append(addresspart.text)
        elif addresspart.tag in invalidParts:
            pywikibot.output(u'Dropping %s, %s' % (addresspart.tag, addresspart.text))
        else:
            pywikibot.warning('%s, %s is not in addressparts lists'
                              % (addresspart.tag, addresspart.text))
    return result

    def loadTypos(self):
        pywikibot.output('Loading typo rules')
        self.typoRules = []

        if self.typos_page_name is None:
            self.typos_page_name = 'Wikipedie:WPCleaner/Typo'
        typos_page = pywikibot.Page(self.site, self.typos_page_name)
        if not typos_page.exists():
            # todo: feedback
            return

        content = typos_page.get()
        load_all = self.load_all is True
        for template, fielddict in textlib.extract_templates_and_params(
            content, remove_disabled_parts=False, strip=False):
            if template.lower() == 'typo':
                try:
                    rule = TypoRule.newFromParameters(fielddict, self.site)
                except IncompleteTypoRuleException as exc:
                    pywikibot.warning(exc.message) # pwb.exception?
                except InvalidExpressionException as exc:
                    if 'fixed-width' not in exc.message:
                        pywikibot.warning('Invalid %s %s: %s' % (
                            exc.aspect, fielddict['1'], exc.message))
                else:
                    rule.id = self.top_id
                    self.top_id += 1
                    if load_all or not rule.needsDecision():
                        self.typoRules.append(rule)

        pywikibot.output('%s typo rules loaded' % len(self.typoRules))
        return self.typoRules

 def testTkdialog(self):
     """Test Tk dialog."""
     try:
         box = Tkdialog('foo', 'tests/data/MP_sounds.png', 'MP_sounds.png')
         box.show_dialog()
     except ImportError as e:
         pywikibot.warning(e)

    def load_values(self, replacements):
        """
        Load the dataset specific replacements into self.variables.

        Does not accept variables other than those in self.variables.
        Automatically handles lat_int and lon_int.

        @param replacements: Dictionary with target variable as key and
            replacement SQL a value. e.g. {"adm0": "'ad'", "lat": "`lat`"}
        """
        required_fields = ('country', 'lang')
        if not all(required in replacements for required in required_fields):
            raise ValueError(
                "All of the required fields '{}' must be replaced".format(
                    "','".join(required_fields)))

        if 'lat' not in replacements:
            self.variables['lat_int'] = None
        if 'lon' not in replacements:
            self.variables['lon_int'] = None

        for variable in self.variables:
            if variable in replacements:
                value = replacements[variable]
                if not isinstance(value, VariableType):
                    raise ValueError(
                        "All variables must be encoded through VariableType, "
                        "'{}' was not".format(variable))
                self.variables[variable] = value.format()

        for target in replacements:
            if target not in self.variables:
                pywikibot.warning(
                    "Unrecognized variable in {table}: {variable}".format(
                        table=self.table, variable=target))

 def status(self):
     """Return Proofread Page status."""
     try:
         return self.site.proofread_levels[self.ql]
     except KeyError:
         pywikibot.warning('Not valid status set for {0!s}: quality level = {1!s}'.format(self.title(asLink=True), self.ql))
         return None

    def display_references(self):
        """
        Display pages which links the current page, sorted per namespace.

        Number of pages to display per namespace is provided by:
        - self.getOption('isorphan')
        """
        refs = self.current_page.ref_table
        if refs:
            total = sum(len(v) for v in refs.values())
            pywikibot.warning('There are %d pages who link to %s.'
                              % (total, self.current_page))
        else:
            return

        show_n_pages = self.getOption('isorphan')
        width = len(max((ns.canonical_prefix() for ns in refs), key=len))
        for ns in sorted(refs):
            n_pages_in_ns = len(refs[ns])
            plural = '' if n_pages_in_ns == 1 else 's'
            ns_name = ns.canonical_prefix() if ns != ns.MAIN else 'Main:'
            ns_id = '[{0}]'.format(ns.id)
            pywikibot.output(
                '    {0!s:<{width}} {1:>6} {2:>10} page{pl}'.format(
                    ns_name, ns_id, n_pages_in_ns, width=width, pl=plural))
            if show_n_pages:  # do not show marker if 0 pages are requested.
                for page in islice_with_ellipsis(refs[ns], show_n_pages):
                    pywikibot.output('      {0!s}'.format(page.title()))

 def run(self):
     for page in self.generator:
         if self.getOption('purge'):
             pywikibot.output(u'Page %s%s purged'
                              % (page.title(asLink=True),
                                 "" if page.purge() else " not"))
             continue
         try:
             # get the page, and save it using the unmodified text.
             # whether or not getting a redirect throws an exception
             # depends on the variable self.touch_redirects.
             page.get(get_redirect=self.getOption('redir'))
             page.save("Pywikibot touch script")
         except pywikibot.NoPage:
             pywikibot.error(u"Page %s does not exist."
                             % page.title(asLink=True))
         except pywikibot.IsRedirectPage:
             pywikibot.warning(u"Page %s is a redirect; skipping."
                               % page.title(asLink=True))
         except pywikibot.LockedPage:
             pywikibot.error(u"Page %s is locked."
                             % page.title(asLink=True))
         except pywikibot.PageNotSaved:
             pywikibot.error(u"Page %s not saved."
                             % page.title(asLink=True))

    def treat_page(self):
        page = self.current_page
        text = page.text
        done_replacements = []
        quickly = self.getOption('quick') is True
        start = time.clock()
        if self.own_generator:
            text = self.currentrule.apply(page.text, done_replacements)
            if page.text == text:
                if quickly:
                    pywikibot.output('Typo not found, not fixing another typos '
                                     'in quick mode')
                    return
            else:
                self.replaced += 1

        for rule in self.typoRules:
            if self.own_generator and rule == self.currentrule: # __eq__
                continue
            if rule.matches(page.title()):
                continue
            if quickly and rule.needsDecision():
                continue

            text = rule.apply(text, done_replacements)
            stop = time.clock()
            if quickly and stop - start > 15:
                pywikibot.warning('Other typos exceeded 15s, skipping')
                break

        self.put_current(
            text, summary='oprava překlepů: %s' % ', '.join(done_replacements))

    def get_sd_template(self):
        """Look for speedy deletion template and return it.

        @return: A valid speedy deletion template.
        @rtype: str or None
        """
        if self.getOption('delete') and not self.site.logged_in(sysop=True):
            sd = self.getOption('sdtemplate')
            if not sd and i18n.twhas_key(self.site,
                                         'redirect-broken-redirect-template'):
                sd = i18n.twtranslate(self.site,
                                      'redirect-broken-redirect-template')
            # TODO: Add bot's signature if needed (Bug: T131517)

            # check whether template exists for this site
            title = None
            if sd:
                template = extract_templates_and_params_regex_simple(sd)
                if template:
                    title = template[0][0]
                    page = pywikibot.Page(self.site, title, ns=10)
                    if page.exists():
                        return sd
            pywikibot.warning(
                'No speedy deletion template {0}available.'
                ''.format('"{0}" '.format(title) if title else ''))
        return None

 def treat_page(self):
     commons = pywikibot.Site(code = u'commons', fam = u'commons')
     today = datetime.date.today()
     # fileTemplate = pywikibot.Page(commons, u'Template:Potd filename')
     # captionTemplate = pywikibot.Page(commons, u'Template:Potd description') # (Potd page, POTD description)
     filePage = pywikibot.Page(commons, u'Template:Potd/%s' % today.isoformat())
     file = get_template_parameter_value(filePage, u'Potd filename', u'1')
     # TODO: use languages instead of lang
     captionPage = pywikibot.Page(commons, u'Template:Potd/%s (%s)'
         % (today.isoformat(), self.current_page.site.lang))
     if self.current_page.site.lang != u'en' and not captionPage.exists():
         pywikibot.warning(u'%s does not exist' % captionPage.title(asLink=True))
         # try en instead
         captionPage = pywikibot.Page(commons, u'Template:Potd/%s (en)' % today.isoformat())
     caption = get_template_parameter_value(captionPage, u'Potd description', u'1')
     # TODO: Complete caption parsing to fix links (if not an interwiki then make it an interwiki to Commons)
     caption = re.sub(r"\[\[([^:])", r"[[:\1", caption, flags=re.UNICODE) # Force links to start with ':'
     caption = re.sub(r"\[\[(:Category:)", r"[[:c\1", caption, flags=re.UNICODE | re.IGNORECASE) # Make category links interwiki links
     # TODO: Use [[d:Q4608595]] to get the local {{Documentation}}
     doc = u'Documentation'
     if file != u'':
         summary = u'Updating Commons picture of the day'
         if caption != u'':
             summary = summary + u', [[:c:%s|caption attribution]]' % captionPage.title()
         else:
             summary = summary + u', failed to parse caption'
             pywikibot.error(u'Failed to parse parameter 1 from {{Potd description}} on %s'
                 % captionPage.title(asLink=True))
         self.put_current(u'<includeonly>{{#switch:{{{1|}}}|caption=%s|#default=%s}}</includeonly><noinclude>\n{{%s}}</noinclude>'
             % (caption, file, doc), summary=summary, minor=False)
     else:
         pywikibot.error(u'Failed to parse parameter 1 from {{Potd filename}} on %s'
             % filePage.title(asLink=True))

    def readFromCache(self, queryStr):
        """
        Check if we have cached this data recently enough, read it
        if we have. Returns None if the data is not there or if it is
        too old
        """

        if self.cacheMaxAge <= 0:
            return None

        cacheFile = self.getCacheFilename(queryStr)

        if os.path.isfile(cacheFile):
            mtime = os.path.getmtime(cacheFile)
            now = time.time()

            if ((now - mtime) / 60) < self.cacheMaxAge:

                try:
                    data = pickle.load(open(cacheFile, 'r'))
                except pickle.UnpicklingError:
                    pywikibot.warning(u"Couldn't read cached data from %s"
                                        % cacheFile)
                    data = None

                return data

        return None

 def load(self, page):
     """
     Loads the given page, does some changes, and saves it.
     """
     try:
         # Load the page
         text = page.get()
     except pywikibot.NoPage:
         if self.create:
             pywikibot.output(u"Page %s doesn't exist yet; creating."
                              % (page.title(asLink=True)))
             text = ''
         else:
             pywikibot.output(u"Page %s does not exist; skipping."
                              % page.title(asLink=True))
     except pywikibot.IsRedirectPage as arg:
         redirTarget = pywikibot.Page(self.site, arg.args[0])
         if self.follow_redirects:
             text = redirTarget.get()
         else:
             pywikibot.warning(u"Page %s is a redirect to %s; skipping."
                               % (page.title(asLink=True),
                                  redirTarget.title(asLink=True)))
     else:
         return text

    def read_file_content(self, file_url=None):
        """Return name of temp file in which remote file is saved."""
        if not file_url:
            file_url = self.url
            pywikibot.warning("file_url is not given. "
                              "Set to self.url by default.")
        pywikibot.output(u'Reading file %s' % file_url)
        resume = False
        rlen = 0
        _contents = None
        dt = 15
        uo = URLopener()
        retrieved = False

        while not retrieved:
            if resume:
                pywikibot.output(u"Resume download...")
                uo.addheader('Range', 'bytes=%s-' % rlen)

            infile = uo.open(file_url)

            if 'text/html' in infile.info().getheader('Content-Type'):
                pywikibot.output(u"Couldn't download the image: "
                                 "the requested URL was not found on server.")
                return

            content_len = infile.info().getheader('Content-Length')
            accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes'

            if resume:
                _contents += infile.read()
            else:
                _contents = infile.read()

            infile.close()
            retrieved = True

            if content_len:
                rlen = len(_contents)
                content_len = int(content_len)
                if rlen < content_len:
                    retrieved = False
                    pywikibot.output(
                        u"Connection closed at byte %s (%s left)"
                        % (rlen, content_len))
                    if accept_ranges and rlen > 0:
                        resume = True
                    pywikibot.output(u"Sleeping for %d seconds..." % dt)
                    time.sleep(dt)
                    if dt <= 60:
                        dt += 15
                    elif dt < 360:
                        dt += 60
            else:
                pywikibot.log(
                    u"WARNING: length check of retrieved data not possible.")
        handle, tempname = tempfile.mkstemp()
        with os.fdopen(handle, "wb") as t:
            t.write(_contents)
        return tempname

 def treat_page(self):
     """Process one page."""
     page = self.current_page
     text = page.get()
     for url in weblinksIn(text):
         ignoreUrl = False
         for ignoreR in ignorelist:
             if ignoreR.match(url):
                 ignoreUrl = True
         if not ignoreUrl:
             # Limit the number of threads started at the same time. Each
             # thread will check one page, then die.
             while threading.activeCount() >= config.max_external_links:
                 time.sleep(config.retry_wait)
             thread = LinkCheckThread(page, url, self.history,
                                      self.HTTPignore, self.day)
             # thread dies when program terminates
             thread.setDaemon(True)
             try:
                 thread.start()
             except threading.ThreadError:
                 pywikibot.warning(
                     "Can't start a new thread.\nPlease decrease "
                     "max_external_links in your user-config.py or use\n"
                     "'-max_external_links:' option with a smaller value. "
                     "Default is 50.")
                 raise

def site_rc_listener(site, total=None):
    """Yield changes received from EventStream.

    @param site: the Pywikibot.Site object to yield live recent changes for
    @type site: Pywikibot.BaseSite
    @param total: the maximum number of changes to return
    @type total: int

    @return: pywikibot.comms.eventstream.rc_listener configured for given site
    """
    if isinstance(EventSource, Exception):
        warning('sseclient is required for EventStreams;\n'
                'install it with "pip install sseclient"\n')
        # fallback to old rcstream method
        # NOTE: this will be deprecated soon
        from pywikibot.comms.rcstream import rc_listener
        return rc_listener(
            wikihost=site.hostname(),
            rchost=site.rcstream_host(),
            rcport=site.rcstream_port(),
            rcpath=site.rcstream_path(),
            total=total,
        )

    stream = EventStreams(stream='recentchange', site=site)
    stream.set_maximum_items(total)
    stream.register_filter(server_name=site.hostname())
    return stream

def main(*args):
    generator = None
    local_args = pywikibot.handle_args(args)
    site = pywikibot.Site()
    if str(site) != "commons:commons":
        pywikibot.warning("The script has not been tested on sites other that "
                          "commons:commons.")

    gen_factory = pagegenerators.GeneratorFactory(site)
    for local_arg in local_args:
        if gen_factory.handleArg(local_arg):
            continue
        arg, sep, value = local_arg.partition(':')
        if arg in ('-showcats',):
            options[arg[1:]] = True
        else:
            raise ValueError('Unknown argument: ' + local_arg)

    generator = gen_factory.getCombinedGenerator(gen=generator)
    if not generator:
        pywikibot.bot.suggest_help(missing_generator=True)
    else:
        pregenerator = pagegenerators.PreloadingGenerator(generator)
        for i, page in enumerate(pregenerator):
            if page.exists():
                log = handle_page(page)
                pywikibot.output('\n'.join(log))
                pywikibot.output("")