def _disable_pywikibot_logging():
    """Tells Pywikibot to not log messages below WARNING level to stderr."""
    # We need to wake up Pywikibot's logging interface so that its logger level
    # won't get overridden by a later logging call:
    import pywikibot
    pywikibot.debug("Disabling routine logging", "logging")
    getLogger("pywiki").setLevel("WARNING")

    def request(self, uri, method="GET", body=None, headers=None,
                max_redirects=None, connection_type=None):
        """Start an HTTP request.

        @param uri: The uri to retrieve
        @param method: (optional) The HTTP method to use. Default is 'GET'
        @param body: (optional) The request body. Default is no body.
        @param headers: (optional) Additional headers to send. Defaults
               include C{connection: keep-alive}, C{user-agent} and
               C{content-type}.
        @param max_redirects: (optional) The maximum number of redirects to
               use for this request. The class instance's max_redirects is
               default
        @param connection_type: (optional) see L{httplib2.Http.request}

        @return: (response, content) tuple

        """
        if max_redirects is None:
            max_redirects = self.max_redirects
        if headers is None:
            headers = {}
        # Prepare headers
        headers.pop('cookie', None)
        req = DummyRequest(uri, headers)
        self.cookiejar.lock.acquire()
        try:
            self.cookiejar.add_cookie_header(req)
        finally:
            self.cookiejar.lock.release()
        headers = req.headers

        # Wikimedia squids: add connection: keep-alive to request headers
        # unless overridden
        headers['connection'] = headers.pop('connection', 'keep-alive')

        # determine connection pool key and fetch connection
        (scheme, authority, request_uri, defrag_uri) = httplib2.urlnorm(
                                                        httplib2.iri2uri(uri))
        conn_key = scheme+":"+authority

        connection = self.connection_pool.pop_connection(conn_key)
        if connection is not None:
            self.connections[conn_key] = connection

        # Redirect hack: we want to regulate redirects
        follow_redirects = self.follow_redirects
        self.follow_redirects = False
        pywikibot.debug(u"%r" % (
                            (uri.replace("%7C","|"), method, body,
                            headers, max_redirects,
                            connection_type),),
                        _logger)
        try:
            (response, content) = httplib2.Http.request(
                                    self, uri, method, body, headers,
                                    max_redirects, connection_type)
        except Exception, e: # what types?
            # return exception instance to be retrieved by the calling thread
            return e

    def add_mbid_claim_to_item(self, item, mbid):
        """
        Adds a claim with pid `pid` with value `mbid` to `item` and call `donefunc`
        with `mbid` to signal the completion.

        :type pid: str
        :type mbid: str
        :type item: pywikibot.ItemPage
        """
        claim = wp.Claim(const.WIKIDATA_DATASITE, self.property_id)
        claim.setTarget(mbid)
        wp.debug(u"Adding property {pid}, value {mbid} to {title}".format
                 (pid=self.property_id, mbid=mbid, title=item.title()),
                 layer="")
        if wp.config.simulate:
            wp.output("Simulation, no property has been added")
            return
        try:
            item.addClaim(claim, True)
        except wp.UserBlocked as e:
            wp.error("I have been blocked")
            exit(1)
        except wp.Error as e:
            wp.warning(e)
            return
        else:
            wp.debug("Adding the source Claim", layer="")
            claim.addSources([const.MUSICBRAINZ_CLAIM, const.RETRIEVED_CLAIM], bot=True)
            self.donefunc(mbid)

def process_monument(params, source, countryconfig, conn, cursor, source_page,
                     header_defaults, unknown_fields):
    """Process a single instance of a monument row template."""
    title = source_page.title(True)

    # Get all the fields
    contents = {}
    # Add the source of information (permalink)
    contents['source'] = source
    for field in countryconfig.get('fields'):
        if field.get('source') in header_defaults:
            contents[field.get('source')] = header_defaults.get(
                field.get('source'))
        else:
            contents[field.get('source')] = ''

    contents['title'] = title

    for param in params:
        (field, value) = extract_elements_from_template_param(param)

        # Check first that field is not empty
        if field.strip():
            # Is it in the fields list?
            if field in contents:
                # Load it with Big fucking escape hack. Stupid mysql lib
                # Do this somewhere else.replace("'", "\\'")
                contents[field] = value
            else:
                # FIXME: Include more information where it went wrong
                pywikibot.debug(
                    'Found unknown field on page {0} : ({1}: {2})'.format(
                        title, field, value),
                    _logger)
                if field not in unknown_fields:
                    unknown_fields[field] = Counter()
                unknown_fields[field][source_page] += 1
                # time.sleep(5)

    # If we truncate we don't have to check for primkey (it's a made up one)
    if countryconfig.get('truncate'):
        update_monument(
            contents, source, countryconfig, conn, cursor, source_page)
    # Check if the primkey is a tuple and if all parts are present
    elif isinstance(countryconfig.get('primkey'), tuple):
        all_keys = True
        for partkey in countryconfig.get('primkey'):
            if not contents.get(lookup_source_field(partkey, countryconfig)):
                all_keys = False
        if all_keys:
            update_monument(
                contents, source, countryconfig, conn, cursor, source_page)
    # Check if the primkey is filled. This only works for a single primkey,
    # not a tuple
    elif contents.get(lookup_source_field(countryconfig.get('primkey'),
                                          countryconfig)):
        update_monument(
            contents, source, countryconfig, conn, cursor, source_page)
    else:
        raise NoPrimkeyException

def output_country_report(unused_images, report_page, max_images=1000):
    """
    Format and output the unused images data for a a single country.

    @param unused_images: the output of group_unused_images
    @param report_page: pywikibot.Page to which the report should be written
    @param max_images: the max number of images to report to a page. Defaults
        to 1000. Note that actual number of images may be slightly higher in
        order to ensure all candidates for a given monument id are presented.
    """
    # People can add a /header template for with more info
    central_page = ':c:Commons:Monuments database/Unused images'
    text = common.instruction_header(central_page)
    total_pages = 0
    total_ids = 0
    totalImages = 0

    if not unused_images:
        text += common.done_message(central_page, 'unused images')
    else:
        for source_page, value in unused_images.iteritems():
            total_pages += 1
            if totalImages < max_images:
                text += u'=== {0} ===\n'.format(source_page)
                text += u'<gallery>\n'
                for monument_id, candidates in value.iteritems():
                    total_ids += 1
                    if totalImages < max_images:
                        for candidate in candidates:
                            text += u'File:{0}|{1}\n'.format(
                                candidate, monument_id)
                    totalImages += len(candidates)
                text += u'</gallery>\n'
            else:
                for monument_id, candidates in value.iteritems():
                    total_ids += 1
                    totalImages += len(candidates)

    if totalImages >= max_images:
        text += (
            u'<!-- Maximum number of images reached: {0}, '
            u'total of unused images: {1} -->\n'.format(
                max_images, totalImages))
        comment = (
            u'Images to be used in monument lists: '
            u'{0} (gallery maximum reached), '
            u'total of unused images: {1}'.format(
                max_images, totalImages))
    else:
        comment = u'Images to be used in monument lists: {0}'.format(
            totalImages)

    pywikibot.debug(text, _logger)
    common.save_to_wiki_or_local(report_page, comment, text, minorEdit=False)

    return {
        'images': totalImages,
        'pages': total_pages,
        'ids': total_ids
    }

def makeStatistics(totals):
    """Make statistics on the number of indexed images and put on Commons."""
    site = pywikibot.Site('commons', 'commons')
    page = pywikibot.Page(
        site, u'Commons:Monuments database/Indexed images/Statistics')

    title_column = [
        'country', ('images', 'total'), 'tracked',
        ('template', 'tracker template'), ('cat', 'tracker category')
    ]
    table = StatisticsTable(title_column, ('images', 'tracked'))

    for (countrycode, countryresults) in sorted(totals.iteritems()):
        table.add_row({
            'country': countrycode,
            'images': countryresults.get('totalImages'),
            'tracked': countryresults.get('tracked_images'),
            'template': u'{{tl|%s}}' % countryresults.get('commonsTemplate'),
            'cat': u'[[:Category:{cat}|{cat}]]'.format(
                cat=countryresults.get('commonsTrackerCategory'))
        })

    text = table.to_wikitext()

    comment = (
        u'Updating indexed image statistics. '
        u'Total indexed images: {}'.format(table.get_sum('tracked')))
    pywikibot.debug(text, _logger)
    common.save_to_wiki_or_local(page, comment, text)

def output_country_report(rows, report_page, max_images=1000):
    """
    Output a gallery of images without id.

    @param rows: list of (image, id, template) or (image, ) tuples.
    @param report_page: pywikibot.Page where report will be outputted.
    @param max_images: the max number of images to report to a page. Defaults
        to 1000.
    """
    # FIXME create this page. Different name?
    central_page = ':c:Commons:Monuments database/Images without id'
    text = common.instruction_header(central_page)

    if rows:
        gallery_rows = [format_gallery_row(*row) for row in rows[:max_images]]
        text += u'<gallery>\n{}\n</gallery>'.format('\n'.join(gallery_rows))
    else:
        text += common.done_message(central_page, 'images without id')

    if len(rows) > max_images:
        text += (
            u'\n<!-- Maximum number of images reached: {0}, '
            u'total of images without id: {1} -->'.format(
                max_images, len(rows)))
        comment = (
            u'Images without an id: {0} (gallery maximum reached), '
            u'total of images without id: {1}'.format(
                max_images, len(rows)))
    else:
        comment = u'Images without an id: {0}'.format(len(rows))

    pywikibot.debug(text, _logger)
    common.save_to_wiki_or_local(
        report_page, comment, text, minorEdit=False)

def p(n):

    pywikibot.debug('going to load %s.' % n, _logger)
    site = pywikibot.Site('en',"wikipedia")
    repo = site.data_repository()

    print 'SD going to load %s' % n
    return pywikibot.ItemPage(repo, n).get()

 def __eq__(self, other):
     """Compare if self is equal to other."""
     if not isinstance(other, LogEntry):
         pywikibot.debug("'{0}' cannot be compared with '{1}'"
                         .format(type(self).__name__, type(other).__name__),
                         _logger)
         return False
     return self.logid() == other.logid() and self.site == other.site

def make_statistics(statistics):
    """
    Output the overall results of the bot as a nice wikitable.

    @param statistics: list of per dataset statistic dicts where the allowed
        keys are: config, totals, report page and cmt.
    """
    site = pywikibot.Site('commons', 'commons')
    page = pywikibot.Page(
        site, u'Commons:Monuments database/Images without id/Statistics')

    title_column = OrderedDict([
        ('code', 'country'),
        ('lang', '[[:en:List of ISO 639-1 codes|lang]]'),
        ('total_with_id', 'Total monuments with suggested id'),
        ('total_without_id', 'Total monuments without suggested id'),
        # ('total_added', 'Total templates automatically added'),
        ('Report page', None),
        ('Commons template', None)
    ])
    numeric = [key for key in title_column.keys() if key.startswith('total_')]
    table = StatisticsTable(title_column, numeric)

    for row in statistics:
        country_config = row.get('config')
        totals = row.get('totals', {})
        total_with_id_or_cmt = row.get('cmt')
        commons_template = None
        report_page = None

        if totals:
            total_with_id_or_cmt = totals.get('with_id')

        if country_config.get('commonsTemplate'):
            commons_template = u'{{tl|%s}}' % (
                country_config.get('commonsTemplate'), )

        if row.get('report_page'):
            report_page = row.get('report_page').title(
                as_link=True, with_ns=False, insite=site)

        table.add_row({
            'code': country_config.get('country'),
            'lang': country_config.get('lang'),
            'total_with_id': total_with_id_or_cmt,
            'total_without_id': totals.get('without_id'),
            # 'total_added': totals.get('added'),
            'Report page': report_page,
            'Commons template': commons_template})

    text = table.to_wikitext()

    comment = (
        u'Updating images without id statistics. Total of {total_with_id} '
        u'images with suggested ids and {total_without_id} without.'.format(
            **table.get_sum()))
    pywikibot.debug(text, _logger)
    common.save_to_wiki_or_local(page, comment, text)

def makeStatistics(statistics):
    """Output the overall results of the bot as a nice wikitable."""
    site = pywikibot.Site('commons', 'commons')
    page = pywikibot.Page(
        site,
        u'Commons:Monuments database/Missing commonscat links/Statistics')

    title_column = OrderedDict([
        ('code', 'country'),
        ('lang', None),
        ('total', None),
        ('report_page', 'page'),
        ('row template', None),
        ('Commons template', None)
    ])
    table = StatisticsTable(title_column, ('total', ))

    for row in statistics:
        countryconfig = row.get('config')
        total_cats_or_cmt = row.get('total_cats')
        row_template = None
        commons_template = None
        report_page = None

        if row.get('total_cats') is None:
            total_cats_or_cmt = row.get('cmt')

        if countryconfig.get('type') != 'sparql':
            row_template = common.get_template_link(
                row.get('lang'),
                countryconfig.get('project', u'wikipedia'),
                countryconfig.get('rowTemplate'),
                site)

        if countryconfig.get('commonsTemplate'):
            commons_template = u'{{tl|%s}}' % (
                countryconfig.get('commonsTemplate'), )

        if row.get('report_page'):
            report_page = row.get('report_page').title(
                as_link=True, with_ns=False, insite=site)

        table.add_row({
            'code': row.get('code'),
            'lang': row.get('lang'),
            'total': total_cats_or_cmt,
            'report_page': report_page,
            'row template': row_template,
            'Commons template': commons_template})

    text = table.to_wikitext()

    comment = (
        u'Updating missing commonscat links statistics. '
        u'Total missing links: {total_cats}'.format(
            total_cats=table.get_sum('total')))
    pywikibot.debug(text, _logger)
    common.save_to_wiki_or_local(page, comment, text)

def make_statistics(statistics):
    """Output the overall results for unknown fields as a nice wikitable."""
    site = pywikibot.Site('commons', 'commons')
    page = pywikibot.Page(
        site, 'Commons:Monuments database/Unknown fields/Statistics')

    title_column = OrderedDict([
        ('code', 'country'),
        ('lang', None),
        ('total_fields', 'Total fields'),
        ('total_usages', 'Total usage of fields'),
        ('total_pages', 'Total pages containing fields'),
        ('report_page', 'Report page'),
        ('row_template', 'Row template'),
        ('header_template', 'Header template')
    ])
    table = StatisticsTable(
        title_column,
        list(filter(lambda col: col.startswith('total'), title_column)))
    for row in statistics:
        if not row:
            # sparql harvests don't generate statistics
            continue
        countryconfig = row.get('config')

        row_template = common.get_template_link(
            countryconfig.get('lang'),
            countryconfig.get('project', u'wikipedia'),
            countryconfig.get('rowTemplate'),
            site)
        header_template = common.get_template_link(
            countryconfig.get('lang'),
            countryconfig.get('project', u'wikipedia'),
            countryconfig.get('headerTemplate'),
            site)
        report_page = row.get('report_page').title(
            as_link=True, with_ns=False, insite=site)

        table.add_row({
            'code': countryconfig.get('country'),
            'lang': countryconfig.get('lang'),
            'total_fields': row.get('total_fields'),
            'total_usages': row.get('total_usages'),
            'total_pages': row.get('total_pages'),
            'report_page': report_page,
            'row_template': row_template,
            'header_template': header_template
        })

    text = table.to_wikitext()

    comment = (
        'Updating unknown fields statistics. Total of {total_fields} '
        'unknown fields used {total_usages} times on {total_pages} different '
        'pages.'.format(**table.get_sum()))
    pywikibot.debug(text, _logger)
    common.save_to_wiki_or_local(page, comment, text)

 def fix_redirect(self, gid, old, new):
     """
     :param gid str:
     :param old str:
     :param new str:
     """
     wp.debug("Fixing the redirect from %s to %s" % (old, new), layer="")
     self.client.edit_url(gid, old, new, self.edit_note % (old, new))
     self._performed_edit()

    def __init__(self, **kwargs):
        """Constructor."""
        try:
            self.site = kwargs.pop("site")
        except KeyError:
            self.site = pywikibot.Site()
        if 'mime_params' in kwargs:
            self.mime_params = kwargs.pop('mime_params')
            # mime may not be different from mime_params
            if 'mime' in kwargs and kwargs.pop('mime') != self.mime:
                raise ValueError('If mime_params is set, mime may not differ '
                                 'from it.')
        else:
            self.mime = kwargs.pop('mime', False)
        self.throttle = kwargs.pop('throttle', True)
        self.max_retries = kwargs.pop("max_retries", pywikibot.config.max_retries)
        self.retry_wait = kwargs.pop("retry_wait", pywikibot.config.retry_wait)
        self.params = {}
        if "action" not in kwargs:
            raise ValueError("'action' specification missing from Request.")
        self.update(**kwargs)
        self._warning_handler = None
        # Actions that imply database updates on the server, used for various
        # things like throttling or skipping actions when we're in simulation
        # mode
        self.write = self.params["action"] in (
            "edit", "move", "rollback", "delete", "undelete",
            "protect", "block", "unblock", "watch", "patrol",
            "import", "userrights", "upload", "emailuser",
            "createaccount", "setnotificationtimestamp",
            "filerevert", "options", "purge", "revisiondelete",
            "wbeditentity", "wbsetlabel", "wbsetdescription",
            "wbsetaliases", "wblinktitles", "wbsetsitelink",
            "wbcreateclaim", "wbremoveclaims", "wbsetclaimvalue",
            "wbsetreference", "wbremovereferences"
        )
        # MediaWiki 1.23 allows assertion for any action,
        # whereas earlier WMF wikis and others used an extension which
        # could only allow assert for action=edit.
        #
        # When we can't easily check whether the extension is loaded,
        # to avoid cyclic recursion in the Pywikibot codebase, assume
        # that it is present, which will cause a API warning emitted
        # to the logging (console) if it is not present, but will not
        # otherwise be a problem.
        # This situation is only tripped when one of the first actions
        # on the site is a write action and the extension isn't installed.
        if ((self.write and LV(self.site.version()) >= LV("1.23")) or
                (self.params['action'] == 'edit' and
                 self.site.has_extension('AssertEdit'))):
            pywikibot.debug(u"Adding user assertion", _logger)
            self.params["assert"] = "user"  # make sure user is logged in

        if (self.site.protocol() == 'http' and (config.use_SSL_always or (
                self.params["action"] == "login" and config.use_SSL_onlogin))
                and self.site.family.name in config.available_ssl_project):
            self.site = EnableSSLSiteWrapper(self.site)

    def __init__(self, maxnum=5):
        """
        @param maxnum: Maximum number of connections per identifier.
                       The pool drops excessive connections added.

        """
        pywikibot.debug(u"Creating connection pool.", _logger)
        self.connections = {}
        self.lock = threading.Lock()
        self.maxnum = maxnum

 def _createFromData(self, logdata):
     """
     Checks for logtype from data, and creates the correct LogEntry
     """
     try:
         logtype = logdata['type']
         return LogEntryFactory._getEntryClass(logtype)(logdata)
     except KeyError:
         pywikibot.debug(u"API log entry received:\n" + logdata,
                         _logger)
         raise Error("Log entry has no 'type' key")

def categorizeImage(
        countrycode, lang, commonsTemplateName, commonsCategoryBase,
        commonsCatTemplates, page, conn, cursor, harvest_type):
    pywikibot.log(u'Working on: %s' % page.title())
    commonsTemplate = _get_commons_template(commonsTemplateName)
    currentcats = list(page.categories())
    if commonsCategoryBase not in currentcats:
        pywikibot.log(u'%s category not found at: %s. Someone probably already categorized it.' % (
            commonsCategoryBase, page.title()))
        return False

    if u'Wikipedia image placeholders for cultural heritage monuments' in currentcats:
        pywikibot.log(u'%s in %s is a placeholder, skipping it.' % (
            page.title(), commonsCategoryBase))
        return False

    templates = page.templates()
    if commonsTemplate not in templates:
        pywikibot.log(u'%s template not found at: %s' % (
            commonsTemplate, page.title()))
        return False

    try:
        monumentId = get_monument_id(page, commonsTemplate)
    except NoMonumentIdentifierFoundException:
        pywikibot.warning(u'Didn\'t find a valid monument identifier at: %s' % (
            page.title(),))
        return False

    monData = getMonData(countrycode, lang, monumentId, conn, cursor)
    if not monData:
        try:
            monumentId = int(monumentId)
            monData = getMonData(countrycode, lang, monumentId, conn, cursor)
        except ValueError:
            pywikibot.debug(
                u'Can\'t convert %s to an integer' % (monumentId,), _logger)

    if not monData:
        # Triage as log since there are plenty of valid reasons for this
        pywikibot.log(
            u'Monument with id %s not in monuments database' % (monumentId, ))
        return False

    (newcats, categorisation_method) = get_new_categories(monumentId, monData, lang, commonsCatTemplates, harvest_type)

    # See if one of the three options worked
    if newcats:
        comment = u'Adding categories based on [[Template:%s]] with identifier %s (method %s)' % (
            commonsTemplateName, monumentId, categorisation_method)
        return replace_default_cat_with_new_categories_in_image(
            page, commonsCategoryBase, newcats, comment, verbose=True)
    else:
        pywikibot.log(u'Categories not found for %s' % page.title())

def getversiondict():
    """Get version info for the package.

    @return:
        - tag (name for the repository),
        - rev (current revision identifier),
        - date (date of current revision),
        - hash (git hash for the current revision)
    @rtype: C{dict} of four C{str}
    """
    global cache
    if cache:
        return cache

    _program_dir = _get_program_dir()
    exceptions = {}

    for vcs_func in (getversion_git,
                     getversion_svn_setuptools,
                     getversion_nightly,
                     getversion_svn,
                     getversion_package):
        try:
            (tag, rev, date, hsh) = vcs_func(_program_dir)
        except Exception as e:
            exceptions[vcs_func] = e
        else:
            break
    else:
        # nothing worked; version unknown (but suppress exceptions)
        # the value is most likely '$Id' + '$', it means that
        # pywikibot was imported without using version control at all.
        tag, rev, date, hsh = (
            '', '-1 (unknown)', '0 (unknown)', '(unknown)')

    # git and svn can silently fail, as it may be a nightly.
    if getversion_package in exceptions:
        warn('Unable to detect version; exceptions raised:\n%r'
             % exceptions, UserWarning)
    elif exceptions:
        pywikibot.debug('version algorithm exceptions:\n%r'
                        % exceptions, _logger)

    if isinstance(date, basestring):
        datestring = date
    elif isinstance(date, time.struct_time):
        datestring = time.strftime('%Y/%m/%d, %H:%M:%S', date)
    else:
        warn('Unable to detect package date', UserWarning)
        datestring = '-2 (unknown)'

    cache = dict(tag=tag, rev=rev, date=datestring, hsh=hsh)
    return cache

 def __missing__(self, key):
     """Debug when the key is missing."""
     pywikibot.debug(u"API log entry received:\n" + repr(self),
                     _logger)
     if ((key in ('ns', 'title', 'pageid', 'logpage', 'params', 'action')
          and 'actionhidden' in self)
             or (key == 'comment' and 'commenthidden' in self)
             or (key == 'user' and 'userhidden' in self)):
         raise HiddenKeyError(
             "Log entry ({0}) has a hidden '{1}' key and you don't have "
             'permission to view it.'.format(self._type, key))
     raise KeyError("Log entry (%s) has no '%s' key" % (self._type, key))

    def storecookiedata(self, data):
        """
        Store cookie data.

        The argument data is the raw data, as returned by getCookie().

        Returns nothing.
        """
        # THIS IS OVERRIDDEN IN data/api.py
        filename = config.datafilepath("pywikibot.lwp")
        pywikibot.debug(u"Storing cookies to %s" % filename, _logger)
        f = open(filename, "w")
        f.write(data)
        f.close()