def refresh(site, sysop=False):
    """Fetch the watchlist."""
    if not site.logged_in(sysop=sysop):
        site.forceLogin(sysop=sysop)

    params = {
        'action': 'query',
        'list': 'watchlistraw',
        'site': site,
        'wrlimit': config.special_page_limit,
    }

    pywikibot.output(u'Retrieving watchlist for %s via API.' % str(site))
    # pywikibot.put_throttle() # It actually is a get, but a heavy one.
    watchlist = []
    while True:
        req = pywikibot.data.api.Request(**params)
        data = req.submit()
        if 'error' in data:
            raise RuntimeError('ERROR: %s' % data)
        watchlist.extend([w['title'] for w in data['watchlistraw']])

        if 'query-continue' in data:
            params.update(data['query-continue']['watchlistraw'])
        else:
            break

    # Save the watchlist to disk
    # The file is stored in the watchlists subdir. Create if necessary.
    with open(config.datafilepath('watchlists',
                                  'watchlist-%s-%s%s.dat'
                                  % (site.family.name, site.code,
                                     '-sysop' if sysop else '')),
              'wb') as f:
        pickle.dump(watchlist, f)

    def __init__(self, site, mindelay=None, maxdelay=None, writedelay=None,
                 multiplydelay=True):
        """Constructor."""
        self.lock = threading.RLock()
        self.mysite = str(site)
        self.ctrlfilename = config.datafilepath('throttle.ctrl')
        self.mindelay = mindelay
        if self.mindelay is None:
            self.mindelay = config.minthrottle
        self.maxdelay = maxdelay
        if self.maxdelay is None:
            self.maxdelay = config.maxthrottle
        self.writedelay = writedelay
        if self.writedelay is None:
            self.writedelay = config.put_throttle
        self.last_read = 0
        self.last_write = 0
        self.next_multiplicity = 1.0

        # Check logfile again after this many seconds:
        self.checkdelay = 300

        # Ignore processes that have not made a check in this many seconds:
        self.dropdelay = 600

        # Free the process id after this many seconds:
        self.releasepid = 1200

        self.lastwait = 0.0
        self.delay = 0
        self.checktime = 0
        self.multiplydelay = multiplydelay
        if self.multiplydelay:
            self.checkMultiplicity()
        self.setDelays()

 def __init__(self, rebuild=False, filename="category.dump.bz2"):
     """Constructor."""
     if not os.path.isabs(filename):
         filename = config.datafilepath(filename)
     self.filename = filename
     if rebuild:
         self.rebuild()

    def dump(self, filename=None):
        """Save the dictionaries to disk if not empty.

        Pickle the contents of the dictionaries superclassDB and catContentDB
        if at least one is not empty. If both are empty, removes the file from
        the disk.

        If the filename is None, it'll use the filename determined in __init__.
        """
        if filename is None:
            filename = self.filename
        elif not os.path.isabs(filename):
            filename = config.datafilepath(filename)
        if self.is_loaded and (self.catContentDB or self.superclassDB):
            pywikibot.output("Dumping to %s, please wait..." % config.shortpath(filename))
            f = bz2.BZ2File(filename, "w")
            databases = {"catContentDB": self.catContentDB, "superclassDB": self.superclassDB}
            # store dump to disk in binary format
            try:
                pickle.dump(databases, f, protocol=config.pickle_protocol)
            except pickle.PicklingError:
                pass
            f.close()
        else:
            try:
                os.remove(filename)
            except EnvironmentError:
                pass
            else:
                pywikibot.output("Database is empty. %s removed" % config.shortpath(filename))

    def dump(self, filename='category.dump.bz2'):
        '''Saves the contents of the dictionaries superclassDB and catContentDB
        to disk.

        '''
        if not os.path.isabs(filename):
            filename = config.datafilepath(filename)
        if self.catContentDB or self.superclassDB:
            pywikibot.output(u'Dumping to %s, please wait...'
                             % config.shortpath(filename))
            f = bz2.BZ2File(filename, 'w')
            databases = {
                'catContentDB': self.catContentDB,
                'superclassDB': self.superclassDB
            }
            # store dump to disk in binary format
            try:
                pickle.dump(databases, f, protocol=pickle.HIGHEST_PROTOCOL)
            except pickle.PicklingError:
                pass
            f.close()
        else:
            try:
                os.remove(filename)
            except EnvironmentError:
                pass
            else:
                pywikibot.output(u'Database is empty. %s removed'
                                 % config.shortpath(filename))

def get(site=None):
    """Load the watchlist, fetching it if necessary."""
    if site is None:
        site = pywikibot.Site()
    if site in cache:
        # Use cached copy if it exists.
        watchlist = cache[site]
    else:
        fn = config.datafilepath('watchlists',
                                 'watchlist-%s-%s.dat'
                                 % (site.family.name, site.code))
        try:
            # find out how old our saved dump is (in seconds)
            file_age = time.time() - os.path.getmtime(fn)
            # if it's older than 1 month, reload it
            if file_age > 30 * 24 * 60 * 60:
                pywikibot.output(
                    u'Copy of watchlist is one month old, reloading')
                refresh(site)
        except OSError:
            # no saved watchlist exists yet, retrieve one
            refresh(site)
        with open(fn, 'rb') as f:
            watchlist = pickle.load(f)
        # create cached copy
        cache[site] = watchlist
    return watchlist

    def __init__(self, disambPage, enabled=False):
        self.disambPage = disambPage
        self.enabled = enabled
        self.ignorelist = []

        folder = config.datafilepath('disambiguations')
        if os.path.exists(folder):
            self._read_ignorelist(folder)

 def __init__(self, catTitle, catDB, filename=None, maxDepth=10):
     self.catTitle = catTitle
     self.catDB = catDB
     if filename and not os.path.isabs(filename):
         filename = config.datafilepath(filename)
     self.filename = filename
     self.maxDepth = maxDepth
     self.site = pywikibot.Site()

 def __init__(self, catTitle, catDB, filename=None, maxDepth=10):
     self.catTitle = catTitle
     self.catDB = catDB
     if filename and not os.path.isabs(filename):
         filename = config.datafilepath(filename)
     self.filename = filename
     # TODO: make maxDepth changeable with a parameter or config file entry
     self.maxDepth = maxDepth
     self.site = pywikibot.getSite()

 def ignore(self, refPage):
     if self.enabled:
         # Skip this occurrence next time.
         filename = config.datafilepath(
             'disambiguations',
             self.disambPage.title(asUrl=True) + '.txt')
         try:
             # Open file for appending. If none exists yet, create a new one.
             f = codecs.open(filename, 'a', 'utf-8')
             f.write(refPage.title(asUrl=True) + '\n')
             f.close()
         except IOError:
             pass

    def storecookiedata(self, data):
        """
        Store cookie data.

        The argument data is the raw data, as returned by getCookie().

        Returns nothing.
        """
        # THIS IS OVERRIDDEN IN data/api.py
        filename = config.datafilepath("pywikibot.lwp")
        pywikibot.debug(u"Storing cookies to %s" % filename, _logger)
        f = open(filename, "w")
        f.write(data)
        f.close()

    def __init__(self, disambPage, enabled=False):
        """Constructor.

        @type disambPage: pywikibot.Page
        @type enabled: bool
        @rtype: None

        """
        self.disambPage = disambPage
        self.enabled = enabled
        self.ignorelist = []

        folder = config.datafilepath('disambiguations')
        if os.path.exists(folder):
            self._read_ignorelist(folder)

    def __init__(self, disambPage, enabled=False):
        self.disambPage = disambPage
        self.enabled = enabled

        self.ignorelist = []
        filename = config.datafilepath(
            'disambiguations',
            self.disambPage.title(as_filename=True) + '.txt')
        try:
            # The file is stored in the disambiguation/ subdir.
            # Create if necessary.
            f = codecs.open(filename, 'r', 'utf-8')
            for line in f.readlines():
                # remove trailing newlines and carriage returns
                while line[-1] in ['\n', '\r']:
                    line = line[:-1]
                # skip empty lines
                if line != '':
                    self.ignorelist.append(line)
            f.close()
        except IOError:
            pass

def refresh_all(new=False, sysop=False):
    """Fetch and locally cache several watchlists."""
    if new:
        pywikibot.output(
            'Downloading all watchlists for your accounts in user-config.py')
        for family in config.usernames:
            for lang in config.usernames[family]:
                refresh(pywikibot.Site(lang, family), sysop=sysop)
        for family in config.sysopnames:
            for lang in config.sysopnames[family]:
                refresh(pywikibot.Site(lang, family), sysop=sysop)

    else:
        import dircache
        filenames = dircache.listdir(
            config.datafilepath('watchlists'))
        watchlist_filenameR = re.compile('watchlist-([a-z\-:]+).dat')
        for filename in filenames:
            match = watchlist_filenameR.match(filename)
            if match:
                arr = match.group(1).split('-')
                family = arr[0]
                lang = '-'.join(arr[1:])
                refresh(pywikibot.Site(lang, family))

    #
    'fckeditor': {
        'regex': True,
        'msg': 'pywikibot-fixes-fckeditor',
        'replacements': [
            # replace <br> with a new line
            (r'(?i)<br>',                      r'\n'),
            # replace &nbsp; with a space
            (r'(?i)&nbsp;',                      r' '),
        ],
    },
}


def _load_file(filename):
    """Load the fixes from the given filename."""
    if os.path.exists(filename):
        # load binary, to let compile decode it according to the file header
        with open(filename, 'rb') as f:
            exec(compile(f.read(), filename, 'exec'))
        return True
    else:
        return False

#
# Load the user fixes file.
if _load_file(config.datafilepath('user-fixes.py')):
    user_fixes_loaded = True
else:
    user_fixes_loaded = False

from pywikibot.tools import deprecate_arg
import pywikibot.version

_logger = "comm.http"


# global variables

numthreads = 1
threads = []

connection_pool = threadedhttp.ConnectionPool()
http_queue = Queue.Queue()

cookie_jar = threadedhttp.LockableCookieJar(
    config.datafilepath("pywikibot.lwp"))
try:
    cookie_jar.load()
except (IOError, cookielib.LoadError):
    pywikibot.debug(u"Loading cookies failed.", _logger)
else:
    pywikibot.debug(u"Loaded cookies from file.", _logger)


# Build up HttpProcessors
pywikibot.log(u'Starting %(numthreads)i threads...' % locals())
for i in range(numthreads):
    proc = threadedhttp.HttpProcessor(http_queue, cookie_jar, connection_pool)
    proc.setDaemon(True)
    threads.append(proc)
    proc.start()

            (u'www.mfa.gov.yu',              u'www.mfa.gov.rs'),
            (u'www.drzavnauprava.sr.gov.yu', u'www.drzavnauprava.gov.rs'),
        ],
    },
    # These replacements will convert HTML tag from FCK-editor to wiki syntax.
    #
    'fckeditor': {
        'regex': True,
        'msg': {
            'en': u'Robot: Fixing rich-editor html',
            'fa': u'ربات: تصحیح اچ‌تی‌ام‌ال ویرایشگر پیشرفته',
        },
        'replacements': [
            # replace <br> with a new line
            (r'(?i)<br>',                      r'\n'),
            # replace &nbsp; with a space
            (r'(?i)&nbsp;',                      r' '),
        ],
    },
}

#
# Load the user fixes file.

from pywikibot import config

try:
    exec(compile(open(config.datafilepath("user-fixes.py")).read(), config.datafilepath("user-fixes.py"), 'exec'))
except IOError:
    pass

            (u'www.mfa.gov.yu',              u'www.mfa.gov.rs'),
            (u'www.drzavnauprava.sr.gov.yu', u'www.drzavnauprava.gov.rs'),
        ],
    },
    # These replacements will convert HTML tag from FCK-editor to wiki syntax.
    #
    'fckeditor': {
        'regex': True,
        'msg': {
            'en': u'Robot: Fixing rich-editor html',
            'fa': u'ربات: تصحیح اچ‌تی‌ام‌ال ویرایشگر پیشرفته',
         },
         'replacements': [
            # replace <br> with a new line
            (r'(?i)<br>',                      r'\n'),
            # replace &nbsp; with a space
            (r'(?i)&nbsp;',                      r' '),
        ],
    },
}

#
# Load the user fixes file.

from pywikibot import config

try:
    execfile(config.datafilepath("user-fixes.py"))
except IOError:
    pass

def init_handlers(strm=None):
    """Initialize logging system for terminal-based bots.

    This function must be called before using pywikibot.output(); and must
    be called again if the destination stream is changed.

    @param strm: Output stream. If None, re-uses the last stream if one
        was defined, otherwise uses sys.stderr

    Note: this function is called by handleArgs(), so it should normally
    not need to be called explicitly

    All user output is routed through the logging module.
    Each type of output is handled by an appropriate handler object.
    This structure is used to permit eventual development of other
    user interfaces (GUIs) without modifying the core bot code.
    The following output levels are defined:
       DEBUG - only for file logging; debugging messages
       STDOUT - output that must be sent to sys.stdout (for bots that may
                have their output redirected to a file or other destination)
       VERBOSE - optional progress information for display to user
       INFO - normal (non-optional) progress information for display to user
       INPUT - prompts requiring user response
       WARN - user warning messages
       ERROR - user error messages
       CRITICAL - fatal error messages
    Accordingly, do ''not'' use print statements in bot code; instead,
    use pywikibot.output function.
    """

    global _handlers_initialized

    moduleName = calledModuleName()
    if not moduleName:
        moduleName = "terminal-interface"

    logging.addLevelName(VERBOSE, "VERBOSE")
    # for messages to be displayed on terminal at "verbose" setting
    # use INFO for messages to be displayed even on non-verbose setting

    logging.addLevelName(STDOUT, "STDOUT")
    # for messages to be displayed to stdout

    logging.addLevelName(INPUT, "INPUT")
    # for prompts requiring user response

    root_logger = logging.getLogger("pywiki")
    root_logger.setLevel(DEBUG + 1)  # all records except DEBUG go to logger
    if hasattr(root_logger, 'captureWarnings'):
        root_logger.captureWarnings(True)  # introduced in Python >= 2.7
    root_logger.handlers = []  # remove any old handlers

    # configure handler(s) for display to user interface
    ui.init_handlers(root_logger, **config.userinterface_init_kwargs)

    # if user has enabled file logging, configure file handler
    if moduleName in config.log or '*' in config.log:
        if config.logfilename:
            logfile = config.datafilepath("logs", config.logfilename)
        else:
            logfile = config.datafilepath("logs", "%s-bot.log" % moduleName)
        file_handler = RotatingFileHandler(filename=logfile,
                                           maxBytes=1024 * config.logfilesize,
                                           backupCount=config.logfilecount)

        file_handler.setLevel(DEBUG)
        form = LoggingFormatter(
            fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
                "in %(caller_name)18s: %(levelname)-8s %(message)s",
            datefmt="%Y-%m-%d %H:%M:%S"
        )
        file_handler.setFormatter(form)
        root_logger.addHandler(file_handler)
        # Turn on debugging for each component requested by user
        # or for all components if nothing was specified
        for component in config.debug_log:
            if component:
                debuglogger = logging.getLogger("pywiki." + component)
            else:
                debuglogger = logging.getLogger("pywiki")
            debuglogger.setLevel(DEBUG)
            debuglogger.addHandler(file_handler)

    _handlers_initialized = True

    writelogheader()

# cf. `openssl errstr 14090086`
SSL_CERT_VERIFY_FAILED = ":14090086:"

# the User-agent: header. The default is
# '<script>/<revision> Pywikipediabot/2.0', where '<script>' is the currently
# executing script and version is the SVN revision of Pywikipediabot.
USER_AGENT_FORMAT = '{script}/r{version[rev]} Pywikipediabot/2.0'
useragent = USER_AGENT_FORMAT.format(script=pywikibot.calledModuleName(),
                                     version=pywikibot.version.getversiondict())
numthreads = 1
threads = []

connection_pool = threadedhttp.ConnectionPool()
http_queue = Queue.Queue()

cookie_jar = threadedhttp.LockableCookieJar(config.datafilepath("pywikibot.lwp"))
try:
    cookie_jar.load()
except (IOError, cookielib.LoadError):
    pywikibot.debug(u"Loading cookies failed.", _logger)
else:
    pywikibot.debug(u"Loaded cookies from file.", _logger)


# Build up HttpProcessors
pywikibot.log('Starting %(numthreads)i threads...' % locals())
for i in range(numthreads):
    proc = threadedhttp.HttpProcessor(http_queue, cookie_jar, connection_pool)
    proc.setDaemon(True)
    threads.append(proc)
    proc.start()