def load_page(self):
     """Load the page to be archived and break it up into threads."""
     self.header = ''
     self.threads = []
     self.archives = {}
     self.archived_threads = 0
     lines = self.get().split('\n')
     found = False  # Reading header
     cur_thread = None
     for line in lines:
         thread_header = re.search('^== *([^=].*?) *== *$', line)
         if thread_header:
             found = True  # Reading threads now
             if cur_thread:
                 self.threads.append(cur_thread)
             cur_thread = DiscussionThread(thread_header.group(1), self.now,
                                           self.timestripper)
         else:
             if found:
                 cur_thread.feed_line(line)
             else:
                 self.header += line + '\n'
     if cur_thread:
         self.threads.append(cur_thread)
     # This extra info is not desirable when run under the unittest
     # framework, which may be run either directly or via setup.py
     if pywikibot.calledModuleName() not in ['archivebot_tests', 'setup']:
         pywikibot.output(u'%d Threads found on %s'
                          % (len(self.threads), self))

 def fixSelfInterwiki(self, text):
     """
     Interwiki links to the site itself are displayed like local links.
     Remove their language code prefix.
     """
     if not self.talkpage and pywikibot.calledModuleName() <> 'interwiki':
         interwikiR = re.compile(r'\[\[%s\s?:([^\[\]\n]*)\]\]' % self.site.lang)
         text = interwikiR.sub(r'[[\1]]', text)
     return text

 def test_default_user_agent(self):
     """Config defined format string test."""
     self.assertTrue(http.user_agent().startswith(
         pywikibot.calledModuleName()))
     self.assertIn('Pywikibot/' + pywikibot.__release__, http.user_agent())
     self.assertNotIn('  ', http.user_agent())
     self.assertNotIn('()', http.user_agent())
     self.assertNotIn('(;', http.user_agent())
     self.assertNotIn(';)', http.user_agent())
     self.assertIn('requests/', http.user_agent())
     self.assertIn('Python/' + str(PYTHON_VERSION[0]), http.user_agent())

 def test_default_user_agent(self):
     """Config defined format string test."""
     self.assertTrue(http.user_agent().startswith(
         pywikibot.calledModuleName()))
     self.assertIn('Pywikibot/' + pywikibot.__release__, http.user_agent())
     self.assertNotIn('  ', http.user_agent())
     self.assertNotIn('()', http.user_agent())
     self.assertNotIn('(;', http.user_agent())
     self.assertNotIn(';)', http.user_agent())
     self.assertIn('httplib2/', http.user_agent())
     self.assertIn('Python/' + str(sys.version_info[0]), http.user_agent())

    def test_user_agent(self):
        """Test different variants of user agents."""
        x = self.get_site()

        x._userinfo = {'name': 'foo'}
        x._username = ('foo', None)

        self.assertEqual('Pywikibot/' + pywikibot.__release__,
                         user_agent(x, format_string='{pwb}'))

        self.assertEqual(x.family.name,
                         user_agent(x, format_string='{family}'))
        self.assertEqual(x.code,
                         user_agent(x, format_string='{lang}'))
        self.assertEqual(x.family.name + ' ' + x.code,
                         user_agent(x, format_string='{family} {lang}'))

        self.assertEqual(x.username(),
                         user_agent(x, format_string='{username}'))

        x._userinfo = {'name': u'!'}
        x._username = (u'!', None)

        self.assertEqual('!', user_agent(x, format_string='{username}'))

        x._userinfo = {'name': u'foo bar'}
        x._username = (u'foo bar', None)

        self.assertEqual('foo_bar', user_agent(x, format_string='{username}'))

        old_config = '{script}/{version} Pywikibot/2.0 (User:{username})'

        pywikibot.version.getversiondict()
        script_value = pywikibot.calledModuleName() + '/' + pywikibot.version.cache['rev']

        self.assertEqual(script_value + ' Pywikibot/2.0 (User:foo_bar)',
                         user_agent(x, format_string=old_config))

        x._userinfo = {'name': u'⁂'}
        x._username = (u'⁂', None)

        self.assertEqual('%E2%81%82',
                         user_agent(x, format_string='{username}'))

        x._userinfo = {'name': u'127.0.0.1'}
        x._username = (None, None)

        self.assertEqual('Foo', user_agent(x, format_string='Foo {username}'))
        self.assertEqual('Foo (' + x.family.name + ':' + x.code + ')',
                         user_agent(x, format_string='Foo ({script_comments})'))

    def putSpacesInLists(self, text):
        """
        For better readability of bullet list and enumeration wiki source code,
        puts a space between the * or # and the text.

        NOTE: This space is recommended in the syntax help on the English,
        German, and French Wikipedia. It might be that it is not wanted on other
        wikis. If there are any complaints, please file a bug report.
        """
        exceptions = ['comment', 'math', 'nowiki', 'pre', 'source', 'timeline']
        if not (self.redirect or self.template) and \
           pywikibot.calledModuleName() != 'capitalize_redirects':
            text = pywikibot.replaceExcept(
                text,
                r'(?m)^(?P<bullet>[:;]*(\*+|#+)[:;\*#]*)(?P<char>[^\s\*#:;].+?)', '\g<bullet> \g<char>',
                exceptions)
        return text

from pywikibot.exceptions import Server504Error
import pywikibot
import cookielib
import threadedhttp
import pywikibot.version

_logger = "comm.http"


# global variables

# the User-agent: header. The default is 
# '<script>/<revision> Pywikipediabot/2.0', where '<script>' is the currently
# executing script and version is the SVN revision of Pywikipediabot.
USER_AGENT_FORMAT = '{script}/r{version[rev]} Pywikipediabot/2.0'
useragent = USER_AGENT_FORMAT.format(script=pywikibot.calledModuleName(),
                                     version=pywikibot.version.getversiondict())
numthreads = 1
threads = []

connection_pool = threadedhttp.ConnectionPool()
http_queue = Queue.Queue()

cookie_jar = threadedhttp.LockableCookieJar(
                 config.datafilepath("pywikibot.lwp"))
try:
    cookie_jar.load()
except (IOError, cookielib.LoadError):
    pywikibot.debug(u"Loading cookies failed.", _logger)
else:
    pywikibot.debug(u"Loaded cookies from file.", _logger)