def test_urlwithfrag(self):
     urlwith_frag = "https://docs.python.org/2/glossary.html#glossary"
     with test_support.transient_internet(urlwith_frag):
         req = urllib2.Request(urlwith_frag)
         res = urllib2.urlopen(req)
         self.assertEqual(res.geturl(),
                 "https://docs.python.org/2/glossary.html#glossary")

 def test_http_basic(self):
     self.assertIsNone(socket.getdefaulttimeout())
     url = "http://www.example.com"
     with test_support.transient_internet(url, timeout=None):
         u = _urlopen_with_retry(url)
         self.assertIsNone(u.fp._sock.fp._sock.gettimeout())
         u.close()

    def _test_urls(self, urls, handlers, retry=True):
        import socket
        import time
        import logging
        debug = logging.getLogger("test_urllib2").debug

        urlopen = urllib2.build_opener(*handlers).open
        if retry:
            urlopen = _wrap_with_retry_thrice(urlopen, urllib2.URLError)

        for url in urls:
            if isinstance(url, tuple):
                url, req, expected_err = url
            else:
                req = expected_err = None
            debug(url)
            try:
                f = urlopen(url, req)
            except EnvironmentError, err:
                debug(err)
                if expected_err:
                    msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" %
                           (expected_err, url, req, type(err), err))
                    self.assert_(isinstance(err, expected_err), msg)
            else:
                with test_support.transient_internet():
                    buf = f.read()
                f.close()
                debug("read %d bytes" % len(buf))
            debug("******** next url coming up...")
            time.sleep(0.1)

 def testPasswordProtectedSite(self):
     test_support.requires('network')
     with test_support.transient_internet('mueblesmoraleda.com'):
         url = 'http://mueblesmoraleda.com'
         robots_url = url + "/robots.txt"
         # First check the URL is usable for our purposes, since the
         # test site is a bit flaky.
         try:
             urlopen(robots_url)
         except HTTPError as e:
             if e.code not in {401, 403}:
                 self.skipTest(
                     "%r should return a 401 or 403 HTTP error, not %r"
                     % (robots_url, e.code))
         else:
             self.skipTest(
                 "%r should return a 401 or 403 HTTP error, not succeed"
                 % (robots_url))
         parser = robotparser.RobotFileParser()
         parser.set_url(url)
         try:
             parser.read()
         except IOError:
             self.skipTest('%s is unavailable' % url)
         self.assertEqual(parser.can_fetch("*", robots_url), False)

 def test_ftp_timeout(self):
     with test_support.transient_internet(self.FTP_HOST):
         try:
             u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
         except:
             raise
         self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
         u.close()

 def testPythonOrg(self):
     support.requires('network')
     with support.transient_internet('www.python.org'):
         parser = robotparser.RobotFileParser(
             "http://www.python.org/robots.txt")
         parser.read()
         self.assertTrue(
             parser.can_fetch("*", "http://www.python.org/robots.txt"))

    def _test_urls(self, urls, handlers, retry=True):
        import time
        import logging

        debug = logging.getLogger("test_urllib2").debug

        urlopen = urllib2.build_opener(*handlers).open
        if retry:
            urlopen = _wrap_with_retry_thrice(urlopen, urllib2.URLError)

        for url in urls:
            if isinstance(url, tuple):
                url, req, expected_err = url
            else:
                req = expected_err = None
            with test_support.transient_internet(url):
                debug(url)
                try:
                    f = urlopen(url, req, TIMEOUT)
                except EnvironmentError as err:
                    debug(err)
                    if expected_err:
                        msg = "Didn't get expected error(s) %s for %s %s, got %s: %s" % (
                            expected_err,
                            url,
                            req,
                            type(err),
                            err,
                        )
                        self.assertIsInstance(err, expected_err, msg)
                except urllib2.URLError as err:
                    if isinstance(err[0], socket.timeout):
                        print >>sys.stderr, "<timeout: %s>" % url
                        continue
                    else:
                        raise
                else:
                    try:
                        with test_support.transient_internet(url):
                            buf = f.read()
                            debug("read %d bytes" % len(buf))
                    except socket.timeout:
                        print >>sys.stderr, "<timeout: %s>" % url
                    f.close()
            debug("******** next url coming up...")
            time.sleep(0.1)

 def test_networked(self):
     # Default settings: requires a valid cert from a trusted CA
     import ssl
     test_support.requires('network')
     with test_support.transient_internet('self-signed.pythontest.net'):
         h = httplib.HTTPSConnection('self-signed.pythontest.net', 443)
         with self.assertRaises(ssl.SSLError) as exc_info:
             h.request('GET', '/')
         self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED')

 def test_ftp_no_timeout(self):
     self.assertIsNone(socket.getdefaulttimeout(),)
     with test_support.transient_internet(self.FTP_HOST):
         socket.setdefaulttimeout(60)
         try:
             u = _urlopen_with_retry(self.FTP_HOST, timeout=None)
         finally:
             socket.setdefaulttimeout(None)
         self.assertIsNone(u.fp.fp._sock.gettimeout())

 def test_networked_trusted_by_default_cert(self):
     # Default settings: requires a valid cert from a trusted CA
     test_support.requires('network')
     with test_support.transient_internet('www.python.org'):
         h = httplib.HTTPSConnection('www.python.org', 443)
         h.request('GET', '/')
         resp = h.getresponse()
         content_type = resp.getheader('content-type')
         self.assertIn('text/html', content_type)

    def test_sites_no_connection_close(self):
        # Some sites do not send Connection: close header.
        # Verify that those work properly. (#issue12576)

        URL = 'http://www.imdb.com' # No Connection:close
        with test_support.transient_internet(URL):
            req = urllib2.urlopen(URL)
            res = req.read()
            self.assertTrue(res)

 def test_http_no_timeout(self):
     self.assertIsNone(socket.getdefaulttimeout())
     url = "http://www.example.com"
     with test_support.transient_internet(url):
         socket.setdefaulttimeout(60)
         try:
             u = _urlopen_with_retry(url, timeout=None)
         finally:
             socket.setdefaulttimeout(None)
         self.assertIsNone(u.fp._sock.fp._sock.gettimeout())

 def testPasswordProtectedSite(self):
     test_support.requires('network')
     with test_support.transient_internet('mueblesmoraleda.com'):
         url = 'http://mueblesmoraleda.com'
         parser = robotparser.RobotFileParser()
         parser.set_url(url)
         try:
             parser.read()
         except IOError:
             self.skipTest('%s is unavailable' % url)
         self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)

 def test_networked_noverification(self):
     # Switch off cert verification
     import ssl
     test_support.requires('network')
     with test_support.transient_internet('self-signed.pythontest.net'):
         context = ssl._create_stdlib_context()
         h = httplib.HTTPSConnection('self-signed.pythontest.net', 443,
                                     context=context)
         h.request('GET', '/')
         resp = h.getresponse()
         self.assertIn('nginx', resp.getheader('server'))

 def test_custom_headers(self):
     url = "http://www.example.com"
     with test_support.transient_internet(url):
         opener = urllib2.build_opener()
         request = urllib2.Request(url)
         self.assertFalse(request.header_items())
         opener.open(request)
         self.assertTrue(request.header_items())
         self.assertTrue(request.has_header('User-agent'))
         request.add_header('User-Agent','Test-Agent')
         opener.open(request)
         self.assertEqual(request.get_header('User-agent'),'Test-Agent')

 def test_networked_bad_cert(self):
     # We feed a "CA" cert that is unrelated to the server's cert
     import ssl
     test_support.requires('network')
     with test_support.transient_internet('self-signed.pythontest.net'):
         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = ssl.CERT_REQUIRED
         context.load_verify_locations(CERT_localhost)
         h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context)
         with self.assertRaises(ssl.SSLError) as exc_info:
             h.request('GET', '/')
         self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED')

 def test_networked_good_cert(self):
     # We feed the server's cert as a validating cert
     import ssl
     test_support.requires('network')
     with test_support.transient_internet('self-signed.pythontest.net'):
         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = ssl.CERT_REQUIRED
         context.load_verify_locations(CERT_selfsigned_pythontestdotnet)
         h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context)
         h.request('GET', '/')
         resp = h.getresponse()
         server_string = resp.getheader('server')
         self.assertIn('nginx', server_string)

    def testRecvTimeout(self):
        # Test recv() timeout
        _timeout = 0.02

        with test_support.transient_internet(self.addr_remote[0]):
            self.sock.connect(self.addr_remote)
            self.sock.settimeout(_timeout)

            _t1 = time.time()
            self.assertRaises(socket.timeout, self.sock.recv, 1024)
            _t2 = time.time()

            _delta = abs(_t1 - _t2)
            self.assertTrue(_delta < _timeout + self.fuzz,
                         "timeout (%g) is %g seconds more than expected (%g)"
                         %(_delta, self.fuzz, _timeout))

 def test_algorithms(self):
     if test_support.verbose:
         sys.stdout.write("test_algorithms disabled, " "as it fails on some old OpenSSL versions")
     return
     # Issue #8484: all algorithms should be available when verifying a
     # certificate.
     # NOTE: https://sha256.tbs-internet.com is another possible test host
     remote = ("sha2.hboeck.de", 443)
     sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem")
     s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED, ca_certs=sha256_cert)
     with test_support.transient_internet():
         try:
             s.connect(remote)
             if test_support.verbose:
                 sys.stdout.write("\nCipher with %r is %r\n" % (remote, s.cipher()))
                 sys.stdout.write("Certificate is:\n%s\n" % pprint.pformat(s.getpeercert()))
         finally:
             s.close()

 def test_connect_default_port(self):
     test_support.get_attribute(smtplib, 'SMTP_SSL')
     with test_support.transient_internet(self.testServer):
         server = smtplib.SMTP_SSL(self.testServer)
     server.ehlo()
     server.quit()