def test_url_join_case01(self):
     u = URL('http://w3af.com/foo.bar')
     self.assertEqual(u.url_join('abc.html').url_string,
                      u'http://w3af.com/abc.html')
     
     self.assertEqual(u.url_join('/abc.html').url_string,
                      u'http://w3af.com/abc.html')

 def test_url_join_case03(self):
     u = URL('http://w3af.com/def/jkl/')
     self.assertEqual(u.url_join('/def/abc.html').url_string,
                      u'http://w3af.com/def/abc.html')
     
     self.assertEqual(u.url_join('def/abc.html').url_string,
                      u'http://w3af.com/def/jkl/def/abc.html')

 def test_encode_decode(self):
     """Encode and Decode should be able to run one on the result of the
     other and return the original"""
     original = URL(u"https://w3af.com:443/file.asp?id=1%202")
     encoded = original.url_encode()
     decoded = URL(encoded).url_decode()
     self.assertEqual(original, decoded)

    def test_can_be_pickled(self):
        # Pickle a URL object that contains a cache
        u = URL('http://www.w3af.com/')
        domain_path = u.get_domain_path()

        cPickle.dumps(u)
        cPickle.dumps(domain_path)

    def test_simplest_url(self):
        u = URL("http://w3af.com/foo/bar.txt")

        self.assertEqual(u.path, "/foo/bar.txt")
        self.assertEqual(u.scheme, "http")
        self.assertEqual(u.get_file_name(), "bar.txt")
        self.assertEqual(u.get_extension(), "txt")

    def do_ALL(self):
        global global_first_request
        if global_first_request:
            global_first_request = False
            om.out.information(
                'The user is navigating through the spider_man proxy.')

        # Convert to url_object
        path = URL(self.path)

        if path == TERMINATE_URL:
            om.out.information('The user terminated the spider_man session.')
            self._send_end()
            self._spider_man.stop_proxy()
            return

        om.out.debug("[spider_man] Handling request: %s %s" %
                    (self.command, path))
        #   Send this information to the plugin so it can send it to the core
        freq = self._create_fuzzable_request()
        self._spider_man.append_fuzzable_request(freq)

        grep = True
        if path.get_domain() != self.server.w3afLayer.target_domain:
            grep = False

        try:
            response = self._send_to_server(grep=grep)
        except Exception, e:
            self._send_error(e)

 def test_default_proto(self):
     """
     http is the default protocol, we can provide URLs with no proto
     """
     u = URL("w3af.com")
     self.assertEqual(u.get_domain(), "w3af.com")
     self.assertEqual(u.get_protocol(), "http")

    def from_httplib_resp(cls, httplibresp, original_url=None):
        """
        Factory function. Build a HTTPResponse object from a
        httplib.HTTPResponse instance
    
        :param httplibresp: httplib.HTTPResponse instance
        :param original_url: Optional 'url_object' instance.
    
        :return: A HTTPResponse instance
        """
        resp = httplibresp
        code, msg, hdrs, body = (resp.code, resp.msg, resp.info(), resp.read())
        hdrs = Headers(hdrs.items())

        if original_url:
            url_inst = URL(resp.geturl(), original_url.encoding)
            url_inst = url_inst.url_decode()
        else:
            url_inst = original_url = URL(resp.geturl())

        httplib_time = DEFAULT_WAIT_TIME
        if hasattr(httplibresp, 'get_wait_time'):
            # This is defined in the keep alive http response object
            httplib_time = httplibresp.get_wait_time()

        if isinstance(resp, urllib2.HTTPError):
            # This is possible because in errors.py I do:
            # err = urllib2.HTTPError(req.get_full_url(), code, msg, hdrs, resp)
            charset = getattr(resp.fp, 'encoding', None)
        else:
            # The encoding attribute is only set on CachedResponse instances
            charset = getattr(resp, 'encoding', None)
        
        return cls(code, body, hdrs, url_inst, original_url,
                   msg, charset=charset, time=httplib_time)

 def test_simplest_url(self):
     u = URL('http://w3af.com/foo/bar.txt')
     
     self.assertEqual(u.path, '/foo/bar.txt')
     self.assertEqual(u.scheme, 'http')
     self.assertEqual(u.get_file_name(), 'bar.txt')
     self.assertEqual(u.get_extension(), 'txt')

    def test_from_url_keep_form(self):
        o = URL('http://w3af.com/foo/bar.txt')
        o.querystring = URLEncodedForm()

        u = URL.from_URL(o)
        self.assertIsInstance(u.querystring, URLEncodedForm)
        self.assertIsNot(u.querystring, o.querystring)
        self.assertEqual(u.querystring, o.querystring)

 def test_remove_fragment(self):
     u = URL('http://w3af.com/foo/bar.txt?id=3#foobar')
     self.assertEqual(u.remove_fragment().url_string,
                      u'http://w3af.com/foo/bar.txt?id=3')
     
     u = URL('http://w3af.com/foo/bar.txt#foobar')
     self.assertEqual(u.remove_fragment().url_string,
                      u'http://w3af.com/foo/bar.txt')

    def http_request(self, req):
        url_instance = URL(req.get_full_url())
        url_instance.set_param(self._url_parameter)

        new_request = HTTPRequest(url_instance, headers=req.headers,
                                  origin_req_host=req.get_origin_req_host(),
                                  unverifiable=req.is_unverifiable())
        return new_request

    def test_memoized(self):
        u = URL('http://www.w3af.com/')
        self.assertEqual(u._cache, dict())

        url = u.uri2url()
        self.assertNotEqual(u._cache, dict())
        self.assertIn(url, u._cache.values())

        second_url = u.uri2url()
        self.assertIs(url, second_url)

        self.assertIsInstance(url, URL)
        self.assertIsInstance(second_url, URL)

 def test_from_url(self):
     o = URL('http://w3af.com/foo/bar.txt')
     u = URL.from_URL(o)
     
     self.assertEqual(u.path, '/foo/bar.txt')
     self.assertEqual(u.scheme, 'http')
     self.assertEqual(u.get_file_name(), 'bar.txt')
     self.assertEqual(u.get_extension(), 'txt')
     
     o = URL('w3af.com')
     u = URL.from_URL(o)
     self.assertEqual(u.get_domain(), 'w3af.com')
     self.assertEqual(u.get_protocol(), 'http')

    def test_from_url(self):
        o = URL("http://w3af.com/foo/bar.txt")
        u = URL.from_URL(o)

        self.assertEqual(u.path, "/foo/bar.txt")
        self.assertEqual(u.scheme, "http")
        self.assertEqual(u.get_file_name(), "bar.txt")
        self.assertEqual(u.get_extension(), "txt")

        o = URL("w3af.com")
        u = URL.from_URL(o)
        self.assertEqual(u.get_domain(), "w3af.com")
        self.assertEqual(u.get_protocol(), "http")

    def test_memoized(self):
        u = URL('http://www.w3af.com/')
        self.assertEqual(u._cache, dict())

        domain_path = u.get_domain_path()
        self.assertNotEqual(u._cache, dict())
        self.assertIn(domain_path, u._cache.values())

        second_domain_path = u.get_domain_path()
        self.assertIs(domain_path, second_domain_path)

        self.assertIsInstance(domain_path, URL)
        self.assertIsInstance(second_domain_path, URL)

    def setUp(self):
        self.kb.cleanup()
        self.w3afcore = w3afCore()
        
        if self.MOCK_RESPONSES:
            httpretty.enable()
            
            url = URL(self.target_url)
            domain = url.get_domain()
            proto = url.get_protocol()
            port = url.get_port()

            self._register_httpretty_uri(proto, domain, port)

    def test_set_params(self):
        u = URL("http://w3af.com/;id=1")
        u.set_param("file=2")

        self.assertEqual(u.get_params_string(), "file=2")

        u = URL("http://w3af.com/xyz.txt;id=1?file=2")
        u.set_param("file=3")

        self.assertEqual(u.get_params_string(), "file=3")
        self.assertEqual(u.get_path_qs(), "/xyz.txt;file=3?file=2")

 def test_set_params(self):
     u = URL('http://w3af.com/;id=1')
     u.set_param('file=2')
     
     self.assertEqual(u.get_params_string(), 'file=2')
     
     u = URL('http://w3af.com/xyz.txt;id=1?file=2')
     u.set_param('file=3')
     
     self.assertEqual(u.get_params_string(), 'file=3')
     self.assertEqual(u.get_path_qs(), '/xyz.txt;file=3?file=2')

    def test_redirect_uri_relative(self):
        ws = web_spider()
        body = ''
        url = URL('http://www.w3af.org')
        redir_url = '/redir'
        headers = Headers([('content-type', 'text/html'),
                           ('uri', redir_url)])
        resp = HTTPResponse(200, body, headers, url, url)

        gen = ws._headers_url_generator(resp, None)

        extracted_data = [i for i in gen]
        expected_data = [(url.url_join(redir_url), None, resp, False)]

        self.assertEqual(extracted_data, expected_data)