def dc_from_hdrs_post(headers, post_data):
    """
    :param headers: HTTP request headers, most importantly containing the
                    content-type info.
    :param post_data: The HTTP request post-data as a string
    :return: The best-match from POST_DATA_CONTAINERS to hold the information
             in self._post_data @ FuzzableRequest
    """
    if headers is None:
        headers = Headers()

    for pdc_klass in POST_DATA_CONTAINERS:
        try:
            return pdc_klass.from_postdata(headers, post_data)
        except (ValueError, TypeError) as e:
            pass
    else:
        content_type, _ = headers.iget("content-type", "None")
        msg = 'Unknown post-data. Content-type: "%s" and/or post-data "%s"'
        om.out.debug(msg % (content_type, post_data[:50]))

        # These lines are for debugging
        # import traceback
        # traceback.print_stack()

        return PlainContainer.from_postdata(headers, post_data)

    def from_httplib_resp(cls, httplibresp, original_url=None):
        """
        Factory function. Build a HTTPResponse object from a
        httplib.HTTPResponse instance
    
        :param httplibresp: httplib.HTTPResponse instance
        :param original_url: Optional 'url_object' instance.
    
        :return: A HTTPResponse instance
        """
        resp = httplibresp
        code, msg, hdrs, body = (resp.code, resp.msg, resp.info(), resp.read())
        hdrs = Headers(hdrs.items())

        if original_url:
            url_inst = URL(resp.geturl(), original_url.encoding)
            url_inst = url_inst.url_decode()
        else:
            url_inst = original_url = URL(resp.geturl())

        httplib_time = DEFAULT_WAIT_TIME
        if hasattr(httplibresp, 'get_wait_time'):
            # This is defined in the keep alive http response object
            httplib_time = httplibresp.get_wait_time()

        if isinstance(resp, urllib2.HTTPError):
            # This is possible because in errors.py I do:
            # err = urllib2.HTTPError(req.get_full_url(), code, msg, hdrs, resp)
            charset = getattr(resp.fp, 'encoding', None)
        else:
            # The encoding attribute is only set on CachedResponse instances
            charset = getattr(resp, 'encoding', None)
        
        return cls(code, body, hdrs, url_inst, original_url,
                   msg, charset=charset, time=httplib_time)

    def test_headers_iget(self):
        upper_headers = Headers([('Abc', 'b')])

        value, real_header = upper_headers.iget('abc')

        self.assertEqual(value, 'b')
        self.assertEqual(real_header, 'Abc')

    def get_all_headers(self):
        """
        :return: Calls get_default_headers to get the default framework headers,
        get_post_data_headers to get the DataContainer headers, merges that info
        with the user specified headers (which live in self._headers) and
        returns a Headers instance which will be sent to the wire.
        """
        wire_headers = Headers()

        for k, v in chain(self._headers.items(),
                          self.get_post_data_headers().items()):

            # Please note that here we're overwriting the headers from the
            # fuzzable request with the headers from the data container,
            # the overwriting is done in this order due to the order in the
            # chain() items above
            #
            # I found a bug where I loaded a request from spider_man, saved
            # it using dump() and then tried to load it again and failed because
            # of this overwriting not being done (the multipart boundary was
            # incorrect).
            #
            # Keep that in mind in case you want to change this overwriting!
            #
            # Overwrite the existing one, case insensitive style
            _, stored_header_name = wire_headers.iget(k, None)
            if stored_header_name is not None:
                wire_headers[stored_header_name] = v
            else:
                wire_headers[k] = v

        return wire_headers

    def get_headers(self):
        """
        Query the spec / operation and return the headers, including the
        content-type, which will be used later to know how to serialize the
        body.
        """
        request_dict = self._bravado_construct_request()
        headers = Headers(request_dict['headers'].items())

        # First, we try to extract content type from a 'consumes'
        # if the operation has one.
        content_type = self.get_consuming_content_type()
        if content_type is not None:
            headers['Content-Type'] = content_type

        content_type, _ = headers.iget('content-type', None)
        if content_type is None and self.parameters:
            # Content-Type is not set yet.
            #
            # There are some specification documents where the consumes
            # section might be empty. This is because the operation doesn't
            # receive anything or because the specification is wrong.
            #
            # If there are parameters then we opt for serializing them as
            # JSON, which is a safe default
            headers['Content-Type'] = self.DEFAULT_CONTENT_TYPE

        return headers

 def test_http_auth_detect_simple(self):
     body = ''
     hdrs = {'content-type': 'text/html', 'www-authenticate': 'realm-w3af'}
     hdrs = Headers(hdrs.items())
     response = HTTPResponse(401, body, hdrs, self.url, self.url, _id=1)
     self.plugin.grep(self.request, response)
     self.assertEqual(len(kb.kb.get('http_auth_detect', 'auth')), 1)
     self.assertEqual(len(kb.kb.get('http_auth_detect', 'userPassUri')), 0)

    def test_headers_update_overlap(self):
        abc_headers = Headers([('Abc', 'b'), ('def', '2')])
        def_headers = Headers([('def', '1')])

        abc_headers.update(def_headers)

        expected_headers = Headers([('Abc', 'b'), ('def', '1')])

        self.assertEqual(expected_headers, abc_headers)

    def test_tokens_to_value(self):
        token = DataToken('a', 'b', ('a',))
        headers = Headers([('a', token)])

        headers.tokens_to_value()

        self.assertIn('a', headers)
        self.assertEqual(headers['a'], 'b')
        self.assertIsInstance(headers['a'], basestring)

    def test_copy_with_token(self):
        dc = Headers([('Abc', 'b')])

        dc.set_token(('Abc',))
        dc_copy = copy.deepcopy(dc)

        self.assertEqual(dc.get_token(), dc_copy.get_token())
        self.assertIsNotNone(dc.get_token())
        self.assertIsNotNone(dc_copy.get_token())
        self.assertEqual(dc_copy.get_token().get_name(), 'Abc')

    def test_analyze_cookies_with_httponly_case_sensitive_expires(self):
        body = ''
        url = URL('https://www.w3af.com/')
        headers = {'content-type': 'text/html',
                   'Set-Cookie': 'name2=value2; Expires=Wed, 09-Jun-2021 10:18:14 GMT;Secure;HttpOnly'}
        headers = Headers(headers.items())
        response = HTTPResponse(200, body, headers, url, url, _id=1)
        request = FuzzableRequest(url, method='GET')

        self.plugin.grep(request, response)

        self.assertEqual(len(kb.kb.get('analyze_cookies', 'cookies')), 1)
        self.assertEqual(len(kb.kb.get('analyze_cookies', 'security')), 0)

    def from_http_request(cls, request):
        """
        :param request: The instance we'll use as base
        :return: An instance of FuzzableRequest based on a urllib2 HTTP request
                 instance.
        """
        headers = request.headers
        headers.update(request.unredirected_hdrs)
        headers = Headers(headers.items())

        post_data = request.get_data() or ""

        return cls.from_parts(request.url_object, method=request.get_method(), headers=headers, post_data=post_data)

    def _urllib_request_to_fr(self, request):
        """
        Convert a urllib2 request object to a FuzzableRequest.
        Used in http_request.

        :param request: A urllib2 request obj.
        :return: A FuzzableRequest.
        """
        headers = request.headers
        headers.update(request.unredirected_hdrs)
        headers = Headers(headers.items())
        fr = FuzzableRequest(request.url_object,
                             request.get_method(),
                             headers)
        fr.set_data(request.get_data() or '')
        return fr

    def mangle_response(self, response):
        """
        This method mangles the response.

        :param response: This is the response to mangle.
        :return: A mangled version of the response.
        """
        body = response.get_body()

        for regex, string in self._manglers['s']['b']:
            body = regex.sub(string, body)

        response.set_body(body)

        header_string = str(response.get_headers())

        for regex, string in self._manglers['s']['h']:
            header_string = regex.sub(string, header_string)

        try:
            mangled_header = Headers.from_string(header_string)
        except ValueError:
            error = 'Your header modifications created an invalid header'\
                    ' string that could NOT be parsed back to a Header object.'
            om.out.error(error)
        else:
            response.set_headers(mangled_header)

        if self._user_option_fix_content_len:
            response = self._fix_content_len(response)

        return response

def create_fuzzable_request_from_request(request, add_headers=None):
    """
    :return: A fuzzable request with the same info as request
    """
    if not isinstance(request, HTTPRequest):
        raise TypeError('Requires HTTPRequest to create FuzzableRequest.')
    
    url = request.url_object
    post_data = str(request.get_data() or '')
    method = request.get_method()
    headers = Headers(request.headers.items())
    headers.update(request.unredirected_hdrs.items())
    headers.update(add_headers or Headers())

    return create_fuzzable_request_from_parts(url, method=method,
                                              post_data=post_data,
                                              add_headers=headers)

    def http_response(self, request, response):

        if len(self._plugin_list) and response._connection.sock is not None:
            # Create the HTTPResponse object
            code, msg, hdrs = response.code, response.msg, response.info()
            hdrs = Headers(hdrs.items())
            url_instance = URL(response.geturl())
            body = response.read()
            # Id is not here, the mangle is done BEFORE logging
            # id = response.id

            http_resp = HTTPResponse(code, body, hdrs, url_instance,
                                     request.url_object, msg=msg)

            for plugin in self._plugin_list:
                plugin.mangle_response(http_resp)

            response = self._HTTPResponse2httplib(response, http_resp)

        return response

    def get_all_headers(self):
        """
        :return: Calls get_default_headers to get the default framework headers,
        get_post_data_headers to get the DataContainer headers, merges that info
        with the user specified headers (which live in self._headers) and
        returns a Headers instance which will be sent to the wire.
        """
        wire_headers = Headers()

        for k, v in chain(self._headers.items(),
                          self.get_post_data_headers().items()):

            # Ignore any keys which are already defined in the user-specified
            # headers
            kvalue, kreal = wire_headers.iget(k, None)
            if kvalue is not None:
                continue

            wire_headers[k] = v

        return wire_headers

    def set_headers(self, headers):
        """
        Sets the headers and also analyzes them in order to get the response
        mime type (text/html , application/pdf, etc).

        :param headers: The headers dict.
        """
        # Fix lowercase in header names from HTTPMessage
        if isinstance(headers, httplib.HTTPMessage):
            self._headers = Headers()
            for header in headers.headers:
                key, value = header.split(':', 1)
                self._headers[key.strip()] = value.strip()
        else:
            self._headers = headers

        find_word = lambda w: content_type.find(w) != -1

        content_type_hvalue, _ = self._headers.iget(CONTENT_TYPE, None)

        # we need exactly content type but not charset
        if content_type_hvalue is not None:
            try:
                self._content_type = content_type_hvalue.split(';', 1)[0].strip().lower()
            except:
                msg = 'Invalid Content-Type value "%s" sent in HTTP response.'
                om.out.debug(msg % (content_type_hvalue,))
            else:
                content_type = self._content_type

                # Set the doc_type
                if content_type.count('image'):
                    self._doc_type = HTTPResponse.DOC_TYPE_IMAGE

                elif content_type.count('pdf'):
                    self._doc_type = HTTPResponse.DOC_TYPE_PDF

                elif content_type.count('x-shockwave-flash'):
                    self._doc_type = HTTPResponse.DOC_TYPE_SWF

                elif any(imap(find_word,
                              ('text', 'html', 'xml', 'txt', 'javascript'))):
                    self._doc_type = HTTPResponse.DOC_TYPE_TEXT_OR_HTML

        # Check if the doc type is still None, that would mean that none of the
        # previous if statements matched.
        #
        # Note that I'm doing this here and not before the other if statements
        # because that triggered a race condition with threads asking if the
        # _doc_type was != None (which it was because I was setting it to
        # DOC_TYPE_OTHER) and that raised all types of errors.
        if self._doc_type is None:
            self._doc_type = HTTPResponse.DOC_TYPE_OTHER

    def __init__(self, uri, method='GET', headers=None, cookie=None, dc=None):
        super(FuzzableRequest, self).__init__()
        
        # Internal variables
        self._dc = dc or DataContainer()
        self._method = method
        self._headers = Headers(headers or ())
        self._cookie = cookie or Cookie()
        self._data = None
        self.set_uri(uri)

        # Set the internal variables
        self._sent_info_comp = None

    def mangle_request(self, request):
        """
        This method mangles the request.

        :param request: This is the request to mangle.
        :return: A mangled version of the request.
        """
        data = request.get_data()
        for regex, string in self._manglers['q']['b']:
            data = regex.sub(string, data)

        header_string = str(request.get_headers())
        
        for regex, string in self._manglers['q']['h']:
            header_string = regex.sub(string, header_string)
        
        headers_inst = Headers.from_string(header_string)

        request.set_headers(headers_inst)
        request.add_data(data)
        return request

    def mangle_request(self, request):
        """
        This method mangles the request.

        :param request: This is the request to mangle.
        :return: A mangled version of the request.
        """
        data = request.get_data()
        for regex, string in self._manglers['q']['b']:
            data = regex.sub(string, data)

        header_string = str(request.get_headers())
        
        for regex, string in self._manglers['q']['h']:
            header_string = regex.sub(string, header_string)
        
        headers_inst = Headers.from_string(header_string)

        return FuzzableRequest.from_parts(request.get_uri(),
                                          method=request.get_method(),
                                          post_data=data, headers=headers_inst)