def get_desc(self, with_id=False):
        if self.TEMPLATE is None:
            return self.first_info.get_desc(with_id=with_id)

        # We render the template using the information set data
        context = {'urls': [smart_unicode(u) for u in self.get_urls()],
                   'uris': [smart_unicode(u) for u in self.get_uris()],
                   'severity': self.get_severity(),
                   'name': self.get_name(),
                   'id': self.get_id(),
                   'method': smart_unicode(self.get_method()),
                   'plugin': self.get_plugin_name()}
        context.update(self.first_info.items())

        template_str = textwrap.dedent(self.TEMPLATE)
        template = self.JINJA2_ENV.from_string(template_str)

        try:
            rendered_desc = template.render(context)
        except UnicodeDecodeError:
            context_pp = pprint.pformat(context, indent=4)
            msg = ('UnicodeDecodeError found while rendering:\n\n%s\n\n'
                   'Using the following context:\n\n%r\n\n')
            om.out.debug(msg % (smart_str(template_str),
                                smart_str(context_pp)))
            raise

        return rendered_desc

    def __setitem__(self, k, v):
        if isinstance(k, basestring):
            k = smart_unicode(k, encoding=self.encoding)
        else:
            raise ValueError('Header name must be a string.')

        if isinstance(v, basestring):
            v = smart_unicode(v, encoding=self.encoding)
        elif isinstance(v, DataToken):
            encoded_str = smart_unicode(v.get_value(), encoding=self.encoding)
            v.set_value(encoded_str)
        else:
            raise ValueError('Header value must be a string.')

        super(Headers, self).__setitem__(k, v)

    def end(self):
        """
        This method is called when the scan has finished, we perform these
        main tasks:
            * Get the target URLs
            * Get the enabled plugins
            * Get the vulnerabilities and infos from the KB
            * Get the debug data
            * Send all the data to jinja2 for rendering the template

        """
        target_urls = [t.url_string for t in cf.cf.get('targets')]
        target_domain = cf.cf.get('target_domains')[0]
        enabled_plugins = self._enabled_plugins
        findings = kb.kb.get_all_findings()
        debug_log = ((t, l, smart_unicode(m)) for (t, l, m) in self._additional_info)
        known_urls = kb.kb.get_all_known_urls()

        context = {'target_urls': target_urls,
                   'target_domain': target_domain,
                   'enabled_plugins': enabled_plugins,
                   'findings': findings,
                   'debug_log': debug_log,
                   'known_urls': known_urls}

        # The file was verified to exist when setting the plugin configuration
        template_fh = file(os.path.expanduser(self._template), 'r')
        output_fh = file(os.path.expanduser(self._output_file_name), 'w')

        self._render_html_file(template_fh, context, output_fh)

 def test_parse_response_with_no_charset_in_header(self):
     # No charset was specified, use the default as well as the default
     # error handling scheme
     for body, charset in TEST_RESPONSES.values():
         html = body.encode(charset)
         resp = self.create_resp(Headers([("Content-Type", "text/xml")]), html)
         self.assertEquals(smart_unicode(html, DEFAULT_CHARSET, ESCAPED_CHAR, on_error_guess=False), resp.body)

def headers_url_generator(resp, fuzzable_req):
    """
    Yields tuples containing:
        * Newly found URL
        * The FuzzableRequest instance passed as parameter
        * The HTTPResponse generated by the FuzzableRequest
        * Boolean indicating if we trust this reference or not

    The newly found URLs are extracted from the http response headers such
    as "Location".

    :param resp: HTTP response object
    :param fuzzable_req: The HTTP request that generated the response
    """
    resp_headers = resp.get_headers()

    for parser, header_names in URL_HEADERS.iteritems():
        for header_name in header_names:

            header_value, _ = resp_headers.iget(header_name, None)
            if header_value is not None:

                header_value = smart_unicode(header_value, encoding=resp.charset)

                for ref in parser(resp, header_name, header_value):
                    yield ref, fuzzable_req, resp, False

    def _headers_url_generator(self, resp, fuzzable_req):
        """
        Yields tuples containing:
            * Newly found URL
            * The FuzzableRequest instance passed as parameter
            * The HTTPResponse generated by the FuzzableRequest
            * Boolean indicating if we trust this reference or not

        The newly found URLs are extracted from the http response headers such
        as "Location".

        :param resp: HTTP response object
        :param fuzzable_req: The HTTP request that generated the response
        """
        # If response was a 30X (i.e. a redirect) then include the
        # corresponding fuzzable request.
        resp_headers = resp.get_headers()

        for url_header_name in URL_HEADERS:
            url_header_value, _ = resp_headers.iget(url_header_name, '')
            if url_header_value:
                url = smart_unicode(url_header_value, encoding=resp.charset)
                try:
                    ref = resp.get_url().url_join(url)
                except ValueError:
                    msg = 'The application sent a "%s" redirect that w3af' \
                          ' failed to correctly parse as an URL, the header' \
                          ' value was: "%s"'
                    om.out.debug(msg % (url_header_name, url))
                else:
                    yield ref, fuzzable_req, resp, False

    def comment(self, elem):
        if self._inside_script:
            # This handles the case where we have:
            # <script><!-- code(); --></script>
            return

        if elem.text is not None:
            self._comments_in_doc.append(smart_unicode(elem.text))

    def clean_values(self, init_val):
        if isinstance(init_val, DataContainer)\
        or isinstance(init_val, dict):
            return init_val

        cleaned_vals = []

        # Cleanup whatever came from the wire into a unicode string
        for key, value in init_val:
            # I can do this key, value thing because the headers do NOT
            # have multiple header values like query strings and post-data
            if isinstance(value, basestring):
                value = smart_unicode(value)
            
            cleaned_vals.append((smart_unicode(key), value))
        
        return cleaned_vals

    def test_invalid_utf8(self):
        invalid_utf8 = "\xf3"
        token = DataToken(self.NAME, invalid_utf8, self.PATH)

        self.assertRaises(UnicodeDecodeError, unicode, token)

        encoded_token = smart_unicode(token)
        self.assertEqual(encoded_token, u"\xf3")

def get_clean_body(mutant, response):
    """
    @see: Very similar to fingerprint_404.py get_clean_body() bug not quite
          the same maybe in the future I can merge both?

    Definition of clean in this method:
        - input:
            - response.get_url() == http://host.tld/aaaaaaa/?id=1 OR 23=23
            - response.get_body() == '...<x>1 OR 23=23</x>...'

        - output:
            - self._clean_body(response) == '...<x></x>...'

    All injected values are removed encoded and 'as is'.

    :param mutant: The mutant where I can get the value from.
    :param response: The HTTPResponse object to clean
    :return: A string that represents the 'cleaned' response body.
    """
    if not response.is_text_or_html():
        return response.body

    body = response.body
    mod_value_1 = mutant.get_token_value()

    # Since the body is already in unicode, when we call body.replace() all
    # arguments are converted to unicode by python. If there are special
    # chars in the mod_value then we end up with an UnicodeDecodeError, so
    # I convert it myself with some error handling
    #
    # https://github.com/andresriancho/w3af/issues/8953
    mod_value_1 = smart_unicode(mod_value_1, errors=PERCENT_ENCODE)

    # unquote, just in case the plugin did an extra encoding of some type.
    # what we want to do here is get the original version of the string
    mod_value_2 = urllib.unquote_plus(mod_value_1)

    payloads_to_replace = set()
    payloads_to_replace.add(mod_value_1)
    payloads_to_replace.add(mod_value_2)

    encoded_payloads = set()

    for payload in payloads_to_replace:
        for encoded_payload in apply_multi_escape_table(payload,
                                                        EXTENDED_TABLE):
            encoded_payloads.add(encoded_payload)

    # uniq sorted by longest len
    encoded_payloads = list(encoded_payloads)
    encoded_payloads.sort(lambda x, y: cmp(len(y), len(x)))

    empty = u''
    replace = unicode.replace
    for to_replace in encoded_payloads:
        body = replace(body, to_replace, empty)

    return body

 def get_path_qs(self):
     """
     :return: Returns the path for the url containing the QS
     """
     res = self.path
     if self.params != u'':
         res += u';' + self.params
     if self.has_query_string():
         res += u'?' + smart_unicode(self.querystring)
     return res

    def test_parse_response_with_wrong_charset(self):
        # A wrong or non-existant charset was set; try to decode the response
        # using the default charset and handling scheme
        from random import choice

        for body, charset in TEST_RESPONSES.values():
            html = body.encode(charset)
            headers = Headers([("Content-Type", "text/xml; charset=%s" % choice(("XXX", "utf-8")))])
            resp = self.create_resp(headers, html)
            self.assertEquals(smart_unicode(html, DEFAULT_CHARSET, ESCAPED_CHAR, on_error_guess=False), resp.body)

    def __setattr__(self, key, value):
        """
        Overriding in order to translate every value to an unicode object

        :param key: The attribute name to set
        :param value: The value (string, unicode or anything else)
        :return: None
        """
        if isinstance(value, basestring):
            value = smart_unicode(value)

        self[key] = value

    def _to_str_with_separators(self, key_val_sep, pair_sep):
        """
        :return: Join all the values stored in this data container using the
                 specified separators.
        """
        lst = []

        for k, v in self.items():
            to_app = u"%s%s%s" % (k, key_val_sep, smart_unicode(v, encoding=UTF8))
            lst.append(to_app)

        return pair_sep.join(lst)

def response_dump(_id):
    """
    :param _id: The ID to query in the database
    :return: The response as unicode
    """
    _history = HistoryItem()

    try:
        details = _history.read(_id)
    except DBException:
        return None

    return smart_unicode(details.response.dump().strip())

    def test_from_dict_encodings(self):
        for body, charset in TEST_RESPONSES.values():
            html = body.encode(charset)
            resp = self.create_resp(Headers([("Content-Type", "text/xml")]), html)

            msg = msgpack.dumps(resp.to_dict())
            loaded_dict = msgpack.loads(msg)

            loaded_resp = HTTPResponse.from_dict(loaded_dict)

            self.assertEquals(
                smart_unicode(html, DEFAULT_CHARSET, ESCAPED_CHAR, on_error_guess=False), loaded_resp.body
            )

    def test_dump_case03(self):
        header_value = ''.join(chr(i) for i in xrange(256))
        
        expected = u'\r\n'.join([u'GET http://w3af.com/a/b/c.php HTTP/1.1',
                                 u'Hola: %s' % smart_unicode(header_value),
                                 u'',
                                 u'a=b'])

        headers = Headers([(u'Hola', header_value)])
        post_data = KeyValueContainer(init_val=[('a', ['b'])])
        fr = FuzzableRequest(self.url, method='GET', post_data=post_data,
                             headers=headers)

        self.assertEqual(fr.dump(), expected)

    def _to_str_with_separators(self, key_val_sep, pair_sep, errors='strict'):
        """
        :return: Join all the values stored in this data container using the
                 specified separators.
        """
        lst = []

        for key, value_list in self.items():
            for value in value_list:
                value = smart_unicode(value, encoding=UTF8, errors=errors)
                to_app = u'%s%s%s' % (key, key_val_sep, value)
                lst.append(to_app)

        return pair_sep.join(lst)

    def test_dump_case03(self):
        header_value = ''.join(chr(i) for i in xrange(256))
        
        expected = u'\r\n'.join([u'GET http://w3af.com/a/b/c.php HTTP/1.1',
                                 u'Hola: %s' % smart_unicode(header_value),
                                 u'',
                                 u''])

        headers = Headers([(u'Hola', header_value)])
        
        #TODO: Note that I'm passing a dc to the FuzzableRequest and it's not
        # appearing in the dump. It might be a bug...
        fr = FuzzableRequest(self.url, method='GET', dc={u'a': ['b']},
                             headers=headers)
        self.assertEqual(fr.dump(), expected)

    def url_string(self):
        """
        :return: A <unicode> representation of the URL
        """
        data = (self.scheme,
                self.netloc,
                self.path,
                self.params,
                self.querystring,
                self.fragment)
        data = [smart_unicode(s) for s in data]

        calc = urlparse.urlunparse(data)

        # ensuring this is actually unicode
        if not isinstance(calc, unicode):
            calc = unicode(calc, self.encoding, 'replace')

        return calc