def _setup_404_detection(self):
        #
        #    NOTE: I need to perform this test here in order to avoid some weird
        #    thread locking that happens when the webspider calls is_404, and
        #    because I want to initialize the is_404 database in a controlled
        #    try/except block.
        #
        from w3af.core.controllers.core_helpers.fingerprint_404 import is_404

        for url in cf.cf.get('targets'):
            try:
                response = self._w3af_core.uri_opener.GET(url, cache=True)
            except ScanMustStopByUserRequest:
                raise
            except Exception, e:
                msg = ('Failed to send HTTP request to the configured target'
                       ' URL "%s", the original exception was: "%s" (%s).')
                args = (url, e, e.__class__.__name__)
                raise ScanMustStopException(msg % args)

            try:
                is_404(response)
            except ScanMustStopByUserRequest:
                raise
            except Exception, e:
                msg = ('Failed to initialize the 404 detection using HTTP'
                       ' response from "%s", the original exception was: "%s"'
                       ' (%s).')
                args = (url, e, e.__class__.__name__)
                raise ScanMustStopException(msg % args)

    def grep(self, request, response):
        """
        Plugin entry point. Get responses, analyze words, create dictionary.

        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None.
        """
        if not self.got_lang():
            return

        # I added the 404 code here to avoid doing some is_404 lookups
        if response.get_code() not in self.BANNED_STATUS \
        and not is_404(response) \
        and request.get_method() in self.ALLOWED_METHODS:

            # Run the plugins
            data = self._run_plugins(response)

            with self._plugin_lock:
                old_data = kb.kb.raw_read(self.get_name(), self.get_name())

                new_data = self.merge_maps(old_data, data, request,
                                           self.captured_lang)

                new_data = self._trim_data(new_data)

                # save the updated map
                kb.kb.raw_write(self, self.get_name(), new_data)

    def _extract_urls(self, fuzzable_request, response):
        """
        Extract information from the server-status page and send FuzzableRequest
        instances to the core.
        """
        self.output_queue.put(FuzzableRequest(response.get_url()))

        # Now really parse the file and create custom made fuzzable requests
        regex = "<td>.*?<td nowrap>(.*?)</td><td nowrap>.*? (.*?) HTTP/1"
        for domain, path in re.findall(regex, response.get_body()):

            if "unavailable" in domain:
                domain = response.get_url().get_domain()

            # Check if the requested domain and the found one are equal.
            if domain == response.get_url().get_domain():
                proto = response.get_url().get_protocol()
                found_url = proto + "://" + domain + path
                found_url = URL(found_url)

                # They are equal, request the URL and create the fuzzable
                # requests
                tmp_res = self._uri_opener.GET(found_url, cache=True)
                if not is_404(tmp_res):
                    self.output_queue.put(FuzzableRequest(found_url))
            else:
                # This is a shared hosting server
                self._shared_hosting_hosts.append(domain)

    def _confirm_file_upload(self, path, mutant, http_response):
        """
        Confirms if the file was uploaded to path

        :param path: The URL where we suspect that a file was uploaded to.
        :param mutant: The mutant that originated the file on the remote end
        :param http_response: The HTTP response asociated with sending mutant
        """
        get_response = self._uri_opener.GET(path, cache=False)

        if not is_404(get_response) and self._has_no_bug(mutant):
            desc = "A file upload to a directory inside the webroot" " was found at: %s" % mutant.found_at()

            v = Vuln.from_mutant(
                "Insecure file upload",
                desc,
                severity.HIGH,
                [http_response.id, get_response.id],
                self.get_name(),
                mutant,
            )

            v["file_dest"] = get_response.get_url()
            v["file_vars"] = mutant.get_file_vars()

            self.kb_append_uniq(self, "file_upload", v)

    def _exists_in_target(self, url):
        """
        Check if a resource still exists in the target web site.

        :param url: The resource to verify.
        :return: None, the result is stored in self.output_queue
        """
        if url in self._already_verified:
            return

        self._already_verified.add(url)

        response = self._uri_opener.GET(url, cache=True)

        if not is_404(response):
            msg = 'The URL: "%s" was found at archive.org and is'\
                  ' STILL AVAILABLE in the target site.'
            om.out.debug(msg % url)

            fr = FuzzableRequest(response.get_uri())
            self.output_queue.put(fr)
        else:
            msg = 'The URL: "%s" was found at archive.org and was'\
                  ' DELETED from the target site.'
            om.out.debug(msg % url)

    def _confirm_file_upload(self, path, mutant, http_response):
        """
        Confirms if the file was uploaded to path

        :param path: The URL where we suspect that a file was uploaded to.
        :param mutant: The mutant that originated the file on the remote end
        :param http_response: The HTTP response asociated with sending mutant
        """
        get_response = self._uri_opener.GET(path, cache=False)

        if not is_404(get_response) and self._has_no_bug(mutant):
            # This is necessary, if I don't do this, the session
            # saver will break cause REAL file objects can't
            # be picked
            mutant.set_mod_value('<file_object>')

            desc = 'A file upload to a directory inside the webroot' \
                   ' was found at: %s' % mutant.found_at()
            
            v = Vuln.from_mutant('Insecure file upload', desc, severity.HIGH,
                                 [http_response.id, get_response.id],
                                 self.get_name(), mutant)
            
            v['file_dest'] = get_response.get_url()
            v['file_vars'] = mutant.get_file_vars()

            self.kb_append_uniq(self, 'file_upload', v)

    def audit(self, freq, orig_response):
        """
        Searches for file upload vulns using a POST to author.dll.

        :param freq: A FuzzableRequest
        """
        domain_path = freq.get_url().get_domain_path()

        if kb.kb.get(self, 'frontpage'):
            # Nothing to do, I have found vuln(s) and I should stop on first
            msg = 'Not verifying if I can upload files to: "%s" using'\
                  ' author.dll. Because I already found a vulnerability.'
            om.out.debug(msg)
            return

        # I haven't found any vulns yet, OR i'm trying to find every
        # directory where I can write a file.
        if domain_path not in self._already_tested:
            self._already_tested.add(domain_path)

            # Find a file that doesn't exist and then try to upload it
            for _ in xrange(3):
                rand_file = rand_alpha(5) + '.html'
                rand_path_file = domain_path.url_join(rand_file)
                res = self._uri_opener.GET(rand_path_file)
                if is_404(res):
                    upload_id = self._upload_file(domain_path, rand_file)
                    self._verify_upload(domain_path, rand_file, upload_id)
                    break
            else:
                msg = 'frontpage plugin failed to find a 404 page. This is'\
                      ' mostly because of an error in 404 page detection.'
                om.out.error(msg)

    def crawl(self, fuzzable_request):
        """
        Get the sitemap.xml file and parse it.

        :param fuzzable_request: A fuzzable_request instance that contains
                                   (among other things) the URL to test.
        """
        base_url = fuzzable_request.get_url().base_url()
        sitemap_url = base_url.url_join('sitemap.xml')
        response = self._uri_opener.GET(sitemap_url, cache=True)

        if '</urlset>' in response and not is_404(response):
            # Send response to core
            fr = FuzzableRequest.from_http_response(response)
            self.output_queue.put(fr)

            om.out.debug('Parsing xml file with xml.dom.minidom.')
            try:
                dom = xml.dom.minidom.parseString(response.get_body())
            except:
                raise BaseFrameworkException('Error while parsing sitemap.xml')
            else:
                raw_url_list = dom.getElementsByTagName("loc")
                parsed_url_list = []
                for url in raw_url_list:
                    try:
                        url = url.childNodes[0].data
                        url = URL(url)
                    except ValueError, ve:
                        msg = 'Sitemap file had an invalid URL: "%s"'
                        om.out.debug(msg % ve)
                    except:
                        om.out.debug('Sitemap file had an invalid format')

    def _is_possible_backdoor(self, response):
        """
        Heuristic to infer if the content of <response> has the pattern of a
        backdoor response.

        :param response: HTTPResponse object
        :return: A bool value
        """
        if not is_404(response):
            body_text = response.get_body()
            dom = response.get_dom()
            if dom is not None:
                for ele, attrs in BACKDOOR_COLLECTION.iteritems():
                    for attrname, attr_vals in attrs.iteritems():
                        # Set of lowered attribute values
                        dom_attr_vals = \
                            set(n.get(attrname).lower() for n in
                                (dom.xpath('//%s[@%s]' % (ele, attrname))))
                        # If at least one elem in intersection return True
                        if (dom_attr_vals and set(attr_vals)):
                            return True

            # If no regex matched then try with keywords. At least 2 should be
            # contained in 'body_text' to succeed.
            times = 0
            for back_kw in KNOWN_OFFENSIVE_WORDS:
                if re.search(back_kw, body_text, re.I):
                    times += 1
                    if times == 2:
                        return True
        return False

    def crawl(self, fuzzable_req):
        """
        Searches for links on the html.

        :param fuzzable_req: A fuzzable_req instance that contains
                             (among other things) the URL to test.
        """
        self._handle_first_run()

        #
        # If it is a form, then smart_fill the parameters to send something that
        # makes sense and will allow us to cover more code.
        #
        data_container = fuzzable_req.get_raw_data()
        if isinstance(data_container, Form):

            if fuzzable_req.get_url() in self._already_filled_form:
                return

            self._already_filled_form.add(fuzzable_req.get_url())
            data_container.smart_fill()

        # Send the HTTP request
        resp = self._uri_opener.send_mutant(fuzzable_req)

        # Nothing to do here...
        if resp.get_code() == http_constants.UNAUTHORIZED:
            return

        # And we don't trust what comes from the core, check if 404
        if is_404(resp):
            return

        self._extract_html_forms(resp, fuzzable_req)
        self._extract_links_and_verify(resp, fuzzable_req)

    def _classic_worker(self, gh, search_term):
        """
        Perform the searches and store the results in the kb.
        """
        google_list = self._google_se.get_n_results(search_term, 9)

        for result in google_list:
            # I found a vuln in the site!
            response = self._uri_opener.GET(result.URL, cache=True)
            if not is_404(response):
                desc = 'ghdb plugin found a vulnerability at URL: "%s".' \
                      ' According to GHDB the vulnerability description'\
                      ' is "%s".'
                desc = desc % (response.get_url(), gh.desc)
                
                v = Vuln('Google hack database match', desc,
                         severity.MEDIUM, response.id, self.get_name())
                v.set_url(response.get_url())
                v.set_method('GET')

                kb.kb.append(self, 'vuln', v)
                om.out.vulnerability(v.get_desc(), severity=severity.LOW)

                # Create the fuzzable requests
                for fr in self._create_fuzzable_requests(response):
                    self.output_queue.put(fr)

    def _send_and_check(self, repo_url, repo_get_files, repo, domain_path):
        """
        Check if a repository index exists in the domain_path.

        :return: None, everything is saved to the self.out_queue.
        """
        http_response = self.http_get_and_parse(repo_url)

        if not is_404(http_response):

            filenames = repo_get_files(http_response.get_body())

            parsed_url_set = set()

            for filename in self._clean_filenames(filenames):
                test_url = domain_path.url_join(filename)
                if test_url not in self._analyzed_filenames:
                    parsed_url_set.add(test_url)
                    self._analyzed_filenames.add(filename)

            self.worker_pool.map(self.http_get_and_parse, parsed_url_set)

            if parsed_url_set:
                desc = 'A %s was found at: "%s"; this could indicate that'\
                       ' a %s is accessible. You might be able to download'\
                       ' the Web application source code.'
                desc = desc % (repo, http_response.get_url(), repo)
                
                v = Vuln('Source code repository', desc, severity.MEDIUM,
                         http_response.id, self.get_name())
                v.set_url(http_response.get_url())
                
                kb.kb.append(self, repo, v)
                om.out.vulnerability(v.get_desc(), severity=v.get_severity())

    def discover(self, fuzzable_request):
        """
        Checks if JBoss Interesting Directories exist in the target server.
        Also verifies some vulnerabilities.
        """
        base_url = fuzzable_request.get_url().base_url()

        args_iter = izip(repeat(base_url), self.JBOSS_VULNS)
        otm_send_request = one_to_many(self.send_request)
        response_pool = self.worker_pool.imap_unordered(otm_send_request,
                                                        args_iter)

        for vuln_db_instance, response in response_pool:

            if is_404(response):
                continue

            vuln_url = base_url.url_join(vuln_db_instance['url'])
            name = vuln_db_instance['name']
            desc = vuln_db_instance['desc']

            if vuln_db_instance['type'] == 'info':
                o = Info(name, desc, response.id, self.get_name())
            else:
                o = Vuln(name, desc, severity.LOW, response.id, self.get_name())

            o.set_url(vuln_url)
            kb.kb.append(self, 'find_jboss', o)

            for fr in self._create_fuzzable_requests(response):
                self.output_queue.put(fr)

    def crawl(self, fuzzable_request):
        """
        Finds the version of a WordPress installation.
        :param fuzzable_request: A fuzzable_request instance that contains
        (among other things) the URL to test.
        """
        if not self._exec:
            # This will remove the plugin from the crawl plugins to be run.
            raise RunOnce()

        #
        # Check if the server is running wp
        #
        domain_path = fuzzable_request.get_url().get_domain_path()

        # Main scan URL passed from w3af + unique wp file
        wp_unique_url = domain_path.url_join('wp-login.php')
        response = self._uri_opener.GET(wp_unique_url, cache=True)

        # If wp_unique_url is not 404, wordpress = true
        if not is_404(response):
            # It was possible to analyze wp-login.php, don't run again
            self._exec = False

            # Analyze the identified wordpress installation
            self._fingerprint_wordpress(domain_path, wp_unique_url,
                                        response)

            # Extract the links
            for fr in self._create_fuzzable_requests(response):
                self.output_queue.put(fr)

    def _do_request(self, url, mutant):
        """
        Perform a simple GET to see if the result is an error or not, and then
        run the actual fuzzing.
        """
        response = self._uri_opener.GET(
            mutant, cache=True, headers=self._headers)

        if not (is_404(response) or
        response.get_code() in (403, 401) or
        self._return_without_eval(mutant)):

            # Create the fuzzable request and send it to the core
            fr = FuzzableRequest.from_http_response(response)
            self.output_queue.put(fr)
            
            #
            #   Save it to the kb (if new)!
            #
            if response.get_url() not in self._seen and response.get_url().get_file_name():
                desc = 'A potentially interesting file was found at: "%s".'
                desc = desc % response.get_url()

                i = Info('Potentially interesting file', desc, response.id,
                         self.get_name())
                i.set_url(response.get_url())
                
                kb.kb.append(self, 'files', i)
                om.out.information(i.get_desc())

                # Report only once
                self._seen.add(response.get_url())

    def _force_disclosures(self, domain_path, potentially_vulnerable_paths):
        """
        :param domain_path: The path to wordpress' root directory
        :param potentially_vulnerable_paths: A list with the paths I'll URL-join
                                             with @domain_path, GET and parse.
        """
        for pvuln_path in potentially_vulnerable_paths:

            pvuln_url = domain_path.url_join(pvuln_path)
            response = self._uri_opener.GET(pvuln_url, cache=True)

            if is_404(response):
                continue

            response_body = response.get_body()
            if 'Fatal error: ' in response_body:
                desc = 'Analyze the HTTP response body to find the full path'\
                       ' where wordpress was installed.'
                i = Info('WordPress path disclosure', desc, response.id,
                         self.get_name())
                i.set_url(pvuln_url)
                
                kb.kb.append(self, 'info', i)
                om.out.information(i.get_desc())
                break

    def _extract_paths(self, domain_path):
        """
        :param domain_path: The URL object pointing to the current wordpress
                            installation
        :return: A list with the paths that might trigger full path disclosures

        TODO: Will fail if WordPress is running on a Windows server due to
              paths manipulation.
        """
        theme_paths = []
        wp_root_response = self._uri_opener.GET(domain_path, cache=True)

        if is_404(wp_root_response):
            return

        response_body = wp_root_response.get_body()

        theme_regexp = '%swp-content/themes/(.*)/style.css' % domain_path
        theme = re.search(theme_regexp, response_body, re.IGNORECASE)
        if theme:
            theme_name = theme.group(1)
            for fname in ('header', 'footer'):
                path_fname = 'wp-content/themes/%s/%s.php' % (theme_name, fname)
                theme_paths.append(path_fname)

        return theme_paths

    def grep(self, request, response):
        """
        Plugin entry point. Get responses, analyze words, create dictionary.

        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None.
        """
        if not self.got_lang():
            return

        # I added the 404 code here to avoid doing some is_404 lookups
        if response.get_code() not in {500, 401, 403, 404} \
        and not is_404(response) and request.get_method() in {'POST', 'GET'}:

            # Run the plugins
            data = self._run_plugins(response)

            with self._plugin_lock:
                old_data = kb.kb.raw_read('password_profiling',
                                          'password_profiling')

                new_data = self.merge_maps(old_data, data, request,
                                           self.captured_lang)

                new_data = self._trim_data(new_data)

                # save the updated map
                kb.kb.raw_write(self, 'password_profiling', new_data)

    def discover(self, fuzzable_request):
        """
        For every directory, fetch a list of files and analyze the response.

        :param fuzzable_request: A fuzzable_request instance that contains
                                    (among other things) the URL to test.
        """
        for domain_path in fuzzable_request.get_url().get_directories():

            if domain_path in self._analyzed_dirs:
                continue

            # Save the domain_path so I know I'm not working in vane
            self._analyzed_dirs.add(domain_path)

            # Request the file
            frontpage_info_url = domain_path.url_join("_vti_inf.html")
            try:
                response = self._uri_opener.GET(frontpage_info_url,
                                                cache=True)
            except BaseFrameworkException, w3:
                fmt = 'Failed to GET Frontpage Server _vti_inf.html file: "%s"'\
                      '. Exception: "%s".'
                om.out.debug(fmt % (frontpage_info_url, w3))
            else:
                # Check if it's a Frontpage Info file
                if not is_404(response):
                    fr = FuzzableRequest(response.get_uri())
                    self.output_queue.put(fr)

                    self._analyze_response(response)

    def grep(self, request, response):
        """
        Plugin entry point, search for meta tags.

        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None
        """
        if not response.is_text_or_html() or is_404(response):
            return

        try:
            dp = parser_cache.dpc.get_document_parser_for(response)
        except BaseFrameworkException:
            return

        meta_tag_list = dp.get_meta_tags()

        for tag in meta_tag_list:
            for attr_name, attr_value in tag.items():

                if not attr_name or not attr_value:
                    # https://github.com/andresriancho/w3af/issues/2012
                    continue

                for word in self.INTERESTING_WORDS:

                    # Check if we have something interesting and WHERE that
                    # thing actually is
                    if word in attr_name:
                        where = ATTR_NAME
                        content = attr_name
                    elif word in attr_value:
                        where = ATTR_VALUE
                        content = attr_value
                    else:
                        # Go to the next one if nothing is found
                        continue

                    # Now... if we found something, report it =)
                    desc = ('The URI: "%s" sent a <meta> tag with the attribute'
                            ' %s set to "%s" which looks interesting.')
                    desc %= (response.get_uri(), where, content)

                    tag_name = self._find_tag_name(tag)
                    usage = self.INTERESTING_WORDS.get(tag_name, None)
                    if usage is not None:
                        desc += ' The tag is used for %s.' % usage

                    i = Info('Interesting META tag', desc, response.id,
                             self.get_name())
                    i.set_uri(response.get_uri())
                    i.add_to_highlight(where, content)
                    i[CONTENT] = content
                    i[WHERE] = where

                    self.kb_append_uniq_group(self, 'meta_tags', i,
                                              group_klass=MetaTagsInfoSet)