def grep(self, request, response):
        """
        Plugin entry point.

        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None, all results are saved in the kb.
        """
        if not response.is_text_or_html():
            return

        uri = response.get_uri()

        for regex in self.RE_LIST:
            for m in regex.findall(response.get_body()):
                user = m[0]

                desc = 'The URL: "%s" contains a SVN versioning signature'\
                       ' with the username "%s".'
                desc = desc % (uri, user)
                
                v = Vuln('SVN user disclosure vulnerability', desc,
                         severity.LOW, response.id, self.get_name())
                v.add_to_highlight(user)
                v.set_uri(uri)
                v[SVNUserInfoSet.ITAG] = user
                
                self.kb_append_uniq_group(self, 'users', v,
                                          group_klass=SVNUserInfoSet)

    def _SEARCH(self, domain_path):
        """
        Test SEARCH method.
        """
        content = "<?xml version='1.0'?>\r\n"
        content += "<g:searchrequest xmlns:g='DAV:'>\r\n"
        content += "<g:sql>\r\n"
        content += "Select 'DAV:displayname' from scope()\r\n"
        content += "</g:sql>\r\n"
        content += "</g:searchrequest>\r\n"

        res = self._uri_opener.SEARCH(domain_path, data=content,
                                      headers=self.CONTENT_TYPE)

        content_matches = '<a:response>' in res or '<a:status>' in res or \
            'xmlns:a="DAV:"' in res

        if content_matches and res.get_code() in xrange(200, 300):
            msg = 'Directory listing with HTTP SEARCH method was found at' \
                  'directory: "%s".' % domain_path
                  
            v = Vuln('Insecure DAV configuration', msg, severity.MEDIUM,
                     res.id, self.get_name())

            v.set_url(res.get_url())
            v.set_method('SEARCH')
            
            self.kb_append(self, 'dav', v)

    def _is_trusted_cert(self):
        plugin = 'certinfo'

        if plugin not in self._plugin_xml_result:
            return

        is_affected = False
        trust_store = {}

        certificate_validation = self._plugin_xml_result[plugin].find("certificateValidation")

        for path_validation in certificate_validation.findall("pathValidation"):
            name = path_validation.get('usingTrustStore', None)
            version = path_validation.get('trustStoreVersion', None)
            result = path_validation.get('validationResult', None)
            if name:
                trust_store[name] = {}
            if version:
                trust_store[name]['version'] = version
            if result:
                trust_store[name]['result'] = result
                is_affected = True if result == 'self signed certificate' else False

        if is_affected:
            desc = 'Host uses self signed certificate.'
            v = Vuln("Invalid SSL certificate", desc, severity.HIGH, self._response_id, self._plugin_name)
            v.set_url(self._target_url)
            self.kb_append(self, 'wg_invalid_ssl', v)

    def grep(self, request, response):
        """
        Plugin entry point, search for directory indexing.
        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None
        """
        if not response.is_text_or_html():
            return
        
        if response.get_url().get_domain_path() in self._already_visited:
            return

        self._already_visited.add(response.get_url().get_domain_path())
        
        html_string = response.get_body()

        for _ in self._multi_in.query(html_string):
            
            desc = 'The URL: "%s" has a directory indexing vulnerability.'
            desc = desc % response.get_url()
            
            v = Vuln('Directory indexing', desc, severity.LOW, response.id,
                     self.get_name())
            v.set_url(response.get_url())

            self.kb_append_uniq(self, 'directory', v, 'URL')
            break

    def _analyze_headers(self, request, response):
        """
        Search for IP addresses in HTTP headers
        """
        # Get the headers string
        headers_string = response.dump_headers()

        #   Match the regular expressions
        for regex in self._regex_list:
            for match in regex.findall(headers_string):

                # If i'm requesting 192.168.2.111 then I don't want to be
                # alerted about it
                if match not in self._ignore_if_match:
                    desc = 'The URL: "%s" returned an HTTP header with a'\
                           ' private IP address: "%s".'
                    desc = desc % (response.get_url(), match)
                    v = Vuln('Private IP disclosure vulnerability', desc,
                             severity.LOW, response.id, self.get_name())

                    v.set_url(response.get_url())

                    v['IP'] = match
                    v.add_to_highlight(match)
                    self.kb_append(self, 'header', v)

    def _http_only(self, request, response, cookie_obj,
                   cookie_header_value, fingerprinted):
        """
        Verify if the cookie has the httpOnly parameter set

        Reference:
            http://www.owasp.org/index.php/HTTPOnly
            http://en.wikipedia.org/wiki/HTTP_cookie

        :param request: The http request object
        :param response: The http response object
        :param cookie_obj: The cookie object to analyze
        :param cookie_header_value: The cookie, as sent in the HTTP response
        :param fingerprinted: True if the cookie was fingerprinted
        :return: None
        """
        if not self.HTTPONLY_RE.search(cookie_header_value):
            
            vuln_severity = severity.MEDIUM if fingerprinted else severity.LOW
            desc = 'A cookie without the HttpOnly flag was sent when ' \
                   ' requesting "%s". The HttpOnly flag prevents potential' \
                   ' intruders from accessing the cookie value through' \
                   ' Cross-Site Scripting attacks.'
            desc = desc % response.get_url()
            
            v = Vuln('Cookie without HttpOnly', desc,
                     vuln_severity, response.id, self.get_name())
            v.set_url(response.get_url())
            
            self._set_cookie_to_rep(v, cobj=cookie_obj)

            kb.kb.append(self, 'security', v)

    def _ssl_cookie_via_http(self, request, response):
        """
        Analyze if a cookie value, sent in a HTTPS request, is now used for
        identifying the user in an insecure page. Example:
            Login is done over SSL
            The rest of the page is HTTP
        """
        if request.get_url().get_protocol().lower() == 'https':
            return
        
        for cookie in kb.kb.get('analyze_cookies', 'cookies'):
            if cookie.get_url().get_protocol().lower() == 'https' and \
            request.get_url().get_domain() == cookie.get_url().get_domain():

                # The cookie was sent using SSL, I'll check if the current
                # request, is using these values in the POSTDATA / QS / COOKIE
                for key in cookie['cookie-object'].keys():

                    value = cookie['cookie-object'][key].value

                    # This if is to create less false positives
                    if len(value) > 6 and value in request.dump():

                        desc = 'Cookie values that were set over HTTPS, are' \
                               ' then sent over an insecure channel in a' \
                               ' request to "%s".'
                        desc = desc % request.get_url()
                    
                        v = Vuln('Secure cookies over insecure channel', desc,
                                 severity.HIGH, response.id, self.get_name())

                        v.set_url(response.get_url())

                        self._set_cookie_to_rep(v, cobj=cookie['cookie-object'])
                        kb.kb.append(self, 'security', v)

    def _universal_allow(self, forged_req, url, origin, response,
                         allow_origin, allow_credentials, allow_methods):
        """
        Check if the allow_origin is set to *.

        :return: A list of vulnerability objects with the identified vulns
                 (if any).
        """
        if allow_origin == '*':
            msg = 'The remote Web application, specifically "%s", returned' \
                  ' an %s header with the value set to "*" which is insecure'\
                  ' and leaves the application open to Cross-domain attacks.'
            msg = msg % (forged_req.get_url(), ACCESS_CONTROL_ALLOW_ORIGIN)
            
            v = Vuln('Access-Control-Allow-Origin set to "*"', msg,
                     severity.LOW, response.get_id(), self.get_name())

            v.set_url(forged_req.get_url())

            self.kb_append(self, 'cors_origin', v)

            return self._filter_report('_universal_allow_counter',
                                       'universal allow-origin',
                                       severity.MEDIUM, [v, ])

        return []

    def _not_secure_over_https(self, request, response, cookie_obj,
                               cookie_header_value):
        """
        Checks if a cookie that does NOT have a secure flag is sent over https.

        :param request: The http request object
        :param response: The http response object
        :param cookie_obj: The cookie object to analyze
        :param cookie_header_value: The cookie, as sent in the HTTP response
        :return: None
        """
        # BUGBUG: See other reference in this file for http://bugs.python.org/issue1028088

        if response.get_url().get_protocol().lower() == 'https' and \
        not self.SECURE_RE.search(cookie_header_value):

            desc = 'A cookie without the secure flag was sent in an HTTPS' \
                   ' response at "%s". The secure flag prevents the browser' \
                   ' from sending a "secure" cookie over an insecure HTTP' \
                   ' channel, thus preventing potential session hijacking' \
                   ' attacks.'
            desc = desc % response.get_url()
            
            v = Vuln('Secure flag missing in HTTPS cookie', desc,
                     severity.HIGH, response.id, self.get_name())

            v.set_url(response.get_url())
            self._set_cookie_to_rep(v, cobj=cookie_obj)
            
            kb.kb.append(self, 'security', v)

    def _lowest_privilege_test(self, response):
        regex_str = 'User/Group </td><td class="v">(.*?)\((\d.*?)\)/(\d.*?)</td>'
        lowest_privilege_test = re.search(regex_str, response.get_body(), re.I)
        if lowest_privilege_test:
            lpt_uname = lowest_privilege_test.group(1)
            lpt_uid = lowest_privilege_test.group(2)
            lpt_uid = int(lpt_uid)
            lpt_gid = lowest_privilege_test.group(3)
            if lpt_uid < 99 or lpt_gid < 99 or \
            re.match('root|apache|daemon|bin|operator|adm', lpt_uname, re.I):

                desc = 'phpinfo()::PHP may be executing as a higher privileged'\
                       ' group. Username: %s, UserID: %s, GroupID: %s.' 
                desc = desc % (lpt_uname, lpt_uid, lpt_gid)
                
                v = Vuln('PHP lowest_privilege_test:fail', desc,
                         severity.MEDIUM, response.id, self.get_name())
                v.set_url(response.get_url())

                kb.kb.append(self, 'phpinfo', v)
                om.out.vulnerability(v.get_desc(), severity=v.get_severity())
            else:
                lpt_name = 'privilege:' + lpt_uname
                lpt_desc = 'phpinfo()::PHP is executing under '
                lpt_desc += 'username: ' + lpt_uname + ', '
                lpt_desc += 'userID: ' + str(lpt_uid) + ', '
                lpt_desc += 'groupID: ' + lpt_gid
                i = Info(lpt_name, lpt_desc, response.id, self.get_name())
                i.set_url(response.get_url())

                kb.kb.append(self, 'phpinfo', i)
                om.out.information(i.get_desc())

    def _check_if_exists(self, web_shell_url):
        """
        Check if the file exists.

        :param web_shell_url: The URL to check
        """
        try:
            response = self._uri_opener.GET(web_shell_url, cache=True)
        except BaseFrameworkException:
            om.out.debug('Failed to GET webshell:' + web_shell_url)
        else:
            signature = self._match_signature(response)
            if signature is None:
                return

            desc = (u'An HTTP response matching the web backdoor signature'
                    u' "%s" was found at: "%s"; this could indicate that the'
                    u' server has been compromised.')
            desc %= (signature, response.get_url())

            # It's probability is higher if we found a long signature
            _severity = severity.HIGH if len(signature) > 8 else severity.MEDIUM

            v = Vuln(u'Potential web backdoor', desc, _severity,
                     response.id, self.get_name())
            v.set_url(response.get_url())

            kb.kb.append(self, 'backdoors', v)
            om.out.vulnerability(v.get_desc(), severity=v.get_severity())

            fr = FuzzableRequest.from_http_response(response)
            self.output_queue.put(fr)

    def _PROPFIND(self, domain_path):
        """
        Test PROPFIND method
        """
        content = "<?xml version='1.0'?>\r\n"
        content += "<a:propfind xmlns:a='DAV:'>\r\n"
        content += "<a:prop>\r\n"
        content += "<a:displayname:/>\r\n"
        content += "</a:prop>\r\n"
        content += "</a:propfind>\r\n"

        headers = copy.deepcopy(self.CONTENT_TYPE)
        headers['Depth'] = '1'

        res = self._uri_opener.PROPFIND(domain_path, data=content,
                                        headers=headers)

        if "D:href" in res and res.get_code() in xrange(200, 300):
            msg = 'Directory listing with HTTP PROPFIND method was found at' \
                  ' directory: "%s".' % domain_path

            v = Vuln('Insecure DAV configuration', msg, severity.MEDIUM,
                     res.id, self.get_name())

            v.set_url(res.get_url())
            v.set_method('PROPFIND')

            self.kb_append(self, 'dav', v)

    def grep(self, request, response):
        """
        Plugin entry point, find the SSN numbers.

        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None.
        """
        if not response.is_text_or_html() or response.get_code() != 200 \
        or response.get_clear_text_body() is None:
            return

        found_ssn, validated_ssn = self._find_SSN(response.get_clear_text_body())
        
        if validated_ssn:
            uri = response.get_uri()
            desc = 'The URL: "%s" possibly discloses a US Social Security'\
                   ' Number: "%s".'
            desc = desc % (uri, validated_ssn)
            v = Vuln('US Social Security Number disclosure', desc,
                     severity.LOW, response.id, self.get_name())
            v.set_uri(uri)

            v.add_to_highlight(found_ssn)
            self.kb_append_uniq(self, 'ssn', v, 'URL')

    def _check_if_exists(self, web_shell_url):
        """
        Check if the file exists.

        :param web_shell_url: The URL to check
        """
        try:
            response = self._uri_opener.GET(web_shell_url, cache=True)
        except BaseFrameworkException:
            om.out.debug('Failed to GET webshell:' + web_shell_url)
        else:
            if self._is_possible_backdoor(response):
                desc = 'A web backdoor was found at: "%s"; this could ' \
                       'indicate that the server has been compromised.'
                desc = desc % response.get_url()

                v = Vuln('Potential web backdoor', desc, severity.HIGH,
                         response.id, self.get_name())
                v.set_url(response.get_url())

                kb.kb.append(self, 'backdoors', v)
                om.out.vulnerability(v.get_desc(), severity=v.get_severity())

                fr = FuzzableRequest.from_http_response(response)
                self.output_queue.put(fr)

    def end(self):
        """
        This method is called when the plugin wont be used anymore.

        The real job of this plugin is done here, where I will try to see if
        one of the error_500 responses were not identified as a vuln by some
        of my audit plugins
        """
        all_vuln_ids = set()

        for info in kb.kb.get_all_findings():
            for _id in info.get_id():
                all_vuln_ids.add(_id)

        for request, error_500_response_id in self._error_500_responses:

            if error_500_response_id not in all_vuln_ids:
                # Found a error 500 that wasn't identified !
                desc = 'An unidentified web application error (HTTP response'\
                       ' code 500) was found at: "%s". Enable all plugins and'\
                       ' try again, if the vulnerability still is not'\
                       ' identified, please verify manually and report it to'\
                       ' the w3af developers.'
                desc = desc % request.get_url()

                v = Vuln('Unhandled error in web application', desc,
                         severity.MEDIUM, error_500_response_id,
                         self.get_name())

                v.set_uri(request.get_uri())

                self.kb_append_uniq(self, 'error_500', v, 'VAR')

        self._error_500_responses.cleanup()

    def _send_and_check(self, repo_url, repo_get_files, repo, domain_path):
        """
        Check if a repository index exists in the domain_path.

        :return: None, everything is saved to the self.out_queue.
        """
        http_response = self.http_get_and_parse(repo_url)

        if not is_404(http_response):

            filenames = repo_get_files(http_response.get_body())

            parsed_url_set = set()

            for filename in self._clean_filenames(filenames):
                test_url = domain_path.url_join(filename)
                if test_url not in self._analyzed_filenames:
                    parsed_url_set.add(test_url)
                    self._analyzed_filenames.add(filename)

            self.worker_pool.map(self.http_get_and_parse, parsed_url_set)

            if parsed_url_set:
                desc = 'A %s was found at: "%s"; this could indicate that'\
                       ' a %s is accessible. You might be able to download'\
                       ' the Web application source code.'
                desc = desc % (repo, http_response.get_url(), repo)
                
                v = Vuln('Source code repository', desc, severity.MEDIUM,
                         http_response.id, self.get_name())
                v.set_url(http_response.get_url())
                
                kb.kb.append(self, repo, v)
                om.out.vulnerability(v.get_desc(), severity=v.get_severity())

    def _analyze_html(self, request, response):
        """
        Search for IP addresses in the HTML
        """
        if not response.is_text_or_html():
            return

        # Performance improvement!
        if not (('10.' in response) or ('172.' in response) or
               ('192.168.' in response) or ('169.254.' in response)):
            return

        for regex in self._regex_list:
            for match in regex.findall(response.get_body()):
                match = match.strip()

                # Some proxy servers will return errors that include headers in the body
                # along with the client IP which we want to ignore
                if re.search("^.*X-Forwarded-For: .*%s" % match, response.get_body(), re.M):
                    continue

                # If i'm requesting 192.168.2.111 then I don't want to be alerted about it
                if match not in self._ignore_if_match and \
                not request.sent(match):
                    desc = 'The URL: "%s" returned an HTML document'\
                           ' with a private IP address: "%s".'
                    desc = desc % (response.get_url(), match)
                    v = Vuln('Private IP disclosure vulnerability', desc,
                             severity.LOW, response.id, self.get_name())

                    v.set_url(response.get_url())

                    v['IP'] = match
                    v.add_to_highlight(match)
                    self.kb_append(self, 'HTML', v)

    def _check_methods(self, url):
        """
        Perform some requests in order to check if we are able to retrieve
        some data with methods that may be wrongly enabled.
        """
        allowed_methods = []
        for method in ['GET', 'POST', 'ABCD', 'HEAD']:
            method_functor = getattr(self._uri_opener, method)
            try:
                response = apply(method_functor, (url,), {})
                code = response.get_code()
            except:
                pass
            else:
                if code in self.SUCCESS_CODES:
                    allowed_methods.append((method, response.id))

        if len(allowed_methods) > 0:
            
            response_ids = [i for m, i in allowed_methods]
            methods = ', '.join([m for m, i in allowed_methods]) + '.'
            desc = 'The resource: "%s" requires authentication but the access'\
                   ' is misconfigured and can be bypassed using these'\
                   ' methods: %s'
            desc = desc % (url, methods)
            
            v = Vuln('Misconfigured access control', desc,
                     severity.MEDIUM, response_ids, self.get_name())

            v.set_url(url)
            v['methods'] = allowed_methods
            
            self.kb_append(self, 'auth', v)

    def crawl(self, fuzzable_request):
        """
        Plugin entry point, perform all the work.
        """
        to_check = self._get_to_check(fuzzable_request.get_url())

        # I found some URLs, create fuzzable requests
        phishtank_matches = self._is_in_phishtank(to_check)
        for ptm in phishtank_matches:
            response = self._uri_opener.GET(ptm.url)
            for fr in self._create_fuzzable_requests(response):
                self.output_queue.put(fr)

        # Only create the vuln object once
        if phishtank_matches:
            desc = 'The URL: "%s" seems to be involved in a phishing scam.' \
                   ' Please see %s for more info.'
            desc = desc % (ptm.url, ptm.more_info_URL)
            
            v = Vuln('Phishing scam', desc, severity.MEDIUM, response.id,
                     self.get_name())
            v.set_url(ptm.url)
            
            kb.kb.append(self, 'phishtank', v)
            om.out.vulnerability(v.get_desc(), severity=v.get_severity())

    def _analyze_SQL(self, request, response, ref, token_name, token_value):
        """
        To find this kind of vulns

        http://thedailywtf.com/Articles/Oklahoma-
            Leaks-Tens-of-Thousands-of-Social-Security-Numbers,-Other-
            Sensitive-Data.aspx

        :return: True if the parameter value contains SQL sentences
        """
        for match in SQL_RE.findall(token_value):
            if request.sent(match):
                continue

            desc = ('The URI: "%s" has a parameter named: "%s" with value:'
                    ' "%s", which is a SQL query.')
            desc %= (response.get_uri(), token_name, token_value)

            v = Vuln('Parameter has SQL sentence', desc, severity.LOW,
                     response.id, self.get_name())
            v['parameter_value'] = token_value
            v.add_to_highlight(token_value)
            v.set_uri(ref)

            self.kb_append(self, 'strange_parameters', v)
            return True

        return False