def filtered_freq_generator(freq_list):
            already_tested = ScalableBloomFilter()

            for freq in freq_list:
                if freq not in already_tested:
                    already_tested.add(freq)
                    yield freq

    def __init__(self):
        CrawlPlugin.__init__(self)

        # Internal variables
        self._already_crawled = ScalableBloomFilter()
        self._already_verified = ScalableBloomFilter()

        # User configured parameters
        self._max_depth = 3

    def test_bloom_filter(self):
        f = ScalableBloomFilter()

        for i in xrange(20000):
            data = (i, i)
            f.add(data)

        for i in xrange(20000):
            data = (i, i)
            data in f

    def __init__(self):
        CrawlPlugin.__init__(self)

        # User configured parameters
        self._wordlist = os.path.join(ROOT_PATH, "plugins", "crawl", "content_negotiation", "common_filenames.db")

        # Internal variables
        self._already_tested_dir = ScalableBloomFilter()
        self._already_tested_resource = ScalableBloomFilter()
        self._content_negotiation_enabled = None
        self._to_bruteforce = Queue.Queue()
        # I want to try 3 times to see if the remote host is vulnerable
        # detection is not thaaaat accurate!
        self._tries_left = 3

    def __init__(self):
        CrawlPlugin.__init__(self)

        # Internal variables
        self._analyzed_dirs = ScalableBloomFilter()
        self._analyzed_filenames = ScalableBloomFilter()

        self._dvcs = {}
        self._dvcs['git repository'] = {}
        self._dvcs['git ignore'] = {}
        self._dvcs['hg repository'] = {}
        self._dvcs['hg ignore'] = {}
        self._dvcs['bzr repository'] = {}
        self._dvcs['bzr ignore'] = {}
        self._dvcs['svn repository'] = {}
        self._dvcs['svn ignore'] = {}
        self._dvcs['cvs repository'] = {}
        self._dvcs['cvs ignore'] = {}

        self._dvcs['git repository']['filename'] = '.git/index'
        self._dvcs['git repository']['function'] = self.git_index

        self._dvcs['git ignore']['filename'] = '.gitignore'
        self._dvcs['git ignore']['function'] = self.ignore_file

        self._dvcs['hg repository']['filename'] = '.hg/dirstate'
        self._dvcs['hg repository']['function'] = self.hg_dirstate

        self._dvcs['hg ignore']['filename'] = '.hgignore'
        self._dvcs['hg ignore']['function'] = self.ignore_file

        self._dvcs['bzr repository']['filename'] = '.bzr/checkout/dirstate'
        self._dvcs['bzr repository']['function'] = self.bzr_checkout_dirstate

        self._dvcs['bzr ignore']['filename'] = '.bzrignore'
        self._dvcs['bzr ignore']['function'] = self.ignore_file

        self._dvcs['svn repository']['filename'] = '.svn/entries'
        self._dvcs['svn repository']['function'] = self.svn_entries

        self._dvcs['svn ignore']['filename'] = '.svnignore'
        self._dvcs['svn ignore']['function'] = self.ignore_file

        self._dvcs['cvs repository']['filename'] = 'CVS/Entries'
        self._dvcs['cvs repository']['function'] = self.cvs_entries

        self._dvcs['cvs ignore']['filename'] = '.cvsignore'
        self._dvcs['cvs ignore']['function'] = self.ignore_file

    def __init__(self):
        CrawlPlugin.__init__(self)
        self._already_visited = ScalableBloomFilter()

        # User options
        self._fuzz_images = False
        self._max_digit_sections = 4

    def __init__(self):
        InfrastructurePlugin.__init__(self)

        # Internal variables
        self._first_exec = True
        self._already_queried = ScalableBloomFilter()
        self._can_resolve_domain_names = False

    def __init__(self, grep_plugins, w3af_core):
        """
        :param grep_plugins: Instances of grep plugins in a list
        :param w3af_core: The w3af core that we'll use for status reporting
        """
        # max_in_queue_size, is the number of items that will be stored in-memory
        # in the consumer queue
        #
        # Any items exceeding max_in_queue_size will be stored on-disk, which
        # is slow but will prevent any high memory usage imposed by this part
        # of the framework
        max_in_queue_size = 20

        # thread_pool_size defines how many threads we'll use to run grep plugins
        thread_pool_size = 2

        # max_pool_queued_tasks defines how many tasks we'll keep in memory waiting
        # for a worker from the pool to be available
        max_pool_queued_tasks = thread_pool_size * 3

        super(grep, self).__init__(grep_plugins,
                                   w3af_core,
                                   create_pool=False,
                                   #max_pool_queued_tasks=max_pool_queued_tasks,
                                   #thread_pool_size=thread_pool_size,
                                   thread_name='Grep',
                                   max_in_queue_size=max_in_queue_size)
        self._already_analyzed = ScalableBloomFilter()

    def __init__(self):
        CrawlPlugin.__init__(self)

        # internal variables
        self._exec = True
        self._already_analyzed = ScalableBloomFilter()

        # User configured parameters
        self._db_file = os.path.join(ROOT_PATH, 'plugins', 'crawl', 'pykto',
                                     'scan_database.db')
        self._extra_db_file = os.path.join(ROOT_PATH, 'plugins', 'crawl',
                                           'pykto', 'w3af_scan_database.db')

        self._cgi_dirs = ['/cgi-bin/']
        self._admin_dirs = ['/admin/', '/adm/']

        self._users = ['adm', 'bin', 'daemon', 'ftp', 'guest', 'listen', 'lp',
                       'mysql', 'noaccess', 'nobody', 'nobody4', 'nuucp',
                       'operator', 'root', 'smmsp', 'smtp', 'sshd', 'sys',
                       'test', 'unknown']

        self._nuke = ['/', '/postnuke/', '/postnuke/html/', '/modules/',
                      '/phpBB/', '/forum/']

        self._mutate_tests = False

    def __init__(self):
        GrepPlugin.__init__(self)

        # Internal variables
        self._comments = DiskDict(table_prefix='html_comments')
        self._already_reported = ScalableBloomFilter()
        self._end_was_called = False

    def __init__(self):
        GrepPlugin.__init__(self)

        self._already_reported = ScalableBloomFilter()

        # regex to split between words
        self._split_re = re.compile('[^\w]')

    def __init__(self):
        CrawlPlugin.__init__(self)

        self._headers = None
        self._first_time = True
        self._fuzz_images = False
        self._seen = ScalableBloomFilter()

class frontpage_version(InfrastructurePlugin):
    """
    Search FrontPage Server Info file and if it finds it will determine its version.
    :author: Viktor Gazdag ( [email protected] )
    """
    VERSION_RE = re.compile('FPVersion="(.*?)"', re.IGNORECASE)
    ADMIN_URL_RE = re.compile('FPAdminScriptUrl="(.*?)"', re.IGNORECASE)
    AUTHOR_URL_RE = re.compile('FPAuthorScriptUrl="(.*?)"', re.IGNORECASE)

    def __init__(self):
        InfrastructurePlugin.__init__(self)

        # Internal variables
        self._analyzed_dirs = ScalableBloomFilter()

    @runonce(exc_class=RunOnce)
    def discover(self, fuzzable_request):
        """
        For every directory, fetch a list of files and analyze the response.

        :param fuzzable_request: A fuzzable_request instance that contains
                                    (among other things) the URL to test.
        """
        for domain_path in fuzzable_request.get_url().get_directories():

            if domain_path in self._analyzed_dirs:
                continue

            # Save the domain_path so I know I'm not working in vane
            self._analyzed_dirs.add(domain_path)

            # Request the file
            frontpage_info_url = domain_path.url_join("_vti_inf.html")
            try:
                response = self._uri_opener.GET(frontpage_info_url,
                                                cache=True)
            except BaseFrameworkException, w3:
                fmt = 'Failed to GET Frontpage Server _vti_inf.html file: "%s"'\
                      '. Exception: "%s".'
                om.out.debug(fmt % (frontpage_info_url, w3))
            else:
                # Check if it's a Frontpage Info file
                if not is_404(response):
                    fr = FuzzableRequest(response.get_uri())
                    self.output_queue.put(fr)

                    self._analyze_response(response)

    def __init__(self):
        InfrastructurePlugin.__init__(self)

        # Internal variables
        self._already_tested = ScalableBloomFilter()
        # On real web applications, if we can't trigger an error in the first
        # MAX_TESTS tests, it simply won't happen and we have to stop testing.
        self.MAX_TESTS = 25

    def __init__(self):
        CrawlPlugin.__init__(self)

        # Internal variables
        self._analyzed_dirs = ScalableBloomFilter()

        # -rw-r--r--    1 andresr   w3af         8139 Apr 12 13:23 foo.zip
        regex_str = '[a-z-]{10}\s*\d+\s*(.*?)\s+(.*?)\s+\d+\s+\w+\s+\d+\s+[0-9:]{4,5}\s+(.*)'
        self._listing_parser_re = re.compile(regex_str)

class blank_body(GrepPlugin):
    """
    Find responses with empty body.

    :author: Andres Riancho ([email protected])
    """

    METHODS = ('GET', 'POST')
    HTTP_CODES = (401, 304, 302, 301, 204, 405)
    
    def __init__(self):
        GrepPlugin.__init__(self)
        self.already_reported = ScalableBloomFilter()

    def grep(self, request, response):
        """
        Plugin entry point, find the blank bodies and report them.

        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None
        """
        if response.get_body() == '' and request.get_method() in self.METHODS\
        and response.get_code() not in self.HTTP_CODES\
        and not response.get_headers().icontains('location')\
        and response.get_url().uri2url() not in self.already_reported:

            self.already_reported.add(response.get_url().uri2url())

            desc = 'The URL: "%s" returned an empty body, this could indicate'\
                   ' an application error.'
            desc = desc % response.get_url()

            i = Info('Blank http response body', desc, response.id,
                     self.get_name())
            i.set_url(response.get_url())
            
            self.kb_append(self, 'blank_body', i)

    def get_long_desc(self):
        """
        :return: A DETAILED description of the plugin functions and features.
        """
        return """

    def __init__(self):
        #
        #   Set the opener, I need it to perform some tests and gain
        #   the knowledge about the server's 404 response bodies.
        #
        self._uri_opener = None
        self._worker_pool = None
        
        #
        #   Internal variables
        #
        self._already_analyzed = False
        self._404_bodies = []
        self._lock = thread.allocate_lock()
        self._fingerprinted_paths = ScalableBloomFilter()
        self._directory_uses_404_codes = ScalableBloomFilter()

        # It is OK to store 200 here, I'm only storing path+filename as the key,
        # and bool as the value.
        self.is_404_LRU = LRU(200)

    def __init__(self):
        self._variants = CachedDiskDict(max_in_memory=self.MAX_IN_MEMORY,
                                        table_prefix='variant_db')
        self._variants_eq = ScalableBloomFilter()
        self._variants_form = CachedDiskDict(max_in_memory=self.MAX_IN_MEMORY,
                                             table_prefix='variant_db_form')

        self.params_max_variants = cf.cf.get('params_max_variants')
        self.path_max_variants = cf.cf.get('path_max_variants')
        self.max_equal_form_variants = cf.cf.get('max_equal_form_variants')

        self._db_lock = threading.RLock()

    def __init__(self):
        InfrastructurePlugin.__init__(self)

        # Already analyzed extensions
        self._already_analyzed_ext = ScalableBloomFilter()

        # Internal DB
        self._db_file = os.path.join(ROOT_PATH, 'plugins', 'infrastructure',
                                     'php_eggs', 'eggs.json')

        # Get data from external JSON file and fill EGG_DB array
        data = self.read_jsondata(self._db_file)
        self.EGG_DB = self.fill_egg_array(data)

    def __init__(self):
        GrepPlugin.__init__(self)

        # For more info regarding this regular expression, please see:
        # https://sourceforge.net/mailarchive/forum.php?thread_name=1955593874.20090122023644%40
        #mlists.olympos.org&forum_name=w3af-develop
        regex_str = '(?<!\.)(?<!\d)(?:(?:10|127)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|192\.168|169\.'
        regex_str += '254|172\.0?(?:1[6-9]|2[0-9]|3[01]))(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-'
        regex_str += '9]?)){2}(?!\d)(?!\.)'
        self._private_ip_address = re.compile(regex_str)
        self._regex_list = [self._private_ip_address, ]

        self._already_inspected = ScalableBloomFilter()
        self._ignore_if_match = None