def __init__(self,regular_expressions):
        """ Initialize the object 

            regular_expressions: an interative set of regular expressions to
                be applied for extracting mutations. These are in the 
                default python syntax (i.e., perl regular expressions), with 
                the single exception being that regular expressions which
                should be performed in a case sensitive manner should be 
                followed by the string '[CASE_SENSITIVE]', with no spaces 
                between it and the regular expression. 
                This can be a list, a file, or any other object which supports
                iteration. For an example, you should refer to the regex.txt
                file in the MutationFinder directory.

        """
        MutationExtractor.__init__(self)
        self._regular_expressions = []

        for regular_expression in regular_expressions:
            if regular_expression.endswith('[CASE_SENSITIVE]'):
                self._regular_expressions.append(\
                 compile(regular_expression[:regular_expression.rindex('[')]))
            else:
                self._regular_expressions.append(\
                 compile(regular_expression,IGNORECASE))

    def run(self):
        if self.results["target"]["category"] == "file":
            return False
 
        self.ie_paths_re = re.compile(r"^c:\\program files(?:\s\(x86\))?\\internet explorer\\iexplore.exe$",re.I)
        #run through re.escape()
        self.white_list_re = ["^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Adobe\\\\Reader\\ \\d+\\.\\d+\\\\Reader\\\\AcroRd32\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Java\\\\jre\\d+\\\\bin\\\\j(?:avaw?|p2launcher)\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Microsoft SilverLight\\\\(?:\\d+\\.)+\\d\\\\agcp.exe$",
                         "^C\\:\\\\Windows\\\\System32\\\\ntvdm\\.exe$",
                         "^C\\:\\\\Windows\\\\system32\\\\rundll32\\.exe$",
                         "^C\\:\\\\Windows\\\\syswow64\\\\rundll32\\.exe$",
                         "^C\\:\\\\Windows\\\\system32\\\\drwtsn32\\.exe$",
                         "^C\\:\\\\Windows\\\\syswow64\\\\drwtsn32\\.exe$",
                         "^C\\:\\\\Windows\\\\system32\\\\dwwin\\.exe$"
                        ]
        #means we can be evaded but also means we can have relatively tight paths between 32-bit and 64-bit
        self.white_list_re_compiled = []
        for entry in self.white_list_re:
            self.white_list_re_compiled.append(re.compile(entry,re.I))
        self.white_list_re_compiled.append(self.ie_paths_re)

        # Sometimes if we get a service loaded we get out of order processes in tree need iterate over IE processes get the path of the initial monitored executable
        self.initialpath = None
        processes = self.results["behavior"]["processtree"]
        if len(processes):
            for p in processes:
                initialpath = p["module_path"].lower()
                if initialpath and self.ie_paths_re.match(initialpath) and p.has_key("children"):
                    self.martians = self.find_martians(p,self.white_list_re_compiled)
                    if len(self.martians) > 0:
                        for martian in self.martians:
                            self.data.append({"ie_martian": martian})
                        return True 
        return False

    def substitute(self, search, replace, flags, text, inputMessage, channel):
        # Apparently re.sub understands escape sequences in the replacement string;
        #  strip all but the backreferences
        replace = replace.replace('\\', '\\\\')
        replace = re.sub(r'\\([1-9][0-9]?([^0-9]|$))', r'\1', replace)

        if channel not in self.messages:
            self.messages[channel] = []
            self.unmodifiedMessages[channel] = []

        messages = self.unmodifiedMessages[channel] if 'o' in flags else self.messages[channel]

        if 'g' in flags:
            count = 0
        else:
            count = 1

        subFlags = 0
        if 'i' in flags:
            subFlags |= re.IGNORECASE
        if 'v' in flags:
            subFlags |= re.VERBOSE

        if 'c' in flags:
            newMessage = copy.copy(inputMessage)

            try:
                searchC = re2.compile(search, subFlags)
                new = searchC.sub(replace, text, count)
            except sre_constants.error as e:
                newMessage.messageString = "[Regex Error in Sed pattern: {}]".format(e.message)
                return newMessage

            if new != text:
                newMessage.messageString = new
                self.storeMessage(newMessage, False)
            else:
                newMessage.messageString = text
                self.storeMessage(newMessage, False)

            return newMessage

        for message in reversed(messages):
            try:
                searchC = re2.compile(search, subFlags)
                new = searchC.sub(replace, message.messageString, count)
            except sre_constants.error as e:
                newMessage = copy.copy(inputMessage)
                newMessage.messageString = "[Regex Error in Sed pattern: {}]".format(e.message)
                return newMessage

            new = new[:300]

            if searchC.search(message.messageString):
                newMessage = copy.copy(message)
                newMessage.messageString = new
                self.storeMessage(newMessage, False)
                return newMessage

        return None

    def _search(self, searchTerms, logPath, files, searchForNick, includeToday, reverse):
        if searchForNick:
            pattern = re2.compile(r"^\[[^]]+\]\s+<(.?{})>\s+.*".format(searchTerms), re.IGNORECASE)
        else:
            pattern = re2.compile(r".*<.*> .*({}).*".format(searchTerms), re.IGNORECASE)
        found = None

        if not includeToday:
            today = "{}.log".format(strftime("%Y-%m-%d"))
            if today in files:
                files.remove(today)

        if reverse:
            files.reverse()
        for filename in files:
            with open(os.path.join(logPath, filename), "r") as logfile:
                if reverse:
                    lines = reversed(logfile.readlines())
                else:
                    lines = logfile.readlines()
            if reverse and includeToday:
                lines = list(lines)[1:]
            for line in lines:
                if pattern.match(line.rstrip()):
                    found = line.rstrip()
                    break
            if found:
                return "[{}] {}".format(filename[:10], found)
        return "Nothing that matches your search terms has been found in the log."

    def test_re_match(self):
        self.assertEqual(re.match('a', 'a').groups(), ())
        self.assertEqual(re.match('(a)', 'a').groups(), ('a',))
        self.assertEqual(re.match(r'(a)', 'a').group(0), 'a')
        self.assertEqual(re.match(r'(a)', 'a').group(1), 'a')
        self.assertEqual(re.match(r'(a)', 'a').group(1, 1), ('a', 'a'))

        pat = re.compile('((a)|(b))(c)?')
        self.assertEqual(pat.match('a').groups(), ('a', 'a', None, None))
        self.assertEqual(pat.match('b').groups(), ('b', None, 'b', None))
        self.assertEqual(pat.match('ac').groups(), ('a', 'a', None, 'c'))
        self.assertEqual(pat.match('bc').groups(), ('b', None, 'b', 'c'))
        self.assertEqual(pat.match('bc').groups(""), ('b', "", 'b', 'c'))

        # A single group
        m = re.match('(a)', 'a')
        self.assertEqual(m.group(0), 'a')
        self.assertEqual(m.group(0), 'a')
        self.assertEqual(m.group(1), 'a')
        self.assertEqual(m.group(1, 1), ('a', 'a'))

        pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
        self.assertEqual(pat.match('a').group(1, 2, 3), ('a', None, None))
        self.assertEqual(pat.match('b').group('a1', 'b2', 'c3'),
                         (None, 'b', None))
        self.assertEqual(pat.match('ac').group(1, 'b2', 3), ('a', None, 'c'))

 def test_bug_926075(self):
     try:
         unicode
     except NameError:
         return # no problem if we have no unicode
     self.assert_(re.compile('bug_926075') is not
                  re.compile(eval("u'bug_926075'")))

 def test_empty_array(self):
     # SF buf 1647541
     import array
     for typecode in 'cbBuhHiIlLfd':
         a = array.array(typecode)
         self.assertEqual(re.compile("bla").match(a), None)
         self.assertEqual(re.compile("").match(a).groups(), ())

    def run(self):
        self.ie_paths_re = re.compile(r"^c:\\program files(?:\s\(x86\))?\\internet explorer\\iexplore.exe$",re.I)
        #run through re.escape()
        self.white_list_re = ["^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Adobe\\\\Reader\\ \\d+\\.\\d+\\\\Reader\\\\AcroRd32\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Java\\\\jre\\d+\\\\bin\\\\j(?:avaw?|p2launcher)\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Microsoft SilverLight\\\\(?:\\d+\\.)+\\d\\\\agcp.exe$",
                         "^C\\:\\\\Windows\\\\System32\\\\ntvdm.exe$",
                        ]
        #means we can be evaded but also means we can have relatively tight paths between 32-bit and 64-bit
        self.white_list_re_compiled = []
        for entry in self.white_list_re:
            self.white_list_re_compiled.append(re.compile(entry,re.I))
        self.white_list_re_compiled.append(self.ie_paths_re)

        # get the path of the initial monitored executable
        self.initialpath = None
        processes = self.results["behavior"]["processtree"]
        if len(processes):
            self.initialpath = processes[0]["module_path"].lower()
        if self.initialpath and self.ie_paths_re.match(self.initialpath) and processes[0].has_key("children"):
           self.martians = self.find_martians(processes,self.white_list_re_compiled)
           if len(self.martians) > 0:
               for martian in self.martians:
                   self.data.append({"ie_martian": martian})
               return True 
        return False

    def _search(self, searchTerms, logPath, files, searchForNick, includeToday, reverse):
        if searchForNick:
            pattern = re2.compile(fr"^\[[^]]+\]\s+<(.?{searchTerms})>\s+.*", re2.IGNORECASE)
        else:
            pattern = re2.compile(fr'.*<.*> .*({searchTerms}).*', re2.IGNORECASE)
        found = None

        if not includeToday:
            today = f"{strftime('%Y-%m-%d')}.log"
            if today in files:
                files.remove(today)

        if reverse:
            files.reverse()
        for filename in files:
            with open(os.path.join(logPath, filename), 'r', errors='ignore') as logfile:
                if reverse:
                    lines = reversed(logfile.readlines())
                else:
                    lines = logfile.readlines()
            if reverse and includeToday:
                lines = list(lines)[1:]
            for line in lines:
                if pattern.match(line.rstrip()):
                    found = line.rstrip()
                    break
            if found:
                return f'[{filename[:10]}] {found}'
        return 'Nothing that matches your search terms has been found in the log.'

 def __init__(self, *args, **kwargs):
     Signature.__init__(self, *args, **kwargs)
     # Named group to extract the URL of the cloned website.
     self.rex = {
         "saved from url": re.compile(r"\<!--\ssaved\sfrom\surl=\(\d+\)(?P<url>[^\s]+)", re.I),
         "mirrored from": re.compile(r"<!--\smirrored\sfrom\s(?P<url>[^\s]+)\sby\sHTTrack", re.I),
     }
     self.hits = set()

def test_regex(regex_array):
    """Ensures the regex strings are validated for proper syntax.

    """
    for regex_entry in regex_array:
        try:
            re.compile(regex_entry, re.MULTILINE | re.UNICODE)
        except re.error:
            logging.error('Invalid Regex Found: %s', regex_entry)
            sys.exit(1)

def _compile_regex(regex_string, flags=0):
    try:
        if re2:
            # default max_mem is 8<<20 = 8*1000*1000
            return re.compile(regex_string, max_mem=60*1000*1000, flags=flags)
        else:
            return re.compile(regex_string, flags=flags)
    except:
        logging.exception("Error compiling with flags %s for string: %s", flags, regex_string)
        raise

 def _prepare_pattern(self, pattern):
     """
     Strip out key:value pairs from the pattern and compile the regular expression.
     """
     regex, _, rest = pattern.partition('\\;')
     try:
         return re.compile(regex, re.I)
     except re.error as e:
         warnings.warn("Caught '{error}' compiling regex: {regex}".format(error=e, regex=regex))
         return re.compile(r'(?!x)x') # regex that never matches: http://stackoverflow.com/a/1845097/413622

    def test_dollar_matches_twice(self):
        "$ matches the end of string, and just before the terminating \n"
        pattern = re.compile('$')
        self.assertEqual(pattern.sub('#', 'a\nb\n'), 'a\nb#\n#')
        self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a\nb\nc#')
        self.assertEqual(pattern.sub('#', '\n'), '#\n#')

        pattern = re.compile('$', re.MULTILINE)
        self.assertEqual(pattern.sub('#', 'a\nb\n' ), 'a#\nb#\n#' )
        self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a#\nb#\nc#')
        self.assertEqual(pattern.sub('#', '\n'), '#\n#')

    def run(self):
        if self.results["target"]["category"] == "url":
            return False

        office_pkgs = ["ppt","doc","xls","eml"]
        if not any(e in self.results["info"]["package"] for e in office_pkgs):
            return False

        self.office_paths_re = re.compile(r"^[A-Z]\:\\Program Files(?:\s\(x86\))?\\Microsoft Office\\(?:Office1[1-5]\\)?(?:WINWORD|OUTLOOK|POWERPNT|EXCEL|WORDVIEW)\.EXE$",re.I)
        #run through re.escape()
        #############################################
        #YOU MAY HAVE TO CUSTOMIZE THIS FOR YOUR ENV#
        #############################################
        self.white_list_re = ["C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Adobe\\\\Reader\\ \\d+\\.\\d+\\\\Reader\\\\AcroRd32\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Java\\\\jre\\d+\\\\bin\\\\j(?:avaw?|p2launcher)\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Microsoft SilverLight\\\\(?:\\d+\\.)+\\d\\\\agcp\\.exe$",
                         "C\\:\\\\Windows\\\\System32\\\\ntvdm\\.exe$",
                         "C\\:\\\\Windows\\\\System32\\\\svchost\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\internet explorer\\\\iexplore\.exe$",
                         # remove this one at some point
                         "C\\:\\\\Windows\\\\System32\\\\rundll32\\.exe$",
                         "C\\:\\\\Windows\\\\System32\\\\drwtsn32\\.exe$",
                         "C\\:\\\\Windows\\\\splwow64\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Common Files\\\\Microsoft Shared\\\\office1[1-6]\\\\off(?:lb|diag)\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Common Files\\\\Microsoft Shared\\\\dw\\\\dw(?:20)?\\.exe$",
                         "C\\:\\\\Windows\\\\system32\\\\dwwin\\.exe$",
                         "C\\:\\\\Windows\\\\system32\\\\WerFault\\.exe$",
                         "C\\:\\\\Windows\\\\syswow64\\\\WerFault\\.exe$"
                        ]
        #means we can be evaded but also means we can have relatively tight paths between 32-bit and 64-bit
        self.white_list_re_compiled = []
        for entry in self.white_list_re:
            try:
                self.white_list_re_compiled.append(re.compile(entry,re.I))
            except Exception as e:
                print "failed to compile expression %s error:%s" % (entry,e)
        self.white_list_re_compiled.append(self.office_paths_re)

        # Sometimes if we get a service loaded we get out of order processes in tree need iterate over IE processes get the path of the initial monitored executable
        self.initialpath = None
        processes = self.results["behavior"]["processtree"]
        if len(processes):
            for p in processes:
                initialpath = p["module_path"].lower()
                if initialpath and self.office_paths_re.match(initialpath) and p.has_key("children"):
                    self.martians = self.find_martians(p,self.white_list_re_compiled)
                    if len(self.martians) > 0:
                        for martian in self.martians:
                            self.data.append({"office_martian": martian})
                        return True
        return False

    def run(self):
        # Check zeus synchronization-mutex.
        # Regexp pattern for zeus synchronization-mutex such as for example:
        # 2CCB0BFE-ECAB-89CD-0261-B06D1C10937F
        exp = re.compile(".*[A-Z0-9]{8}-([A-Z0-9]{4}-){3}[A-Z0-9]{12}", re.IGNORECASE)
        mutexes = self.results["behavior"]["summary"]["mutexes"]
        mutexset = set()
        count = 0
        for mutex in mutexes:
            if exp.match(mutex):
                mutexset.add(mutex)
                count += 1 

        # Check if there are at least 5 mutexes opened matching the pattern?   
        if count < 5:
            return False
        
        # Check for UDP Traffic on remote port greater than 1024.
        # TODO: this might be faulty without checking whether the destination
        # IP is really valid.
        count = 0
        if "network" in self.results:
            for udp in self.results["network"]["udp"]:
                if udp["dport"] > 1024:
                    count += 1
            
        if count < 4:
            return False

        for mutex in mutexset:
            self.data.append({"mutex": mutex})

        return True

    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        # We only want web content from the target
        if srcModuleName != "sfp_spider":
            return None

        eventSource = event.sourceEvent.data
        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if eventSource not in self.results.keys():
            self.results[eventSource] = list()

        # We only want web content for pages on the target site
        if not self.getTarget().matches(self.sf.urlFQDN(eventSource)):
            self.sf.debug("Not collecting web content information for external sites.")
            return None

        for regexpGrp in regexps.keys():
            if regexpGrp in self.results[eventSource]:
                continue

            for regex in regexps[regexpGrp]:
                pat = re.compile(regex, re.IGNORECASE)
                matches = re.findall(pat, eventData)
                if len(matches) > 0 and regexpGrp not in self.results[eventSource]:
                    self.sf.info("Matched " + regexpGrp + " in content from " + eventSource)
                    self.results[eventSource].append(regexpGrp)
                    evt = SpiderFootEvent("ERROR_MESSAGE", regexpGrp,
                                          self.__name__, event.sourceEvent)
                    self.notifyListeners(evt)

        return None

def build_smile_re(dsmile):
    out = {}
    for name, lsmile in dsmile.items():
        out[name] = re.compile(r'(?: %s)' % (r'| '.join(lsmile)))
        #print name, r'(?:\s%s)' % (r'|\s'.join(lsmile))

    return out

    def execute(self, message: IRCMessage):
        comicLimit = 8
        params = list(message.parameterList)
        if len(params) > 0 and string.isNumber(params[0]):
            comicLimit = int(params.pop(0))

        messages = self.getMessages(message.replyTo)
        if len(params) > 0:
            regex = re2.compile(" ".join(params), re2.IGNORECASE)
            matches = list(filter(regex.search, [msg[1] for msg in messages]))
            if len(matches) == 0:
                return IRCResponse(ResponseType.Say,
                                   "Sorry, that didn't match anything in my message buffer.",
                                   message.replyTo)
            elif len(matches) > 1:
                return IRCResponse(ResponseType.Say,
                                   "Sorry, that matches too many lines in my message buffer.",
                                   message.replyTo)

            index = [msg[1] for msg in messages].index(matches[0])
            lastIndex = index + comicLimit
            if lastIndex > len(messages):
                lastIndex = len(messages)
            messages = messages[index:lastIndex]
        else:
            messages = messages[comicLimit * -1:]
        if messages:
            comicBytes = self.makeComic(messages)
            return IRCResponse(ResponseType.Say, self.postComic(comicBytes), message.replyTo)
        else:
            return IRCResponse(ResponseType.Say,
                               "There are no messages in the buffer to create a comic with.",
                               message.replyTo)

def score_pattern(pattern, pos,neg):
	"""Calculates a one tailed p-value for a pattern using a Fisher's exact test
	
	:param pattern: regular expression search pattern
	:type pattern: string
	:param pos: list of strings where the outcome was positive
	:type pos: list of strings
	:param neg: list of strings where the outcome was negative
	:type neg: list of strings
	:returns: one tailed p-value
	:rtype: float
	"""
	p=re.compile(pattern)
	pp=0
	pn=0
	for l in pos:
		if p.search(l):
			pp+=1
	for l in neg:
		if p.search(l):
			pn+=1
	s1=getLogPvalue(pp, len(pos), pn, len(neg))
	s2=getLogPvalue(pn, len(neg),pp, len(pos))

	return min(s1,s2)