def remove_readmes(self):
     for path, _, files in os.walk(self.search_dir):
         for filename in files:
             if filename == "README.txt":
                 filepath = os.path.join(path, filename)
                 Logger.info("Deleting the file:", filepath)
                 os.remove(filepath)

 def divide_by_exploitability(self, function=shutil.move):
     if self.output_dir is not None and not os.path.exists(self.output_dir):
         os.mkdir(self.output_dir)
     for path, _, files in os.walk(self.search_dir):
         for filename in files:
             if filename.endswith(self.config.run_extension):
                 continue
             filepath = os.path.join(path, filename)
             gdb_out_filepath = filepath+self.config.get_gdb_exploitable_file_extension()
             if os.path.exists(gdb_out_filepath):
                 file_content = file(gdb_out_filepath, "rb").read()
                 out_dir_main = self.output_dir
                 if out_dir_main is None:
                     out_dir_main = path
                 out_dir = os.path.join(out_dir_main, "UNCATEGORIZED") + os.path.sep
                 for classification in self.classifications:
                     if self._get_search_string_for_classification(classification) in file_content:
                         out_dir = os.path.join(out_dir_main, classification) + os.path.sep
                         break
                 if not os.path.exists(out_dir):
                     os.mkdir(out_dir)
                 Logger.debug("Moving", filepath+"* to", out_dir, debug_level=4)
                 for file_all_extensions in glob.glob(filepath+"*"):
                     function(file_all_extensions, out_dir)
             else:
                 Logger.warning("Seems like there is no gdb output file %s, can not find exploitability" % gdb_out_filepath)

 def divide_by_signal(self, confirmation_loops=0, function=shutil.copyfile):
     if self.output_dir is not None and not os.path.exists(self.output_dir):
         os.mkdir(self.output_dir)
     ex = Executer(self.config)
     for path, _, files in os.walk(self.search_dir):
         for filename in files:
             if filename.endswith(self.config.run_extension):
                 continue
             filepath = os.path.join( path, filename )
             command = self.config.get_command_line(self.binary_to_use, filepath)
             Logger.debug("Executing:", command, debug_level=4)
             Logger.busy()
             signal = ex.get_signal_for_run(command, env=self.config.env)
             while confirmation_loops > 0:
                 Logger.busy()
                 new_signal = ex.get_signal_for_run(command, env=self.config.env)
                 if new_signal == signal:
                     signal = new_signal
                     confirmation_loops -= 1
                 else:
                     Logger.info("Detected varying return codes for exactly the same run")
                     signal = SignalFinder.VARYING_SIGNAL
                     break
             Logger.debug("We consider signal %i for input file %s" % (signal, filename), debug_level=5)
             destination_dir = self.get_folder_path_for_signal(signal)
             if not os.path.exists(destination_dir):
                 os.mkdir(destination_dir)
             function(filepath, os.path.join(destination_dir, filename))

    def run_forest_run(self):
        if self.output_dir is not None and not os.path.exists(self.output_dir):
            os.mkdir(self.output_dir)
        new_file_path = os.path.join(self.config.tmp_dir, "feelingLucky.txt")
        cmd = self.config.get_gdb_command_line(self.config.get_most_standard_binary(), new_file_path, self.gdb_script_path)
        for path, _, files in os.walk(self.search_dir):
            for filename in files:
                eips = []
                indexes = []
                if filename.endswith(self.config.run_extension):
                    continue
                Logger.info("Trying my luck with", filename)
                filepath = os.path.join(path, filename)
                orig_file = file(filepath, "rb").read()
                Logger.debug(filepath, debug_level=4)
                for index in xrange(0,len(orig_file)-len(self.lucky_hex_values)):
                    new_file = orig_file[:index] + self.lucky_hex_values + orig_file[index+len(self.lucky_hex_values):]
                    #Logger.debug(new_file[:100])
                    file(new_file_path, "w").write(new_file)
                    crash_eip = self.get_crash_eip(cmd)
                    if crash_eip:
                        if not crash_eip in eips:
                            eips.append(crash_eip)
                            indexes.append(index)
                        if self.lucky_hex_values <= crash_eip and crash_eip <= self.lucky_hex_values_upper_bound:
                            o = os.path.join(self.output_dir, filename)
                            Logger.info("WTF, we actually were able to control EIP! See file ", o)
                            file(o, "w").write(new_file)
#                        else:
#                            Logger.debug("Binary crashed, but at eip:", hex(crash_eip), "index to put lucky hex value in file:", index, debug_level=7)
                Logger.info("Seen the following crashing eips for this file:", list_as_intervals(eips, as_hex=True))
                Logger.info("File indexes that lead to different crashes for this file:", list_as_intervals(indexes))

 def delete_duplicates_recursively(self):
     Logger.info("Removing duplicates in", self.search_dir)
     for duplicate in self.find_duplicate_contents(self.search_dir):
         if duplicate.endswith(self.config.run_extension):
             continue
         Logger.info("Deleting the duplicate file:", duplicate)
         os.remove(duplicate)

 def run_command(self, command, timeout=None, env={}, stdout=file("/dev/null"), stderr=file("/dev/null")):
     #TODO: make stdout / stderr configurable
     if not timeout:
         timeout = self.config.run_timeout
     process = subprocess.Popen(command, stdin=None, shell=False, stdout=stdout, stderr=stderr)
     self.current_process = process
     signal.signal(signal.SIGALRM, self._handle_alarm)
     #We also had a problem that memory corruptions...
     signal.signal(signal.SIGTTOU, self._handle_sigttou)
     signal.alarm(timeout)
     self.timeout_flag = False
     self.sigttou_flag = False
     #TODO: get rid of magic number
     ret_signal = self.TIMEOUT_SIGNAL
     #blocking call:
     process.communicate()
     signal.alarm(0)
     #This line is reached when timeout_flag was set by _handle_alarm if it was called
     if self.timeout_flag:
         Logger.debug("Process was killed as it exceeded the time limit", debug_level=3)
         ret_signal = self.TIMEOUT_SIGNAL
     elif self.sigttou_flag:
         Logger.debug("Some memory corruption resulted in a SIGTTOU signal being thrown (usually stops process). We caught it.", debug_level=3)
         ret_signal = signal.SIGTTOU
     else:
         ret_signal = process.returncode
     return ret_signal

 def _handle_alarm(self, signum, frame):
     # If the alarm is triggered, we're still in the communicate()
     # call, so use kill() to end the process
     self.timeout_flag = True
     try:
         self.current_process.kill()
     except OSError as ose:
         Logger.info("Kill failed. Sometimes the process exactly exits before we try to kill it... coward. Nothing to worry about.", ose)

 def rename_all_files(self, extension=""):
     i = 1
     for path, _, files in os.walk(self.search_dir):
         for filename in files:
             formatstr = "%0"+str(self.config.max_digets+4)+"d"
             new_filename = formatstr % i
             shutil.move(os.path.join(path, filename), os.path.join(path, new_filename+extension))
             i = i+1
     Logger.info("Renamed all files starting from 1, last file was named", new_filename+extension)

 def get_crash_eip(self, cmd):
     eip = None
     Logger.busy()
     gdb_output = self.executer.get_output_for_run(cmd, self.executer.pipe, self.config.run_timeout_tmin, env=self.config.env, stderr=self.executer.pipe)
     #Logger.debug("GDB output:", gdb_output)
     m = self.regular_expr.search(gdb_output)
     if m:
         eip = m.group(1)
         eip = int(eip, 16)
     #if not signal == SignalFinder.TIMEOUT_SIGNAL:
     #    Logger.error("Minimizing this file took too long, aborted")
     return eip

 def rename_same_name_files(self):
     filenames = []
     for path, _, files in os.walk(self.search_dir):
         for filename in files:
             i = 1
             new_filename = filename
             name, extension = os.path.splitext(filename)
             while new_filename in filenames:
                 formatstr = "%0"+str(self.config.max_digets)+"d"
                 new_number = formatstr % i
                 new_filename = name + "_" + new_number + extension
                 i += 1
             if not new_filename == filename:
                 Logger.info("Found filename that is already taken, renaming", filename, "to", new_filename)
                 shutil.move(os.path.join(path, filename), os.path.join(path, new_filename))
             filenames.append(new_filename)

 def _handle_sigttou(self, signum, frame):
     #Had some issues that when memory corruptions occured in a subprocess
     #(no matter if shielded by multiprocess and subprocess module), 
     #that a SIGTTOU was sent to the entire Python main process.
     #According to https://en.wikipedia.org/wiki/SIGTTOU this
     #results in the process being stopped (and it looks like SIGSTP on the cmd):
     #[1]+  Stopped                 ./AflCrashAnalyzer.py
     #Of course we don't want that. Debugging was hard but then
     #realized after this program was stopped:
     #$ echo $?
     #150
     #So that's SIGTTOU on Linux at least.
     #This handler will prevent the process to stop.
     self.sigttou_flag = True
     try:
         self.current_process.kill()
     except OSError as ose:
         Logger.info("Kill failed. Sometimes the process exactly exits before we try to kill it... coward. Nothing to worry about.", ose)

 def do_sane_output_runs(self):
     if self.output_dir is not None and not os.path.exists(self.output_dir):
         os.mkdir(self.output_dir)
     if self.config.target_binary_plain is None and self.config.target_binary_asan is None:
         Logger.warning("You didn't specify any non-instrumented binary, running tests with instrumented binaries")
         self.instrumented_combined_stdout_stderr()
         self.instrumented_combined_stdout_stderr(gdb_run=True)
     else:
         Logger.info("Plain run")
         self.plain_combined_stdout_stderr()
         Logger.info("Plain gdb run")
         self.plain_combined_stdout_stderr(gdb_run=True)
         Logger.info("ASAN run")
         self.asan_combined_stdout_stderr()

 def _combined_stdout_stderr(self, binary, gdb_run, hint):
     executer = Executer(self.config)
     for path, _, files in os.walk(self.search_dir):
         for filename in files:
             if filename.endswith(self.config.run_extension):
                 continue
             filepath = os.path.join(path, filename)
             if gdb_run:
                 command = self.config.get_gdb_command_line(binary, filepath)
                 new_filename = filename+"-"+os.path.basename(binary)+hint+self.config.gdb_prefix
             else:
                 command = self.config.get_command_line(binary, filepath)
                 new_filename = filename+"-"+os.path.basename(binary)+hint
             Logger.debug("Looking for stdout/stderr output:", command, debug_level=4)
             if self.output_dir:
                 output_file_name = get_new_output_file_name(self.output_dir, new_filename, self.config.run_extension, self.config.max_digets)
                 new_filepath = os.path.join(self.output_dir, output_file_name)
             else:
                 output_file_name = get_new_output_file_name(path, new_filename, self.config.run_extension, self.config.max_digets)
                 new_filepath = os.path.join(path, output_file_name)
             fp = file(new_filepath, "w")
             executer.get_output_for_run(command, fp, env=self.config.env)
             fp.close()

 def notify_restaurant(self):
     if self.restaurant.has_gps_printer:
         # Send GPS push
         pass
         return True
     elif self.restaurant.has_merchant_app:
         # Send GCM Push
         pass
         return True
     else:
         try:
             for contact in self.restaurant.numbers.all():
                 if contact.number_type == NUMBER_TYPE[1][0]:
                     s = SMS()
                     s.send(mobile_number=contact.number, sms_text=self.order_text())
                     OrderLog.objects.create(order=self, message="Restaurant notified via SMS, to"
                                                                 " be followed up by a call",
                                             owner_type='system', owner_id=self.agent.id)
                     return True
         except ObjectDoesNotExist:
             Logger.log_ticket(order=self, message=dict(STOCK_MESSAGES)['failed_to_notify'],
                               owner=self.agent, owner_type='system', ticket_type=TICKET_TYPE[1][0])
             return False

 def minimize_testcases(self):
     if self.output_dir is not None and not os.path.exists(self.output_dir):
         os.mkdir(self.output_dir)
     executer = Executer(self.config)
     for path, _, files in os.walk(self.search_dir):
         for filename in files:
             if filename.endswith(self.config.run_extension):
                 continue
             Logger.info("Minimizing", filename)
             filepath = os.path.join(path, filename)
             cmd = self.config.get_afl_tmin_command_line(filepath, os.path.join(self.output_dir, filename))
             Logger.debug("Executing:", cmd)
             Logger.busy()
             signal = executer.run_command(cmd, timeout=self.config.run_timeout_tmin, env=self.config.env)
             if signal == SignalFinder.TIMEOUT_SIGNAL:
                 Logger.error("Minimizing this file took too long, aborted")

    def should_call(self):
        ticket_id, numbers = None, None

        if self.restaurant.has_to_be_called:
            numbers = [c.number for c in self.restaurant.numbers.all() if c.number_type == NUMBER_TYPE[2][0]]
            ticket_id = Logger.log_ticket(order=self, message="System ticket created to place order via "
                                                  "outbound call with restaurant", owner=self.agent,
                              owner_type="Agent", ticket_type='place_order')
        #else:
        #self.notify_restaurant()

        return {
                "order": self,
                "estimated_delivery_time": 20, #abs((self.expected_delivery_time - datetime.now()).minutes),
                "contact_numbers": numbers,
                "ticket_id": ticket_id
                }

def analyze_output_and_exploitability(config, signal_finder, uninteresting_signals, message_prefix=""):
    for signal, signal_folder in signal_finder.get_folder_paths_for_signals_if_exist(uninteresting_signals):
        skip = False
        for cat in ExploitableGdbPlugin.get_classifications():
            if os.path.exists(os.path.join(signal_folder, cat)):
                Logger.warning("Seems like there are already exploitability analysis results, skipping. If you want to rerun: rm -r %s" % os.path.join(signal_folder, cat))
                skip = True
        if not skip:
            Logger.info(message_prefix, "Discover stdout, stderr, gdb and ASAN output (signal %s)" % signal)
            wildcard_for_run_output_files = os.path.join(signal_folder, "*" + config.run_extension)
            if glob.glob(wildcard_for_run_output_files):
                Logger.warning("Seems like there are already results from running the binaries, skipping. If you want to rerun: rm", wildcard_for_run_output_files)
            else:
                of = OutputFinder(config, signal_folder)
                of.do_sane_output_runs()
            
            Logger.info(message_prefix, "Analyzing exploitability (signal %s)" % signal)
            egp = ExploitableGdbPlugin(config, signal_folder)
            egp.divide_by_exploitability()

 def __init__(self, config, search_dir=None, output_dir=None):
     self.config = config
     self.search_dir = search_dir
     if self.search_dir is None:
         self.search_dir = self.config.original_crashes_directory
     self.output_dir = output_dir
     if self.output_dir is None:
         self.output_dir = self.config.default_signal_directory
     if config.target_binary_plain:
         Logger.debug("Using", self.config.target_binary_plain, "for signal run")
         self.binary_to_use = self.config.target_binary_plain
     elif config.target_binary_asan:
         Logger.debug("Using", self.config.target_binary_asan, "for signal run")
         self.binary_to_use = self.config.target_binary_asan
     else:
         Logger.debug("Using", self.config.target_binary_instrumented, "for signal run")
         self.binary_to_use = self.config.target_binary_instrumented

def main():
    #Read the README before you start.
    
    Logger.info("Setting up configuration")

    gdb_script_64bit = r"""printf "[+] Disabling verbose and complaints\n"
set verbose off
set complaints 0
printf "[+] Backtrace:\n"
bt
printf "[+] info reg:\n"
info reg
printf "[+] exploitable:\n"
exploitable
printf "[+] disassemble $rip, $rip+16:\n"
disassemble $rip, $rip+16
"""
    gdb_script_32bit = r"""printf "[+] Disabling verbose and complaints\n"
set verbose off
set complaints 0
printf "[+] Backtrace:\n"
bt
printf "[+] info reg:\n"
info reg
printf "[+] exploitable:\n"
exploitable
printf "[+] disassemble $eip, $eip+16:\n"
disassemble $eip, $eip+16
"""
    where_this_python_script_lives = os.path.dirname(os.path.realpath(__file__))
    
    gdb_command = "gdb"
    gdb_command_osx = "/opt/local/bin/gdb-apple"
    
    config_gm = CrashAnalysisConfig(where_this_python_script_lives, 
                            target_binary_instrumented=where_this_python_script_lives+"/test-cases/gm/graphicsmagick-afl/utilities/gm", 
                            args_before="identify", 
                            args_after="", 
                            target_binary_plain=where_this_python_script_lives+"/test-cases/gm/graphicsmagick-plain/utilities/gm", 
                            target_binary_asan=where_this_python_script_lives+"/test-cases/gm/graphicsmagick-asan/utilities/gm",
                            env={"ASAN_SYMBOLIZER_PATH": "/usr/bin/llvm-symbolizer-3.4", "ASAN_OPTIONS": "symbolize=1:redzone=512:quarantine_size=512Mb:exitcode=1"},
                            crash_dir=where_this_python_script_lives+"/test-cases/gm/crashes",
                            gdb_script=gdb_script_32bit,
                            gdb_binary=gdb_command
                            )
    
#    config_ffmpeg = CrashAnalysisConfig(where_this_python_script_lives, 
#                        target_binary_instrumented=where_this_python_script_lives+"/test-cases/ffmpeg/ffmpeg-afl/ffmpeg", 
#                        args_before="-i", 
#                        args_after="-loglevel quiet", 
#                        target_binary_plain=where_this_python_script_lives+"/test-cases/ffmpeg/ffmpeg-plain/ffmpeg", 
##                        target_binary_asan=where_this_python_script_lives+"/test-cases/ffmpeg/ffmpeg-asan/ffmpeg",
#                        env={"ASAN_SYMBOLIZER_PATH": "/usr/bin/llvm-symbolizer-3.4", "ASAN_OPTIONS": "symbolize=1:redzone=512:quarantine_size=512Mb:exitcode=1"},
#                        crash_dir=where_this_python_script_lives+"/test-cases/ffmpeg/crashes",
#                        gdb_script=gdb_script_32bit,
#                        gdb_binary=gdb_command
#                        )

    #
    Logger.info("Input crashes directory operations")
    #
    
    Logger.info("Removing README.txt files")
    fdf = FileDuplicateFinder(config_gm, config_gm.original_crashes_directory)
    fdf.remove_readmes()
    
    Logger.info("Removing duplicates from original crashes folder (same file size + MD5)")
    fdf.delete_duplicates_recursively()
    
    Logger.info("Renaming files from original crashes folder so that the filename is a unique identifier. This allows us to copy all crash files into one directory (eg. for tmin output) if necessary, without name collisions")
    fdf.rename_same_name_files()
    
    #
    Logger.info("Finding interesting signals (all crashes)")
    #
    sf_all_crashes = SignalFinder(config_gm)
    if os.path.exists(config_gm.default_signal_directory):
        Logger.warning("Seems like all crashes were already categorized by signal, skipping. If you want to rerun: rm -r", config_gm.default_signal_directory)
    else:
        Logger.debug("Dividing files to output folder according to their signal")
        sf_all_crashes.divide_by_signal()
    
    #Interestings signals: negative on OSX, 129 and above for Linux
    #Uninteresting signals: We usually don't care about signals 0, 1, 2, etc. up to 128
    uninteresting_signals = range(0,129)
    
    analyze_output_and_exploitability(config_gm, sf_all_crashes, uninteresting_signals, message_prefix="Interesting signals /")
        
    Logger.info("Interesting signals / Minimizing input (afl-tmin)")
    if os.path.exists(config_gm.default_minimized_crashes_directory):
        Logger.warning("Seems like crashes were already minimized, skipping. If you want to rerun: rm -r", config_gm.default_minimized_crashes_directory)
    else:
        for signal, signal_folder in sf_all_crashes.get_folder_paths_for_signals_if_exist(uninteresting_signals):
            Logger.debug("Minimizing inputs resulting in signal %i" % signal)
            im = InputMinimizer(config_gm, signal_folder)
            im.minimize_testcases()
        
        Logger.info("Interesting signals / Minimized inputs / Deduplication")
        fdf_minimized = FileDuplicateFinder(config_gm, config_gm.default_minimized_crashes_directory)
        fdf_minimized.delete_duplicates_recursively()
#.........这里部分代码省略.........

    def sanity_check(self):
        ##
        # Sanity checks and initial setup
        ##
        if not os.access(self.target_binary_instrumented, os.R_OK):
            Logger.fatal("AFL target binary not accessible:", self.target_binary_instrumented+". Did you configure the CrashAnalysisConfig class?")
        if not self.target_binary_plain is None and not os.access(self.target_binary_plain, os.R_OK):
            Logger.fatal("Target binary not accessible:", self.target_binary_plain+". Did you configure the CrashAnalysisConfig class?")
        if not self.target_binary_asan is None and not os.access(self.target_binary_asan, os.R_OK):
            Logger.fatal("ASAN target binary not accessible:", self.target_binary_asan+". Did you configure the CrashAnalysisConfig class?")
        if not os.access(self.main_dir, os.F_OK):
            Logger.fatal("Your main_dir doesn't exist:", self.main_dir)
        if not os.access(self.original_crashes_directory, os.F_OK):
            Logger.fatal("Your original_crashes_directory doesn't exist:", self.original_crashes_directory)

        if os.path.exists(self.output_dir):
            Logger.warning("Your output directory already exists, did you want to move it before running?", self.output_dir)
        else:
            Logger.info("Output folder will be:", self.output_dir)
            os.mkdir(self.output_dir)
        if not os.path.exists(self.tmp_dir):
            os.mkdir(self.tmp_dir)
        self.prepare_gdb_script()