def testCreateNewOoid(self):
        new_ooid = ooid.create_new_ooid()
        ndate = ooid.dateFromOoid(new_ooid)
        ndepth = ooid.depthFromOoid(new_ooid)
        assert self.nowstamp == ndate
        assert ooid.defaultDepth == ndepth

        new_ooid = ooid.create_new_ooid(timestamp=self.xmas05)
        ndate = ooid.dateFromOoid(new_ooid)
        ndepth = ooid.depthFromOoid(new_ooid)
        assert self.xmas05 == ndate
        assert ooid.defaultDepth == ndepth

        for d in range(1, 5):
            ooid0 = ooid.create_new_ooid(depth=d)
            ooid1 = ooid.create_new_ooid(timestamp=self.xmas05, depth=d)
            ndate0 = ooid.dateFromOoid(ooid0)
            ndepth0 = ooid.depthFromOoid(ooid0)
            ndate1 = ooid.dateFromOoid(ooid1)
            ndepth1 = ooid.depthFromOoid(ooid1)
            assert self.nowstamp == ndate0
            assert self.xmas05 == ndate1
            assert ndepth0 == ndepth1
            assert d == ndepth0
        assert ooid.depthFromOoid(self.badooid0) is None
        assert ooid.depthFromOoid(self.badooid1) is None

  def testCreateNewOoid(self):
    ooid = oo.createNewOoid()
    ndate = oo.dateFromOoid(ooid)
    ndepth = oo.depthFromOoid(ooid)
    assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate)
    assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)

    ooid = oo.createNewOoid(timestamp=self.xmas05)
    ndate = oo.dateFromOoid(ooid)
    ndepth = oo.depthFromOoid(ooid)
    assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate)
    assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)

    for d in range(1,5):
      ooid0 = oo.createNewOoid(depth=d)
      ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d)
      ndate0 = oo.dateFromOoid(ooid0)
      ndepth0 = oo.depthFromOoid(ooid0)
      ndate1 = oo.dateFromOoid(ooid1)
      ndepth1 = oo.depthFromOoid(ooid1)
      assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0)
      assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1)
      assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1)
      assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0)
    assert None == oo.depthFromOoid(self.badooid0)
    assert None == oo.depthFromOoid(self.badooid1)

 def _get_datestamp(self, crashid):
     """Retrieves the datestamp from a crashid or raises an exception"""
     datestamp = dateFromOoid(crashid)
     if datestamp is None:
         # We should never hit this situation unless the crashid is a bad crashid
         raise CrashidMissingDatestamp('%s is missing datestamp' % crashid)
     return datestamp

  def processJob (self, jobTuple):
    """ This function is run only by a worker thread.
        Given a job, fetch a thread local database connection and the json document.  Use these
        to create the record in the 'reports' table, then start the analysis of the dump file.

        input parameters:
          jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the
              jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer)
    """
    threadName = threading.currentThread().getName()
    try:
      threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair()
    except:
      self.quit = True
      socorro.lib.util.reportExceptionAndAbort(logger) # can't continue without a database connection
    try:
      processorErrorMessages = []
      jobId, jobUuid, jobPriority = jobTuple
      logger.info("%s - starting job: %s, %s", threadName, jobId, jobUuid)
      startedDateTime = datetime.datetime.now()
      threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId))
      threadLocalDatabaseConnection.commit()

      jobPathname = self.jsonPathForUuidInJsonDumpStorage(jobUuid)
      dumpfilePathname = self.dumpPathForUuidInJsonDumpStorage(jobUuid)
      jsonFile = open(jobPathname)
      try:
        jsonDocument = simplejson.load(jsonFile)
      finally:
        jsonFile.close()

      try:
        date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"])
      except KeyError:
        date_processed = ooid.dateFromOoid(jobUuid)

      reportId = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, jobPathname, date_processed, processorErrorMessages)
      threadLocalDatabaseConnection.commit()
      truncated = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, threadLocalCursor, date_processed, processorErrorMessages)
      self.quitCheck()
      #finished a job - cleanup
      threadLocalCursor.execute("update jobs set completeddatetime = %s, success = True where id = %s", (datetime.datetime.now(), jobId))
      threadLocalCursor.execute("update reports set started_datetime = timestamp without time zone '%s', completed_datetime = timestamp without time zone '%s', success = True, truncated = %s where id = %s and date_processed = timestamp without time zone '%s'" % (startedDateTime, datetime.datetime.now(), truncated, reportId, date_processed))
      #self.updateRegistrationNoCommit(threadLocalCursor)
      threadLocalDatabaseConnection.commit()
      logger.info("%s - succeeded and committed: %s, %s", threadName, jobId, jobUuid)
    except (KeyboardInterrupt, SystemExit):
      logger.info("%s - quit request detected", threadName)
      self.quit = True
      try:
        logger.info("%s - abandoning job with rollback: %s, %s", threadName, jobId, jobUuid)
        threadLocalDatabaseConnection.rollback()
        threadLocalDatabaseConnection.close()
      except:
        pass
    except DuplicateEntryException, x:
      logger.warning("%s - duplicate entry: %s", threadName, jobUuid)

 def _get_base(self, crash_id):
     """this method overrides the base method to define the daily file
     system root directory name.  While the default class is to use a
     YYYYMMDD form, this class substitutes a simple DD form.  This is the
     mechanism of directory recycling as at the first day of a new month
     we return to the same directiory structures that were created on the
     first day of the previous month"""
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%02d" % (date.day,)
     return [self.config.fs_root, date_formatted]

 def dailyPart(self, ooid, timestamp=None):
   """
   return YYYYMMDD
   use the timestamp if any, else the ooid's last 6 chars if reasonable, else now()
   """
   year,month,day = None,None,None
   if not timestamp:
     timestamp = socorro_ooid.dateFromOoid(ooid)
   if not timestamp:
     timestamp = utc_now()
   (year,month,day) = (timestamp.year,timestamp.month,timestamp.day)
   return "%4d%02d%02d"%(year,month,day)

 def build_keys(self, prefix, name_of_thing, id):
     """return a list of one key. The reason for not returning more than
     one is that this key builder class is going to be used for
     something new so it has no legacy."""
     datestamp = dateFromOoid(id)
     if datestamp is None:
         # the id did not have a date component in it
         datestamp = datetime.datetime.utcnow()
     date = datestamp.strftime('%Y%m%d')
     keys = [
         '%s/v1/%s/%s/%s' % (
             prefix, name_of_thing, date, id
         )
     ]
     return keys

 def newEntry(self, ooid, timestamp=None, webheadName=None):
     """
     Sets up the name and date storage directory branches for the given
       ooid.
     Creates any needed directories along the path to the appropriate
       storage location. Sets gid and mode if specified
     Creates one symbolic link in the date leaf directory with name ooid
       and referencing the name leaf directory
     returns (nameDir,dateDir)
     """
     if not timestamp:
         timestamp = socorro_ooid.dateFromOoid(ooid)
         if not timestamp:
             timestamp = utc_now()
     if not self.osModule.path.isdir(self.root):
         um = self.osModule.umask(0)
         try:
             self.osModule.mkdir(self.root, self.dirPermissions)
         finally:
             self.osModule.umask(um)
     nameDir, nparts = self.makeNameDir(ooid, timestamp)
     dateDir, dparts = self.makeDateDir(timestamp, webheadName)
     # adjust the current subslot only when inserting a new entry
     if self.subSlotCount:
         k = dparts[-1].split('_')[0]
         curcount = self.currentSubSlots.setdefault(k, 0)
         self.currentSubSlots[k] = (curcount + 1) % self.subSlotCount
     parts = [os.path.pardir, ] * (len(dparts) - 2)  # lose root / dailypart
     parts.append(self.indexName)
     parts.extend(self.relativeNameParts(ooid))
     relNameDir = os.sep.join(parts)
     try:
         self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
     except OSError, x:
         if errno.ENOENT == x.errno:
         # maybe another thread cleaned this out from under us. Try again
             nameDir = self.makeNameDir(ooid)  # might be overkill,
                                               # but cheap insurance
             dateDir = self.makeDateDir(timestamp)
             self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
         elif errno.EEXIST == x.errno:
             self.osModule.unlink(os.path.join(dateDir, ooid))
             self.osModule.symlink(relNameDir, os.path.join(dateDir, ooid))
         else:
             raise

 def lookupOoidInDatePath(self, date, ooid, webheadName=None):
     """Look for the date path holding a symbolic link named 'ooid', return
     datePath, dateParts on failure return None,[]"""
     if not date:
         date = socorro_ooid.dateFromOoid(ooid)
     if date:
         datePath, dateParts = self.datePath(date, webheadName)
         if self.osModule.path.exists(os.path.join(datePath, ooid)):
             return datePath, dateParts
     for d in self.osModule.listdir(self.root):
         # We don't know webhead if any, so avoid confusion by looking
         # everywhere
         for dir, dirs, files in os.walk(
           os.sep.join((self.root, d, self.dateName))
         ):
             if ooid in dirs or ooid in files:  # probably dirs
                 dirPath = dir
                 dirParts = dir.split(os.sep)
                 return dirPath, dirParts
     return None, []

    def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):

        processor_notes = processor_meta.processor_notes

        processed_crash.submitted_timestamp = raw_crash.get(
            'submitted_timestamp',
            dateFromOoid(raw_crash.uuid)
        )
        if isinstance(processed_crash.submitted_timestamp, basestring):
            processed_crash.submitted_timestamp = datetime_from_isodate_string(
                processed_crash.submitted_timestamp
            )
        processed_crash.date_processed = processed_crash.submitted_timestamp
        # defaultCrashTime: must have crashed before date processed
        submitted_timestamp_as_epoch = int(
            time.mktime(processed_crash.submitted_timestamp.timetuple())
        )
        try:
            timestampTime = int(
                raw_crash.get('timestamp', submitted_timestamp_as_epoch)
            )  # the old name for crash time
        except ValueError:
            timestampTime = 0
            processor_notes.append('non-integer value of "timestamp"')
        try:
            crash_time = int(
                self._get_truncate_or_warn(
                    raw_crash,
                    'CrashTime',
                    processor_notes,
                    timestampTime,
                    10
                )
            )
        except ValueError:
            crash_time = 0
            processor_notes.append(
                'non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime
            )

        processed_crash.crash_time = crash_time
        if crash_time == submitted_timestamp_as_epoch:
            processor_notes.append("client_crash_date is unknown")
        # StartupTime: must have started up some time before crash
        try:
            startupTime = int(raw_crash.get('StartupTime', crash_time))
        except ValueError:
            startupTime = 0
            processor_notes.append('non-integer value of "StartupTime"')
        # InstallTime: must have installed some time before startup
        try:
            installTime = int(raw_crash.get('InstallTime', startupTime))
        except ValueError:
            installTime = 0
            processor_notes.append('non-integer value of "InstallTime"')
        processed_crash.client_crash_date = datetime.datetime.fromtimestamp(
            crash_time,
            UTC
        )
        processed_crash.install_age = crash_time - installTime
        processed_crash.uptime = max(0, crash_time - startupTime)
        try:
            last_crash = int(raw_crash.SecondsSinceLastCrash)
        except (KeyError, TypeError, ValueError):
            last_crash = None
            processor_notes.append(
                'non-integer value of "SecondsSinceLastCrash"'
            )
        if last_crash > maxint:
            last_crash = None
            processor_notes.append(
                '"SecondsSinceLastCrash" larger than MAXINT - set to NULL'
            )
        processed_crash.last_crash = last_crash

        return True

 def _get_base(self, crash_id):
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
     return [self.config.fs_root, date_formatted]

 def testGetDate(self):
   for ooid in self.yyyyoids:
     assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid))
     assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid))
   assert None == oo.dateFromOoid(self.badooid0)
   assert None == oo.dateFromOoid(self.badooid1)

  def processJob (self, jobTuple):
    """ This function is run only by a worker thread.
        Given a job, fetch a thread local database connection and the json document.  Use these
        to create the record in the 'reports' table, then start the analysis of the dump file.

        input parameters:
          jobTuple: a tuple containing up to three items: the jobId (the primary key from the jobs table), the
              jobUuid (a unique string with the json file basename minus the extension) and the priority (an integer)
    """
    if self.quit:
      return Processor.quit
    threadName = threading.currentThread().getName()

    try:
      threadLocalDatabaseConnection, threadLocalCursor = self.databaseConnectionPool.connectionCursorPair()
      threadLocalCrashStorage = self.crashStorePool.crashStorage(threadName)
    except sdb.exceptions_eligible_for_retry:
      logger.critical("something's gone horribly wrong with the database connection")
      sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL)
      return Processor.criticalError
    except hbc.FatalException:
      logger.critical("something's gone horribly wrong with the HBase connection")
      sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL)
      return Processor.criticalError
    except Exception:
      self.quit = True
      sutil.reportExceptionAndContinue(logger, loggingLevel=logging.CRITICAL)
      return Processor.quit

    try:
      self.quitCheck()
      newReportRecordAsDict = {}
      processorErrorMessages = []
      jobId, jobUuid, jobPriority = jobTuple
      logger.info("starting job: %s", jobUuid)
      startedDateTime = self.nowFunc()
      threadLocalCursor.execute("update jobs set starteddatetime = %s where id = %s", (startedDateTime, jobId))
      threadLocalDatabaseConnection.commit()

      jsonDocument = threadLocalCrashStorage.get_meta(jobUuid)

      self.config.logger.debug('about to apply rules')
      self.json_transform_rule_system.apply_all_rules(jsonDocument, self)
      self.config.logger.debug('done applying transform rules')

      try:
        date_processed = sdt.datetimeFromISOdateString(jsonDocument["submitted_timestamp"])
      except KeyError:
        date_processed = ooid.dateFromOoid(jobUuid)

      newReportRecordAsDict = self.insertReportIntoDatabase(threadLocalCursor, jobUuid, jsonDocument, date_processed, processorErrorMessages)

      if jobUuid in self.priority_job_set:
        processorErrorMessages.append('Priority Job')
        self.priority_job_set.remove(jobUuid)

      threadLocalDatabaseConnection.commit()
      reportId = newReportRecordAsDict["id"]
      newReportRecordAsDict['dump'] = ''
      newReportRecordAsDict["startedDateTime"] = startedDateTime

      try:
        newReportRecordAsDict["ReleaseChannel"] = jsonDocument["ReleaseChannel"]
      except KeyError:
        newReportRecordAsDict["ReleaseChannel"] = 'unknown'

      if self.config.collectAddon:
        #logger.debug("collecting Addons")
        addonsAsAListOfTuples = self.insertAdddonsIntoDatabase(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages)
        newReportRecordAsDict["addons"] = addonsAsAListOfTuples

      if self.config.collectCrashProcess:
        #logger.debug("collecting Crash Process")
        crashProcessAsDict = self.insertCrashProcess(threadLocalCursor, reportId, jsonDocument, date_processed, processorErrorMessages)
        newReportRecordAsDict.update( crashProcessAsDict )

      try:
        dumpfilePathname = threadLocalCrashStorage.dumpPathForUuid(jobUuid,
                                                                   self.config.temporaryFileSystemStoragePath)
        #logger.debug('about to doBreakpadStackDumpAnalysis')
        isHang = 'hangid' in newReportRecordAsDict and bool(newReportRecordAsDict['hangid'])
        # hangType values: -1 if old style hang with hangid and Hang not present
        #                  else hangType == jsonDocument.Hang
        hangType = int(jsonDocument.get("Hang", -1 if isHang else 0))
        java_stack_trace = jsonDocument.setdefault('JavaStackTrace', None)
        additionalReportValuesAsDict = self.doBreakpadStackDumpAnalysis(reportId, jobUuid, dumpfilePathname, hangType, java_stack_trace, threadLocalCursor, date_processed, processorErrorMessages)
        newReportRecordAsDict.update(additionalReportValuesAsDict)
      finally:
        newReportRecordAsDict["completeddatetime"] = completedDateTime = self.nowFunc()
        threadLocalCrashStorage.cleanUpTempDumpStorage(jobUuid, self.config.temporaryFileSystemStoragePath)

      #logger.debug('finished a job - cleanup')
      #finished a job - cleanup
      threadLocalCursor.execute("update jobs set completeddatetime = %s, success = %s where id = %s", (completedDateTime, newReportRecordAsDict['success'], jobId))
      # Bug 519703: Collect setting for topmost source filename(s), addon compatibility check override, flash version
      reportsSql = """
      update reports set
        signature = %%s,
        processor_notes = %%s,
        started_datetime = timestamp with time zone %%s,
#.........这里部分代码省略.........

    def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dump):
        """ This function is run only by a worker thread.
            Given a job, fetch a thread local database connection and the json
            document.  Use these to create the record in the 'reports' table,
            then start the analysis of the dump file.

            input parameters:
        """
        try:
            self.quit_check()
            crash_id = raw_crash.uuid
            processor_notes = []
            processed_crash = DotDict()
            processed_crash.uuid = raw_crash.uuid
            processed_crash.success = False

            started_timestamp = self._log_job_start(crash_id)

            #self.config.logger.debug('about to apply rules')
            self.raw_crash_transform_rule_system.apply_all_rules(raw_crash,
                                                                 self)
            #self.config.logger.debug('done applying transform rules')

            try:
                submitted_timestamp = datetimeFromISOdateString(
                    raw_crash.submitted_timestamp
                )
            except KeyError:
                submitted_timestamp = dateFromOoid(crash_id)

            # formerly the call to 'insertReportIntoDatabase'
            processed_crash_update = self._create_basic_processed_crash(
                crash_id,
                raw_crash,
                submitted_timestamp,
                started_timestamp,
                processor_notes
            )
            processed_crash.update(processed_crash_update)

            temp_dump_pathname = self._get_temp_dump_pathname(
                crash_id,
                raw_dump
            )
            try:
                #logger.debug('about to doBreakpadStackDumpAnalysis')
                processed_crash_update_dict = \
                    self._do_breakpad_stack_dump_analysis(
                        crash_id,
                        temp_dump_pathname,
                        processed_crash.hang_type,
                        processed_crash.java_stack_trace,
                        submitted_timestamp,
                        processor_notes
                    )
                processed_crash.update(processed_crash_update_dict)
            finally:
                self._cleanup_temp_file(temp_dump_pathname)

            processed_crash.topmost_filenames = "|".join(
                processed_crash.get('topmost_filenames', [])
            )
            try:
                processed_crash.Winsock_LSP = raw_crash.Winsock_LSP
            except KeyError:
                pass  # if it's not in the original raw_crash,
                        # it does get into the jsonz

        #except (KeyboardInterrupt, SystemExit):
            #self.config.logger.info("quit request detected")
            #raise
        except Exception, x:
            self.config.logger.warning(
                'Error while processing %s: %s',
                crash_id,
                str(x),
                exc_info=True
            )
            processor_notes.append(str(x))

 def testGetDate(self):
     for this_ooid in self.yyyyoids:
         assert self.baseDate == ooid.dateFromOoid(this_ooid)
         assert 4 == ooid.depthFromOoid(this_ooid)
     assert ooid.dateFromOoid(self.badooid0) is None
     assert ooid.dateFromOoid(self.badooid1) is None