def check_for_crashes(self, test_name=None):
     test_name = test_name or self.last_test
     dump_dir = self.device.pull_minidumps()
     crashed = BaseRunner.check_for_crashes(
         self, dump_directory=dump_dir, test_name=test_name)
     mozfile.remove(dump_dir)
     return crashed

def process_test_job(data):
    global logger
    logger = logger or structuredlog.get_default_logger()

    build_name = "{}-{} {}".format(data['platform'], data['buildtype'], data['test'])
    logger.debug("now processing a '{}' job".format(build_name))

    log_url = None
    for name, url in data['blobber_files'].iteritems():
        if name in settings['structured_log_names']:
            log_url = url
            break
    log_path = _download_log(log_url)

    try:
        backend = settings['datastore']
        db_args = config.database
        store = get_storage_backend(backend, **db_args)

        # TODO commit metadata about the test run

        handler = StoreResultsHandler(store)
        with open(log_path, 'r') as log:
            iterator = reader.read(log)
            reader.handle_log(iterator, handler)
    finally:
        mozfile.remove(log_path)

 def vendor(self, ignore_modified=False):
     self.populate_logger()
     self.log_manager.enable_unstructured()
     if not ignore_modified:
         self.check_modified_files()
     cargo = self.get_cargo_path()
     if not self.check_cargo_version(cargo):
         self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)')
         return
     else:
         self.log(logging.DEBUG, 'cargo_version', {}, 'cargo is new enough')
     have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines())
     if not have_vendor:
         self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor')
         self.run_process(args=[cargo, 'install', 'cargo-vendor'])
     else:
         self.log(logging.DEBUG, 'cargo_vendor', {}, 'cargo-vendor already intalled')
     vendor_dir = mozpath.join(self.topsrcdir, 'third_party/rust')
     self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % vendor_dir)
     mozfile.remove(vendor_dir)
     # Once we require a new enough cargo to switch to workspaces, we can
     # just do this once on the workspace root crate.
     for crate_root in ('toolkit/library/rust/',
                        'toolkit/library/gtest/rust'):
         path = mozpath.join(self.topsrcdir, crate_root)
         self._run_command_in_srcdir(args=[cargo, 'generate-lockfile', '--manifest-path', mozpath.join(path, 'Cargo.toml')])
         self._run_command_in_srcdir(args=[cargo, 'vendor', '--sync', mozpath.join(path, 'Cargo.lock'), vendor_dir])
     #TODO: print stats on size of files added/removed, warn or error
     # when adding very large files (bug 1306078)
     self.repository.add_remove_files(vendor_dir)

 def python_test(self, *args, **kwargs):
     try:
         tempdir = os.environ[b'PYTHON_TEST_TMP'] = str(tempfile.mkdtemp(suffix='-python-test'))
         return self.run_python_tests(*args, **kwargs)
     finally:
         import mozfile
         mozfile.remove(tempdir)

    def computeSNRAndDelay(self):
        snr_delay = "-1.000,-1"

        if not os.path.exists(_MEDIA_TOOLS_):
            return False, "SNR Tool not found"

        cmd = [_MEDIA_TOOLS_, '-c', 'snr', '-r', _INPUT_FILE_, '-t',
               _RECORDED_NO_SILENCE_]
        cmd = [str(s) for s in cmd]

        output = subprocess.check_output(cmd)
        # SNR_Delay=1.063,5
        result = re.search('SNR_DELAY=(\d+\.\d+),(\d+)', output)

        # delete the recorded file with no silence
        mozfile.remove(_RECORDED_NO_SILENCE_)

        if result:
            snr_delay = str(result.group(1)) + ',' + str(result.group(2))
            return True, snr_delay
        else:
            """
            We return status as True since SNR computation went through
            successfully but scores computation failed due to severly
            degraded audio quality.
            """
            return True, snr_delay

    def setupProfile(self, prefs=None):
        """Sets up the user profile on the device.

        :param prefs: String of user_prefs to add to the profile. Defaults to a standard b2g testing profile.
        """
        # currently we have no custom prefs to set (when bug 800138 is fixed,
        # we will probably want to enable marionette on an external ip by
        # default)
        if not prefs:
            prefs = ""

        #remove previous user.js if there is one
        if not self.profileDir:
            self.profileDir = tempfile.mkdtemp()
        our_userJS = os.path.join(self.profileDir, "user.js")
        mozfile.remove(our_userJS)
        #copy profile
        try:
            self.getFile(self.userJS, our_userJS)
        except subprocess.CalledProcessError:
            pass
        #if we successfully copied the profile, make a backup of the file
        if os.path.exists(our_userJS):
            self.shellCheckOutput(['dd', 'if=%s' % self.userJS, 'of=%s.orig' % self.userJS])
        with open(our_userJS, 'a') as user_file:
            user_file.write("%s" % prefs)

        self.pushFile(our_userJS, self.userJS)
        self.restartB2G()
        self.setupMarionette()

    def get_gaia_info(self, app_zip):
        tempdir = tempfile.mkdtemp()
        try:
            gaia_commit = os.path.join(tempdir, 'gaia_commit.txt')
            try:
                zip_file = zipfile.ZipFile(app_zip.name)
                with open(gaia_commit, 'w') as f:
                    f.write(zip_file.read('resources/gaia_commit.txt'))
            except zipfile.BadZipfile:
                self._logger.info('Unable to unzip application.zip, falling '
                                  'back to system unzip')
                from subprocess import call
                call(['unzip', '-j', app_zip.name, 'resources/gaia_commit.txt',
                      '-d', tempdir])

            with open(gaia_commit) as f:
                changeset, date = f.read().splitlines()
                self._info['gaia_changeset'] = re.match(
                    '^\w{40}$', changeset) and changeset or None
                self._info['gaia_date'] = date
        except KeyError:
                self._logger.warning(
                    'Unable to find resources/gaia_commit.txt in '
                    'application.zip')
        finally:
            mozfile.remove(tempdir)

 def pushDir(self, localDir, remoteDir, retryLimit=None):
     # adb "push" accepts a directory as an argument, but if the directory
     # contains symbolic links, the links are pushed, rather than the linked
     # files; we either zip/unzip or re-copy the directory into a temporary
     # one to get around this limitation
     retryLimit = retryLimit or self.retryLimit
     if not self.dirExists(remoteDir):
         self.mkDirs(remoteDir+"/x")
     if self._useZip:
         try:
             localZip = tempfile.mktemp() + ".zip"
             remoteZip = remoteDir + "/adbdmtmp.zip"
             subprocess.Popen(["zip", "-r", localZip, '.'], cwd=localDir,
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
             self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False)
             mozfile.remove(localZip)
             data = self._runCmd(["shell", "unzip", "-o", remoteZip,
                                  "-d", remoteDir]).stdout.read()
             self._checkCmd(["shell", "rm", remoteZip],
                            retryLimit=retryLimit)
             if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
                 raise Exception("unzip failed, or permissions error")
         except:
             self._logger.info("zip/unzip failure: falling back to normal push")
             self._useZip = False
             self.pushDir(localDir, remoteDir, retryLimit=retryLimit)
     else:
         tmpDir = tempfile.mkdtemp()
         # copytree's target dir must not already exist, so create a subdir
         tmpDirTarget = os.path.join(tmpDir, "tmp")
         shutil.copytree(localDir, tmpDirTarget)
         self._checkCmd(["push", tmpDirTarget, remoteDir], retryLimit=retryLimit)
         mozfile.remove(tmpDir)

    def remove_addon(self, addon_id):
        """Remove the add-on as specified by the id

        :param addon_id: id of the add-on to be removed
        """
        path = self.get_addon_path(addon_id)
        mozfile.remove(path)

 def getDirectory(self, remoteDir, localDir, checkDir=True):
     localDir = os.path.normpath(localDir)
     remoteDir = os.path.normpath(remoteDir)
     copyRequired = False
     originalLocal = localDir
     if self._adb_version >= '1.0.36' and \
        os.path.isdir(localDir) and self.dirExists(remoteDir):
         # See do_sync_pull in
         # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp
         # Work around change in behavior in adb 1.0.36 where if
         # the local destination directory exists, adb pull will
         # copy the source directory *into* the destination
         # directory otherwise it will copy the source directory
         # *onto* the destination directory.
         #
         # If the destination directory does exist, pull to its
         # parent directory. If the source and destination leaf
         # directory names are different, pull the source directory
         # into a temporary directory and then copy the temporary
         # directory onto the destination.
         localName = os.path.basename(localDir)
         remoteName = os.path.basename(remoteDir)
         if localName != remoteName:
             copyRequired = True
             tempParent = tempfile.mkdtemp()
             localDir = os.path.join(tempParent, remoteName)
         else:
             localDir = '/'.join(localDir.rstrip('/').split('/')[:-1])
     self._runCmd(["pull", remoteDir, localDir]).wait()
     if copyRequired:
         dir_util.copy_tree(localDir, originalLocal)
         mozfile.remove(tempParent)

    def clean(self):
        """Clean up addons in the profile."""

        # Remove all add-ons installed
        for addon in self._addons:
            # TODO (bug 934642)
            # Once we have a proper handling of add-ons we should kill the id
            # from self._addons once the add-on is removed. For now lets forget
            # about the exception
            try:
                self.remove_addon(addon)
            except IOError:
                pass

        # Remove all downloaded add-ons
        for addon in self.downloaded_addons:
            mozfile.remove(addon)

        # restore backups
        if self.backup_dir and os.path.isdir(self.backup_dir):
            extensions_path = os.path.join(self.profile, 'extensions', 'staged')

            for backup in os.listdir(self.backup_dir):
                backup_path = os.path.join(self.backup_dir, backup)
                shutil.move(backup_path, extensions_path)

            if not os.listdir(self.backup_dir):
                mozfile.remove(self.backup_dir)

        # reset instance variables to defaults
        self._internal_init()

    def __iter__(self):
        for path, extra in self.dump_files:
            rv = self._process_dump_file(path, extra)
            yield rv

        if self.remove_symbols:
            mozfile.remove(self.symbols_path)

def cleanup_pending_crash_reports():
    """
    Delete any pending crash reports.

    The presence of pending crash reports may be reported by the browser,
    affecting test results; it is best to ensure that these are removed
    before starting any browser tests.

    Firefox stores pending crash reports in "<UAppData>/Crash Reports".
    If the browser is not running, it cannot provide <UAppData>, so this
    code tries to anticipate its value.

    See dom/system/OSFileConstants.cpp for platform variations of <UAppData>.
    """
    if mozinfo.isWin:
        location = os.path.expanduser("~\\AppData\\Roaming\\Mozilla\\Firefox\\Crash Reports")
    elif mozinfo.isMac:
        location = os.path.expanduser("~/Library/Application Support/firefox/Crash Reports")
    else:
        location = os.path.expanduser("~/.mozilla/firefox/Crash Reports")
    logger = get_logger()
    if os.path.exists(location):
        try:
            mozfile.remove(location)
            logger.info("Removed pending crash reports at '%s'" % location)
        except:
            pass

 def fetch_and_unpack(self, revision, target):
     '''Fetch and unpack upstream source'''
     url = self.upstream_snapshot(revision)
     self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
     prefix = 'aom-' + revision
     filename = prefix + '.tar.gz'
     with open(filename, 'wb') as f:
         req = requests.get(url, stream=True)
         for data in req.iter_content(4096):
             f.write(data)
     tar = tarfile.open(filename)
     bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
                        tar.getnames())
     if any(bad_paths):
         raise Exception("Tar archive contains non-local paths,"
                         "e.g. '%s'" % bad_paths[0])
     self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
     mozfile.remove(target)
     self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
     tar.extractall(target)
     # Github puts everything properly down a directory; move it up.
     if all(map(lambda name: name.startswith(prefix), tar.getnames())):
         tardir = mozpath.join(target, prefix)
         os.system('mv %s/* %s/.* %s' % (tardir, tardir, target))
         os.rmdir(tardir)
     # Remove the tarball.
     mozfile.remove(filename)

        def _run_tests(tags):
            application_folder = None

            try:
                # Backup current tags
                test_tags = self.test_tags

                application_folder = self.duplicate_application(source_folder)
                self.bin = mozinstall.get_binary(application_folder, 'Firefox')

                self.test_tags = tags
                super(UpdateTestRunner, self).run_tests(tests)

            except Exception:
                self.exc_info = sys.exc_info()
                self.logger.error('Failure during execution of the update test.',
                                  exc_info=self.exc_info)

            finally:
                self.test_tags = test_tags

                self.logger.info('Removing copy of the application at "%s"' % application_folder)
                try:
                    mozfile.remove(application_folder)
                except IOError as e:
                    self.logger.error('Cannot remove copy of application: "%s"' % str(e))

    def download(self, url, target_folder=None):
        """
        Downloads an add-on from the specified URL to the target folder

        :param url: URL of the add-on (XPI file)
        :param target_folder: Folder to store the XPI file in

        """
        response = urllib2.urlopen(url)
        fd, path = tempfile.mkstemp(suffix='.xpi')
        os.write(fd, response.read())
        os.close(fd)

        if not self.is_addon(path):
            mozfile.remove(path)
            raise AddonFormatError('Not a valid add-on: %s' % url)

        # Give the downloaded file a better name by using the add-on id
        details = self.addon_details(path)
        new_path = path.replace('.xpi', '_%s.xpi' % details.get('id'))

        # Move the add-on to the target folder if requested
        if target_folder:
            new_path = os.path.join(target_folder, os.path.basename(new_path))

        os.rename(path, new_path)

        return new_path

 def clean(self):
     """
     Clean up temp folders created with the instance creation.
     """
     mozfile.remove(self.option('dir'))
     for symbol_path in self.symbol_paths.values():
         mozfile.remove(symbol_path)

    def bootstrap(self):
        try:
            name = self.boot.instance.__class__.__name__
            name = name[:name.index('Bootstrapper')].lower()

            if hasattr(self, 'pre_bootstrap_%s' % name):
                getattr(self, 'pre_bootstrap_%s' % name)()
            else:
                self.boot.finished = PLATFORM_NOT_IMPLEMENTED + SEE_MDN_DOCS
            self.boot.instance.packages.extend(self.extra_packages)

            if name == 'osx':
                # for osx, use the shell script provided in the documentation
                self.boot.instance.ensure_xcode()
                self._download('https://raw.github.com/mozilla-b2g/B2G/master/scripts/bootstrap-mac.sh')
                print('Running the mac bootstrap script...')
                ret = subprocess.call(['bash', 'bootstrap-mac.sh'])
                os.remove('bootstrap-mac.sh')
                print(self.boot.finished)
                return ret

            self.boot.bootstrap()

            if hasattr(self, 'post_bootstrap_%s' % name):
                getattr(self, 'post_bootstrap_%s' % name)()
        finally:
            if self.mozboot_dir:
                import mozfile
                mozfile.remove(self.mozboot_dir)

  def __init__(self, userName, password, serverUri,
               tbVersion, binary, cachePath="profileCache", reset=False, *args, **kwargs):
    self.profileName = "%s-tb%d-%s" % (userName, tbVersion, time.strftime("%Y-%m-%d", time.localtime()))
    profilePath = os.path.join(cachePath, self.profileName)

    if reset:
      print "Reseting profile in",profilePath
      mozfile.remove(profilePath)

    super(ObmProfile, self).__init__(profile=profilePath, *args, **kwargs)
    self.userName = userName
    self.password = password
    self.serverUri = serverUri
    self.tbVersion = tbVersion

    # Thunderbird 3 doesn't have 64-bit NSS libraries on mac, use the old
    # signons file for this version
    if self.tbVersion > 3:
      self.signons = SignonsSQLFile(profilePath, os.path.dirname(binary))
    else:
      self.signons = Signons3File(os.path.join(profilePath, "signons3.txt"))

    self.overrides = CertOverrideFile(os.path.join(profilePath,"cert_override.txt"))

    self.initProfile()
    self.flush()

    def set_default_settings(self):
        filename = 'settings.json'
        defaults = DEFAULT_SETTINGS.copy()
        defaults.update(self.testvars.get('settings', {}))
        defaults = self.modify_settings(defaults)

        if self.locale != 'undefined':
                defaults['language.current'] = self.locale

        if self.device.is_desktop_b2g:
            directory = self.marionette.instance.profile_path
            path = os.path.join(directory, filename)
        else:
            directory = '/system/b2g/defaults'
            path = posixpath.join(directory, filename)

        settings = json.loads(self.device.file_manager.pull_file(path))
        for name, value in defaults.items():
            self.logger.debug('Setting %s to %s' % (name, value))
            settings[name] = value
        td = tempfile.mkdtemp()
        try:
            tf = os.path.join(td, filename)
            with open(tf, 'w') as f:
                json.dump(settings, f)
            if not self.device.is_desktop_b2g:
                self.device.manager.remount()
            self.device.file_manager.push_file(tf, directory)
        finally:
            mozfile.remove(td)