def create_dmg_from_staged(stagedir, output_dmg, tmpdir, volume_name):
    'Given a prepared directory stagedir, produce a DMG at output_dmg.'
    if not is_linux:
        # Running on OS X
        hybrid = os.path.join(tmpdir, 'hybrid.dmg')
        subprocess.check_call(['hdiutil', 'makehybrid', '-hfs',
                               '-hfs-volume-name', volume_name,
                               '-hfs-openfolder', stagedir,
                               '-ov', stagedir,
                               '-o', hybrid])
        subprocess.check_call(['hdiutil', 'convert', '-format', 'UDBZ',
                               '-imagekey', 'bzip2-level=9',
                               '-ov', hybrid, '-o', output_dmg])
    else:
        # The dmg tool doesn't create the destination directories, and silently
        # returns success if the parent directory doesn't exist.
        ensureParentDir(output_dmg)

        hfs = os.path.join(tmpdir, 'staged.hfs')
        subprocess.check_call([
            buildconfig.substs['HFS_TOOL'], hfs, 'addall', stagedir])
        subprocess.check_call([
            buildconfig.substs['DMG_TOOL'],
            'build',
            hfs,
            output_dmg
        ],
                              # dmg is seriously chatty
                              stdout=open(os.devnull, 'wb'))

    def dump_cache(self):
        if self._skip_cache:
            self.log(logging.DEBUG, "artifact", {}, "Skipping cache: ignoring dump_cache!")
            return

        ensureParentDir(self._cache_filename)
        pickle.dump(list(reversed(list(self._cache.items()))), open(self._cache_filename, "wb"), -1)

    def updateManifest(self, manifestPath, chromebasepath, register):
        '''updateManifest replaces the % in the chrome registration entries
        with the given chrome base path, and updates the given manifest file.
        '''

        ensureParentDir(manifestPath)
        lock = lock_file(manifestPath + '.lck')
        try:
            myregister = dict.fromkeys(map(lambda s: s.replace('%',
                    chromebasepath), register.iterkeys()))
            manifestExists = os.path.isfile(manifestPath)
            mode = manifestExists and 'r+b' or 'wb'
            mf = open(manifestPath, mode)
            if manifestExists:
                # import previous content into hash, ignoring empty ones and comments
                imf = re.compile('(#.*)?$')
                for l in re.split('[\r\n]+', mf.read()):
                    if imf.match(l):
                        continue
                    myregister[l] = None
                mf.seek(0)
            for k in sorted(myregister.iterkeys()):
                mf.write(k + os.linesep)
            mf.close()
        finally:
            lock = None

def repackage_installer(topsrcdir, tag, setupexe, package, output):
    if package and not zipfile.is_zipfile(package):
        raise Exception("Package file %s is not a valid .zip file." % package)

    # We need the full path for the tag and output, since we chdir later.
    tag = mozpath.realpath(tag)
    output = mozpath.realpath(output)
    ensureParentDir(output)

    tmpdir = tempfile.mkdtemp()
    old_cwd = os.getcwd()
    try:
        if package:
            z = zipfile.ZipFile(package)
            z.extractall(tmpdir)
            z.close()

        # Copy setup.exe into the root of the install dir, alongside the
        # package.
        shutil.copyfile(setupexe, mozpath.join(tmpdir, mozpath.basename(setupexe)))

        # archive_exe requires us to be in the directory where the package is
        # unpacked (the tmpdir)
        os.chdir(tmpdir)

        sfx_package = mozpath.join(topsrcdir, 'other-licenses/7zstub/firefox/7zSD.sfx')

        package_name = 'firefox' if package else None
        archive_exe(package_name, tag, sfx_package, output)

    finally:
        os.chdir(old_cwd)
        shutil.rmtree(tmpdir)

def explode(aar, destdir):
    # Take just the support-v4-22.2.1 part.
    name, _ = os.path.splitext(os.path.basename(aar))

    destdir = mozpath.join(destdir, name)
    if os.path.exists(destdir):
        # We always want to start fresh.
        shutil.rmtree(destdir)
    ensureParentDir(destdir)
    with zipfile.ZipFile(aar) as zf:
        zf.extractall(destdir)

    # classes.jar is always present.  However, multiple JAR files with the same
    # name confuses our staged Proguard process in
    # mobile/android/base/Makefile.in, so we make the names unique here.
    classes_jar = mozpath.join(destdir, name + '-classes.jar')
    os.rename(mozpath.join(destdir, 'classes.jar'), classes_jar)

    # Embedded JAR libraries are optional.
    finder = FileFinder(mozpath.join(destdir, 'libs'))
    for p, _ in finder.find('*.jar'):
        jar = mozpath.join(finder.base, name + '-' + p)
        os.rename(mozpath.join(finder.base, p), jar)

    # Frequently assets/ is present but empty.  Protect against meaningless
    # changes to the AAR files by deleting empty assets/ directories.
    assets = mozpath.join(destdir, 'assets')
    try:
        os.rmdir(assets)
    except OSError, e:
        if e.errno in (errno.ENOTEMPTY, errno.ENOENT):
            pass
        else:
            raise

def main(argv):
    parser = argparse.ArgumentParser(
        description='Produce test archives')
    parser.add_argument('archive', help='Which archive to generate')
    parser.add_argument('outputfile', help='File to write output to')

    args = parser.parse_args(argv)

    if not args.outputfile.endswith('.zip'):
        raise Exception('expected zip output file')

    file_count = 0
    t_start = time.time()
    ensureParentDir(args.outputfile)
    with open(args.outputfile, 'wb') as fh:
        # Experimentation revealed that level 5 is significantly faster and has
        # marginally larger sizes than higher values and is the sweet spot
        # for optimal compression. Read the detailed commit message that
        # introduced this for raw numbers.
        with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
            res = find_files(args.archive)
            for p, f in res:
                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
                file_count += 1

    duration = time.time() - t_start
    zip_size = os.path.getsize(args.outputfile)
    basename = os.path.basename(args.outputfile)
    print('Wrote %d files in %d bytes to %s in %.2fs' % (
          file_count, zip_size, basename, duration))

    def install_from_file(self, filename, distdir):
        self.log(logging.INFO, "artifact", {"filename": filename}, "Installing from {filename}")

        # Do we need to post-process?
        processed_filename = filename + PROCESSED_SUFFIX

        if self._skip_cache and os.path.exists(processed_filename):
            self.log(
                logging.DEBUG,
                "artifact",
                {"path": processed_filename},
                "Skipping cache: removing cached processed artifact {path}",
            )
            os.remove(processed_filename)

        if not os.path.exists(processed_filename):
            self.log(logging.INFO, "artifact", {"filename": filename}, "Processing contents of {filename}")
            self.log(
                logging.INFO,
                "artifact",
                {"processed_filename": processed_filename},
                "Writing processed {processed_filename}",
            )
            self._artifact_job.process_artifact(filename, processed_filename)

        self.log(
            logging.INFO,
            "artifact",
            {"processed_filename": processed_filename},
            "Installing from processed {processed_filename}",
        )

        # Copy all .so files, avoiding modification where possible.
        ensureParentDir(mozpath.join(distdir, ".dummy"))

        with zipfile.ZipFile(processed_filename) as zf:
            for info in zf.infolist():
                if info.filename.endswith(".ini"):
                    continue
                n = mozpath.join(distdir, info.filename)
                fh = FileAvoidWrite(n, mode="rb")
                shutil.copyfileobj(zf.open(info), fh)
                file_existed, file_updated = fh.close()
                self.log(
                    logging.INFO,
                    "artifact",
                    {"updating": "Updating" if file_updated else "Not updating", "filename": n},
                    "{updating} {filename}",
                )
                if not file_existed or file_updated:
                    # Libraries and binaries may need to be marked executable,
                    # depending on platform.
                    perms = info.external_attr >> 16  # See http://stackoverflow.com/a/434689.
                    perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH  # u+w, a+r.
                    os.chmod(n, perms)
        return 0

 def add(self, path, content=None):
     # Put foo/qux files under $tmp/b.
     if path.startswith('foo/qux/'):
         real_path = mozpath.join('b', path[8:])
     else:
         real_path = mozpath.join('a', path)
     ensureParentDir(self.tmppath(real_path))
     if not content:
         content = path
     open(self.tmppath(real_path), 'wb').write(content)

 def test_composed_finder(self):
     self.prepare_match_test()
     # Also add files in $tmp/a/foo/qux because ComposedFinder is
     # expected to mask foo/qux entirely with content from $tmp/b.
     ensureParentDir(self.tmppath('a/foo/qux/hoge'))
     open(self.tmppath('a/foo/qux/hoge'), 'wb').write('hoge')
     open(self.tmppath('a/foo/qux/bar'), 'wb').write('not the right content')
     self.finder = ComposedFinder({
         '': FileFinder(self.tmppath('a')),
         'foo/qux': FileFinder(self.tmppath('b')),
     })
     self.do_match_test()

    def install_from_file(self, filename, distdir):
        self.log(logging.INFO, 'artifact',
            {'filename': filename},
            'Installing from {filename}')

        # Do we need to post-process?
        processed_filename = filename + PROCESSED_SUFFIX

        if self._skip_cache and os.path.exists(processed_filename):
            self.log(logging.DEBUG, 'artifact',
                {'path': processed_filename},
                'Skipping cache: removing cached processed artifact {path}')
            os.remove(processed_filename)

        if not os.path.exists(processed_filename):
            self.log(logging.INFO, 'artifact',
                {'filename': filename},
                'Processing contents of {filename}')
            self.log(logging.INFO, 'artifact',
                {'processed_filename': processed_filename},
                'Writing processed {processed_filename}')
            self._artifact_job.process_artifact(filename, processed_filename)

        self._artifact_cache._persist_limit.register_file(processed_filename)

        self.log(logging.INFO, 'artifact',
            {'processed_filename': processed_filename},
            'Installing from processed {processed_filename}')

        # Copy all .so files, avoiding modification where possible.
        ensureParentDir(mozpath.join(distdir, '.dummy'))

        with zipfile.ZipFile(processed_filename) as zf:
            for info in zf.infolist():
                if info.filename.endswith('.ini'):
                    continue
                n = mozpath.join(distdir, info.filename)
                fh = FileAvoidWrite(n, mode='rb')
                shutil.copyfileobj(zf.open(info), fh)
                file_existed, file_updated = fh.close()
                self.log(logging.INFO, 'artifact',
                    {'updating': 'Updating' if file_updated else 'Not updating', 'filename': n},
                    '{updating} {filename}')
                if not file_existed or file_updated:
                    # Libraries and binaries may need to be marked executable,
                    # depending on platform.
                    perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689.
                    perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r.
                    os.chmod(n, perms)
        return 0

def process_package_overload(src, dst, version, app_buildid):
    ensureParentDir(dst)
    # First replace numeric version like '1.3'
    # Then replace with 'slashed' version like '1_4'
    # Finally set the full length addon version like 1.3.20131230
    defines = {
        "NUM_VERSION": version,
        "SLASH_VERSION": version.replace(".", "_"),
        "FULL_VERSION": ("%s.%s" % (version, app_buildid))
    }
    pp = Preprocessor(defines=defines)
    pp.do_filter("substitution")
    with open(dst, "w") as output:
        with open(src, "r") as input:
            pp.processFile(input=input, output=output)

def _generate_geckoview_classes_jar(distdir, base_path):
    base_folder = FileFinder(base_path, ignore=['gecko-R.jar'])

    # Unzip all jar files into $(DISTDIR)/geckoview_aar_classes.
    geckoview_aar_classes_path = os.path.join(distdir, 'geckoview_aar_classes')
    shutil.rmtree(geckoview_aar_classes_path, ignore_errors=True)
    util.ensureParentDir(geckoview_aar_classes_path)

    for p, f in base_folder.find('*.jar'):
        with zipfile.ZipFile(f.path) as zf:
            zf.extractall(geckoview_aar_classes_path)

    # Rezip them into a single classes.jar file.
    classes_jar_path =  os.path.join(distdir, 'classes.jar')
    _zipdir(geckoview_aar_classes_path, classes_jar_path)
    return File(classes_jar_path)

def preprocess_file(src, dst, version, app_buildid, update_url):
    ensureParentDir(dst)

    defines = {
        "ADDON_ID": "fxos_" + version.replace(".", "_") + "[email protected]",
        # (reduce the app build id to only the build date
        # as addon manager doesn't handle big ints in addon versions)
        "ADDON_VERSION": ("%s.%s" % (version, app_buildid[:8])),
        "ADDON_NAME": "Firefox OS " + version + " Simulator",
        "ADDON_DESCRIPTION": "a Firefox OS " + version + " simulator",
        "ADDON_UPDATE_URL": update_url
    }
    pp = Preprocessor(defines=defines)
    pp.do_filter("substitution")
    with open(dst, "w") as output:
        with open(src, "r") as input:
            pp.processFile(input=input, output=output)

def addEntriesToListFile(listFile, entries):
  """Given a file |listFile| containing one entry per line,
  add each entry in |entries| to the file, unless it is already
  present."""
  ensureParentDir(listFile)
  lock = lock_file(listFile + ".lck")
  try:
    if os.path.exists(listFile):
      f = open(listFile)
      existing = set(x.strip() for x in f.readlines())
      f.close()
    else:
      existing = set()
    for e in entries:
      if e not in existing:
        existing.add(e)
    with open(listFile, 'wb') as f:
      f.write("\n".join(sorted(existing))+"\n")
  finally:
    lock = None

    def install_from_file(self, filename, distdir):
        self.log(logging.INFO, 'artifact',
            {'filename': filename},
            'Installing from {filename}')

        # Copy all .so files to dist/bin, avoiding modification where possible.
        ensureParentDir(os.path.join(distdir, 'bin', '.dummy'))

        with zipfile.ZipFile(filename) as zf:
            for info in zf.infolist():
                if not info.filename.endswith('.so'):
                    continue
                n = os.path.join(distdir, 'bin', os.path.basename(info.filename))
                fh = FileAvoidWrite(n, mode='r')
                shutil.copyfileobj(zf.open(info), fh)
                file_existed, file_updated = fh.close()
                self.log(logging.INFO, 'artifact',
                    {'updating': 'Updating' if file_updated else 'Not updating', 'filename': n},
                    '{updating} {filename}')
        return 0

    def test_faster_recursive_make(self):
        substs = list(BASE_SUBSTS) + [
            ('BUILD_BACKENDS', 'FasterMake+RecursiveMake'),
        ]
        with self.do_test_backend(get_backend_class(
                'FasterMake+RecursiveMake'), substs=substs) as config:
            buildid = mozpath.join(config.topobjdir, 'config', 'buildid')
            ensureParentDir(buildid)
            with open(buildid, 'w') as fh:
                fh.write('20100101012345\n')

            build = MozbuildObject(config.topsrcdir, None, None,
                                   config.topobjdir)
            overrides = [
                'install_manifest_depends=',
                'MOZ_JAR_MAKER_FILE_FORMAT=flat',
                'TEST_MOZBUILD=1',
            ]
            with self.line_handler() as handle_make_line:
                build._run_make(directory=config.topobjdir, target=overrides,
                                silent=False, line_handler=handle_make_line)

            self.validate(config)

    def test_faster_make(self):
        substs = list(BASE_SUBSTS) + [
            ('MOZ_BUILD_APP', 'dummy_app'),
            ('MOZ_WIDGET_TOOLKIT', 'dummy_widget'),
        ]
        with self.do_test_backend(RecursiveMakeBackend, FasterMakeBackend,
                                  substs=substs) as config:
            buildid = mozpath.join(config.topobjdir, 'config', 'buildid')
            ensureParentDir(buildid)
            with open(buildid, 'w') as fh:
                fh.write('20100101012345\n')

            build = MozbuildObject(config.topsrcdir, None, None,
                                   config.topobjdir)
            overrides = [
                'TEST_MOZBUILD=1',
            ]
            with self.line_handler() as handle_make_line:
                build._run_make(directory=mozpath.join(config.topobjdir,
                                                       'faster'),
                                target=overrides, silent=False,
                                line_handler=handle_make_line)

            self.validate(config)

 def add(self, path):
     with self._client() as c:
         ensureParentDir(self.tmppath(path))
         with open(self.tmppath(path), 'wb') as fh:
             fh.write(path)
         c.add(self.tmppath(path))

 def create_one(self, which, path, content):
     file = self.tmppath(os.path.join(which, path))
     ensureParentDir(file)
     open(file, 'wb').write(content)

 def dump_cache(self):
     ensureParentDir(self._cache_filename)
     pickle.dump(list(reversed(list(self._cache.items()))), open(self._cache_filename, 'wb'), -1)