def __init__(self, topsrcdir, topobjdir, dist, group=Grouping.NO,
                 abspaths=False):
        topsrcdir = mozpath.normsep(os.path.normcase(os.path.abspath(topsrcdir)))
        topobjdir = mozpath.normsep(os.path.normcase(os.path.abspath(topobjdir)))
        dist = mozpath.normsep(os.path.normcase(os.path.abspath(dist)))
        if abspaths:
            topsrcdir_value = topsrcdir
            topobjdir_value = topobjdir
            dist_value = dist
        else:
            topsrcdir_value = '$(topsrcdir)'
            topobjdir_value = '$(DEPTH)'
            dist_value = '$(DIST)'

        self._normpaths = {
            topsrcdir: topsrcdir_value,
            topobjdir: topobjdir_value,
            dist: dist_value,
            '$(topsrcdir)': topsrcdir_value,
            '$(DEPTH)': topobjdir_value,
            '$(DIST)': dist_value,
            '$(depth)': topobjdir_value, # normcase may lowercase variable refs when
            '$(dist)': dist_value,       # they are in the original dependency file
            mozpath.relpath(topsrcdir, os.curdir): topsrcdir_value,
            mozpath.relpath(topobjdir, os.curdir): topobjdir_value,
            mozpath.relpath(dist, os.curdir): dist_value,
        }

        Makefile.__init__(self)
        self._group = group
        self._targets = OrderedDict()

    def consume_object(self, obj):
        if isinstance(obj, JARManifest) and \
                obj.install_target.startswith('dist/bin'):
            self._consume_jar_manifest(obj)

        elif isinstance(obj, (FinalTargetFiles,
                              FinalTargetPreprocessedFiles)) and \
                obj.install_target.startswith('dist/bin'):
            defines = obj.defines or {}
            if defines:
                defines = defines.defines
            for path, files in obj.files.walk():
                for f in files:
                    if isinstance(obj, FinalTargetPreprocessedFiles):
                        self._add_preprocess(obj, f.full_path, path,
                                             target=f.target_basename,
                                             defines=defines)
                    elif '*' in f:
                        def _prefix(s):
                            for p in mozpath.split(s):
                                if '*' not in p:
                                    yield p + '/'
                        prefix = ''.join(_prefix(f.full_path))

                        self._install_manifests[obj.install_target] \
                            .add_pattern_symlink(
                                prefix,
                                f.full_path[len(prefix):],
                                mozpath.join(path, f.target_basename))
                    else:
                        self._install_manifests[obj.install_target].add_symlink(
                            f.full_path,
                            mozpath.join(path, f.target_basename)
                        )
                    if isinstance(f, ObjDirPath):
                        dep_target = 'install-%s' % obj.install_target
                        self._dependencies[dep_target].append(
                            mozpath.relpath(f.full_path,
                                            self.environment.topobjdir))

        elif isinstance(obj, ChromeManifestEntry) and \
                obj.install_target.startswith('dist/bin'):
            top_level = mozpath.join(obj.install_target, 'chrome.manifest')
            if obj.path != top_level:
                entry = 'manifest %s' % mozpath.relpath(obj.path,
                                                        obj.install_target)
                self._manifest_entries[top_level].add(entry)
            self._manifest_entries[obj.path].add(str(obj.entry))

        elif isinstance(obj, XPIDLFile):
            self._has_xpidl = True
            # We're not actually handling XPIDL files.
            return False

        else:
            return False

        return True

 def _process_library_definition(self, libdef, backend_file):
     backend_file.write("LIBRARY_NAME = %s\n" % libdef.basename)
     thisobjdir = libdef.objdir
     topobjdir = libdef.topobjdir.replace(os.sep, "/")
     for objdir, basename in libdef.static_libraries:
         # If this is an external objdir (i.e., comm-central), use the other
         # directory instead of $(DEPTH).
         if objdir.startswith(topobjdir + "/"):
             relpath = "$(DEPTH)/%s" % mozpath.relpath(objdir, topobjdir)
         else:
             relpath = mozpath.relpath(objdir, thisobjdir)
         backend_file.write("SHARED_LIBRARY_LIBS += %s/$(LIB_PREFIX)%s.$(LIB_SUFFIX)\n" % (relpath, basename))

 def add(self, path, content):
     chrome = self._chromepath(path)
     if chrome:
         jar = chrome + '.jar'
         if not self.copier.contains(jar):
             self.copier.add(jar, Jarrer(self._compress, self._optimize))
         if not self.copier[jar].contains(mozpath.relpath(path,
                                                               chrome)):
             self.copier[jar].add(mozpath.relpath(path, chrome),
                                  content)
     else:
         FlatFormatter.add(self, path, content)

    def process_tests_artifact(self, filename, processed_filename):
        from mozbuild.action.test_archive import OBJDIR_TEST_FILES
        added_entry = False

        with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
            reader = JarReader(filename)
            for filename, entry in reader.entries.iteritems():
                for pattern, (src_prefix, dest_prefix) in self.test_artifact_patterns:
                    if not mozpath.match(filename, pattern):
                        continue
                    destpath = mozpath.relpath(filename, src_prefix)
                    destpath = mozpath.join(dest_prefix, destpath)
                    self.log(logging.INFO, 'artifact',
                             {'destpath': destpath},
                             'Adding {destpath} to processed archive')
                    mode = entry['external_attr'] >> 16
                    writer.add(destpath.encode('utf-8'), reader[filename], mode=mode)
                    added_entry = True
                    break
                for files_entry in OBJDIR_TEST_FILES.values():
                    origin_pattern = files_entry['pattern']
                    leaf_filename = filename
                    if 'dest' in files_entry:
                        dest = files_entry['dest']
                        origin_pattern = mozpath.join(dest, origin_pattern)
                        leaf_filename = filename[len(dest) + 1:]
                    if mozpath.match(filename, origin_pattern):
                        destpath = mozpath.join('..', files_entry['base'], leaf_filename)
                        mode = entry['external_attr'] >> 16
                        writer.add(destpath.encode('utf-8'), reader[filename], mode=mode)

        if not added_entry:
            raise ValueError('Archive format changed! No pattern from "{patterns}"'
                             'matched an archive path.'.format(
                                 patterns=LinuxArtifactJob.test_artifact_patterns))

    def _write_file(self, path=None, fh=None):
        """Context manager to write a file.

        This is a glorified wrapper around FileAvoidWrite with integration to
        update the BackendConsumeSummary on this instance.

        Example usage:

            with self._write_file('foo.txt') as fh:
                fh.write('hello world')
        """

        if path is not None:
            assert fh is None
            fh = FileAvoidWrite(path)
        else:
            assert fh is not None

        dirname = mozpath.dirname(fh.name)
        try:
            os.makedirs(dirname)
        except OSError as error:
            if error.errno != errno.EEXIST:
                raise

        yield fh

        self._backend_output_files.add(mozpath.relpath(fh.name, self.environment.topobjdir))
        existed, updated = fh.close()
        if not existed:
            self.summary.created_count += 1
        elif updated:
            self.summary.updated_count += 1
        else:
            self.summary.unchanged_count += 1

    def process_package_artifact(self, filename, processed_filename):
        added_entry = False

        with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
            with tarfile.open(filename) as reader:
                for f in reader:
                    if not f.isfile():
                        continue

                    if not any(mozpath.match(f.name, p) for p in self.package_artifact_patterns):
                        continue

                    # We strip off the relative "firefox/" bit from the path,
                    # but otherwise preserve it.
                    destpath = mozpath.join('bin',
                                            mozpath.relpath(f.name, "firefox"))
                    self.log(logging.INFO, 'artifact',
                             {'destpath': destpath},
                             'Adding {destpath} to processed archive')
                    writer.add(destpath.encode('utf-8'), reader.extractfile(f), mode=f.mode)
                    added_entry = True

        if not added_entry:
            raise ValueError('Archive format changed! No pattern from "{patterns}" '
                             'matched an archive path.'.format(
                                 patterns=LinuxArtifactJob.package_artifact_patterns))

 def all_files(self, base):
     all_files = set()
     for root, dirs, files in os.walk(base):
         for f in files:
             all_files.add(
                 mozpath.join(mozpath.relpath(root, base), f))
     return all_files

 def normalize_path(path):
     '''
     Remove any bin/ prefix.
     '''
     if mozpath.basedir(path, ['bin']) == 'bin':
         return mozpath.relpath(path, 'bin')
     return path

 def is_resource(self, path, base=None):
     '''
     Return whether the given path corresponds to a resource to be put in an
     omnijar archive.
     '''
     if base is None:
         base = self._get_base(path)
     path = mozpath.relpath(path, base)
     if any(mozpath.match(path, p.replace('*', '**'))
            for p in self._non_resources):
         return False
     path = mozpath.split(path)
     if path[0] == 'chrome':
         return len(path) == 1 or path[1] != 'icons'
     if path[0] == 'components':
         return path[-1].endswith(('.js', '.xpt'))
     if path[0] == 'res':
         return len(path) == 1 or \
             (path[1] != 'cursors' and path[1] != 'MainMenu.nib')
     if path[0] == 'defaults':
         return len(path) != 3 or \
             not (path[2] == 'channel-prefs.js' and
                  path[1] in ['pref', 'preferences'])
     return path[0] in [
         'modules',
         'greprefs.js',
         'hyphenation',
         'update.locale',
     ] or path[0] in STARTUP_CACHE_PATHS

    def _get_files_info(self, paths):
        from mozpack.files import FileFinder

        # Normalize to relative from topsrcdir.
        relpaths = []
        for p in paths:
            a = mozpath.abspath(p)
            if not mozpath.basedir(a, [self.topsrcdir]):
                raise InvalidPathException('path is outside topsrcdir: %s' % p)

            relpaths.append(mozpath.relpath(a, self.topsrcdir))

        finder = FileFinder(self.topsrcdir, find_executables=False)

        # Expand wildcards.
        allpaths = []
        for p in relpaths:
            if '*' not in p:
                if p not in allpaths:
                    allpaths.append(p)
                continue

            for path, f in finder.find(p):
                if path not in allpaths:
                    allpaths.append(path)

        reader = self._get_reader()
        return reader.files_info(allpaths)

 def test_relpath(self):
     self.assertEqual(relpath("foo", "foo"), "")
     self.assertEqual(relpath(os.path.join("foo", "bar"), "foo/bar"), "")
     self.assertEqual(relpath(os.path.join("foo", "bar"), "foo"), "bar")
     self.assertEqual(relpath(os.path.join("foo", "bar", "baz"), "foo"), "bar/baz")
     self.assertEqual(relpath(os.path.join("foo", "bar"), "foo/bar/baz"), "..")
     self.assertEqual(relpath(os.path.join("foo", "bar"), "foo/baz"), "../bar")
     self.assertEqual(relpath("foo/", "foo"), "")
     self.assertEqual(relpath("foo/bar/", "foo"), "bar")

    def consume_object(self, obj):
        if isinstance(obj, JARManifest) and obj.install_target.startswith("dist/bin"):
            self._consume_jar_manifest(obj)

        elif isinstance(obj, (FinalTargetFiles, FinalTargetPreprocessedFiles)) and obj.install_target.startswith(
            "dist/bin"
        ):
            defines = obj.defines or {}
            if defines:
                defines = defines.defines
            for path, files in obj.files.walk():
                for f in files:
                    if isinstance(obj, FinalTargetPreprocessedFiles):
                        self._add_preprocess(obj, f.full_path, path, target=f.target_basename, defines=defines)
                    elif "*" in f:

                        def _prefix(s):
                            for p in mozpath.split(s):
                                if "*" not in p:
                                    yield p + "/"

                        prefix = "".join(_prefix(f.full_path))

                        self._install_manifests[obj.install_target].add_pattern_symlink(
                            prefix, f.full_path[len(prefix) :], mozpath.join(path, f.target_basename)
                        )
                    else:
                        self._install_manifests[obj.install_target].add_symlink(
                            f.full_path, mozpath.join(path, f.target_basename)
                        )
                    if isinstance(f, ObjDirPath):
                        dep_target = "install-%s" % obj.install_target
                        self._dependencies[dep_target].append(mozpath.relpath(f.full_path, self.environment.topobjdir))

        elif isinstance(obj, ChromeManifestEntry) and obj.install_target.startswith("dist/bin"):
            top_level = mozpath.join(obj.install_target, "chrome.manifest")
            if obj.path != top_level:
                entry = "manifest %s" % mozpath.relpath(obj.path, obj.install_target)
                self._manifest_entries[top_level].add(entry)
            self._manifest_entries[obj.path].add(str(obj.entry))

        elif isinstance(obj, XPIDLFile):
            self._has_xpidl = True

        # We currently ignore a lot of object types, so just acknowledge
        # everything.
        return True

 def add_manifest(self, entry):
     if isinstance(entry, ManifestBinaryComponent):
         formatter, base = super(OmniJarFormatter, self), ''
     else:
         formatter, base, path = self._get_formatter(entry.base,
                                                     is_resource=True)
     entry = entry.move(mozpath.relpath(entry.base, base))
     formatter.add_manifest(entry)

 def contains(self, path):
     assert '*' not in path
     if self.copier.contains(path):
         return True
     for base, copier in self.omnijars.iteritems():
         if copier.contains(mozpath.relpath(path, base)):
             return True
     return False

 def _get_base(self, path):
     '''
     Return the deepest base directory containing the given path.
     '''
     self._frozen_bases = True
     base = mozpath.basedir(path, self._sub_formatter.keys())
     relpath = mozpath.relpath(path, base) if base else path
     return base, relpath

def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
    """Installs the requested test files to the objdir. This is invoked by
    test runners to avoid installing tens of thousands of test files when
    only a few tests need to be run.
    """
    flavor_info = {flavor: (root, prefix, install)
                   for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
    objdir_dest = mozpath.join(topobjdir, tests_root)

    converter = SupportFilesConverter()
    install_info = TestInstallInfo()
    for o in test_objs:
        flavor = o['flavor']
        if flavor not in flavor_info:
            # This is a test flavor that isn't installed by the build system.
            continue
        root, prefix, install = flavor_info[flavor]
        if not install:
            # This flavor isn't installed to the objdir.
            continue

        manifest_path = o['manifest']
        manifest_dir = mozpath.dirname(manifest_path)

        out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
        file_relpath = o['file_relpath']
        source = mozpath.join(topsrcdir, file_relpath)
        dest = mozpath.join(root, prefix, file_relpath)
        if 'install-to-subdir' in o:
            out_dir = mozpath.join(out_dir, o['install-to-subdir'])
            manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
            dest = mozpath.join(out_dir, manifest_relpath)

        install_info.installs.append((source, dest))
        install_info |= converter.convert_support_files(o, root,
                                                        manifest_dir,
                                                        out_dir)

    manifest = InstallManifest()

    for source, dest in set(install_info.installs):
        if dest in install_info.external_installs:
            continue
        manifest.add_symlink(source, dest)
    for base, pattern, dest in install_info.pattern_installs:
        manifest.add_pattern_symlink(base, pattern, dest)

    _resolve_installs(install_info.deferred_installs, topobjdir, manifest)

    # Harness files are treated as a monolith and installed each time we run tests.
    # Fortunately there are not very many.
    manifest |= InstallManifest(mozpath.join(topobjdir,
                                             '_build_manifests',
                                             'install', tests_root))
    copier = FileCopier()
    manifest.populate_registry(copier)
    copier.copy(objdir_dest,
                remove_unaccounted=False)

 def _get_preprocessor(self, obj):
     '''Returns a preprocessor with a few predefined values depending on
     the given BaseConfigSubstitution(-like) object, and all the substs
     in the current environment.'''
     pp = Preprocessor()
     srcdir = mozpath.dirname(obj.input_path)
     pp.context.update(obj.config.substs)
     pp.context.update(
         top_srcdir=obj.topsrcdir,
         srcdir=srcdir,
         relativesrcdir=mozpath.relpath(srcdir, obj.topsrcdir) or '.',
         DEPTH=mozpath.relpath(obj.topobjdir, mozpath.dirname(obj.output_path)) or '.',
     )
     pp.do_filter('attemptSubstitution')
     pp.setMarker(None)
     with self._write_file(obj.output_path) as fh:
         pp.out = fh
         yield pp

 def _handle_webidl_build(self, bindings_dir, unified_source_mapping,
                          webidls, expected_build_output_files,
                          global_define_files):
     include_dir = mozpath.join(self.environment.topobjdir, 'dist',
         'include')
     for f in expected_build_output_files:
         if f.startswith(include_dir):
             manifest, reltarget = self._get_manifest_from_target('dist/include')
             manifest.add_optional_exists(mozpath.join(reltarget, mozpath.relpath(f, include_dir)))

    def consume_object(self, obj):
        if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
            defines = self._defines.get(obj.objdir, {})
            if defines:
                defines = defines.defines

        if isinstance(obj, Defines):
            self._defines[obj.objdir] = obj

            # We're assuming below that Defines come first for a given objdir,
            # which is kind of set in stone from the order things are treated
            # in emitter.py.
            assert obj.objdir not in self._seen_directories

        elif isinstance(obj, JARManifest) and \
                obj.install_target.startswith('dist/bin'):
            self._consume_jar_manifest(obj, defines)

        elif isinstance(obj, (FinalTargetFiles,
                              FinalTargetPreprocessedFiles)) and \
                obj.install_target.startswith('dist/bin'):
            for path, files in obj.files.walk():
                for f in files:
                    if isinstance(obj, FinalTargetPreprocessedFiles):
                        self._add_preprocess(obj, f.full_path, path,
                                             defines=defines)
                    else:
                        self._install_manifests[obj.install_target].add_symlink(
                            f.full_path,
                            mozpath.join(path, mozpath.basename(f))
                        )

        elif isinstance(obj, ChromeManifestEntry) and \
                obj.install_target.startswith('dist/bin'):
            top_level = mozpath.join(obj.install_target, 'chrome.manifest')
            if obj.path != top_level:
                entry = 'manifest %s' % mozpath.relpath(obj.path,
                                                        obj.install_target)
                if entry not in self._manifest_entries[top_level]:
                    self._manifest_entries[top_level].append(entry)
            self._manifest_entries[obj.path].append(str(obj.entry))

        elif isinstance(obj, XPIDLFile):
            self._has_xpidl = True
            # XPIDL are emitted before Defines, which breaks the assert in the
            # branch for Defines. OTOH, we don't actually care about the
            # XPIDLFile objects just yet, so we can just pretend we didn't see
            # an object in the directory yet.
            return True

        else:
            # We currently ignore a lot of object types, so just acknowledge
            # everything.
            return True

        self._seen_directories.add(obj.objdir)
        return True