def _synchronize_docs(self):
        m = InstallManifest()

        m.add_symlink(self._conf_py_path, 'conf.py')

        for dest, source in sorted(self._trees.items()):
            source_dir = os.path.join(self._topsrcdir, source)
            for root, dirs, files in os.walk(source_dir):
                for f in files:
                    source_path = os.path.join(root, f)
                    rel_source = source_path[len(source_dir) + 1:]

                    m.add_symlink(source_path, os.path.join(dest, rel_source))

        stage_dir = os.path.join(self._output_dir, 'staging')
        copier = FileCopier()
        m.populate_registry(copier)
        copier.copy(stage_dir)

        with open(self._index_path, 'rb') as fh:
            data = fh.read()

        indexes = ['%s/index' % p for p in sorted(self._trees.keys())]
        indexes = '\n   '.join(indexes)

        packages = [os.path.basename(p) for p in self._python_package_dirs]
        packages = ['python/%s' % p for p in packages]
        packages = '\n   '.join(sorted(packages))
        data = data.format(indexes=indexes, python_packages=packages)

        with open(os.path.join(stage_dir, 'index.rst'), 'wb') as fh:
            fh.write(data)

    def _synchronize_docs(self):
        m = InstallManifest()

        m.add_symlink(self._conf_py_path, "conf.py")

        for dest, source in sorted(self._trees.items()):
            source_dir = os.path.join(self._topsrcdir, source)
            for root, dirs, files in os.walk(source_dir):
                for f in files:
                    source_path = os.path.join(root, f)
                    rel_source = source_path[len(source_dir) + 1 :]

                    m.add_symlink(source_path, os.path.join(dest, rel_source))

        copier = FileCopier()
        m.populate_registry(copier)
        copier.copy(self._docs_dir)

        with open(self._index_path, "rb") as fh:
            data = fh.read()

        indexes = ["%s/index" % p for p in sorted(self._trees.keys())]
        indexes = "\n   ".join(indexes)

        packages = [os.path.basename(p) for p in self._python_package_dirs]
        packages = ["python/%s" % p for p in packages]
        packages = "\n   ".join(sorted(packages))
        data = data.format(indexes=indexes, python_packages=packages)

        with open(os.path.join(self._docs_dir, "index.rst"), "wb") as fh:
            fh.write(data)

def unpack(source):
    '''
    Transform a jar chrome or omnijar packaged directory into a flat package.
    '''
    copier = FileCopier()
    unpack_to_registry(source, copier)
    copier.copy(source, skip_if_older=False)

    def test_symlink_directory_replaced(self):
        """Directory symlinks in destination are replaced if they need to be
        real directories."""
        if not self.symlink_supported:
            return

        dest = self.tmppath('dest')

        copier = FileCopier()
        copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))

        os.makedirs(self.tmppath('dest/foo'))
        dummy = self.tmppath('dummy')
        os.mkdir(dummy)
        link = self.tmppath('dest/foo/bar')
        os.symlink(dummy, link)

        result = copier.copy(dest)

        st = os.lstat(link)
        self.assertFalse(stat.S_ISLNK(st.st_mode))
        self.assertTrue(stat.S_ISDIR(st.st_mode))

        self.assertEqual(self.all_files(dest), set(copier.paths()))

        self.assertEqual(result.removed_directories, set())
        self.assertEqual(len(result.updated_files), 1)

    def test_optional_exists_creates_unneeded_directory(self):
        """Demonstrate that a directory not strictly required, but specified
        as the path to an optional file, will be unnecessarily created.

        This behaviour is wrong; fixing it is tracked by Bug 972432;
        and this test exists to guard against unexpected changes in
        behaviour.
        """

        dest = self.tmppath('dest')

        copier = FileCopier()
        copier.add('foo/bar', ExistingFile(required=False))

        result = copier.copy(dest)

        st = os.lstat(self.tmppath('dest/foo'))
        self.assertFalse(stat.S_ISLNK(st.st_mode))
        self.assertTrue(stat.S_ISDIR(st.st_mode))

        # What's worse, we have no record that dest was created.
        self.assertEquals(len(result.updated_files), 0)

        # But we do have an erroneous record of an optional file
        # existing when it does not.
        self.assertIn(self.tmppath('dest/foo/bar'), result.existing_files)

    def output_changes(self, verbose=True):
        '''
        Return an iterator of `FasterBuildChange` instances as outputs
        from the faster build system are updated.
        '''
        for change in self.input_changes(verbose=verbose):
            now = datetime.datetime.utcnow()

            for unrecognized in sorted(change.unrecognized):
                print_line('watch', '! {}'.format(unrecognized), now=now)

            all_outputs = set()
            for input in sorted(change.input_to_outputs):
                outputs = change.input_to_outputs[input]

                print_line('watch', '< {}'.format(input), now=now)
                for output in sorted(outputs):
                    print_line('watch', '> {}'.format(output), now=now)
                all_outputs |= outputs

            if all_outputs:
                partial_copier = FileCopier()
                for output in all_outputs:
                    partial_copier.add(output, self.file_copier[output])

                self.incremental_copy(partial_copier, force=True, verbose=verbose)
                yield change

def process_manifest(destdir, paths, remove_unaccounted=True):
    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier)
    return copier.copy(destdir, remove_unaccounted=remove_unaccounted)

def strip(dir):
    copier = FileCopier()
    # The FileFinder will give use ExecutableFile instances for files
    # that can be stripped, and copying ExecutableFiles defaults to
    # stripping them unless buildconfig.substs['PKG_SKIP_STRIP'] is set.
    for p, f in FileFinder(dir):
        copier.add(p, f)
    copier.copy(dir)

def process_manifest(destdir, *paths):
    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier)
    return copier.copy(destdir)

def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
    """Installs the requested test files to the objdir. This is invoked by
    test runners to avoid installing tens of thousands of test files when
    only a few tests need to be run.
    """
    flavor_info = {flavor: (root, prefix, install)
                   for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
    objdir_dest = mozpath.join(topobjdir, tests_root)

    converter = SupportFilesConverter()
    install_info = TestInstallInfo()
    for o in test_objs:
        flavor = o['flavor']
        if flavor not in flavor_info:
            # This is a test flavor that isn't installed by the build system.
            continue
        root, prefix, install = flavor_info[flavor]
        if not install:
            # This flavor isn't installed to the objdir.
            continue

        manifest_path = o['manifest']
        manifest_dir = mozpath.dirname(manifest_path)

        out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
        file_relpath = o['file_relpath']
        source = mozpath.join(topsrcdir, file_relpath)
        dest = mozpath.join(root, prefix, file_relpath)
        if 'install-to-subdir' in o:
            out_dir = mozpath.join(out_dir, o['install-to-subdir'])
            manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
            dest = mozpath.join(out_dir, manifest_relpath)

        install_info.installs.append((source, dest))
        install_info |= converter.convert_support_files(o, root,
                                                        manifest_dir,
                                                        out_dir)

    manifest = InstallManifest()

    for source, dest in set(install_info.installs):
        if dest in install_info.external_installs:
            continue
        manifest.add_symlink(source, dest)
    for base, pattern, dest in install_info.pattern_installs:
        manifest.add_pattern_symlink(base, pattern, dest)

    _resolve_installs(install_info.deferred_installs, topobjdir, manifest)

    # Harness files are treated as a monolith and installed each time we run tests.
    # Fortunately there are not very many.
    manifest |= InstallManifest(mozpath.join(topobjdir,
                                             '_build_manifests',
                                             'install', tests_root))
    copier = FileCopier()
    manifest.populate_registry(copier)
    copier.copy(objdir_dest,
                remove_unaccounted=False)

def unpack(source):
    '''
    Transform a jar chrome or omnijar packaged directory into a flat package.
    '''
    copier = FileCopier()
    finder = UnpackFinder(source)
    packager = SimplePackager(FlatFormatter(copier))
    for p, f in finder.find('*'):
        if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS:
            packager.add(p, f)
    packager.close()
    copier.copy(source, skip_if_older=False)

    def test_copier_application(self):
        dest = self.tmppath('dest')
        os.mkdir(dest)

        to_delete = self.tmppath('dest/to_delete')
        with open(to_delete, 'a'):
            pass

        with open(self.tmppath('s_source'), 'wt') as fh:
            fh.write('symlink!')

        with open(self.tmppath('c_source'), 'wt') as fh:
            fh.write('copy!')

        with open(self.tmppath('p_source'), 'wt') as fh:
            fh.write('#define FOO 1\npreprocess!')

        with open(self.tmppath('dest/e_dest'), 'a'):
            pass

        with open(self.tmppath('dest/o_dest'), 'a'):
            pass

        m = self._get_test_manifest()
        c = FileCopier()
        m.populate_registry(c)
        result = c.copy(dest)

        self.assertTrue(os.path.exists(self.tmppath('dest/s_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/c_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/p_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/e_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/o_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/content')))
        self.assertFalse(os.path.exists(to_delete))

        with open(self.tmppath('dest/s_dest'), 'rt') as fh:
            self.assertEqual(fh.read(), 'symlink!')

        with open(self.tmppath('dest/c_dest'), 'rt') as fh:
            self.assertEqual(fh.read(), 'copy!')

        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
            self.assertEqual(fh.read(), 'preprocess!')

        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in (
            'dest/s_dest', 'dest/c_dest', 'dest/p_dest', 'dest/content')))
        self.assertEqual(result.existing_files,
            set([self.tmppath('dest/e_dest'), self.tmppath('dest/o_dest')]))
        self.assertEqual(result.removed_files, {to_delete})
        self.assertEqual(result.removed_directories, set())

    def test_copier_application(self):
        dest = self.tmppath("dest")
        os.mkdir(dest)

        to_delete = self.tmppath("dest/to_delete")
        with open(to_delete, "a"):
            pass

        with open(self.tmppath("s_source"), "wt") as fh:
            fh.write("symlink!")

        with open(self.tmppath("c_source"), "wt") as fh:
            fh.write("copy!")

        with open(self.tmppath("p_source"), "wt") as fh:
            fh.write("#define FOO 1\npreprocess!")

        with open(self.tmppath("dest/e_dest"), "a"):
            pass

        with open(self.tmppath("dest/o_dest"), "a"):
            pass

        m = self._get_test_manifest()
        c = FileCopier()
        m.populate_registry(c)
        result = c.copy(dest)

        self.assertTrue(os.path.exists(self.tmppath("dest/s_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/c_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/p_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/e_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/o_dest")))
        self.assertFalse(os.path.exists(to_delete))

        with open(self.tmppath("dest/s_dest"), "rt") as fh:
            self.assertEqual(fh.read(), "symlink!")

        with open(self.tmppath("dest/c_dest"), "rt") as fh:
            self.assertEqual(fh.read(), "copy!")

        with open(self.tmppath("dest/p_dest"), "rt") as fh:
            self.assertEqual(fh.read(), "preprocess!")

        self.assertEqual(
            result.updated_files, set(self.tmppath(p) for p in ("dest/s_dest", "dest/c_dest", "dest/p_dest"))
        )
        self.assertEqual(result.existing_files, set([self.tmppath("dest/e_dest"), self.tmppath("dest/o_dest")]))
        self.assertEqual(result.removed_files, {to_delete})
        self.assertEqual(result.removed_directories, set())

def process_manifest(destdir, paths,
        remove_unaccounted=True,
        remove_all_directory_symlinks=True,
        remove_empty_directories=True):
    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier)
    return copier.copy(destdir,
        remove_unaccounted=remove_unaccounted,
        remove_all_directory_symlinks=remove_all_directory_symlinks,
        remove_empty_directories=remove_empty_directories)

    def test_no_remove(self):
        copier = FileCopier()
        copier.add('foo', GeneratedFile('foo'))

        with open(self.tmppath('bar'), 'a'):
            pass

        os.mkdir(self.tmppath('emptydir'))

        result = copier.copy(self.tmpdir, remove_unaccounted=False)

        self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar']))
        self.assertEqual(result.removed_files, set())
        self.assertEqual(result.removed_directories,
            set([self.tmppath('emptydir')]))

def process_manifest(
    destdir,
    paths,
    track=None,
    remove_unaccounted=True,
    remove_all_directory_symlinks=True,
    remove_empty_directories=True,
    defines={},
):

    if track:
        if os.path.exists(track):
            # We use the same format as install manifests for the tracking
            # data.
            manifest = InstallManifest(path=track)
            remove_unaccounted = FileRegistry()
            dummy_file = BaseFile()

            finder = FileFinder(destdir, find_executables=False, find_dotfiles=True)
            for dest in manifest._dests:
                for p, f in finder.find(dest):
                    remove_unaccounted.add(p, dummy_file)

        else:
            # If tracking is enabled and there is no file, we don't want to
            # be removing anything.
            remove_unaccounted = False
            remove_empty_directories = False
            remove_all_directory_symlinks = False

    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier, defines_override=defines)
    result = copier.copy(
        destdir,
        remove_unaccounted=remove_unaccounted,
        remove_all_directory_symlinks=remove_all_directory_symlinks,
        remove_empty_directories=remove_empty_directories,
    )

    if track:
        manifest.write(path=track)

    return result

def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
    '''
    Replace localized data from the `source` directory with localized data
    from `l10n` and `extra_l10n`.

    The `source` argument points to a directory containing a packaged
    application (in omnijar, jar or flat form).
    The `l10n` argument points to a directory containing the main localized
    data (usually in the form of a language pack addon) to use to replace
    in the packaged application.
    The `extra_l10n` argument contains a dict associating relative paths in
    the source to separate directories containing localized data for them.
    This can be used to point at different language pack addons for different
    parts of the package application.
    The `non_resources` argument gives a list of relative paths in the source
    that should not be added in an omnijar in case the packaged application
    is in that format.
    The `non_chrome` argument gives a list of file/directory patterns for
    localized files that are not listed in a chrome.manifest.
    '''
    app_finder = UnpackFinder(source)
    l10n_finder = UnpackFinder(l10n)
    if extra_l10n:
        finders = {
            '': l10n_finder,
        }
        for base, path in extra_l10n.iteritems():
            finders[base] = UnpackFinder(path)
        l10n_finder = ComposedFinder(finders)
    copier = FileCopier()
    compress = min(app_finder.compressed, JAR_DEFLATED)
    if app_finder.kind == 'flat':
        formatter = FlatFormatter(copier)
    elif app_finder.kind == 'jar':
        formatter = JarFormatter(copier,
                                 optimize=app_finder.optimizedjars,
                                 compress=compress)
    elif app_finder.kind == 'omni':
        formatter = OmniJarFormatter(copier, app_finder.omnijar,
                                     optimize=app_finder.optimizedjars,
                                     compress=compress,
                                     non_resources=non_resources)

    with errors.accumulate():
        _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
    copier.copy(source, skip_if_older=False)
    generate_precomplete(source)

def repack(source, l10n, non_resources=[], non_chrome=set()):
    app_finder = UnpackFinder(source)
    l10n_finder = UnpackFinder(l10n)
    copier = FileCopier()
    if app_finder.kind == 'flat':
        formatter = FlatFormatter(copier)
    elif app_finder.kind == 'jar':
        formatter = JarFormatter(copier, optimize=app_finder.optimizedjars)
    elif app_finder.kind == 'omni':
        formatter = OmniJarFormatter(copier, app_finder.omnijar,
                                     optimize=app_finder.optimizedjars,
                                     non_resources=non_resources)

    with errors.accumulate():
        _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
    copier.copy(source, skip_if_older=False)
    generate_precomplete(source)

    def test_pattern_expansion(self):
        source = self.tmppath('source')
        os.mkdir(source)
        os.mkdir('%s/base' % source)
        os.mkdir('%s/base/foo' % source)

        with open('%s/base/foo/file1' % source, 'a'):
            pass

        with open('%s/base/foo/file2' % source, 'a'):
            pass

        m = InstallManifest()
        m.add_pattern_symlink('%s/base' % source, '**', 'dest')

        c = FileCopier()
        m.populate_registry(c)
        self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])

    def test_permissions(self):
        """Ensure files without write permission can be deleted."""
        with open(self.tmppath('dummy'), 'a'):
            pass

        p = self.tmppath('no_perms')
        with open(p, 'a'):
            pass

        # Make file and directory unwritable. Reminder: making a directory
        # unwritable prevents modifications (including deletes) from the list
        # of files in that directory.
        os.chmod(p, 0400)
        os.chmod(self.tmpdir, 0400)

        copier = FileCopier()
        copier.add('dummy', GeneratedFile('content'))
        result = copier.copy(self.tmpdir)
        self.assertEqual(result.removed_files_count, 1)
        self.assertFalse(os.path.exists(p))