def main(options, out, err):
    if options.debug:
        out.write('starting scanning distdir %s...' % options.distdir)
    files = set(basename(file) for file in listdir_files(options.distdir))

    if options.debug:
        out.write('scanning repo...')

    pfiles = set()
    for pkg in options.repo.itermatch(options.restrict, sorter=sorted):
        try:
            pfiles.update(fetchable.filename for fetchable in
                        iflatten_instance(pkg.fetchables, fetchable_kls))
        except ParseChksumError as e:
            err.write("got corruption error '%s', with package %s " %
                (e, pkg.cpvstr))
            if options.ignore_failures:
                err.write("skipping...")
                err.write()
            else:
                err.write("aborting...")
                return 1
        except Exception as e:
            err.write("got error '%s', parsing package %s in repo '%s'" %
                (e, pkg.cpvstr, pkg.repo))
            raise

    d = options.distdir
    for file in (files - pfiles):
        out.write(pjoin(d, file))

    def _add_sets(self):
        self["world"] = basics.AutoConfigSection({
            "class": "pkgcore.pkgsets.filelist.WorldFile",
            "location": pjoin(self.root, econst.WORLD_FILE.lstrip('/'))})
        self["system"] = basics.AutoConfigSection({
            "class": "pkgcore.pkgsets.system.SystemSet",
            "profile": "profile"})
        self["installed"] = basics.AutoConfigSection({
            "class": "pkgcore.pkgsets.installed.Installed",
            "vdb": "vdb"})
        self["versioned-installed"] = basics.AutoConfigSection({
            "class": "pkgcore.pkgsets.installed.VersionedInstalled",
            "vdb": "vdb"})

        set_fp = pjoin(self.dir, "sets")
        try:
            for setname in listdir_files(set_fp):
                # Potential for name clashes here, those will just make
                # the set not show up in config.
                if setname in ("system", "world"):
                    logger.warning(
                        "user defined set %r is disallowed; ignoring",
                        pjoin(set_fp, setname))
                    continue
                self[setname] = basics.AutoConfigSection({
                    "class": "pkgcore.pkgsets.filelist.FileList",
                    "location": pjoin(set_fp, setname)})
        except FileNotFoundError:
            pass

 def licenses(self):
     """Return the set of all defined licenses in a repo."""
     try:
         content = listdir_files(self.licenses_dir)
     except EnvironmentError:
         content = ()
     return frozenset(content)

 def iter_vulnerabilities(self):
     """generator yielding each GLSA restriction"""
     for path in self.paths:
         for fn in listdir_files(path):
             # glsa-1234-12.xml
             if not (fn.startswith("glsa-") and fn.endswith(".xml")):
                 continue
             # This verifies the filename is of the correct syntax.
             try:
                 [int(x) for x in fn[5:-4].split("-")]
             except ValueError:
                 continue
             root = etree.parse(pjoin(path, fn))
             glsa_node = root.getroot()
             if glsa_node.tag != 'glsa':
                 raise ValueError("glsa without glsa rootnode")
             for affected in root.findall('affected'):
                 for pkg in affected.findall('package'):
                     try:
                         pkgname = str(pkg.get('name')).strip()
                         pkg_vuln_restrict = \
                             self.generate_intersects_from_pkg_node(
                                 pkg, tag="glsa(%s)" % fn[5:-4])
                         if pkg_vuln_restrict is None:
                             continue
                         pkgatom = atom.atom(pkgname)
                         yield fn[5:-4], pkgname, pkgatom, pkg_vuln_restrict
                     except (TypeError, ValueError) as v:
                         # thrown from cpv.
                         logger.warning(
                             "invalid glsa- %s, package %s: error %s",
                             fn, pkgname, v)
                         del v

 def _reload_state(self):
     try:
         self.__set_stage_state__([x[1:]
             for x in listdir_files(self.builddir) if x.startswith(".")])
     except EnvironmentError as e:
         if e.errno not in (errno.ENOTDIR, errno.ENOENT):
             raise

    def trigger(self, engine, existing_cset, install_cset):
        # hackish, but it works.
        protected_filter = gen_config_protect_filter(
            engine.offset, self.extra_protects, self.extra_disables).match
        ignore_filter = gen_collision_ignore_filter(engine.offset).match
        protected = {}

        for x in existing_cset.iterfiles():
            if not ignore_filter(x.location) and protected_filter(x.location):
                replacement = install_cset[x]
                if not simple_chksum_compare(replacement, x):
                    protected.setdefault(
                        pjoin(engine.offset,
                              os.path.dirname(x.location).lstrip(os.path.sep)),
                        []).append((os.path.basename(replacement.location),
                                    replacement))

        for dir_loc, entries in protected.iteritems():
            updates = {x[0]: [] for x in entries}
            try:
                existing = sorted(x for x in listdir_files(dir_loc)
                                  if x.startswith("._cfg"))
            except OSError as oe:
                if oe.errno != errno.ENOENT:
                    raise
                # this shouldn't occur.
                continue

            for x in existing:
                try:
                    # ._cfg0000_filename
                    count = int(x[5:9])
                    if x[9] != "_":
                        raise ValueError
                    fn = x[10:]
                except (ValueError, IndexError):
                    continue
                if fn in updates:
                    updates[fn].append((count, fn))

            # now we rename.
            for fname, entry in entries:
                # check for any updates with the same chksums.
                count = 0
                for cfg_count, cfg_fname in updates[fname]:
                    if simple_chksum_compare(livefs.gen_obj(
                            pjoin(dir_loc, cfg_fname)), entry):
                        count = cfg_count
                        break
                    count = max(count, cfg_count + 1)
                try:
                    install_cset.remove(entry)
                except KeyError:
                    # this shouldn't occur...
                    continue
                new_fn = pjoin(dir_loc, "._cfg%04i_%s" % (count, fname))
                new_entry = entry.change_attributes(location=new_fn)
                install_cset.add(new_entry)
                self.renames[new_entry] = entry
            del updates

def collapse_envd(base):
    collapsed_d = {}
    try:
        env_d_files = sorted(listdir_files(base))
    except OSError, oe:
        if oe.errno != errno.ENOENT:
            raise

def add_sets(config, root, portage_base_dir):
    config["world"] = basics.AutoConfigSection({
        "class": "pkgcore.pkgsets.filelist.WorldFile",
        "location": pjoin(root, const.WORLD_FILE)})
    config["system"] = basics.AutoConfigSection({
        "class": "pkgcore.pkgsets.system.SystemSet",
        "profile": "profile"})
    config["installed"] = basics.AutoConfigSection({
        "class": "pkgcore.pkgsets.installed.Installed",
        "vdb": "vdb"})
    config["versioned-installed"] = basics.AutoConfigSection({
        "class": "pkgcore.pkgsets.installed.VersionedInstalled",
        "vdb": "vdb"})

    set_fp = pjoin(portage_base_dir, "sets")
    try:
        for setname in listdir_files(set_fp):
            # Potential for name clashes here, those will just make
            # the set not show up in config.
            if setname in ("system", "world"):
                logger.warning(
                    "user defined set %s is disallowed; ignoring" %
                    pjoin(set_fp, setname))
                continue
            config[setname] = basics.AutoConfigSection({
                "class": "pkgcore.pkgsets.filelist.FileList",
                "location": pjoin(set_fp, setname)})
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

    def regen(self, binary, basepath):
        ignores = ("dir", "dir.old")
        try:
            files = listdir_files(basepath)
        except OSError as oe:
            if oe.errno == errno.ENOENT:
                return
            raise

        if self.should_skip_directory(basepath, files):
            return

        # wipe old indexes.
        for x in set(ignores).intersection(files):
            os.remove(pjoin(basepath, x))

        index = pjoin(basepath, 'dir')
        for x in files:
            if x in ignores or x.startswith("."):
                continue

            ret, data = spawn.spawn_get_output(
                [binary, '--quiet', pjoin(basepath, x), '--dir-file', index],
                collect_fds=(1,2), split_lines=False)

            if not data or "already exists" in data or \
                    "warning: no info dir entry" in data:
                continue
            yield pjoin(basepath, x)

def _scan_directory(path):
    files = []
    for x in listdir_files(path):
        match = valid_updates_re.match(x)
        if match is not None:
            files.append(((match.group(2), match.group(1)), x))
    files.sort(key=itemgetter(0))
    return [x[1] for x in files]

 def raw_use_expand_desc(self):
     base = pjoin(self.profiles_base, 'desc')
     try:
         targets = sorted(listdir_files(base))
     except EnvironmentError, e:
         if e.errno != errno.ENOENT:
             raise
         return ()

 def regen(self, binary, basepath):
     ignores = ("dir", "dir.old")
     try:
         files = listdir_files(basepath)
     except OSError, oe:
         if oe.errno == errno.ENOENT:
             return
         raise

 def _load_eclasses(self):
     """Force an update of the internal view of on disk/remote eclasses."""
     ec = {}
     eclass_len = len(".eclass")
     try:
         files = listdir_files(self.eclassdir)
     except EnvironmentError, e:
         if e.errno not in (errno.ENOENT, errno.ENOTDIR):
             raise
         return ImmutableDict()

def collapse_envd(base):
    collapsed_d = {}
    try:
        env_d_files = sorted(listdir_files(base))
    except OSError as oe:
        if oe.errno != errno.ENOENT:
            raise
    else:
        for x in env_d_files:
            if x.endswith(".bak") or x.endswith("~") or x.startswith("._cfg") \
                    or len(x) <= 2 or not x[0:2].isdigit():
                continue
            d = read_bash_dict(pjoin(base, x))
            # inefficient, but works.
            for k, v in d.iteritems():
                collapsed_d.setdefault(k, []).append(v)
            del d

    loc_incrementals = set(incrementals)
    loc_colon_parsed = set(colon_parsed)

    # split out env.d defined incrementals..
    # update incrementals *and* colon parsed for colon_separated;
    # incrementals on its own is space separated.

    for x in collapsed_d.pop("COLON_SEPARATED", []):
        v = x.split()
        if v:
            loc_colon_parsed.update(v)

    loc_incrementals.update(loc_colon_parsed)

    # now space.
    for x in collapsed_d.pop("SPACE_SEPARATED", []):
        v = x.split()
        if v:
            loc_incrementals.update(v)

    # now reinterpret.
    for k, v in collapsed_d.iteritems():
        if k not in loc_incrementals:
            collapsed_d[k] = v[-1]
            continue
        if k in loc_colon_parsed:
            collapsed_d[k] = filter(None, iflatten_instance(
                x.split(':') for x in v))
        else:
            collapsed_d[k] = filter(None, iflatten_instance(
                x.split() for x in v))

    return collapsed_d, loc_incrementals, loc_colon_parsed

 def _get_versions(self, catpkg):
     cppath = pjoin(self.base, catpkg[0], catpkg[1])
     pkg = f'{catpkg[-1]}-'
     lp = len(pkg)
     extension = self.extension
     ext_len = -len(extension)
     try:
         return tuple(
             x[lp:ext_len] for x in listdir_files(cppath)
             if x[ext_len:] == extension and x[:lp] == pkg)
     except EnvironmentError as e:
         raise KeyError(
             "failed fetching versions for package %s: %s" %
             (pjoin(self.base, '/'.join(catpkg)), str(e))) from e

 def _load_eclasses(self):
     """Force an update of the internal view of on disk/remote eclasses."""
     ec = {}
     eclass_len = len(".eclass")
     try:
         files = listdir_files(self.eclassdir)
     except (FileNotFoundError, NotADirectoryError):
         return ImmutableDict()
     for y in files:
         if not y.endswith(".eclass"):
             continue
         ys = y[:-eclass_len]
         ec[intern(ys)] = LazilyHashedPath(
             pjoin(self.eclassdir, y), eclassdir=self.eclassdir)
     return ImmutableDict(ec)

def parse_moves(location):
    pjoin = os.path.join

    # schwartzian comparison, convert it into YYYY-QQ
    def get_key(fname):
        return tuple(reversed(fname.split('-')))

    moves = {}
    for update_file in sorted(listdir_files(location), key=get_key):
        for line in iter_read_bash(pjoin(location, update_file)):
            line = line.split()
            if line[0] != 'move':
                continue
            moves[atom(line[1])] = atom(line[2])
    return moves

 def _load_eclasses(self):
     """Force an update of the internal view of on disk/remote eclasses."""
     ec = {}
     eclass_len = len(".eclass")
     try:
         files = listdir_files(self.eclassdir)
     except EnvironmentError as e:
         if e.errno not in (errno.ENOENT, errno.ENOTDIR):
             raise
         return ImmutableDict()
     for y in files:
         if not y.endswith(".eclass"):
             continue
         ys = y[:-eclass_len]
         ec[intern(ys)] = LazilyHashedPath(pjoin(self.eclassdir, y),
                                           eclassdir=self.eclassdir)
     return ImmutableDict(ec)

    def raw_use_expand_desc(self):
        """USE_EXPAND settings for the repo."""
        base = pjoin(self.profiles_base, 'desc')
        try:
            targets = sorted(listdir_files(base))
        except FileNotFoundError:
            return ()

        def f():
            for use_group in targets:
                group = use_group.split('.', 1)[0] + "_"

                def converter(key):
                    return (packages.AlwaysTrue, group + key)

                for x in self._split_use_desc_file(f'desc/{use_group}', converter):
                    yield x

        return tuple(f())

    def _get_packages(self, category):
        cpath = pjoin(self.base, category.lstrip(os.path.sep))
        l = set()
        d = {}
        lext = len(self.extension)
        bad = False
        try:
            for x in listdir_files(cpath):
                # don't use lstat; symlinks may exist
                if (x.endswith(".lockfile") or
                        not x[-lext:].lower() == self.extension or
                        x.startswith(".tmp.")):
                    continue
                pv = x[:-lext]
                try:
                    pkg = versioned_CPV(category+"/"+pv)
                except InvalidCPV:
                    bad = True
                if bad or not pkg.fullver:
                    if '-scm' in pv:
                        bad = 'scm'
                    elif '-try' in pv:
                        bad = 'try'
                    else:
                        raise InvalidCPV(
                            "%s/%s: no version component" %
                            (category, pv))
                    if self.ignore_paludis_versioning:
                        bad = False
                        continue
                    raise InvalidCPV(
                        "%s/%s: -%s version component is "
                        "not standard." % (category, pv, bad))
                l.add(pkg.package)
                d.setdefault((category, pkg.package), []).append(pkg.fullver)
        except EnvironmentError as e:
            raise_from(KeyError(
                "failed fetching packages for category %s: %s" %
                (pjoin(self.base, category.lstrip(os.path.sep)), str(e))))

        self._versions_tmp_cache.update(d)
        return tuple(l)