def collect(src, ui):
    seplen = len(os.path.sep)
    candidates = []
    live = len(src['tip'].manifest())
    # Your average repository has some files which were deleted before
    # the tip revision. We account for that by assuming that there are
    # 3 tracked files for every 2 live files as of the tip version of
    # the repository.
    #
    # mozilla-central as of 2010-06-10 had a ratio of just over 7:5.
    total = live * 3 // 2
    src = src.store.path
    pos = 0
    ui.status(_("tip has %d files, estimated total number of files: %d\n")
              % (live, total))
    for dirpath, dirnames, filenames in os.walk(src):
        dirnames.sort()
        relpath = dirpath[len(src) + seplen:]
        for filename in sorted(filenames):
            if filename[-2:] not in ('.d', '.i'):
                continue
            st = os.stat(os.path.join(dirpath, filename))
            if not stat.S_ISREG(st.st_mode):
                continue
            pos += 1
            candidates.append((os.path.join(relpath, filename), st))
            ui.progress(_('collecting'), pos, filename, _('files'), total)

    ui.progress(_('collecting'), None)
    ui.status(_('collected %d candidate storage files\n') % len(candidates))
    return candidates

def readurltoken(repo):
    """return conduit url, token and make sure they exist

    Currently read from [auth] config section. In the future, it might
    make sense to read from .arcconfig and .arcrc as well.
    """
    url = repo.ui.config(b'phabricator', b'url')
    if not url:
        raise error.Abort(_(b'config %s.%s is required')
                          % (b'phabricator', b'url'))

    res = httpconnectionmod.readauthforuri(repo.ui, url, util.url(url).user)
    token = None

    if res:
        group, auth = res

        repo.ui.debug(b"using auth.%s.* for authentication\n" % group)

        token = auth.get(b'phabtoken')

    if not token:
        token = readlegacytoken(repo, url)
        if not token:
            raise error.Abort(_(b'Can\'t find conduit token associated to %s')
                              % (url,))

    return url, token

    def recover(self, repo):
        '''commit working directory using journal metadata'''
        node, user, date, message, parents = self.readlog()
        merge = len(parents) == 2

        if not user or not date or not message or not parents[0]:
            raise util.Abort(_('transplant log file is corrupt'))

        extra = {'transplant_source': node}
        wlock = repo.wlock()
        try:
            p1, p2 = repo.dirstate.parents()
            if p1 != parents[0]:
                raise util.Abort(
                    _('working dir not at transplant parent %s') %
                                 revlog.hex(parents[0]))
            if merge:
                repo.dirstate.setparents(p1, parents[1])
            n = repo.commit(None, message, user, date, extra=extra)
            if not n:
                raise util.Abort(_('commit failed'))
            if not merge:
                self.transplants.set(n, node)
            self.unlog()

            return n, node
        finally:
            del wlock

def catfile(ui, repo, type=None, r=None, **opts):
    """cat a specific revision"""
    # in stdin mode, every line except the commit is prefixed with two
    # spaces.  This way the our caller can find the commit without magic
    # strings
    #
    prefix = ""
    if opts['stdin']:
        try:
            (type, r) = raw_input().split(' ')
            prefix = "    "
        except EOFError:
            return

    else:
        if not type or not r:
            ui.warn(_("cat-file: type or revision not supplied\n"))
            commands.help_(ui, 'cat-file')

    while r:
        if type != "commit":
            ui.warn(_("aborting hg cat-file only understands commits\n"))
            return 1
        n = repo.lookup(r)
        catcommit(ui, repo, n, prefix)
        if opts['stdin']:
            try:
                (type, r) = raw_input().split(' ')
            except EOFError:
                break
        else:
            break

def getkeys(ui, repo, mygpg, sigdata, context):
    """get the keys who signed a data"""
    fn, ln = context
    node, version, sig = sigdata
    prefix = "%s:%d" % (fn, ln)
    node = hgnode.bin(node)

    data = node2txt(repo, node, version)
    sig = binascii.a2b_base64(sig)
    err, keys = mygpg.verify(data, sig)
    if err:
        ui.warn("%s:%d %s\n" % (fn, ln, err))
        return None

    validkeys = []
    # warn for expired key and/or sigs
    for key in keys:
        if key[0] == "BADSIG":
            ui.write(_('%s Bad signature from "%s"\n') % (prefix, key[2]))
            continue
        if key[0] == "EXPSIG":
            ui.write(_("%s Note: Signature has expired" ' (signed by: "%s")\n') % (prefix, key[2]))
        elif key[0] == "EXPKEYSIG":
            ui.write(_("%s Note: This key has expired" ' (signed by: "%s")\n') % (prefix, key[2]))
        validkeys.append((key[1], key[2], key[3]))
    return validkeys

    def verify(self, revs, contents=False):
        '''Verify the existence (and, optionally, contents) of every big
        file revision referenced by every changeset in revs.
        Return 0 if all is well, non-zero on any errors.'''
        failed = False

        self.ui.status(
            _('searching %d changesets for largefiles\n') % len(revs))
        verified = set()  # set of (filename, filenode) tuples

        for rev in revs:
            cctx = self.repo[rev]
            cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))

            for standin in cctx:
                if self._verifyfile(cctx, cset, contents, standin, verified):
                    failed = True

        numrevs = len(verified)
        numlfiles = len(set([fname for (fname, fnode) in verified]))
        if contents:
            self.ui.status(
                _('verified contents of %d revisions of %d largefiles\n') %
                (numrevs, numlfiles))
        else:
            self.ui.status(
                _('verified existence of %d revisions of %d largefiles\n') %
                (numrevs, numlfiles))
        return int(failed)

    def run(self):
        if not self.repo.local:
            raise util.Abort(_('Repository "%s" is not local') % self.repo.root)

        self._check_changed()
        upload, remove = self._get_files()

        if self.opts.get('show'):
            self.ui.write(_('Upload files:\n'))
            for f in upload:
                self.ui.write('\t%s\n' % f)
            self.ui.write(_('\nDelete files:\n'))
            for f in remove:
                self.ui.write('\t%s\n' % f)

        self.ui.write(_('Upload files: %s, delete files: %s\n') %
                (len(upload), len(remove)) )

        if self.opts.get('upload'):
            if upload or remove:
                self._ftp(upload, remove)

            if not self.opts.get('only'):
                commands.tag(self.ui, self.repo, self.tagname,
                        local=not self.useGlobal,
                        rev=str(self.selected),
                        force=True)

                self.ui.write(_('Added tag %s for changeset %d:%s\n') %
                        (self.tagname, self.selected, self.selected))

def toposort_postorderreverse(ui, rl):
    # reverse-postorder of the reverse directed graph

    children = {}
    roots = set()
    ui.status(_('reading revs\n'))
    try:
        for rev in rl:
            ui.progress(_('reading'), rev, total=len(rl))
            (p1, p2) = rl.parentrevs(rev)
            if p1 == p2 == node.nullrev:
                roots.add(rev)
            children[rev] = []
            if p1 != node.nullrev:
                children[p1].append(rev)
            if p2 != node.nullrev:
                children[p2].append(rev)
    finally:
        ui.progress(_('reading'), None)

    roots = list(roots)
    roots.sort()

    ui.status(_('sorting revs\n'))
    result = postorder(roots, children)
    result.reverse()
    return result

    def _verifyfiles(self, contents, filestocheck):
        failed = False
        expectedhashes = [expectedhash
                          for cset, filename, expectedhash in filestocheck]
        localhashes = self._hashesavailablelocally(expectedhashes)
        stats = self._stat([expectedhash for expectedhash in expectedhashes
                            if expectedhash not in localhashes])

        for cset, filename, expectedhash in filestocheck:
            if expectedhash in localhashes:
                filetocheck = (cset, filename, expectedhash)
                verifyresult = self._lstore._verifyfiles(contents,
                                                         [filetocheck])
                if verifyresult:
                    failed = True
            else:
                stat = stats[expectedhash]
                if stat:
                    if stat == 1:
                        self.ui.warn(
                            _('changeset %s: %s: contents differ\n')
                            % (cset, filename))
                        failed = True
                    elif stat == 2:
                        self.ui.warn(
                            _('changeset %s: %s missing\n')
                            % (cset, filename))
                        failed = True
                    else:
                        raise RuntimeError('verify failed: unexpected response '
                                           'from statlfile (%r)' % stat)
        return failed

def _pullreviews(repo):
    reviews = repo.reviews
    if not reviews.remoteurl:
        raise util.Abort(_("We don't know of any review servers. Try " "creating a review first."))

    reviewdata = _pullreviewidentifiers(repo, sorted(reviews.identifiers))
    repo.ui.write(_("updated %d reviews\n") % len(reviewdata))

def toposort_reversepostorder(ui, rl):
    # postorder of the reverse directed graph

    # map rev to list of parent revs (p2 first)
    parents = {}
    heads = set()
    ui.status(_('reading revs\n'))
    try:
        for rev in rl:
            ui.progress(_('reading'), rev, total=len(rl))
            (p1, p2) = rl.parentrevs(rev)
            if p1 == p2 == node.nullrev:
                parents[rev] = ()       # root node
            elif p1 == p2 or p2 == node.nullrev:
                parents[rev] = (p1,)    # normal node
            else:
                parents[rev] = (p2, p1) # merge node
            heads.add(rev)
            for p in parents[rev]:
                heads.discard(p)
    finally:
        ui.progress(_('reading'), None)

    heads = list(heads)
    heads.sort(reverse=True)

    ui.status(_('sorting revs\n'))
    return postorder(heads, parents)

def lfpull(ui, repo, source="default", **opts):
    """pull largefiles for the specified revisions from the specified source

    Pull largefiles that are referenced from local changesets but missing
    locally, pulling from a remote repository to the local cache.

    If SOURCE is omitted, the 'default' path will be used.
    See :hg:`help urls` for more information.

    .. container:: verbose

      Some examples:

      - pull largefiles for all branch heads::

          hg lfpull -r "head() and not closed()"

      - pull largefiles on the default branch::

          hg lfpull -r "branch(default)"
    """
    repo.lfpullsource = source

    revs = opts.get('rev', [])
    if not revs:
        raise util.Abort(_('no revisions specified'))
    revs = scmutil.revrange(repo, revs)

    numcached = 0
    for rev in revs:
        ui.note(_('pulling largefiles for revision %s\n') % rev)
        (cached, missing) = cachelfiles(ui, repo, rev)
        numcached += len(cached)
    ui.status(_("%d largefiles cached\n") % numcached)

def uploadlfiles(ui, rsrc, rdst, files):
    '''upload largefiles to the central store'''

    if not files:
        return

    store = basestore._openstore(rsrc, rdst, put=True)

    at = 0
    ui.debug("sending statlfile command for %d largefiles\n" % len(files))
    retval = store.exists(files)
    files = filter(lambda h: not retval[h], files)
    ui.debug("%d largefiles need to be uploaded\n" % len(files))

    for hash in files:
        ui.progress(_('uploading largefiles'), at, unit='largefile',
                    total=len(files))
        source = lfutil.findfile(rsrc, hash)
        if not source:
            raise util.Abort(_('largefile %s missing from store'
                               ' (needs to be uploaded)') % hash)
        # XXX check for errors here
        store.put(source, hash)
        at += 1
    ui.progress(_('uploading largefiles'), None)

def abort(repo, originalwd, target, state):
    'Restore the repository to its original state'
    dstates = [s for s in state.values() if s > nullrev]
    immutable = [d for d in dstates if not repo[d].mutable()]
    cleanup = True
    if immutable:
        repo.ui.warn(_("warning: can't clean up immutable changesets %s\n")
                     % ', '.join(str(repo[r]) for r in immutable),
                     hint=_('see hg help phases for details'))
        cleanup = False

    descendants = set()
    if dstates:
        descendants = set(repo.changelog.descendants(dstates))
    if descendants - set(dstates):
        repo.ui.warn(_("warning: new changesets detected on target branch, "
                       "can't strip\n"))
        cleanup = False

    if cleanup:
        # Update away from the rebase if necessary
        if inrebase(repo, originalwd, state):
            merge.update(repo, repo[originalwd].rev(), False, True, False)

        # Strip from the first rebased revision
        rebased = filter(lambda x: x > -1 and x != target, state.values())
        if rebased:
            strippoints = [c.node()  for c in repo.set('roots(%ld)', rebased)]
            # no backup of rebased cset versions needed
            repair.strip(repo.ui, repo, strippoints)

    clearstatus(repo)
    repo.ui.warn(_('rebase aborted\n'))
    return 0

def extsetup(ui):
    commands.globalopts.append(
        ('', 'color', 'auto',
         # i18n: 'always', 'auto', and 'never' are keywords and should
         # not be translated
         _("when to colorize (boolean, always, auto, or never)"),
         _('TYPE')))

        def readsparseconfig(self, raw):
            """Takes a string sparse config and returns the includes,
            excludes, and profiles it specified.
            """
            includes = set()
            excludes = set()
            current = includes
            profiles = []
            for line in raw.split('\n'):
                line = line.strip()
                if not line or line.startswith('#'):
                    # empty or comment line, skip
                    continue
                elif line.startswith('%include '):
                    line = line[9:].strip()
                    if line:
                        profiles.append(line)
                elif line == '[include]':
                    if current != includes:
                        raise error.Abort(_('.hg/sparse cannot have includes ' +
                            'after excludes'))
                    continue
                elif line == '[exclude]':
                    current = excludes
                elif line:
                    if line.strip().startswith('/'):
                        self.ui.warn(_('warning: sparse profile cannot use' +
                                       ' paths starting with /, ignoring %s\n')
                                       % line)
                        continue
                    current.add(line)

            return includes, excludes, profiles

def _checkhook(ui, repo, node, headsonly):
    # Get revisions to check and touched files at the same time
    files = set()
    revs = set()
    for rev in xrange(repo[node].rev(), len(repo)):
        revs.add(rev)
        if headsonly:
            ctx = repo[rev]
            files.update(ctx.files())
            for pctx in ctx.parents():
                revs.discard(pctx.rev())
    failed = []
    for rev in revs:
        ctx = repo[rev]
        eol = parseeol(ui, repo, [ctx.node()])
        if eol:
            failed.extend(eol.checkrev(repo, ctx, files))

    if failed:
        eols = {'to-lf': 'CRLF', 'to-crlf': 'LF'}
        msgs = []
        for node, target, f in failed:
            msgs.append(_("  %s in %s should not have %s line endings") %
                        (f, node, eols[target]))
        raise util.Abort(_("end-of-line check failed:\n") + "\n".join(msgs))

    def __init__(self, ui, path, revs=None):
        super(gnuarch_source, self).__init__(ui, path, revs=revs)

        if not os.path.exists(os.path.join(path, '{arch}')):
            raise common.NoRepo(_("%s does not look like a GNU Arch repository")
                         % path)

        # Could use checktool, but we want to check for baz or tla.
        self.execmd = None
        if util.findexe('baz'):
            self.execmd = 'baz'
        else:
            if util.findexe('tla'):
                self.execmd = 'tla'
            else:
                raise error.Abort(_('cannot find a GNU Arch tool'))

        common.commandline.__init__(self, ui, self.execmd)

        self.path = os.path.realpath(path)
        self.tmppath = None

        self.treeversion = None
        self.lastrev = None
        self.changes = {}
        self.parents = {}
        self.tags = {}
        self.catlogparser = email.Parser.Parser()
        self.encoding = encoding.encoding
        self.archives = []

    def get(self, files):
        '''Get the specified largefiles from the store and write to local
        files under repo.root.  files is a list of (filename, hash)
        tuples.  Return (success, missing), lists of files successfully
        downloaded and those not found in the store.  success is a list
        of (filename, hash) tuples; missing is a list of filenames that
        we could not get.  (The detailed error message will already have
        been presented to the user, so missing is just supplied as a
        summary.)'''
        success = []
        missing = []
        ui = self.ui

        at = 0
        available = self.exists(set(hash for (_filename, hash) in files))
        for filename, hash in files:
            ui.progress(
                _('getting largefiles'), at, unit='lfile', total=len(files))
            at += 1
            ui.note(_('getting %s:%s\n') % (filename, hash))

            if not available.get(hash):
                ui.warn(
                    _('%s: largefile %s not available from %s\n') %
                    (filename, hash, util.hidepassword(self.url)))
                missing.append(filename)
                continue

            if self._gethash(filename, hash):
                success.append((filename, hash))
            else:
                missing.append(filename)

        ui.progress(_('getting largefiles'), None)
        return (success, missing)

def hook(ui, repo, hooktype, node=None, source=None, **kwargs):
    '''send email notifications to interested subscribers.

    if used as changegroup hook, send one email for all changesets in
    changegroup. else send one email per changeset.'''
    n = notifier(ui, repo, hooktype)
    if not n.subs:
        ui.debug(_('notify: no subscribers to repo %s\n') % n.root)
        return
    if n.skipsource(source):
        ui.debug(_('notify: changes have source "%s" - skipping\n') %
                 source)
        return
    node = bin(node)
    ui.pushbuffer()
    if hooktype == 'changegroup':
        start = repo.changelog.rev(node)
        end = repo.changelog.count()
        count = end - start
        for rev in xrange(start, end):
            n.node(repo.changelog.node(rev))
        n.diff(node, repo.changelog.tip())
    else:
        count = 1
        n.node(node)
        n.diff(node, node)
    data = ui.popbuffer()
    n.send(node, count, data)