def openlfdirstate(ui, repo):
    '''
    Return a dirstate object that tracks largefiles: i.e. its root is
    the repo root, but it is saved in .hg/largefiles/dirstate.
    '''
    admin = repo.join(longname)
    opener = scmutil.opener(admin)
    lfdirstate = largefiles_dirstate(opener, ui, repo.root,
                                     repo.dirstate._validate)

    # If the largefiles dirstate does not exist, populate and create
    # it. This ensures that we create it on the first meaningful
    # largefiles operation in a new clone. It also gives us an easy
    # way to forcibly rebuild largefiles state:
    #   rm .hg/largefiles/dirstate && hg status
    # Or even, if things are really messed up:
    #   rm -rf .hg/largefiles && hg status
    if not os.path.exists(os.path.join(admin, 'dirstate')):
        util.makedirs(admin)
        matcher = getstandinmatcher(repo)
        for standin in dirstate_walk(repo.dirstate, matcher):
            lfile = splitstandin(standin)
            hash = readstandin(repo, lfile)
            lfdirstate.normallookup(lfile)
            try:
                if hash == hashfile(lfile):
                    lfdirstate.normal(lfile)
            except IOError, err:
                if err.errno != errno.ENOENT:
                    raise

        lfdirstate.write()

def _move_pending(ui, repo, bfdirstate, ctx, standin, filename):
    '''
    Update bfiles administrative area (.hg/bfiles) to reflect a commit
    that affects the big file filename (tracked by standin).
    Specifically:
      - if the big file was added/modified by this changeset,
        move the pending revision from .hg/bfiles/pending to
        .hg/bfiles/committed (reflects state change from pending/uncommitted
        to pending/committed)
      - if the big file was removed by this changeset, remove
        it from .hg/bfiles/dirstate
    '''
    try:
        fctx = ctx[standin]
    except error.LookupError:
        # Standin file not in this changeset: it was removed.  Make
        # sure the bfiles dirstate no longer tracks it.
        dirstate_drop(bfdirstate, _split_standin(standin))
        return

    hash = fctx.data()[0:40]
    pending = repo.join(os.path.join('bfiles', 'pending', filename, hash))
    if os.path.exists(pending):
        committed = repo.join(os.path.join(
            'bfiles', 'committed', filename, hash))
        util.makedirs(os.path.dirname(committed))
        ui.debug('moving %s -> %s\n' % (pending, committed))
        os.rename(pending, committed)
        try:
            os.removedirs(os.path.dirname(pending))
        except OSError:                 # probably not empty, so ignore it
            pass

def putlfile(repo, proto, sha):
    '''Put a largefile into a repository's local store and into the
    user cache.'''
    proto.redirect()

    path = lfutil.storepath(repo, sha)
    util.makedirs(os.path.dirname(path))
    tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)

    try:
        try:
            proto.getfile(tmpfp)
            tmpfp._fp.seek(0)
            if sha != lfutil.hexsha1(tmpfp._fp):
                raise IOError(0, _('largefile contents do not match hash'))
            tmpfp.close()
            lfutil.linktousercache(repo, sha)
        except IOError, e:
            repo.ui.warn(_('largefiles: failed to put %s into store: %s\n') %
                         (sha, e.strerror))
            return wireproto.pushres(1)
    finally:
        tmpfp.discard()

    return wireproto.pushres(0)

def openlfdirstate(ui, repo):
    '''
    Return a dirstate object that tracks largefiles: i.e. its root is
    the repo root, but it is saved in .hg/largefiles/dirstate.
    '''
    admin = repo.join(longname)
    opener = scmutil.opener(admin)
    lfdirstate = largefiles_dirstate(opener, ui, repo.root,
                                     repo.dirstate._validate)

    # If the largefiles dirstate does not exist, populate and create
    # it. This ensures that we create it on the first meaningful
    # largefiles operation in a new clone.
    if not os.path.exists(os.path.join(admin, 'dirstate')):
        util.makedirs(admin)
        matcher = getstandinmatcher(repo)
        for standin in dirstate_walk(repo.dirstate, matcher):
            lfile = splitstandin(standin)
            hash = readstandin(repo, lfile)
            lfdirstate.normallookup(lfile)
            try:
                if hash == hashfile(repo.wjoin(lfile)):
                    lfdirstate.normal(lfile)
            except OSError, err:
                if err.errno != errno.ENOENT:
                    raise

def _updatebigrepo(ui, repo, files, brepo, bigfiles, ds):
    for file in files:
        f = repo.wjoin(file)
        hash = accelerated_hash(repo, file, os.lstat(f), ds)
        bigfiles[file] = hash
        rf = "%s/%s.%s" % (brepo, file, hash)
        util.makedirs(os.path.dirname(rf))
        try:
            ext = f.split('.')[-1]
            dont_pack=['gz', 'zip', 'tgz', '7z', 'jpg', 'jpeg', 'gif', 
                'mpg', 'mpeg', 'avi', 'rar', 'cab']
            if ext in dont_pack:
                util.copyfile(f, rf)
            else:
                fo = open(f, 'rb')
                rfo_fileobj = open(rf+'.gz', 'wb')
                rfo = gzip.GzipFile(file+'.'+hash, 'wb', 9, rfo_fileobj)
                def read10Mb():
                    return fo.read(1024*1024*10)
                for chunk in iter(read10Mb, ''):
                    rfo.write(chunk)
                fo.close()
                rfo.close()
                rfo_fileobj.close()
        except:
            ui.write(_('failed to store %s\n') % f)

    def _gethash(self, filename, hash):
        """Get file with the provided hash and store it in the local repo's
        store and in the usercache.
        filename is for informational messages only.
        """
        util.makedirs(lfutil.storepath(self.repo, ''))
        storefilename = lfutil.storepath(self.repo, hash)

        tmpname = storefilename + '.tmp'
        tmpfile = util.atomictempfile(
            tmpname, createmode=self.repo.store.createmode)

        try:
            gothash = self._getfile(tmpfile, filename, hash)
        except StoreError as err:
            self.ui.warn(err.longmessage())
            gothash = ""
        tmpfile.close()

        if gothash != hash:
            if gothash != "":
                self.ui.warn(
                    _('%s: data corruption (expected %s, got %s)\n') %
                    (filename, hash, gothash))
            util.unlink(tmpname)
            return False

        util.rename(tmpname, storefilename)
        lfutil.linktousercache(self.repo, hash)
        return True

def copytostoreabsolute(repo, file, hash):
    util.makedirs(os.path.dirname(storepath(repo, hash)))
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    else:
        shutil.copyfile(file, storepath(repo, hash))
        os.chmod(storepath(repo, hash), os.stat(file).st_mode)
        linktousercache(repo, hash)

    def put(self, source, filename, hash):
        destdir = os.path.join(self.url, filename)
        dest = os.path.join(destdir, hash)
        if os.path.exists(dest):
            # No big deal: this could happen if someone restores a big
            # file to a previous revision.
            return

        util.makedirs(destdir)
        shutil.copy(source, dest)

def copytostoreabsolute(repo, file, hash):
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    else:
        util.makedirs(os.path.dirname(storepath(repo, hash)))
        with open(file, 'rb') as srcf:
            with util.atomictempfile(storepath(repo, hash),
                                     createmode=repo.store.createmode) as dstf:
                for chunk in util.filechunkiter(srcf):
                    dstf.write(chunk)
        linktousercache(repo, hash)

def copytostoreabsolute(repo, file, hash):
    util.makedirs(os.path.dirname(storepath(repo, hash)))
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    else:
        dst = util.atomictempfile(storepath(repo, hash))
        for chunk in util.filechunkiter(open(file, 'rb')):
            dst.write(chunk)
        dst.close()
        util.copymode(file, storepath(repo, hash))
        linktousercache(repo, hash)

def copytostoreabsolute(repo, file, hash):
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    elif not getattr(repo, "_isconverting", False):
        util.makedirs(os.path.dirname(storepath(repo, hash)))
        dst = util.atomictempfile(storepath(repo, hash),
                                  createmode=repo.store.createmode)
        for chunk in util.filechunkiter(open(file, 'rb')):
            dst.write(chunk)
        dst.close()
        linktousercache(repo, hash)

def link(src, dest):
    util.makedirs(os.path.dirname(dest))
    try:
        util.oslink(src, dest)
    except OSError:
        # if hardlinks fail, fallback on atomic copy
        dst = util.atomictempfile(dest)
        for chunk in util.filechunkiter(open(src, 'rb')):
            dst.write(chunk)
        dst.close()
        os.chmod(dest, os.stat(src).st_mode)

def findfile(repo, hash):
    if instore(repo, hash):
        repo.ui.note(_('Found %s in store\n') % hash)
        return storepath(repo, hash)
    elif inusercache(repo.ui, hash):
        repo.ui.note(_('Found %s in system cache\n') % hash)
        path = storepath(repo, hash)
        util.makedirs(os.path.dirname(path))
        link(usercachepath(repo.ui, hash), path)
        return path
    return None

def link(src, dest):
    """Try to create hardlink - if that fails, efficiently make a copy."""
    util.makedirs(os.path.dirname(dest))
    try:
        util.oslink(src, dest)
    except OSError:
        # if hardlinks fail, fallback on atomic copy
        with open(src, 'rb') as srcf:
            with util.atomictempfile(dest) as dstf:
                for chunk in util.filechunkiter(srcf):
                    dstf.write(chunk)
        os.chmod(dest, os.stat(src).st_mode)

def copyfromcache(repo, hash, filename):
    '''Copy the specified largefile from the repo or system cache to
    filename in the repository. Return true on success or false if the
    file was not found in either cache (which should not happened:
    this is meant to be called only after ensuring that the needed
    largefile exists in the cache).'''
    path = findfile(repo, hash)
    if path is None:
        return False
    util.makedirs(os.path.dirname(repo.wjoin(filename)))
    shutil.copy(path, repo.wjoin(filename))
    return True

def writehash(hash, filename, executable):
    util.makedirs(os.path.dirname(filename))
    if os.path.exists(filename):
        os.unlink(filename)
    wfile = open(filename, 'wb')

    try:
        wfile.write(hash)
        wfile.write('\n')
    finally:
        wfile.close()
    if os.path.exists(filename):
        os.chmod(filename, getmode(executable))

    def get(self, files):
        '''Get the specified largefiles from the store and write to local
        files under repo.root.  files is a list of (filename, hash)
        tuples.  Return (success, missing), lists of files successfully
        downloaded and those not found in the store.  success is a list
        of (filename, hash) tuples; missing is a list of filenames that
        we could not get.  (The detailed error message will already have
        been presented to the user, so missing is just supplied as a
        summary.)'''
        success = []
        missing = []
        ui = self.ui

        util.makedirs(lfutil.storepath(self.repo, ''))

        at = 0
        available = self.exists(set(hash for (_filename, hash) in files))
        for filename, hash in files:
            ui.progress(_('getting largefiles'), at, unit='lfile',
                total=len(files))
            at += 1
            ui.note(_('getting %s:%s\n') % (filename, hash))

            if not available.get(hash):
                ui.warn(_('%s: largefile %s not available from %s\n')
                        % (filename, hash, self.url))
                missing.append(filename)
                continue

            storefilename = lfutil.storepath(self.repo, hash)
            tmpfile = util.atomictempfile(storefilename + '.tmp',
                                          createmode=self.repo.store.createmode)

            try:
                hhash = self._getfile(tmpfile, filename, hash)
            except StoreError, err:
                ui.warn(err.longmessage())
                hhash = ""
            tmpfile.close()

            if hhash != hash:
                if hhash != "":
                    ui.warn(_('%s: data corruption (expected %s, got %s)\n')
                            % (filename, hash, hhash))
                util.unlink(storefilename + '.tmp')
                missing.append(filename)
                continue

            util.rename(storefilename + '.tmp', storefilename)
            lfutil.linktousercache(self.repo, hash)
            success.append((filename, hhash))

    def _gethash(self, filename, hash):
        """Get file with the provided hash and store it in the local repo's
        store and in the usercache.
        filename is for informational messages only.
        """
        util.makedirs(lfutil.storepath(self.repo, ""))
        storefilename = lfutil.storepath(self.repo, hash)

        tmpname = storefilename + ".tmp"
        tmpfile = util.atomictempfile(tmpname, createmode=self.repo.store.createmode)

        try:
            gothash = self._getfile(tmpfile, filename, hash)
        except StoreError, err:
            self.ui.warn(err.longmessage())
            gothash = ""

def _write_hash(hhash, fn, mode=None):
    if mode is None:
        mode = 0666
    util.makedirs(os.path.dirname(fn))
    if os.path.exists(fn):
        os.unlink(fn)
    if os.name == 'posix':
        # Yuck: on Unix, go through open(2) to ensure that the caller's mode is 
        # filtered by umask() in the kernel, where it's supposed to be done.
        wfile = os.fdopen(os.open(fn, os.O_WRONLY|os.O_CREAT, mode), 'wb')
    elif os.name == 'nt':
        # But on Windows, use open() directly, since passing mode='wb' to os.fdopen()
        # does not work.  (Python bug?)
        wfile = open(fn, 'wb')
    wfile.write(hhash)
    wfile.write('\n')
    wfile.close()

 def do_bfput(self):
     """respond to the bfput command: send a file
     """
     key, fname = self.getarg()
     self.log('do_bfput: key=%r, fname=%r', key, fname)
     if os.path.exists(fname):
         self.log('dest file exists: returning early')
         self.respond('skip')
         return
     destdir = os.path.dirname(fname)
     try:
         self.log('opening dest file %r', fname)
         util.makedirs(destdir)
         fd = open(fname, "wb")
     except (OSError, IOError), err:
         self.log('error opening dest file: %s', err)
         self.respond('cannot create file: %s' % err)
         return