def copytree(src, dst, symlinks=False, ignore=None):
    """
    copytree that works even if folder already exists
    """
    # http://stackoverflow.com/questions/1868714/how-do-i-copy-an-entire-directory-of-files-into-an-existing-directory-using-pyth
    if not os.path.exists(dst):
        os.makedirs(dst)
        shutil.copystat(src, dst)
    lst = os.listdir(src)
    if ignore:
        excl = ignore(src, lst)
        lst = [x for x in lst if x not in excl]
    for item in lst:
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if symlinks and os.path.islink(s):
            if os.path.lexists(d):
                os.remove(d)
            os.symlink(os.readlink(s), d)
            try:
                st = os.lstat(s)
                mode = stat.S_IMODE(st.st_mode)
                os.lchmod(d, mode)
            except:
                pass  # lchmod not available
        elif os.path.isdir(s):
            copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

    def backup(self, datasetType, dataId):
        """Rename any existing object with the given type and dataId.

        The CameraMapper implementation saves objects in a sequence of e.g.:
          foo.fits
          foo.fits~1
          foo.fits~2
        All of the backups will be placed in the output repo, however, and will
        not be removed if they are found elsewhere in the _parent chain.  This
        means that the same file will be stored twice if the previous version was
        found in an input repo.
        """
        n = 0
        suffix = ""
        newLocation = self.map(datasetType, dataId, write=True)
        newPath = newLocation.getLocations()[0]
        path = self._parentSearch(newPath)
        oldPaths = []
        while path is not None:
            n += 1
            oldPaths.append((n, path))
            path = self._parentSearch("%s~%d" % (newPath, n))
        for n, oldPath in reversed(oldPaths):
            newDir, newFile = os.path.split(newPath)
            if not os.path.exists(newDir):
                os.makedirs(newDir)
            shutil.copy(oldPath, "%s~%d" % (newPath, n))
            shutil.copystat(oldPath, "%s~%d" % (newPath, n))

def overwriteCopy(src, dest, symlinks=False, ignore=None):
    if not os.path.exists(dest):
        os.makedirs(dest)
        shutil.copystat(src, dest)
    sub_list = os.listdir(src)
    if ignore:
        excl = ignore(src, sub_list)
        sub_list = [x for x in sub_list if x not in excl]
    for i_sub in sub_list:
        s_path = os.path.join(src, i_sub)
        d_path = os.path.join(dest, i_sub)
        if symlinks and os.path.islink(s_path):
            if os.path.lexists(d_path):
                os.remove(d_path)
            os.symlink(os.readlink(s_path), d_path)
            try:
                s_path_s = os.lstat(s_path)
                s_path_mode = stat.S_IMODE(s_path_s.st_mode)
                os.lchmod(d_path, s_path_mode)
            except Exception:
                pass
        elif os.path.isdir(s_path):
            overwriteCopy(s_path, d_path, symlinks, ignore)
        else:
            shutil.copy2(s_path, d_path)

def sed_inplace(filename, pattern, repl):
    """Perform the pure-Python equivalent of in-place `sed` substitution: e.g.,
    `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.

    Examples
    --------
    sed_inplace('/etc/apt/sources.list', r'^\# deb', 'deb')

    """
    # For efficiency, precompile the passed regular expression.
    pattern_compiled = re.compile(pattern)

    # For portability, NamedTemporaryFile() defaults to mode "w+b" (i.e., binary
    # writing with updating). This is usually a good thing. In this case,
    # however, binary writing imposes non-trivial encoding constraints trivially
    # resolved by switching to text writing. Let's do that.
    with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
        with open(filename) as src_file:
            for line in src_file:
                tmp_file.write(pattern_compiled.sub(repl, line))

    # Overwrite the original file with the munged temporary file in a
    # manner preserving file attributes (e.g., permissions).
    shutil.copystat(filename, tmp_file.name)
    shutil.move(tmp_file.name, filename)

 def assemble(self):
     if _check_path_overlap(self.name) and os.path.isdir(self.name):
         _rmtree(self.name)
     logger.info("Building COLLECT %s", self.tocbasename)
     os.makedirs(self.name)
     toc = add_suffix_to_extensions(self.toc)
     for inm, fnm, typ in toc:
         if not os.path.exists(fnm) or not os.path.isfile(fnm) and is_path_to_egg(fnm):
             # file is contained within python egg, it is added with the egg
             continue
         if os.pardir in os.path.normpath(inm) or os.path.isabs(inm):
             raise SystemExit('Security-Alert: try to store file outside '
                              'of dist-directory. Aborting. %r' % inm)
         tofnm = os.path.join(self.name, inm)
         todir = os.path.dirname(tofnm)
         if not os.path.exists(todir):
             os.makedirs(todir)
         if typ in ('EXTENSION', 'BINARY'):
             fnm = checkCache(fnm, strip=self.strip_binaries,
                              upx=(self.upx_binaries and (is_win or is_cygwin)),
                              dist_nm=inm)
         if typ != 'DEPENDENCY':
             shutil.copy(fnm, tofnm)
             try:
                 shutil.copystat(fnm, tofnm)
             except OSError:
                 logger.warn("failed to copy flags of %s", fnm)
         if typ in ('EXTENSION', 'BINARY'):
             os.chmod(tofnm, 0o755)

    def _createType(self, meta_name, root, movie_info, group, file_type, i):  # Get file path
        camelcase_method = underscoreToCamel(file_type.capitalize())
        name = getattr(self, 'get' + camelcase_method + 'Name')(meta_name, root, i)

        if name and (self.conf('meta_' + file_type) or self.conf('meta_' + file_type) is None):

            # Get file content
            content = getattr(self, 'get' + camelcase_method)(movie_info = movie_info, data = group, i = i)
            if content:
                log.debug('Creating %s file: %s', (file_type, name))
                if os.path.isfile(content):
                    content = sp(content)
                    name = sp(name)

                    if not os.path.exists(os.path.dirname(name)):
                        os.makedirs(os.path.dirname(name))

                    shutil.copy2(content, name)
                    shutil.copyfile(content, name)

                    # Try and copy stats seperately
                    try: shutil.copystat(content, name)
                    except: pass
                else:
                    self.createFile(name, content)
                    group['renamed_files'].append(name)

                try:
                    os.chmod(sp(name), Env.getPermission('file'))
                except:
                    log.debug('Failed setting permissions for %s: %s', (name, traceback.format_exc()))

def copyDigestedFile(src, dst, copystat=1):
    """ Copy data from `src` to `dst`, adding a fingerprint to `dst`.
        If `copystat` is true, the file status is copied, too
        (like shutil.copy2).
    """
    if os.path.isdir(dst):
        dst = os.path.join(dst, os.path.basename(src))

    dummy, ext = os.path.splitext(src)
    if ext not in digested_file_types:
        if copystat:
            return shutil.copy2(src, dst)
        else:
            return shutil.copyfile(src, dst)

    fsrc = None
    fdst = None
    try:
        fsrc = open(src, 'r')
        fdst = DigestFile(dst)
        shutil.copyfileobj(fsrc, fdst)
    finally:
        if fdst: fdst.close()
        if fsrc: fsrc.close()

    if copystat: shutil.copystat(src, dst)

def copystatRecursive(src, dst):
    if S_ISDIR(os.stat(src).st_mode):
        srcEntries = os.listdir(src)
        dstEntries = os.listdir(dst)
        for i in xrange(0, len(srcEntries)):
            copystatRecursive(os.path.join(src, srcEntries[i]), os.path.join(dst, dstEntries[i]))
    shutil.copystat(src, dst)

def safe_copyfile(src, dest):
    """safely copy src to dest using a temporary intermediate and then renaming
    to dest"""
    fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(dest))
    shutil.copyfileobj(open(src, 'rb'), os.fdopen(fd, 'wb'))
    shutil.copystat(src, tmpname)
    os.rename(tmpname, dest)

        def destaging_harness(backup, func):
            path = backup[0:-len('.release')]
            trace(path)

            def relocate_for_release(token):
                newtoken = token.replace(self.staged_prefix, self.prefix)

                if newtoken != token:
                    trace('%s:\n\t%s\t->\t%s' %
                          (os.path.basename(path), token, newtoken))

                return newtoken

            try:
                trace('Destaging %s' % path)
                func(path, relocate_for_release)
                if os.path.exists(path + '.stage'):
                    os.remove(path)
                    shutil.move(path + '.stage', path)
                    shutil.copystat(backup, path)
                os.remove(backup)

            except Exception as e:
                warn ('Critical: Destaging failed for ''%s''' % path)
                raise

 def copy2(src, dst):
     """
     shutil.copy2 does not copy the file attributes on windows, so we
     hack into the shutil module to fix the problem
     """
     old(src, dst)
     shutil.copystat(src, dst)

        def staging_harness(path, func, failure_count=failure_count):
            def relocate_to_profile(token):
                if token.find(package.staged_prefix) == -1 and token.find(package.staged_profile) == -1:
                    newtoken = token.replace(
                        package.package_prefix, package.staged_profile)
                else:
                    newtoken = token.replace(
                        package.staged_prefix, package.staged_profile)

                if newtoken != token:
                    package.trace('%s:\n\t%s\t->\t%s' %
                                  (os.path.basename(path), token, newtoken))
                return newtoken

            if (path.endswith('.release')):
                error('Staging backup exists in dir we''re trying to stage: %s' % path)

            backup = path + '.release'
            shutil.copy2(path, backup)
            try:
                trace('Staging %s' % path)
                func(path, relocate_to_profile)
                if os.path.exists(path + '.stage'):
                    os.remove(path)
                    shutil.move(path + '.stage', path)
                    shutil.copystat(backup, path)
            except CommandException as e:
                package.rm_if_exists(path)
                shutil.copy2(backup, path)
                package.rm(backup)
                warn('Staging failed for %s' % os.path.basename(path))
                error(str(e))
                failure_count = failure_count + 1
                if failure_count > 10:
                    error('Possible staging issue, >10 staging failures')

def make_thumbnails(files, thumbfunc):
    """
    For any file without a matching image file, invokes function to generate
    a thumbnail image.
    """
    dirs_changed = {}  # {path name: os.stat_result}
    imageexts = [x.replace("*", "") for x in TYPEGROUPS["image"]]
    for video in files:
        path, tail = os.path.split(video)
        base = os.path.splitext(tail)[0]
        if any(os.path.isfile(os.path.join(path, base + x)) for x in imageexts):
            continue  # for video

        pathstat = dirs_changed.get(path) or os.stat(path)
        image = os.path.join(path, base + ".jpg")
        tempimage = os.path.join(TEMP_DIR, uuid.uuid4().hex[:8] + ".jpg")
        if os.path.exists(tempimage): os.remove(tempimage)
        print("Creating thumb for video \"%s\"." % video)
        attempts = 3
        while attempts:
            try:
                thumbfunc(video, tempimage), shutil.move(tempimage, image)
                break  # while attempts
            except Exception:
                attempts -= 1
        if os.path.exists(image):
            shutil.copystat(video, image)  # Set video file timestamps to image
            dirs_changed[path] = pathstat
        else:
            print("Failed to produce \"%s\"." % image)
    for path, stat in dirs_changed.items():  # Restore directory timestamps
        os.utime(path, (stat.st_atime, stat.st_mtime))

def copytree(src, dst, symlinks=False):
    names = os.listdir(src)
    os.makedirs(dst)
    errors = []
    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and islink(srcname):
                linkto = readlink(srcname)
                symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                copytree(srcname, dstname, symlinks)
            else:
                copy2(srcname, dstname)
        except OSError as why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except Error as err:
            errors.extend(err.args[0])
    try:
        copystat(src, dst)
    except OSError as why:
        # can't copy file access times on Windows
        if why.winerror is None:
            errors.extend((src, dst, str(why)))
    if errors:
        raise Error(errors)

 def do_copyfile(self, from_file, to_file):
     outdir = os.path.split(to_file)[0]
     if not os.path.isfile(from_file) and not os.path.islink(from_file):
         raise RuntimeError('Tried to install something that isn\'t a file:'
                            '{!r}'.format(from_file))
     # copyfile fails if the target file already exists, so remove it to
     # allow overwriting a previous install. If the target is not a file, we
     # want to give a readable error.
     if os.path.exists(to_file):
         if not os.path.isfile(to_file):
             raise RuntimeError('Destination {!r} already exists and is not '
                                'a file'.format(to_file))
         if self.should_preserve_existing_file(from_file, to_file):
             append_to_log(self.lf, '# Preserving old file %s\n' % to_file)
             print('Preserving existing file %s.' % to_file)
             return False
         os.remove(to_file)
     print('Installing %s to %s' % (from_file, outdir))
     if os.path.islink(from_file):
         if not os.path.exists(from_file):
             # Dangling symlink. Replicate as is.
             shutil.copy(from_file, outdir, follow_symlinks=False)
         else:
             # Remove this entire branch when changing the behaviour to duplicate
             # symlinks rather than copying what they point to.
             print(symlink_warning)
             shutil.copyfile(from_file, to_file)
             shutil.copystat(from_file, to_file)
     else:
         shutil.copyfile(from_file, to_file)
         shutil.copystat(from_file, to_file)
     selinux_updates.append(to_file)
     append_to_log(self.lf, to_file)
     return True

    def _sieve_path(self, mailbox):
        """Retrieve the sieve script path for a mailbox

           The default value if not configured is ~/.dovecot.sieve

        :returns: the path where the sieve path will be stored, the
                  backup if the user already that filename and the active
                  sieve script if required
        :rtype: tuple
        """
        # Read the sieve script path template from config
        sieve_script_path_template = self.conf['sieve_script_path']
        sieve_script_path_mkdir = self.conf['sieve_script_path_mkdir']
        log.debug("Sieve script path template is '%s'" %
                  sieve_script_path_template)

        # Build the expansion variables for template
        (user, domain) = mailbox.split('@')

        # Substitute in template
        t = Template(sieve_script_path_template)
        sieve_script_path = t.substitute(domain=domain, user=user,
                                         fulluser=mailbox)
        log.debug("Expanded sieve script path for mailbox '%s' is '%s'" %
                  (mailbox, sieve_script_path))

        # If sieve script path mkdir enabled create hierarchy if it does not exist
        (head, tail) = os.path.split(sieve_script_path)
        if (sieve_script_path_mkdir):
            r_mkdir(head)
        else:
            if not os.path.isdir(head):
                raise Exception("Sieve script directory '%s' does not exist" %
                                head)

        sieve_user_backup = None
        if os.path.isfile(sieve_script_path):
            if not self._isOofScript(sieve_script_path):
                if os.path.islink(sieve_script_path):
                    target_sieve_script_path = os.path.realpath(sieve_script_path)
                    log.info('Activate the OOF script and change the link for %s' % target_sieve_script_path)
                    (sieve_user_backup, _) = os.path.splitext(os.path.basename(target_sieve_script_path))
                else:
                    log.info('Backing up already created "%s" to "%s.sieve"' % (sieve_script_path,
                                                                                sieve_script_path))
                    sieve_path_backup = sieve_script_path + '.sieve'
                    shutil.copyfile(sieve_script_path, sieve_path_backup)
                    shutil.copystat(sieve_script_path, sieve_path_backup)
                    sieve_user_backup = os.path.basename(sieve_script_path)
        elif os.path.exists(sieve_script_path):
            raise Exception("Sieve script path '%s' exists and it is "
                            "not a regular file" % sieve_script_path)

        # Active the script if necessary
        active_sieve_script_path = None
        if tail == 'sieve-script':  # Dovecot only?
            active_sieve_script_path = sieve_script_path
            sieve_script_path = os.path.join(head, SIEVE_SCRIPT_NAME + '.sieve')

        return (sieve_script_path, sieve_user_backup, active_sieve_script_path)

 def __copydir(self, src, dst, passes=None, fails=None, dry_run=False):
     """
     Make the given dst directory and copy stats from src
     Append dst to ``fails`` on error
     """
     if self.progressfnc:
         self.progressfnc('Copying to {0}'.format(dst), self.__getProgPercent())
     try:
         if not dry_run:
             os.mkdir(dst)
     except Exception as e:
         LOG.error(e)
         if fails is not None:
             fails.append(dst)
     else:
         if os.path.exists(dst) and self.runstngs['forceOwnership']:
             # make writable
             fileAtt = os.stat(dst)[0]
             if (not fileAtt & stat.S_IWRITE):
                 try:
                     os.chmod(dst, stat.S_IWRITE)
                 except Exception as e:
                     LOG.error('could not make file writable {0}: {1}'.format(dst, e))
                     return False
         shutil.copystat(src, dst)
         if passes is not None:
             passes.append(dst)
         LOG.debug('Created Directory: {0}'.format(dst))

def copyfile(source, dest, replace=None, replace_with=None):
    """
    Copy `source` to `dest`, preserving the modification time.

    If `replace` is given, instances of `replace` in the file contents are
    replaced with `replace_with`.
    """

    sfn = os.path.join("..", source)
    dfn = os.path.join("..", dest)

    if os.path.exists(dfn):
        if os.path.getmtime(sfn) <= os.path.getmtime(dfn):
            return

    sf = file(sfn, "rb")
    data = sf.read()
    sf.close()

    if replace:
        data = data.replace(replace, replace_with)

    df = file(dfn, "wb")
    df.write("# This file was automatically generated from " + source + "\n")
    df.write("# Modifications will be automatically overwritten.\n\n")
    df.write(data)
    df.close()

    import shutil
    shutil.copystat(sfn, dfn)

 def make_fake_special_file(f_type, f_path, src_dir, dst_dir):
     tgt_path = os.path.join(dst_dir, f_path + '.' + f_type)
     tgt_dir = os.path.dirname(tgt_path)
     if not os.path.exists(tgt_dir):
         os.makedirs(tgt_dir)
     open(tgt_path, 'w') # make zero byte file, immediately closed
     shutil.copystat(os.path.join(src_dir, f_path), tgt_path)

def _migrate_files(snap_files, snap_dirs, srcdir, dstdir, missing_ok=False,
                   follow_symlinks=False, fixup_func=lambda *args: None):

    for directory in snap_dirs:
        src = os.path.join(srcdir, directory)
        dst = os.path.join(dstdir, directory)
        os.makedirs(dst, exist_ok=True)
        shutil.copystat(src, dst, follow_symlinks=follow_symlinks)

    for snap_file in snap_files:
        src = os.path.join(srcdir, snap_file)
        dst = os.path.join(dstdir, snap_file)
        os.makedirs(os.path.dirname(dst), exist_ok=True)
        shutil.copystat(os.path.dirname(src), os.path.dirname(dst),
                        follow_symlinks=follow_symlinks)

        if missing_ok and not os.path.exists(src):
            continue

        # If the file is already here and it's a symlink, leave it alone.
        if os.path.islink(dst):
            continue

        # Otherwise, remove and re-link it.
        if os.path.exists(dst):
            os.remove(dst)

        if src.endswith('.pc'):
            shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
        else:
            common.link_or_copy(src, dst, follow_symlinks=follow_symlinks)

        fixup_func(dst)