def set_awk_path():
    # find awk
    awk_path = ''
    for path in os.environ['PATH'].split(':'):
        if os.path.isfile(os.path.join(path,'awk')):
            awk_path = os.path.join(path,'awk')
            break

    if awk_path == '':
        print >> sys.stderr, 'Cannot find Awk'
        exit(1)
    else:
        # change all scripts
        for awkf in glob.glob('scripts/*.awk'):
            os.rename(awkf,awkf+'.tmp')

            newf = open(awkf, 'w')
            print >> newf, '#!%s -f' % awk_path

            oldf = open(awkf+'.tmp')
            line = oldf.readline()
            line = oldf.readline()
            while line:
                print >> newf, line,
                line = oldf.readline()
            oldf.close()

            newf.close()
            shutil.copymode(awkf+'.tmp', awkf)
            os.remove(awkf+'.tmp')

    def copyf(src, dst, root):
        if dst.startswith("/"):
            dst = dst[1:]

        if os.path.isdir(src):
            #
            # Copy entire src directory to target directory
            #
            dstpath = os.path.join(root, dst)
            logger.debug("Copytree %s -> %s" % (src, dstpath))
            shutil.copytree(src, dstpath)
        else:
            #
            # If the destination ends in a '/' it means copy the filename
            # as-is to that directory.
            #
            # If not, its a full rename to the destination.
            #
            if dst.endswith("/"):
                dstpath = os.path.join(root, dst)
                if not os.path.exists(dstpath):
                    os.makedirs(dstpath)
                shutil.copy(src, dstpath)
            else:
                dstpath = os.path.join(root, os.path.dirname(dst))
                if not os.path.exists(dstpath):
                    os.makedirs(dstpath)
                shutil.copyfile(src, os.path.join(root, dst))
                shutil.copymode(src, os.path.join(root, dst))

def __process_template_folder(path, subs):
    items = os.listdir(path)
    processed_items = []
    for item in list(items):
        item = os.path.abspath(os.path.join(path, item))
        if os.path.basename(item) in ['.', '..', '.git', '.svn']:
            continue
        if os.path.isdir(item):
            sub_items = __process_template_folder(item, subs)
            processed_items.extend([os.path.join(item, s) for s in sub_items])
        if not item.endswith(TEMPLATE_EXTENSION):
            continue
        with open(item, 'r') as f:
            template = f.read()
        # Remove extension
        template_path = item[:-len(TEMPLATE_EXTENSION)]
        # Expand template
        info("Expanding '{0}' -> '{1}'".format(
            os.path.relpath(item),
            os.path.relpath(template_path)))
        result = em.expand(template, **subs)
        # Write the result
        with open(template_path, 'w') as f:
            f.write(result.encode('utf8'))
        # Copy the permissions
        shutil.copymode(item, template_path)
        processed_items.append(item)
    return processed_items

 def run(self):
     values = list()
     for argument in self.user_options:
         if argument[0].endswith('='):
             print argument[0][:-1],'is',
             print getattr(self, argument[0][:-1])
             values.append((argument[0][:-1], getattr(self, argument[0][:-1].replace('-','_'))))
         else:
             print "Found switch",argument,getattr(self, argument[0].replace('-','_'))
             values.append((argument[0], bool(getattr(self, argument[0].replace('-','_')))))
     print 'Replacing values in template files...'
     for item in os.listdir('in'):
         if item.endswith('.in'):
             print 'Replacing values in',item,
             original_name = os.path.join('in',item)
             item_in = open(original_name, 'r')
             final_name = item[:-3].replace('=','/')
             print final_name
             item_out = open(final_name, 'w')
             for line in item_in.readlines():
                 for item, value in values:
                     line = line.replace('%' + str(item.upper().replace('-','_')) + '%', str(value))
                 item_out.write(line)
             item_out.close()
             item_in.close()
             shutil.copymode(original_name, final_name)

def do_conf_file(src, dst, confdata, format):
    try:
        with open(src, encoding='utf-8') as f:
            data = f.readlines()
    except Exception as e:
        raise MesonException('Could not read input file %s: %s' % (src, str(e)))
    # Only allow (a-z, A-Z, 0-9, _, -) as valid characters for a define
    # Also allow escaping '@' with '\@'
    if format in ['meson', '[email protected]']:
        regex = re.compile(r'(?:\\\\)+(?=\\[email protected])|\\@|@([-a-zA-Z0-9_]+)@')
    elif format == 'cmake':
        regex = re.compile(r'(?:\\\\)+(?=\\?\$)|\\\${|\${([-a-zA-Z0-9_]+)}')
    else:
        raise MesonException('Format "{}" not handled'.format(format))

    search_token = '#mesondefine'
    if format != 'meson':
        search_token = '#cmakedefine'

    result = []
    missing_variables = set()
    for line in data:
        if line.startswith(search_token):
            line = do_mesondefine(line, confdata)
        else:
            line, missing = do_replacement(regex, line, format, confdata)
            missing_variables.update(missing)
        result.append(line)
    dst_tmp = dst + '~'
    with open(dst_tmp, 'w', encoding='utf-8') as f:
        f.writelines(result)
    shutil.copymode(src, dst_tmp)
    replace_if_different(dst, dst_tmp)
    return missing_variables

def sed_i(files, expr, replace_exp, only_first_occurrence=False):
    """
    Massively search/replace matching lines in files.

    Similar to:

    sed -i "s/expr/replace_expr/g" files...

    :type files: enumerate or list
    :param files: file names generator
    :type expr: str or pattern
    :type replace_exp: str
    :param only_first_occurrence: replace only first occurrence per line
    """
    r = _compiled_re(expr)
    for f in files:
        with open(f, 'r') as source:
            tmp_f = f + '.pygrep.tmp'
            with open(tmp_f, 'w') as dest:
                sed(source, r, replace_exp, dest, only_first_occurrence)

        shutil.copymode(f, tmp_f)
        ori_f = f + '.pygrep.ori'
        os.rename(f, ori_f)
        os.rename(tmp_f, f)
        os.remove(ori_f)

def copy_template(app_template, copy_to, app_name):
  """copies the specified template directory to the copy_to location"""

  app_name_spaces = " ".join(word.capitalize() for word in app_name.split("_"))
  app_name_camel = "".join(word.capitalize() for word in app_name.split("_"))

  # walks the template structure and copies it
  for directory, subdirs, files in os.walk(app_template):
    relative_dir = directory[len(app_template)+1:].replace('app_name_camel', app_name_camel).replace('app_name',app_name)
    if not os.path.exists(os.path.join(copy_to, relative_dir)):
      os.mkdir(os.path.join(copy_to, relative_dir))
    for f in files:
      if f.endswith('.pyc') or f.startswith("."):
        continue
        
      path_old = os.path.join(directory, f)
      path_new = os.path.join(copy_to, relative_dir, f.replace('app_name_camel', app_name_camel).replace('app_name', app_name))

      LOG.info("Writing %s" % path_new)
      fp_new = open(path_new, 'w')
      if path_old.endswith(".png"):
        shutil.copyfileobj(file(path_old), fp_new)
      else:
        fp_new.write( Template(filename=path_old).render(app_name=app_name, app_name_camel=app_name_camel, app_name_spaces=app_name_spaces) )
      fp_new.close()
        
      shutil.copymode(path_old, path_new)

    def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder):
        self.parent = parent
        if lib_builder:
            native_args = native_args + lib_builder(self.name, native=True, env_init={"CC": self.cc, "CXX": self.cxx})
        if not native_exec:
            compiler = self.cxx if filename.endswith("cpp") else self.cc
            process = Popen(
                [compiler, "-fno-math-errno", filename, "-o", filename + ".native"]
                + self.args
                + shared_args
                + native_args,
                stdout=PIPE,
                stderr=parent.stderr_redirect,
            )
            output = process.communicate()
            if process.returncode is not 0:
                print >> sys.stderr, "Building native executable with command failed"
                print "Output: " + output[0]
        else:
            shutil.copyfile(native_exec, filename + ".native")
            shutil.copymode(native_exec, filename + ".native")

        final = os.path.dirname(filename) + os.path.sep + self.name + "_" + os.path.basename(filename) + ".native"
        shutil.move(filename + ".native", final)
        self.filename = final

def copy_template_file(src, dest, replace=None):
    """
    Copy a source file to a new destination file.

    To replace boilerplate strings in the source data, pass a dictionary to the
    ``replace`` argument where each key is the boilerplate string and the
    corresponding value is the string which should replace it.
    """
    replace = replace or {}
    # Read the data from the source file.
    src_file = open(src, 'r')
    data = src_file.read()
    src_file.close()

    # Replace boilerplate strings.
    for old_val, new_val in replace.items():
        data = data.replace(old_val, new_val)

    # Write the data to the destination file.
    dest_file = open(dest, 'w')
    dest_file.write(data)
    dest_file.close()
    # Copy permissions from source file.
    shutil.copymode(src, dest)

    # Make new file writable.
    if os.access(dest, os.W_OK):
        st = os.stat(dest)
        new_permissions = stat.S_IMODE(st.st_mode) | stat.S_IWUSR
        os.chmod(dest, new_permissions)

  def WriteIndex(self):
    """Generate an index for libnss-cache from this map."""
    for index_name in self._indices:
      # index file write to tmp file first, magic string ".ix"
      tmp_index_filename = '%s.ix%s.tmp' % (self.GetCacheFilename(), index_name)
      self.log.debug('Writing index %s', tmp_index_filename)

      index = self._indices[index_name]
      key_length = LongestLength(index.keys())
      pos_length = LongestLength(index.values())
      max_length = key_length + pos_length
      # Open for write/truncate
      index_file = open(tmp_index_filename, 'w')
      # setup permissions
      try:
        shutil.copymode(self.GetCompatFilename(), tmp_index_filename)
        stat_info = os.stat(self.GetCompatFilename())
        uid = stat_info.st_uid
        gid = stat_info.st_gid
        os.chown(tmp_index_filename, uid, gid)
      except OSError, e:
        if e.errno == errno.ENOENT:
          os.chmod(tmp_index_filename,
                   stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH)
      for key in sorted(index):
        pos = index[key]
        index_line = ('%s\0%s\0%s\n' %
                      (key, pos,
                       '\0' * (max_length - len(key) - len(pos))))
        index_file.write(index_line)
      index_file.close()

def main():
    DB = db.open(config)
    tmppath = TXTPATH+".tmp"
    outfile = open(tmppath, 'w')
    count = util.Counter()
    storage = util.Counter()

    def onStatusUpdate(asset, status, db_asset):
        if status.status == bithorde.message.SUCCESS:
            if int(count):
                outfile.write(',\n')
            count.inc()
            storage.inc(status.size)
            json.dump(db_asset, outfile, cls=Encoder, indent=2)

    outfile.write('[')

    client = bithorde.BitHordeIteratorClient(list_db(DB), onStatusUpdate)
    bithorde.connectUNIX(UNIXSOCKET, client)
    bithorde.reactor.run()

    outfile.write(']')
    outfile.close()

    if os.path.exists(TXTPATH):
        shutil.copymode(TXTPATH, tmppath)
    os.rename(tmppath, TXTPATH)

    print "Exported %d assets, with %.2fGB worth of data." % (count, storage.inGibi())

        def compile_src(srcs, exe, for_evaluation, lang, assume=None):
            if lang != 'pas' or len(srcs) == 1:
                call(base_dir, get_compilation_command(
                    lang,
                    srcs,
                    exe,
                    for_evaluation=for_evaluation))

            # When using Pascal with graders, file naming conventions
            # require us to do a bit of trickery, i.e., performing the
            # compilation in a separate temporary directory
            else:
                tempdir = tempfile.mkdtemp()
                task_name = detect_task_name(base_dir)
                new_srcs = [os.path.split(srcs[0])[1],
                            '%s.pas' % (task_name)]
                new_exe = os.path.split(srcs[1])[1][:-4]
                shutil.copyfile(os.path.join(base_dir, srcs[0]),
                                os.path.join(tempdir, new_srcs[0]))
                shutil.copyfile(os.path.join(base_dir, srcs[1]),
                                os.path.join(tempdir, new_srcs[1]))
                lib_filename = '%slib.pas' % (task_name)
                if os.path.exists(os.path.join(SOL_DIRNAME, lib_filename)):
                    shutil.copyfile(os.path.join(SOL_DIRNAME, lib_filename),
                                    os.path.join(tempdir, lib_filename))
                call(tempdir, get_compilation_command(
                    lang,
                    new_srcs,
                    new_exe,
                    for_evaluation=for_evaluation))
                shutil.copyfile(os.path.join(tempdir, new_exe),
                                os.path.join(base_dir, exe))
                shutil.copymode(os.path.join(tempdir, new_exe),
                                os.path.join(base_dir, exe))
                shutil.rmtree(tempdir)

def buildpyMySQL(mysqlLocation):

    # Unzip the pyMySQL zip file
    zipfile.ZipFile('lib/pyMySQL.zip').extractall(path='lib')

    # Change to the directory of the src code for pyMySQL
    os.chdir('lib/pyMySQL/src')
    
    # Builds the arg list
    buildArgs = [sys.executable, 'pyMySQL_setup.py', 'build']
    
    if mysqlLocation is not None:
        buildArgs.append('--with-mysql=' + mysqlLocation)
    
    # Attempts to build the pyMySQL module
    retcode = subprocess.call(buildArgs)

    if retcode != 0:
        sys.exit('Error building pyMySQL C extension module')
        
    # Gets the filename of library
    libfiles = glob.glob('build/lib.*/pyMySQL.so')
        
    # Checks the file exists
    if len(libfiles) == 0:
        sys.exit('Error building pyMySQL C extension module')
        
    # Copies the file to the lib directory
    shutil.copyfile(libfiles[0], '../../pyMySQL.so')
    shutil.copymode(libfiles[0], '../../pyMySQL.so')

    # Return to the original directory
    os.chdir('../../../')

    def process_config_file(self, f, install_dir, **kwargs):

        # The path where the weewx.conf configuration file will be installed
        install_path = os.path.join(install_dir, os.path.basename(f))
    
        new_config = merge_config_files(f, install_path, install_dir)
        # Get a temporary file:
        tmpfile = tempfile.NamedTemporaryFile("w", 1)
        
        # Write the new configuration file to it:
        new_config.write(tmpfile)
        
        # Save the old config file if it exists:
        if os.path.exists(install_path):
            backup_path = save_path(install_path)
            print "Saved old configuration file as %s" % backup_path
            
        # Now install the temporary file (holding the merged config data)
        # into the proper place:
        rv = install_data.copy_file(self, tmpfile.name, install_path, **kwargs)
        
        # Set the permission bits unless this is a dry run:
        if not self.dry_run:
            shutil.copymode(f, install_path)

        return rv

def copy_template_file(src, dest, replace=None):
    """
    Copy a source file to a new destination file.
    
    To replace boilerplate strings in the source data, pass a dictionary to the
    ``replace`` argument where each key is the boilerplate string and the
    corresponding value is the string which should replace it.
    
    """
    replace = replace or {}
    # Read the data from the source file.
    src_file = open(src, 'r')
    data = src_file.read()
    src_file.close()
    # Replace boilerplate strings.
    for old_val, new_val in replace.items():
        data = data.replace(old_val, new_val)

    # Generate SECRET_KEY for settings file
    secret_key = ''.join([choice('[email protected]#$%^&*(-_=+)') for i in range(50)])
    data = re.sub(r"(?<=SECRET_KEY = ')'", secret_key + "'", data)

    # Write the data to the destination file.
    dest_file = open(dest, 'w')
    dest_file.write(data)
    dest_file.close()
    # Copy permissions from source file.
    shutil.copymode(src, dest)
    # Make new file writable.
    if os.access(dest, os.W_OK):
        st = os.stat(dest)
        new_permissions = stat.S_IMODE(st.st_mode) | stat.S_IWUSR
        os.chmod(dest, new_permissions)

def copytree(src, dest, symlink=None):
    """ 
    This is the same as shutil.copytree, but doesn't error out if the 
    directories already exist.

    """
    for root, dirs, files in os.walk(src, True):
        for d in dirs:
            if d.startswith('.'):
                continue
            srcpath = os.path.join(root, d)
            destpath = os.path.join(dest, root, d)
            if symlink and os.path.islink(srcpath):
                if os.path.exists(destpath):
                    os.remove(destpath)
                os.symlink(os.readlink(srcpath), destpath)
            elif not os.path.isdir(destpath):
                os.makedirs(destpath)
                try: 
                    shutil.copymode(srcpath, destpath)
                except: pass
                try: 
                    shutil.copystat(srcpath, destpath)
                except: pass
        for f in files:
            if f.startswith('.'):
                continue
            srcpath = os.path.join(root, f)
            destpath = os.path.join(dest, root, f)
            if symlink and os.path.islink(srcpath):
                if os.path.exists(destpath):
                    os.remove(destpath)
                os.symlink(os.readlink(srcpath), destpath)
            else:
                shutil.copy2(srcpath, destpath)

def save_file(filename, data, encoding, keep_tmp=False):
    tmpfilename = realpath(filename) + '.bak'

    try:
        f = open(tmpfilename, 'w')
    except IOError:
        dname = dirname(tmpfilename)
        if not exists(dname):
            os.makedirs(dname, mode=0755)
            f = open(tmpfilename, 'w')
        else:
            raise

    f.write(data.encode(encoding))
    f.close()

    if exists(filename):
        try:
            shutil.copymode(filename, tmpfilename)
        except OSError:
            pass

    if keep_tmp:
        return tmpfilename
    else:
        os.rename(tmpfilename, filename)

def copy_template(template_name, copy_to, tag_library_name):
    """copies the specified template directory to the copy_to location"""
    import django_extensions
    import shutil

    template_dir = os.path.join(django_extensions.__path__[0], 'conf', template_name)

    # walks the template structure and copies it
    for d, subdirs, files in os.walk(template_dir):
        relative_dir = d[len(template_dir) + 1:]
        if relative_dir and not os.path.exists(os.path.join(copy_to, relative_dir)):
            os.mkdir(os.path.join(copy_to, relative_dir))
        for i, subdir in enumerate(subdirs):
            if subdir.startswith('.'):
                del subdirs[i]
        for f in files:
            if f.endswith('.pyc') or f.startswith('.DS_Store'):
                continue
            path_old = os.path.join(d, f)
            path_new = os.path.join(copy_to, relative_dir, f.replace('sample', tag_library_name))
            if os.path.exists(path_new):
                path_new = os.path.join(copy_to, relative_dir, f)
                if os.path.exists(path_new):
                    continue
            path_new = path_new.rstrip(".tmpl")
            fp_old = open(path_old, 'r')
            fp_new = open(path_new, 'w')
            fp_new.write(fp_old.read())
            fp_old.close()
            fp_new.close()
            try:
                shutil.copymode(path_old, path_new)
                _make_writeable(path_new)
            except OSError:
                sys.stderr.write("Notice: Couldn't set permission bits on %s. You're probably using an uncommon filesystem setup. No problem.\n" % path_new)

    def autoconf(self, args=[], inplace=False):
        for dirname, filename in util.walk_files(self._path):
            path = os.path.join(dirname, filename)
            if filename == 'config.sub':
                # Replace config.sub files by an up-to-date copy. The copy
                # provided by the tarball rarely supports CloudABI.
                shutil.copy(
                    os.path.join(
                        config.DIR_RESOURCES,
                        'config.sub'),
                    path)
            elif filename == 'configure':
                # Patch up configure scripts to remove constructs that are known
                # to fail, for example due to functions being missing.
                with open(path, 'r') as fin:
                    with open(path + '.new', 'w') as fout:
                        for l in fin.readlines():
                            # Bad C99 features test.
                            if l.startswith('#define showlist(...)'):
                                l = '#define showlist(...) fputs (stderr, #__VA_ARGS__)\n'
                            elif l.startswith('#define report(test,...)'):
                                l = '#define report(...) fprintf (stderr, __VA_ARGS__)\n'
                            fout.write(l)
                shutil.copymode(path, path + '.new')
                os.rename(path + '.new', path)

        # Run the configure script in a separate directory.
        builddir = self._path if inplace else self._builder.get_new_directory()
        self._builder.autoconf(
            builddir, os.path.join(self._path, 'configure'), args)
        return FileHandle(self._builder, builddir)

 def apply(self):
     "merges all decisions for this proposal (and those with lower revisions)"
     file_path = self.get_file_path()
     merged_configpath = file_path + '.merged'
     bak_configpath = file_path + '.bak'
     fd = open(merged_configpath, 'w')
     fd.writelines(self.get_merged_content())
     fd.close()
     try:
         shutil.copymode(file_path, merged_configpath)
         shutil.copystat(file_path, merged_configpath)
         shutil.move(file_path, bak_configpath)
     except OSError:
         pass
     shutil.move(merged_configpath, file_path)
     try:
         os.unlink(file_path + '.bak')
     except OSError:
         pass
     self.base_lines = None
     self._changes = None
     try:
         os.unlink(self.path)
     except OSError:
         pass
     self.clear_state()