def env_init(site_name=SITE_NAME):
    '''Initialize with this site hostname.'''
    print(green("Initializing new site configuration..."))

    #
    # Generate secret key and update config file
    #
    import random
    import string

    CHARS = string.ascii_letters + string.digits
    SECRET_KEY = "".join([random.choice(CHARS) for i in range(50)])

    print(blue("Configuring the secret key..."))
    os.chdir(PROJ_DIR)
    try:
        sh.sed("-i ",
               "s/SECRET_KEY *=.*/SECRET_KEY = '{0}'/g".format(SECRET_KEY),
               "{0}/config.py".format(APP_NAME))
    except sh.ErrorReturnCode:
        print(red("Could not configure SECRET_KEY for config.py"))
        exit(1)

    #
    # Set the site name, the user defined site hostname
    #
    print(blue("Configuring the SITE_NAME '{0}'.".format(site_name)))
    try:
        sh.sed("-i ",
               "s/SITE_NAME *=.*/SITE_NAME = '{0}'/g".format(site_name),
               "{0}/config.py".format(APP_NAME))
    except sh.ErrorReturnCode:
        print(red("Could not configure SITE_NAME for config.py"))
        exit(1)

def main(logger=mylog.default_logger()):
    # load arguments and logger
    arguments = docopt(__doc__, version='0.0')
    # print arguments
    # script self name
    self_name=os.path.basename(sys.argv[0])
    # log
    # logfile=self_name.replace('py','log')
    # logger=set_mylogger(arguments,logfile)
    # load config
    # main_config=load_config('.ll')

    # set filename varibles
    # main_title=os.path.basename(os.getcwd())
    # markdown_file_name=os.path.join(os.getcwd(),'build',main_title+'.markdown')
    # docx_file_name=main_title+'.docx'
    # rtf_file_name=main_title+'.rtf'

    md_pattern=re.compile(r'\.md$')

    markdown_file_name=arguments.get('<filename>')
    if markdown_file_name:
        if os.path.isdir(markdown_file_name):
            walk_sed(markdown_file_name)
        else:
            sh.sed('-i','-e','s/^#//',markdown_file_name)
            # sh.sed('-i','-e','s/^#/'+'#/',markdown_file_name)
    else:
        walk_sed('.')

def check_upgrade():
        server_file = curl(PIFM_HOST + '/client_agent/pifm_agent.py')
        server_sum = awk(
                        md5sum(
                            grep(server_file, '-v', '^PIFM_HOST')
                        ), '{print $1}'
        )

        local_sum = awk(
                        md5sum(
                            grep('-v', '^PIFM_HOST', OUR_SCRIPT)
                        ), '{print $1}'
        )

        if str(server_sum) != str(local_sum):
            logging.info(
                "server: {server}, local: {local}, should update.".format(
                    server=server_sum,
                    local=local_sum
                )
            )
            with open(TMP_SCRIPT, 'w') as f:
                f.write(str(server_file))
            sed('-i',
                "0,/def/ s#http://pi_director#{myhost}#".format(myhost=PIFM_HOST),
                OUR_SCRIPT
                )
            status = python('-m', 'py_compile', TMP_SCRIPT)
            if (status.exit_code == 0):
                shutil.copy(TMP_SCRIPT, OUR_SCRIPT)
                os.chmod(OUR_SCRIPT, 0755)
                sys.exit(0)

    def html_output(self):
        ext = '.html'
        today = datetime.date.today().isoformat()
        sha = self.test_file + ".html.sha"
        # cannot recover if generating html fails
        options = (['--zip'] + self.options
                   + ['-f', 'html', self.test_file,
                      self.test_out + ext + '.zip'])
        try:
            self.gdoc_to(*options,
                         _err=self.test_err + ".html.log")
            # XXX it hangs without -n, didn't have time to figure out why
            out_dir = os.path.dirname(self.test_out)
            sh.unzip('-n', '-d', out_dir, self.test_out + ext + '.zip')
            sh.sed('-i', '-e', 's/%s/TODAYS_DATE/g' % today,
                   self.test_out + ext)
            test_result = slurp('%s.html' % self.test_out)
        except ErrorReturnCode as e:
            self.say(red("gdoc-to failed: {}. See {}.html.log"),
                     e, self.test_err)
            self.say(red("Ran in {}"), os.getcwd())
            self.failed = True
            sh.rm('-f', sha)
            return
        try:
            html5check(self.test_out + ext,
                       _out=self.test_out + ".html.errors")
        except ErrorReturnCode:
            self.say(red("Test output did not validate as XHTML5!"))
            self.say(red("\tSee {}.html.errors"), self.test_out)
            self.failed = True

        if test_result != slurp(self.test_file + ext):
            # the file changed, but the change might be okay
            spit(self._canonical_body(self.test_out + ext),
                 self.test_out + ".body")
            spit(self._canonical_body(self.test_file + ext),
                 self.test_out + ".canon.body")

            if (slurp(self.test_out + '.body')
                    == slurp(self.test_out + '.canon.body')):
                self.say(yellow("File changed. Updating canonical file."))
                sh.cp(self.test_out + ext, self.test_file + ext)
            else:
                self.say(red("HTML body changed!"))
                self.say(red("\tSee {}.*"), fail_path(self.test_name))
                sh.cp(self.test_out + ext, fail_path(self.test_name + ext))
                sh.diff('-u', self.test_file + ext, self.test_out + ext,
                        _out=fail_path(self.test_name + ".html.diff"),
                        _ok_code=[0, 1])
                sh.cp(self.test_out + ".body",
                      fail_path(self.test_name + ".body"))
                sh.cp(self.test_out + ".canon.body",
                      fail_path(self.test_name + ".body.expected"))
                sh.diff('-u', self.test_out + ".canon.body",
                        self.test_out + ".body",
                        _out=fail_path(self.test_name + '.body.diff'),
                        _ok_code=[0, 1])
                self.failed = True

def walk_sed(dirname):
    md_pattern=re.compile(r'\.md$')
    list_dirs=os.walk(dirname)
    for root,dirs,files in list_dirs:
        for f in files:
            if md_pattern.search(f):

                sh.sed('-i','-e','s/^#//',os.path.join(root, f))

    def _add_description(self):
        """ adds description to control file"""
        if not self.description or not len(self.description) <= 60:
            logger.warning('bad description, using default pattern')
            self.description = '{} Package'.format(self.package_name)

        sh.sed('-i', r'/^Description/c\\Description: {}'
               .format(self.description),
               self.debian_package_path + '/debian/control').wait()

    def _add_deb_dependencies(self):
        """ adds deb dependencies to control file """
        if self.dependencies:
            debian_dependencies = self.dependencies_to_debian_format(
                self.dependencies)
            dependencies_str = ', '.join(debian_dependencies)

            sh.sed('-i', r'/^Depends/c\\Depends: ${{misc:Depends}}, {}'
                   .format(dependencies_str),
                   self.debian_package_path + '/debian/control').wait()

def add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec):
    sh.curl(
        '-s', '-k', '-L',
        "https://github.com/crowbar/%s/compare/%s...%s.patch" % (
            repo, pr_branch, head_sha1),
        '-o', 'prtest.patch')
    sh.sed('-i', '-e', 's,Url:.*,%define _default_patch_fuzz 2,',
           '-e', 's,%patch[0-36-9].*,,', spec)
    Command('/usr/lib/build/spec_add_patch')(spec, 'prtest.patch')
    iosc('vc', '-m', "added PR test patch from %s#%s (%s)" % (
        repo, pr_id, head_sha1))

 def phred_13_to_18_sed(self, new_path=None, in_place=True):
     """Illumina-1.3 format conversion to Illumina-1.8 format via sed (faster)."""
     # String #
     sed_command = r"""4~4y/@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghi/!"#$%&'\''()*+,-.\/0123456789:;<=>[email protected]/"""
     # Faster with bash utilities #
     if in_place is True:
         sh.sed('-i', sed_command, self.path)
         return self
     # New file #
     if new_path is None: new_fastq = self.__class__(new_temp_path())
     else:                new_fastq = self.__class__(new_path)
     sh.sed(sed_command + " " + new_fastq, self.path)
     return new_fastq

def dfs_dir(root_dir,markdown_file,level,logger=mylog.default_logger()):
    # generate the filelist, ignore the hidden files.
    hiden_file=re.compile(r'^\.|main_title\.md|\.synopsis$|\.markdown')
    effective_file=re.compile(r'(^.*_+|^[0-9]*_*)(.*)\.md$')
    efflists=[ filename for filename in os.listdir(root_dir) if  effective_file.match(filename)]
    efflists_title=[ effective_file.match(filename).group(2) for filename in os.listdir(root_dir) if  effective_file.match(filename)]

    alllists=[ filename for filename in os.listdir(root_dir) if not hiden_file.match(filename)]

    #
    for filename,title in zip(efflists,efflists_title):
        path = os.path.join(root_dir, filename)
        # print logger
        logger.debug(filename+title)


        # write the file content according to level
        #TODO: 文件夹的处理,文件夹名+.md, .synopsis的处理
        if os.path.isdir(path):
            markdown_file.write('#'*level+title)
            markdown_file.write('\n')
            # if os.path.exists(str(path)+'/00.content'):
                # markdown_file.write(str(sh.cat(path+'/00.content'))+'\n')
            dfs_dir(path, markdown_file, level+1)
        else:
            if title:
                markdown_file.write('#'*level+title)
            markdown_file.write('\n')
            markdown_file.write(str(sh.sed('s/^#/'+'#'*level+'#/',path)))

 def remove_trailing_stars(self, new_path=None, in_place=True, check=False):
     """Remove the bad character that can be inserted by some programs at the
     end of sequences."""
     # Optional check #
     if check and int(sh.grep('-c', '\*', self.path, _ok_code=[0,1])) == 0: return self
     # Faster with bash utilities #
     if in_place is True:
         sh.sed('-i', 's/\*$//g', self.path)
         return self
     # Standard way #
     if new_path is None: new_fasta = self.__class__(new_temp_path())
     else:                new_fasta = self.__class__(new_path)
     new_fasta.create()
     for seq in self: new_fasta.add_str(str(seq.seq).rstrip('*'), seq.id)
     new_fasta.close()
     return new_fasta

    def capture_screen(self):
        """
        Use adb shell screencap
        :return: CV2Img
        """
        img = CV2Img()
        result = sed(adb("shell", "screencap", "-p"), 's/\r$//')

        return img.load_binary(result.stdout)

def device_list():
    """
    Get udid from iPhone and iPad plugged into the computer
    Returns a list
    """

    # TODO: separate iPhone and iPad
    raw = sh.sed(sh.system_profiler("SPUSBDataType"), "-n", "-e",
                 '/iPad/,/Serial/p', "-e", '/iPhone/,/Serial/p')
    devices = sh.awk(
        sh.grep(raw, "Serial Number:"), "-F", ": ", "{print $2}").split('\n')
    return [device for device in devices if device]

	def import_csv(self, username, userpass, filestream):
		retCode = True
		import_xml_command = sh.Command("/usr/share/ds-matricula-plugin/matricula-common-scripts/1-itaca2mysql")
		# set a temporary filename
		TMP_CSV = tempfile.mkstemp()[1]
		TMP_CSV2 = tempfile.mkstemp()[1]
		if self._put_file(TMP_CSV, filestream):
			TMP_XML = tempfile.mkstemp()[1]
			sh.sed(sh.iconv("-f", "ISO-8859-15", "-t", "UTF-8", TMP_CSV), "-e", "1{s%[[:blank:]]%%g;s%\.%%g;s%[ñÑ]%n%g;s%.*%\L&%}", _out=TMP_CSV2)

			if self._csv2xml(TMP_CSV2, TMP_XML):
				try:
					import_xml_command(TMP_XML)
				except ErrorReturnCode:
					# some error happened
					retCode = False

			os.remove(TMP_XML)
		os.remove(TMP_CSV)
		os.remove(TMP_CSV2)
		return retCode

 def build_arch(self, arch):
     options_iphoneos = (
         "-isysroot {}".format(arch.sysroot),
         "-DOPENSSL_THREADS",
         "-D_REENTRANT",
         "-DDSO_DLFCN",
         "-DHAVE_DLFCN_H",
         "-fomit-frame-pointer",
         "-fno-common",
         "-O3"
     )
     build_env = arch.get_env()
     target = arch_mapper[arch.arch]
     shprint(sh.env, _env=build_env)
     sh.perl(join(self.build_dir, "Configure"),
             target,
             _env=build_env)
     if target == 'iphoneos-cross':
         sh.sed("-ie", "s!^CFLAG=.*!CFLAG={} {}!".format(build_env['CFLAGS'],
                " ".join(options_iphoneos)),
                "Makefile")
         sh.sed("-ie", "s!static volatile sig_atomic_t intr_signal;!static volatile intr_signal;! ",
                "crypto/ui/ui_openssl.c")
     else:
         sh.sed("-ie", "s!^CFLAG=!CFLAG={} !".format(build_env['CFLAGS']),
                "Makefile")
     shprint(sh.make, "clean")
     shprint(sh.make, "-j4", "build_libs")

    def set_hostname(self, hostname):
        """Update hostname

            Args:
                hostname (str): hostname to be updated
        """
        try:
            old_hostname = self.get_hostname()

            is_valid_hostname(hostname)

            sh.hostname("-b", hostname)
            sh.echo(hostname, _out="/etc/hostname")

            try:
                # sed -i 's/ old$/ new/g' /etc/hosts
                sh.sed("-i", "s/ {}$/ {}/g".format(old_hostname, hostname),
                       "/etc/hosts")
            except:
                with open("/etc/hosts", "a") as f:
                    f.write("127.0.0.1       localhost {}\n".format(hostname))
            self.update(id=1, newObj={"hostname": hostname})
        except Exception as e:
            raise e

def get_container_info():
    container_info = {}

    try:
        container_info['container_name'] = tail(sed(grep(cat('/proc/self/cgroup'), 'docker'), 's/^.*\///'), '-n1')
    except:
        container_info['container_name'] = 'unknown'

    try:
        container_info['app_version'] =  open('version', 'r').read()
    except:
        container_info['app_version'] = 'unknown'


    return container_info

def dfs_dir_byplist(root_dir,markdown_file,level,logger=mylog.default_logger()):
    # generate the filelist, ignore the hidden files.

    effective_file=re.compile(r'(^.*_+|^[0-9]*_*)(.*)\.md$')
    plist_file_name=os.path.join(root_dir,'.Ulysses-Group.plist')
    print plist_file_name
    print root_dir, level
    filename_list,dirname_list=get_name_from_plist(plist_file_name)
    print filename_list
    alllists=filename_list+dirname_list
    efflists=[ filename for filename in filename_list if  effective_file.match(filename)]+dirname_list
    print efflists
    # efflists_title=[ effective_file.match(filename).group(2) for filename in alllists if  effective_file.match(filename)]


    #

    for filename in efflists:
        path = os.path.join(root_dir, filename)
        # print logger
        # logger.debug(filename+title)


        # write the file content according to level
        #TODO: 文件夹的处理,文件夹名+.md, .synopsis的处理
        if os.path.isdir(path):
            # markdown_file.write('#'*level+title)
            # markdown_file.write('\n\n')
            # if os.path.exists(str(path)+'/00.content'):
                # markdown_file.write(str(sh.cat(path+'/00.content'))+'\n')
            dfs_dir_byplist(path, markdown_file, level+1)
        else:
            # if title:
                # markdown_file.write('#'*level+title)
            # markdown_file.write('\n\n')
            # markdown_file.write(str(sh.sed('s/^#/'+'#'*level+'#/',path)))
            markdown_file.write(str(sh.sed('s/^#/'+'#'+'/',path)))
            markdown_file.write('\n\n')

 def add(source_repo, f, apt_file):
     if not re.search(source_repo, f.read()):
         lgr.debug('Adding source repository {0}'.format(source_repo))
         sh.sed('-i', "2i {0}".format(source_repo), apt_file)
     else:
         lgr.debug('Repo {0} already added.'.format(source_repo))

 def test_stdin_from_string(self):
     from sh import sed
     self.assertEqual(sed(_in="test", e="s/test/lol/").strip(), "lol")