def make_doc(ctx):
    with cd("docs_rst"):
        ctx.run("cp ../CHANGES.rst change_log.rst")
        ctx.run("sphinx-apidoc -d 6 -o . -f ../matgendb")
        ctx.run("rm matgendb*.tests.rst")
        for f in glob.glob("*.rst"):
            if f.startswith('matgendb') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("pymatgen") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        ctx.run("make html")
        ctx.run("cp _static/* ../docs/html/_static")

    with cd("docs"):
        ctx.run("cp -r html/* .")
        ctx.run("rm -r html")
        # Avoid ths use of jekyll so that _dir works as intended.
        ctx.run("touch .nojekyll")

def update_doc(ctx):
    with cd("docs/_build/html/"):
        ctx.run("git pull")
    make_doc(ctx)
    with cd("docs/_build/html/"):
        ctx.run("git add .")
        ctx.run('git commit -a -m "Update dev docs"')
        ctx.run("git push origin gh-pages")

def make_doc(ctx):
    with open("CHANGES.rst") as f:
        contents = f.read()

    toks = re.split("\-{3,}", contents)
    n = len(toks[0].split()[-1])
    changes = [toks[0]]
    changes.append("\n" + "\n".join(toks[1].strip().split("\n")[0:-1]))
    changes = ("-" * n).join(changes)

    with open("docs_rst/latest_changes.rst", "w") as f:
        f.write(changes)

    with cd("examples"):
       ctx.run("jupyter nbconvert --to html *.ipynb")
       ctx.run("mv *.html ../docs_rst/_static")

    with cd("docs_rst"):
        ctx.run("cp ../CHANGES.rst change_log.rst")
        ctx.run("sphinx-apidoc --separate -d 6 -o . -f ../pymatgen")
        ctx.run("rm pymatgen*.tests.*rst")
        for f in glob.glob("*.rst"):
            if f.startswith('pymatgen') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("pymatgen") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        ctx.run("make html")
        
        ctx.run("cp _static/* ../docs/html/_static")

    with cd("docs"):
        ctx.run("cp -r html/* .")
        ctx.run("rm -r html")
        ctx.run("rm -r doctrees")
        ctx.run("rm -r _sources")

        # This makes sure pymatgen.org works to redirect to the Gihub page
        ctx.run("echo \"pymatgen.org\" > CNAME")
        # Avoid the use of jekyll so that _dir works as intended.
        ctx.run("touch .nojekyll")

def make_doc(ctx):
    with cd("docs_rst"):
        ctx.run("sphinx-apidoc -o . -f ../fireworks")
        ctx.run("make html")

    with cd("docs"):
        ctx.run("cp -r html/* .")
        ctx.run("rm -r html")
        ctx.run("rm -r doctrees")

        # Avoid the use of jekyll so that _dir works as intended.
        ctx.run("touch .nojekyll")

def make_doc(ctx):
    with cd("docs_rst"):
        ctx.run("python featurizer_summary.py > featurizer_summary.rst")
        ctx.run("python dataset_summary.py > dataset_summary.rst")
        ctx.run("sphinx-apidoc -o . -f ../matminer")
        ctx.run("make html")
        ctx.run("cp _static/* ../docs/html/_static")

    with cd("docs"):
        ctx.run("cp -r html/* .")
        ctx.run("rm -r html")
        ctx.run("rm -r doctrees")

        # Avoid the use of jekyll so that _dir works as intended.
        ctx.run("touch .nojekyll")

def make_doc(ctx):
    with cd("docs"):
        ctx.run("sphinx-apidoc -o . -f ../monty")
        for f in glob.glob("docs/*.rst"):
            if f.startswith('docs/monty') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("monty") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        ctx.run("make html")

        #This makes sure monty.org works to redirect to the Gihub page
        ctx.run("echo \"monty.org\" > _build/html/CNAME")
        #Avoid ths use of jekyll so that _dir works as intended.
        ctx.run("touch _build/html/.nojekyll")

    def gnuplot(self):
        """
        Plot the results with gnuplot.
        Based on the `replot.sh` script provided by the oncvpsp code.
        """
        outfile = self.filepath
        base = os.path.basename(outfile)
        gnufile = base + ".scr"
        plotfile = base + ".plot"
        temp = base + ".tmp"

        from monty.os import cd
        from subprocess import check_call
        workdir = tempfile.mkdtemp()
        print("Working in %s" % workdir)

        with cd(workdir):
            check_call("awk 'BEGIN{out=0};/GNUSCRIPT/{out=0}; {if(out == 1) {print}}; \
                                /DATA FOR PLOTTING/{out=1}' %s > %s" % (outfile, plotfile), shell=True)

            check_call("awk 'BEGIN{out=0};/END_GNU/{out=0}; {if(out == 1) {print}}; \
                                /GNUSCRIPT/{out=1}' %s > %s" % (outfile, temp), shell=True)

            check_call('sed -e 1,1000s/t1/"%s"/ %s > %s' % (plotfile, temp, gnufile), shell=True)

            try:
                check_call(["gnuplot", gnufile])
            except KeyboardInterrupt:
                print("Received KeyboardInterrupt")

        os.rmdir(workdir)

def makedoc(ctx):
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "matgendb.webui.settings")
    with cd("docs"):
        ctx.run("sphinx-apidoc -o . -f ../matgendb")
        ctx.run("rm matgendb*.tests.rst")
        for f in glob.glob("docs/*.rst"):
            if f.startswith('docs/matgendb') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("matgendb") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))

        ctx.run("make html")
        ctx.run("cp favicon.ico _build/html/_static/favicon.ico")

def pytest(ctx):
    pytest_cmd = r"""\
pytest -n 2 --cov-config=.coveragerc --cov=abiflows -v --doctest-modules abiflows \
    --ignore=abiflows/fireworks/integration_tests
"""
    with cd(ROOTDIR):
        ctx.run(pytest_cmd, pty=True)

def make_doc(ctx):
    with cd("docs"):
        ctx.run("sphinx-apidoc -o . -f ../custodian")
        ctx.run("rm custodian*.tests.rst")
        for f in glob.glob("docs/*.rst"):
            if f.startswith('docs/custodian') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("custodian") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))

        ctx.run("make html")

def write_input(temp, params, xkeys, parr=True, func='pbe'):
    '''
    Method to write QE input file in scratch dir from template.
    :param temp (list): Returned from get_template() method.
    :param params (dict): Dict of parameters to pass into template. Keys
        are the strings within the placeholders in the template.
    :param xkeys ([str]): Parameters to distinguish between different
        input files.
    :param parr (bool): Whether use parallel mode (send the calculation
        to computational nodes). Default to True. Note in non-parallel
        (serial, run the calculation on login nodes), you need to cp the
        psp files into scratch dir.
    :param func (str): Functional used. Default to GGA. Refer to the
        format of filename of template files.
    :return: None
    '''
    t, info = temp
    pl = [str(params[k]) for k in xkeys]
    jobname = "_".join(info+pl)
    if parr:
        c = Composition(info[0])
        elements = [e.symbol for e in c.elements]
        psp = {}
        for e in elements:
            psp[e] = os.path.abspath(glob.glob('../%s.%s*.UPF' % (e, func))[0])
        os.makedirs(jobname)
        with cd(jobname):
            _write_input_from_temp(t, jobname, params)
            for e in psp.keys():
                shutil.copyfile(psp[e],os.path.split(psp[e])[1])
    else:
        _write_input_from_temp(t, jobname, params)

 def test_continue(self):
     # Test the continuation functionality
     with cd(os.path.join(test_dir, 'postprocess')):
         # Test default functionality
         with ScratchDir('.', copy_from_current_on_enter=True) as d:
             v = VaspJob("hello", auto_continue=True)
             v.setup()
             self.assertTrue(os.path.exists("continue.json"), "continue.json not created")
             v.setup()
             self.assertEqual(Poscar.from_file("CONTCAR").structure,
                              Poscar.from_file("POSCAR").structure)
             self.assertEqual(Incar.from_file('INCAR')['ISTART'], 1)
             v.postprocess()
             self.assertFalse(os.path.exists("continue.json"),
                              "continue.json not deleted after postprocessing")
         # Test explicit action functionality
         with ScratchDir('.', copy_from_current_on_enter=True) as d:
             v = VaspJob("hello", auto_continue=[{"dict": "INCAR",
                                                  "action": {"_set": {"ISTART": 1}}}])
             v.setup()
             v.setup()
             self.assertNotEqual(Poscar.from_file("CONTCAR").structure,
                                 Poscar.from_file("POSCAR").structure)
             self.assertEqual(Incar.from_file('INCAR')['ISTART'], 1)
             v.postprocess()

 def test_cd_exception(self):
     try:
         with cd(test_dir):
             self.assertTrue(os.path.exists("empty_file.txt"))
             raise RuntimeError()
     except:
         pass
     self.assertFalse(os.path.exists("empty_file.txt"))

def pytest(ctx):
    pytest_cmd = r"""\
pytest -n 2 --cov-config=.coveragerc --cov=abipy -v --doctest-modules abipy \
    --ignore=abipy/integration_tests --ignore=abipy/data/refs --ignore=abipy/scripts/ \
    --ignore=abipy/examples/plot --ignore=abipy/examples/flows --ignore=abipy/gui
"""
    with cd(ABIPY_ROOTDIR):
        ctx.run(pytest_cmd, pty=True)

 def test_postprocess(self):
     with cd(test_dir):
         with ScratchDir('.', copy_from_current_on_enter=True):
             f = FeffJob("hello", backup=True, gzipped=True)
             f.postprocess()
             self.assertTrue(os.path.exists('feff_out.1.tar.gz'))
             f.postprocess()
             self.assertTrue(os.path.exists('feff_out.2.tar.gz'))

 def test_setup(self):
     with cd(test_dir):
         with ScratchDir('.', copy_from_current_on_enter=True) as d:
             v = VaspJob("hello")
             v.setup()
             incar = Incar.from_file("INCAR")
             count = multiprocessing.cpu_count()
             if count > 1:
                 self.assertGreater(incar["NPAR"], 1)

def proc_dic(rootfolder, filename, sub):

    filename_pre = rootfolder.rsplit('/', 1)[-1]
    rename = filename_pre + '.' + filename.split('.')[-1]
    command = 'mv F.out {}'.format(rename)
    # print filename
    with cd(rootfolder):
        os.system(command)
    print command

 def test_setup(self):
     with cd(test_dir):
         with ScratchDir('.', copy_from_current_on_enter=True) as d:
             v = VaspJob("hello", auto_npar=True)
             v.setup()
             incar = Incar.from_file("INCAR")
             count = multiprocessing.cpu_count()
             # Need at least 3 CPUs for NPAR to be greater than 1
             if count > 3:
                 self.assertGreater(incar["NPAR"], 1)

def contribute_dash(ctx):
    make_dash(ctx)
    ctx.run('cp pymatgen.tgz ../Dash-User-Contributions/docsets/pymatgen/pymatgen.tgz')
    with cd("../Dash-User-Contributions/docsets/pymatgen"):
        with open("docset.json", "rt") as f:
            data = json.load(f)
            data["version"] = NEW_VER
        with open("docset.json", "wt") as f:
            json.dump(data, f, indent=4)
        ctx.run('git commit -a -m "Update to v%s"' % NEW_VER)
        ctx.run('git push')
    ctx.run("rm pymatgen.tgz")

def make_doc(ctx):
    with cd("docs_rst"):
        ctx.run("sphinx-apidoc --separate -d 6 -o . -f ../monty")
        for f in glob.glob("*.rst"):
            if f.startswith('monty') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith(
                                    "monty") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        ctx.run("make html")

        # ctx.run("cp _static/* ../docs/html/_static")

    with cd("docs"):
        ctx.run("cp -r html/* .")
        ctx.run("rm -r html")
        ctx.run("rm -r doctrees")
        ctx.run("rm -r _sources")

        # This makes sure pymatgen.org works to redirect to the Gihub page
        # ctx.run("echo \"pymatgen.org\" > CNAME")
        # Avoid the use of jekyll so that _dir works as intended.
        ctx.run("touch .nojekyll")