def main_diff(afn, bfn, dfn=None):
    for fn in (afn, bfn):
        if not os.path.exists(fn):
            print("Missing file {}".format(fn))
            return 1

    a = nbformat.read(afn, as_version=4)
    b = nbformat.read(bfn, as_version=4)

    # TODO: Split lines here?
    #a = split_lines(a)
    #b = split_lines(b)

    d = diff_notebooks(a, b)

    verbose = True
    if verbose:
        pretty_print_notebook_diff(afn, bfn, a, d)

    if dfn:
        with open(dfn, "w") as df:
            # Compact version:
            #json.dump(d, df)
            # Verbose version:
            json.dump(d, df, indent=2, separators=(",", ": "))
    return 0

def main():
    if len(sys.argv) > 1:
        if sys.argv[1] in ['help', '-h', '--help']:
            print(__doc__, file=sys.stderr)
            sys.exit(1)
        if sys.argv[1] in ['install', '--install']:
            sys.exit(install())

        force = False
        filenames = sys.argv[1:]
        if filenames[0] in ['-f', '--force']:
            force = True
            filenames.pop(0)

        for filename in filenames:
            if not force and not filename.endswith('.ipynb'):
                continue
            try:
                with io.open(filename, 'r', encoding='utf8') as f:
                    nb = read(f, as_version=NO_CONVERT)
                nb = strip_output(nb)
                with io.open(filename, 'w', encoding='utf8') as f:
                    write(nb, f)
            except Exception:
                # Ignore exceptions for non-notebook files.
                print("Could not strip '{}'".format(filename))
                raise
    else:
        write(strip_output(read(sys.stdin, as_version=NO_CONVERT)), sys.stdout)

def main_diff(args):
    afn = args.base
    bfn = args.remote
    dfn = args.output

    for fn in (afn, bfn):
        if not os.path.exists(fn):
            print("Missing file {}".format(fn))
            return 1

    a = nbformat.read(afn, as_version=4)
    b = nbformat.read(bfn, as_version=4)

    d = diff_notebooks(a, b)

    if dfn:
        with io.open(dfn, "w", encoding="utf8") as df:
            # Compact version:
            # json.dump(d, df)
            # Verbose version:
            json.dump(d, df, indent=2, separators=(",", ": "))
    else:
        # This printer is to keep the unit tests passing,
        # some tests capture output with capsys which doesn't
        # pick up on sys.stdout.write()
        class Printer:
            def write(self, text):
                print(text, end="")

        pretty_print_notebook_diff(afn, bfn, a, d, Printer())

    return 0

 def setUp(self):
     self.data_dir = tempfile.mkdtemp()
     self.notary = sign.NotebookNotary(
         db_file=':memory:',
         secret=b'secret',
         data_dir=self.data_dir,
     )
     with self.fopen(u'test3.ipynb', u'r') as f:
         self.nb = read(f, as_version=4)
     with self.fopen(u'test3.ipynb', u'r') as f:
         self.nb3 = read(f, as_version=3)

def main():
    parser = ArgumentParser(epilog=__doc__, formatter_class=RawDescriptionHelpFormatter)
    task = parser.add_mutually_exclusive_group()
    task.add_argument('--install', action='store_true',
                      help="""Install nbstripout in the current repository (set
                              up the git filter and attributes)""")
    task.add_argument('--uninstall', action='store_true',
                      help="""Uninstall nbstripout from the current repository
                              (remove the git filter and attributes)""")
    task.add_argument('--is-installed', action='store_true',
                      help='Check if nbstripout is installed in current repository')
    task.add_argument('--status', action='store_true',
                      help='Print status of nbstripout installation in current repository and configuration summary if installed')
    parser.add_argument('--attributes', metavar='FILEPATH', help="""Attributes
        file to add the filter to (in combination with --install/--uninstall),
        defaults to .git/info/attributes""")
    task.add_argument('--version', action='store_true',
                      help='Print version')
    parser.add_argument('--force', '-f', action='store_true',
                        help='Strip output also from files with non ipynb extension')
    parser.add_argument('files', nargs='*', help='Files to strip output from')
    args = parser.parse_args()

    if args.install:
        sys.exit(install(args.attributes))
    if args.uninstall:
        sys.exit(uninstall(args.attributes))
    if args.is_installed:
        sys.exit(status(verbose=False))
    if args.status:
        sys.exit(status(verbose=True))
    if args.version:
        print(__version__)
        sys.exit(0)

    for filename in args.files:
        if not (args.force or filename.endswith('.ipynb')):
            continue
        try:
            with io.open(filename, 'r', encoding='utf8') as f:
                nb = read(f, as_version=NO_CONVERT)
            nb = strip_output(nb)
            with io.open(filename, 'w', encoding='utf8') as f:
                write(nb, f)
        except Exception:
            # Ignore exceptions for non-notebook files.
            print("Could not strip '{}'".format(filename))
            raise
    if not args.files:
        write(strip_output(read(input_stream, as_version=NO_CONVERT)), output_stream)

def main_merge(args):
    bfn = args.base
    lfn = args.local
    rfn = args.remote
    mfn = args.output

    for fn in (bfn, lfn, rfn):
        if not os.path.exists(fn):
            print("Cannot find file '{}'".format(fn))
            print(_usage)
            return 1

    b = nbformat.read(bfn, as_version=4)
    l = nbformat.read(lfn, as_version=4)
    r = nbformat.read(rfn, as_version=4)

    # TODO: Split lines here?
    #b = split_lines(b)
    #l = split_lines(l)
    #r = split_lines(r)

    m, lc, rc = merge_notebooks(b, l, r, args)

    if lc or rc:
        print("Conflicts occured during merge operation.")
    else:
        print("Merge completed successfully with no unresolvable conflicts.")

    if mfn:
        if lc or rc:
            # Write partial merge and conflicts to a foo.ipynb-merge file
            result = {
                "merged": m,
                "local_conflicts": lc,
                "remote_conflicts": rc
                }
            with open(mfn+"-merge", "w") as mf:
                json.dump(result, mf)
        else:
            # Write fully completed merge to given foo.ipynb filename
            with open(mfn, "w") as mf:
                nbformat.write(m, mf)
    else:
        # FIXME: Display conflicts in a useful way
        if lc or rc:
            print("Local conflicts:")
            pprint(lc)
            print("Remote conflicts:")
            pprint(rc)
    return 0

def main():
    if len(sys.argv) > 1:
        if sys.argv[1] in ['help', '-h', '--help']:
            print(__doc__, file=sys.stderr)
            sys.exit(1)
        if sys.argv[1] in ['install', '--install']:
            sys.exit(install())
        filename = sys.argv[1]
        with io.open(filename, 'r', encoding='utf8') as f:
            nb = read(f, as_version=NO_CONVERT)
        nb = strip_output(nb)
        with io.open(filename, 'w', encoding='utf8') as f:
            write(nb, f)
    else:
        write(strip_output(read(sys.stdin, as_version=NO_CONVERT)), sys.stdout)

def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''):
    """
    Execute notebook in `src` and write the output to `dst`

    Parameters
    ----------
    src, dst: str
        path to notebook
    allow_errors: bool
    timeout: int
    kernel_name: str
        defualts to value set in notebook metadata

    Returns
    -------
    dst: str
    """
    import nbformat
    from nbconvert.preprocessors import ExecutePreprocessor

    with io.open(src, encoding='utf-8') as f:
        nb = nbformat.read(f, as_version=4)

    ep = ExecutePreprocessor(allow_errors=allow_errors,
                             timeout=timeout,
                             kernel_name=kernel_name)
    ep.preprocess(nb, resources={})

    with io.open(dst, 'wt', encoding='utf-8') as f:
        nbformat.write(nb, f)
    return dst

def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
    """
    Execute notebook in `src` and write the output to `dst`

    Parameters
    ----------
    src, dst: str
        path to notebook
    allow_errors: bool
    timeout: int
    kernel_name: str
        defualts to value set in notebook metadata

    Returns
    -------
    dst: str
    """
    with io.open(src, encoding="utf-8") as f:
        nb = nbformat.read(f, as_version=4)

    ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name)
    ep.preprocess(nb, {"metadta": {"path": "notebooks/"}})

    with io.open(dst, "wt", encoding="utf-8") as f:
        nbformat.write(nb, f)
    return dst

    def test_iter_validation_error(self):
        with self.fopen(u'invalid.ipynb', u'r') as f:
            nb = read(f, as_version=4)

        errors = list(iter_validate(nb))
        assert len(errors) == 3
        assert {e.ref for e in errors} == {'markdown_cell', 'heading_cell', 'bad stream'}

def _notebook_run(path):
    """
    Execute a notebook via nbconvert and collect output.

    Taken from
    https://blog.thedataincubator.com/2016/06/testing-jupyter-notebooks/

    Args:
        path (str): file path for the notebook object

    Returns: (parsed nb object, execution errors)

    """
    dirname, __ = os.path.split(path)
    os.chdir(dirname)
    with tempfile.NamedTemporaryFile(suffix=".ipynb") as fout:
        args = ["jupyter", "nbconvert", "--to", "notebook", "--execute",
                "--ExecutePreprocessor.timeout=60",
                "--output", fout.name, path]
        subprocess.check_call(args)

        fout.seek(0)
        nb = nbformat.read(fout, nbformat.current_nbformat)

    errors = [output for cell in nb.cells if "outputs" in cell
              for output in cell["outputs"]\
              if output.output_type == "error"]

    return nb, errors

    def load_module(self, fullname):
        """import a notebook as a module"""
        path = find_notebook(fullname, self.path)

        # load the notebook object
        with io.open(path, 'r', encoding='utf-8') as f:
            nb = nbformat.read(f, 4)

        # create the module and add it to sys.modules
        # if name in sys.modules:
        #    return sys.modules[name]
        mod = types.ModuleType(fullname)
        mod.__file__ = path
        mod.__loader__ = self
        mod.__dict__['get_ipython'] = get_ipython
        sys.modules[fullname] = mod

        # extra work to ensure that magics that would affect the user_ns
        # actually affect the notebook module's ns
        save_user_ns = self.shell.user_ns
        self.shell.user_ns = mod.__dict__

        try:
            for cell in nb.cells:
                if cell.cell_type == 'code':
                    # transform the input to executable Python
                    code = self.shell.input_transformer_manager.transform_cell(cell.source)
                    # run the code in themodule
                    exec(code, mod.__dict__)
        finally:
            self.shell.user_ns = save_user_ns
        return mod

    def _read_notebook(self, os_path, as_version=4):
        """Read a notebook from an os path."""
        with self.open(os_path, 'r', encoding='utf-8') as f:
            try:
                return nbformat.read(f, as_version=as_version)
            except Exception as e:
                e_orig = e

            # If use_atomic_writing is enabled, we'll guess that it was also
            # enabled when this notebook was written and look for a valid
            # atomic intermediate.
            tmp_path = path_to_intermediate(os_path)

            if not self.use_atomic_writing or not os.path.exists(tmp_path):
                raise HTTPError(
                    400,
                    u"Unreadable Notebook: %s %r" % (os_path, e_orig),
                )

            # Move the bad file aside, restore the intermediate, and try again.
            invalid_file = path_to_invalid(os_path)
            # Rename over existing file doesn't work on Windows
            if os.name == 'nt' and os.path.exists(invalid_file):
                os.remove(invalid_file)
            os.rename(os_path, invalid_file)
            os.rename(tmp_path, os_path)
            return self._read_notebook(os_path, as_version)

def test_mergedriver(git_repo):
    p = filespath()
    # enable diff/merge drivers
    nbdime.gitdiffdriver.main(['config', '--enable'])
    nbdime.gitmergedriver.main(['config', '--enable'])
    # run merge with no conflicts
    out = get_output('git merge remote-no-conflict', err=True)
    assert 'nbmergeapp' in out
    with open('merge-no-conflict.ipynb') as f:
        merged = f.read()

    with open(os.path.join(p, 'multilevel-test-merged.ipynb')) as f:
        expected = f.read()

    # verify merge success
    assert merged == expected

    # reset
    call('git reset local --hard')

    # run merge with conflicts
    with pytest.raises(CalledProcessError):
        call('git merge remote-conflict')
    
    status = get_output('git status')
    assert 'merge-conflict.ipynb' in status
    out = get_output('git diff HEAD')
    assert 'nbdiff' in out
    # verify that the conflicted result is a valid notebook
    nb = nbformat.read('merge-conflict.ipynb', as_version=4)
    nbformat.validate(nb)

def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
    '''
    Execute notebook in `src` and write the output to `dst`

    Parameters
    ----------
    src, dst: str
        path to notebook
    allow_errors: bool
    timeout: int
    kernel_name: str
        defualts to value set in notebook metadata

    Returns
    -------
    dst: str
    '''
    with io.open(src, encoding='utf-8') as f:
        nb = nbformat.read(f, as_version=4)

    ep = ExecutePreprocessor(allow_errors=False,
                             timeout=timeout,
                             kernel_name=kernel_name)
    ep.preprocess(nb, {'metadata': {'path': SOURCE_DIR}})

    with io.open(dst, 'wt', encoding='utf-8') as f:
        nbformat.write(nb, f)
    return dst

    def test_custom_kernel_manager(self):
        from .fake_kernelmanager import FakeCustomKernelManager

        current_dir = os.path.dirname(__file__)

        filename = os.path.join(current_dir, 'files', 'HelloWorld.ipynb')

        with io.open(filename) as f:
            input_nb = nbformat.read(f, 4)

        preprocessor = self.build_preprocessor({
            'kernel_manager_class': FakeCustomKernelManager
        })

        cleaned_input_nb = copy.deepcopy(input_nb)
        for cell in cleaned_input_nb.cells:
            if 'execution_count' in cell:
                del cell['execution_count']
            cell['outputs'] = []

        # Override terminal size to standardise traceback format
        with modified_env({'COLUMNS': '80', 'LINES': '24'}):
            output_nb, _ = preprocessor(cleaned_input_nb,
                                        self.build_resources())

        expected = FakeCustomKernelManager.expected_methods.items()

        for method, call_count in expected:
            self.assertNotEqual(call_count, 0, '{} was called'.format(method))

def main_patch(args):
    bfn = args.base
    dfn = args.patch
    afn = args.output

    for fn in (bfn, dfn):
        if not os.path.exists(fn):
            print("Missing file {}".format(fn))
            return 1

    before = nbformat.read(bfn, as_version=4)
    with open(dfn) as df:
        d = json.load(df)
    d = to_diffentry_dicts(d)

    # TODO: Split lines here? Must be consistent with the diff for patch_notebook to work correctly!?
    #before = split_lines(before)

    after = patch_notebook(before, d)

    if afn:
        nbformat.write(after, afn)
    else:
        print(after)

    return 0

    def _runTest(self):
        kernel = 'python%d' % sys.version_info[0]
        cur_dir = os.path.dirname(self.nbfile)

        with open(self.nbfile) as f:
            nb = nbformat.read(f, as_version=4)
            if self.cov:
                covdict = {'cell_type': 'code', 'execution_count': 1,
                           'metadata': {'collapsed': True}, 'outputs': [],
                           'nbsphinx': 'hidden',
                           'source': 'import coverage\n'
                                     'coverage.process_startup()\n'
                                     'import sys\n'
                                     'sys.path.append("{0}")\n'.format(cur_dir)
                           }
                nb['cells'].insert(0, nbformat.from_dict(covdict))

            exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600)

            try:
                run_dir = os.getenv('WRADLIB_BUILD_DIR', cur_dir)
                exproc.preprocess(nb, {'metadata': {'path': run_dir}})
            except CellExecutionError as e:
                raise e

        if self.cov:
            nb['cells'].pop(0)

        with io.open(self.nbfile, 'wt') as f:
            nbformat.write(nb, f)

        self.assertTrue(True)

    def setUp(self):
        if self.metadata is None:
            metadata = self.default_metadata
        else:
            metadata = self.metadata
        self.orig = nbformat.from_dict({
            "nbformat": NBFORMAT_VERSION,
            "nbformat_minor": 0,
            "metadata": metadata,
            "cells": self.cells
        })

        with tempfile.TemporaryDirectory() as d:
            ipynb0_name = d + "/0"
            rmd_name = d + "/1"
            ipynb1_name = d + "/2"

            with open(ipynb0_name, "w") as f:
                nbformat.write(self.orig, f)

            if self.use_rmd:
                ipyrmd.ipynb_to_rmd(ipynb0_name, rmd_name)
                ipyrmd.rmd_to_ipynb(rmd_name, ipynb1_name)
            else:
                ipyrmd.ipynb_to_spin(ipynb0_name, rmd_name)
                ipyrmd.spin_to_ipynb(rmd_name, ipynb1_name)

            with open(rmd_name) as f:
                self.rmd = f.read()

            with open(ipynb1_name) as f:
                self.roundtrip = nbformat.read(f, NBFORMAT_VERSION)

def copy_notebooks():
    nblist = sorted(nb for nb in os.listdir(NB_SOURCE_DIR)
                    if nb.endswith('.ipynb'))
    name_map = {nb: nb.rsplit('.', 1)[0].lower() + '.html'
                for nb in nblist}

    figsource = abspath_from_here('..', 'notebooks', 'figures')
    figdest = abspath_from_here('content', 'figures')

    if os.path.exists(figdest):
        shutil.rmtree(figdest)
    shutil.copytree(figsource, figdest)

    figurelist = os.listdir(abspath_from_here('content', 'figures'))
    figure_map = {os.path.join('figures', fig) : os.path.join('/PythonDataScienceHandbook/figures', fig)
                  for fig in figurelist}

    for nb in nblist:
        base, ext = os.path.splitext(nb)
        print('-', nb)

        content = nbformat.read(os.path.join(NB_SOURCE_DIR, nb),
                                as_version=4)

        if nb == 'Index.ipynb':
            cells = '1:'
            template = 'page'
            title = 'Python Data Science Handbook'
            content.cells[2].source = INTRO_TEXT
        else:
            cells = '2:'
            template = 'booksection'
            title = content.cells[2].source
            if not title.startswith('#') or len(title.splitlines()) > 1:
                raise ValueError('title not found in third cell')
            title = title.lstrip('#').strip()

            # put nav below title
            content.cells[0], content.cells[1], content.cells[2] = content.cells[2], content.cells[0], content.cells[1]

        # Replace internal URLs and figure links in notebook
        for cell in content.cells:
            if cell.cell_type == 'markdown':
                for nbname, htmlname in name_map.items():
                    if nbname in cell.source:
                        cell.source = cell.source.replace(nbname, htmlname)
                for figname, newfigname in figure_map.items():
                    if figname in cell.source:
                        cell.source = cell.source.replace(figname, newfigname)
                        
        nbformat.write(content, os.path.join(NB_DEST_DIR, nb))

        pagefile = os.path.join(PAGE_DEST_DIR, base + '.md')
        htmlfile = base.lower() + '.html'
        with open(pagefile, 'w') as f:
            f.write(PAGEFILE.format(title=title,
                                    htmlfile=htmlfile,
                                    notebook_file=nb,
                                    template=template,
                                    cells=cells))