def read_notebook(self, arg):
        # Currently assuming arg is a filename relative to
        # where the server was started from, later we may
        # want to accept urls or full notebooks as well.
        if not isinstance(arg, string_types):
            raise web.HTTPError(400, 'Expecting a filename or a URL.')

        try:
            # Check that file exists
            if arg == EXPLICIT_MISSING_FILE:
                path = arg
            else:
                path = os.path.join(self.curdir, arg)
                if not os.path.exists(path):
                    if '://' not in arg:
                        raise ValueError('Supplied argument cannot be read: %r' % arg)
                    # Assume file is URI
                    r = requests.get(arg)

            # Let nbformat do the reading and validation
            if os.path.exists(path):
                nb = nbformat.read(path, as_version=4)
            elif path == EXPLICIT_MISSING_FILE:
                nb = nbformat.v4.new_notebook()
            else:
                nb = nbformat.reads(r.text, as_version=4)
        except Exception as e:
            self.log.exception(e)
            raise web.HTTPError(422, 'Invalid notebook: %s' % arg)

        return nb

def bundle_notebook(vid, fileid):
    """Return a file from the bundle"""
    from ambry.orm.file import File
    import nbformat
    from traitlets.config import Config
    from nbconvert import HTMLExporter

    b = aac.library.bundle(vid)

    nbfile = b.build_source_files.file_by_id(fileid)

    notebook = nbformat.reads(nbfile.unpacked_contents, as_version=4)

    html_exporter = HTMLExporter()
    html_exporter.template_file = 'basic'

    (body, resources) = html_exporter.from_notebook_node(notebook)

    cxt = dict(
        vid=vid,
        b=b,
        fileid=fileid,
        nbfile=nbfile,
        notebooks=b.build_source_files.list_records(File.BSFILE.NOTEBOOK),
        notebook=notebook,
        notebook_html=body,
        **aac.cc

    )

    return aac.render('bundle/notebook.html', **cxt)

def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack):
    model_string = inspect.getsource(model)
    model_string = remove_imports(model_string)

    if notebook_name:
        notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name)
        with open(notebook_path, 'r') as f:
            notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
            exporter = PythonExporter()
            source, _ = exporter.from_notebook_node(notebook)
    else:
        calling_script_file = os.path.abspath(inspect.stack()[stack][1])
        with open(calling_script_file, 'r') as f:
            source = f.read()

    cleaned_source = remove_all_comments(source)
    imports = extract_imports(cleaned_source, verbose)

    parts = hyperparameter_names(model_string)
    aug_parts = augmented_names(parts)

    hyperopt_params = get_hyperparameters(model_string)
    space = get_hyperopt_space(parts, hyperopt_params, verbose)

    functions_string = retrieve_function_string(functions, verbose)
    data_string = retrieve_data_string(data, verbose)
    model = hyperopt_keras_model(model_string, parts, aug_parts, verbose)

    temp_str = temp_string(imports, model, data_string, functions_string, space)
    return temp_str

    def get_notebook_argument(self, argname):
        # Assuming a request on the form "{'argname':arg}"
        body = json.loads(escape.to_unicode(self.request.body))
        arg = body[argname]

        # Currently assuming arg is a filename relative to
        # where the server was started from, later we may
        # want to accept urls or full notebooks as well.
        if not isinstance(arg, string_types):
            raise web.HTTPError(400, "Expecting a filename.")

        # Check that file exists
        path = os.path.join(self.params["cwd"], arg)
        if not os.path.exists(path):
            # Assume file is URI
            r = requests.get(arg)

        # Let nbformat do the reading and validation
        try:
            if os.path.exists(path):
                nb = nbformat.read(path, as_version=4)
            else:
                nb = nbformat.reads(r.text, as_version=4)
        except:
            raise web.HTTPError(400, "Invalid notebook: %s" % truncate_filename(arg))

        return nb

 def update_jupyter(self, s, keywords):
     '''Update @jupyter node in the vr pane.'''
     pc = self
     c = pc.c
     if pc.must_change_widget(QtWebKitWidgets.QWebView):
         # g.trace('===== instantiating QWebView')
         w = QtWebKitWidgets.QWebView()
         n = c.config.getInt('qweb_view_font_size')
         if n:
             settings = w.settings()
             settings.setFontSize(settings.DefaultFontSize, n)
         pc.embed_widget(w)
         assert(w == pc.w)
     else:
         w = pc.w
     url = g.getUrlFromNode(c.p)
     if url and nbformat:
         s = urlopen(url).read().decode()
         try:
             nb = nbformat.reads(s, as_version=4)
             e = HTMLExporter()
             (s, junk_resources) = e.from_notebook_node(nb)
         except nbformat.reader.NotJSONError:
             # Assume the result is html.
             pass
     elif url:
         s = 'can not import nbformt: %r' % url
     else:
         s = g.u('')
     if isQt5:
         w.hide() # This forces a proper update.
     w.setHtml(s)
     w.show()
     c.bodyWantsFocusNow()

    def parse(self, inputstring, document):
        """Parse `inputstring`, write results to `document`."""
        nb = nbformat.reads(inputstring, as_version=_ipynbversion)
        env = document.settings.env
        srcdir = os.path.dirname(env.doc2path(env.docname))
        auxdir = os.path.join(env.doctreedir, 'nbsphinx')
        sphinx.util.ensuredir(auxdir)

        resources = {}
        # Working directory for ExecutePreprocessor
        resources['metadata'] = {'path': srcdir}
        # Sphinx doesn't accept absolute paths in images etc.
        resources['output_files_dir'] = os.path.relpath(auxdir, srcdir)
        resources['unique_key'] = env.docname.replace('/', '_')

        exporter = Exporter(allow_errors=env.config.nbsphinx_allow_errors,
                            timeout=env.config.nbsphinx_timeout,
                            codecell_lexer=env.config.nbsphinx_codecell_lexer)

        try:
            rststring, resources = exporter.from_notebook_node(nb, resources)
        except NotebookError as e:
            env.warn(env.docname, str(e))
            return  # document is unchanged (i.e. empty)

        # Create additional output files (figures etc.),
        # see nbconvert.writers.FilesWriter.write()
        for filename, data in resources.get('outputs', {}).items():
            dest = os.path.normpath(os.path.join(srcdir, filename))
            with open(dest, 'wb') as f:
                f.write(data)

        rst.Parser.parse(self, rststring, document)

    def _load_notebook(self, uri):
        """
        Loads a local or remote notebook. Raises RuntimeError if no installed 
        kernel can handle the language specified in the notebook. Otherwise,
        returns the notebook object.
        """
        parts = urlparse(self.seed_uri)

        if parts.netloc == "" or parts.netloc == "file":
            # Local file
            with open(parts.path) as nb_fh:
                notebook = nbformat.read(nb_fh, 4)
        else:
            # Remote file
            import requests

            resp = requests.get(uri)
            resp.raise_for_status()
            notebook = nbformat.reads(resp.text, 4)

        # Error if no kernel spec can handle the language requested
        kernel_name = notebook["metadata"]["kernelspec"]["name"]
        self.kernel_spec_manager.get_kernel_spec(kernel_name)

        return notebook

def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'):
    '''
    This functions executes a Jupyter notebook and creates the related
    HTML file.

    Args:
        root (str): name of the file without the .ipynb extension
        template (str): name of the template (to be in the current folder as template.tpl)
        version (int): version of the notebook
        timeout (float): maximum time spent per cell
        kernel (str)

    Returns:
        None

    The function executes root.ipynb into root_exe.ipynb and creates the file root.html.
    '''
    with open(root + '.ipynb') as f:
        nb = nbformat.read(f, as_version=version)

    ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
    ep.preprocess(nb, {'metadata': {'path': '.'}})

    with open(root + '_exe.ipynb', 'wt') as f:
        nbformat.write(nb, f)

    html_exporter = HTMLExporter()
    html_exporter.template_file = template

    with open(root + '_exe.ipynb', mode='r') as f:
        notebook = nbformat.reads(''.join(f.readlines()), as_version=version)
        (body, _) = html_exporter.from_notebook_node(notebook)
        codecs.open(root + '.html', 'w', encoding='utf-8').write(body)

    def downloadIntroNotebooks(self):
        #download the Intro.txt file that contains all the notebooks to download
        response = requests.get("https://github.com/ibm-watson-data-lab/pixiedust/raw/master/notebook/Intro.txt")
        if not response.ok:
            raise Exception("Unable to read the list of Intro Notebooks")

        notebookNames = response.content.decode().split("\n")
        introNotebooksUrls = [
            "https://github.com/ibm-watson-data-lab/pixiedust/raw/master/notebook/" + n for n in notebookNames if n != ""
        ]
        for url in introNotebooksUrls:
            print("...{0}".format(url))
            try:
                path = self.downloadFileToDir(url, targetDir=self.pixiedust_notebooks_dir)
                #update kernel name and display_name
                f = open(path, 'r')
                contents = f.read()
                f.close()
                nb=nbformat.reads(contents, as_version=4)
                nb.metadata.kernelspec.name=self.kernelInternalName
                nb.metadata.kernelspec.display_name = self.kernelName
                f = open(path, 'w')
                f.write(json.dumps(nb))
                f.close()
                print("\033[F\033[F")
                print("...{0} : {1}".format(url, self.hilite("done")))
            except Exception as e:
                print("\033[F\033[F")
                print("...{0} : {1}".format(url, self.hilite("Error {}".format(e))))

def check_one_notebook(filepath):
    folder, filename = os.path.split(filepath)
    os.chdir(folder)
    with open(filename) as f:
        nb = nbformat.reads(f.read(), nbformat.NO_CONVERT)
    run_notebook(nb)
    os.chdir('../../../')

    def write_notebook(self, include_html=True):
        suffix = "_responses_with_names" if self.include_usernames else "_responses"
        nb_name = self.nb_name_stem + suffix
        output_file = os.path.join(PROCESSED_NOTEBOOK_DIR, nb_name + '.ipynb')
        html_output = os.path.join(PROCESSED_NOTEBOOK_DIR, nb_name + '.html')

        remove_duplicate_answers = not self.include_usernames

        filtered_cells = []
        for prompt in self.question_prompts:
            filtered_cells += prompt.cells
            answers = prompt.answers_without_duplicates if remove_duplicate_answers else prompt.answers
            for gh_username, response_cells in answers.items():
                if self.include_usernames:
                    filtered_cells.append(
                        NotebookUtils.markdown_heading_cell(self.gh_username_to_fullname(gh_username), 4))
                filtered_cells.extend(response_cells)

        answer_book = deepcopy(self.template)
        answer_book['cells'] = filtered_cells
        nb = nbformat.from_dict(answer_book)

        print "Writing", output_file
        with io.open(output_file, 'wt') as fp:
            nbformat.write(nb, fp, version=4)

        if include_html:
            # TODO why is the following necessary?
            nb = nbformat.reads(nbformat.writes(nb, version=4), as_version=4)
            html_content, _ = nbconvert.export_html(nb)
            print "Writing", html_output
            with io.open(html_output, 'w') as fp:
                fp.write(html_content)

    def _load_notebook(self, uri):
        """Loads a notebook from the local filesystem or HTTP URL.

        Raises
        ------
        RuntimeError if no installed kernel can handle the language specified
        in the notebook.

        Returns
        -------
        object
            Notebook object from nbformat
        """
        parts = urlparse(uri)

        if parts.netloc == '' or parts.netloc == 'file':
            # Local file
            with open(parts.path) as nb_fh:
                notebook = nbformat.read(nb_fh, 4)
        else:
            # Remote file
            import requests
            resp = requests.get(uri)
            resp.raise_for_status()
            notebook = nbformat.reads(resp.text, 4)

        # Error if no kernel spec can handle the language requested
        kernel_name = notebook['metadata']['kernelspec']['name']
        self.kernel_spec_manager.get_kernel_spec(kernel_name)

        return notebook

    def parse(self, inputstring, document):
        """Parse `inputstring`, write results to `document`."""
        nb = nbformat.reads(inputstring, as_version=_ipynbversion)
        nbsphinx_metadata = nb.metadata.get('nbsphinx', {})
        resources = {}
        env = document.settings.env
        srcdir = os.path.dirname(env.doc2path(env.docname))
        auxdir = os.path.join(env.doctreedir, 'nbsphinx')
        sphinx.util.ensuredir(auxdir)

        # Execute notebook only if there are no outputs:
        if not any(c.outputs for c in nb.cells if 'outputs' in c):
            resources.setdefault('metadata', {})['path'] = srcdir
            allow_errors = nbsphinx_metadata.get('allow_errors', False)
            pp = nbconvert.preprocessors.ExecutePreprocessor(
                allow_errors=allow_errors)
            nb, resources = pp.preprocess(nb, resources)

        # Remove hidden cells
        nb.cells[:] = (cell for cell in nb.cells
                       if cell.metadata.get('nbsphinx') != 'hidden')

        # Sphinx doesn't accept absolute paths in images etc.
        resources['output_files_dir'] = os.path.relpath(auxdir, srcdir)
        resources['unique_key'] = env.docname.replace('/', '_')

        def get_empty_lines(s):
            """Get number of empty lines before and after code."""
            before = 0
            lines = s.split('\n')
            for line in lines:
                if line.strip():
                    break
                before += 1
            after = 0
            for line in reversed(lines[before:]):
                if line.strip():
                    break
                after += 1
            return before, after

        resources['get_empty_lines'] = get_empty_lines

        loader = jinja2.DictLoader({'nbsphinx-rst.tpl': RST_TEMPLATE})
        exporter = nbconvert.RSTExporter(template_file='nbsphinx-rst',
                                         extra_loaders=[loader])
        rststring, resources = exporter.from_notebook_node(nb, resources)

        if nbsphinx_metadata.get('orphan', False):
            rststring = ':orphan:\n\n' + rststring

        # Create additional output files (figures etc.),
        # see nbconvert.writers.FilesWriter.write()
        for filename, data in resources.get('outputs', {}).items():
            dest = os.path.normpath(os.path.join(srcdir, filename))
            with open(dest, 'wb') as f:
                f.write(data)

        rst.Parser.parse(self, rststring, document)

    def finish_notebook(self, json_notebook, download_url, provider_url=None, 
                        provider_icon=None, provider_label=None, msg=None,
                        breadcrumbs=None, public=False, format=None, request=None):
        """render a notebook from its JSON body.

        download_url is required, provider_url is not.

        msg is extra information for the log message when rendering fails.
        """

        if msg is None:
            msg = download_url

        try:
            nb = reads(json_notebook, current_nbformat)
        except ValueError:
            app_log.error("Failed to render %s", msg, exc_info=True)
            raise web.HTTPError(400, "Error reading JSON notebook")

        try:
            app_log.debug("Requesting render of %s", download_url)
            with time_block("Rendered %s" % download_url):
                app_log.info("rendering %d B notebook from %s", len(json_notebook), download_url)
                nbhtml, config = yield self.pool.submit(render_notebook,
                    self.formats[format], nb, download_url,
                    config=self.config,
                )
        except NbFormatError as e:
            app_log.error("Invalid notebook %s: %s", msg, e)
            raise web.HTTPError(400, str(e))
        except Exception as e:
            app_log.error("Failed to render %s", msg, exc_info=True)
            raise web.HTTPError(400, str(e))
        else:
            app_log.debug("Finished render of %s", download_url)

        html = self.render_template(
            "formats/%s.html" % format,
            body=nbhtml,
            nb=nb,
            download_url=download_url,
            provider_url=provider_url,
            provider_label=provider_label,
            provider_icon=provider_icon,
            format=self.format,
            default_format=self.default_format,
            format_prefix=format_prefix,
            formats=dict(self.filter_formats(nb, json_notebook)),
            format_base=self.request.uri.replace(self.format_prefix, ""),
            date=datetime.utcnow().strftime(date_fmt),
            breadcrumbs=breadcrumbs,
            **config)

        if 'content_type' in self.formats[format]:
            self.set_header('Content-Type', self.formats[format]['content_type'])
        yield self.cache_and_finish(html)

        # Index notebook
        self.index.index_notebook(download_url, nb, public)

def nb2rst(filepath):
    with open(filepath) as fh:
        nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
    exporter = RSTExporter()
    # source is a tuple of python source code
    # meta contains metadata
    source, meta = exporter.from_notebook_node(nb)
    return [line + '\n' for line in source.split('\n')]

def convertNotebook(notebookPath):
    with open(notebookPath) as fh:
        nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)

    exporter = PythonExporter()
    source, meta = exporter.from_notebook_node(nb)
    lines = source.split("\n")
    for line in lines:
        print(line)

def load_ipynb_url(url, proxies):
    response = retryrequests.get(url, proxies=proxies)
    response.raise_for_status()

    try:
        return (nbformat.reads(response.text, as_version=4), len(response.content))
    except IOError as e:
        _schema_not_found_error_handler(e)
        raise

    def create_post(self, path, **kw):
        """Create a new post."""
        if flag is None:
            req_missing(["ipython[notebook]>=2.0.0"], "build this site (compile ipynb)")
        content = kw.pop("content", None)
        onefile = kw.pop("onefile", False)
        kernel = kw.pop("ipython_kernel", None)
        # is_page is not needed to create the file
        kw.pop("is_page", False)

        metadata = {}
        metadata.update(self.default_metadata)
        metadata.update(kw)

        makedirs(os.path.dirname(path))

        if content.startswith("{"):
            # imported .ipynb file, guaranteed to start with "{" because it’s JSON.
            nb = nbformat.reads(content, current_nbformat)
        else:
            if ipy_modern:
                nb = nbformat.v4.new_notebook()
                nb["cells"] = [nbformat.v4.new_markdown_cell(content)]
            else:
                nb = nbformat.new_notebook()
                nb["worksheets"] = [nbformat.new_worksheet(cells=[nbformat.new_text_cell("markdown", [content])])]

            if kernelspec is not None:
                if kernel is None:
                    kernel = self.default_kernel
                    self.logger.notice('No kernel specified, assuming "{0}".'.format(kernel))

                IPYNB_KERNELS = {}
                ksm = kernelspec.KernelSpecManager()
                for k in ksm.find_kernel_specs():
                    IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict()
                    IPYNB_KERNELS[k]["name"] = k
                    del IPYNB_KERNELS[k]["argv"]

                if kernel not in IPYNB_KERNELS:
                    self.logger.error('Unknown kernel "{0}". Maybe you mispelled it?'.format(kernel))
                    self.logger.info("Available kernels: {0}".format(", ".join(sorted(IPYNB_KERNELS))))
                    raise Exception('Unknown kernel "{0}"'.format(kernel))

                nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel]
            else:
                # Older IPython versions don’t need kernelspecs.
                pass

        if onefile:
            nb["metadata"]["nikola"] = metadata

        with io.open(path, "w+", encoding="utf8") as fd:
            if ipy_modern:
                nbformat.write(nb, fd, 4)
            else:
                nbformat.write(nb, fd, "ipynb")

def run():
    content = sys.stdin.read()
    version = json.loads(content)['nbformat']
    content = reads(content, version)
    for command in SHEET_COMMANDS:
        command(content)
    for cell in content.cells:
        for command in CELL_COMMANDS:
            command(cell)
    write(content, sys.stdout, version)

def notebook_node_from_string_list(string_list):
    """
    Reads a notebook from a string list and returns the NotebookNode
    object.

    :param string_list: The notebook file contents as list of strings
                        (linewise).
    :return:            The notebook as NotebookNode.
    """
    return nbformat.reads(''.join(string_list), nbformat.NO_CONVERT)