def extract_forms(url, follow = "false", cookie_jar = None, filename = "forms.json"):
	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))
	
	if cookie_jar == None:
		try:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl form -o {} -a start_url="{}" -a follow={} -a proxy={}'.format(filename, url, follow, HTTP_PROXY)), EXTRACT_WAIT_TIME)
		except:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl form -o {} -a start_url="{}" -a follow={}'.format(filename, url, follow)), EXTRACT_WAIT_TIME)
	else:
		cookie_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename.replace('.json', '.txt'))
		cookie_jar.save(cookie_jar_path)
		out = utils.run_command('{} && {}'.format(
			utils.cd(os.path.dirname(os.path.abspath(__file__))),
			'scrapy crawl form_with_cookie -o {} -a start_url="{}" -a cookie_jar={}'.format(filename, url, cookie_jar_path)), EXTRACT_WAIT_TIME)

	with open(os.path.join(os.path.dirname(__file__), filename)) as json_forms:
		forms = json.load(json_forms)

	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))
		
	return forms

 def sync_server(self, path):
     LOG.info('Syncing server ...')
     command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py syncdb --noinput'.format(
         utils.to_env(self.base_path), utils.cd(path))
     output = utils.run_command(command)
     if 'Unknown command' in output[2]:
         command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py migrate --noinput'.format(
         utils.to_env(self.base_path), utils.cd(path))
     return utils.run_command(command)

def test_workbench(enaml_run, qtbot):
    from enaml.qt.QtCore import Qt

    # Run normally to generate cache files
    dir_path = os.path.abspath(os.path.split(os.path.dirname(__file__))[0])
    example = os.path.join(dir_path, 'examples', 'workbench')

    def handler(app, window):
        widget = window.proxy.widget
        qtbot.wait(1000)
        for i in range(1, 4):
            qtbot.keyClick(widget, str(i), Qt.ControlModifier)
            qtbot.wait(100)
            # TODO: Verify each screen somehow

        qtbot.keyClick(widget, 'q', Qt.ControlModifier)
        # Wait for exit, otherwise it unregisters the commands
        qtbot.wait(100)

    enaml_run.run = handler

    # Add to example folder to the sys path or we get an import error
    with cd(example, add_to_sys_path=True):
        # Now run from cache
        mod = importlib.import_module('sample')
        mod.main()

    def __call__(self, project, patch):
        src = basename(project.dir)
        logger.info('applying patch to {} source'.format(src))

        environment = dict(os.environ)
        dirpath = tempfile.mkdtemp()
        patch_file = join(dirpath, 'patch')
        with open(patch_file, 'w') as file:
            for e, p in patch.items():
                file.write('{} {} {} {}\n'.format(*e))
                file.write(p + "\n")

        if self.config['semfix']:
            environment['ANGELIX_SEMFIX_MODE'] = 'YES'

        environment['ANGELIX_PATCH'] = patch_file

        with cd(project.dir):
            return_code = subprocess.call(['apply-patch', project.buggy],
                                          stderr=self.subproc_output,
                                          stdout=self.subproc_output,
                                          env=environment)
        if return_code != 0:
            if self.config['ignore_trans_errors']:
                logger.error("transformation of {} failed".format(relpath(project.dir)))
            else:
                logger.error("transformation of {} failed".format(relpath(project.dir)))
                raise TransformationError()

        shutil.rmtree(dirpath)

        pass

    def test_single_relative_force(self):
        """
        Simple auto-gen key; relative paths; with force flag to overwrite source file
        """
        # use only the first test file
        test_file_path = self.file_contents.keys()[0]

        with cd(self.working_dir):
            # encrypt the test file
            encrypt(
                inputfiles=[test_file_path],
                outputfile='sesame.encrypted',
                keys=[self.key],
            )

            # sleep before decrypt to ensure file ctime is different
            time.sleep(1)

            # decrypt the test file
            decrypt(
                inputfile='sesame.encrypted',
                keys=[self.key],
                output_dir=os.getcwd(),         # default in argparse
                force=True,
            )

            # ensure file has been overwritten
            assert self.file_timestamps[test_file_path] < os.stat(test_file_path).st_ctime

            # verify decrypted contents
            with open(test_file_path, 'r') as f:
                assert self.file_contents[test_file_path] == f.read()

 def clone(self):
     if exists(self.name):
         with cd(self.name):
             shell('git fetch origin --tags')
     else:
         shell('git clone --depth=1 --branch {} {}'.format(self.branch,
                                                           self.url))

    def umount(self):
        """
        umounts fs
        if method commit() was executed else makes working suvolume default
        (saves changes on disk), else deletes working subvolume (restores
        its state)
        """
        if self.save_changes:
            tdir = self.mpoint
        else:
            tdir = "/"

        with cd(tdir):
            if self.save_changes:
                shell_exec('btrfs subvolume set-default "{0}" "{1}"'.format(self.snap, self.mpoint))
                self.save_changes = False
            else:
                shell_exec('umount "{0}"'.format(self.mpoint))
                shell_exec('mount "{0}" "{1}"'.format(self.device, self.mpoint))

                os.chdir(self.mpoint)

                shell_exec('btrfs subvolume delete "{0}"'.format(self.snap))

            os.chdir("/")
            shell_exec('umount "{0}"'.format(self.mpoint))

        if self.image_file is not None:
            shell_exec('losetup -d "{0}"'.format(self.device))

    def test_single_relative(self):
        """
        Simple auto-gen key; relative paths; deletes source file
        """
        # use only the first test file
        test_file_path = self.file_contents.keys()[0]

        with cd(self.working_dir):
            # encrypt the test file
            encrypt(
                inputfiles=[test_file_path],
                outputfile='sesame.encrypted',
                keys=[self.key],
            )

            # delete the source file
            os.remove(test_file_path)

            # decrypt the test file
            decrypt(
                inputfile='sesame.encrypted',
                keys=[self.key],
                output_dir=os.getcwd(),         # default in argparse
            )

            # ensure file has been created
            assert os.path.exists(test_file_path)

            # verify decrypted contents
            with open(test_file_path, 'r') as f:
                assert self.file_contents[test_file_path] == f.read()

def apply_compute_changes():
    # checkout the files needed for the compute_nodes
    path_dict = {'virt_driver.py': 'compute/monitors/membw',
                 '__init__.py': 'compute/monitors/membw',
                 'driver.py': 'virt/libvirt',
                 'pcp_utils.py': 'virt/libvirt',
                 'driver.py': 'virt',
                 '__init__.py': 'compute/monitors',
                 'base.py': 'compute/monitors',
                 'claims.py': 'compute'}
    git_repo = "https://github.com/sudswas/nova.git"

    utils.helper.git_clone(git_repo, 'nova', "stable/liberty")
    # Copy the changes now assuming all the files have been
    # copied into the present directory.
    dir_to_create = py_path + '/compute/monitors/membw'
    utils.helper.execute_command("mkdir " + dir_to_create)
    with utils.cd('nova/nova'):
        for file_name, dir in path_dict.iteritems():
            rel_path = dir + "/" + file_name
            sys_file_path = py_path + '/' + rel_path
            utils.helper.execute_command("mv " +
                                         rel_path + " " + sys_file_path)
    utils.helper.execute_command("openstack-config " + "--set " +
                                 "/etc/nova/nova.conf " + "DEFAULT" + " "
                                 + "compute_monitors" + " " +
                                 "membw.virt_driver")

    print "Please restart nova-compute"

    def test_multiple_relative(self):
        """
        Test a directory hierarchy with relative paths
        """
        with cd(self.working_dir):
            # encrypt all the test files
            encrypt(
                inputfiles=self.file_contents.keys(),
                outputfile='sesame.encrypted',
                keys=[self.key],
            )

            # delete the source files
            for path in self.file_contents.keys():
                delete_path(path)

            # decrypt the test files
            decrypt(
                inputfile='sesame.encrypted',
                keys=[self.key],
                output_dir=os.getcwd(),         # default in argparse
            )

            for test_file_path in self.file_contents.keys():
                # ensure files have been created
                assert os.path.exists(test_file_path)

                # verify decrypted contents
                with open(test_file_path, 'r') as f:
                    assert self.file_contents[test_file_path] == f.read()

    def test_multiple_absolute(self):
        """
        Test a directory hierarchy with absolute paths
        """
        # convert the files list to absolute paths
        test_input_files = [
            os.path.join(self.working_dir, path) for path in self.file_contents.keys()
        ]

        with cd(self.working_dir):
            # encrypt all the test files
            encrypt(
                inputfiles=test_input_files,
                outputfile='sesame.encrypted',
                keys=[self.key],
            )

            # delete the source files
            for path in self.file_contents.keys():
                delete_path(path)

            # decrypt the test files
            decrypt(
                inputfile='sesame.encrypted',
                keys=[self.key],
                output_dir=os.getcwd(),         # default in argparse
            )

            for test_file_path in self.file_contents.keys():
                # the file will be extracted on the absolute path
                test_file_path_abs = os.path.join(self.working_dir, test_file_path)[1:]

                # verify decrypted contents at the absolute extracted path
                with open(test_file_path_abs, 'r') as f:
                    assert self.file_contents[test_file_path] == f.read()

    def test_single_relative_output_dir(self):
        """
        Simple auto-gen key; relative paths; deletes source file; change output directory
        """
        # use only the first test file
        test_file_path = self.file_contents.keys()[0]

        with cd(self.working_dir):
            # encrypt the test file
            encrypt(
                inputfiles=[test_file_path],
                outputfile='sesame.encrypted',
                keys=[self.key],
            )

            # create a new temporary directory to extract into
            with make_secure_temp_directory() as output_dir:
                # decrypt the test file
                decrypt(
                    inputfile='sesame.encrypted',
                    keys=[self.key],
                    output_dir=output_dir
                )

                # ensure file has been created in the output_dir
                assert os.path.exists(os.path.join(output_dir, test_file_path))

                # verify decrypted contents
                with open(os.path.join(output_dir, test_file_path), 'r') as f:
                    assert self.file_contents[test_file_path] == f.read()

    def test_single_absolute(self):
        """
        Simple auto-gen key; absolute paths
        """
        # use only the first test file
        test_file_path = self.file_contents.keys()[0]

        with cd(self.working_dir):
            # encrypt the test file
            encrypt(
                inputfiles=[os.path.join(self.working_dir, test_file_path)],
                outputfile=os.path.join(self.working_dir, 'sesame.encrypted'),
                keys=[self.key],
            )

            # delete the source file
            os.remove(test_file_path)

            # sleep before decrypt to ensure file ctime is different
            time.sleep(1)

            # decrypt the test file
            decrypt(
                inputfile=os.path.join(self.working_dir, 'sesame.encrypted'),
                keys=[self.key],
                output_dir=os.getcwd(),         # default in argparse
            )

            # the file will be extracted on the absolute path
            test_file_path_abs = os.path.join(self.working_dir, test_file_path)[1:]

            # verify decrypted contents at the absolute extracted path
            with open(test_file_path_abs, 'r') as f:
                assert self.file_contents[test_file_path] == f.read()

    def test_single_relative_overwrite_false(self):
        """
        Simple auto-gen key; relative paths; answer no to overwrite the source file
        """
        # use only the first test file
        test_file_path = self.file_contents.keys()[0]

        with cd(self.working_dir):
            # encrypt the test file
            encrypt(
                inputfiles=[test_file_path],
                outputfile='sesame.encrypted',
                keys=[self.key],
            )

            # sleep before decrypt to ensure file ctime is different
            time.sleep(1)

            # decrypt the test file; mock responds no to overwrite the existing file
            with mock.patch('__builtin__.raw_input', return_value='n'):
                # decrypt the test file
                decrypt(
                    inputfile='sesame.encrypted',
                    keys=[self.key],
                    output_dir=os.getcwd(),         # default in argparse
                )

            # ensure no file has been decrypted
            assert self.file_timestamps[test_file_path] == os.stat(test_file_path).st_ctime

def install_libpfm():
    git_path = "git://git.code.sf.net/u/hkshaw1990/perfmon2 perfmon2-libpfm4"

    utils.helper.git_clone(git_path, "perfmon2-libpfm4")

    commands = ['make', 'make install PREFIX=']
    with utils.cd("~/perfmon2-libpfm4"):
        utils.helper.execute_command(commands)

    with utils.cd("~/perfmon2-libpfm4/examples"):
        out, err = utils.helper.execute_command('./check_events')
        if out:
            if "POWERPC_NEST_MEM_BW" in out:
                print "Libpfm is ready for Memory BW measurement"
        else:
            print "There was an error during make of libpfm", err

def apply_scheduler_changes():
    # checkout the files needed for the compute_nodes
    path_dict = {'virt/': 'hardware.py',
                 'scheduler/filters' : 'numa_topology_filter.py'}
    git_repo = "https://github.com/openstack/nova.git"

    utils.helper.git_clone(git_repo, 'nova', "stable/liberty")
    # Copy the changes now assuming all the files have been
    # copied into the present directory.
    with utils.cd('nova/nova'):
        for dir, file_name in path_dict.iteritems():
            rel_path = dir + "/" + file_name
            sys_file_path = py_path + rel_path
            utils.helper.execute_command("mv " +
                                         rel_path + " " + sys_file_path)

    filters = utils.helper.execute_command("openstack-config " + "--get " +
                                    "/etc/nova/nova.conf" + "DEFAULT" + " "
                                    + "default_scheduler_filters")
    if "NUMA" not in filters:
        utils.helper.execute_command("openstack-config " + "--set " +
                                 "/etc/nova/nova.conf" + "DEFAULT" + " "
                                 + "default_scheduler_filters" + " " +
                                 + filters + ",NUMATopologyFilter")

    print "please restart openstack-nova-scheduler"

def test_tutorials(enaml_run, tmpdir, tutorial):
    # Run normally to generate cache files

    dir_path = os.path.abspath(os.path.split(os.path.dirname(__file__))[0])
    source = os.path.join(dir_path, 'examples', 'tutorial', tutorial)
    example = os.path.join(tmpdir.strpath, tutorial)

    # Copy to a tmp dir
    shutil.copytree(source, example)
    clean_cache(example)  # To be safe

    # Run compileall
    compileall.compile_dir(example)

    # Remove source files
    clean_source(example)

    # Add to example folder to the sys path or we get an import error
    with cd(example, add_to_sys_path=True):
        if IS_PY3:
            # PY3 only uses pyc files if copied from the pycache folder
            for f in os.listdir('__pycache__'):
                cf = ".".join(f.split(".")[:-2]) + ".pyc"
                shutil.copy(os.path.join('__pycache__', f), cf)

        # Verify it's clean
        assert not os.path.exists(tutorial+".py")
        assert not os.path.exists(tutorial+"_view.enaml")

        # Now run from cache
        mod = importlib.import_module(tutorial)
        mod.main()

 def run_server(self, path, port):
     self.configure_network()
     LOG.info('Running server ...')
     command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py runserver 0.0.0.0:{}'.format(
         utils.to_env(self.base_path),
         utils.cd(path),
         port)
     return utils.run_command_async(command)

 def create_superuser(self, path):
     LOG.info('Creating superuser ...')
     command = '{} && {} && unset DJANGO_SETTINGS_MODULE && {}'.format(
         utils.to_env(self.base_path),
         utils.cd(path),
         """
         echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', '[email protected]', 'admin')" | python manage.py shell
         """)
     return utils.run_command(command)

 def predict(self, target_word, test_xml):
     if target_word in self.target_words:
         curr_dir = os.getcwd()
         target_model_dir = os.path.join(curr_dir, self.model_dir, target_word)
         test_xml = os.path.join(curr_dir, test_xml)
         with cd(self.ims_lib_path):
             out = "/tmp"
             command = "{} {} {} {}".format(self.test_sh, target_model_dir, test_xml, out)
             check_output(command.split())