def collect_dependencies(project, internal_report_file, external_report_file):
    source_dir = project.expand_path("$dir_source_main_python")
    internal_command = ["sfood", "--internal"]
    external_command = ["sfood", "--external"]

    execute_command(internal_command, internal_report_file, cwd=source_dir)
    execute_command(external_command, external_report_file, cwd=source_dir)

 def test_execute_command(self, popen, _):
     popen.return_value = Mock()
     popen.return_value.wait.return_value = 0
     self.assertEquals(execute_command(["test", "commands"]), 0)
     self.assertEquals(execute_command(["test", "commands"], outfile_name="test.out"), 0)
     self.assertEquals(
         execute_command(["test", "commands"], outfile_name="test.out", error_file_name="test.out.err"), 0)

def generate_manpages(project, logger):
    """
        Uses the ronn script to convert a markdown source to a gzipped manpage.
    """
    logger.info('Generating manpages')
    try:
        os.makedirs(project.get_property('dir_manpages'))
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise
    ronn_report_file = project.expand_path("$dir_reports/{0}".format('generate_manpage'))
    generate_manpages_command = build_generate_manpages_command(project)
    execute_command(generate_manpages_command, ronn_report_file, shell=True)

def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=False, insecure_installs=None,
                force_reinstall=False, target_dir=None, verbose=False, trusted_host=None, constraint_file=None,
                eager_upgrade=False,
                logger=None, outfile_name=None, error_file_name=None, env=None, cwd=None):
    pip_command_line = list()
    pip_command_line.extend(PIP_EXEC_STANZA)
    pip_command_line.append("install")
    pip_command_line.extend(build_pip_install_options(index_url=index_url,
                                                      extra_index_url=extra_index_url,
                                                      upgrade=upgrade,
                                                      insecure_installs=insecure_installs,
                                                      force_reinstall=force_reinstall,
                                                      target_dir=target_dir,
                                                      verbose=verbose,
                                                      trusted_host=trusted_host,
                                                      constraint_file=constraint_file,
                                                      eager_upgrade=eager_upgrade
                                                      ))
    for install_target in as_list(install_targets):
        pip_command_line.extend(as_pip_install_target(install_target))

    if env is None:
        env = os.environ

    if logger:
        logger.debug("Invoking pip: %s", pip_command_line)
    return execute_command(pip_command_line, outfile_name=outfile_name, env=env, cwd=cwd,
                           error_file_name=error_file_name, shell=False)

def _do_install_dependency(logger, project, dependency, upgrade, force_reinstall, target_dir, log_file):
    batch = isinstance(dependency, collections.Iterable)

    pip_command_line = list()
    pip_command_line.extend(PIP_EXEC_STANZA)
    pip_command_line.append("install")
    pip_command_line.extend(build_pip_install_options(project.get_property("install_dependencies_index_url"),
                                                      project.get_property("install_dependencies_extra_index_url"),
                                                      upgrade,
                                                      project.get_property(
                                                          "install_dependencies_insecure_installation"),
                                                      force_reinstall,
                                                      target_dir,
                                                      project.get_property("verbose"),
                                                      project.get_property("install_dependencies_trusted_host")
                                                      ))
    pip_command_line.extend(as_pip_install_target(dependency))
    logger.debug("Invoking pip: %s", pip_command_line)
    exit_code = execute_command(pip_command_line, log_file, env=os.environ, shell=False)

    if exit_code != 0:
        if batch:
            dependency_name = " batch dependencies."
        else:
            dependency_name = " dependency '%s'." % dependency.name

        if project.get_property("verbose"):
            print_file_content(log_file)
            raise BuildFailedException("Unable to install%s" % dependency_name)
        else:
            raise BuildFailedException("Unable to install%s See %s for details.",
                                       dependency_name,
                                       log_file)

def run_single_test(logger, project, reports_dir, test, ):
    name, _ = os.path.splitext(os.path.basename(test))
    logger.info("Running acceptance test %s", name)
    env = prepare_environment(project)
    test_time = Timer.start()
    command_and_arguments = (sys.executable, test)
    report_file_name = os.path.join(reports_dir, name)
    error_file_name = report_file_name + ".err"
    return_code = execute_command(
        command_and_arguments, report_file_name, env, error_file_name=error_file_name)
    test_time.stop()
    report_item = {
        "test": name,
        "test_file": test,
        "time": test_time.get_millis(),
        "success": True
    }
    if return_code != 0:
        logger.error("acceptance test failed: %s", test)
        report_item["success"] = False

        if project.get_property("verbose"):
            print_file_content(report_file_name)
            print_text_line()
            print_file_content(error_file_name)

    return report_item

    def run(self, outfile_name):
        error_file_name = "{0}.err".format(outfile_name)
        return_code = execute_command(self.parts, outfile_name)
        error_file_lines = read_file(error_file_name)
        outfile_lines = read_file(outfile_name)

        return ExternalCommandResult(return_code, outfile_name, outfile_lines, error_file_name, error_file_lines)

def run_sphinx_build(build_command, task_name, logger, project):
    logger.info("Running %s" % task_name)
    log_file = project.expand_path(
        "$dir_target/reports/{0}".format(task_name))
    if project.get_property("verbose"):
        logger.info(build_command)
        exit_code = execute_command(build_command, log_file, shell=True)
        if exit_code != 0:
            raise BuildFailedException("Sphinx build command failed. See %s for details.", log_file)

def run_py2dsc_deb_build(build_command, task_name, logger, project):
    logger.info("Running %s" % task_name)
    log_file = project.expand_path("$dir_target", "reports", task_name)
    if project.get_property("verbose"):
        logger.info(build_command)
        exit_code = execute_command(build_command, log_file, shell=True)
        if exit_code != 0:
            raise BuildFailedException(
                "py2dsc_deb build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))

def execute_pymetrics(project, logger):
    logger.info("Executing pymetrics on project sources")
    source_dir = project.expand_path("$dir_source_main_python")

    files_to_scan = []
    for root, _, files in os.walk(source_dir):
        for file_name in files:
            if file_name.endswith(".py"):
                files_to_scan.append(os.path.join(root, file_name))

    csv_file = project.expand_path("$dir_reports/pymetrics.csv")

    command = ["pymetrics", "--nosql", "-c", csv_file] + files_to_scan

    report_file = project.expand_path("$dir_reports/pymetrics")

    env = {"PYTHONPATH": source_dir}
    execute_command(command, report_file, env=env)

def run_sphinx_build(build_command, task_name, logger, project, builder=None):
    logger.info("Running %s" % task_name)
    log_file = project.expand_path("$dir_target", "reports", task_name)

    build_command = [sys.executable, "-m"] + build_command
    if project.get_property("verbose"):
        logger.debug(build_command)

    exit_code = execute_command(build_command, log_file, shell=False)
    if exit_code != 0:
        raise BuildFailedException("Sphinx build command failed. See %s for details.", log_file)

def execute_tool_on_modules(project, name, command_and_arguments, extend_pythonpath=True):
    source_dir = project.expand_path("$dir_source_main_python")
    modules = discover_modules(source_dir)
    command = as_list(command_and_arguments) + modules

    report_file = project.expand_path("$dir_reports/%s" % name)

    env = os.environ
    if extend_pythonpath:
        env["PYTHONPATH"] = source_dir
    return execute_command(command, report_file, env=env), report_file

def pdoc_compile_docs(project, logger):
    logger.info("Generating pdoc documentation")

    if not project.get_property("pdoc_module_name"):
        raise BuildFailedException("'pdoc_module_name' must be specified")

    pdoc_command_args = project.get_property("pdoc_command_args", [])
    pdoc_output_dir = project.expand_path("$pdoc_output_dir")

    command_and_arguments = ["pdoc"] + pdoc_command_args
    if "--html" in pdoc_command_args:
        command_and_arguments += ["--html-dir", pdoc_output_dir]
    command_and_arguments += [project.get_property("pdoc_module_name")]

    source_directory = project.expand_path("$pdoc_source")
    environment = {"PYTHONPATH": source_directory,
                   "PATH": os.environ["PATH"]}

    logger.debug("Executing pdoc as: %s", command_and_arguments)
    execute_command(command_and_arguments, outfile_name=project.expand_path("$dir_reports", "pdoc"), env=environment,
                    cwd=pdoc_output_dir)

def _install_external_plugin(name, logger):
    if not name.startswith(PYPI_PLUGIN_PROTOCOL):
        message = "Only plugins starting with '{0}' are currently supported"
        raise MissingPluginException(name, message.format(PYPI_PLUGIN_PROTOCOL))
    plugin_name_on_pypi = name.replace(PYPI_PLUGIN_PROTOCOL, "")
    log_file = tempfile.NamedTemporaryFile(delete=False).name
    result = execute_command(
        "pip install {0}".format(plugin_name_on_pypi), log_file, error_file_name=log_file, shell=True
    )
    if result != 0:
        logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file)))
        message = "Failed to install from PyPI".format(plugin_name_on_pypi)
        raise MissingPluginException(name, message)

def install_dependency(logger, project, dependency):
    logger.info("Installing dependency '%s'%s", dependency.name, " from %s" % dependency.url if dependency.url else "")
    log_file = project.expand_path("$dir_install_logs", dependency.name)

    pip_command_line = "pip install {0}'{1}'".format(build_pip_install_options(project), as_pip_argument(dependency))
    exit_code = execute_command(pip_command_line, log_file, shell=True)
    if exit_code != 0:
        if project.get_property("verbose"):
            print_file_content(log_file)
            raise BuildFailedException("Unable to install dependency '%s'.", dependency.name)
        else:
            raise BuildFailedException("Unable to install dependency '%s'. See %s for details.",
                                       dependency.name,
                                       log_file)

def sphinx_generate(project, logger):
    """Runs sphinx-build against rst sources for the given project.
    """
    logger.info("Running sphinx-build")

    log_file = project.expand_path("$dir_target/reports/{0}".format("sphinx-build"))
    build_command = get_sphinx_build_command(project)

    if project.get_property("verbose"):
        logger.info(build_command)

    exit_code = execute_command(build_command, log_file, shell=True)
    if exit_code != 0:
        raise BuildFailedException("Sphinx build command failed. See %s for details.", log_file)

def execute_tool_on_source_files(project, name, command_and_arguments, logger=None, include_test_sources=False):
    files = discover_affected_files(include_test_sources, project)

    command = as_list(command_and_arguments) + [f for f in files]

    report_file = project.expand_path("$dir_reports/{0}".format(name))

    execution_result = execute_command(command, report_file), report_file

    report_file = execution_result[1]
    report_lines = read_file(report_file)

    if project.get_property(name + "_verbose_output") and logger:
        log_report(logger, name, report_lines)

    return execution_result

def run_single_test(logger, project, reports_dir, test, output_test_names=True):
    additional_integrationtest_commandline_text = project.get_property("integrationtest_additional_commandline", "")

    if additional_integrationtest_commandline_text:
        additional_integrationtest_commandline = tuple(additional_integrationtest_commandline_text.split(" "))
    else:
        additional_integrationtest_commandline = ()

    name, _ = os.path.splitext(os.path.basename(test))

    if output_test_names:
        logger.info("Running integration test %s", name)

    env = prepare_environment(project)
    test_time = Timer.start()
    command_and_arguments = (sys.executable, test)
    command_and_arguments += additional_integrationtest_commandline

    report_file_name = os.path.join(reports_dir, name)
    error_file_name = report_file_name + ".err"
    return_code = execute_command(
        command_and_arguments, report_file_name, env, error_file_name=error_file_name)
    test_time.stop()
    report_item = {
        "test": name,
        "test_file": test,
        "time": test_time.get_millis(),
        "success": True
    }
    if return_code != 0:
        logger.error("Integration test failed: %s", test)
        report_item["success"] = False

        if project.get_property("verbose") or project.get_property("integrationtest_always_verbose"):
            print_file_content(report_file_name)
            print_text_line()
            print_file_content(error_file_name)
            report_item['exception'] = ''.join(read_file(error_file_name)).replace('\'', '')
    elif project.get_property("integrationtest_always_verbose"):
        print_file_content(report_file_name)
        print_text_line()
        print_file_content(error_file_name)

    return report_item

def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=False, insecure_installs=None,
                force_reinstall=False, target_dir=None, verbose=False, logger=None, outfile_name=None,
                error_file_name=None, env=None, cwd=None):
    pip_command_line = list()
    pip_command_line.append(PIP_EXECUTABLE)
    pip_command_line.append("install")
    pip_command_line.extend(build_pip_install_options(index_url,
                                                      extra_index_url,
                                                      upgrade,
                                                      insecure_installs,
                                                      force_reinstall,
                                                      target_dir,
                                                      verbose
                                                      ))
    for install_target in as_list(install_targets):
        pip_command_line.extend(as_pip_install_target(install_target))

    if logger:
        logger.debug("Invoking pip: %s", pip_command_line)
    return execute_command(pip_command_line, outfile_name=outfile_name, env=env, cwd=cwd,
                           error_file_name=error_file_name, shell=False)

def install_dependency(logger, project, dependency):
    url = getattr(dependency, "url", None)
    logger.info("Installing dependency '%s'%s", dependency.name,
                " from %s" % url if url else "")
    log_file = project.expand_path("$dir_install_logs", dependency.name)

    if sys.platform.startswith("win"):
        # we can't use quotes on windows
        pip_dependency = as_pip_argument(dependency)
    else:
        # on linux we need quotes because version pinning (>=) would be an IO redirect
        pip_dependency = "'{0}'".format(as_pip_argument(dependency))
    pip_command_line = "pip install {0}{1}".format(build_pip_install_options(project, pip_dependency), pip_dependency)
    exit_code = execute_command(pip_command_line, log_file, shell=True)
    if exit_code != 0:
        if project.get_property("verbose"):
            print_file_content(log_file)
            raise BuildFailedException("Unable to install dependency '%s'.", dependency.name)
        else:
            raise BuildFailedException("Unable to install dependency '%s'. See %s for details.",
                                       getattr(dependency, "name", dependency),
                                       log_file)