def dump_symbols_from_binary(binary, pool_dir, dump_syms_executable=None):
    """ Dump sympobls from the binary.
    Results can be found in
    <pool_dir>/<binary name>/<id>/<binary name>.sym

    """
    cmd = [dump_syms_executable, binary]
    ui.debug(cmd)
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
    (out, err) = process.communicate()
    if process.returncode != 0:
        ui.error("Failed to dump symbols", err)
        return

    # First line looks like:
    # MODULE Linux x86_64  ID  binary
    lines = out.splitlines()
    first_line = lines[0]
    uuid = first_line.split()[3]
    name = first_line.split()[4]

    to_make = os.path.join(pool_dir, name, uuid)
    qisys.sh.mkdir(to_make, recursive=True)
    with open(os.path.join(to_make, name + ".sym"), "w") as fp:
        fp.write(out)

 def target():
     ui.debug("Starting thread.")
     ui.debug("Calling:", subprocess.list2cmdline(self.cmd))
     try:
         opts = dict()
         if os.name == 'posix':
             opts = {
                 'preexec_fn': os.setsid,
                 'close_fds': True
             }
         elif os.name == 'nt':
             opts = {
                 'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
             }
         kwargs = {
                 "cwd": self.cwd,
                 "env" : self.env
         }
         kwargs.update(opts)
         if self.capture:
             kwargs["stdout"] = subprocess.PIPE
             kwargs["stderr"] = subprocess.STDOUT
         self._process = subprocess.Popen(self.cmd, **kwargs)
     except Exception, e:
         self.exception = e
         self.return_type = Process.NOT_RUN
         return

def jar(jar_path, files, paths):
    """ Search each files using qibuild find and
        add them into a jar using qisys
    """

    # Create command line
    jar_path = qisys.sh.to_native_path(jar_path)
    args = ["cvfM"]
    args += [jar_path]

    if not files:
        raise Exception("Missing arguments : Files to package")
    for wanted_file in files:
        ui.info("Searching for " + wanted_file + "...")
        path = qibuild.find.find(paths, wanted_file, expect_one=False)[0]
        if not path:
            ui.error("Cannot find " + wanted_file + " in worktree")
            return None
        ui.debug("Found : " + path)
        dirname = os.path.dirname(path)
        basename = os.path.basename(path)
        args += ["-C", dirname, basename]
        ui.debug("Added -C " + dirname + " " + wanted_file + " to command line")

    qisys.command.call(["jar"] + args, ignore_ret_code=False)
    return jar_path

 def call(self, *args, **kwargs):
     """ Call """
     if "cwd" not in kwargs.keys():
         kwargs["cwd"] = self.path
     ui.debug("svn", " ".join(args), "in", kwargs["cwd"])
     if "quiet" not in kwargs.keys():
         kwargs["quiet"] = False
     svn = qisys.command.find_program("svn", raises=True)
     cmd = [svn]
     cmd.extend(args)
     raises = kwargs.get("raises")
     if raises is False:
         del kwargs["raises"]
         del kwargs["quiet"]
         process = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT,
                                    **kwargs)
         out = process.communicate()[0]
         # Don't want useless blank lines
         out = out.rstrip("\n")
         ui.debug("out:", out)
         return process.returncode, out
     else:
         if "raises" in kwargs:
             del kwargs["raises"]
         qisys.command.call(cmd, **kwargs)
     return None

 def on_dependent_job_finished(self, job):
     with self.lock:
         try:
             self.deps.remove(job)
         except ValueError:
             ui.debug(ui.red, "Job not in the deps list!", self.deps, job)
             pass

def _compress_zip(directory, quiet=True, verbose=False, flat=False, output=None):
    """Compress directory in a .zip file

    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param quiet:            quiet mode (print nothing)

    :return: path to the generated archive (archive_basepath.zip)

    """
    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED)
    for root, _, filenames in os.walk(directory):
        for filename in filenames:
            full_path = os.path.join(root, filename)
            # Do not zip ourselves
            if full_path == output:
                continue
            rel_path  = os.path.relpath(full_path, directory)
            if flat:
                arcname = rel_path
            else:
                arcname = os.path.join(os.path.basename(directory), rel_path)
            if not quiet:
                sys.stdout.write("adding {0}\n".format(rel_path))
                sys.stdout.flush()
            if not qisys.sh.broken_symlink(full_path):
                archive.write(full_path, arcname)
    archive.close()
    return output

def run(projects, binary, bin_args, env=None, exec_=True):
    """ Find binary in worktree and
        exec it with given arguments.
    """
    paths = list()
    for proj in projects:
        paths += [proj.sdk_directory]
    if os.path.exists(binary):
        bin_path = qisys.sh.to_native_path(binary)
    else:
        bin_path = None
        candidates = qibuild.find.find_bin(paths, binary, expect_one=False)
        if len(candidates) == 1:
            bin_path = candidates[0]
        if len(candidates) > 1:
            bin_path = qisys.interact.ask_choice(candidates,
                                                 "Please select a binary to run")
    if not bin_path:
        bin_path = qisys.command.find_program(binary)
    if not bin_path:
        raise Exception("Cannot find " + binary + " binary")
    cmd = [bin_path] + bin_args
    if exec_:
      ui.debug("exec", cmd)
      os.execve(bin_path,  cmd, env)
    else:
      qisys.command.call(cmd, env=env)

def handle_pure_python(venv_path, python_worktree, env=None):
    """ Add the paths of all python projects to the virtualenv """
    lib_path = virtualenv.path_locations(venv_path)[1]
    qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
    res = True
    with open(qi_pth_dest, "w") as fp:
        fp.write("")
        for i, project in enumerate(python_worktree.python_projects):
            ui.info_count(i, len(python_worktree.python_projects),
                          ui.blue, project.name)
            if project.setup_with_distutils:
                cmd = [python_worktree.pip, "install"]
                if not ui.CONFIG["verbose"]:
                    cmd.append("--quiet")
                cmd.extend(["--editable", "."])
                rc = qisys.command.call(cmd, cwd=project.path, ignore_ret_code=True,
                                        env=env)
                if rc != 0:
                    ui.warning("Failed to run pip install on", project.src)
                    res = False
            else:
                ui.debug("Adding python path for project", project.name, ":\n",
                         project.python_path)
                for path in project.python_path:
                    fp.write(path + "\n")
    return res

def find_program(executable, env=None, raises=False, build_config=None):
    """
    Get the full path of an executable by
    looking at PATH environment variable
    (and PATHEXT on windows)
    Toolchain binaries from build_config are also prepend to path.
    :return: None if program was not found,
      the full path to executable otherwise
    """
    if executable in _FIND_PROGRAM_CACHE:
        return _FIND_PROGRAM_CACHE[executable]
    res = None
    if not env:
        env = dict(qibuild.config.get_build_env())
        if not env:
            env = dict(os.environ)
    toolchain_paths = get_toolchain_binary_paths(build_config)
    if toolchain_paths:
        env["PATH"] = os.pathsep.join((toolchain_paths, env.get("PATH", "")))
    for path in env["PATH"].split(os.pathsep):
        res = _find_program_in_path(executable, path)
        if res and _is_runnable(res):
            ui.debug("Use %s from: %s" % (executable, res))
            _FIND_PROGRAM_CACHE[executable] = res
            return res
    if raises:
        raise NotInPath(executable, env=env)
    return None

 def _call(self, *args, **kwargs):
     """ Helper for self.call """
     ui.debug("git", " ".join(args))
     if not "cwd" in kwargs.keys():
         kwargs["cwd"] = self.repo
     if not "quiet" in kwargs.keys():
         kwargs["quiet"] = False
     git = qisys.command.find_program("git", raises=True)
     cmd = [git]
     cmd.extend(args)
     raises = kwargs.get("raises")
     if raises is False:
         del kwargs["raises"]
         del kwargs["quiet"]
         process = subprocess.Popen(cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             **kwargs)
         out = process.communicate()[0]
         # Don't want useless blank lines
         out = out.rstrip("\n")
         ui.debug("out:", out)
         return (process.returncode, out)
     else:
         if "raises" in kwargs:
             del kwargs["raises"]
         qisys.command.call(cmd, **kwargs)

 def run(self, timeout=None):
     def target():
         ui.debug("Starting thread.")
         ui.debug("Calling:", " ".join(self.cmd))
         try:
             opts = dict()
             if os.name == 'posix':
                 opts = {
                     'preexec_fn': os.setsid,
                     'close_fds': True
                 }
             elif os.name == 'nt':
                 opts = {
                     'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
                 }
             self._process = subprocess.Popen(self.cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 cwd=self.cwd,
                 env=self.env,
                 **opts)
         except Exception, e:
             self.exception = e
             self.return_type = Process.NOT_RUN
             return
         self.out = self._process.communicate()[0]
         self.returncode = self._process.returncode
         if self.returncode == 0:
             self.return_type = Process.OK
         ui.debug("Thread terminated.")

def find_installed_cmake_qibuild_dir(python_dir):
    ui.debug("looking for cmake code from", python_dir)
    for candidate in [
        # python in qibuild/python, cmake in qibuild/cmake
        ("..", "..", "cmake"),
        # python in lib/python-2.7/{dist,site}-packages,
        # cmake in share/cmake/
        # (default pip)
        ("..", "..", "..", "..", "share", "cmake"),
        # python in local/lib/python-2.7/{dist,site}-packages,
        # cmake in share/cmake
        # (debian's pip)
        ("..", "..", "..", "..", "..", "share", "cmake"),
        # python in Python27\Lib\{dist,site}-packages
        # cmake in Python27\share\cmake
        # (windows' pip)
        ("..", "..", "..", "share", "cmake"),
        # python in qibuild.egg/qibuild,
        # cmake in qibuild.egg/share/cmake
        # (pip with setuptools)
        ("..", "share", "cmake"),
        # pip on mac
        (sys.prefix, "share", "cmake")
        ]:

        rel_path = os.path.join(*candidate)
        res = os.path.join(python_dir, rel_path)
        res = qisys.sh.to_native_path(res)
        qibuild_config = os.path.join(res, "qibuild", "qibuild-config.cmake")
        ui.debug("trying", qibuild_config)
        if os.path.exists(qibuild_config):
            return res

 def get_server_access(self, server_name):
     """ Return the access settings of a server. """
     server = self.servers.get(server_name)
     ui.debug("access for", server_name, ":", server)
     if not server:
         return None
     return server.access

 def _resolve_job_build_dependencies(self, job):
     for p in job.project.build_depends:
         dep_job = self._find_job_by_name(p)
         if dep_job:
             job.add_dependency(dep_job)
         else:
             ui.debug("Job {job}: Couldn't find the job for the project dependency {dep}". \
                 format(job=job.project.name, dep=p))

 def _kill_subprocess(self):
     if self._thread and self._process:
         ui.debug('Terminating process.')
         self._process.terminate()
         self.return_type = Process.TIME_OUT
         self._thread.join(5)
         if self._thread.is_alive():
             self._destroy_zombie()

    def install(self, destdir, prefix="/", components=None, num_jobs=1,
                split_debug=False):
        """ Install the project

        :param project: project name.
        :param destdir: destination. Note that when using `qibuild install`,
          we will first call `cmake` to make sure `CMAKE_INSTALL_PREFIX` is
          ``/``. But this function simply calls ``cmake --target install``
          in the simple case.
        :param runtime: Whether to install the project as a runtime
           package or not.
           (see :ref:`cmake-install` section for the details)
        :package split_debug: split the debug symbols out of the binaries
            useful for `qibuild deploy`

        """
        installed = list()
        if components is None:
            components = list()
        # DESTDIR=/tmp/foo and CMAKE_PREFIX="/usr/local" means
        # dest = /tmp/foo/usr/local
        destdir = qisys.sh.to_native_path(destdir)
        build_env = self.build_env.copy()
        build_env["DESTDIR"] = destdir
        # Must make sure prefix is not seen as an absolute path here:
        dest = os.path.join(destdir, prefix[1:])
        dest = qisys.sh.to_native_path(dest)

        cprefix = qibuild.cmake.get_cached_var(self.build_directory,
                                               "CMAKE_INSTALL_PREFIX")
        if cprefix != prefix:
            qibuild.cmake.cmake(self.path, self.build_directory,
                ['-DCMAKE_INSTALL_PREFIX=%s' % prefix],
                clean_first=False,
                env=build_env)
        else:
            mess = "Skipping configuration of project %s\n" % self.name
            mess += "CMAKE_INSTALL_PREFIX is already correct"
            ui.debug(mess)

        # Hack for http://www.cmake.org/Bug/print_bug_page.php?bug_id=13934
        if "Unix Makefiles" in self.cmake_generator:
            self.build(target="preinstall", num_jobs=num_jobs, env=build_env)
        if components:
            for component in components:
                files = self._install_component(destdir, component)
                installed.extend(files)
        else:
            self.build(target="install", env=build_env)
            manifest_path = os.path.join(self.build_directory, "install_manifest.txt")
            installed.extend(read_install_manifest(manifest_path, destdir))
        if "test" in components:
            self._install_qitest_json(destdir)

        if split_debug:
            self.split_debug(destdir, file_list=installed)

        return installed

def configure_virtualenv(config, python_worktree,  build_worktree=None,
                         remote_packages=None, site_packages=True):
    """ Main entry point. Called by ``qipy bootstrap``

    :param: remote_packages List of third-party packages to add in the virtualenv
    :param: site_packages Allow access to global site packages

    """
    ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
    if not remote_packages:
        remote_packages = list()

    # create a new virtualenv
    python_worktree.config = config
    venv_path = python_worktree.venv_path
    pip = python_worktree.pip

    try:
        virtualenv.create_environment(python_worktree.venv_path,
                                      site_packages=site_packages)
    except:
        ui.error("Failed to create virtualenv")
        return False


    ui.info("Adding python projects")
    # Write a qi.pth file containing path to C/C++ extensions and
    # path to pure python modules or packages
    pure_python_ok = handle_pure_python(venv_path, python_worktree)
    if build_worktree:
        handle_extensions(venv_path, python_worktree, build_worktree)

    ui.info("Adding other requirements: " + ", ".join(remote_packages))
    binaries_path = virtualenv.path_locations(venv_path)[-1]
    pip_binary = os.path.join(binaries_path, "pip")
    remote_ok = True
    if remote_packages:
        cmd = [pip_binary, "install"] + remote_packages
        rc = qisys.command.call(cmd, ignore_ret_code=True)
        remote_ok = (rc == 0)
    if pure_python_ok and remote_ok:
        ui.info(ui.green, "Done")
    if not pure_python_ok:
        ui.info(ui.red, "Failed to add some python projects")
    if not remote_ok:
        ui.info(ui.red, "Failed to add some third party requirements")

    requirements_ok = True
    for project in python_worktree.python_projects:
        path = os.path.join(project.path, "requirements.txt")
        if os.path.isfile( path ):
            ui.info(ui.green, " * Installing dependencies from " + path)
            cmd = [pip_binary, "install", "--requirement", path]
            rc = qisys.command.call(cmd, ignore_ret_code=True)
            requirements_ok = (rc == 0)
        else:
            ui.debug(ui.yellow, " * missing " + path)
    return (pure_python_ok and remote_ok and requirements_ok)

 def build(self, **kwargs):
     """ Run sphinx.main() with the correct arguments """
     try:
         import sphinx
     except ImportError as e:
         ui.error(e, "skipping build")
         return
     build_type = kwargs.get("build_type", None)
     language = kwargs.get("language", None)
     spellcheck = kwargs.get("spellcheck", False)
     werror = kwargs.get("werror", False)
     pdb = kwargs.get("pdb", False)
     if self.prebuild_script:
         ui.info(ui.green, "Running pre-build script:",
                 ui.white, self.prebuild_script)
         cmd = [sys.executable, self.prebuild_script]
         qisys.command.call(cmd, cwd=self.path)
         ui.info()
     self.generate_examples_zips()
     if self.translated and language and language != "en" \
             and language not in self.linguas:
         raise UnknownLingua(self, language)
     if self.translated:
         self.intl_build(language)
     qisys.sh.mkdir(self.html_dir, recursive=True)
     spell_dir = os.path.join(self.build_dir, "spellcheck")
     qisys.sh.mkdir(spell_dir, recursive=True)
     cmd = [sys.executable, "-c", self.build_dir]
     if spellcheck:
         cmd.extend(("-b", "spelling"))
     else:
         cmd.extend(("-b", "html"))
     if werror:
         cmd.append("-W")
     if language:
         cmd.append("-Dlanguage=%s" % language)
     if pdb:
         cmd.append("-P")
     cmd.append(self.source_dir)
     if spellcheck:
         cmd.append(spell_dir)
     else:
         cmd.append(self.html_dir)
     if build_type:
         os.environ["build_type"] = build_type
     ui.debug("launching:", cmd)
     rc = 0
     try:
         sphinx.main(argv=cmd)
     except SystemExit as e:
         rc = e.code
     if spellcheck:
         num_errors = get_num_spellcheck_errors(self.build_dir)
         if num_errors != 0:
             raise SphinxBuildError(self)
     if rc != 0:
         raise SphinxBuildError(self)

def _compress_zip(directory, quiet=True, verbose=False, display_progress=False,
                  flat=False, output=None):
    """
    Compress directory in a .zip file
    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param quiet:            quiet mode (print nothing)
    :return: path to the generated archive (archive_basepath.zip)
    """
    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, allowZip64=True)
    # a list of tuple src, arcname to be added in the archive
    to_add = list()
    for root, directories, filenames in os.walk(directory):
        entries = directories
        entries.extend(filenames)
        for entry in entries:
            full_path = os.path.join(root, entry)
            # Do not zip ourselves
            if full_path == output:
                continue
            rel_path = os.path.relpath(full_path, directory)
            if flat:
                arcname = rel_path
            else:
                arcname = os.path.join(os.path.basename(directory), rel_path)
            to_add.append((full_path, arcname))
    for i, (full_path, arcname) in enumerate(to_add):
        if os.path.islink(full_path):
            content = os.readlink(full_path)  # pylint:disable=no-member
            attr = zipfile.ZipInfo(arcname)
            attr.create_system = 3
            # long type of hex val of '0xA1ED0000L',
            # say, symlink attr magic..
            attr.external_attr = 2716663808
            zip_call = archive.writestr
        elif os.path.isdir(full_path):
            continue
        else:
            attr = full_path
            content = arcname
            zip_call = archive.write
        if not quiet and not display_progress:
            rel_path = os.path.relpath(full_path, directory)
            sys.stdout.write("adding {0}\n".format(rel_path.encode('ascii', "ignore")))
            sys.stdout.flush()
        if display_progress:
            ui.info_progress(i, len(to_add), "Done")
        zip_call(attr, content.encode('ascii', "ignore"))
    archive.close()
    return output

def _fix_rpaths(package_path):
    """ Search all dylib in lib directory and fix rpaths. """
    ui.info("Fix RPATH for", package_path, "librairies")
    # pylint: disable=W0612
    for root, dirs, files in os.walk(package_path):
        for basename in files:
            if basename.endswith("dylib"):
                ui.debug("Fixing RPATH for", basename)
                filename = os.path.join(root, basename)
                _fix_rpath(filename, package_path)