def decompressor_for(path):
    """Get the appropriate decompressor for a path."""
    if path.endswith(".zip"):
        unzip = which('unzip', required=True)
        return unzip
    tar = which('tar', required=True)
    tar.add_default_arg('-xf')
    return tar

    def __call__(self, stage, url):
        """Try to guess the type of build system used by a project based on
        the contents of its archive or the URL it was downloaded from."""

        # Most octave extensions are hosted on Octave-Forge:
        #     http://octave.sourceforge.net/index.html
        # They all have the same base URL.
        if 'downloads.sourceforge.net/octave/' in url:
            self.build_system = 'octave'
            return

        # A list of clues that give us an idea of the build system a package
        # uses. If the regular expression matches a file contained in the
        # archive, the corresponding build system is assumed.
        # NOTE: Order is important here. If a package supports multiple
        # build systems, we choose the first match in this list.
        clues = [
            (r'/CMakeLists\.txt$',    'cmake'),
            (r'/configure$',          'autotools'),
            (r'/configure\.(in|ac)$', 'autoreconf'),
            (r'/Makefile\.am$',       'autoreconf'),
            (r'/SConstruct$',         'scons'),
            (r'/waf$',                'waf'),
            (r'/setup\.py$',          'python'),
            (r'/NAMESPACE$',          'r'),
            (r'/WORKSPACE$',          'bazel'),
            (r'/Build\.PL$',          'perlbuild'),
            (r'/Makefile\.PL$',       'perlmake'),
            (r'/.*\.pro$',            'qmake'),
            (r'/(GNU)?[Mm]akefile$',  'makefile'),
            (r'/DESCRIPTION$',        'octave'),
            (r'/meson\.build$',       'meson'),
        ]

        # Peek inside the compressed file.
        if stage.archive_file.endswith('.zip'):
            try:
                unzip  = which('unzip')
                output = unzip('-lq', stage.archive_file, output=str)
            except ProcessError:
                output = ''
        else:
            try:
                tar    = which('tar')
                output = tar('--exclude=*/*/*', '-tf',
                             stage.archive_file, output=str)
            except ProcessError:
                output = ''
        lines = output.split('\n')

        # Determine the build system based on the files contained
        # in the archive.
        for pattern, bs in clues:
            if any(re.search(pattern, l) for l in lines):
                self.build_system = bs
                break

def decompressor_for(path, extension=None):
    """Get the appropriate decompressor for a path."""
    if ((extension and re.match(r'\.?zip$', extension)) or
            path.endswith('.zip')):
        unzip = which('unzip', required=True)
        unzip.add_default_arg('-q')
        return unzip
    if extension and re.match(r'gz', extension):
        gunzip = which('gunzip', required=True)
        return gunzip
    tar = which('tar', required=True)
    tar.add_default_arg('-xf')
    return tar

def _debug_tarball_suffix():
    now = datetime.now()
    suffix = now.strftime('%Y-%m-%d-%H%M%S')

    git = which('git')
    if not git:
        return 'nobranch-nogit-%s' % suffix

    with working_dir(spack.paths.prefix):
        if not os.path.isdir('.git'):
            return 'nobranch.nogit.%s' % suffix

        # Get symbolic branch name and strip any special chars (mainly '/')
        symbolic = git(
            'rev-parse', '--abbrev-ref', '--short', 'HEAD', output=str).strip()
        symbolic = re.sub(r'[^\w.-]', '-', symbolic)

        # Get the commit hash too.
        commit = git(
            'rev-parse', '--short', 'HEAD', output=str).strip()

        if symbolic == commit:
            return "nobranch.%s.%s" % (commit, suffix)
        else:
            return "%s.%s.%s" % (symbolic, commit, suffix)

    def setUp(self):
        super(InstallTest, self).setUp()

        self.stage = Stage('not_a_real_url')
        archive_dir = join_path(self.stage.path, dir_name)
        dummy_configure = join_path(archive_dir, 'configure')

        mkdirp(archive_dir)
        with closing(open(dummy_configure, 'w')) as configure:
            configure.write(
                "#!/bin/sh\n"
                "prefix=$(echo $1 | sed 's/--prefix=//')\n"
                "cat > Makefile <<EOF\n"
                "all:\n"
                "\techo Building...\n\n"
                "install:\n"
                "\tmkdir -p $prefix\n"
                "\ttouch $prefix/dummy_file\n"
                "EOF\n")
        os.chmod(dummy_configure, 0755)

        with working_dir(self.stage.path):
            tar = which('tar')
            tar('-czf', archive_name, dir_name)

        # We use a fake package, so skip the checksum.
        spack.do_checksum = False

        # Use a fake install directory to avoid conflicts bt/w
        # installed pkgs and mock packages.
        self.tmpdir = tempfile.mkdtemp()
        self.orig_layout = spack.install_layout
        spack.install_layout = SpecHashDirectoryLayout(self.tmpdir)

    def _default_target_from_env(self):
        '''Set and return the default CrayPE target loaded in a clean login
        session.

        A bash subshell is launched with a wiped environment and the list of
        loaded modules is parsed for the first acceptable CrayPE target.
        '''
        # Based on the incantation:
        # echo "$(env - USER=$USER /bin/bash -l -c 'module list -lt')"
        if getattr(self, 'default', None) is None:
            env = which('env')
            env.add_default_arg('-')
            # CAUTION - $USER is generally needed in the sub-environment.
            # There may be other variables needed for general success.
            output = env('USER=%s' % os.environ['USER'],
                         'HOME=%s' % os.environ['HOME'],
                         '/bin/bash', '--noprofile', '--norc', '-c',
                         '. /etc/profile; module list -lt',
                         output=str, error=str)
            self._defmods = _get_modules_in_modulecmd_output(output)
            targets = []
            _fill_craype_targets_from_modules(targets, self._defmods)
            self.default = targets[0] if targets else None
            tty.debug("Found default modules:",
                      *["     %s" % mod for mod in self._defmods])
        return self.default

def get_origin_url():
    git_dir = join_path(spack.prefix, '.git')
    git = which('git', required=True)
    origin_url = git(
        '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url',
        return_output=True)
    return origin_url.strip()

    def setUp(self):
        """This sets up a mock archive to fetch, and a mock temp space for use
           by the Stage class.  It doesn't actually create the Stage -- that
           is done by individual tests.
        """
        if os.path.exists(test_files_dir):
            shutil.rmtree(test_files_dir)

        mkdirp(test_files_dir)
        mkdirp(archive_dir_path)
        mkdirp(test_tmp_path)

        with closing(open(test_readme, 'w')) as readme:
            readme.write(readme_text)

        with working_dir(test_files_dir):
            tar = which('tar')
            tar('czf', archive_name, archive_dir)

        # Make spack use the test environment for tmp stuff.
        self.old_tmp_dirs = spack.tmp_dirs
        spack.tmp_dirs = [test_tmp_path]

        # record this since this test changes to directories that will
        # be removed.
        self.working_dir = os.getcwd()

    def setUp(self):
        super(InstallTest, self).setUp()

        self.stage = Stage('not_a_real_url')
        archive_dir = join_path(self.stage.path, dir_name)
        dummy_configure = join_path(archive_dir, 'configure')

        mkdirp(archive_dir)
        with closing(open(dummy_configure, 'w')) as configure:
            configure.write(
                "#!/bin/sh\n"
                "prefix=$(echo $1 | sed 's/--prefix=//')\n"
                "cat > Makefile <<EOF\n"
                "all:\n"
                "\techo Building...\n\n"
                "install:\n"
                "\tmkdir -p $prefix\n"
                "\ttouch $prefix/dummy_file\n"
                "EOF\n")
        os.chmod(dummy_configure, 0755)

        with working_dir(self.stage.path):
            tar = which('tar')
            tar('-czf', archive_name, dir_name)

        # We use a fake pacakge, so skip the checksum.
        spack.do_checksum = False

def test_create_db_tarball(tmpdir, database):
    with tmpdir.as_cwd():
        debug('create-db-tarball')

        # get the first non-dotfile to avoid coverage files in the directory
        files = os.listdir(os.getcwd())
        tarball_name = next(f for f in files if not f.startswith('.'))

        # debug command made an archive
        assert os.path.exists(tarball_name)

        # print contents of archive
        tar = which('tar')
        contents = tar('tzf', tarball_name, output=str)

        # DB file is included
        assert 'index.json' in contents

        # spec.yamls from all installs are included
        for spec in database.query():
            # externals won't have a spec.yaml
            if spec.external:
                continue

            spec_suffix = '%s/.spack/spec.yaml' % spec.dag_hash()
            assert spec_suffix in contents

    def __call__(self, stage):
        """Try to guess the type of build system used by the project, and return
           an appropriate configure line.
        """
        autotools = "configure('--prefix=%s' % prefix)"
        cmake     = "cmake('.', *std_cmake_args)"
        python    = "python('setup.py', 'install', '--prefix=%s' % prefix)"

        config_lines = ((r'/configure$',      'autotools', autotools),
                        (r'/CMakeLists.txt$', 'cmake',     cmake),
                        (r'/setup.py$',       'python',    python))

        # Peek inside the tarball.
        tar = which('tar')
        output = tar(
            "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True)
        lines = output.split("\n")

        # Set the configure line to the one that matched.
        for pattern, bs, cl in config_lines:
            if any(re.search(pattern, l) for l in lines):
                config_line = cl
                build_system = bs
                break
        else:
            # None matched -- just put both, with cmake commented out
            config_line =  "# FIXME: Spack couldn't guess one, so here are some options:\n"
            config_line += "        # " + autotools + "\n"
            config_line += "        # " + cmake
            build_system = 'unknown'

        self.configure = config_line
        self.build_system = build_system

def bootstrap(parser, args):
    origin_url, branch = get_origin_info(args.remote)
    prefix = args.prefix

    tty.msg("Fetching spack from '%s': %s" % (args.remote, origin_url))

    if os.path.isfile(prefix):
        tty.die("There is already a file at %s" % prefix)

    mkdirp(prefix)

    if os.path.exists(join_path(prefix, '.git')):
        tty.die("There already seems to be a git repository in %s" % prefix)

    files_in_the_way = os.listdir(prefix)
    if files_in_the_way:
        tty.die("There are already files there! "
                "Delete these files before boostrapping spack.",
                *files_in_the_way)

    tty.msg("Installing:",
            "%s/bin/spack" % prefix,
            "%s/lib/spack/..." % prefix)

    os.chdir(prefix)
    git = which('git', required=True)
    git('init', '--shared', '-q')
    git('remote', 'add', 'origin', origin_url)
    git('fetch', 'origin', '%s:refs/remotes/origin/%s' % (branch, branch),
                           '-n', '-q')
    git('reset', '--hard', 'origin/%s' % branch, '-q')
    git('checkout', '-B', branch, 'origin/%s' % branch, '-q')

    tty.msg("Successfully created a new spack in %s" % prefix,
            "Run %s/bin/spack to use this installation." % prefix)

    def __init__(self):
        self.tempdir = tempfile.mkdtemp()

        self.directory = os.path.join(self.tempdir, 'dir')
        mkdirp(self.directory)

        # Script with short shebang
        self.short_shebang = os.path.join(self.tempdir, 'short')
        with open(self.short_shebang, 'w') as f:
            f.write(short_line)
            f.write(last_line)

        # Script with long shebang
        self.long_shebang = os.path.join(self.tempdir, 'long')
        with open(self.long_shebang, 'w') as f:
            f.write(long_line)
            f.write(last_line)

        # Lua script with long shebang
        self.lua_shebang = os.path.join(self.tempdir, 'lua')
        with open(self.lua_shebang, 'w') as f:
            f.write(lua_line)
            f.write(last_line)

        # Script already using sbang.
        self.has_sbang = os.path.join(self.tempdir, 'shebang')
        with open(self.has_sbang, 'w') as f:
            f.write(sbang_line)
            f.write(long_line)
            f.write(last_line)

        # Fake binary file.
        self.binary = os.path.join(self.tempdir, 'binary')
        tar = which('tar', required=True)
        tar('czf', self.binary, self.has_sbang)

    def __call__(self, stage, url):
        """Try to guess the type of build system used by a project based on
        the contents of its archive or the URL it was downloaded from."""

        # Most octave extensions are hosted on Octave-Forge:
        #     http://octave.sourceforge.net/index.html
        # They all have the same base URL.
        if 'downloads.sourceforge.net/octave/' in url:
            self.build_system = 'octave'
            return

        # A list of clues that give us an idea of the build system a package
        # uses. If the regular expression matches a file contained in the
        # archive, the corresponding build system is assumed.
        clues = [
            (r'/configure$',      'autotools'),
            (r'/CMakeLists.txt$', 'cmake'),
            (r'/SConstruct$',     'scons'),
            (r'/setup.py$',       'python'),
            (r'/NAMESPACE$',      'r'),
            (r'/WORKSPACE$',      'bazel')
        ]

        # Peek inside the compressed file.
        if stage.archive_file.endswith('.zip'):
            try:
                unzip  = which('unzip')
                output = unzip('-lq', stage.archive_file, output=str)
            except:
                output = ''
        else:
            try:
                tar    = which('tar')
                output = tar('--exclude=*/*/*', '-tf',
                             stage.archive_file, output=str)
            except:
                output = ''
        lines = output.split('\n')

        # Determine the build system based on the files contained
        # in the archive.
        build_system = 'generic'
        for pattern, bs in clues:
            if any(re.search(pattern, l) for l in lines):
                build_system = bs

        self.build_system = build_system

def set_module_variables_for_package(pkg):
    """Populate the module scope of install() with some useful functions.
       This makes things easier for package writers.
    """
    m = pkg.module

    # number of jobs spack will to build with.
    jobs = multiprocessing.cpu_count()
    if not pkg.parallel:
        jobs = 1
    elif pkg.make_jobs:
        jobs = pkg.make_jobs
    m.make_jobs = jobs

    # TODO: make these build deps that can be installed if not found.
    m.make = MakeExecutable("make", jobs)
    m.gmake = MakeExecutable("gmake", jobs)

    # easy shortcut to os.environ
    m.env = os.environ

    # Find the configure script in the archive path
    # Don't use which for this; we want to find it in the current dir.
    m.configure = Executable("./configure")

    # TODO: shouldn't really use "which" here.  Consider adding notion
    # TODO: of build dependencies, as opposed to link dependencies.
    # TODO: Currently, everything is a link dependency, but tools like
    # TODO: this shouldn't be.
    m.cmake = which("cmake")

    # standard CMake arguments
    m.std_cmake_args = ["-DCMAKE_INSTALL_PREFIX=%s" % pkg.prefix, "-DCMAKE_BUILD_TYPE=RelWithDebInfo"]
    if platform.mac_ver()[0]:
        m.std_cmake_args.append("-DCMAKE_FIND_FRAMEWORK=LAST")

    # Set up CMake rpath
    m.std_cmake_args.append("-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE")
    m.std_cmake_args.append("-DCMAKE_INSTALL_RPATH=%s" % ":".join(get_rpaths(pkg)))

    # Emulate some shell commands for convenience
    m.pwd = os.getcwd
    m.cd = os.chdir
    m.mkdir = os.mkdir
    m.makedirs = os.makedirs
    m.remove = os.remove
    m.removedirs = os.removedirs
    m.symlink = os.symlink

    m.mkdirp = mkdirp
    m.install = install
    m.install_tree = install_tree
    m.rmtree = shutil.rmtree
    m.move = shutil.move

    # Useful directories within the prefix are encapsulated in
    # a Prefix object.
    m.prefix = pkg.prefix

def list_packages(rev):
    pkgpath = os.path.join(spack.paths.packages_path, 'packages')
    relpath = pkgpath[len(spack.paths.prefix + os.path.sep):] + os.path.sep

    git = which('git', required=True)
    with working_dir(spack.paths.prefix):
        output = git('ls-tree', '--full-tree', '--name-only', rev, relpath,
                     output=str)
    return sorted(line[len(relpath):] for line in output.split('\n') if line)

    def configure(self, spec, prefix):
        aspell = spec['aspell'].prefix.bin.aspell
        prezip = spec['aspell'].prefix.bin.prezip
        destdir = prefix

        sh = which('sh')
        sh('./configure', '--vars', "ASPELL={0}".format(aspell),
           "PREZIP={0}".format(prezip),
           "DESTDIR={0}".format(destdir))

    def git(self):
        if not self._git:
            self._git = which('git', required=True)

            # If the user asked for insecure fetching, make that work
            # with git as well.
            if not spack.config.get('config:verify_ssl'):
                self._git.add_default_env('GIT_SSL_NO_VERIFY', 'true')

        return self._git

def pkg_add(args):
    for pkg_name in args.packages:
        filename = spack.repo.path.filename_for_package_name(pkg_name)
        if not os.path.isfile(filename):
            tty.die("No such package: %s.  Path does not exist:" %
                    pkg_name, filename)

        git = which('git', required=True)
        with working_dir(spack.paths.prefix):
            git('-C', spack.paths.packages_path, 'add', filename)

 def _avail_targets(self):
     '''Return a list of available CrayPE CPU targets.'''
     if getattr(self, '_craype_targets', None) is None:
         module = which('modulecmd', required=True)
         module.add_default_arg('python')
         output = module('avail', '-t', 'craype-', output=str, error=str)
         craype_modules = _get_modules_in_modulecmd_output(output)
         self._craype_targets = targets = []
         _fill_craype_targets_from_modules(targets, craype_modules)
     return self._craype_targets