def list_software_txt(software, detailed=False):
    """
    Return overview of supported software in plain text

    :param software: software information (structured like list_software does)
    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
    :return: multi-line string presenting requested info
    """

    lines = ['']
    for key in sorted(software, key=lambda x: x.lower()):
        lines.append('* %s' % key)
        if detailed:
            lines.extend([
                '',
                ' '.join(software[key][-1]['description'].split('\n')),
                '',
                "homepage: %s" % software[key][-1]['homepage'],
                '',
            ])
            pairs = nub((x['version'], x['versionsuffix']) for x in software[key])
            for ver, vsuff in sorted((LooseVersion(v), vs) for (v, vs) in pairs):
                tcs = [x['toolchain'] for x in software[key] if x['version'] == ver and x['versionsuffix'] == vsuff]

                line = "  * %s v%s" % (key, ver)
                if vsuff:
                    line += " (versionsuffix: '%s')" % vsuff
                line += ": %s" % ', '.join(sorted(nub(tcs)))
                lines.append(line)
            lines.append('')

    return '\n'.join(lines)

    def configure_step(self, srcdir=None, builddir=None):
        """Configure build using cmake"""

        if builddir is not None:
            self.log.nosupport("CMakeMake.configure_step: named argument 'builddir' (should be 'srcdir')", "2.0")

        # Set the search paths for CMake
        tc_ipaths = self.toolchain.get_variable("CPPFLAGS", list)
        tc_lpaths = self.toolchain.get_variable("LDFLAGS", list)
        cpaths = os.getenv('CPATH', '').split(os.pathsep)
        lpaths = os.getenv('LD_LIBRARY_PATH', '').split(os.pathsep)
        include_paths = os.pathsep.join(nub(tc_ipaths + cpaths))
        library_paths = os.pathsep.join(nub(tc_lpaths + lpaths))
        setvar("CMAKE_INCLUDE_PATH", include_paths)
        setvar("CMAKE_LIBRARY_PATH", library_paths)

        default_srcdir = '.'
        if self.cfg.get('separate_build_dir', False):
            objdir = os.path.join(self.builddir, 'easybuild_obj')
            try:
                os.mkdir(objdir)
                os.chdir(objdir)
            except OSError, err:
                raise EasyBuildError("Failed to create separate build dir %s in %s: %s", objdir, os.getcwd(), err)
            default_srcdir = self.cfg['start_dir']

 def set_mod_paths(self, mod_paths=None):
     """Set mod_paths, based on $MODULEPATH unless a list of module paths is specified."""
     # make sure we don't have the same path twice, using nub
     if mod_paths is not None:
         self.mod_paths = nub(mod_paths)
         for mod_path in self.mod_paths:
             self.prepend_module_path(mod_path)
     else:
         # no paths specified, so grab list of (existing) module paths from $MODULEPATH
         self.mod_paths = [p for p in nub(curr_module_paths()) if os.path.exists(p)]
     self.log.debug("$MODULEPATH after set_mod_paths: %s" % os.environ.get('MODULEPATH', ''))

    def test_conflicts(self):
        """Check whether any conflicts occur in software dependency graphs."""

        if not single_tests_ok:
            print "(skipped conflicts test)"
            return

        if self.ordered_specs is None:
            self.process_all_easyconfigs()

        def mk_dep_mod_name(spec):
            return tuple(EasyBuildMNS().det_full_module_name(spec).split(os.path.sep))

        # construct a dictionary: (name, installver) tuple to (build) dependencies
        depmap = {}
        for spec in self.ordered_specs:
            # exclude external modules, since we can't check conflicts on them (we don't even know the software name)
            build_deps = [mk_dep_mod_name(d) for d in spec['builddependencies'] if not d.get('external_module', False)]
            deps = [mk_dep_mod_name(d) for d in spec['ec'].all_dependencies if not d.get('external_module', False)]

            # separate runtime deps from build deps
            runtime_deps = [d for d in deps if d not in build_deps]
            key = tuple(spec['full_mod_name'].split(os.path.sep))
            depmap.update({key: [build_deps, runtime_deps]})

        # iteratively expand list of dependencies
        depmap_last = None
        while depmap != depmap_last:
            depmap_last = copy.deepcopy(depmap)
            for (spec, (build_deps, runtime_deps)) in depmap_last.items():
                # extend runtime dependencies with non-build dependencies of own runtime dependencies
                for dep in runtime_deps:
                    depmap[spec][1].extend([d for d in depmap[dep][1] if d not in depmap[dep][0]])
                depmap[spec][1] = sorted(nub(depmap[spec][1]))
                # extend build dependencies with non-build dependencies of own build dependencies
                for dep in build_deps:
                    depmap[spec][0].extend([d for d in depmap[dep][1] if d not in depmap[dep][0]])
                depmap[spec][0] = sorted(nub(depmap[spec][0]))

        def check_conflict((name, installver), (name1, installver1), (name2, installver2)):
            """Check whether dependencies with given name/(install) version conflict with each other."""
            # dependencies with the same name should have the exact same install version
            # if not => CONFLICT!
            if name1 == name2 and installver1 != installver2:
                specname = '%s-%s' % (name, installver)
                vs_msg = "%s-%s vs %s-%s" % (name1, installver1, name2, installver2)
                print "Conflict found for dependencies of %s: %s" % (specname, vs_msg)
                return True
            else:
                return False

def get_convert_class(class_name):
    """Return the Convert class with specified class name class_name"""
    res = [x for x in nub(get_subclasses(Convert)) if x.__name__ == class_name]
    if len(res) == 1:
        return res[0]
    else:
        _log.error('More then one Convert subclass found for name %s: %s' % (class_name, res))

def list_toolchains_rst(tcs):
    """ Returns overview of all toolchains in rst format """
    title = "List of known toolchains"

    # figure out column names
    table_titles = ['name', 'compiler', 'MPI']
    for tc in tcs.values():
        table_titles.extend(tc.keys())

    col_names = {
        'COMPILER_CUDA': 'CUDA compiler',
        'SCALAPACK': 'ScaLAPACK',
    }

    table_titles = nub(table_titles)

    table_values = [[] for i in range(len(table_titles))]
    table_values[0] = ['**%s**' % tcname for tcname in tcs.keys()]

    for idx in range(1, len(table_titles)):
        for tc in tcs.values():
            table_values[idx].append(', '.join(tc.get(table_titles[idx].upper(), [])))

    table_titles = [col_names.get(col, col) for col in table_titles]
    doc = rst_title_and_table(title, table_titles, table_values)

    return '\n'.join(doc)

    def __init__(self, mod_paths=None):
        """
        Create a ModulesTool object
        @param mod_paths: A list of paths where the modules can be located
        @type mod_paths: list
        """
        self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
        # make sure we don't have the same path twice
        if mod_paths:
            self.mod_paths = nub(mod_paths)
        else:
            self.mod_paths = None

        # DEPRECATED!
        self._modules = []

        self.check_module_path()

        # actual module command (i.e., not the 'module' wrapper function, but the binary)
        self.cmd = None

        # shell that should be used to run module command (specified above) in (if any)
        self.shell = None

        # version of modules tool
        self.version = None

        # terse command line option
        self.add_terse_opt_fn = lambda x: x.insert(0, '--terse')

    def check_module_path(self):
        """
        Check if MODULEPATH is set and change it if necessary.
        """
        if not 'MODULEPATH' in os.environ:
            errormsg = 'MODULEPATH not found in environment'
            # check if environment-modules is found
            module_regexp = re.compile(r"^module is a function\s*\nmodule\s*()")
            cmd = "type module"
            (out, ec) = run_cmd(cmd, log_all=False, log_ok=False)
            if ec != 0 or not module_regexp.match(out):
                errormsg += "; environment-modules doesn't seem to be installed: "
                errormsg += "'%s' failed with exit code %s and output: '%s'" % (cmd, ec, out.strip('\n'))
            self.log.error(errormsg)

        if self.mod_paths:
            # set the module path environment accordingly
            os.environ['MODULEPATH'] = ':'.join(self.mod_paths)
            self.log.debug("$MODULEPATH set based on supplied list of module paths: %s" % os.environ['MODULEPATH'])
        else:
            # take module path from environment
            self.mod_paths = nub(os.environ['MODULEPATH'].split(':'))
            self.log.debug("self.mod_paths set based on $MODULEPATH: %s" % self.mod_paths)

        if not 'LOADEDMODULES' in os.environ:
            os.environ['LOADEDMODULES'] = ''

    def test_env_variables(self):
        """ Test the passing of (extra) variables """
        fake_mpirun_env = """#!/bin/bash
        echo 'fake mpirun called with args:' [email protected]
        env
        """
        install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=fake_mpirun_env)

        os.environ['PYTHONPATH'] = '/just/an/example:%s' % os.getenv('PYTHONPATH', '')

        command = [
            sys.executable,
            self.mympiscript,
            "--variablesprefix=USER",
            "hostname",
        ]
        ec, out = run(command)

        for key in nub(filter(os.environ.has_key, MPI.OPTS_FROM_ENV_BASE)):
            self.assertTrue(key in out, "%s is not in out" % key)

        regex = r'.*-envlist [^ ]*USER.*'
        self.assertTrue(re.search(regex, out), "Variablesprefix USER isn't passed to mympirun script env")

        regex = r'PYTHONPATH=/just/an/example:.*'
        self.assertTrue(re.search(regex, out), "PYTHONPATH isn't passed to mympirun script env correctly: %s" % out)

    def list_filesets(self, devices=None, filesetnames=None, update=False):
        """Get all the filesets for one or more specific devices

        @type devices: list of devices (if string: 1 device; if None: all found devices)
        @type filesetnames: report only on specific filesets (if string: 1 filesetname)

            set self.gpfslocalfilesets is dict with
                key = filesystemName value is dict with
                    key = id value is dict
                        key = remaining header entries and corresponding values
        """

        if not update and self.gpfslocalfilesets:
            return self.gpfslocalfilesets

        opts = []

        if devices is None:
            # get all devices from all filesystems
            if self.gpfslocalfilesystems is None:
                self.list_filesystems()

            devices = self.gpfslocalfilesystems.keys()
        else:
            if isinstance(devices, str):
                devices = [devices]

        if filesetnames is not None:
            if isinstance(filesetnames, str):
                filesetnames = [filesetnames]

            filesetnamestxt = ','.join(filesetnames)
            opts.append(filesetnamestxt)

        self.log.debug("Looking up filesets for devices %s" % (devices))

        listm = Monoid([], lambda xs, ys: xs + ys)
        info = MonoidDict(listm)
        for device in devices:
            opts_ = copy.deepcopy(opts)
            opts_.insert(1, device)
            res = self._executeY('mmlsfileset', opts_)
            # for v3.5 filesystemName:filesetName:id:rootInode:status:path:parentId:created:inodes:dataInKB:comment:filesetMode:afmTarget:afmState:afmMode:afmFileLookupRefreshInterval:afmFileOpenRefreshInterval:afmDirLookupRefreshInterval:afmDirOpenRefreshInterval:afmAsyncDelay:reserved:afmExpirationTimeout:afmRPO:afmLastPSnapId:inodeSpace:isInodeSpaceOwner:maxInodes:allocInodes:inodeSpaceMask:afmShowHomeSnapshots:afmNumReadThreads:afmNumReadGWs:afmReadBufferSize:afmWriteBufferSize:afmReadSparseThreshold:afmParallelReadChunkSize:afmParallelReadThreshold:snapId:
            self.log.debug("list_filesets res keys = %s " % (res.keys()))
            for (key, value) in res.items():
                info[key] = value

        datakeys = info.keys()
        datakeys.remove('filesystemName')
        datakeys.remove('id')

        fss = nub(info.get('filesystemName', []))
        res = dict([(fs, {}) for fs in fss])  # build structure

        for idx, (fs, qid) in enumerate(zip(info['filesystemName'], info['id'])):
            details = dict([(k, info[k][idx]) for k in datakeys])
            res[fs][qid] = details

        self.gpfslocalfilesets = res
        return res

    def test_make_machine_file(self):
        """test if the machinefile is made and if it contains the same amount of nodes as mpinodes"""
        mpi_instance = getinstance(mpim.MPI, Local, MympirunOption())
        mpi_instance.make_machine_file()
        self.assertTrue(os.path.isfile(mpi_instance.mpiexec_node_filename), msg="the nodefile has not been created")

        # test if amount of lines in nodefile matches amount of nodes
        with open(mpi_instance.mpiexec_node_filename) as file:
            index = 0
            for index, _ in enumerate(file):
                pass
            self.assertEqual(len(mpi_instance.mpinodes), index+1,
                             msg="mpinodes doesn't match the amount of nodes in the nodefile")

        # disable make_mympirundir
        mpi_instance.make_mympirundir = lambda: True
        mpi_instance.mympirundir = '/does/not/exist/'
        self.assertErrorRegex(IOError, "failed to write nodefile", mpi_instance.make_machine_file)

        # openmpi oversubscribing
        mpi_instance = getinstance(OpenMpiOversubscribe, Local, MympirunOption())
        mpi_instance.options.double = True
        mpi_instance.set_multiplier()
        mpi_instance.make_machine_file()

        with open(mpi_instance.mpiexec_node_filename) as file:
            n_slots = mpi_instance.ppn
            regex = re.compile("slots=%s" % n_slots)
            machinefile = file.read()
            self.assertTrue(regex.search(machinefile), "Regex %s not found in %s" % (regex.pattern, machinefile))

            self.assertEqual(len(nub(mpi_instance.mpinodes)), len(machinefile.strip().split('\n')),
                             msg="mpinodes doesn't match the amount of nodes in the nodefile")

    def validate_iterate_opts_lists(self):
        """
        Configure/build/install options specified as lists should have same length.
        """

        # configure/build/install options may be lists, in case of an iterated build
        # when lists are used, they should be all of same length
        # list of length 1 are treated as if it were strings in EasyBlock
        opt_counts = []
        for opt in ITERATE_OPTIONS:

            # anticipate changes in available easyconfig parameters (e.g. makeopts -> buildopts?)
            if self.get(opt, None) is None:
                self.log.error("%s not available in self.cfg (anymore)?!" % opt)

            # keep track of list, supply first element as first option to handle
            if isinstance(self[opt], (list, tuple)):
                opt_counts.append((opt, len(self[opt])))

        # make sure that options that specify lists have the same length
        list_opt_lengths = [length for (opt, length) in opt_counts if length > 1]
        if len(nub(list_opt_lengths)) > 1:
            self.log.error("Build option lists for iterated build should have same length: %s" % opt_counts)

        return True

def get_convert_class(class_name):
    """Return the Convert class with specified class name class_name"""
    res = [x for x in nub(get_subclasses(Convert)) if x.__name__ == class_name]
    if len(res) == 1:
        return res[0]
    else:
        raise EasyBuildError("More than one Convert subclass found for name %s: %s", class_name, res)

def tweak(easyconfigs, build_specs):
    """Tweak list of easyconfigs according to provided build specifications."""

    # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
    toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
    if len(toolchains) > 1:
        _log.error("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains)

    # obtain full dependency graph for specified easyconfigs
    # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first
    orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True)

    # determine toolchain based on last easyconfigs
    toolchain = orig_ecs[-1]['ec']['toolchain']
    _log.debug("Filtering using toolchain %s" % toolchain)

    # filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies
    if toolchain['name'] != DUMMY_TOOLCHAIN_NAME:
        while orig_ecs[0]['ec']['toolchain'] != toolchain:
            orig_ecs = orig_ecs[1:]

    # generate tweaked easyconfigs, and continue with those instead
    easyconfigs = []
    for orig_ec in orig_ecs:
        new_ec_file = tweak_one(orig_ec['spec'], None, build_specs)
        new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs)
        easyconfigs.extend(new_ecs)

    return easyconfigs

    def make_mpdboot_file(self):
        """
        Make an mpdbootfile.

        Parses the list of unique nodes and writes this information to a mpdbootfile
        (based on hydra and universe options).
        """
        self.make_mympirundir()

        if self.mpinodes is None:
            self.set_mpinodes()

        mpdboottxt = '\n'.join(nub(self.mpinodes))

        mpdfn = os.path.join(self.mympirundir, 'mpdboot')
        try:
            fp = open(mpdfn, 'w')
            fp.write(mpdboottxt)
            fp.close()
        except IOError as err:
            msg = 'make_mpdboot_file: failed to write mpbboot file %s: %s' % (mpdfn, err)
            self.log.raiseException(msg)

        self.mpdboot_node_filename = mpdfn
        self.log.debug("make_mpdboot_file: wrote mpdbootfile %s:\n%s", mpdfn, mpdboottxt)

 def test_get_universe_ncpus(self):
     """ Test mpinode scheduling for --universe option """
     inst = getinstance(mpim.MPI, Local, MympirunOption())
     inst.nodes = [
         'node1',
         'node1',
         'node1',
         'node2',
         'node2',
         'node2',
         'node2',
     ]
     inst.nodes_tot_cnt = len(inst.nodes)
     inst.nodes_uniq = nub(inst.nodes)
     options = {
         2: {'node1': 1, 'node2': 1},
         3: {'node1': 2, 'node2': 1},
         6: {'node1': 3, 'node2': 3}
     }
     for opt in options:
         inst.options.universe = opt
         inst.set_ppn()
         inst.set_mpinodes()
         universe_ppn = inst.get_universe_ncpus()
         self.assertEqual(universe_ppn, options[opt])

    def make_machine_file(self, nodetxt=None, universe=None):
        """
        Make the machinefile.

        Parses the list of nodes that run an MPI process and writes this information to a machinefile.
        """
        if not self.mympirundir:
            self.make_mympirundir()

        if self.mpinodes is None:
            self.set_mpinodes()

        if nodetxt is None:
            if universe is not None and universe > 0:
                universe_ppn = self.get_universe_ncpus()
                nodes = []
                for node in nub(self.mpinodes):
                    nodes.extend([node] * universe_ppn[node])
            else:
                nodes = self.mpinodes

            nodetxt = '\n'.join(nodes)

        nodefn = os.path.join(self.mympirundir, 'nodes')
        try:
            fp = open(nodefn, 'w')
            fp.write(nodetxt)
            fp.close()
        except IOError as err:
            msg = 'make_machine_file: failed to write nodefile %s: %s' % (nodefn, err)
            self.log.raiseException(msg)

        self.mpiexec_node_filename = nodefn
        self.log.debug("make_machine_file: wrote nodefile %s:\n%s", nodefn, nodetxt)

def expand_glob_paths(glob_paths):
    """Expand specified glob paths to a list of unique non-glob paths to only files."""
    paths = []
    for glob_path in glob_paths:
        paths.extend([f for f in glob.glob(glob_path) if os.path.isfile(f)])

    return nub(paths)

    def list_filesystems(self, device='all', update=False):
        """List all filesystems.

        Set self.gpfslocalfilesystems to a convenient dict structure of the returned dict
        where the key is the deviceName, the value is a dict
            where the key is the fieldName and the values are the corresponding value, i.e., the
        """

        if not update and self.gpfslocalfilesystems:
            return self.gpfslocalfilesystems

        info = self._executeY('mmlsfs', [device])
        # for v3.5 deviceName:fieldName:data:remarks:

        # set the gpfsdevices
        gpfsdevices = nub(info.get('deviceName', []))
        if len(gpfsdevices) == 0:
            self.log.raiseException("No devices found. Returned info %s" % info, GpfsOperationError)
        else:
            self.log.debug("listAllFilesystems found devices %s" % gpfsdevices)

        res = dict([(dev, {}) for dev in gpfsdevices])  # build structure
        for dev, k, v in zip(info['deviceName'], info['fieldName'], info['data']):
            res[dev][k] = v

        self.gpfslocalfilesystems = res
        return res

def search_toolchain(name):
    """
    Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains.
    :param name: toolchain name
    :return: Toolchain instance (or None), found_toolchains
    """

    package = easybuild.tools.toolchain
    check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX

    if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name):
        # import all available toolchains, so we know about them
        tc_modules = import_available_modules('easybuild.toolchains')

        # make sure all defined toolchain constants are available in toolchain module
        tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX)
        for tc_mod in tc_modules:
            # determine classes imported in this module
            mod_classes = []
            for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
                if hasattr(elem, '__module__'):
                    # exclude the toolchain class defined in that module
                    if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
                        _log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__)
                        mod_classes.append(elem)

            # look for constants in modules of imported classes, and make them available
            for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
                for elem in dir(mod_class_mod):
                    res = tc_const_re.match(elem)
                    if res:
                        tc_const_name = res.group(1)
                        tc_const_value = getattr(mod_class_mod, elem)
                        _log.debug("Found constant %s ('%s') in module %s, adding it to %s",
                                   tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__)
                        if hasattr(package, tc_const_name):
                            cur_value = getattr(package, tc_const_name)
                            if not tc_const_value == cur_value:
                                raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.",
                                                     package.__name__, tc_const_name, cur_value, tc_const_value)
                        else:
                            setattr(package, tc_const_name, tc_const_value)

        # indicate that processing of toolchain constants is done, so it's not done again
        setattr(package, check_attr_name, True)
    else:
        _log.debug("Skipping importing of toolchain modules, processing of toolchain constants is already done.")

    # obtain all subclasses of toolchain
    found_tcs = nub(get_subclasses(Toolchain))

    # filter found toolchain subclasses based on whether they can be used a toolchains
    found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]

    for tc in found_tcs:
        if tc._is_toolchain_for(name):
            return tc, found_tcs

    return None, found_tcs