def test_ampliconnoise_install(self):
        """ AmpliconNoise install looks sane."""
        url = "http://qiime.org/install/install.html#ampliconnoise-install-notes"

        pyro_lookup_file = getenv('PYRO_LOOKUP_FILE')
        self.assertTrue(pyro_lookup_file is not None,
                        "$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url)
        self.assertTrue(exists(pyro_lookup_file),
                        "$PYRO_LOOKUP_FILE variable is not set to an existing filepath.")

        seq_lookup_file = getenv('SEQ_LOOKUP_FILE')
        self.assertTrue(seq_lookup_file is not None,
                        "$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url)
        self.assertTrue(exists(seq_lookup_file),
                        "$SEQ_LOOKUP_FILE variable is not set to an existing filepath.")

        self.assertTrue(which("SplitKeys.pl"),
                        "Couldn't find SplitKeys.pl. " +
                        "Perhaps AmpliconNoise Scripts directory isn't in $PATH?" +
                        " See %s for help." % url)

        self.assertTrue(which("FCluster"),
                        "Couldn't find FCluster. " +
                        "Perhaps the AmpliconNoise bin directory isn't in $PATH?" +
                        " See %s for help." % url)

        self.assertTrue(which("Perseus"),
                        "Couldn't find Perseus. " +
                        "Perhaps the AmpliconNoise bin directory isn't in $PATH?" +
                        " See %s for help." % url)

def set_sff_trimpoints_with_sfftools(
        sff_dir, technical_lengths, sffinfo_path='sffinfo', sfffile_path='sfffile',
        debug=False):
    """Set trimpoints to end of technical read for all SFF files in directory.

    This function essentially provides the reference implementation.
    It uses the official sfftools from Roche to process the SFF files.
    """
    if not (exists(sffinfo_path) or which(sffinfo_path)):
        raise ApplicationNotFoundError(
            'sffinfo executable not found. Is it installed and in your $PATH?')
    if not (exists(sfffile_path) or which(sfffile_path)):
        raise ApplicationNotFoundError(
            'sfffile executable not found. Is it installed and in your $PATH?')

    for lib_id, sff_fp in get_per_lib_sff_fps(sff_dir):
        try:
            readlength = technical_lengths[lib_id]
        except KeyError:
            continue

        sffinfo_args = [sffinfo_path, '-s', sff_fp]
        if debug:
            print "Running sffinfo command %s" % sffinfo_args
        sffinfo_output_file = TemporaryFile()
        check_call(sffinfo_args, stdout=sffinfo_output_file)
        sffinfo_output_file.seek(0)

        seqlengths = {}
        for line in sffinfo_output_file:
            if line.startswith('>'):
                fields = line[1:].split()
                seq_len = fields[1].split('=')[1]
                seqlengths[fields[0]] = seq_len

        trim_fp = sff_fp + '.trim'
        trim_file = open(trim_fp, 'w')
        for id_, length in seqlengths.items():
            curr_length = int(seqlengths[id_])
            # Sfftools use 1-based index
            left_trim = readlength + 1
            # Key sequence not included in FASTA length
            right_trim = curr_length + 4
            if curr_length > left_trim:
                trim_file.write(
                    "%s\t%s\t%s\n" % (id_, left_trim, right_trim))
            else:
                stderr.write(
                    'Rejected read %s with trim points %s and %s (orig '
                    'length %s)' % (id_, left_trim, curr_length, length))
        trim_file.close()

        trimmed_sff_fp = sff_fp + '.trimmed'
        sfffile_args = [
            sfffile_path, '-t', trim_fp, '-o', trimmed_sff_fp, sff_fp]
        if debug:
            print "Running sfffile command:", sfffile_args
        check_call(sfffile_args, stdout=open(devnull, 'w'))
        remove(sff_fp)
        rename(trimmed_sff_fp, sff_fp)

    def test_FastTree_supported_version(self):
        """FastTree is in path and version is supported """
        acceptable_version = (2, 1, 3)
        self.assertTrue(which('FastTree'),
                        "FastTree not found. This may or may not be a problem depending on " +
                        "which components of QIIME you plan to use.")

        # If FastTree is run interactively, it outputs the following line:
        #     Usage for FastTree version 2.1.3 SSE3:
        #
        # If run non-interactively:
        #     FastTree Version 2.1.3 SSE3
        command = "FastTree 2>&1 > %s | grep -i version" % devnull
        proc = Popen(command, shell=True, universal_newlines=True,
                     stdout=PIPE, stderr=STDOUT)
        stdout = proc.stdout.read().strip()

        version_str_matches = re.findall('ersion\s+(\S+)\s+', stdout)
        self.assertEqual(len(version_str_matches), 1,
                         "Could not find FastTree version info in usage text "
                         "'%s'." % stdout)

        version_str = version_str_matches[0]

        try:
            version = tuple(map(int, version_str.split('.')))
            pass_test = version == acceptable_version
        except ValueError:
            pass_test = False

        acceptable_version_str = '.'.join(map(str, acceptable_version))
        self.assertTrue(pass_test,
                        "Unsupported FastTree version. %s is required, but "
                        "running %s." % (acceptable_version_str, version_str))

 def test_R_supported_version(self):
     """R is in path and version is supported """
     minimum_version = (2, 12, 0)
     self.assertTrue(which('R'),
                     "R not found. This may or may not be a problem depending on " +
                     "which components of QIIME you plan to use.")
     command = "R --version | grep 'R version' | awk '{print $3}'"
     proc = Popen(command, shell=True, universal_newlines=True,
                  stdout=PIPE, stderr=STDOUT)
     stdout = proc.stdout.read()
     version_string = stdout.strip()
     try:
         version = tuple(map(int, version_string.split('.')))
         pass_test = False
         if version[0] == minimum_version[0]:
             if version[1] == minimum_version[1]:
                 if version[2] >= minimum_version[2]:
                     pass_test = True
             elif version[1] > minimum_version[1]:
                 pass_test = True
         elif version[0] > minimum_version[0]:
             pass_test = True
     except ValueError:
         pass_test = False
         version_string = stdout
     self.assertTrue(pass_test,
                     "Unsupported R version. %s or greater is required, but running %s."
                     % ('.'.join(map(str, minimum_version)), version_string))

    def test_blastall_fp(self):
        """blastall_fp is set to a valid path"""

        blastall = self.config["blastall_fp"]
        if not self.config["blastall_fp"].startswith("/"):
            # path is relative, figure out absolute path
            blast_all = which(blastall)
            if not blast_all:
                raise ApplicationNotFoundError(
                    "blastall_fp set to %s, but is not in your PATH. Either use an absolute path to or put it in your PATH." %
                    blastall)
            self.config["blastall_fp"] = blast_all

        test_qiime_config_variable("blastall_fp", self.config, self, X_OK)

def wait_for_cluster_ids(ids, interval=10):
    """Puts process to sleep until jobs with ids are done.

    ids:  list of ids to wait for

    interval: time to sleep in seconds

    NOT USED ANYMORE
    """
    if which("qstat"):
        for id in ids:
            while(getoutput("qstat %s" % id).startswith("Job")):
                sleep(interval)
    else:
        raise ApplicationNotFoundError("qstat not available. Is it installed?\n" +
                                       "This test may fail if not run on a cluster.")

def submit_jobs(filenames, verbose=False):
    """Submit jobs in filenames.

    filenames: list of prepared qsub job scripts, ready to be submitted

    verbose: a binary verbose flag
    """
    if not which("qsub"):
        raise ApplicationNotFoundError("qsub not found. Can't submit jobs.")

    for file in filenames:
        command = 'qsub %s' % file
        result = Popen(command, shell=True, universal_newlines=True,
                       stdout=PIPE, stderr=STDOUT).stdout.read()
        if verbose:
            print result

    def test_chimeraSlayer_install(self):
        """no obvious problems with ChimeraSlayer install """

        # The ChimerSalyer app requires that all its components are installed
        # relative to the main program ChimeraSlayer.pl.
        # We therefore check that at least one the files is there.
        # However, if the directory structure of ChimeraSlayer changes, this test will most
        # likely fail as well and need to be updated.
        # Tested with the version of microbiomeutil_2010-04-29

        chim_slay = which("ChimeraSlayer.pl")
        self.assertTrue(chim_slay, "ChimeraSlayer was not found in your $PATH")
        dir, app_name = split(chim_slay)
        self.assertTrue(
            exists(dir + "/ChimeraParentSelector/chimeraParentSelector.pl"),
            "ChimeraSlayer depends on external files in directoryies relative to its "
            "install directory. These do not appear to be present.")

def make_cidx_file(fp):
    """make a CS index file.

    fp: reference DB to make an index of

    ChimeraSlayer implicetely performs this task when the file is not
    already created and deletes it when done. Especially in a parallel
    environment it mkaes sense to manually make and delete the file.
    """

    if which("cdbfasta"):
        # cdbfasta comes with the microbiometools/CS
        # should always be installed when using CS
        args = ["cdbfasta", fp]
        # cdbfasta write one line to stderr
        stdout, stderr = Popen(args, stderr=PIPE, stdout=PIPE).communicate()
    else:
        raise ApplicationNotFoundError

    def _error_on_missing_application(self, params):
        """Raise an ApplicationNotFoundError if the app is not accessible

        In this case, checks for the java runtime and the RDP jar file.
        """
        if not (os.path.exists('java') or which('java')):
            raise ApplicationNotFoundError(
                "Cannot find java runtime. Is it installed? Is it in your "
                "path?")
        jar_fp = self._get_jar_fp()
        if jar_fp is None:
            raise ApplicationNotFoundError(
                "JAR file not found in current directory and the RDP_JAR_PATH "
                "environment variable is not set.  Please set RDP_JAR_PATH to "
                "the full pathname of the JAR file.")
        if not os.path.exists(jar_fp):
            raise ApplicationNotFoundError(
                "JAR file %s does not exist." % jar_fp)

    def test_cluster_jobs_script(self):
        """cluster_jobs_fp is set to a good value"""

        qiime_config = load_qiime_config()
        submit_script = qiime_config['cluster_jobs_fp']

        if (submit_script):
            full_path = which(submit_script)
            if full_path:
                submit_script = full_path
            self.assertTrue(exists(submit_script),
                            "cluster_jobs_fp is not set to a valid path in qiime config: %s" % submit_script)
            # check if executable
            self.assertTrue(access(submit_script, X_OK),
                            "cluster_jobs_fp is not executable: %s" % submit_script)
        else:
            # Can't run in parallel, but not a critical error
            pass

    def test_cluster_jobs_fp(self):
        """cluster_jobs_fp is set to a valid path and is executable"""
        fp = self.config["cluster_jobs_fp"]

        if fp:
            full_path = which(fp)
            if full_path:
                fp = full_path

            # test if file exists or is in $PATH
            self.assertTrue(exists(fp),
                            "cluster_jobs_fp set to an invalid file path or is not in $PATH: %s" % fp)

            modes = {R_OK: "readable",
                     W_OK: "writable",
                     X_OK: "executable"}
            # test if file readable
            self.assertTrue(access(fp, X_OK),
                            "cluster_jobs_fp is not %s: %s" % (modes[X_OK], fp))

 def test_rtax_supported_version(self):
     """rtax is in path and version is supported """
     acceptable_version = [(0, 984)]
     self.assertTrue(which('rtax'),
                     "rtax not found. This may or may not be a problem depending on " +
                     "which components of QIIME you plan to use.")
     command = "rtax 2>&1 > %s | grep Version | awk '{print $2}'" % devnull
     proc = Popen(command, shell=True, universal_newlines=True,
                  stdout=PIPE, stderr=STDOUT)
     stdout = proc.stdout.read()
     version_string = stdout.strip()
     try:
         version = tuple(map(int, version_string.split('.')))
         pass_test = version in acceptable_version
     except ValueError:
         pass_test = False
         version_string = stdout
     self.assertTrue(pass_test,
                     "Unsupported rtax version. %s is required, but running %s."
                     % ('.'.join(map(str, acceptable_version)), version_string))

 def test_usearch_supported_version(self):
     """usearch is in path and version is supported """
     acceptable_version = [(5, 2, 236), (5, 2, 236)]
     self.assertTrue(which('usearch'),
                     "usearch not found. This may or may not be a problem depending on " +
                     "which components of QIIME you plan to use.")
     command = "usearch --version"
     proc = Popen(command, shell=True, universal_newlines=True,
                  stdout=PIPE, stderr=STDOUT)
     stdout = proc.stdout.read()
     version_string = stdout.split('v')[1]
     try:
         version = tuple(map(int, version_string.split('.')))
         pass_test = version in acceptable_version
     except ValueError:
         pass_test = False
         version_string = stdout
     self.assertTrue(pass_test,
                     "Unsupported usearch version. %s is required, but running %s."
                     % ('.'.join(map(str, acceptable_version)), version_string))

 def test_clearcut_supported_version(self):
     """clearcut is in path and version is supported """
     acceptable_version = (1, 0, 9)
     self.assertTrue(which('clearcut'),
                     "clearcut not found. This may or may not be a problem depending on " +
                     "which components of QIIME you plan to use.")
     command = "clearcut -V"
     proc = Popen(command, shell=True, universal_newlines=True,
                  stdout=PIPE, stderr=STDOUT)
     stdout = proc.stdout.read()
     version_string = stdout.strip().split(' ')[2].strip()
     try:
         version = tuple(map(int, version_string.split('.')))
         pass_test = version == acceptable_version
     except ValueError:
         pass_test = False
         version_string = stdout
     self.assertTrue(pass_test,
                     "Unsupported clearcut version. %s is required, but running %s."
                     % ('.'.join(map(str, acceptable_version)), version_string))

    def test_denoise_worker(self):
        """denoiser_worker.py is where it belongs and is callable."""
        DENOISE_WORKER = "denoiser_worker.py"

        self.assertTrue(which(DENOISE_WORKER) is not None,
                        "%s is not accessible via the PATH environment "
                        "variable." % DENOISE_WORKER)

        # test if its callable and actually works
        command = "%s -h" % DENOISE_WORKER
        proc = Popen(command, shell=True, universal_newlines=True,
                     stdout=PIPE, stderr=STDOUT)

        if (proc.wait() != 0):
            self.fail("Calling %s failed. Check permissions and that it is in fact an executable."
                      % DENOISE_WORKER)

        result = proc.stdout.read()
        # check that the help string looks correct
        self.assertTrue(result.startswith("Usage"))

 def test_raxmlHPC_supported_version(self):
     """raxmlHPC is in path and version is supported """
     acceptable_version = [(7, 3, 0), (7, 3, 0)]
     self.assertTrue(which('raxmlHPC'),
                     "raxmlHPC not found. This may or may not be a problem depending on " +
                     "which components of QIIME you plan to use.")
     command = "raxmlHPC -v | grep version"
     proc = Popen(command, shell=True, universal_newlines=True,
                  stdout=PIPE, stderr=STDOUT)
     stdout = proc.stdout.read()
     version_string = stdout.strip().split(' ')[4].strip()
     try:
         version = tuple(map(int, version_string.split('.')))
         pass_test = version in acceptable_version
     except ValueError:
         pass_test = False
         version_string = stdout
     self.assertTrue(pass_test,
                     "Unsupported raxmlHPC version. %s is required, but running %s."
                     % ('.'.join(map(str, acceptable_version)), version_string))

    def test_flowgramAli_bin(self):
        """Check if we have a working FlowgramAligner"""
        ali_exe = get_flowgram_ali_exe()

        self.assertTrue(which(ali_exe) is not None, "The alignment program %s "
                        "is not accessible via the PATH environment variable."
                        % ali_exe)

        # test if its callable and actually works
        command = "%s -h" % ali_exe
        proc = Popen(command, shell=True, universal_newlines=True,
                     stdout=PIPE, stderr=STDOUT)

        if (proc.wait() != 0):
            self.fail("Calling %s failed. Check permissions and that it is in fact an executable."
                      % ali_exe)

        result = proc.stdout.read()
        # check that the help string looks correct
        self.assertTrue(result.startswith("Usage"))

    def test_ParsInsert_supported_version(self):
        """ParsInsert is in path and version is supported """
        acceptable_version = ["1.04"]
        self.assertTrue(which('ParsInsert'),
                        "ParsInsert not found. This may or may not be a problem depending on " +
                        "which components of QIIME you plan to use.")
        command = "ParsInsert -v | grep App | awk '{print $3}'"
        proc = Popen(command, shell=True, universal_newlines=True,
                     stdout=PIPE, stderr=STDOUT)
        stdout = proc.stdout.read()

        # remove log file generated
        remove_files(['ParsInsert.log'], error_on_missing=False)

        version_string = stdout.strip()
        try:
            pass_test = version_string in acceptable_version
        except ValueError:
            pass_test = False
            version_string = stdout
        self.assertTrue(pass_test,
                        "Unsupported ParsInsert version. %s is required, but running %s."
                        % ('.'.join(map(str, acceptable_version)), version_string))

    def test_mothur_supported_version(self):
        """mothur is in path and version is supported """
        acceptable_version = (1, 25, 0)
        self.assertTrue(which('mothur'),
                        "mothur not found. This may or may not be a problem depending on " +
                        "which components of QIIME you plan to use.")
        # mothur creates a log file in cwd, so create a tmp and cd there first
        log_file = join(get_qiime_temp_dir(), 'mothur.log')
        command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
        stdout, stderr, exit_Status = qiime_system_call(command)

        # remove log file
        remove_files([log_file], error_on_missing=False)

        version_string = stdout.strip().split(' ')[1].strip('v.')
        try:
            version = tuple(map(int, version_string.split('.')))
            pass_test = version == acceptable_version
        except ValueError:
            pass_test = False
            version_string = stdout
        self.assertTrue(pass_test,
                        "Unsupported mothur version. %s is required, but running %s."
                        % ('.'.join(map(str, acceptable_version)), version_string))