def silly_function(ui):
for c_value in ui.series(coloring_values):
sample_ids = sample_ids_from_metadata_description(open(mapping_fp, 'U'),
'%s:%s' % (coloring_header_name, c_value))
_headers, _data = filter_mapping_file(data, headers, sample_ids, True)
per_color_subject_values = list(set([row[subject_index] for row in _data]))
fd = open(join(output_path, 'color_by_'+c_value+'.txt'), 'w')
for s in ui.series(per_color_subject_values):
fd.write('%s\n' % s)
fd.close()
if not suppress_trajectory_files:
for s in ui.series(per_color_subject_values):
filename = join(output_path, s+'.txt')
if opts.verbose:
print 'Working on printing', filename
COMMAND_CALL = FILTER_CMD % (coords_fp, mapping_fp,
'%s:%s' % (subject_header_name, s), filename,
sorting_category)
o, e, r = qiime_system_call(COMMAND_CALL)
if opts.verbose and e:
print 'Error happened on filtering step: \n%s' % e
continue
COMMAND_CALL = CONVERSION_CMD % (filename, filename)
o, e, r = qiime_system_call(COMMAND_CALL)
if opts.verbose and e:
print 'Error happened on conversion step: \n%s' % e
continue # useless here but just in case
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if opts.submit_jobs and not opts.make_jobs:
option_parser.error('Must pass -m if passing -s. (Sorry about this, '
'it\'s for backwards-compatibility.)')
min_args = 2
if len(args) != min_args:
option_parser.error('Program requires <commands file> and '
'<job prefix>')
if (len(args[1]) > 10 or len(args[1]) == 0):
option_parser.error('job prefix must be 1-10 characters long')
if(not exists(opts.job_dir)):
try:
makedirs(opts.job_dir)
except OSError:
exit(" Jobs directory can not be created. "
"Check for permissions or file with the same name: %s\n"
% opts.job_dir)
commands = list(open(args[0]))
job_prefix = args[1]
if opts.mem_per_cpu:
mem_per_cpu = " --mem_per_cpu=" + opts.mem_per_cpu
else:
mem_per_cpu = ""
if opts.queue:
queue = " -p " + opts.queue
else:
queue = ""
if (opts.make_jobs):
filenames = make_jobs(
commands,
job_prefix,
opts.queue,
opts.job_dir)
else:
exit("Should we ever get here???")
if (opts.submit_jobs):
for f in filenames:
qiime_system_call("".join([
"sbatch",
queue,
" -J ", job_prefix,
mem_per_cpu,
" -o ", normpath(opts.job_dir), sep, job_prefix, "_%j.out",
" ", f
]), shell=True)
def test_mothur_supported_version(self):
"""mothur is in path and version is supported """
acceptable_version = (1, 25, 0)
self.assertTrue(
which("mothur"),
"mothur not found. This may or may not be a problem depending on "
+ "which components of QIIME you plan to use.",
)
# mothur creates a log file in cwd, so create a tmp and cd there first
log_file = join(get_qiime_temp_dir(), "mothur.log")
command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
stdout, stderr, exit_Status = qiime_system_call(command)
# remove log file
remove_files([log_file], error_on_missing=False)
version_string = stdout.strip().split(" ")[1].strip("v.")
try:
version = tuple(map(int, version_string.split(".")))
pass_test = version == acceptable_version
except ValueError:
pass_test = False
version_string = stdout
self.assertTrue(
pass_test,
"Unsupported mothur version. %s is required, but running %s."
% (".".join(map(str, acceptable_version)), version_string),
)
def copy_support_files(file_path):
"""Copy the support files to a named destination
file_path: path where you want the support files to be copied to
Will raise EmperorSupportFilesError if a problem is found whilst trying to
copy the files.
"""
file_path = join(file_path, "emperor_required_resources")
if exists(file_path) == False:
create_dir(file_path, False)
# shutil.copytree does not provide an easy way to copy the contents of a
# directory into another existing directory, hence the system call.
# use double quotes for the paths to escape any invalid chracter(s)/spaces
cmd = 'cp -R "%s/"* "%s"' % (get_emperor_support_files_dir(), abspath(file_path))
cmd_o, cmd_e, cmd_r = qiime_system_call(cmd)
if cmd_e:
raise EmperorSupportFilesError, "Error found whilst trying to copy " + "the support files:\n%s\n Could not execute: %s" % (
cmd_e,
cmd,
)
return
def _iter_ids_over_system_call(cmd_fmt, sample_ids, opts):
"""Iteratively execute a system call over sample IDs
Parameters
----------
cmd_fmt : str
The format of the command to execute. It is expected that there
is a single string format to be done and it should take a sample
ID
sample_ids : Iterable of str
A list of sample IDs of interest
Returns
-------
dict
A dict containing each sample ID and any errors observed or None if
no error was observed for the sample. {str: str or None}
"""
results = {}
for id_ in sample_ids:
cmd = cmd_fmt % {'result_path': _result_path(opts, id_),
'id': id_}
stdout, stderr, return_value = qiime_system_call(cmd)
if return_value != 0:
msg = stderr.splitlines()
results[id_] = 'FAILED (%s): %s' % (msg[-1] if msg else '', cmd)
else:
results[id_] = None
return results
def call_commands_serially(commands,
status_update_callback,
logger,
close_logger_on_success=True):
"""Run list of commands, one after another """
logger.write("Executing commands.\n\n")
for c in commands:
for e in c:
status_update_callback('%s\n%s' % e)
logger.write('# %s command \n%s\n\n' % e)
stdout, stderr, return_value = qiime_system_call(e[1])
if return_value != 0:
msg = "\n\n*** ERROR RAISED DURING STEP: %s\n" % e[0] +\
"Command run was:\n %s\n" % e[1] +\
"Command returned exit status: %d\n" % return_value +\
"Stdout:\n%s\nStderr\n%s\n" % (stdout,stderr)
logger.write(msg)
logger.close()
raise WorkflowError, msg
# in the no error case, we write commands' output to the log
# and also echo to this proc's stdout/stderr
else:
# write stdout and stderr to log file
logger.write("Stdout:\n%s\nStderr:\n%s\n" % (stdout,stderr))
# write stdout to stdout
if stdout:
print stdout
# write stderr to stderr
if stderr:
sys.stderr.write(stderr)
if close_logger_on_success: logger.close()
开发者ID:Jorge-C，项目名称:qiime，代码行数:31，代码来源:util.py
示例7: generate_random_password
def generate_random_password(min_len=8, max_len=12):
"""Returns a random alphanumeric password of random length.
Returns both unencrypted and encrypted password. Encryption is performed
via Apache's htpasswd command, using their custom MD5 algorithm.
Length will be randomly chosen from within the specified bounds
(inclusive).
"""
# Modified from
# http://code.activestate.com/recipes/59873-random-password-generation
chars = letters + digits
length = randint(min_len, max_len)
password = ''.join([choice(chars) for i in range(length)])
# This is hackish but should work for now...
stdout, stderr, ret_val = qiime_system_call('htpasswd -nbm foobarbaz %s' %
password)
if ret_val != 0:
raise ValueError("Error executing htpasswd command. Do you have this "
"command on your machine?")
# Will be in the form foobarbaz:<encrypted password>
encrypted_password = stdout.strip().split('foobarbaz:', 1)[1]
return password, encrypted_password
def run_commands(output_dir,commands,run_id,submit_jobs,keep_temp,queue_name):
"""
"""
job_fps, paths_to_remove = write_job_files(output_dir,commands,run_id,queue_name)
# Call the jobs
if submit_jobs:
for job_fp in job_fps:
qiime_system_call(' '.join(['qsub', job_fp]))
# clean up the shell scripts that were created
if not keep_temp:
for p in paths_to_remove:
try:
# p is file
remove(p)
except OSError:
# p is directory
rmtree(p)
return
def _submit_jobs(self, jobs_fp, job_prefix):
""" Submit the jobs to the queue using cluster_jobs.py
"""
cmd = "%s -ms %s %s" % (self._cluster_jobs_fp, jobs_fp, job_prefix)
stdout, stderr, return_value = qiime_system_call(cmd)
if return_value != 0:
msg = (
"\n\n*** Could not start parallel jobs. \n"
+ "Command run was:\n %s\n" % cmd
+ "Command returned exit status: %d\n" % return_value
+ "Stdout:\n%s\nStderr\n%s\n" % (stdout, stderr)
)
raise RuntimeError, msg
# Leave this comments in as they're useful for debugging.
# print 'Return value: %d\n' % return_value
# print 'STDOUT: %s\n' % stdout
# print 'STDERR: %s\n' % stderr
return stdout, stderr, return_value
开发者ID:qinjunjie，项目名称:qiime，代码行数:20，代码来源:util.py
示例15: get_cluster_ratio
def get_cluster_ratio(fasta_seqs, min_difference_in_clusters):
"""
Uses uclust to calculate cluster ratio
cluster_ratio =
num_of_seq_in_cluster_with_max_seq
divided by
num_of_seq_in cluster_with_second_higest_seq
Parameters
----------
fasta_seqs: list
list of fasta sequences
min_difference_in_clusters: float
percent identity threshold for cluster formation
Returns
----------
cluster_ratio: float
cluster ratio of the sequences using uclust
cluster_ratio =
num_of_seq_in_cluster_with_max_seq /
num_of_seq_in cluster_with_second_higest_seq
"""
cluster_percent_id = min_difference_in_clusters
temp_dir = get_qiime_temp_dir()
fd_uc, uclust_tempfile_name = mkstemp(dir=temp_dir, suffix='.uc')
close(fd_uc)
fd_fas, fasta_tempfile_name = mkstemp(dir=temp_dir, suffix='.uc')
close(fd_fas)
with open(fasta_tempfile_name, 'w') as fasta_tempfile:
fasta_tempfile.write(fasta_seqs)
fasta_tempfile.close()
count = 0
command = "uclust --usersort --input {} --uc {} --id 0.98".format(
fasta_tempfile_name, uclust_tempfile_name)
# In the function, I am calling uclust a large number of times.
# Initially I was using from bfillings.get_clusters_from_fasta_filepath
# but due to issue (biocore/bfillingss#31), I have temporarily
# reverted to qiime_system_call.
count_lookup = defaultdict(int)
qiime_system_call(command)
uclust_tempfile = open(uclust_tempfile_name, 'r')
for line in uclust_tempfile:
if search(r'^C', line):
pieces = line.split('\t')
count_lookup[pieces[1]] += int(pieces[2])
count += 1
uclust_tempfile.close()
files_to_be_removed = list()
files_to_be_removed.append(uclust_tempfile_name)
remove_files(files_to_be_removed)
sorted_counts_in_clusters = sorted(
count_lookup.iteritems(),
key=lambda x: x[1], reverse=True)
try:
max_cluster_count = \
float(str(sorted_counts_in_clusters[0][1]))
second_cluster_count = \
float(str(sorted_counts_in_clusters[1][1]))
return max_cluster_count / second_cluster_count
except IndexError:
return 1
def main():
option_parser, opts, args =\
parse_command_line_parameters(**script_info)
if (opts.suppress_unit_tests and opts.suppress_script_usage_tests):
option_parser.error(
"You're suppressing both test types. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile('OK\s*$')
application_not_found_pattern = re.compile('ApplicationNotFoundError')
python_name = 'python'
bad_tests = []
missing_application_tests = []
# Run through all of QIIME's unit tests, and keep track of any files which
# fail unit tests.
if not opts.suppress_unit_tests:
unittest_names = []
if not opts.unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith('test_') and name.endswith('.py'):
unittest_names.append(join(root, name))
else:
for fp in glob(opts.unittest_glob):
fn = split(fp)[1]
if fn.startswith('test_') and fn.endswith('.py'):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = '%s %s -v' % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
qiime_test_data_dir = join(get_qiime_project_dir(), 'qiime_test_data')
qiime_test_data_dir_exists = exists(qiime_test_data_dir)
if not opts.suppress_script_usage_tests and qiime_test_data_dir_exists:
if opts.script_usage_tests is not None:
script_usage_tests = opts.script_usage_tests.split(',')
else:
script_usage_tests = None
# Run the script usage testing functionality
script_usage_result_summary, has_script_usage_example_failures = \
run_script_usage_tests(
test_data_dir=qiime_test_data_dir,
scripts_dir=get_qiime_scripts_dir(),
working_dir=qiime_config['temp_dir'],
verbose=True,
tests=script_usage_tests,
force_overwrite=True,
timeout=240)
print "==============\nResult summary\n=============="
if not opts.suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" % '\n'.join(bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due " +\
"to missing external applications.\nDepending on the QIIME features " +\
"you plan to use, this may not be critical.\n%s"\
% '\n'.join(missing_application_tests)
if not (missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
if not opts.suppress_script_usage_tests:
if qiime_test_data_dir_exists:
print "\nScript usage test result summary\n--------------------------------\n"
print script_usage_result_summary
else:
print "\nCould not run script usage tests because the directory %s does not exist." % qiime_test_data_dir
print ""
# If script usage tests weren't suppressed, the qiime_test_data dir must
# exist and we can't have any failures.
script_usage_tests_success = (opts.suppress_script_usage_tests or
(qiime_test_data_dir_exists and
not has_script_usage_example_failures))
# If any of the unit tests or script usage tests fail, or if we have any
# missing application errors, use return code 1 (as python's unittest
# module does to indicate one or more failures).
return_code = 1
if (len(bad_tests) == 0 and len(missing_application_tests) == 0 and
script_usage_tests_success):
return_code = 0
#.........这里部分代码省略.........
def main():
option_parser, opts, args = parse_command_line_parameters(**script_info)
if opts.suppress_unit_tests and opts.suppress_script_tests and opts.suppress_script_usage_tests:
option_parser.error("You're suppressing all three test types. Nothing to run.")
test_dir = abspath(dirname(__file__))
unittest_good_pattern = re.compile("OK\s*$")
application_not_found_pattern = re.compile("ApplicationNotFoundError")
python_name = "python"
bad_tests = []
missing_application_tests = []
# Run through all of QIIME's unit tests, and keep track of any files which
# fail unit tests.
if not opts.suppress_unit_tests:
unittest_names = []
if not opts.unittest_glob:
for root, dirs, files in walk(test_dir):
for name in files:
if name.startswith("test_") and name.endswith(".py"):
unittest_names.append(join(root, name))
else:
for fp in glob(opts.unittest_glob):
fn = split(fp)[1]
if fn.startswith("test_") and fn.endswith(".py"):
unittest_names.append(abspath(fp))
unittest_names.sort()
for unittest_name in unittest_names:
print "Testing %s:\n" % unittest_name
command = "%s %s -v" % (python_name, unittest_name)
stdout, stderr, return_value = qiime_system_call(command)
print stderr
if not unittest_good_pattern.search(stderr):
if application_not_found_pattern.search(stderr):
missing_application_tests.append(unittest_name)
else:
bad_tests.append(unittest_name)
bad_scripts = []
if not opts.suppress_script_tests:
# Run through all of QIIME's scripts, and pass -h to each one. If the
# resulting stdout does not being with the Usage text, that is an
# indicator of something being wrong with the script. Issues that would
# cause that are bad import statements in the script, SyntaxErrors, or
# other failures prior to running qiime.util.parse_command_line_parameters.
try:
scripts_dir = get_qiime_scripts_dir()
script_directory_found = True
except AssertionError:
script_directory_found = False
if script_directory_found:
script_names = []
script_names = glob("%s/*py" % scripts_dir)
script_names.sort()
for script_name in script_names:
script_good_pattern = re.compile("^Usage: %s" % split(script_name)[1])
print "Testing %s." % script_name
command = "%s %s -h" % (python_name, script_name)
stdout, stderr, return_value = qiime_system_call(command)
if not script_good_pattern.search(stdout):
bad_scripts.append(script_name)
num_script_usage_example_failures = 0
qiime_test_data_dir = qiime_config["qiime_test_data_dir"]
if not opts.suppress_script_usage_tests and qiime_test_data_dir != None:
# Run the script usage testing functionality
script_usage_result_summary, num_script_usage_example_failures = run_script_usage_tests(
qiime_test_data_dir=qiime_test_data_dir,
qiime_scripts_dir=qiime_config["qiime_scripts_dir"],
working_dir=qiime_config["temp_dir"],
verbose=True,
tests=None, # runs all
failure_log_fp=None,
force_overwrite=True,
)
print "==============\nResult summary\n=============="
if not opts.suppress_unit_tests:
print "\nUnit test result summary\n------------------------\n"
if bad_tests:
print "\nFailed the following unit tests.\n%s" % "\n".join(bad_tests)
if missing_application_tests:
print "\nFailed the following unit tests, in part or whole due " + "to missing external applications.\nDepending on the QIIME features " + "you plan to use, this may not be critical.\n%s" % "\n".join(
missing_application_tests
)
if not (missing_application_tests or bad_tests):
print "\nAll unit tests passed.\n\n"
if not opts.suppress_script_tests:
print "\nBasic script test result summary\n--------------------------------\n"
#.........这里部分代码省略.........
#.........这里部分代码省略.........
# Allow the user to override the default job_prefix (defined by the
# base classes)
if job_prefix is None:
job_prefix = self._get_random_job_prefix(self._job_prefix)
# A temporary output directory is created in output_dir named
# job_prefix. Output files are then moved from the temporary
# directory to the output directory when they are complete,
# allowing a poller to detect when runs complete by the presence
# of their output files.
working_dir = join(output_dir,job_prefix)
try:
makedirs(working_dir)
self.files_to_remove.append(working_dir)
except OSError:
# working dir already exists
pass
# Split the input file into the individual job input files. Add the
# individual job files to the files_to_remove list
input_fps, remove_input_on_completion = self._input_splitter(
input_fp,
params,
self._jobs_to_start,
job_prefix,
working_dir)
if remove_input_on_completion:
self.files_to_remove += input_fps
# Perform any method-specific setup (e.g., formatting a BLAST database)
self._precommand_initiation(input_fp,output_dir,working_dir,params)
# Generate the list of commands to be pushed out to workers
# and the list of output files generated by each job.
commands, job_result_filepaths = self._get_job_commands(input_fps,
output_dir,
params,
job_prefix,
working_dir)
self.files_to_remove += \
self._identify_files_to_remove(job_result_filepaths,params)
# Generate the output clean-up files
merge_map_filepath, deletion_list_filepath, expected_files_filepath =\
self._initialize_output_cleanup_files(job_result_filepaths,
output_dir,
working_dir,
input_file_basename,
params)
# Set up poller apparatus if the user does not suppress polling
if not self._suppress_polling:
poller_command = self._initiate_polling(job_result_filepaths,
working_dir,
poll_directly,
merge_map_filepath,
deletion_list_filepath,
expected_files_filepath)
# If the poller should be run in the same way as the other commands
# (rather than by the current process), add it to the list of commands
if not poll_directly:
commands.append(poller_command)
# Build the filepath for the 'jobs script'. Add that file to the
# files_to_remove list.
jobs_fp = join(working_dir,job_prefix + 'jobs.txt')
self._write_jobs_file(commands,jobs_fp)
self.files_to_remove.append(jobs_fp)
# submit the jobs file using cluster_jobs, if not suppressed by the
# user
if not suppress_submit_jobs:
stdout, stderr, return_value = self._submit_jobs(
jobs_fp=jobs_fp, job_prefix=job_prefix)
# If the poller is going to be run by the current process,
# start polling
if poll_directly:
# IMPORTANT: the following line MUST use qiime_system_call()
# instead of subprocess.call, .check_call, or .check_output in case
# we are invoked in a child process with PIPEs (a deadlock will
# occur otherwise). This can happen if this code is tested by
# all_tests.py, for example.
stdout, stderr, return_value = qiime_system_call(poller_command)
if return_value != 0:
print '**Error occuring when calling the poller directly. '+\
'Jobs may have been submitted, but are not being polled.'
print stderr
print poller_command
exit(-1)
self.files_to_remove = []
# Perform any method-specific cleanup. This should prevent the need to
# overwrite __call__
self._call_cleanup(input_fp,
output_dir,
params,
job_prefix,
poll_directly,
suppress_submit_jobs)
客服电话
APP下载
官方微信
请发表评论