def deploy_task(self, task, transfer_se, transfer_sb):
		# HACK
		self._output_fn_list = lmap(lambda d_s_t: d_s_t[2], self._get_out_transfer_info_list(task))
		task.validate_variables()

		# add task SE files to SM
		self._sm_se_in.add_file_list(lmap(lambda d_s_t: d_s_t[2], task.get_se_in_fn_list()))
		# Transfer common SE files
		if transfer_se:
			self._sm_se_in.do_transfer(task.get_se_in_fn_list())

		def _convert(fn_list):
			for fn in fn_list:
				if isinstance(fn, str):
					yield (fn, os.path.basename(fn))
				else:
					yield (fn, os.path.basename(fn.name))

		# Package sandbox tar file
		self._log.log(logging.INFO1, 'Packing sandbox')
		sandbox = self._get_sandbox_name(task)
		ensure_dir_exists(os.path.dirname(sandbox), 'sandbox directory')
		if not os.path.exists(sandbox) or transfer_sb:
			sandbox_file_list = self._get_sandbox_file_list(task, [self._sm_se_in, self._sm_se_out])
			create_tarball(_convert(sandbox_file_list), name=sandbox)

	def __init__(self, fn, format = 'sniffed'):
		(self._fn, self._format) = (fn, format)
		fp = open(fn)
		try:
			first_line = fp.readline()
			sniffed = csv.Sniffer().sniff(first_line)
			csv.register_dialect('sniffed', sniffed)
			tmp = list(csv.DictReader(fp, first_line.strip().split(sniffed.delimiter) + [None], dialect = format))
		finally:
			fp.close()
		for entry in tmp:
			entry.pop(None, None)
			if None in entry.values():
				raise Exception('Malformed entry in csv file %r: %r' % (fn, entry))

		def cleanupDict(d):
			# strip all key value entries
			tmp = tuple(imap(lambda item: lmap(str.strip, item), d.items()))
			# filter empty parameters
			return lfilter(lambda k_v: k_v[0] != '', tmp)
		keys = []
		if len(tmp):
			keys = lmap(ParameterMetadata, tmp[0].keys())
		values = lmap(lambda d: dict(cleanupDict(d)), tmp)
		InternalParameterSource.__init__(self, values, keys)

	def makeJDL(self, jobNum, module):
		cfgPath = os.path.join(self._jobPath, 'job_%d.var' % jobNum)
		sbIn = lmap(lambda d_s_t: d_s_t[1], self._getSandboxFilesIn(module))
		sbOut = lmap(lambda d_s_t: d_s_t[2], self._getSandboxFilesOut(module))
		wcList = lfilter(lambda x: '*' in x, sbOut)
		if len(wcList):
			self._writeJobConfig(cfgPath, jobNum, module, {'GC_WC': str.join(' ', wcList)})
			sandboxOutJDL = lfilter(lambda x: x not in wcList, sbOut) + ['GC_WC.tar.gz']
		else:
			self._writeJobConfig(cfgPath, jobNum, module, {})
			sandboxOutJDL = sbOut
		# Warn about too large sandboxes
		sbSizes = lmap(os.path.getsize, sbIn)
		if sbSizes and (self._warnSBSize > 0) and (sum(sbSizes) > self._warnSBSize * 1024 * 1024):
			if not utils.getUserBool('Sandbox is very large (%d bytes) and can cause issues with the WMS! Do you want to continue?' % sum(sbSizes), False):
				sys.exit(os.EX_OK)
			self._warnSBSize = 0

		reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum), WMS.SITES)
		formatStrList = lambda strList: '{ %s }' % str.join(', ', imap(lambda x: '"%s"' % x, strList))
		contents = {
			'Executable': '"gc-run.sh"',
			'Arguments': '"%d"' % jobNum,
			'StdOutput': '"gc.stdout"',
			'StdError': '"gc.stderr"',
			'InputSandbox': formatStrList(sbIn + [cfgPath]),
			'OutputSandbox': formatStrList(sandboxOutJDL),
			'VirtualOrganisation': '"%s"' % self.vo,
			'Rank': '-other.GlueCEStateEstimatedResponseTime',
			'RetryCount': 2
		}
		return self._jdl_writer.format(reqs, contents)

	def cancel(self, wms, jobs, interactive = False, showJobs = True):
		if len(jobs) == 0:
			return
		if showJobs:
			self._reportClass(self.jobDB, self._task, jobs).display()
		if interactive and not utils.getUserBool('Do you really want to cancel these jobs?', True):
			return

		def mark_cancelled(jobNum):
			jobObj = self.jobDB.get(jobNum)
			if jobObj is None:
				return
			self._update(jobObj, jobNum, Job.CANCELLED)
			self._eventhandler.onJobUpdate(wms, jobObj, jobNum, {'reason': 'cancelled'})

		jobs.reverse()
		for (jobNum, wmsId) in wms.cancelJobs(self._wmsArgs(jobs)):
			# Remove deleted job from todo list and mark as cancelled
			assert(self.jobDB.get(jobNum).wmsId == wmsId)
			jobs.remove(jobNum)
			mark_cancelled(jobNum)

		if len(jobs) > 0:
			self._log_user.warning('There was a problem with cancelling the following jobs:')
			self._reportClass(self.jobDB, self._task, jobs).display()
			if (interactive and utils.getUserBool('Do you want to mark them as cancelled?', True)) or not interactive:
				lmap(mark_cancelled, jobs)
		if interactive:
			utils.wait(2)

	def _get_sandbox_file_list(self, task, sm_list):
		# Prepare all input files
		dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
		dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
			lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
		task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
			*imap(lambda x: x.get_task_dict(), [task] + sm_list))
		task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
			'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
		task_config_str_list = DictFormat(escape_strings=True).format(
			task_config_dict, format='export %s%s%s\n')
		vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
			self._remote_event_handler.get_mon_env_dict().keys()))
		vn_alias_dict.update(task.get_var_alias_map())
		vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')

		# Resolve wildcards in task input files
		def _get_task_fn_list():
			for fpi in task.get_sb_in_fpi_list():
				matched = glob.glob(fpi.path_abs)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield fpi.path_abs
		return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
			VirtualFile('_config.sh', sorted(task_config_str_list)),
			VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])

	def _cancel(self, task, wms, jobnum_list, interactive, show_jobs):
		if len(jobnum_list) == 0:
			return
		if show_jobs:
			self._abort_report.show_report(self.job_db, jobnum_list)
		if interactive and not self._uii.prompt_bool('Do you really want to cancel these jobs?', True):
			return

		def _mark_cancelled(jobnum):
			job_obj = self.job_db.get_job(jobnum)
			if job_obj is not None:
				self._update(task, job_obj, jobnum, Job.CANCELLED)
				self._local_event_handler.on_job_update(task, wms, job_obj, jobnum, {'reason': 'cancelled'})

		jobnum_list.reverse()
		map_gc_id2jobnum = self._get_map_gc_id_jobnum(jobnum_list)
		gc_id_list = sorted(map_gc_id2jobnum, key=lambda gc_id: -map_gc_id2jobnum[gc_id])
		for (gc_id,) in wms.cancel_jobs(gc_id_list):
			# Remove cancelledd job from todo list and mark as cancelled
			_mark_cancelled(map_gc_id2jobnum.pop(gc_id))

		if map_gc_id2jobnum:
			jobnum_list = list(map_gc_id2jobnum.values())
			self._log.warning('There was a problem with cancelling the following jobs:')
			self._abort_report.show_report(self.job_db, jobnum_list)
			if (not interactive) or self._uii.prompt_bool('Do you want to mark them as cancelled?', True):
				lmap(_mark_cancelled, jobnum_list)
		if interactive:
			wait(2)

def join_config_locations(opt_first, *opt_list):
	if isinstance(opt_first, (list, tuple)):  # first option is a list - expand the first parameter
		if not opt_list:  # only first option -> clean and return
			return lmap(str.strip, opt_first)
		return lchain(imap(lambda opt: join_config_locations(opt.strip(), *opt_list), opt_first))
	if not opt_list:  # only first option -> clean and return
		return [opt_first.strip()]
	return lmap(lambda opt: (opt_first + ' ' + opt).strip(), join_config_locations(*opt_list))

	def process(self, pNum, splitInfo, result):
		fl = splitInfo[DataSplitter.FileList]
		locations = splitInfo.get(DataSplitter.Locations)
		if not locations:
			splitInfo[DataSplitter.FileList] = lmap(lambda fn: self._lookup(fn, None), fl)
		else:
			for location in locations:
				splitInfo[DataSplitter.FileList] = lmap(lambda fn: self._lookup(fn, location), fl)

	def process(self, pnum, partition_info, result):
		url_list = partition_info[DataSplitter.FileList]
		locations = partition_info.get(DataSplitter.Locations)
		if not locations:
			partition_info[DataSplitter.FileList] = lmap(lambda url: self._lookup(url, None), url_list)
		else:
			for location in locations:
				partition_info[DataSplitter.FileList] = lmap(lambda url: self._lookup(url, location), url_list)

def wait(timeout):
	shortStep = lmap(lambda x: (x, 1), irange(max(timeout - 5, 0), timeout))
	for x, w in lmap(lambda x: (x, 5), irange(0, timeout - 5, 5)) + shortStep:
		if abort():
			return False
		log = ActivityLog('waiting for %d seconds' % (timeout - x))
		time.sleep(w)
		del log
	return True

	def __init__(self, arg, **kwargs):
		def parseTerm(term):
			cmpValue = utils.QM(term[0] == '~', False, True)
			term = term.lstrip('~')
			selectorType = utils.QM(term[0].isdigit(), 'id', 'state')
			if ':' in term:
				selectorType = term.split(':', 1)[0]
			selector = JobSelector.createInstance(selectorType, term.split(':', 1)[-1], **kwargs)
			return lambda jobNum, jobObj: selector.__call__(jobNum, jobObj) == cmpValue
		orTerms = str.join('+', imap(str.strip, arg.split('+'))).split()
		self.js = lmap(lambda orTerm: lmap(parseTerm, orTerm.split('+')), orTerms)

			def _getPartition(self, key):
				if not self._cacheKey == key / 100:
					self._cacheKey = key / 100
					subTarFileObj = self._tar.extractfile('%03dXX.tgz' % (key / 100))
					subTarFileObj = BytesBuffer(gzip.GzipFile(fileobj = subTarFileObj).read()) # 3-4x speedup for sequential access
					self._cacheTar = tarfile.open(mode = 'r', fileobj = subTarFileObj)
				data = self._fmt.parse(self._cacheTar.extractfile('%05d/info' % key).readlines(),
					keyParser = {None: int}, valueParser = self._parserMap)
				fileList = lmap(bytes2str, self._cacheTar.extractfile('%05d/list' % key).readlines())
				if DataSplitter.CommonPrefix in data:
					fileList = imap(lambda x: '%s/%s' % (data[DataSplitter.CommonPrefix], x), fileList)
				data[DataSplitter.FileList] = lmap(str.strip, fileList)
				return data

	def finaliseJobSplitting(self, block, splitInfo, files = None):
		# Copy infos from block
		for prop in ['Dataset', 'BlockName', 'Nickname', 'Locations']:
			if getattr(DataProvider, prop) in block:
				splitInfo[getattr(DataSplitter, prop)] = block[getattr(DataProvider, prop)]
		if DataProvider.Metadata in block:
			splitInfo[DataSplitter.MetadataHeader] = block[DataProvider.Metadata]
		# Helper for very simple splitter
		if files:
			splitInfo[DataSplitter.FileList] = lmap(lambda x: x[DataProvider.URL], files)
			splitInfo[DataSplitter.NEntries] = sum(imap(lambda x: x[DataProvider.NEntries], files))
			if DataProvider.Metadata in block:
				splitInfo[DataSplitter.Metadata] = lmap(lambda x: x[DataProvider.Metadata], files)
		return splitInfo

	def _finish_partition(self, block, partition, fi_list=None):
		# Copy infos from block
		for (dp_prop, ds_prop) in self._dp_ds_prop_list:
			if dp_prop in block:
				partition[ds_prop] = block[dp_prop]
		if DataProvider.Metadata in block:
			partition[DataSplitter.MetadataHeader] = block[DataProvider.Metadata]
		# Helper for very simple splitter
		if fi_list:
			partition[DataSplitter.FileList] = lmap(itemgetter(DataProvider.URL), fi_list)
			partition[DataSplitter.NEntries] = sum(imap(itemgetter(DataProvider.NEntries), fi_list))
			if DataProvider.Metadata in block:
				partition[DataSplitter.Metadata] = lmap(itemgetter(DataProvider.Metadata), fi_list)
		return partition

	def __init__(self, arg, **kwargs):
		def _parse_term(term):
			negate = (term[0] == '~')
			term = term.lstrip('~')
			selector_type = 'state'
			if term[0].isdigit():
				selector_type = 'id'
			elif ':' in term:
				selector_type = term.split(':', 1)[0]
			selector = JobSelector.create_instance(selector_type, term.split(':', 1)[-1], **kwargs)
			if negate:
				return lambda jobnum, job_obj: not selector.__call__(jobnum, job_obj)
			return selector
		(self._arg, or_term_list) = (arg, str.join('+', imap(str.strip, arg.split('+'))).split())
		self._js = lmap(lambda orTerm: lmap(_parse_term, orTerm.split('+')), or_term_list)

	def __init__(self, fn, format = 'sniffed'):
		sniffed = csv.Sniffer().sniff(open(fn).read(1024))
		csv.register_dialect('sniffed', sniffed)
		tmp = list(csv.DictReader(open(fn), dialect = format))

		def cleanupDict(d):
			# strip all key value entries
			tmp = tuple(imap(lambda item: imap(str.strip, item), d.items()))
			# filter empty parameters
			return lfilter(lambda k_v: k_v[0] != '', tmp)
		keys = []
		if len(tmp):
			keys = lmap(ParameterMetadata, tmp[0].keys())
		values = lmap(lambda d: dict(cleanupDict(d)), tmp)
		InternalParameterSource.__init__(self, values, keys)

def list_parameters(opts, psource):
	(result, needGCParam) = get_parameters(opts, psource)
	enabledOutput = opts.output.split(',')
	output = lfilter(lambda k: not opts.output or k in enabledOutput, psource.getJobKeys())
	stored = lfilter(lambda k: k.untracked == False, output)
	untracked = lfilter(lambda k: k.untracked == True, output)

	if opts.collapse > 0:
		result_old = result
		result = {}
		result_nicks = {}
		head = [('COLLATE_JOBS', '# of jobs')]
		if 'DATASETSPLIT' in stored:
			stored.remove('DATASETSPLIT')
			if opts.collapse == 1:
				stored.append('DATASETNICK')
				head.append(('DATASETNICK', 'DATASETNICK'))
			elif opts.collapse == 2:
				head.append(('COLLATE_NICK', '# of nicks'))
		for pset in result_old:
			if ('DATASETSPLIT' in pset) and (opts.collapse == 1):
				pset.pop('DATASETSPLIT')
			nickname = None
			if ('DATASETNICK' in pset) and (opts.collapse == 2):
				nickname = pset.pop('DATASETNICK')
			h = md5_hex(repr(lmap(pset.get, stored)))
			result.setdefault(h, []).append(pset)
			result_nicks.setdefault(h, set()).add(nickname)

		def doCollate(h):
			tmp = result[h][0]
			tmp['COLLATE_JOBS'] = len(result[h])
			tmp['COLLATE_NICK'] = len(result_nicks[h])
			return tmp
		result = lmap(doCollate, result)
	else:
		head = [('GC_JOB_ID', '#')]
		if needGCParam:
			head.append(('GC_PARAM', 'GC_PARAM'))
	if opts.active:
		head.append((ParameterInfo.ACTIVE, 'ACTIVE'))
	if opts.visible:
		stored = opts.visible.split(',')
	head.extend(sorted(izip(stored, stored)))
	if opts.untracked:
		head.extend(sorted(imap(lambda n: (n, '(%s)' % n), ifilter(lambda n: n not in ['GC_PARAM', 'GC_JOB_ID'], untracked))))
	utils.vprint('')
	utils.printTabular(head, result)

def partition_check(splitter):
		fail = utils.set()
		for jobNum in irange(splitter.getMaxJobs()):
			splitInfo = splitter.getSplitInfo(jobNum)
			try:
				(events, skip, files) = (0, 0, [])
				for line in open(os.path.join(opts.checkSplitting, 'jobs', 'job_%d.var' % jobNum)).readlines():
					if 'MAX_EVENTS' in line:
						events = int(line.split('MAX_EVENTS', 1)[1].replace('=', ''))
					if 'SKIP_EVENTS' in line:
						skip = int(line.split('SKIP_EVENTS', 1)[1].replace('=', ''))
					if 'FILE_NAMES' in line:
						files = line.split('FILE_NAMES', 1)[1].replace('=', '').replace('\"', '').replace('\\', '')
						files = lmap(lambda x: x.strip().strip(','), files.split())
				def printError(curJ, curS, msg):
					if curJ != curS:
						logging.warning('%s in job %d (j:%s != s:%s)', msg, jobNum, curJ, curS)
						fail.add(jobNum)
				printError(events, splitInfo[DataSplitter.NEntries], 'Inconsistent number of events')
				printError(skip, splitInfo[DataSplitter.Skipped], 'Inconsistent number of skipped events')
				printError(files, splitInfo[DataSplitter.FileList], 'Inconsistent list of files')
			except Exception:
				logging.warning('Job %d was never initialized!', jobNum)
		if fail:
			logging.warning('Failed: ' + str.join('\n', imap(str, fail)))

	def __init__(self, config, name):
		self._name = name # needed for changeView calls before the constructor
		head = [('DATASETNICK', 'Nickname')]

		# Mapping between nickname and config files:
		self._nmCfg = config.getLookup('nickname config', {}, defaultMatcher = 'regex',
			parser = lambda x: lmap(str.strip, x.split(',')), strfun = lambda x: str.join(',', x))
		if not self._nmCfg.empty():
			allConfigFiles = sorted(set(ichain(self._nmCfg.get_values())))
			config.set('config file', str.join('\n', allConfigFiles))
			head.append((1, 'Config file'))
		elif config.get('config file', ''):
			raise ConfigError("Please use 'nickname config' instead of 'config file'")

		# Mapping between nickname and constants - only display - work is handled by the 'normal' parameter factory
		nmCName = config.getList('nickname constants', [], onChange = None)
		param_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, addSections = ['parameters'])
		param_config.set('constants', str.join(' ', nmCName), '+=')
		for cName in nmCName:
			param_config.set(cName + ' matcher', 'regex')
			param_config.set(cName + ' lookup', 'DATASETNICK')
			head.append((cName, cName))

		# Mapping between nickname and lumi filter - only display - work is handled by the 'normal' lumi filter
		config.set('lumi filter matcher', 'regex')
		if 'nickname lumi filter' in config.getOptions():
			config.set('lumi filter', strDictLong(config.getDict('nickname lumi filter', {}, onChange = None)))
		self._nmLumi = config.getLookup('lumi filter', {}, parser = parseLumiFilter, strfun = strLumi, onChange = None)
		if not self._nmLumi.empty():
			head.append((2, 'Lumi filter'))

		CMSSW.__init__(self, config, name)
		self._displaySetup(config.getWorkPath('datacache.dat'), head)

	def _parseLine(self, exceptionIntro, configContent, configFile, idx, line):
		exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line)
		try:
			line = rsplit(line, ';', 1)[0].rstrip()
		except Exception:
			raise ConfigError(exceptionIntroLineInfo + '\nUnable to strip comments!')
		exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line) # removed comment
		if line.lstrip().startswith(';') or line.lstrip().startswith('#') or not line.strip():
			return # skip empty lines or comment lines
		elif line[0].isspace():
			try:
				self._currentValue += '\n' + line.strip()
				self._currentIndices += [idx]
			except Exception:
				raise ConfigError(exceptionIntroLineInfo + '\nInvalid indentation!')
		elif line.startswith('['):
			if self._currentOption:
				self._storeOption(exceptionIntroLineInfo, configContent, configFile)
			try:
				self._currentSection = line[1:line.index(']')].strip()
				self._parseLine(exceptionIntro, configContent, configFile,
					idx, line[line.index(']') + 1:].strip())
			except Exception:
				raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config section!')
		elif '=' in line:
			if self._currentOption:
				self._storeOption(exceptionIntroLineInfo, configContent, configFile)
			try:
				(self._currentOption, self._currentValue) = lmap(str.strip, line.split('=', 1))
				self._currentIndices = [idx]
			except Exception:
				raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config option!')
		else:
			raise ConfigError(exceptionIntroLineInfo + '\nPlease use "key = value" syntax or indent values!')