import org.apache.commons.lang3.text.StrMatcher; //导入依赖的package包/类
/** Split string x in tokens. Effectively just a friendly wrapper around StrTokenizer.
 * Use *single* quotes for avoiding splitting. 
 */
public static ArrayList<String> tokenize(String x, String delimiterString){
	
	if(x == null){
		return null;
	}
	
	// This is a hack to allow empty tokens to be passed at the command line. 
	// An empty 
	x= x.replace("''", "' '");
	
	// See also http://stackoverflow.com/questions/38161437/inconsistent-behaviour-of-strtokenizer-to-split-string
	StrTokenizer str= new StrTokenizer(x);
   	str.setTrimmerMatcher(StrMatcher.spaceMatcher());
	str.setDelimiterString(delimiterString);
	str.setQuoteChar('\'');
	// str.setIgnoreEmptyTokens(false);
	ArrayList<String> tokens= (ArrayList<String>) str.getTokenList();
	for(int i= 0; i < tokens.size(); i++){
		String tok= tokens.get(i).trim();
		tokens.set(i, tok);
	}
	return tokens;

}
 

import org.apache.commons.lang3.text.StrMatcher; //导入依赖的package包/类
static String[] _tokenizeString(String string) {
	final StrTokenizer _tokenizer = new StrTokenizer().
			setDelimiterMatcher(StrMatcher.trimMatcher()).
			setQuoteMatcher(StrMatcher.quoteMatcher()).
			setTrimmerMatcher(StrMatcher.trimMatcher()).
			setIgnoredMatcher(StrMatcher.quoteMatcher());
	_tokenizer.reset(string.toLowerCase());
	return _tokenizer.getTokenArray();
}
 

import org.apache.commons.lang3.text.StrMatcher; //导入依赖的package包/类
public String getFirstMatch(StrMatcher strMatcher) {
	int length = strMatcher.isMatch(buffer, currentPos);
	if (length > 0) {
		String result = substring(currentPos, currentPos + length);
		currentPos += length;
		return result;				
	}

	return null;
}
 

import org.apache.commons.lang3.text.StrMatcher; //导入依赖的package包/类
public static <V> StrSubstitutor newStrSubstitutor(String start, String end, char escape, final Map<String, V> valueMap){
	StrSubstitutor substitutor = new StrSubstitutor(StrLookup.mapLookup(valueMap), StrMatcher.stringMatcher(start), StrMatcher.stringMatcher(end), escape);
	return substitutor;
}
 

import org.apache.commons.lang3.text.StrMatcher; //导入依赖的package包/类
public StrMatcher getSeparator() {
    return separator;
}
 

import org.apache.commons.lang3.text.StrMatcher; //导入依赖的package包/类
public void setSeparator(StrMatcher separator) {
    this.separator = separator;
}
 

import org.apache.commons.lang3.text.StrMatcher; //导入依赖的package包/类
/**
 * Processes the specified CSV file. For every line but the header line (which is required), the
 * specified command is executed.
 * 
 * @param reader
 *            the reader for loading the CSV data
 * @param delimiter
 *            the column separator
 * @param quoteChar
 *            the quote character ('\0' for no quoting)
 * @param command
 *            the command (i. e. a Groovy closure if used in a Groovy script) to be executed for
 *            every processed line
 */
public void processFile(final Reader reader, final String delimiter, final char quoteChar, final Runnable command) {
	try {
		List<String> inputLines = CharStreams.readLines(reader);

		StrTokenizer st = StrTokenizer.getCSVInstance();
		st.setDelimiterString(delimiter);
		if (quoteChar != '\0') {
			st.setQuoteChar(quoteChar);
		} else {
			st.setQuoteMatcher(StrMatcher.noneMatcher());
		}

		// extract header
		String headerLine = inputLines.remove(0);
		List<Column> columns = initColumns(st, headerLine);
		for (String line : inputLines) {
			st.reset(line);
			String[] colArray = st.getTokenArray();
			int len = colArray.length;
			checkState(len == columns.size(), "Mismatch between number of header columns and number of line columns.");

			DataSource dataSource = dataSourceProvider.get();
			Configuration config = configProvider.get();
			for (int i = 0; i < len; ++i) {
				String value = StringUtils.trimToEmpty(colArray[i]);

				String dataSetKey = columns.get(i).dataSetKey;
				String key = columns.get(i).key;
				if (dataSetKey != null) {
					if ("<auto>".equals(value)) {
						dataSource.resetFixedValue(dataSetKey, key);
					} else {
						log.debug("Setting data set entry for " + this + " to value=" + value);
						dataSource.setFixedValue(dataSetKey, key, value);
					}
				} else {
					log.debug("Setting property for " + this + " to value=" + value);
					config.put(key, value);
				}
			}

			command.run();
		}
	} catch (IOException ex) {
		throw new JFunkException("Error processing CSV data", ex);
	}
}