import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public Tokenizer create(AttributeFactory factory, Reader input) {
	MMSegTokenizer tokenizer = tokenizerLocal.get();
	if(tokenizer == null) {
		tokenizer = newTokenizer(input);
	} else {
		try {
			tokenizer.setReader(input);
		} catch (IOException e) {
			tokenizer = newTokenizer(input);
			log.info("MMSegTokenizer.reset i/o error by:"+e.getMessage());
		}
	}

	return tokenizer;
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
static Object[] newTokenizerArgs(Random random, Reader reader, Class<?>[] paramTypes) {
  Object[] args = new Object[paramTypes.length];
  for (int i = 0; i < args.length; i++) {
    Class<?> paramType = paramTypes[i];
    if (paramType == Reader.class) {
      args[i] = reader;
    } else if (paramType == AttributeFactory.class) {
      // TODO: maybe the collator one...???
      args[i] = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
    } else if (paramType == AttributeSource.class) {
      // TODO: args[i] = new AttributeSource();
      // this is currently too scary to deal with!
      args[i] = null; // force IAE
    } else {
      args[i] = newRandomArg(random, paramType);
    }
  }
  return args;
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public Tokenizer create(AttributeFactory factory, Reader input) {
  if (luceneMatchVersion.onOrAfter(Version.LUCENE_44)) {
    if (!EdgeNGramTokenFilter.Side.FRONT.getLabel().equals(side)) {
      throw new IllegalArgumentException(EdgeNGramTokenizer.class.getSimpleName() + " does not support backward n-grams as of Lucene 4.4");
    }
    return new EdgeNGramTokenizer(luceneMatchVersion, input, minGramSize, maxGramSize);
  } else {
    return new Lucene43EdgeNGramTokenizer(luceneMatchVersion, input, side, minGramSize, maxGramSize);
  }
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
/** Creates the {@link TokenStream} of n-grams from the given {@link Reader} and {@link AttributeFactory}. */
@Override
public Tokenizer create(AttributeFactory factory, Reader input) {
  if (luceneMatchVersion.onOrAfter(Version.LUCENE_44)) {
    return new NGramTokenizer(luceneMatchVersion, factory, input, minGramSize, maxGramSize);
  } else {
    return new Lucene43NGramTokenizer(factory, input, minGramSize, maxGramSize);
  }
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public Tokenizer create(AttributeFactory factory, Reader input) {
  if (reverse) {
    return new ReversePathHierarchyTokenizer(factory, input, delimiter, replacement, skip);
  }
  return new PathHierarchyTokenizer(factory, input, delimiter, replacement, skip);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
public Tokenizer create(AttributeFactory arg0, Reader input)
{
  Tokenizer _IKTokenizer = new IKTokenizer(input, this.useSmart);
  return _IKTokenizer;
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public CJKTokenizer create(AttributeFactory factory, Reader in) {
  return new CJKTokenizer(factory, in);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public ArabicLetterTokenizer create(AttributeFactory factory, Reader input) {
  return new ArabicLetterTokenizer(luceneMatchVersion, factory, input);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
/**
 * Split the input using configured pattern
 */
@Override
public PatternTokenizer create(final AttributeFactory factory, final Reader in) {
  return new PatternTokenizer(factory, in, pattern, group);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public RussianLetterTokenizer create(AttributeFactory factory, Reader in) {
  return new RussianLetterTokenizer(luceneMatchVersion, factory, in);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public WikipediaTokenizer create(AttributeFactory factory, Reader input) {
  return new WikipediaTokenizer(factory, input, WikipediaTokenizer.TOKENS_ONLY, 
      Collections.<String>emptySet());
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public StandardTokenizer create(AttributeFactory factory, Reader input) {
  StandardTokenizer tokenizer = new StandardTokenizer(luceneMatchVersion, factory, input); 
  tokenizer.setMaxTokenLength(maxTokenLength);
  return tokenizer;
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public ClassicTokenizer create(AttributeFactory factory, Reader input) {
  ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, factory, input); 
  tokenizer.setMaxTokenLength(maxTokenLength);
  return tokenizer;
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public UAX29URLEmailTokenizer create(AttributeFactory factory, Reader input) {
  UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, factory, input); 
  tokenizer.setMaxTokenLength(maxTokenLength);
  return tokenizer;
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public ICUTokenizer create(AttributeFactory factory, Reader input) {
  assert config != null : "inform must be called first!";
  return new ICUTokenizer(factory, input, config);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public ChineseTokenizer create(AttributeFactory factory, Reader in) {
  return new ChineseTokenizer(factory, in);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public SentenceTokenizer create(AttributeFactory factory, Reader input) {
  return new SentenceTokenizer(factory, input);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public UIMAAnnotationsTokenizer create(AttributeFactory factory, Reader input) {
  return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters, factory, input);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
@Override
public UIMATypeAwareAnnotationsTokenizer create(AttributeFactory factory, Reader input) {
  return new UIMATypeAwareAnnotationsTokenizer
      (descriptorPath, tokenType, featurePath, configurationParameters, factory, input);
}
 

import org.apache.lucene.util.AttributeSource.AttributeFactory; //导入依赖的package包/类
/** Creates a TokenStream of the specified input using the default attribute factory. */
public final Tokenizer create(Reader input) {
  return create(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input);
}