import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
protected String lemmatize(String query) {
	ItalianAnalyzer analyzer = new ItalianAnalyzer();
	TokenStream tokenStream = analyzer.tokenStream("label", query);
	
	
	StringBuilder sb = new StringBuilder();
	CharTermAttribute token = tokenStream.getAttribute(CharTermAttribute.class);
	try {
    	tokenStream.reset();
		while (tokenStream.incrementToken()) {
		    if (sb.length() > 0) {
		        sb.append(" ");
		    }
		    sb.append(token.toString());
		}
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	
	return sb.toString();
}
 

import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
/**
 * Initialize the classifiers. This static method initializes the
 * italian and the english classifiers under the hood. You must
 * call this function after you have constructed an instance of the
 * TMFVariables class as described in TMFVariables docs.
 *
 * If you don't call this method, when you use the classifier you
 * will get a NullPointerException in Classifier().
 *
 * @since 2.0.0.0.
 */
public static void init() {
	unchecked(() -> {
		LOG.debug("[initializator] - BEGIN");

		// build italian searcher
		Directory contextIndexDirIT = LuceneManager.pickDirectory(new File(TMFVariables.CORPUS_INDEX_IT));
		LOG.info("Corpus index used for italian: " + contextIndexDirIT);
		LuceneManager contextLuceneManagerIT = new LuceneManager(contextIndexDirIT);
		contextLuceneManagerIT.setLuceneDefaultAnalyzer(new ItalianAnalyzer(Version.LUCENE_36, TMFVariables.STOPWORDS_IT));
		ITALIAN_CORPUS_INDEX_SEARCHER = new SimpleSearcher(contextLuceneManagerIT);

		// build english searcher
		Directory contextIndexDirEN = LuceneManager.pickDirectory(new File(TMFVariables.CORPUS_INDEX_EN));
		LOG.info("Corpus index used for english: " + contextIndexDirEN);
		LuceneManager contextLuceneManagerEN = new LuceneManager(contextIndexDirEN);
		contextLuceneManagerEN.setLuceneDefaultAnalyzer(new EnglishAnalyzer(Version.LUCENE_36, TMFVariables.STOPWORDS_EN));
		ENGLISH_CORPUS_INDEX_SEARCHER = new SimpleSearcher(contextLuceneManagerEN);

		LOG.debug("[initializator] - END");
	});
}
 

import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
public ItalianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
    super(indexSettings, name, settings);
    analyzer = new ItalianAnalyzer(
        Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, ItalianAnalyzer.getDefaultStopSet()),
        Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
    );
    analyzer.setVersion(version);
}
 

import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
@Inject
public ItalianAnalyzerProvider(Index index, IndexSettingsService indexSettingsService, Environment env, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);
    analyzer = new ItalianAnalyzer(Analysis.parseStopWords(env, settings, ItalianAnalyzer.getDefaultStopSet()),
                                   Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
    analyzer.setVersion(version);
}
 

import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
public static String lemmatize(String query) {
   	StringBuilder sb = new StringBuilder();
   	
	ItalianAnalyzer analyzer = new ItalianAnalyzer(Version.LUCENE_44);
	TokenStream tokenStream;
	try {
		tokenStream = analyzer.tokenStream("label", query);
		
		CharTermAttribute token = tokenStream.getAttribute(CharTermAttribute.class);
		
    	tokenStream.reset();
		while (tokenStream.incrementToken()) {
		    if (sb.length() > 0) {
		        sb.append(" ");
		    }
		    sb.append(token.toString());
		}
		
		analyzer.close();
	} catch (IOException e) {
		log.error(e.getMessage(), e);
		sb = new StringBuilder();
		sb.append(query);
	}
	
	
	return sb.toString();
}
 

import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
@Override
public ItalianAnalyzer get() {
    return this.analyzer;
}
 

import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
/**
 * Returns the default stopwords set used by Lucene language analyzer for the specified language.
 *
 * @param language The language for which the stopwords are. The supported languages are English, French, Spanish,
 *                 Portuguese, Italian, Romanian, German, Dutch, Swedish, Norwegian, Danish, Russian, Finnish,
 *                 Irish, Hungarian, Turkish, Armenian, Basque and Catalan.
 * @return The default stopwords set used by Lucene language analyzers.
 */
private static CharArraySet getDefaultStopwords(String language) {
    switch (language) {
        case "English":
            return EnglishAnalyzer.getDefaultStopSet();
        case "French":
            return FrenchAnalyzer.getDefaultStopSet();
        case "Spanish":
            return SpanishAnalyzer.getDefaultStopSet();
        case "Portuguese":
            return PortugueseAnalyzer.getDefaultStopSet();
        case "Italian":
            return ItalianAnalyzer.getDefaultStopSet();
        case "Romanian":
            return RomanianAnalyzer.getDefaultStopSet();
        case "German":
            return GermanAnalyzer.getDefaultStopSet();
        case "Dutch":
            return DutchAnalyzer.getDefaultStopSet();
        case "Swedish":
            return SwedishAnalyzer.getDefaultStopSet();
        case "Norwegian":
            return NorwegianAnalyzer.getDefaultStopSet();
        case "Danish":
            return DanishAnalyzer.getDefaultStopSet();
        case "Russian":
            return RussianAnalyzer.getDefaultStopSet();
        case "Finnish":
            return FinnishAnalyzer.getDefaultStopSet();
        case "Irish":
            return IrishAnalyzer.getDefaultStopSet();
        case "Hungarian":
            return HungarianAnalyzer.getDefaultStopSet();
        case "Turkish":
            return SpanishAnalyzer.getDefaultStopSet();
        case "Armenian":
            return SpanishAnalyzer.getDefaultStopSet();
        case "Basque":
            return BasqueAnalyzer.getDefaultStopSet();
        case "Catalan":
            return CatalanAnalyzer.getDefaultStopSet();
        default:
            return CharArraySet.EMPTY_SET;
    }
}
 

import org.apache.lucene.analysis.it.ItalianAnalyzer; //导入依赖的package包/类
public Tokenizer() {
	
	Analyzer en_analyzer = new EnglishAnalyzer(Version.LUCENE_44);
	Analyzer es_analyzer = new SpanishAnalyzer(Version.LUCENE_44);
	Analyzer de_analyzer = new GermanAnalyzer(Version.LUCENE_44);
	Analyzer da_analyzer = new DanishAnalyzer(Version.LUCENE_44);
	Analyzer el_analyzer = new GreekAnalyzer(Version.LUCENE_44);
	Analyzer fr_analyzer = new FrenchAnalyzer(Version.LUCENE_44);
	Analyzer it_analyzer = new ItalianAnalyzer(Version.LUCENE_44);
	Analyzer pt_analyzer = new PortugueseAnalyzer(Version.LUCENE_44);
	Analyzer ru_analyzer = new RussianAnalyzer(Version.LUCENE_44);
	Analyzer fa_analyzer = new PersianAnalyzer(Version.LUCENE_44);	
	Analyzer ar_analyzer = new ArabicAnalyzer(Version.LUCENE_44);
	Analyzer id_analyzer = new IndonesianAnalyzer(Version.LUCENE_44);
	Analyzer pl_analyzer = new MorfologikAnalyzer(Version.LUCENE_44);
	Analyzer nl_analyzer = new DutchAnalyzer(Version.LUCENE_44);
	Analyzer no_analyzer = new NorwegianAnalyzer(Version.LUCENE_44);
	Analyzer ro_analyzer = new RomanianAnalyzer(Version.LUCENE_44);
	Analyzer sv_analyzer = new SwedishAnalyzer(Version.LUCENE_44);
	Analyzer fi_analyzer = new FinnishAnalyzer(Version.LUCENE_44);
	Analyzer tr_analyzer = new TurkishAnalyzer(Version.LUCENE_44);
	Analyzer hu_analyzer = new HungarianAnalyzer(Version.LUCENE_44);
	Analyzer bg_analyzer = new BulgarianAnalyzer(Version.LUCENE_44);
	
	analyzers.put("en", en_analyzer);
	analyzers.put("es", es_analyzer);
	analyzers.put("de", de_analyzer);
	analyzers.put("da", da_analyzer);
	analyzers.put("el", el_analyzer);
	analyzers.put("fr", fr_analyzer);
	analyzers.put("it", it_analyzer);
	analyzers.put("pt", pt_analyzer);
	analyzers.put("ru", ru_analyzer);
	analyzers.put("fa", fa_analyzer);
	analyzers.put("ar", ar_analyzer);
	analyzers.put("id", id_analyzer);
	analyzers.put("pl", pl_analyzer);
	analyzers.put("nl", nl_analyzer);
	analyzers.put("no", no_analyzer);
	analyzers.put("ro", ro_analyzer);
	analyzers.put("sv", sv_analyzer);
	analyzers.put("fi", fi_analyzer);
	analyzers.put("tr", tr_analyzer);
	analyzers.put("hu", hu_analyzer);
	analyzers.put("bg", bg_analyzer);
}