import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
/** Attempts to translate a text using the word list. */
public String translate(Tokenizer tok, String text)
{
	StringBuilder res = new StringBuilder();
	
	for(Token t : tok.tokenize(text)) {
		if(Transliterator.hasAsianChar(t.getSurface())) {
			// attempt (partial) translation
			String s = t.getSurface();
			String r = StringUtils.replaceEach(s, srcEverywhere, trlEverywhere);
			r = replaceAnyOfAtStart(r, srcPrefix, trlPrefix);
			r = replaceAnyOfAtEnd(r, srcSuffix, trlSuffix);
			if(r != s && verbose > 1)
				System.out.println("translation list: translated " + s + " to " + r);
			res.append(r);				
		}
		else // tokens without asian chars can be ignored here and will be copied directly
			res.append(t.getSurface());
	}
	return res.toString();
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
private List<VocabularyDTO> asDTO(List<Token> tokens) {
    List<VocabularyDTO> result = new ArrayList<>();

    for (Token p : tokens) {
        //ei töötle lühikesi hiraganas lõike
        if (p.getBaseForm() != null && (JapaneseCharacter.isKanji(p.getBaseForm().charAt(0)) || p.getBaseForm().length() > 2)) {
            VocabularyDTO dto = new VocabularyDTO();
            dto.txt = p.getBaseForm();
            dto.rdng = toHiragana(p.getReading());
            List<EN_Essum> essums = jmDictEnDB.getEssumByKanjAndRdng(dto.txt, dto.rdng);
            List<String> glosses = essums.stream().map(EN_Essum::getGloss).collect(Collectors.toList());
            dto.gloss = Joiner.on("; ").join(glosses);
            result.add(dto);
        }
    }

    return result;
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
/**
 * Creates a default builder
 */
public Builder() {
    totalFeatures = DictionaryEntry.TOTAL_FEATURES;
    readingFeature = DictionaryEntry.READING_FEATURE;
    partOfSpeechFeature = DictionaryEntry.PART_OF_SPEECH_FEATURE;

    tokenFactory = new TokenFactory<Token>() {
        @Override
        public Token createToken(int wordId, String surface, ViterbiNode.Type type, int position,
                        Dictionary dictionary) {
            return new Token(wordId, surface, type, position, dictionary);
        }
    };
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
private boolean isOkPartsOfSpeech(Token token)
{
    if (!task.getOkPartsOfSpeech().isPresent()) {
        return true;
    }
    for (String okPartsOfSpeech : task.getOkPartsOfSpeech().get()) {
        if (token.getAllFeaturesArray()[0].equals(okPartsOfSpeech)) {
            return true;
        }
    }
    return false;
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
public static List<String> analyze(final String sentence) {
    Tokenizer search = getTokenizer();
    final List<Token> tokens = search.tokenize(sentence);
    final List<String> analiezed = Lists.newArrayList();
    for (final Token token : tokens) {
        final String word = token.getSurface();
        analiezed.add(word);
    }
    return analiezed;
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
public String transliterate(String jaName) throws IOException {
   	String trName = jaName; 
   	
   	trName = prenormalize(trName);
   	
	// Use Kuromoji for preprocessing of kanji and for tokenization
	StringBuilder sb = new StringBuilder();
	for(Token t : tokKanji.tokenize(trName)) {
		if(hasKanji(t.getSurface()) && !(t.getReading().equals("*"))) {
			if(sb.length() > 0)
				sb.append(' ');
			sb.append(t.getReading());
		}
		/*if(hasKanji(t.getSurface()) && !(t.getLemmaReadingForm().equals("*"))) {
			if(sb.length() > 0)
				sb.append(' ');
			sb.append(t.getLemmaReadingForm());
		}*/
		else {
			String surf = t.getSurface();
			if(sb.length() > 0 && surf.length() > 0 && !Character.isWhitespace(surf.charAt(0)))
				sb.append(' ');
			sb.append(surf);
		}
		/*sb.append(t.getReading());
		sb.append('[');
		sb.append(t.getSurface());
		sb.append(']');
		sb.append(' ');*/
	}
	trName = sb.toString();
	
	// Use Kakasi for rest of conversion
	trName = trHiragana.doString(trName);
	trName = trKatakana.doString(trName);
	
	// Add spaces between words (but not within abbreviations)
	trName = trName.replaceAll("([a-z0-9])([A-Z])", "$1 $2");
	trName = postnormalize(trName);
	
	/*// debugging option: marking remaining asian chars in the output.  
	StringBuilder sbOut = new StringBuilder();
	for(char c : trName.toCharArray()) {
		sbOut.append(c);
		if(c > 0x200) {
			sbOut.append(String.format("[\\u%04x]", (int)c));
		}
			
	} 

	return sbOut.toString(); */
	return trName; 
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
/**
 * @param builder
 */
private void setValue(PageBuilder builder)
{
    if (task.getKeepInput()) {
        for (Column inputColumn : inputSchema.getColumns()) {
            if (reader.isNull(inputColumn)) {
                builder.setNull(inputColumn);
                continue;
            }
            if (Types.STRING.equals(inputColumn.getType())) {
                builder.setString(inputColumn, reader.getString(inputColumn));
            }
            else if (Types.BOOLEAN.equals(inputColumn.getType())) {
                builder.setBoolean(inputColumn, reader.getBoolean(inputColumn));
            }
            else if (Types.DOUBLE.equals(inputColumn.getType())) {
                builder.setDouble(inputColumn, reader.getDouble(inputColumn));
            }
            else if (Types.LONG.equals(inputColumn.getType())) {
                builder.setLong(inputColumn, reader.getLong(inputColumn));
            }
            else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
                builder.setTimestamp(inputColumn, reader.getTimestamp(inputColumn));
            }
            else if (Types.JSON.equals(inputColumn.getType())) {
                builder.setJson(inputColumn, reader.getJson(inputColumn));
            }
        }
    }

    for (Column column : keyNameColumns) {
        final String source = reader.getString(column);
        List<Token> tokens = tokenizer.tokenize(source);
        logger.debug("{} => {}", source, tokens);
        for (Map<String, String> setting : task.getSettings()) {
            String suffix = setting.get("suffix");
            String method = setting.get("method");
            Column outputColumn = outputSchema.lookupColumn(column.getName() + MoreObjects.firstNonNull(suffix, ""));
            List<Value> outputs = Lists.newArrayList();
            for (Token token : tokens) {
                logger.debug("token => {}, {}", token, token.getAllFeatures());
                if (!isOkPartsOfSpeech(token)) {
                    continue;
                }
                String word = null;
                if ("base_form".equals(method)) {
                    word = MoreObjects.firstNonNull(token.getBaseForm(), token.getSurface());
                }
                else if ("reading".equals(method)) {
                    word = MoreObjects.firstNonNull(token.getReading(), token.getSurface());
                }
                else if ("surface_form".equals(method)) {
                    word = token.getSurface();
                }
                outputs.add(ValueFactory.newString(word));
            }
            if (outputColumn.getType().equals(Types.STRING)) {
                Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
                builder.setString(outputColumn, joiner.join(outputs));
            }
            else if (outputColumn.getType().equals(Types.JSON)) {
                builder.setJson(outputColumn, ValueFactory.newArray(outputs));
            }
        }
    }
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
private List<Token> tokenize(String sentence) {
    List<Token> result = tokenizer().tokenize(sentence);
    log.debug("tokenize: result.size=" + result.size() + ", sentence=" + sentence);
    return result;
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
private String getToken(int i) {
    Token t = tokens.get(i);
    String ret = (useBaseForm) ? t.getBaseForm() : t.getSurface();
    return (preProcessor == null) ? ret : preProcessor.preProcess(ret);
}
 

import com.atilika.kuromoji.ipadic.Token; //导入依赖的package包/类
/**
 * Tokenizes the provided text and returns a list of tokens with various feature information
 * <p>
 * This method is thread safe
 *
 * @param text  text to tokenize
 * @return list of Token, not null
 */
@Override
public List<Token> tokenize(String text) {
    return createTokenList(text);
}