import com.chenlb.mmseg4j.Word; //导入依赖的package包/类
@Override
public final boolean incrementToken() throws IOException {
	clearAttributes();
	Word word = mmSeg.next();
	if(word != null) {
		//lucene 3.0
		//termAtt.setTermBuffer(word.getSen(), word.getWordOffset(), word.getLength());
		//lucene 3.1
		termAtt.copyBuffer(word.getSen(), word.getWordOffset(), word.getLength());
		offsetAtt.setOffset(word.getStartOffset(), word.getEndOffset());
		typeAtt.setType(word.getType());
		return true;
	} else {
		end();
		return false;
	}
}
 

import com.chenlb.mmseg4j.Word; //导入依赖的package包/类
public String segWords(Reader input, String wordSpilt) throws IOException {
	StringBuilder sb = new StringBuilder();
	Seg seg = getSeg();	//取得不同的分词具体算法
	MMSeg mmSeg = new MMSeg(input, seg);
	Word word = null;
	boolean first = true;
	while((word=mmSeg.next())!=null) {
		if(!first) {
			sb.append(wordSpilt);
		}
		String w = word.getString();
		sb.append(w);
		first = false;
		
	}
	return sb.toString();
}
 

import com.chenlb.mmseg4j.Word; //导入依赖的package包/类
private void addToken(Token oriToken, int termBufferOffset, int termBufferLength, byte type) {
	Token token = new Token(oriToken.buffer(), termBufferOffset, termBufferLength,
			oriToken.startOffset()+termBufferOffset, oriToken.startOffset()+termBufferOffset+termBufferLength);

	if(type == Character.DECIMAL_DIGIT_NUMBER) {
		token.setType(Word.TYPE_DIGIT);
	} else {
		token.setType(Word.TYPE_LETTER);
	}

	tokenQueue.offer(token);
}
 

import com.chenlb.mmseg4j.Word; //导入依赖的package包/类
@Override
public final boolean incrementToken() throws IOException {
	clearAttributes();
	Word word = mmSeg.next();
	if(word != null) {
		termAtt.copyBuffer(word.getSen(), word.getWordOffset(), word.getLength());
		offsetAtt.setOffset(word.getStartOffset(), word.getEndOffset());
		typeAtt.setType(word.getType());
		return true;
	} else {
		return false;
	}
}
 

import com.chenlb.mmseg4j.Word; //导入依赖的package包/类
private void addToken(PackedTokenAttributeImpl oriToken, int termBufferOffset, int termBufferLength, byte type) {
	PackedTokenAttributeImpl token = TokenUtils.subToken(oriToken, termBufferOffset, termBufferLength);

	if(type == Character.DECIMAL_DIGIT_NUMBER) {
		token.setType(Word.TYPE_DIGIT);
	} else {
		token.setType(Word.TYPE_LETTER);
	}

	tokenQueue.offer(token);
}
 

import com.chenlb.mmseg4j.Word; //导入依赖的package包/类
public static List<String> toMMsegWords(String txt, Seg seg) {
	List<String> words = new ArrayList<String>();
	MMSeg mmSeg = new MMSeg(new StringReader(txt), seg);
	Word word = null;
	try {
		while ((word = mmSeg.next()) != null) {
			String w = word.getString();
			words.add(w);
		}
	} catch (IOException e) {
		e.printStackTrace();
	}
	return words;
}