import cc.mallet.extract.Tokenization; //导入依赖的package包/类
public Instance pipe (Instance carrier)
{
  Object data = carrier.getData ();
  if (data instanceof Tokenization) {
    // we're done
  } else if (data instanceof TokenSequence) {
    StringBuffer buf = new StringBuffer ();
    TokenSequence ts = (TokenSequence) data;
    StringTokenization spans = new StringTokenization (buf);  // I can use a StringBuffer as the doc! Awesome!

    for (int i = 0; i < ts.size(); i++) {
      Token token = ts.get(i);

      int start = buf.length ();
      buf.append (token.getText());
      int end = buf.length();

      StringSpan span = new StringSpan (buf, start, end);
      span.setFeatures (token.getFeatures ());
      span.setProperties (token.getProperties ());

      spans.add (span);
      buf.append (" ");
    }

    carrier.setData (spans);
  } else {
    throw new IllegalArgumentException ("Can't convert "+data+" to Tokenization.");
  }

  return carrier;
}
 

import cc.mallet.extract.Tokenization; //导入依赖的package包/类
public Instance pipe (Instance carrier)
{
  Tokenization tok = (Tokenization) carrier.getData ();
  carrier.setProperty ("TOKENIZATION", tok);
  return carrier;
}