import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
private static CRF4 train(List<Sentence> sentences, int order, boolean useFeatureInduction, TagFormat format, Pipe pipe, boolean reverse)
{
    InstanceList instances = new InstanceList(pipe);
    for (Sentence sentence : sentences)
    {
        String text = sentence.getTrainingText(format, reverse);
        instances.add(new Instance(text, null, sentence.getTag(), null, pipe));
    }
    CRF4 crf = new CRF4(pipe, null);
    if (order == 1)
        crf.addStatesForLabelsConnectedAsIn(instances);
    else if (order == 2)
        crf.addStatesForBiLabelsConnectedAsIn(instances);
    else
        throw new IllegalArgumentException("Order must be equal to 1 or 2");
    if (useFeatureInduction)
        crf.trainWithFeatureInduction(instances, null, null, null, 99999, 100, 10, 1000, 0.5, false, new double[] {.2, .5, .8});
    else
        crf.train(instances, null, null, (MultiSegmentationEvaluator)null, 99999, 10, new double[] {.2, .5, .8});
    return crf;
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
/** Creates a PagedInstanceList where "instancesPerPage" instances
 * are swapped to disk in directory "swapDir" if the amount of free
 * system memory drops below "minFreeMemory" bytes
 * @param pipe instance pipe
 * @param instancesPerPage number of Instances to store in each
 * page. If -1, determine at first call to
 * <code>swapOutExcept</code>
 * @param swapDir where the pages on disk live.
 */
public PagedInstanceList (Pipe pipe, int size, int instancesPerPage, File swapDir)
{
	super (pipe, size);
	inMemory = new BitSet();
	pageNotInMemory = new BitSet();
	this.instancesPerPage = instancesPerPage;
	this.swapDir = swapDir;
	try {
		if (!swapDir.exists()) {
			swapDir.mkdir();
		}
	} catch (SecurityException e) {
		System.err.println ("No permission to make directory " + swapDir);
		System.exit(-1);
	}
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
/**
  * Creates a list consisting of randomly-generated
  * <code>FeatureVector</code>s.
  */
// xxx Perhaps split these out into a utility class
public InstanceList (Random r,
										 // the generator of all random-ness used here
										 Dirichlet classCentroidDistribution,
										 // includes a Alphabet
										 double classCentroidAverageAlphaMean,
										 // Gaussian mean on the sum of alphas
										 double classCentroidAverageAlphaVariance,
										 // Gaussian variance on the sum of alphas
										 double featureVectorSizePoissonLambda,
										 double classInstanceCountPoissonLambda,
										 String[] classNames)
{
	this (new SerialPipes (new Pipe[]	{
		new TokenSequence2FeatureSequence (),
		new FeatureSequence2FeatureVector (),
		new Target2Label()}));
	//classCentroidDistribution.print();
	PipeInputIterator iter = new RandomTokenSequenceIterator (
		r, classCentroidDistribution,
		classCentroidAverageAlphaMean, classCentroidAverageAlphaVariance,
		featureVectorSizePoissonLambda, classInstanceCountPoissonLambda,
		classNames);
	this.add (iter);
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public void pipeOutputAccumulate (Instance carrier,	Pipe iteratedPipe)
{
	// xxx ??? assert (iteratedPipe == pipe);
	// The assertion above won't be true when using IteratedPipe...
	//logger.fine ("pipeOutputAccumulate target="+target);
	// These various add() methods below will make sure that the Pipes match appropriately
	if (carrier.getData() instanceof InstanceList)
		add ((InstanceList)carrier.getData());
	else if (carrier.getData() instanceof PipeInputIterator)
		add ((PipeInputIterator)carrier.getData());
	else if (carrier.getData() instanceof Instance)
		add ((Instance)carrier.getData());
	else {
		if (pipe == notYetSetPipe)
			pipe = iteratedPipe;
		//System.out.println ("Instance.pipeOuputAccumulate carrier.getSource()="+carrier.getSource());
     // Carrier has already been piped; make sure not to repipe it.
		add (carrier);
	}
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public Instance pipe(Instance carrier, int startingIndex)
{
	// System.err.println(pipes.size());
	for (int i = startingIndex; i < pipes.size(); i++)
	{
		// System.err.println("Pipe: " + i);
		Pipe p = (Pipe) pipes.get(i);
		if (p == null)
		{
			System.err.println("Pipe is null");
		} else
		{
			carrier = p.pipe(carrier);
		}
	}
	return carrier;
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
protected Alphabet resolveDataAlphabet ()
{
	if (dataAlphabetResolved)
		throw new IllegalStateException ("Alphabet already resolved.");
	// Set this before we ask our children anything so they can use it
	if (parent != null)
		dataDict = parent.dataDict;
	// Start at the first of the parallel pipelines to allow first-added Pipe's
	// to create a dictionary first.
	// xxx The below implies that either all feature dictionaries
	// must be null, or none of them.  Is this right?
	for (int i = 0; i < pipes.size(); i++) {
		Alphabet fd = ((Pipe)pipes.get(i)).resolveDataAlphabet ();
		if (dataDict == null)
			dataDict = fd;
		else if (! dataDict.equals (fd))
			throw new IllegalArgumentException
				("ParallelPipes pipe " + pipes.get(i).getClass().getName()
				 + "does not have same output Alphabet as previous pipes.");
	}
	dataAlphabetResolved = true;
	return dataDict;
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
protected Alphabet resolveTargetAlphabet ()
{
	if (targetAlphabetResolved)
		throw new IllegalStateException ("Target Alphabet already resolved.");
	// Set this before we ask our children anything so they can use it
	if (parent != null)
		targetDict = parent.targetDict;
	// Start at the first of the parallel pipelines to allow first-added Pipe's
	// to create a dictionary first.
	// xxx The below implies that either all feature dictionaries
	// must be null, or none of them.  Is this right?
	for (int i = 0; i < pipes.size(); i++) {
		Alphabet ld = ((Pipe)pipes.get(i)).resolveTargetAlphabet ();
		if (targetDict == null)
			targetDict = ld;
		else if (! targetDict.equals (ld))
			throw new IllegalArgumentException
				("ParallelPipes pipe " + pipes.get(i).getClass().getName()
				 + "does not have same target Alphabet as previous pipes.");
	}
	targetAlphabetResolved = true;
	return targetDict;
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public void testParenGroupIterator ()
{
	String input = "(a (b c) ((d))  ) f\n\n (3\n 4) (  6) ";
	Reader reader = new StringReader (input);
	ParenGroupIterator it = new ParenGroupIterator (reader);
	Pipe pipe = new Noop();
	pipe.setTargetProcessing (false);

	InstanceList lst = new InstanceList (pipe);
	lst.add (it);

	assertEquals (3, lst.size());
	assertEquals ("(a (b c) ((d))  )", lst.getInstance(0).getData());
	assertEquals ("(3\n 4)", lst.getInstance(1).getData());
	assertEquals ("(  6)", lst.getInstance(2).getData());
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
static CRFExtractor hackCrfExtor (CRF4 crf)
{
  Pipe[] newPipes = new Pipe [3];

  SerialPipes pipes = (SerialPipes) crf.getInputPipe ();
  for (int i = 0; i < 3; i++) {
    Pipe p0 = pipes.getPipe (0);
    pipes.removePipe (0);
    p0.setParent (null);
    newPipes[i] = p0;
  }

  Pipe tokPipe = new SerialPipes (newPipes);

  CRFExtractor extor = new CRFExtractor (crf, tokPipe);
  return extor;
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public void testSpaceViewer () throws IOException
{
  Pipe pipe = TestMEMM.makeSpacePredictionPipe ();
  String[] data0 = { TestCRF.data[0] };
  String[] data1 = { TestCRF.data[1] };

  InstanceList training = new InstanceList (pipe);
  training.add (new ArrayIterator (data0));
  InstanceList testing = new InstanceList (pipe);
  testing.add (new ArrayIterator (data1));

  CRF4 crf = new CRF4 (pipe, null);
  crf.addFullyConnectedStatesForLabels ();
  crf.train (training, null, null, null);

  CRFExtractor extor = TestLatticeViewer.hackCrfExtor (crf);
  Extraction extraction = extor.extract (new ArrayIterator (data1));
  DocumentViewer.writeExtraction (new File (outputDir), extraction);
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
  int version = in.readInt();
  if (version != CURRENT_SERIAL_VERSION)
    throw new ClassNotFoundException("Mismatched NaiveBayesTrainer versions: wanted " +
                                     CURRENT_SERIAL_VERSION + ", got " +
                                     version);

    //default selections for the kind of Estimator used
    featureEstimator = (Multinomial.Estimator) in.readObject();
    priorEstimator = (Multinomial.Estimator) in.readObject();

    // These are the counts formed after NaiveBayes training.
    me = (Multinomial.Estimator []) in.readObject();
    pe = (Multinomial.Estimator) in.readObject();

    // pipe and alphabets
    instancePipe = (Pipe) in.readObject();
    dataAlphabet = (Alphabet) in.readObject();
    targetAlphabet = (Alphabet) in.readObject();
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
/** Construct a KMeans object
    * 
    * @param instancePipe Pipe for the instances being clustered
    * @param numClusters  Number of clusters to use
    * @param metric       Metric object to measure instance distances
    * @param emptyAction  Specify what should happen when an empty cluster 
    *                     occurs
    */
   public KMeans( Pipe instancePipe, int numClusters, Metric metric, 
	   int emptyAction ) {

super(instancePipe);

if (emptyAction == EMPTY_SINGLE)
    throw new UnsupportedOperationException("EMPTY_SINGLE not yet implemented");

this.emptyAction = emptyAction;
this.metric = metric;
this.numClusters = numClusters;

this.clusterMeans = new ArrayList(numClusters);
this.randinator = new Random();

   }
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
private static void setupPipes(ArrayList<Pipe> pipes, String regexFilename)
{
	try
	{
		if (regexFilename != null)
			pipes.add(new ConfigurableRegexMatches(regexFilename));
	} catch (IOException e)
	{
		throw new RuntimeException(e);
	}
	pipes.add(new TokenTextCharPrefix("2PREFIX=", 2));
	pipes.add(new TokenTextCharPrefix("3PREFIX=", 3));
	pipes.add(new TokenTextCharPrefix("4PREFIX=", 4));
	pipes.add(new TokenTextCharSuffix("2SUFFIX=", 2));
	pipes.add(new TokenTextCharSuffix("3SUFFIX=", 3));
	pipes.add(new TokenTextCharSuffix("4SUFFIX=", 4));
	pipes.add(new TokenTextCharNGrams("CHARNGRAM=", new int[] { 2, 3 }, true));
	// pipes.add(new LexiconMembership()); // Use this for determining
	// whether word in a lexicon
	pipes.add(new RegexMatches("ROMAN", Pattern.compile("[IVXDLCM]+", Pattern.CASE_INSENSITIVE)));
	pipes.add(new RegexMatches("GREEK", Pattern.compile(GREEK, Pattern.CASE_INSENSITIVE)));
	pipes.add(new RegexMatches("ISPUNCT", Pattern.compile("[`[email protected]#$%^&*()-=_+\\[\\]\\\\{}|;\':\\\",./<>?]+")));
	pipes.add(new OffsetConjunctions(new int[][] { { -2 }, { -1 }, { 1 }, { 2 } }));
	pipes.add(new TokenSequence2FeatureVectorSequence(true, true));
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public CRF (Pipe inputPipe, Pipe outputPipe)
{
	this.inputPipe = inputPipe;
	this.outputPipe = outputPipe;
	this.inputAlphabet = inputPipe.getDataAlphabet();
	this.outputAlphabet = inputPipe.getTargetAlphabet();
	this.defaultFeatureIndex = inputAlphabet.size();
	//inputAlphabet.stopGrowth();
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public CRFByGISUpdate (Pipe inputPipe, Pipe outputPipe)
{
	this.inputPipe = inputPipe;
	this.outputPipe = outputPipe;
	this.inputAlphabet = inputPipe.getDataAlphabet();
	this.outputAlphabet = inputPipe.getTargetAlphabet();
	//this.defaultFeatureIndex = inputAlphabet.size();
	//inputAlphabet.stopGrowth();
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public HMM (Pipe inputPipe, Pipe outputPipe)
{
	this.inputPipe = inputPipe;
	this.outputPipe = outputPipe;
	this.inputAlphabet = inputPipe.getDataAlphabet();
	this.outputAlphabet = inputPipe.getTargetAlphabet();
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public CRF3 (Pipe inputPipe, Pipe outputPipe)
{
	this.inputPipe = inputPipe;
	this.outputPipe = outputPipe;
	this.inputAlphabet = inputPipe.getDataAlphabet();
	this.outputAlphabet = inputPipe.getTargetAlphabet();
	//this.defaultFeatureIndex = inputAlphabet.size();
	//inputAlphabet.stopGrowth();
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
public CRF2 (Pipe inputPipe, Pipe outputPipe)
{
	this.inputPipe = inputPipe;
	this.outputPipe = outputPipe;
	this.inputAlphabet = inputPipe.getDataAlphabet();
	this.outputAlphabet = inputPipe.getTargetAlphabet();
	//this.defaultFeatureIndex = inputAlphabet.size();
	//inputAlphabet.stopGrowth();
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
private void readObject (ObjectInputStream in) throws IOException, ClassNotFoundException {
	int i, size;
	int version = in.readInt ();
	id = (VMID) in.readObject ();
	instances = (ArrayList) in.readObject();
	instanceWeights = (DoubleList) in.readObject();
	pipe = (Pipe) in.readObject();
	// memory attributes
	instancesPerPage = in.readInt ();
	swapDir = (File) in.readObject ();
	inMemory = (BitSet) in.readObject ();
	pageNotInMemory = (BitSet) in.readObject ();
	collectGarbage = in.readBoolean ();
}
 

import edu.umass.cs.mallet.base.pipe.Pipe; //导入依赖的package包/类
/** Initialize the slots with the given four values, then put the
		Instance through the given pipe, then lock the instance. */
public Instance (Object data, Object target, Object name, Object source,
								 Pipe p)
{
	this (data, target, name, source);
	if (p != null) {
		p.pipe (this);
		locked = true;
	}
	pipe = p;
}