import edu.stanford.nlp.stats.Distribution; //导入依赖的package包/类
private Distribution<Integer> getSegmentedWordLengthDistribution(Treebank tb) {
  // CharacterLevelTagExtender ext = new CharacterLevelTagExtender();
  ClassicCounter<Integer> c = new ClassicCounter<Integer>();
  for (Iterator iterator = tb.iterator(); iterator.hasNext();) {
    Tree gold = (Tree) iterator.next();
    StringBuilder goldChars = new StringBuilder();
    Sentence goldYield = gold.yield();
    for (Iterator wordIter = goldYield.iterator(); wordIter.hasNext();) {
      Word word = (Word) wordIter.next();
      goldChars.append(word);
    }
    Sentence ourWords = segmentWords(goldChars.toString());
    for (int i = 0; i < ourWords.size(); i++) {
      c.incrementCount(Integer.valueOf(ourWords.get(i).toString().length()));
    }
  }
  return Distribution.getDistribution(c);
}
 

import edu.stanford.nlp.stats.Distribution; //导入依赖的package包/类
@Override
public void finishTraining() {
  lex.finishTraining();

  int numTags = tagIndex.size();
  POSes = new HashSet<String>(tagIndex.objectsList());
  initialPOSDist = Distribution.laplaceSmoothedDistribution(initial, numTags, 0.5);
  markovPOSDists = new HashMap<String, Distribution>();
  Set entries = ruleCounter.lowestLevelCounterEntrySet();
  for (Iterator iter = entries.iterator(); iter.hasNext();) {
    Map.Entry entry = (Map.Entry) iter.next();
    //      Map.Entry<List<String>, Counter> entry = (Map.Entry<List<String>, Counter>) iter.next();
    Distribution d = Distribution.laplaceSmoothedDistribution((ClassicCounter) entry.getValue(), numTags, 0.5);
    markovPOSDists.put(((List<String>) entry.getKey()).get(0), d);
  }
}
 

import edu.stanford.nlp.stats.Distribution; //导入依赖的package包/类
private Distribution<Integer> getSegmentedWordLengthDistribution(Treebank tb) {
  // CharacterLevelTagExtender ext = new CharacterLevelTagExtender();
  ClassicCounter<Integer> c = new ClassicCounter<Integer>();
  for (Iterator iterator = tb.iterator(); iterator.hasNext();) {
    Tree gold = (Tree) iterator.next();
    StringBuilder goldChars = new StringBuilder();
    ArrayList goldYield = gold.yield();
    for (Iterator wordIter = goldYield.iterator(); wordIter.hasNext();) {
      Word word = (Word) wordIter.next();
      goldChars.append(word);
    }
    List<HasWord> ourWords = segment(goldChars.toString());
    for (int i = 0; i < ourWords.size(); i++) {
      c.incrementCount(Integer.valueOf(ourWords.get(i).word().length()));
    }
  }
  return Distribution.getDistribution(c);
}
 

import edu.stanford.nlp.stats.Distribution; //导入依赖的package包/类
private boolean argVectorsDiffer(Counter<String> args1, Counter<String> args2) {
  System.out.println("argVectorsDiffer top!");
  Distribution<String> dist1 = Distribution.getDistribution(args1);
  Distribution<String> dist2 = Distribution.getDistribution(args2);

  Set<String> argdiffs = new HashSet<String>();

  for( String token : dist1.keySet() ) {
    double prob1 = dist1.getCount(token);
    if( dist1.getCount(token) > 0.02 ) {
      double prob2 = dist2.getCount(token);
      double ratio = (prob1 < prob2 ? prob1 / prob2 : prob2 / prob1);
      System.out.printf("- %s\t%.4f\t%.4f\tratio=%.4f\n", token, prob1, prob2, ratio);
      if( ratio < 0.2 ) {
        argdiffs.add(token);
        System.out.println("  arg differs: " + token);
      }
    }
  }

  if( argdiffs.size() >= 2 ) {
    System.out.println("Arg vectors differ!!");
    return true;
  }

  return false;
}
 

import edu.stanford.nlp.stats.Distribution; //导入依赖的package包/类
public void train(Collection<Tree> trees) {
  Numberer tagNumberer = Numberer.getGlobalNumberer("tags");
  lex.train(trees);
  ClassicCounter<String> initial = new ClassicCounter<String>();
  GeneralizedCounter ruleCounter = new GeneralizedCounter(2);
  for (Tree tree : trees) {
    List<Label> tags = tree.preTerminalYield();
    String last = null;
    for (Label tagLabel : tags) {
      String tag = tagLabel.value();
      tagNumberer.number(tag);
      if (last == null) {
        initial.incrementCount(tag);
      } else {
        ruleCounter.incrementCount2D(last, tag);
      }
      last = tag;
    }
  }
  int numTags = tagNumberer.total();
  POSes = new HashSet<String>(ErasureUtils.<Collection<String>>uncheckedCast(tagNumberer.objects()));
  initialPOSDist = Distribution.laplaceSmoothedDistribution(initial, numTags, 0.5);
  markovPOSDists = new HashMap<String, Distribution>();
  Set entries = ruleCounter.lowestLevelCounterEntrySet();
  for (Iterator iter = entries.iterator(); iter.hasNext();) {
    Map.Entry entry = (Map.Entry) iter.next();
    //      Map.Entry<List<String>, Counter> entry = (Map.Entry<List<String>, Counter>) iter.next();
    Distribution d = Distribution.laplaceSmoothedDistribution((ClassicCounter) entry.getValue(), numTags, 0.5);
    markovPOSDists.put(((List<String>) entry.getKey()).get(0), d);
  }
}
 

import edu.stanford.nlp.stats.Distribution; //导入依赖的package包/类
protected Distribution<String> computeInputPrior(Map<String, List<List<String>>> allTrainPaths) {
  ClassicCounter<String> result = new ClassicCounter<String>();
  for (Iterator<List<List<String>>> catI = allTrainPaths.values().iterator(); catI.hasNext();) {
    List<List<String>> pathList = catI.next();
    for (List<String> path : pathList) {
      for (String input : path) {
        result.incrementCount(input);
      }
    }
  }
  return Distribution.laplaceSmoothedDistribution(result, result.size() * 2, 0.5);
}
 

import edu.stanford.nlp.stats.Distribution; //导入依赖的package包/类
protected static Distribution<String> computeInputPrior(Map<String, List<List<String>>> allTrainPaths) {
  ClassicCounter<String> result = new ClassicCounter<String>();
  for (List<List<String>> pathList : allTrainPaths.values()) {
    for (List<String> path : pathList) {
      for (String input : path) {
        result.incrementCount(input);
      }
    }
  }
  return Distribution.laplaceSmoothedDistribution(result, result.size() * 2, 0.5);
}