import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
public void run(String[] args) {
  Params params = validateAndParseParams(args, Params.class);

  File dictInFile = params.getInputFile();

  CmdLineUtil.checkInputFile("dictionary input file", dictInFile);
  Path metadataPath = DictionaryMetadata.getExpectedMetadataLocation(dictInFile.toPath());
  CmdLineUtil.checkInputFile("dictionary metadata (.info) input file", metadataPath.toFile());

  MorfologikDictionayBuilder builder = new MorfologikDictionayBuilder();
  try {
    builder.build(dictInFile.toPath(), params.getOverwrite(),
        params.getValidate(), params.getAcceptBOM(), params.getAcceptCR(),
        params.getIgnoreEmpty());
  } catch (Exception e) {
    throw new TerminateToolException(-1,
        "Error while creating Morfologik POS Dictionay: " + e.getMessage(), e);
  }

}
 

import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
public static void main(String[] args) {
if (args.length < 2) {
    System.out.println("usage: <input> <output>\n");
    System.exit(0);
}

String input = args[0];
String output = args[1];

TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(0));
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(100));
//params.put(TrainingParameters.ALGORITHM_PARAM, NaiveBayesTrainer.NAIVE_BAYES_VALUE);

AgeClassifyModel model;
try {
    model = AgeClassifySparkTrainer.createModel("en", input, 
        "opennlp.tools.tokenize.SentenceTokenizer", "opennlp.tools.tokenize.BagOfWordsTokenizer", params);
} catch (IOException e) {
    throw new TerminateToolException(-1,
        "IO error while reading training data or indexing data: " + e.getMessage(), e);
}
CmdLineUtil.writeModel("age classifier", new File(output), model);
   }
 

import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
public static void serializeEntityGazetteers(Path dictionaryFile)
    throws IOException {
  Map<String, String> dictionary = new HashMap<String, String>();
  InputStream inputStream = CmdLineUtil.openInFile(dictionaryFile.toFile());
  BufferedReader breader = new BufferedReader(
      new InputStreamReader(inputStream, Charset.forName("UTF-8")));
  String line;
  while ((line = breader.readLine()) != null) {
    String[] lineArray = tabPattern.split(line);
    if (lineArray.length == 2) {
      String normalizedToken = dotInsideI.matcher(lineArray[0])
          .replaceAll("i");
      dictionary.put(normalizedToken.toLowerCase(), lineArray[1].intern());
    } else {
      System.err.println(lineArray[0] + " is not well formed!");
    }
  }
  String outputFile = dictionaryFile.toString() + SER_GZ;
  IOUtils.writeClusterToFile(dictionary, outputFile, IOUtils.TAB_DELIMITER);
  breader.close();
}
 

import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
public static void serializeLemmaDictionary(Path lemmaDict)
    throws IOException {
  Map<List<String>, String> dictMap = new HashMap<List<String>, String>();
  InputStream inputStream = CmdLineUtil.openInFile(lemmaDict.toFile());
  BufferedReader breader = new BufferedReader(
      new InputStreamReader(inputStream, Charset.forName("UTF-8")));
  String line;
  while ((line = breader.readLine()) != null) {
    final String[] elems = tabPattern.split(line);
    if (elems.length == 3) {
      String normalizedToken = dotInsideI.matcher(elems[0]).replaceAll("I");
      dictMap.put(Arrays.asList(normalizedToken, elems[2]), elems[1]);
    } else {
      System.err.println(elems[0] + " is not well formed!");
    }
  }
  String outputFile = lemmaDict.toString() + SER_GZ;
  IOUtils.writeDictionaryLemmatizerToFile(dictMap, outputFile,
      IOUtils.TAB_DELIMITER);
  breader.close();
}
 

import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
/**
 * Main entry point for training.
 * 
 * @throws IOException
 *           throws an exception if errors in the various file inputs.
 */
public final void train() throws IOException {
  // load training parameters file
  final String paramFile = this.parsedArguments.getString("params");
  final TrainingParameters params = InputOutputUtils
      .loadTrainingParameters(paramFile);
  String outModel = null;
  if (params.getSettings().get("OutputModel") == null
      || params.getSettings().get("OutputModel").length() == 0) {
    outModel = Files.getNameWithoutExtension(paramFile) + ".bin";
    params.put("OutputModel", outModel);
  } else {
    outModel = Flags.getModel(params);
  }
  final Trainer chunkerTrainer = new DefaultTrainer(params);
  final ChunkerModel trainedModel = chunkerTrainer.train(params);
  CmdLineUtil.writeModel("ixa-pipe-chunk", new File(outModel), trainedModel);
}
 

import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
static ObjectStream<POSSample> openSampleData(String sampleDataName, File sampleDataFile, Charset encoding) {
    CmdLineUtil.checkInputFile(sampleDataName + " Data", sampleDataFile);
    FileInputStream sampleDataIn = CmdLineUtil.openInFile(sampleDataFile);
    ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn.getChannel(), encoding);
    return new WordTagSampleStream(lineStream);
}
 

import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
public void train() throws IOException {
    if (languageCode == null) {
        throw new IllegalStateException("languageCode is not provided");
    }
    if (modelOutFile == null) {
        throw new IllegalStateException("model output path is not provided");
    }
    if (trainParams == null) {
        throw new IllegalStateException("training parameters are not set");
    }
    if (sentenceStream == null) {
        throw new IllegalStateException("sentence stream is not configured");
    }
    if (taggerFactory == null) {
        throw new IllegalStateException("tagger factory is not configured");
    }
    Map<String, String> manifestInfoEntries = new HashMap<>();
    BeamSearchContextGenerator<Token> contextGenerator = taggerFactory.getContextGenerator();

    MaxentModel posModel;
    try {
        if (TrainerFactory.TrainerType.EVENT_MODEL_TRAINER.equals(
                TrainerFactory.getTrainerType(trainParams.getSettings()))) {

            ObjectStream<Event> es = new POSTokenEventStream<>(sentenceStream, contextGenerator);
            EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams.getSettings(), manifestInfoEntries);
            posModel = trainer.train(es);
        } else {
            throw new UnsupportedOperationException("Sequence training");
            //POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator);
            // posModel = TrainUtil.train(ss, trainParams.getSettings(), manifestInfoEntries);
        }
    } finally {
        sentenceStream.close();
    }
    POSModel modelAggregate = new POSModel(languageCode,
            posModel, manifestInfoEntries, taggerFactory);
    CmdLineUtil.writeModel("PoS-tagger", modelOutFile, modelAggregate);
}
 

import opennlp.tools.cmdline.CmdLineUtil; //导入依赖的package包/类
/**
 * Do not print a sentence if is less than 90% lowercase.
 * 
 * @param sentences
 *          the list of sentences
 * @throws IOException
 */
private static void brownCleanUpperCase(Path inFile) throws IOException {
  StringBuilder precleantext = new StringBuilder();
  InputStream inputStream = CmdLineUtil.openInFile(inFile.toFile());
  BufferedReader breader = new BufferedReader(
      new InputStreamReader(inputStream, Charset.forName("UTF-8")));
  String line;
  while ((line = breader.readLine()) != null) {
    double lowercaseCounter = 0;
    StringBuilder sb = new StringBuilder();
    String[] lineArray = line.split(" ");
    for (String word : lineArray) {
      if (lineArray.length > 0) {
        sb.append(word);
      }
    }
    char[] lineCharArray = sb.toString().toCharArray();
    for (char lineArr : lineCharArray) {
      if (Character.isLowerCase(lineArr)) {
        lowercaseCounter++;
      }
    }
    double percent = lowercaseCounter / (double) lineCharArray.length;
    if (percent >= 0.90) {
      precleantext.append(line).append("\n");
    }
  }
  Path outfile = Files.createFile(Paths.get(inFile.toString() + ".clean"));
  Files.write(outfile,
      precleantext.toString().getBytes(StandardCharsets.UTF_8));
  System.err.println(">> Wrote clean document to " + outfile);
  breader.close();
}