import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
private void init(String location) {
    csv = new File(location);
    pair = new HashMap<>();
    sentenceToPhrase = new HashMap<>();

    CSV csv1 = CSV.separator('\t')
            .ignoreLeadingWhiteSpace().skipLines(1)
            .create();
    //PhraseId	SentenceId	Phrase	Sentiment
    csv1.read(csv,new CSVReadProc() {
        @Override
        public void procRow(int rowIndex, String... values) {
            //sentence id -> phrase id
            List<String> phrases = sentenceToPhrase.get(values[1]);
            if(phrases == null) {
                phrases = new ArrayList<>();
                sentenceToPhrase.put(values[1],phrases);
            }

            phrases.add(values[2]);

            pair.put(values[0],new Pair<>(values[2],values[3]));
        }
    });
}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
public List<Pair<String, Double>> predict(@NotNull String name, @NotNull SourceSegment source, @NotNull List<var> inputs) {

     /*
      Now we'll iterate over unlabeled data, and check which label it could be assigned to
      Please note: for many domains it's normal to have 1 document fall into few labels at once,
      with different "weight" for each.
     */
        MeansBuilder meansBuilder = new MeansBuilder((InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable(),
                                                     tokenizerFactory);
        LabelSeeker seeker = new LabelSeeker(iterator.getLabelsSource().getLabels(),
                                             (InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable());


        LabelledDocument document = new LabelledDocument();
        document.setContent(signatureToText(name, inputs));
        INDArray documentAsCentroid = meansBuilder.documentAsVector(document);
        List<Pair<String, Double>> scores = seeker.getScores(documentAsCentroid);
        return scores;

    }
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
/**
 * This method accepts vector, that represents any document,
 * and returns distances between this document, and previously trained categories
 * @return
 */
public List<Pair<String, Double>> getScores(@NonNull INDArray vector) {
    List<Pair<String, Double>> result = new ArrayList<>();
    for (String label: labelsUsed) {
        INDArray vecLabel = lookupTable.vector(label);
        if (vecLabel == null) throw new IllegalStateException("Label '"+ label+"' has no known vector!");

        double sim = Transforms.cosineSim(vector, vecLabel);
        result.add(new Pair<String, Double>(label, sim));
    }
    return result;
}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
@Test
public void testRetrieval() throws  Exception {
    Map<String,Pair<String,String>> data = retriever.data();
    Pair<String,String> phrase2 = data.get("2");
    assertEquals("A series of escapades demonstrating the adage that what is good for the goose",phrase2.getFirst());
    assertEquals("2",phrase2.getSecond());

}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
/**
 * This method accepts vector, that represents any document,
 * and returns distances between this document, and previously trained categories
 *
 * @return
 */
@Nonnull
public List<Pair<String, Double>> getScores(@Nonnull INDArray vector) {
    List<Pair<String, Double>> result = new ArrayList<>();
    for (String label : labelsUsed) {
        INDArray vecLabel = lookupTable.vector(label);
        if (vecLabel == null) throw new IllegalStateException("Label '" + label + "' has no known vector!");

        double sim = Transforms.cosineSim(vector, vecLabel);
        result.add(new Pair<>(label, sim));
    }
    return result;
}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
@Nullable
@Override
public TypePrediction predict(@NotNull String name, @NotNull SourceSegment source, @NotNull List<var> inputs) {
    try {
        List<Pair<String, Double>> scores = classifier.predict(name, source, inputs);
        log.info("Predictions: ");
        log.info("{}", scores);
        return new SmartTypePrediction(scores);
    } catch (Exception e) {
        log.debug(e.getMessage(), e);
        return new SingleValueTypePrediction(Type._ANY);
    }


}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
@Override
public @NotNull Double probability(@NotNull Type type) {
    for (Pair<String, Double> score : scores) {
        if (score.getFirst().equals(type.name())) {
            return score.getSecond() / 2 + 0.5;
        }
    }
    return 0.0;
}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
@Override
public @Nullable Type probableType() {
    Pair<String, Double> max = scores.stream().max(Comparator.comparing(Pair::getSecond)).orElse(null);
    if (max != null) {
        return Type.of(max.getFirst());
    }
    return null;
}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
void checkUnlabeledData() throws FileNotFoundException {
  /*
  At this point we assume that we have model built and we can check
  which categories our unlabeled document falls into.
  So we'll start loading our unlabeled documents and checking them
 */
    ClassPathResource unClassifiedResource = new ClassPathResource("paravec/unlabeled");
    FileLabelAwareIterator unClassifiedIterator = new FileLabelAwareIterator.Builder()
                                                          .addSourceFolder(unClassifiedResource.getFile())
                                                          .build();

 /*
  Now we'll iterate over unlabeled data, and check which label it could be assigned to
  Please note: for many domains it's normal to have 1 document fall into few labels at once,
  with different "weight" for each.
 */
    MeansBuilder meansBuilder = new MeansBuilder(
                                                        (InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable(),
                                                        tokenizerFactory);
    LabelSeeker seeker = new LabelSeeker(iterator.getLabelsSource().getLabels(),
                                         (InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable());

    while (unClassifiedIterator.hasNextDocument()) {
        LabelledDocument document = unClassifiedIterator.nextDocument();
        INDArray documentAsCentroid = meansBuilder.documentAsVector(document);
        List<Pair<String, Double>> scores = seeker.getScores(documentAsCentroid);

     /*
      please note, document.getLabel() is used just to show which document we're looking at now,
      as a substitute for printing out the whole document name.
      So, labels on these two documents are used like titles,
      just to visualize our classification done properly
     */
        log.info("Document '" + document.getLabel() + "' falls into the following categories: ");
        for (Pair<String, Double> score : scores) {
            log.info("        " + score.getFirst() + ": " + score.getSecond());
        }
    }

}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

        ClassPathResource resource = new ClassPathResource("paravec/labeled");

        iter = new FileLabelAwareIterator.Builder()
                .addSourceFolder(resource.getFile())
                .build();

        tFact = new DefaultTokenizerFactory();
        tFact.setTokenPreProcessor(new CommonPreprocessor());

        pVect = new ParagraphVectors.Builder()
                .learningRate(0.025)
                .minLearningRate(0.001)
                .batchSize(1000)
                .epochs(20)
                .iterate(iter)
                .trainWordVectors(true)
                .tokenizerFactory(tFact)
                .build();

        pVect.fit();


        ClassPathResource unlabeledText = new ClassPathResource("paravec/unlabeled");
        FileLabelAwareIterator unlabeledIter = new FileLabelAwareIterator.Builder()
                .addSourceFolder(unlabeledText.getFile())
                .build();


        MeansBuilder mBuilder = new MeansBuilder(
                (InMemoryLookupTable<VocabWord>) pVect.getLookupTable(),
                tFact);
        LabelSeeker lSeeker = new LabelSeeker(iter.getLabelsSource().getLabels(),
                (InMemoryLookupTable<VocabWord>) pVect.getLookupTable());

        while (unlabeledIter.hasNextDocument()) {
            LabelledDocument doc = unlabeledIter.nextDocument();
            INDArray docCentroid = mBuilder.documentAsVector(doc);
            List<Pair<String, Double>> scores = lSeeker.getScores(docCentroid);

            out.println("Document '" + doc.getLabel() + "' falls into the following categories: ");
            for (Pair<String, Double> score : scores) {
                out.println("        " + score.getFirst() + ": " + score.getSecond());
            }

        }
    }
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
public Map<String,Pair<String,String>> data() {
    return pair;
}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
private void checkUnlabelledData(Word2Vec paragraphVectors, LabelAwareIterator iterator, TokenizerFactory tokenizerFactory) throws FileNotFoundException {
  ClassPathResource unClassifiedResource = new ClassPathResource("papers/unlabeled");
  FileLabelAwareIterator unClassifiedIterator = new FileLabelAwareIterator.Builder()
      .addSourceFolder(unClassifiedResource.getFile())
      .build();

  MeansBuilder meansBuilder = new MeansBuilder(
      (InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable(),
      tokenizerFactory);
  LabelSeeker seeker = new LabelSeeker(iterator.getLabelsSource().getLabels(),
      (InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable());

  System.out.println(paragraphVectors + " classification results");
  double cc = 0;
  double size = 0;
  while (unClassifiedIterator.hasNextDocument()) {
    LabelledDocument document = unClassifiedIterator.nextDocument();
    INDArray documentAsCentroid = meansBuilder.documentAsVector(document);
    List<Pair<String, Double>> scores = seeker.getScores(documentAsCentroid);

    double max = -Integer.MAX_VALUE;
    String cat = null;
    for (Pair<String, Double> p : scores) {
      if (p.getSecond() > max) {
        max = p.getSecond();
        cat = p.getFirst();
      }
    }
    if (document.getLabels().contains(cat)) {
      cc++;
    }
    size++;

  }
  System.out.println("acc:" + (cc / size));

}
 

import org.deeplearning4j.berkeley.Pair; //导入依赖的package包/类
public SmartTypePrediction(List<Pair<String, Double>> scores) {this.scores = scores;}