import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testCommonTermsQuery() throws IOException {
    Directory dir = newDirectory();
    String value = "The quick brown fox.";
    Analyzer analyzer = new StandardAnalyzer();
    IndexReader ir = indexOneDoc(dir, "text", value, analyzer);
    CommonTermsQuery query = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
    query.add(new Term("text", "quick"));
    query.add(new Term("text", "brown"));
    query.add(new Term("text", "fox"));
    IndexSearcher searcher = newSearcher(ir);
    TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
    assertThat(topDocs.totalHits, equalTo(1));
    int docId = topDocs.scoreDocs[0].doc;
    CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
    CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
        passageFormatter, null, value, false);
    Snippet[] snippets = highlighter.highlightField("text", query, docId, 5);
    assertThat(snippets.length, equalTo(1));
    assertThat(snippets[0].getText(), equalTo("The <b>quick</b> <b>brown</b> <b>fox</b>."));
    ir.close();
    dir.close();
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
@Override
protected SpanQuery convertUnknownQuery(String field, Query query) {
  if (query instanceof CommonTermsQuery) {

    // specialized since rewriting would change the result query
    // this query is TermContext sensitive.
    CommonTermsQuery ctq = (CommonTermsQuery) query;

    Set<Term> terms = new HashSet<>();
    try {
      Weight w = ctq.createWeight(searcher, false, 1.0f);
      w.extractTerms(terms);
    } catch (IOException e) {
      throw new RuntimeException("IOException on searcher!!!", e);
    }
    List<SpanQuery> spanQs = new LinkedList<>();

    for (Term term : terms) {
      if (term.field().equals(field)) {
        spanQs.add(new SpanTermQuery(term));
      }
    }
    if (spanQs.size() == 0) {
      return getEmptySpanQuery();
    } else if (spanQs.size() == 1) {
      return spanQs.get(0);
    } else {
      return new SpanOrQuery(spanQs.toArray(new SpanQuery[spanQs.size()]));
    }
  }
  super.convertUnknownQuery(field, query);
  return null;
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testExtractQueryMetadata_commonTermsQuery() {
    CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100);
    commonTermsQuery.add(new Term("_field", "_term1"));
    commonTermsQuery.add(new Term("_field", "_term2"));
    Result result = analyze(commonTermsQuery);
    assertThat(result.verified, is(false));
    List<Term> terms = new ArrayList<>(result.terms);
    Collections.sort(terms);
    assertThat(terms.size(), equalTo(2));
    assertThat(terms.get(0).field(), equalTo("_field"));
    assertThat(terms.get(0).text(), equalTo("_term1"));
    assertThat(terms.get(1).field(), equalTo("_field"));
    assertThat(terms.get(1).text(), equalTo("_term2"));
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testHighlightingCommonTermsQuery() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);
  query.add(new Term(FIELD_NAME, "this"));
  query.add(new Term(FIELD_NAME, "long"));
  query.add(new Term(FIELD_NAME, "very"));

  searcher = newSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
  Highlighter highlighter = new Highlighter(scorer);

  Document doc = searcher.doc(hits.scoreDocs[0].doc);
  String storedField = doc.get(FIELD_NAME);

  TokenStream stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[0].doc, FIELD_NAME, doc, analyzer);
  Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
  highlighter.setTextFragmenter(fragmenter);
  String fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("Hello <B>this</B> is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
  
  doc = searcher.doc(hits.scoreDocs[1].doc);
  storedField = doc.get(FIELD_NAME);

  stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[1].doc, FIELD_NAME, doc, analyzer);
  highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));
  fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("<B>This</B> piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testHighlightingCommonTermsQuery() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);
  query.add(new Term(FIELD_NAME, "this"));
  query.add(new Term(FIELD_NAME, "long"));
  query.add(new Term(FIELD_NAME, "very"));

  searcher = new IndexSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
  Highlighter highlighter = new Highlighter(scorer);

  Document doc = searcher.doc(hits.scoreDocs[0].doc);
  String storedField = doc.get(FIELD_NAME);

  TokenStream stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[0].doc, FIELD_NAME, doc, analyzer);
  Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
  highlighter.setTextFragmenter(fragmenter);
  String fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("Hello <B>this</B> is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
  
  doc = searcher.doc(hits.scoreDocs[1].doc);
  storedField = doc.get(FIELD_NAME);

  stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[1].doc, FIELD_NAME, doc, analyzer);
  highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));
  fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("<B>This</B> piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public LuceneResponse similarDocuments(String identifier) throws Throwable {
    SearcherAndTaxonomy reference = data.getManager().acquire();
    try {
        Query idQuery = new TermQuery(new Term(ID_FIELD, identifier));
        TopDocs topDocs = reference.searcher.search(idQuery, 1);
        if (topDocs.totalHits == 0)
            return new LuceneResponse(0);
        int docId = topDocs.scoreDocs[0].doc;
        IndexReader reader = reference.searcher.getIndexReader();
        CommonTermsQuery commonQuery = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD, 0.1f);
        Fields termVectors = reader.getTermVectors(docId);
        if (termVectors == null)
            return new LuceneResponse(0);
        for (String field : termVectors) {
            TermsEnum iterator = termVectors.terms(field).iterator();
            BytesRef b;
            while ((b = iterator.next()) != null) {
                Term term = new Term(field, b.utf8ToString());
                commonQuery.add(term);
            }
        }
        BooleanQuery.Builder query = new BooleanQuery.Builder();
        query.add(idQuery, Occur.MUST_NOT);
        query.add(commonQuery, Occur.MUST);
        return executeQuery(query.build());
    } finally {
        data.getManager().release(reference);
    }
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
static Function<Query, Result> commonTermsQuery() {
    return query -> {
        List<Term> terms = ((CommonTermsQuery) query).getTerms();
        return new Result(false, new HashSet<>(terms));
    };
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testDuelSpecificQueries() throws Exception {
    List<ParseContext.Document> documents = new ArrayList<>();

    CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
    commonTermsQuery.add(new Term("field", "quick"));
    commonTermsQuery.add(new Term("field", "brown"));
    commonTermsQuery.add(new Term("field", "fox"));
    addQuery(commonTermsQuery, documents);

    BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"),
            new Term("field", "brown"), new Term("field", "fox")}, false);
    addQuery(blendedTermQuery, documents);

    SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true)
            .addClause(new SpanTermQuery(new Term("field", "quick")))
            .addClause(new SpanTermQuery(new Term("field", "brown")))
            .addClause(new SpanTermQuery(new Term("field", "fox")))
            .build();
    addQuery(spanNearQuery, documents);

    SpanNearQuery spanNearQuery2 = new SpanNearQuery.Builder("field", true)
            .addClause(new SpanTermQuery(new Term("field", "the")))
            .addClause(new SpanTermQuery(new Term("field", "lazy")))
            .addClause(new SpanTermQuery(new Term("field", "doc")))
            .build();
    SpanOrQuery spanOrQuery = new SpanOrQuery(
            spanNearQuery,
            spanNearQuery2
    );
    addQuery(spanOrQuery, documents);

    SpanNotQuery spanNotQuery = new SpanNotQuery(spanNearQuery, spanNearQuery);
    addQuery(spanNotQuery, documents);

    long lowerLong = randomIntBetween(0, 256);
    long upperLong = lowerLong + randomIntBetween(0, 32);
    addQuery(LongPoint.newRangeQuery("long_field", lowerLong, upperLong), documents);

    indexWriter.addDocuments(documents);
    indexWriter.close();
    directoryReader = DirectoryReader.open(directory);
    IndexSearcher shardSearcher = newSearcher(directoryReader);
    // Disable query cache, because ControlQuery cannot be cached...
    shardSearcher.setQueryCache(null);

    Document document = new Document();
    document.add(new TextField("field", "the quick brown fox jumps over the lazy dog", Field.Store.NO));
    long randomLong = randomIntBetween((int) lowerLong, (int) upperLong);
    document.add(new LongPoint("long_field", randomLong));
    MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
/**
 * Translate custom queries in queries that are supported by the unified highlighter.
 */
private Collection<Query> rewriteCustomQuery(Query query) {
    if (query instanceof MultiPhrasePrefixQuery) {
        MultiPhrasePrefixQuery mpq = (MultiPhrasePrefixQuery) query;
        Term[][] terms = mpq.getTerms();
        int[] positions = mpq.getPositions();
        SpanQuery[] positionSpanQueries = new SpanQuery[positions.length];
        int sizeMinus1 = terms.length - 1;
        for (int i = 0; i < positions.length; i++) {
            SpanQuery[] innerQueries = new SpanQuery[terms[i].length];
            for (int j = 0; j < terms[i].length; j++) {
                if (i == sizeMinus1) {
                    innerQueries[j] = new SpanMultiTermQueryWrapper(new PrefixQuery(terms[i][j]));
                } else {
                    innerQueries[j] = new SpanTermQuery(terms[i][j]);
                }
            }
            if (innerQueries.length > 1) {
                positionSpanQueries[i] = new SpanOrQuery(innerQueries);
            } else {
                positionSpanQueries[i] = innerQueries[0];
            }
        }
        // sum position increments beyond 1
        int positionGaps = 0;
        if (positions.length >= 2) {
            // positions are in increasing order.   max(0,...) is just a safeguard.
            positionGaps = Math.max(0, positions[positions.length - 1] - positions[0] - positions.length + 1);
        }

        //if original slop is 0 then require inOrder
        boolean inorder = (mpq.getSlop() == 0);
        return Collections.singletonList(new SpanNearQuery(positionSpanQueries,
            mpq.getSlop() + positionGaps, inorder));
    } else if (query instanceof CommonTermsQuery) {
        CommonTermsQuery ctq = (CommonTermsQuery) query;
        List<Query> tqs = new ArrayList<> ();
        for (Term term : ctq.getTerms()) {
            tqs.add(new TermQuery(term));
        }
        return tqs;
    } else if (query instanceof AllTermQuery) {
        AllTermQuery atq = (AllTermQuery) query;
        return Collections.singletonList(new TermQuery(atq.getTerm()));
    } else if (query instanceof FunctionScoreQuery) {
        return Collections.singletonList(((FunctionScoreQuery) query).getSubQuery());
    } else if (query instanceof FiltersFunctionScoreQuery) {
        return Collections.singletonList(((FiltersFunctionScoreQuery) query).getSubQuery());
    } else {
        return null;
    }
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testCommonTermsQueryHighlight() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
  FieldType type = new FieldType(TextField.TYPE_STORED);
  type.setStoreTermVectorOffsets(true);
  type.setStoreTermVectorPositions(true);
  type.setStoreTermVectors(true);
  type.freeze();
  String[] texts = {
      "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
      "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
      "JFK has been shot", "John Kennedy has been shot",
      "This text has a typo in referring to Keneddy",
      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
  for (int i = 0; i < texts.length; i++) {
    Document doc = new Document();
    Field field = new Field("field", texts[i], type);
    doc.add(field);
    writer.addDocument(doc);
  }
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 2);
  query.add(new Term("field", "text"));
  query.add(new Term("field", "long"));
  query.add(new Term("field", "very"));
 
  FastVectorHighlighter highlighter = new FastVectorHighlighter();
  IndexReader reader = DirectoryReader.open(writer, true);
  IndexSearcher searcher = newSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  FieldQuery fieldQuery  = highlighter.getFieldQuery(query, reader);
  String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
  assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);

  fieldQuery  = highlighter.getFieldQuery(query, reader);
  bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[1].doc, "field", 1000, 1);
  assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);

  reader.close();
  writer.close();
  dir.close();
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testCommonTermsQueryHighlightTest() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,  new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
  FieldType type = new FieldType(TextField.TYPE_STORED);
  type.setStoreTermVectorOffsets(true);
  type.setStoreTermVectorPositions(true);
  type.setStoreTermVectors(true);
  type.freeze();
  String[] texts = {
      "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
      "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
      "JFK has been shot", "John Kennedy has been shot",
      "This text has a typo in referring to Keneddy",
      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
  for (int i = 0; i < texts.length; i++) {
    Document doc = new Document();
    Field field = new Field("field", texts[i], type);
    doc.add(field);
    writer.addDocument(doc);
  }
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 2);
  query.add(new Term("field", "text"));
  query.add(new Term("field", "long"));
  query.add(new Term("field", "very"));
 
  FastVectorHighlighter highlighter = new FastVectorHighlighter();
  IndexReader reader = DirectoryReader.open(writer, true);
  IndexSearcher searcher = new IndexSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  FieldQuery fieldQuery  = highlighter.getFieldQuery(query, reader);
  String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
  assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);

  fieldQuery  = highlighter.getFieldQuery(query, reader);
  bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[1].doc, "field", 1000, 1);
  assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);

  reader.close();
  writer.close();
  dir.close();
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入依赖的package包/类
public void testCommonTermsQueryHighlightTest() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,  new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
  FieldType type = new FieldType(TextField.TYPE_STORED);
  type.setStoreTermVectorOffsets(true);
  type.setStoreTermVectorPositions(true);
  type.setStoreTermVectors(true);
  type.freeze();
  String[] texts = {
      "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
      "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
      "JFK has been shot", "John Kennedy has been shot",
      "This text has a typo in referring to Keneddy",
      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
  for (int i = 0; i < texts.length; i++) {
    Document doc = new Document();
    Field field = new Field("field", texts[i], type);
    doc.add(field);
    writer.addDocument(doc);
  }
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 2);
  query.add(new Term("field", "text"));
  query.add(new Term("field", "long"));
  query.add(new Term("field", "very"));
 
  FastVectorHighlighter highlighter = new FastVectorHighlighter();
  IndexReader reader = DirectoryReader.open(writer, true);
  IndexSearcher searcher = newSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  FieldQuery fieldQuery  = highlighter.getFieldQuery(query, reader);
  String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
  assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);

  fieldQuery  = highlighter.getFieldQuery(query, reader);
  bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[1].doc, "field", 1000, 1);
  assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);

  reader.close();
  writer.close();
  dir.close();
}