Apache Lucene open-source search software
Optimise phrasescorer matches (#15861)
Prithvi7b61978b14ff
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 4ba9ce4b205..66e89f00a6c 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -301,6 +301,8 @@ ImprovementsOptimizations---------------------+* GITHUB#15861: Optimise PhraseScorer by short circuiting non competitive documents in TOP_SCORES mode. (Prithvi S)+* GITHUB#15637: Lazily allocate ByteArrayDataInputs in SegmentTermsEnumFrame#15637 (Misha Dmitriev)* GITHUB#15718 Skip per-document stored field reads on sorted indices when no stored fields are present (Francisco Fernández Castaño)diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PhraseScorerBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PhraseScorerBenchmark.javanew file mode 100644index 00000000000..2b0ba46b948--- /dev/null+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PhraseScorerBenchmark.java@@ -0,0 +1,141 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.benchmark.jmh;++import java.io.IOException;+import java.util.concurrent.TimeUnit;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.TextField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.PhraseQuery;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.search.TopFieldCollectorManager;+import org.apache.lucene.search.TopScoreDocCollectorManager;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.MMapDirectory;+import org.openjdk.jmh.annotations.Benchmark;+import org.openjdk.jmh.annotations.BenchmarkMode;+import org.openjdk.jmh.annotations.Fork;+import org.openjdk.jmh.annotations.Level;+import org.openjdk.jmh.annotations.Measurement;+import org.openjdk.jmh.annotations.Mode;+import org.openjdk.jmh.annotations.OutputTimeUnit;+import org.openjdk.jmh.annotations.Scope;+import org.openjdk.jmh.annotations.Setup;+import org.openjdk.jmh.annotations.State;+import org.openjdk.jmh.annotations.TearDown;+import org.openjdk.jmh.annotations.Warmup;++@BenchmarkMode(Mode.Throughput)+@OutputTimeUnit(TimeUnit.MILLISECONDS)+@State(Scope.Benchmark)+@Warmup(iterations = 5, time = 3)+@Measurement(iterations = 10, time = 5)+@Fork(+ value = 3,+ jvmArgsAppend = {"-Xmx2g", "-Xms2g"})+public class PhraseScorerBenchmark {++ private static final int NUM_HITS = 10;++ private Directory dir;+ private IndexReader reader;+ private IndexSearcher searcher;+ private PhraseQuery exactQuery;+ private PhraseQuery sloppyQuery;++ @Setup(Level.Trial)+ public void setUp() throws IOException {+ dir = new MMapDirectory(java.nio.file.Files.createTempDirectory("benchmark"));+ IndexWriterConfig config = new IndexWriterConfig();+ try (IndexWriter writer = new IndexWriter(dir, config)) {+ // Create a corpus where most docs contain the individual query terms but only a small+ // fraction contain the actual phrase. This maximises the number of documents whose maxFreq+ // upper-bound check allows short-circuiting.+ for (int i = 0; i < 1_000_000; i++) {+ Document doc = new Document();+ if (i % 1000 == 0) {+ // 0.1% of docs: exact phrase match+ doc.add(+ new TextField(+ "text", "the quick brown fox jumped over the lazy dog", Field.Store.NO));+ } else if (i % 2 == 0) {+ // 50% of docs: terms present but not as a phrase (high freq, no match)+ StringBuilder sb = new StringBuilder("quick ");+ for (int j = 0; j < 100; j++) sb.append("padding ");+ sb.append("fox");+ doc.add(new TextField("text", sb.toString(), Field.Store.NO));+ } else {+ // 50% of docs: no query terms at all+ doc.add(new TextField("text", "unrelated words", Field.Store.NO));+ }+ writer.addDocument(doc);+ }+ }+ reader = DirectoryReader.open(dir);+ searcher = new IndexSearcher(reader);+ exactQuery = new PhraseQuery("text", "quick", "brown", "fox");+ sloppyQuery = new PhraseQuery(10, "text", "quick", "fox");+ }++ @TearDown(Level.Trial)+ public void tearDown() throws IOException {+ reader.close();+ dir.close();+ }++ @Benchmark+ public TopDocs benchmarkExactTopScores() throws IOException {+ return searcher.search(exactQuery, 10);+ }++ @Benchmark+ public TopDocs benchmarkSloppyTopScores() throws IOException {+ return searcher.search(sloppyQuery, 10);+ }++ @Benchmark+ public TopDocs benchmarkExactComplete() throws IOException {+ return searcher.search(+ exactQuery, new TopScoreDocCollectorManager(NUM_HITS, Integer.MAX_VALUE));+ }++ @Benchmark+ public TopDocs benchmarkExactCompleteNoScores() throws IOException {+ return searcher.search(+ exactQuery, new TopFieldCollectorManager(Sort.INDEXORDER, NUM_HITS, Integer.MAX_VALUE));+ }++ @Benchmark+ public TopDocs benchmarkSloppyComplete() throws IOException {+ return searcher.search(+ sloppyQuery, new TopScoreDocCollectorManager(NUM_HITS, Integer.MAX_VALUE));+ }++ @Benchmark+ public TopDocs benchmarkSloppyCompleteNoScores() throws IOException {+ return searcher.search(+ sloppyQuery, new TopFieldCollectorManager(Sort.INDEXORDER, NUM_HITS, Integer.MAX_VALUE));+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseMatcher.java b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseMatcher.javaindex 8148bbf9fec..aa7af30b264 100644--- a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseMatcher.java+++ b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseMatcher.java@@ -50,6 +50,7 @@ public final class ExactPhraseMatcher extends PhraseMatcher {private final PostingsAndPosition[] postings;private final DocIdSetIterator approximation;private final ImpactsDISI impactsApproximation;+ private boolean freqsLoaded;/** Expert: Creates ExactPhraseMatcher instance */public ExactPhraseMatcher(@@ -93,14 +94,40 @@ public final class ExactPhraseMatcher extends PhraseMatcher {}@Override- float maxFreq() {- int minFreq = postings[0].freq;+ float maxFreq() throws IOException {+ // Load freqs eagerly so maxFreq() can be called before resetPositions() in TOP_SCORES+ // mode. PhraseScorer uses this to short-circuit non-competitive documents+ // before paying the cost of resetPositions() + nextMatch().+ int minFreq = postings[0].postings.freq();+ postings[0].freq = minFreq;for (int i = 1; i < postings.length; i++) {- minFreq = Math.min(minFreq, postings[i].freq);+ int f = postings[i].postings.freq();+ postings[i].freq = f;+ minFreq = Math.min(minFreq, f);}+ freqsLoaded = true;return minFreq;}+ @Override+ public void resetPositions() throws IOException {+ if (freqsLoaded) {+ // Freqs already loaded by maxFreq(). Only reset position state.+ freqsLoaded = false;+ for (PostingsAndPosition posting : postings) {+ posting.pos = -1;+ posting.upTo = 0;+ }+ } else {+ // Freqs not yet loaded. Original single-loop path.+ for (PostingsAndPosition posting : postings) {+ posting.freq = posting.postings.freq();+ posting.pos = -1;+ posting.upTo = 0;+ }+ }+ }+/*** Advance the given pos enum to the first position on or after {@code target}. Return {@code* false} if the enum was exhausted before reaching {@code target} and {@code true} otherwise.@@ -118,15 +145,6 @@ public final class ExactPhraseMatcher extends PhraseMatcher {return true;}- @Override- public void reset() throws IOException {- for (PostingsAndPosition posting : postings) {- posting.freq = posting.postings.freq();- posting.pos = -1;- posting.upTo = 0;- }- }-@Overridepublic boolean nextMatch() throws IOException {final PostingsAndPosition lead = postings[0];diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseMatcher.java b/lucene/core/src/java/org/apache/lucene/search/PhraseMatcher.javaindex 68b69036afc..d9804f6138d 100644--- a/lucene/core/src/java/org/apache/lucene/search/PhraseMatcher.java+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseMatcher.java@@ -23,8 +23,8 @@ import java.io.IOException;* Base class for exact and sloppy phrase matching** <p>To find matches on a document, first advance {@link #approximation} to the relevant document,- * then call {@link #reset()}. Clients can then call {@link #nextMatch()} to iterate over the- * matches+ * then call {@link #resetPositions()}. Clients can then call {@link #nextMatch()} to iterate over+ * the matches** @lucene.internal*/@@ -42,11 +42,18 @@ public abstract class PhraseMatcher {/** Approximation that is aware of impacts. */abstract ImpactsDISI impactsApproximation();- /** An upper bound on the number of possible matches on this document */+ /**+ * An upper bound on the number of possible matches on this document.+ *+ * <p>This may be called before {@link #resetPositions()} to enable early termination of+ * non-competitive documents in {@link ScoreMode#TOP_SCORES} mode, as long as the {@link+ * #approximation()} has been advanced to the target document. Implementations must lazily load+ * any required state (e.g. term frequencies) on first access per document.+ */abstract float maxFreq() throws IOException;- /** Called after {@link #approximation} has been advanced */- public abstract void reset() throws IOException;+ /** Called after {@link #approximation} has been advanced to load positions for matching. */+ public abstract void resetPositions() throws IOException;/** Find the next match on the current document, returning {@code false} if there are none. */public abstract boolean nextMatch() throws IOException;diff --git a/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java b/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.javaindex be6892f19f4..60c61002cca 100644--- a/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java+++ b/lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java@@ -31,6 +31,7 @@ final class PhrasePositions {int rptGroup = -1; // >=0 indicates that this is a repeating PPint rptInd; // index in the rptGroupfinal Term[] terms; // for repetitions initialization+ int freq; // cached frequency for the current documentPhrasePositions(PostingsEnum postings, int o, int ord, Term[] terms) {this.postings = postings;@@ -40,7 +41,7 @@ final class PhrasePositions {}final void firstPosition() throws IOException {- count = postings.freq(); // read first pos+ count = freq; // use cached frequencynextPosition();}diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.javaindex 69f07e624f2..8d9b2b5ff8d 100644--- a/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java@@ -52,7 +52,6 @@ class PhraseScorer extends Scorer {return new TwoPhaseIterator(approximation) {@Overridepublic boolean matches() throws IOException {- matcher.reset();if (scoreMode == ScoreMode.TOP_SCORES && minCompetitiveScore > 0) {float maxFreq = matcher.maxFreq();long norm = 1L;@@ -64,6 +63,7 @@ class PhraseScorer extends Scorer {return false;}}+ matcher.resetPositions();freq = 0;return matcher.nextMatch();}diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseWeight.java b/lucene/core/src/java/org/apache/lucene/search/PhraseWeight.javaindex 75439ca7eb0..beaafa89d96 100644--- a/lucene/core/src/java/org/apache/lucene/search/PhraseWeight.java+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseWeight.java@@ -75,7 +75,7 @@ public abstract class PhraseWeight extends Weight {if (matcher == null || matcher.approximation().advance(doc) != doc) {return Explanation.noMatch("no matching terms");}- matcher.reset();+ matcher.resetPositions();if (matcher.nextMatch() == false) {return Explanation.noMatch("no matching phrase");}@@ -111,7 +111,7 @@ public abstract class PhraseWeight extends Weight {if (matcher == null || matcher.approximation().advance(doc) != doc) {return null;}- matcher.reset();+ matcher.resetPositions();if (matcher.nextMatch() == false) {return null;}diff --git a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseMatcher.java b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseMatcher.javaindex 25558668829..b99ad42b88c 100644--- a/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseMatcher.java+++ b/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseMatcher.java@@ -79,6 +79,7 @@ public final class SloppyPhraseMatcher extends PhraseMatcher {private boolean positioned;private int matchLength;+ private boolean freqsLoaded;public SloppyPhraseMatcher(PhraseQuery.PostingsAndFreq[] postings,@@ -167,18 +168,29 @@ public final class SloppyPhraseMatcher extends PhraseMatcher {@Overridefloat maxFreq() throws IOException {- // every term position in each postings list can be at the head of at most- // one matching phrase, so the maximum possible phrase freq is the sum of- // the freqs of the postings lists.+ // Load freqs eagerly so maxFreq() can be called before resetPositions() in TOP_SCORES+ // mode. PhraseScorer uses this to short-circuit non-competitive documents+ // before paying the cost of resetPositions() + initPhrasePositions().float maxFreq = 0;for (PhrasePositions phrasePosition : phrasePositions) {- maxFreq += phrasePosition.postings.freq();+ phrasePosition.freq = phrasePosition.postings.freq();+ maxFreq += phrasePosition.freq;}+ freqsLoaded = true;return maxFreq;}@Override- public void reset() throws IOException {+ public void resetPositions() throws IOException {+ if (freqsLoaded) {+ // Freqs already loaded by maxFreq().+ freqsLoaded = false;+ } else {+ // Freqs not yet loaded. Load them now.+ for (PhrasePositions phrasePosition : phrasePositions) {+ phrasePosition.freq = phrasePosition.postings.freq();+ }+ }this.positioned = initPhrasePositions();this.matchLength = Integer.MAX_VALUE;this.leadPosition = Integer.MAX_VALUE;diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhraseMatcherContract.java b/lucene/core/src/test/org/apache/lucene/search/TestPhraseMatcherContract.javanew file mode 100644index 00000000000..b17b7323d47--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestPhraseMatcherContract.java@@ -0,0 +1,163 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.TextField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.Term;+import org.apache.lucene.index.TermStates;+import org.apache.lucene.search.similarities.BM25Similarity;+import org.apache.lucene.search.similarities.Similarity.SimScorer;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.tests.util.LuceneTestCase;++/**+ * Tests the {@link PhraseMatcher} contract, particularly the ability to call {@link+ * PhraseMatcher#maxFreq()} before {@link PhraseMatcher#resetPositions()}.+ */+public class TestPhraseMatcherContract extends LuceneTestCase {++ public void testExactPhraseMatcherContract() throws IOException {+ try (Directory dir = newDirectory()) {+ try (RandomIndexWriter fw = new RandomIndexWriter(random(), dir)) {+ Document doc = new Document();+ doc.add(new TextField("body", "the quick brown fox", Field.Store.NO));+ fw.addDocument(doc);+ }+ try (IndexReader reader = DirectoryReader.open(dir)) {+ LeafReaderContext context = reader.leaves().get(0);+ PhraseQuery.PostingsAndFreq[] postings = new PhraseQuery.PostingsAndFreq[2];++ Term t1 = new Term("body", "quick");+ Term t2 = new Term("body", "brown");++ org.apache.lucene.index.Terms bodyTerms = context.reader().terms("body");+ org.apache.lucene.index.TermsEnum te1 = bodyTerms.iterator();+ te1.seekExact(t1.bytes());+ PostingsEnum pe1 = te1.impacts(PostingsEnum.ALL);++ org.apache.lucene.index.TermsEnum te2 = bodyTerms.iterator();+ te2.seekExact(t2.bytes());+ PostingsEnum pe2 = te2.impacts(PostingsEnum.ALL);++ postings[0] =+ new PhraseQuery.PostingsAndFreq(pe1, (org.apache.lucene.index.ImpactsEnum) pe1, 0, t1);+ postings[1] =+ new PhraseQuery.PostingsAndFreq(pe2, (org.apache.lucene.index.ImpactsEnum) pe2, 1, t2);++ IndexSearcher searcher = new IndexSearcher(reader);+ FieldStats fieldStats = searcher.fieldStats("body");+ assertNotNull(fieldStats);+ // Same stats path as PhraseWeight#getStats (PhraseQuery#createWeight)+ TermStates ts1 = TermStates.build(searcher, t1, true);+ TermStates ts2 = TermStates.build(searcher, t2, true);+ assertTrue(ts1.docFreq() > 0);+ assertTrue(ts2.docFreq() > 0);+ SimScorer scorer =+ new BM25Similarity()+ .scorer(+ 1f,+ fieldStats,+ searcher.termStats(t1, ts1.docFreq(), ts1.totalTermFreq()),+ searcher.termStats(t2, ts2.docFreq(), ts2.totalTermFreq()));+ ExactPhraseMatcher matcher =+ new ExactPhraseMatcher(postings, ScoreMode.TOP_SCORES, scorer, 1f);++ DocIdSetIterator approximation = matcher.approximation();+ int docId = approximation.nextDoc();+ assertNotEquals(DocIdSetIterator.NO_MORE_DOCS, docId);++ // CALL maxFreq BEFORE resetPositions+ float maxFreq = matcher.maxFreq();+ assertTrue(maxFreq >= 1.0f);++ // Then resetPositions+ matcher.resetPositions();+ assertTrue("Should have a match in doc " + docId, matcher.nextMatch());+ assertFalse(matcher.nextMatch());+ }+ }+ }++ public void testSloppyPhraseMatcherContract() throws IOException {+ try (Directory dir = newDirectory()) {+ try (RandomIndexWriter fw = new RandomIndexWriter(random(), dir)) {+ Document doc = new Document();+ doc.add(new TextField("body", "the quick silver fox", Field.Store.NO));+ fw.addDocument(doc);+ }+ try (IndexReader reader = DirectoryReader.open(dir)) {+ LeafReaderContext context = reader.leaves().get(0);+ Term t1 = new Term("body", "quick");+ Term t2 = new Term("body", "fox");+ PhraseQuery.PostingsAndFreq[] postings = new PhraseQuery.PostingsAndFreq[2];++ org.apache.lucene.index.Terms bodyTerms = context.reader().terms("body");+ org.apache.lucene.index.TermsEnum te1 = bodyTerms.iterator();+ te1.seekExact(t1.bytes());+ PostingsEnum pe1 = te1.impacts(PostingsEnum.ALL);++ org.apache.lucene.index.TermsEnum te2 = bodyTerms.iterator();+ te2.seekExact(t2.bytes());+ PostingsEnum pe2 = te2.impacts(PostingsEnum.ALL);++ postings[0] =+ new PhraseQuery.PostingsAndFreq(pe1, (org.apache.lucene.index.ImpactsEnum) pe1, 0, t1);+ postings[1] =+ new PhraseQuery.PostingsAndFreq(pe2, (org.apache.lucene.index.ImpactsEnum) pe2, 1, t2);++ IndexSearcher searcher = new IndexSearcher(reader);+ FieldStats fieldStats = searcher.fieldStats("body");+ assertNotNull(fieldStats);+ TermStates ts1 = TermStates.build(searcher, t1, true);+ TermStates ts2 = TermStates.build(searcher, t2, true);+ assertTrue(ts1.docFreq() > 0);+ assertTrue(ts2.docFreq() > 0);+ SimScorer scorer =+ new BM25Similarity()+ .scorer(+ 1f,+ fieldStats,+ searcher.termStats(t1, ts1.docFreq(), ts1.totalTermFreq()),+ searcher.termStats(t2, ts2.docFreq(), ts2.totalTermFreq()));+ // Slop 2 allows "quick silver fox"+ SloppyPhraseMatcher matcher =+ new SloppyPhraseMatcher(postings, 2, ScoreMode.TOP_SCORES, scorer, 1f, false);++ DocIdSetIterator approximation = matcher.approximation();+ int docId = approximation.nextDoc();+ assertNotEquals(DocIdSetIterator.NO_MORE_DOCS, docId);++ // CALL maxFreq BEFORE resetPositions+ float maxFreq = matcher.maxFreq();+ assertTrue(maxFreq >= 1.0f);++ // Then resetPositions+ matcher.resetPositions();+ assertTrue("Should have a match in doc " + docId, matcher.nextMatch());+ assertFalse(matcher.nextMatch());+ }+ }+ }+}