Apache Lucene open-source search software
Speed up top-k retrieval on filtered conjunctions. (#13994)
Adrien Grand4400d552970e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 66f13d264e5..547b8308145 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -93,6 +93,9 @@ Optimizationslongs that would pack two integers. We are now moving back to integers to beable to take advantage of 2x more lanes with the vector API. (Adrien Grand)+* GITHUB#13994: Speed up top-k retrieval of filtered conjunctions.+ (Adrien Grand)+Bug Fixes---------------------* GITHUB#13832: Fixed an issue where the DefaultPassageFormatter.format method did not format passages as intendeddiff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java b/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.javaindex 0d6daa07ba1..b50b0530a2d 100644--- a/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java@@ -268,6 +268,11 @@ public class BooleanQuery extends Query implements Iterable<BooleanClause> {return new MatchNoDocsQuery("empty BooleanQuery");}+ // Queries with no positive clauses have no matches+ if (clauses.size() == clauseSets.get(Occur.MUST_NOT).size()) {+ return new MatchNoDocsQuery("pure negative BooleanQuery");+ }+// optimize 1-clause queriesif (clauses.size() == 1) {BooleanClause c = clauses.get(0);@@ -283,8 +288,6 @@ public class BooleanQuery extends Query implements Iterable<BooleanClause> {// no scoring clauses, so return a score of 0return new BoostQuery(new ConstantScoreQuery(query), 0);case MUST_NOT:- // no positive clauses- return new MatchNoDocsQuery("pure negative BooleanQuery");default:throw new AssertionError();}@@ -539,8 +542,7 @@ public class BooleanQuery extends Query implements Iterable<BooleanClause> {builder.setMinimumNumberShouldMatch(minimumNumberShouldMatch);boolean actuallyRewritten = false;for (BooleanClause clause : clauses) {- if (clause.occur() == Occur.SHOULD && clause.query() instanceof BooleanQuery) {- BooleanQuery innerQuery = (BooleanQuery) clause.query();+ if (clause.occur() == Occur.SHOULD && clause.query() instanceof BooleanQuery innerQuery) {if (innerQuery.isPureDisjunction()) {actuallyRewritten = true;for (BooleanClause innerClause : innerQuery.clauses()) {@@ -558,6 +560,46 @@ public class BooleanQuery extends Query implements Iterable<BooleanClause> {}}+ // Inline required / prohibited clauses. This helps run filtered conjunctive queries more+ // efficiently by providing all clauses to the block-max AND scorer.+ {+ BooleanQuery.Builder builder = new BooleanQuery.Builder();+ builder.setMinimumNumberShouldMatch(minimumNumberShouldMatch);+ boolean actuallyRewritten = false;+ for (BooleanClause outerClause : clauses) {+ if (outerClause.isRequired() && outerClause.query() instanceof BooleanQuery innerQuery) {+ // Inlining prohibited clauses is not legal if the query is a pure negation, since pure+ // negations have no matches. It works because the inner BooleanQuery would have first+ // rewritten to a MatchNoDocsQuery if it only had prohibited clauses.+ assert innerQuery.getClauses(Occur.MUST_NOT).size() != innerQuery.clauses().size();+ if (innerQuery.getMinimumNumberShouldMatch() == 0+ && innerQuery.getClauses(Occur.SHOULD).isEmpty()) {++ actuallyRewritten = true;+ for (BooleanClause innerClause : innerQuery) {+ Occur innerOccur = innerClause.occur();+ if (innerOccur == Occur.FILTER+ || innerOccur == Occur.MUST_NOT+ || outerClause.occur() == Occur.MUST) {+ builder.add(innerClause);+ } else {+ assert outerClause.occur() == Occur.FILTER && innerOccur == Occur.MUST;+ // In this case we need to change the occur of the inner query from MUST to FILTER.+ builder.add(innerClause.query(), Occur.FILTER);+ }+ }+ } else {+ builder.add(outerClause);+ }+ } else {+ builder.add(outerClause);+ }+ }+ if (actuallyRewritten) {+ return builder.build();+ }+ }+// SHOULD clause count less than or equal to minimumNumberShouldMatch// Important(this can only be processed after nested clauses have been flattened){diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.javaindex a8169ad227f..7a53bc9a485 100644--- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java@@ -333,10 +333,15 @@ final class BooleanScorerSupplier extends ScorerSupplier {requiredScoring.add(ss.get(leadCost));}if (scoreMode == ScoreMode.TOP_SCORES- && requiredNoScoring.isEmpty()&& requiredScoring.size() > 1// Only specialize top-level conjunctions for clauses that don't have a two-phase iterator.+ && requiredNoScoring.stream().map(Scorer::twoPhaseIterator).allMatch(Objects::isNull)&& requiredScoring.stream().map(Scorer::twoPhaseIterator).allMatch(Objects::isNull)) {+ // Turn all filters into scoring clauses with a score of zero, so that+ // BlockMaxConjunctionBulkScorer is applicable.+ for (Scorer filter : requiredNoScoring) {+ requiredScoring.add(new ConstantScoreScorer(0f, ScoreMode.COMPLETE, filter.iterator()));+ }return new BlockMaxConjunctionBulkScorer(maxDoc, requiredScoring);}if (scoreMode != ScoreMode.TOP_SCORESdiff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.javaindex 9083a62e37a..f36c7539c7d 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java@@ -330,8 +330,8 @@ public class TestBooleanRewrites extends LuceneTestCase {int depth = TestUtil.nextInt(random(), 10, 30);TestRewriteQuery rewriteQueryExpected = new TestRewriteQuery();TestRewriteQuery rewriteQuery = new TestRewriteQuery();- Query expectedQuery =- new BooleanQuery.Builder().add(rewriteQueryExpected, Occur.FILTER).build();+ BooleanQuery.Builder expectedQueryBuilder =+ new BooleanQuery.Builder().add(rewriteQueryExpected, Occur.FILTER);Query deepBuilder =new BooleanQuery.Builder().add(rewriteQuery, Occur.SHOULD)@@ -345,21 +345,19 @@ public class TestBooleanRewrites extends LuceneTestCase {.add(tq, Occur.SHOULD).add(deepBuilder, Occur.SHOULD);deepBuilder = bq.build();- BooleanQuery.Builder expectedBq = new BooleanQuery.Builder().add(tq, Occur.FILTER);+ expectedQueryBuilder.add(tq, Occur.FILTER);if (i == depth) {- expectedBq.add(rewriteQuery, Occur.FILTER);- } else {- expectedBq.add(expectedQuery, Occur.FILTER);+ expectedQueryBuilder.add(rewriteQuery, Occur.FILTER);}- expectedQuery = expectedBq.build();}BooleanQuery bq = new BooleanQuery.Builder().add(deepBuilder, Occur.FILTER).build();- expectedQuery = new BoostQuery(new ConstantScoreQuery(expectedQuery), 0.0f);+ Query expectedQuery =+ new BoostQuery(new ConstantScoreQuery(expectedQueryBuilder.build()), 0.0f);Query rewritten = searcher.rewrite(bq);assertEquals(expectedQuery, rewritten);// the SHOULD clauses cause more rewrites because they incrementally change to `MUST` and then- // `FILTER`- assertEquals("Depth=" + depth, depth + 1, rewriteQuery.numRewrites);+ // `FILTER`, plus the flattening of required clauses+ assertEquals("Depth=" + depth, depth * 2, rewriteQuery.numRewrites);}public void testDeeplyNestedBooleanRewrite() throws IOException {@@ -369,27 +367,26 @@ public class TestBooleanRewrites extends LuceneTestCase {int depth = TestUtil.nextInt(random(), 10, 30);TestRewriteQuery rewriteQueryExpected = new TestRewriteQuery();TestRewriteQuery rewriteQuery = new TestRewriteQuery();- Query expectedQuery =- new BooleanQuery.Builder().add(rewriteQueryExpected, Occur.FILTER).build();+ BooleanQuery.Builder expectedQueryBuilder =+ new BooleanQuery.Builder().add(rewriteQueryExpected, Occur.FILTER);Query deepBuilder = new BooleanQuery.Builder().add(rewriteQuery, Occur.MUST).build();for (int i = depth; i > 0; i--) {TermQuery tq = termQueryFunction.apply(i);BooleanQuery.Builder bq =new BooleanQuery.Builder().add(tq, Occur.MUST).add(deepBuilder, Occur.MUST);deepBuilder = bq.build();- BooleanQuery.Builder expectedBq = new BooleanQuery.Builder().add(tq, Occur.FILTER);+ expectedQueryBuilder.add(tq, Occur.FILTER);if (i == depth) {- expectedBq.add(rewriteQuery, Occur.FILTER);- } else {- expectedBq.add(expectedQuery, Occur.FILTER);+ expectedQueryBuilder.add(rewriteQuery, Occur.FILTER);}- expectedQuery = expectedBq.build();}BooleanQuery bq = new BooleanQuery.Builder().add(deepBuilder, Occur.FILTER).build();- expectedQuery = new BoostQuery(new ConstantScoreQuery(expectedQuery), 0.0f);+ Query expectedQuery =+ new BoostQuery(new ConstantScoreQuery(expectedQueryBuilder.build()), 0.0f);Query rewritten = searcher.rewrite(bq);assertEquals(expectedQuery, rewritten);- assertEquals("Depth=" + depth, 1, rewriteQuery.numRewrites);+ // `depth` rewrites because of the flattening+ assertEquals("Depth=" + depth, depth, rewriteQuery.numRewrites);}public void testRemoveMatchAllFilter() throws IOException {@@ -691,6 +688,110 @@ public class TestBooleanRewrites extends LuceneTestCase {assertSame(query, searcher.rewrite(query));}+ public void testFlattenInnerConjunctions() throws IOException {+ IndexSearcher searcher = newSearcher(new MultiReader());++ Query inner =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.MUST)+ .build();+ Query query =+ new BooleanQuery.Builder()+ .add(inner, Occur.MUST)+ .add(new TermQuery(new Term("foo", "baz")), Occur.FILTER)+ .build();+ Query expectedRewritten =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "baz")), Occur.FILTER)+ .build();+ assertEquals(expectedRewritten, searcher.rewrite(query));++ query =+ new BooleanQuery.Builder()+ .setMinimumNumberShouldMatch(0)+ .add(inner, Occur.MUST)+ .add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD)+ .build();+ expectedRewritten =+ new BooleanQuery.Builder()+ .setMinimumNumberShouldMatch(0)+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD)+ .build();+ assertEquals(expectedRewritten, searcher.rewrite(query));++ query =+ new BooleanQuery.Builder()+ .add(inner, Occur.MUST)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST_NOT)+ .build();+ expectedRewritten =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST_NOT)+ .build();+ assertEquals(expectedRewritten, searcher.rewrite(query));++ inner =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.FILTER)+ .build();+ query =+ new BooleanQuery.Builder()+ .add(inner, Occur.MUST)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST)+ .build();+ expectedRewritten =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.FILTER)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST)+ .build();+ assertEquals(expectedRewritten, searcher.rewrite(query));++ inner =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.FILTER)+ .build();+ query =+ new BooleanQuery.Builder()+ .add(inner, Occur.FILTER)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST)+ .build();+ expectedRewritten =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)+ .add(new TermQuery(new Term("foo", "quux")), Occur.FILTER)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST)+ .build();+ assertEquals(expectedRewritten, searcher.rewrite(query));++ inner =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)+ .add(new TermQuery(new Term("foo", "quux")), Occur.MUST_NOT)+ .build();+ query =+ new BooleanQuery.Builder()+ .add(inner, Occur.FILTER)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST)+ .build();+ expectedRewritten =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("foo", "bar")), Occur.FILTER)+ .add(new TermQuery(new Term("foo", "quux")), Occur.MUST_NOT)+ .add(new TermQuery(new Term("foo", "baz")), Occur.MUST)+ .build();+ assertEquals(expectedRewritten, searcher.rewrite(query));+ }+public void testDiscardShouldClauses() throws IOException {IndexSearcher searcher = newSearcher(new MultiReader());