Apache Lucene open-source search software
Fix LogOddsFusionQuery.rewrite() to filter out MatchNoDocsQuery and preserve parallel arrays (#16106)
Prithvi S3c607cfb3882
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 1f9dbae99a7..2bbfea5a2c6 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -338,6 +338,10 @@ ImprovementsAdd per-signal weights and logit normalization to LogOddsFusionQuery.(Jaepil Jeong)+* GITHUB#16106: Fix LogOddsFusionQuery.rewrite() to filter out MatchNoDocsQuery clauses and+ correctly preserve per-signal weights and logit normalization bounds for the surviving clauses.+ (Prithvi S)+* GITHUB#15823: Implement method to add all stream elements into a PriorityQueue.Call PriorityQueue#addAll with mapped stream in DisjunctionMaxBulkScorer's constructor. (Zhou Hui)diff --git a/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.java b/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.javaindex 5fb575d5f28..f311ecf9055 100644--- a/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.java@@ -362,17 +362,57 @@ public final class LogOddsFusionQuery extends Query implements Iterable<Query> {boolean actuallyRewritten = false;List<Query> rewrittenClauses = new ArrayList<>();- for (Query sub : orderedClauses) {+ List<Float> newWeights = signalWeights != null ? new ArrayList<>() : null;+ List<Float> newLogitMin = logitMin != null ? new ArrayList<>() : null;+ List<Float> newLogitMax = logitMax != null ? new ArrayList<>() : null;++ for (int i = 0; i < orderedClauses.size(); i++) {+ Query sub = orderedClauses.get(i);Query rewrittenSub = sub.rewrite(indexSearcher);- actuallyRewritten |= rewrittenSub != sub;- rewrittenClauses.add(rewrittenSub);+ if (rewrittenSub != sub || sub.getClass() == MatchNoDocsQuery.class) {+ actuallyRewritten = true;+ }+ if (rewrittenSub.getClass() != MatchNoDocsQuery.class) {+ rewrittenClauses.add(rewrittenSub);+ if (newWeights != null) {+ newWeights.add(signalWeights[i]);+ }+ if (newLogitMin != null) {+ newLogitMin.add(logitMin[i]);+ newLogitMax.add(logitMax[i]);+ }+ }}- if (actuallyRewritten) {- return new LogOddsFusionQuery(rewrittenClauses, alpha, signalWeights, logitMin, logitMax);+ if (actuallyRewritten == false) {+ return super.rewrite(indexSearcher);+ }+ if (rewrittenClauses.isEmpty()) {+ return new MatchNoDocsQuery("empty LogOddsFusionQuery");+ }+ if (rewrittenClauses.size() == 1) {+ return rewrittenClauses.get(0);+ }++ float[] filteredWeights = toFloatArray(newWeights);+ if (filteredWeights != null) {+ float sum = 0;+ for (float w : filteredWeights) {+ sum += w;+ }+ if (sum > 0) {+ for (int i = 0; i < filteredWeights.length; i++) {+ filteredWeights[i] /= sum;+ }+ }}- return super.rewrite(indexSearcher);+ return new LogOddsFusionQuery(+ rewrittenClauses,+ alpha,+ filteredWeights,+ toFloatArray(newLogitMin),+ toFloatArray(newLogitMax));}@Overridediff --git a/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.javaindex e5d1b59eff9..79606a7f977 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.java@@ -129,7 +129,75 @@ public class TestLogOddsFusionQuery extends LuceneTestCase {public void testEmptyClauseRewrite() throws Exception {LogOddsFusionQuery loq = new LogOddsFusionQuery(Collections.emptyList(), 0.5f);Query rewritten = searcher.rewrite(loq);- assertTrue("empty should rewrite to MatchNoDocsQuery", rewritten instanceof MatchNoDocsQuery);+ assertEquals(MatchNoDocsQuery.INSTANCE, rewritten);+ }++ public void testRewriteAllMatchNoDocs() throws Exception {+ Query q1 = new MatchNoDocsQuery("q1");+ Query q2 = new MatchNoDocsQuery("q2");+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.5f);+ Query rewritten = searcher.rewrite(loq);+ assertEquals(MatchNoDocsQuery.INSTANCE, rewritten);+ }++ public void testRewriteSingleSurvivor() throws Exception {+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));+ Query q2 = new MatchNoDocsQuery("q2");+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.5f);+ Query rewritten = searcher.rewrite(loq);+ assertEquals(q1, rewritten);+ }++ public void testRewriteFilterMatchNoDocs() throws Exception {+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));+ Query q3 = new MatchNoDocsQuery("q3");+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.5f);+ Query rewritten = searcher.rewrite(loq);+ LogOddsFusionQuery expected = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.5f);+ assertEquals(expected, rewritten);+ }++ public void testRewriteFilterMatchNoDocsWithAlpha() throws Exception {+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));+ Query q3 = new MatchNoDocsQuery("q3");+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.3f);+ Query rewritten = searcher.rewrite(loq);+ LogOddsFusionQuery expected = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.3f);+ assertEquals(expected, rewritten);+ }++ public void testRewriteFilterMatchNoDocsWithWeights() throws Exception {+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));+ Query q3 = new MatchNoDocsQuery("q3");+ float[] weights = {0.4f, 0.4f, 0.2f};+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.5f, weights);+ Query rewritten = searcher.rewrite(loq);+ assertTrue("should rewrite to LogOddsFusionQuery", rewritten instanceof LogOddsFusionQuery);+ LogOddsFusionQuery result = (LogOddsFusionQuery) rewritten;+ assertEquals(2, result.getClauses().size());+ float[] resultWeights = result.getWeights();+ assertNotNull(resultWeights);+ assertEquals(2, resultWeights.length);+ assertEquals(0.5f, resultWeights[0], 1e-6f);+ assertEquals(0.5f, resultWeights[1], 1e-6f);+ }++ public void testRewriteFilterMatchNoDocsWithLogitBounds() throws Exception {+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));+ Query q3 = new MatchNoDocsQuery("q3");+ float[] weights = {0.4f, 0.4f, 0.2f};+ float[] logitMin = {-5f, -4f, -3f};+ float[] logitMax = {5f, 4f, 3f};+ LogOddsFusionQuery loq =+ new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.5f, weights, logitMin, logitMax);+ Query rewritten = searcher.rewrite(loq);+ assertTrue("should rewrite to LogOddsFusionQuery", rewritten instanceof LogOddsFusionQuery);+ LogOddsFusionQuery result = (LogOddsFusionQuery) rewritten;+ assertEquals(2, result.getClauses().size());}public void testAlphaValues() throws Exception {