Skip to content

Apache Lucene open-source search software

Fix LogOddsFusionQuery.rewrite() to filter out MatchNoDocsQuery and preserve parallel arrays (#16106)

Prithvi S3c607cfb3882
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 1f9dbae99a7..2bbfea5a2c6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -338,6 +338,10 @@ Improvements
Add per-signal weights and logit normalization to LogOddsFusionQuery.
(Jaepil Jeong)
+* GITHUB#16106: Fix LogOddsFusionQuery.rewrite() to filter out MatchNoDocsQuery clauses and
+ correctly preserve per-signal weights and logit normalization bounds for the surviving clauses.
+ (Prithvi S)
+
* GITHUB#15823: Implement method to add all stream elements into a PriorityQueue.
Call PriorityQueue#addAll with mapped stream in DisjunctionMaxBulkScorer's constructor. (Zhou Hui)
diff --git a/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.java b/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.java
index 5fb575d5f28..f311ecf9055 100644
--- a/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/LogOddsFusionQuery.java
@@ -362,17 +362,57 @@ public final class LogOddsFusionQuery extends Query implements Iterable<Query> {
boolean actuallyRewritten = false;
List<Query> rewrittenClauses = new ArrayList<>();
- for (Query sub : orderedClauses) {
+ List<Float> newWeights = signalWeights != null ? new ArrayList<>() : null;
+ List<Float> newLogitMin = logitMin != null ? new ArrayList<>() : null;
+ List<Float> newLogitMax = logitMax != null ? new ArrayList<>() : null;
+
+ for (int i = 0; i < orderedClauses.size(); i++) {
+ Query sub = orderedClauses.get(i);
Query rewrittenSub = sub.rewrite(indexSearcher);
- actuallyRewritten |= rewrittenSub != sub;
- rewrittenClauses.add(rewrittenSub);
+ if (rewrittenSub != sub || sub.getClass() == MatchNoDocsQuery.class) {
+ actuallyRewritten = true;
+ }
+ if (rewrittenSub.getClass() != MatchNoDocsQuery.class) {
+ rewrittenClauses.add(rewrittenSub);
+ if (newWeights != null) {
+ newWeights.add(signalWeights[i]);
+ }
+ if (newLogitMin != null) {
+ newLogitMin.add(logitMin[i]);
+ newLogitMax.add(logitMax[i]);
+ }
+ }
}
- if (actuallyRewritten) {
- return new LogOddsFusionQuery(rewrittenClauses, alpha, signalWeights, logitMin, logitMax);
+ if (actuallyRewritten == false) {
+ return super.rewrite(indexSearcher);
+ }
+ if (rewrittenClauses.isEmpty()) {
+ return new MatchNoDocsQuery("empty LogOddsFusionQuery");
+ }
+ if (rewrittenClauses.size() == 1) {
+ return rewrittenClauses.get(0);
+ }
+
+ float[] filteredWeights = toFloatArray(newWeights);
+ if (filteredWeights != null) {
+ float sum = 0;
+ for (float w : filteredWeights) {
+ sum += w;
+ }
+ if (sum > 0) {
+ for (int i = 0; i < filteredWeights.length; i++) {
+ filteredWeights[i] /= sum;
+ }
+ }
}
- return super.rewrite(indexSearcher);
+ return new LogOddsFusionQuery(
+ rewrittenClauses,
+ alpha,
+ filteredWeights,
+ toFloatArray(newLogitMin),
+ toFloatArray(newLogitMax));
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.java
index e5d1b59eff9..79606a7f977 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestLogOddsFusionQuery.java
@@ -129,7 +129,75 @@ public class TestLogOddsFusionQuery extends LuceneTestCase {
public void testEmptyClauseRewrite() throws Exception {
LogOddsFusionQuery loq = new LogOddsFusionQuery(Collections.emptyList(), 0.5f);
Query rewritten = searcher.rewrite(loq);
- assertTrue("empty should rewrite to MatchNoDocsQuery", rewritten instanceof MatchNoDocsQuery);
+ assertEquals(MatchNoDocsQuery.INSTANCE, rewritten);
+ }
+
+ public void testRewriteAllMatchNoDocs() throws Exception {
+ Query q1 = new MatchNoDocsQuery("q1");
+ Query q2 = new MatchNoDocsQuery("q2");
+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.5f);
+ Query rewritten = searcher.rewrite(loq);
+ assertEquals(MatchNoDocsQuery.INSTANCE, rewritten);
+ }
+
+ public void testRewriteSingleSurvivor() throws Exception {
+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));
+ Query q2 = new MatchNoDocsQuery("q2");
+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.5f);
+ Query rewritten = searcher.rewrite(loq);
+ assertEquals(q1, rewritten);
+ }
+
+ public void testRewriteFilterMatchNoDocs() throws Exception {
+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));
+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));
+ Query q3 = new MatchNoDocsQuery("q3");
+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.5f);
+ Query rewritten = searcher.rewrite(loq);
+ LogOddsFusionQuery expected = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.5f);
+ assertEquals(expected, rewritten);
+ }
+
+ public void testRewriteFilterMatchNoDocsWithAlpha() throws Exception {
+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));
+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));
+ Query q3 = new MatchNoDocsQuery("q3");
+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.3f);
+ Query rewritten = searcher.rewrite(loq);
+ LogOddsFusionQuery expected = new LogOddsFusionQuery(Arrays.asList(q1, q2), 0.3f);
+ assertEquals(expected, rewritten);
+ }
+
+ public void testRewriteFilterMatchNoDocsWithWeights() throws Exception {
+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));
+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));
+ Query q3 = new MatchNoDocsQuery("q3");
+ float[] weights = {0.4f, 0.4f, 0.2f};
+ LogOddsFusionQuery loq = new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.5f, weights);
+ Query rewritten = searcher.rewrite(loq);
+ assertTrue("should rewrite to LogOddsFusionQuery", rewritten instanceof LogOddsFusionQuery);
+ LogOddsFusionQuery result = (LogOddsFusionQuery) rewritten;
+ assertEquals(2, result.getClauses().size());
+ float[] resultWeights = result.getWeights();
+ assertNotNull(resultWeights);
+ assertEquals(2, resultWeights.length);
+ assertEquals(0.5f, resultWeights[0], 1e-6f);
+ assertEquals(0.5f, resultWeights[1], 1e-6f);
+ }
+
+ public void testRewriteFilterMatchNoDocsWithLogitBounds() throws Exception {
+ Query q1 = bayesian(new TermQuery(new Term("body", "alpha")));
+ Query q2 = bayesian(new TermQuery(new Term("body", "beta")));
+ Query q3 = new MatchNoDocsQuery("q3");
+ float[] weights = {0.4f, 0.4f, 0.2f};
+ float[] logitMin = {-5f, -4f, -3f};
+ float[] logitMax = {5f, 4f, 3f};
+ LogOddsFusionQuery loq =
+ new LogOddsFusionQuery(Arrays.asList(q1, q2, q3), 0.5f, weights, logitMin, logitMax);
+ Query rewritten = searcher.rewrite(loq);
+ assertTrue("should rewrite to LogOddsFusionQuery", rewritten instanceof LogOddsFusionQuery);
+ LogOddsFusionQuery result = (LogOddsFusionQuery) rewritten;
+ assertEquals(2, result.getClauses().size());
}
public void testAlphaValues() throws Exception {