Skip to content

Apache Lucene open-source search software

Fix docFreq in score calculation after rewrite of boolean query consisting of blended query and boosted term query (#12354)

Rafał Harabień3e077fdc076d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 174b14812a4..5b64315c514 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -120,6 +120,9 @@ Bug Fixes
* GITHUB#15125: Handle inconsistent schema on flush with index sorts (Nhat Nguyen)
+* GITHUB#10309: Do not merge term queries with different docFreq. Fixes wrong scores in some queries that included
+ term query and fuzzy query. (Rafał Harabień)
+
* GITHUB#15434: Don't override boost of conflicting expanded phrases by taking the max boost in markTerminal for FVHighlighter (Luana Fragoso)
* GITHUB#12561: UAX29URLEmailTokenizer matched emails with commas and invalid
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermStates.java b/lucene/core/src/java/org/apache/lucene/index/TermStates.java
index 2321104d30f..511a4433f20 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermStates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermStates.java
@@ -257,4 +257,14 @@ public final class TermStates {
return sb.toString();
}
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof TermStates && ((TermStates) obj).docFreq == docFreq;
+ }
+
+ @Override
+ public int hashCode() {
+ return Integer.hashCode(docFreq);
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
index 6c29c23095b..6d5cc2d9094 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
@@ -340,11 +340,16 @@ public class TermQuery extends Query {
/** Returns true iff <code>other</code> is equal to <code>this</code>. */
@Override
public boolean equals(Object other) {
- return sameClassAs(other) && term.equals(((TermQuery) other).term);
+ return sameClassAs(other) && equalsTo((TermQuery) other);
+ }
+
+ private boolean equalsTo(TermQuery other) {
+ return Objects.equals(term, other.term)
+ && Objects.equals(perReaderTermState, other.perReaderTermState);
}
@Override
public int hashCode() {
- return classHash() ^ term.hashCode();
+ return classHash() ^ term.hashCode() ^ Objects.hashCode(perReaderTermState);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java
index 78fab5a5633..1ef784f7639 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java
@@ -17,8 +17,11 @@
package org.apache.lucene.search;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilteredTermsEnum;
import org.apache.lucene.index.IndexReader;
@@ -32,6 +35,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -131,8 +135,10 @@ public class TestMultiTermQueryRewrites extends LuceneTestCase {
System.out.println("multi segment with duplicates: " + q3);
}
assertEquals("The multi-segment case must produce same rewritten query", q1, q2);
- assertEquals(
- "The multi-segment case with duplicates must produce same rewritten query", q1, q3);
+ assertNotEquals(
+ "The multi-segment case with duplicates must produce rewritten query with different term freqs",
+ q1,
+ q3);
checkBooleanQueryOrder(q1);
checkBooleanQueryOrder(q2);
checkBooleanQueryOrder(q3);
@@ -270,4 +276,46 @@ public class TestMultiTermQueryRewrites extends LuceneTestCase {
checkNoMaxClauseLimitation(new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(1024));
checkNoMaxClauseLimitation(new MultiTermQuery.TopTermsBoostOnlyBooleanQueryRewrite(1024));
}
+
+ public void testDocFreqAfterTermAndFuzzyRewrite() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ for (String s : new String[] {"aaaa", "aaab", "aaab"}) {
+ Document doc = new Document();
+ doc.add(new StringField("text", s, Field.Store.NO));
+ w.addDocument(doc);
+ }
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ BooleanQuery bq =
+ new BooleanQuery.Builder()
+ .add(
+ new BoostQuery(new TermQuery(new Term("text", "aaaa")), 2.0f),
+ BooleanClause.Occur.SHOULD)
+ .add(new FuzzyQuery(new Term("text", "aaaa"), 1), BooleanClause.Occur.SHOULD)
+ .build();
+
+ BooleanQuery rewritten = (BooleanQuery) searcher.rewrite(bq);
+ List<TermQuery> termQueries = extractTermQueries(rewritten, new ArrayList<>());
+ // check if "aaaa" term query was kept and not merged with blended term query
+ // with wrong docFreq produced from fuzzy query
+ boolean found =
+ termQueries.stream()
+ .anyMatch(q -> q.getTerm().text().equals("aaaa") && q.getTermStates() == null);
+ assertTrue(found);
+
+ IOUtils.close(reader, w, dir);
+ }
+
+ private List<TermQuery> extractTermQueries(Query query, List<TermQuery> result) {
+ if (query instanceof BooleanQuery) {
+ ((BooleanQuery) query).clauses().forEach(c -> extractTermQueries(c.query(), result));
+ } else if (query instanceof BoostQuery) {
+ extractTermQueries(((BoostQuery) query).getQuery(), result);
+ } else if (query instanceof TermQuery) {
+ result.add((TermQuery) query);
+ }
+ return result;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
index 513dde82a04..f87acb83bdf 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
@@ -57,7 +57,7 @@ public class TestTermQuery extends LuceneTestCase {
try (MultiReader multiReader = new MultiReader()) {
context = multiReader.getContext();
IndexSearcher searcher = new IndexSearcher(context);
- QueryUtils.checkEqual(
+ QueryUtils.checkUnequal(
new TermQuery(new Term("foo", "bar")),
new TermQuery(
new Term("foo", "bar"), TermStates.build(searcher, new Term("foo", "bar"), true)));