Apache Lucene open-source search software
Fix docFreq in score calculation after rewrite of boolean query consisting of blended query and boosted term query (#12354)
Rafał Harabień3e077fdc076d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 174b14812a4..5b64315c514 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -120,6 +120,9 @@ Bug Fixes* GITHUB#15125: Handle inconsistent schema on flush with index sorts (Nhat Nguyen)+* GITHUB#10309: Do not merge term queries with different docFreq. Fixes wrong scores in some queries that included+ term query and fuzzy query. (Rafał Harabień)+* GITHUB#15434: Don't override boost of conflicting expanded phrases by taking the max boost in markTerminal for FVHighlighter (Luana Fragoso)* GITHUB#12561: UAX29URLEmailTokenizer matched emails with commas and invaliddiff --git a/lucene/core/src/java/org/apache/lucene/index/TermStates.java b/lucene/core/src/java/org/apache/lucene/index/TermStates.javaindex 2321104d30f..511a4433f20 100644--- a/lucene/core/src/java/org/apache/lucene/index/TermStates.java+++ b/lucene/core/src/java/org/apache/lucene/index/TermStates.java@@ -257,4 +257,14 @@ public final class TermStates {return sb.toString();}++ @Override+ public boolean equals(Object obj) {+ return obj instanceof TermStates && ((TermStates) obj).docFreq == docFreq;+ }++ @Override+ public int hashCode() {+ return Integer.hashCode(docFreq);+ }}diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.javaindex 6c29c23095b..6d5cc2d9094 100644--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java@@ -340,11 +340,16 @@ public class TermQuery extends Query {/** Returns true iff <code>other</code> is equal to <code>this</code>. */@Overridepublic boolean equals(Object other) {- return sameClassAs(other) && term.equals(((TermQuery) other).term);+ return sameClassAs(other) && equalsTo((TermQuery) other);+ }++ private boolean equalsTo(TermQuery other) {+ return Objects.equals(term, other.term)+ && Objects.equals(perReaderTermState, other.perReaderTermState);}@Overridepublic int hashCode() {- return classHash() ^ term.hashCode();+ return classHash() ^ term.hashCode() ^ Objects.hashCode(perReaderTermState);}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.javaindex 78fab5a5633..1ef784f7639 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java@@ -17,8 +17,11 @@package org.apache.lucene.search;import java.io.IOException;+import java.util.ArrayList;+import java.util.List;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;+import org.apache.lucene.document.StringField;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.FilteredTermsEnum;import org.apache.lucene.index.IndexReader;@@ -32,6 +35,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.util.AttributeSource;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;import org.junit.AfterClass;import org.junit.BeforeClass;@@ -131,8 +135,10 @@ public class TestMultiTermQueryRewrites extends LuceneTestCase {System.out.println("multi segment with duplicates: " + q3);}assertEquals("The multi-segment case must produce same rewritten query", q1, q2);- assertEquals(- "The multi-segment case with duplicates must produce same rewritten query", q1, q3);+ assertNotEquals(+ "The multi-segment case with duplicates must produce rewritten query with different term freqs",+ q1,+ q3);checkBooleanQueryOrder(q1);checkBooleanQueryOrder(q2);checkBooleanQueryOrder(q3);@@ -270,4 +276,46 @@ public class TestMultiTermQueryRewrites extends LuceneTestCase {checkNoMaxClauseLimitation(new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(1024));checkNoMaxClauseLimitation(new MultiTermQuery.TopTermsBoostOnlyBooleanQueryRewrite(1024));}++ public void testDocFreqAfterTermAndFuzzyRewrite() throws Exception {+ Directory dir = newDirectory();+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);+ for (String s : new String[] {"aaaa", "aaab", "aaab"}) {+ Document doc = new Document();+ doc.add(new StringField("text", s, Field.Store.NO));+ w.addDocument(doc);+ }++ DirectoryReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);+ BooleanQuery bq =+ new BooleanQuery.Builder()+ .add(+ new BoostQuery(new TermQuery(new Term("text", "aaaa")), 2.0f),+ BooleanClause.Occur.SHOULD)+ .add(new FuzzyQuery(new Term("text", "aaaa"), 1), BooleanClause.Occur.SHOULD)+ .build();++ BooleanQuery rewritten = (BooleanQuery) searcher.rewrite(bq);+ List<TermQuery> termQueries = extractTermQueries(rewritten, new ArrayList<>());+ // check if "aaaa" term query was kept and not merged with blended term query+ // with wrong docFreq produced from fuzzy query+ boolean found =+ termQueries.stream()+ .anyMatch(q -> q.getTerm().text().equals("aaaa") && q.getTermStates() == null);+ assertTrue(found);++ IOUtils.close(reader, w, dir);+ }++ private List<TermQuery> extractTermQueries(Query query, List<TermQuery> result) {+ if (query instanceof BooleanQuery) {+ ((BooleanQuery) query).clauses().forEach(c -> extractTermQueries(c.query(), result));+ } else if (query instanceof BoostQuery) {+ extractTermQueries(((BoostQuery) query).getQuery(), result);+ } else if (query instanceof TermQuery) {+ result.add((TermQuery) query);+ }+ return result;+ }}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.javaindex 513dde82a04..f87acb83bdf 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java@@ -57,7 +57,7 @@ public class TestTermQuery extends LuceneTestCase {try (MultiReader multiReader = new MultiReader()) {context = multiReader.getContext();IndexSearcher searcher = new IndexSearcher(context);- QueryUtils.checkEqual(+ QueryUtils.checkUnequal(new TermQuery(new Term("foo", "bar")),new TermQuery(new Term("foo", "bar"), TermStates.build(searcher, new Term("foo", "bar"), true)));