Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9635: BM25FQuery - Mask encoded norm long value in array lookup to avoid negative norms in long documents (#2138)

yiluncui894b6b5c888a
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ac45db983df..826d86b23f2 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -189,6 +189,9 @@ Bug fixes
* LUCENE-9365: FuzzyQuery was missing matches when prefix length was equal to the term length
(Mark Harwood, Mike Drob)
+* LUCENE-9635: BM25FQuery - Mask encoded norm long value in array lookup.
+ (Yilun Cui)
+
Other
* LUCENE-9631: Properly override slice() on subclasses of OffsetRange. (Dawid Weiss)
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/MultiNormsLeafSimScorer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/MultiNormsLeafSimScorer.java
index a2dc38125ae..d17ac05ce85 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/MultiNormsLeafSimScorer.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/MultiNormsLeafSimScorer.java
@@ -128,7 +128,7 @@ final class MultiNormsLeafSimScorer {
for (int i = 0; i < normsArr.length; i++) {
boolean found = normsArr[i].advanceExact(target);
assert found;
- normValue += weightArr[i] * LENGTH_TABLE[(byte) normsArr[i].longValue()];
+ normValue += weightArr[i] * LENGTH_TABLE[Byte.toUnsignedInt((byte) normsArr[i].longValue())];
}
current = SmallFloat.intToByte4(Math.round(normValue));
return true;
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestBM25FQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestBM25FQuery.java
index 18ed94c52ed..96bfc93d352 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestBM25FQuery.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestBM25FQuery.java
@@ -16,32 +16,36 @@
*/
package org.apache.lucene.sandbox.search;
-import java.io.IOException;
-
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.sandbox.search.BM25FQuery;
import org.apache.lucene.search.CheckHits;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import java.io.IOException;
+
public class TestBM25FQuery extends LuceneTestCase {
public void testInvalid() {
BM25FQuery.Builder builder = new BM25FQuery.Builder();
@@ -176,4 +180,80 @@ public class TestBM25FQuery extends LuceneTestCase {
w.close();
dir.close();
}
+
+ public void testDocWithNegativeNorms() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(new NegativeNormSimilarity());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ String queryString = "foo";
+
+ Document doc = new Document();
+ //both fields must contain tokens that match the query string "foo"
+ doc.add(new TextField("f", "foo", Store.NO));
+ doc.add(new TextField("g", "foo baz", Store.NO));
+ w.addDocument(doc);
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ BM25FQuery query = new BM25FQuery.Builder()
+ .addField("f")
+ .addField("g")
+ .addTerm(new BytesRef(queryString))
+ .build();
+ TopDocs topDocs = searcher.search(query, 10);
+ CheckHits.checkDocIds("queried docs do not match", new int[]{0}, topDocs.scoreDocs);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMultipleDocsNegativeNorms() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(new NegativeNormSimilarity());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ String queryString = "foo";
+
+ Document doc0 = new Document();
+ doc0.add(new TextField("f", "foo", Store.NO));
+ doc0.add(new TextField("g", "foo baz", Store.NO));
+ w.addDocument(doc0);
+
+ Document doc1 = new Document();
+ // add another match on the query string to the second doc
+ doc1.add(new TextField("f", "foo is foo", Store.NO));
+ doc1.add(new TextField("g", "foo baz", Store.NO));
+ w.addDocument(doc1);
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ BM25FQuery query = new BM25FQuery.Builder()
+ .addField("f")
+ .addField("g")
+ .addTerm(new BytesRef(queryString))
+ .build();
+ TopDocs topDocs = searcher.search(query, 10);
+ //Return doc1 ahead of doc0 since its tf is higher
+ CheckHits.checkDocIds("queried docs do not match", new int[]{1,0}, topDocs.scoreDocs);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ private static final class NegativeNormSimilarity extends Similarity {
+ @Override
+ public long computeNorm(FieldInvertState state) {
+ return -128;
+ }
+
+ @Override
+ public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new BM25Similarity().scorer(boost, collectionStats, termStats);
+ }
+ }
}