Skip to content

Apache Lucene open-source search software

Fix IBSimilarity with DistributionSPL producing negative scores (#16139)

Prithvi S81b810d73f33
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c17c33c1935..5944d5453a6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -561,6 +561,9 @@ Bug Fixes
* GITHUB#16577: Fix incorrect position increment generated by FixedShingleFilter. (Kishor Kharbas)
+* GITHUB#16139: Fix IBSimilarity with DistributionSPL producing negative scores due to
+ floating-point rounding errors. (Prithvi S)
+
Build
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java
index 11f451eef24..a42de7cda2b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java
@@ -21,8 +21,10 @@ package org.apache.lucene.search.similarities;
* in the original paper.
*
* <p>Unlike for DFR, the natural logarithm is used, as it is faster to compute and the original
- * paper does not express any preference to a specific base. WARNING: this model currently returns
- * infinite scores for very small tf values and negative scores for very large tf values
+ * paper does not express any preference to a specific base.
+ *
+ * <p>WARNING: this model may return very large (effectively infinite) scores for very small tf
+ * values.
*
* @lucene.experimental
*/
@@ -55,7 +57,9 @@ public class DistributionSPL extends Distribution {
}
}
- return -Math.log((pow - lambda) / (1 - lambda));
+ double fraction = (pow - lambda) / (1 - lambda);
+ // Clamp to avoid negative scores due to floating-point rounding errors
+ return Math.max(0.0, -Math.log(fraction));
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java
index 92b6924f87e..e0c21b29252 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java
@@ -22,4 +22,43 @@ public class TestDistributionSPL extends DistributionTestCase {
protected Distribution getDistribution() {
return new DistributionSPL();
}
+
+ /** Test that DistributionSPL never returns negative scores, even for extreme inputs. */
+ public void testNoNegativeScores() {
+ DistributionSPL dist = new DistributionSPL();
+ BasicStats stats = new BasicStats("field", 1.0);
+ stats.setNumberOfDocuments(100);
+ stats.setDocFreq(50);
+ stats.setTotalTermFreq(500);
+ stats.setNumberOfFieldTokens(5000);
+ stats.setAvgFieldLength(50);
+
+ // Test lambda values extremely close to 1, which can cause floating-point
+ // rounding artifacts that produce -0.0 without the clamping fix.
+ for (double lambda :
+ new double[] {
+ 0.99,
+ 0.999,
+ 0.9999,
+ 0.99999,
+ 0.999999,
+ Math.nextDown(1.0),
+ Math.nextUp(1.0),
+ 1.001,
+ 1.01,
+ 1.1
+ }) {
+ for (double tfn : new double[] {1e-10, 1e-5, 0.1, 1, 10, 1e6, 1e10, 1e15}) {
+ double score = dist.score(stats, tfn, lambda);
+ assertTrue(
+ "score must be non-negative for lambda="
+ + lambda
+ + ", tfn="
+ + tfn
+ + ", but got "
+ + score,
+ score >= 0);
+ }
+ }
+ }
}