Apache Lucene open-source search software
Fix IBSimilarity with DistributionSPL producing negative scores (#16139)
Prithvi S81b810d73f33
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c17c33c1935..5944d5453a6 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -561,6 +561,9 @@ Bug Fixes* GITHUB#16577: Fix incorrect position increment generated by FixedShingleFilter. (Kishor Kharbas)+* GITHUB#16139: Fix IBSimilarity with DistributionSPL producing negative scores due to+ floating-point rounding errors. (Prithvi S)+Build---------------------diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.javaindex 11f451eef24..a42de7cda2b 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java@@ -21,8 +21,10 @@ package org.apache.lucene.search.similarities;* in the original paper.** <p>Unlike for DFR, the natural logarithm is used, as it is faster to compute and the original- * paper does not express any preference to a specific base. WARNING: this model currently returns- * infinite scores for very small tf values and negative scores for very large tf values+ * paper does not express any preference to a specific base.+ *+ * <p>WARNING: this model may return very large (effectively infinite) scores for very small tf+ * values.** @lucene.experimental*/@@ -55,7 +57,9 @@ public class DistributionSPL extends Distribution {}}- return -Math.log((pow - lambda) / (1 - lambda));+ double fraction = (pow - lambda) / (1 - lambda);+ // Clamp to avoid negative scores due to floating-point rounding errors+ return Math.max(0.0, -Math.log(fraction));}@Overridediff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.javaindex 92b6924f87e..e0c21b29252 100644--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java@@ -22,4 +22,43 @@ public class TestDistributionSPL extends DistributionTestCase {protected Distribution getDistribution() {return new DistributionSPL();}++ /** Test that DistributionSPL never returns negative scores, even for extreme inputs. */+ public void testNoNegativeScores() {+ DistributionSPL dist = new DistributionSPL();+ BasicStats stats = new BasicStats("field", 1.0);+ stats.setNumberOfDocuments(100);+ stats.setDocFreq(50);+ stats.setTotalTermFreq(500);+ stats.setNumberOfFieldTokens(5000);+ stats.setAvgFieldLength(50);++ // Test lambda values extremely close to 1, which can cause floating-point+ // rounding artifacts that produce -0.0 without the clamping fix.+ for (double lambda :+ new double[] {+ 0.99,+ 0.999,+ 0.9999,+ 0.99999,+ 0.999999,+ Math.nextDown(1.0),+ Math.nextUp(1.0),+ 1.001,+ 1.01,+ 1.1+ }) {+ for (double tfn : new double[] {1e-10, 1e-5, 0.1, 1, 10, 1e6, 1e10, 1e15}) {+ double score = dist.score(stats, tfn, lambda);+ assertTrue(+ "score must be non-negative for lambda="+ + lambda+ + ", tfn="+ + tfn+ + ", but got "+ + score,+ score >= 0);+ }+ }+ }}