Apache Solr open-source search software
LUCENE-9646: Set BM25Similarity discountOverlaps via the constructor
Patrick Marty227256d951ac
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 9dec819134b..71630e7002b 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -82,6 +82,8 @@ API Changes* LUCENE-9317 LUCENE-9318 LUCENE-9319 LUCENE-9558 LUCENE-9600 : Clean up package name conflictsbetween modules. See MIGRATE.md for details. (David Ryan, Tomoko Uchida, Uwe Schindler, Dawid Weiss)+* LUCENE-9646: Set BM25Similarity discountOverlaps via the constructor (Patrick Marty via Bruno Roustant)+Improvements* LUCENE-9665 LUCENE-9676 Hunspell improvements: support default encoding, improve stemming of all-caps wordsdiff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.mdindex 09cce551fcd..c5317359cfe 100644--- a/lucene/MIGRATE.md+++ b/lucene/MIGRATE.md@@ -7,6 +7,11 @@ NativeUnixDirectory in the misc module was therefore removed and replacedby DirectIODirectory. To use it, you need a JVM and operating system thatsupports Direct IO.+## BM25Similarity.setDiscountOverlaps and LegacyBM25Similarity.setDiscountOverlaps methods removed (LUCENE-9646)++The discount discountOverlaps parameter for both BM25Similarity and LegacyBM25Similarity+is now set by the constructor of those classes.+## Packages in misc module are renamed (LUCENE-9600)Following package names in misc module are renamed.diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.javaindex 1aa37de0241..4d48004c523 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java@@ -34,16 +34,19 @@ import org.apache.lucene.util.SmallFloat;public class BM25Similarity extends Similarity {private final float k1;private final float b;+ private final boolean discountOverlaps;/*** BM25 with the supplied parameter values.** @param k1 Controls non-linear term frequency normalization (saturation).* @param b Controls to what degree document length normalizes tf values.+ * @param discountOverlaps True if overlap tokens (tokens with a position of increment of zero)+ * are discounted from the document's length.* @throws IllegalArgumentException if {@code k1} is infinite or negative, or if {@code b} is not* within the range {@code [0..1]}*/- public BM25Similarity(float k1, float b) {+ public BM25Similarity(float k1, float b, boolean discountOverlaps) {if (Float.isFinite(k1) == false || k1 < 0) {throw new IllegalArgumentException("illegal k1 value: " + k1 + ", must be a non-negative finite value");@@ -53,6 +56,19 @@ public class BM25Similarity extends Similarity {}this.k1 = k1;this.b = b;+ this.discountOverlaps = discountOverlaps;+ }++ /**+ * BM25 with the supplied parameter values.+ *+ * @param k1 Controls non-linear term frequency normalization (saturation).+ * @param b Controls to what degree document length normalizes tf values.+ * @throws IllegalArgumentException if {@code k1} is infinite or negative, or if {@code b} is not+ * within the range {@code [0..1]}+ */+ public BM25Similarity(float k1, float b) {+ this(k1, b, true);}/**@@ -62,9 +78,27 @@ public class BM25Similarity extends Similarity {* <li>{@code k1 = 1.2}* <li>{@code b = 0.75}* </ul>+ *+ * and the supplied parameter value:+ *+ * @param discountOverlaps True if overlap tokens (tokens with a position of increment of zero)+ * are discounted from the document's length.+ */+ public BM25Similarity(boolean discountOverlaps) {+ this(1.2f, 0.75f, discountOverlaps);+ }++ /**+ * BM25 with these default values:+ *+ * <ul>+ * <li>{@code k1 = 1.2}+ * <li>{@code b = 0.75}+ * <li>{@code discountOverlaps = true}+ * </ul>*/public BM25Similarity() {- this(1.2f, 0.75f);+ this(1.2f, 0.75f, true);}/** Implemented as <code>log(1 + (docCount - docFreq + 0.5)/(docFreq + 0.5))</code>. */@@ -82,24 +116,10 @@ public class BM25Similarity extends Similarity {return (float) (collectionStats.sumTotalTermFreq() / (double) collectionStats.docCount());}- /**- * True if overlap tokens (tokens with a position of increment of zero) are discounted from the- * document's length.- */- protected boolean discountOverlaps = true;-- /**- * Sets whether overlap tokens (Tokens with 0 position increment) are ignored when computing norm.- * By default this is true, meaning overlap tokens do not count when computing norms.- */- public void setDiscountOverlaps(boolean v) {- discountOverlaps = v;- }-/*** Returns true if overlap tokens are discounted from the document's length.*- * @see #setDiscountOverlaps+ * @see #BM25Similarity(float, float, boolean)*/public boolean getDiscountOverlaps() {return discountOverlaps;diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.javaindex 3b9a394acf5..16316c7ae67 100644--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestBooleanSimilarity.java@@ -102,7 +102,6 @@ public class TestBooleanSimilarity extends BaseSimilarityTestCase {public void testSameNormsAsBM25() {BooleanSimilarity sim1 = new BooleanSimilarity();BM25Similarity sim2 = new BM25Similarity();- sim2.setDiscountOverlaps(true);for (int iter = 0; iter < 100; ++iter) {final int length = TestUtil.nextInt(random(), 1, 100);final int position = random().nextInt(length);diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.javaindex 1020a5133a6..f37a3fe4cab 100644--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java@@ -172,7 +172,6 @@ public class TestClassicSimilarity extends BaseSimilarityTestCase {public void testSameNormsAsBM25() {ClassicSimilarity sim1 = new ClassicSimilarity();BM25Similarity sim2 = new BM25Similarity();- sim2.setDiscountOverlaps(true);for (int iter = 0; iter < 100; ++iter) {final int length = TestUtil.nextInt(random(), 1, 1000);final int position = random().nextInt(length);diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.javaindex 8d9a8550705..77788f3b347 100644--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java@@ -512,17 +512,16 @@ public class TestSimilarityBase extends LuceneTestCase {// LUCENE-5221public void testDiscountOverlapsBoost() throws IOException {- BM25Similarity expected = new BM25Similarity();+ BM25Similarity expected = new BM25Similarity(false);SimilarityBase actual =new DFRSimilarity(new BasicModelIne(), new AfterEffectB(), new NormalizationH2());- expected.setDiscountOverlaps(false);actual.setDiscountOverlaps(false);FieldInvertState state =new FieldInvertState(Version.LATEST.major, "foo", IndexOptions.DOCS_AND_FREQS);state.setLength(5);state.setNumOverlap(2);assertEquals(expected.computeNorm(state), actual.computeNorm(state));- expected.setDiscountOverlaps(true);+ expected = new BM25Similarity();actual.setDiscountOverlaps(true);assertEquals(expected.computeNorm(state), actual.computeNorm(state));}diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java b/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.javaindex 86c67aa811c..bbd038fa34a 100644--- a/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java+++ b/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java@@ -398,8 +398,8 @@ public final class SearchImpl extends LukeModel implements Search {tfidf.setDiscountOverlaps(config.isDiscountOverlaps());similarity = tfidf;} else {- BM25Similarity bm25 = new BM25Similarity(config.getK1(), config.getB());- bm25.setDiscountOverlaps(config.isDiscountOverlaps());+ BM25Similarity bm25 =+ new BM25Similarity(config.getK1(), config.getB(), config.isDiscountOverlaps());similarity = bm25;}diff --git a/lucene/misc/src/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java b/lucene/misc/src/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.javaindex 13387ead238..fe3ea1c2725 100644--- a/lucene/misc/src/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java+++ b/lucene/misc/src/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java@@ -40,6 +40,7 @@ public final class LegacyBM25Similarity extends Similarity {* <ul>* <li>{@code k1 = 1.2}* <li>{@code b = 0.75}+ * <li>{@code discountOverlaps = true}* </ul>*/public LegacyBM25Similarity() {@@ -58,6 +59,20 @@ public final class LegacyBM25Similarity extends Similarity {this.bm25Similarity = new BM25Similarity(k1, b);}+ /**+ * BM25 with the supplied parameter values.+ *+ * @param k1 Controls non-linear term frequency normalization (saturation).+ * @param b Controls to what degree document length normalizes tf values.+ * @param discountOverlaps True if overlap tokens (tokens with a position of increment of zero)+ * are discounted from the document's length.+ * @throws IllegalArgumentException if {@code k1} is infinite or negative, or if {@code b} is not+ * within the range {@code [0..1]}+ */+ public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) {+ this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps);+ }+@Overridepublic long computeNorm(FieldInvertState state) {return bm25Similarity.computeNorm(state);@@ -87,18 +102,10 @@ public final class LegacyBM25Similarity extends Similarity {return bm25Similarity.getB();}- /**- * Sets whether overlap tokens (Tokens with 0 position increment) are ignored when computing norm.- * By default this is true, meaning overlap tokens do not count when computing norms.- */- public void setDiscountOverlaps(boolean v) {- bm25Similarity.setDiscountOverlaps(v);- }-/*** Returns true if overlap tokens are discounted from the document's length.*- * @see #setDiscountOverlaps+ * @see #LegacyBM25Similarity(float, float, boolean)*/public boolean getDiscountOverlaps() {return bm25Similarity.getDiscountOverlaps();diff --git a/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.javaindex fefe8936e21..2ba51fa4079 100644--- a/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.java+++ b/solr/core/src/java/org/apache/solr/search/similarities/BM25SimilarityFactory.java@@ -32,33 +32,27 @@ import org.apache.solr.schema.SimilarityFactory;* The default is <code>1.2</code>* <li>b (float): Controls to what degree document length normalizes tf values.* The default is <code>0.75</code>- * </ul>- * <p>- * Optional settings:- * <ul>- * <li>discountOverlaps (bool): Sets- * {@link BM25Similarity#setDiscountOverlaps(boolean)}</li>+ * <li>discountOverlaps (bool): True if overlap tokens (tokens with a position of increment of zero) are+ * discounted from the document's length.+ * The default is <code>true</code>* </ul>* @lucene.experimental* @since 8.0.0*/public class BM25SimilarityFactory extends SimilarityFactory {- private boolean discountOverlaps;- private float k1;- private float b;+ private BM25Similarity similarity;@Overridepublic void init(SolrParams params) {super.init(params);- discountOverlaps = params.getBool("discountOverlaps", true);- k1 = params.getFloat("k1", 1.2f);- b = params.getFloat("b", 0.75f);+ boolean discountOverlaps = params.getBool("discountOverlaps", true);+ float k1 = params.getFloat("k1", 1.2f);+ float b = params.getFloat("b", 0.75f);+ similarity = new BM25Similarity(k1, b, discountOverlaps);}@Overridepublic Similarity getSimilarity() {- BM25Similarity sim = new BM25Similarity(k1, b);- sim.setDiscountOverlaps(discountOverlaps);- return sim;+ return similarity;}}diff --git a/solr/core/src/java/org/apache/solr/search/similarities/LegacyBM25SimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/LegacyBM25SimilarityFactory.javaindex 66dd802922e..170e67b9595 100644--- a/solr/core/src/java/org/apache/solr/search/similarities/LegacyBM25SimilarityFactory.java+++ b/solr/core/src/java/org/apache/solr/search/similarities/LegacyBM25SimilarityFactory.java@@ -32,33 +32,27 @@ import org.apache.solr.schema.SimilarityFactory;* The default is <code>1.2</code>* <li>b (float): Controls to what degree document length normalizes tf values.* The default is <code>0.75</code>- * </ul>- * <p>- * Optional settings:- * <ul>- * <li>discountOverlaps (bool): Sets- * {@link LegacyBM25Similarity#setDiscountOverlaps(boolean)}</li>+ * <li>discountOverlaps (bool): True if overlap tokens (tokens with a position of increment of zero) are+ * discounted from the document's length.+ * The default is <code>true</code>* </ul>* @lucene.experimental* @since 8.0.0*/public class LegacyBM25SimilarityFactory extends SimilarityFactory {- private boolean discountOverlaps;- private float k1;- private float b;+ private LegacyBM25Similarity similarity;@Overridepublic void init(SolrParams params) {super.init(params);- discountOverlaps = params.getBool("discountOverlaps", true);- k1 = params.getFloat("k1", 1.2f);- b = params.getFloat("b", 0.75f);+ boolean discountOverlaps = params.getBool("discountOverlaps", true);+ float k1 = params.getFloat("k1", 1.2f);+ float b = params.getFloat("b", 0.75f);+ similarity = new LegacyBM25Similarity(k1, b, discountOverlaps);}@Overridepublic Similarity getSimilarity() {- LegacyBM25Similarity sim = new LegacyBM25Similarity(k1, b);- sim.setDiscountOverlaps(discountOverlaps);- return sim;+ return similarity;}}