Apache Lucene open-source search software
feat: implement asBulkSimScorer on FeatureFields's SimScorers (#15137)
Aditya Teltiaf59ff9220ec6
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 7015efea8ca..04b94c2a9d4 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -270,6 +270,10 @@ Optimizations* GITHUB#15045: Use FixedBitSet#cardinality for counting liveDocs in CheckIndex (Zhang Chao)++* GITHUB#15117: Score computations are now more reliably vectorized in FeatureField Scorer's+ (Aditya Teltia)+* GITHUB#15116, GITHUB#15138: Rewrite of the GroupVInt optimization without lambdas, varhandlesand no code in subclasses. The optimization now auto-detects if an IndexInput supports random accessand uses an optimized branchless approach. Any subclasses that have implemented the optimized methoddiff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureField.java b/lucene/core/src/java/org/apache/lucene/document/FeatureField.javaindex e2c7bce6913..da114008231 100644--- a/lucene/core/src/java/org/apache/lucene/document/FeatureField.java+++ b/lucene/core/src/java/org/apache/lucene/document/FeatureField.java@@ -34,6 +34,7 @@ import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;import org.apache.lucene.search.SortField;import org.apache.lucene.search.similarities.BM25Similarity;+import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.search.similarities.Similarity.SimScorer;/**@@ -262,9 +263,27 @@ public final class FeatureField extends Field {@OverrideSimScorer scorer(float w) {return new SimScorer() {+ private float doScore(float f) {+ return (w * f);+ }+@Overridepublic float score(float freq, long norm) {- return (w * decodeFeatureValue(freq));+ float f = decodeFeatureValue(freq);+ return doScore(f);+ }++ @Override+ public Similarity.BulkSimScorer asBulkSimScorer() {+ return new Similarity.BulkSimScorer() {+ @Override+ public void score(int size, float[] freqs, long[] norms, float[] scores) {+ for (int i = 0; i < size; ++i) {+ float f = decodeFeatureValue(freqs[i]);+ scores[i] = doScore(f);+ }+ }+ };}};}@@ -333,9 +352,27 @@ public final class FeatureField extends Field {@OverrideSimScorer scorer(float weight) {return new SimScorer() {+ private float doScore(float f) {+ return (float) (weight * Math.log(scalingFactor + f));+ }+@Overridepublic float score(float freq, long norm) {- return (float) (weight * Math.log(scalingFactor + decodeFeatureValue(freq)));+ float f = decodeFeatureValue(freq);+ return doScore(f);+ }++ @Override+ public Similarity.BulkSimScorer asBulkSimScorer() {+ return new Similarity.BulkSimScorer() {+ @Override+ public void score(int size, float[] freqs, long[] norms, float[] scores) {+ for (int i = 0; i < size; ++i) {+ float f = decodeFeatureValue(freqs[i]);+ scores[i] = doScore(f);+ }+ }+ };}};}@@ -405,14 +442,32 @@ public final class FeatureField extends Field {}final float pivot = this.pivot; // unboxreturn new SimScorer() {- @Override- public float score(float freq, long norm) {- float f = decodeFeatureValue(freq);++ private float doScore(float f) {// should be f / (f + k) but we rewrite it to// 1 - k / (f + k) to make sure it doesn't decrease// with f in spite of roundingreturn weight * (1 - pivot / (f + pivot));}++ @Override+ public float score(float freq, long norm) {+ float f = decodeFeatureValue(freq);+ return doScore(f);+ }++ @Override+ public Similarity.BulkSimScorer asBulkSimScorer() {+ return new Similarity.BulkSimScorer() {+ @Override+ public void score(int size, float[] freqs, long[] norms, float[] scores) {+ for (int i = 0; i < size; ++i) {+ float f = decodeFeatureValue(freqs[i]);+ scores[i] = doScore(f);+ }+ }+ };+ }};}@@ -469,14 +524,31 @@ public final class FeatureField extends Field {@OverrideSimScorer scorer(float weight) {return new SimScorer() {- @Override- public float score(float freq, long norm) {- float f = decodeFeatureValue(freq);+ private float doScore(float f) {// should be f^a / (f^a + k^a) but we rewrite it to// 1 - k^a / (f + k^a) to make sure it doesn't decrease// with f in spite of roundingreturn (float) (weight * (1 - pivotPa / (Math.pow(f, a) + pivotPa)));}++ @Override+ public float score(float freq, long norm) {+ float f = decodeFeatureValue(freq);+ return doScore(f);+ }++ @Override+ public Similarity.BulkSimScorer asBulkSimScorer() {+ return new Similarity.BulkSimScorer() {+ @Override+ public void score(int size, float[] freqs, long[] norms, float[] scores) {+ for (int i = 0; i < size; ++i) {+ float f = decodeFeatureValue(freqs[i]);+ scores[i] = doScore(f);+ }+ }+ };+ }};}diff --git a/lucene/core/src/test/org/apache/lucene/document/TestFeatureField.java b/lucene/core/src/test/org/apache/lucene/document/TestFeatureField.javaindex bd0696f66d1..7b03487c795 100644--- a/lucene/core/src/test/org/apache/lucene/document/TestFeatureField.java+++ b/lucene/core/src/test/org/apache/lucene/document/TestFeatureField.java@@ -22,6 +22,7 @@ import static org.hamcrest.Matchers.equalTo;import java.io.IOException;import java.util.List;+import java.util.Random;import org.apache.lucene.document.Field.Store;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.LeafReaderContext;@@ -37,11 +38,14 @@ import org.apache.lucene.search.TermQuery;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.Weight;import org.apache.lucene.search.similarities.BM25Similarity;+import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.search.similarities.Similarity.SimScorer;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.search.QueryUtils;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IOUtils;@@ -528,4 +532,65 @@ public class TestFeatureField extends LuceneTestCase {IOUtils.close(reader, dir);}++ public void testLinearBulkScorer() {+ FeatureField.LinearFunction func = new FeatureField.LinearFunction();+ SimScorer scorer = func.scorer(2f); // weight = 2+ Similarity.BulkSimScorer bulkScorer = scorer.asBulkSimScorer();+ doTestBulkScorer(scorer, bulkScorer);+ }++ public void testLogBulkScorer() {+ FeatureField.LogFunction func = new FeatureField.LogFunction(4.5f);+ SimScorer scorer = func.scorer(3f); // weight = 3+ Similarity.BulkSimScorer bulkScorer = scorer.asBulkSimScorer();+ doTestBulkScorer(scorer, bulkScorer);+ }++ public void testSaturationBulkScorer() {+ FeatureField.SaturationFunction func = new FeatureField.SaturationFunction("foo", "bar", 4.5f);+ SimScorer scorer = func.scorer(3f);+ Similarity.BulkSimScorer bulkScorer = scorer.asBulkSimScorer();+ doTestBulkScorer(scorer, bulkScorer);+ }++ public void testSigmoidBulkScorer() {+ FeatureField.SigmoidFunction func = new FeatureField.SigmoidFunction(4.5f, 0.6f);+ SimScorer scorer = func.scorer(3f);+ Similarity.BulkSimScorer bulkScorer = scorer.asBulkSimScorer();+ doTestBulkScorer(scorer, bulkScorer);+ }++ private void doTestBulkScorer(SimScorer scorer, Similarity.BulkSimScorer bulkScorer) {+ Random random = random();+ int iters = atLeast(3);+ float[] freqs = new float[0];+ long[] norms = new long[0];+ float[] scores = new float[0];++ for (int iter = 0; iter < iters; ++iter) {+ int size = TestUtil.nextInt(random, 0, 200);+ if (size > freqs.length) {+ freqs = new float[ArrayUtil.oversize(size, Float.BYTES)];+ norms = new long[freqs.length];+ scores = new float[freqs.length];+ }+ for (int i = 0; i < size; ++i) {+ freqs[i] = TestUtil.nextInt(random, 1, 1000); // freq values+ norms[i] = TestUtil.nextLong(random, 1, 255); // norms in byte range+ }++ float[] expected = new float[size];+ for (int i = 0; i < size; ++i) {+ expected[i] = scorer.score(freqs[i], norms[i]);+ }++ bulkScorer.score(size, freqs, norms, scores);++ assertArrayEquals(+ ArrayUtil.copyOfSubArray(expected, 0, size),+ ArrayUtil.copyOfSubArray(scores, 0, size),+ 0f);+ }+ }}