Apache Lucene open-source search software
Add remaining bulk float32 off-heap scoring similarities (#15037)
Chris Hegarty41fcc22c76f8
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 2fc76299be3..e096bd41701 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -237,7 +237,7 @@ Optimizations* GITHUB#15001: Remove full integrity check from SortingStoredFieldsConsumer (Martijn van Groningen)-* GITHUB#14980: Add bulk off-heap scoring for float32 vectors (Chris Hegarty)+* GITHUB#14980, GITHUB#15037: Add bulk off-heap scoring for float32 vectors (Chris Hegarty)* GITHUB#15004: Wraps all iterator with likelyImpactsEnum under BlockMaxConjunctionBulkScorer. (Ge Song)diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.javaindex 215f754c3fe..92dc5b6de5a 100644--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java@@ -16,7 +16,10 @@*/package org.apache.lucene.benchmark.jmh;+import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;+import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;+import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT;import java.io.IOException;import java.nio.ByteBuffer;@@ -75,6 +78,11 @@ import org.openjdk.jmh.annotations.Warmup;"-XX:+AlwaysPreTouch","--add-modules=jdk.incubator.vector"})+/**+ * Benchmark to compare the performance of float32 vector scoring using the default and optimized+ * scorers. While there are benchmark methods for each of the similarities, it is often most useful+ * to compare equivalent subsets, e.g. .*dot.*+ */public class VectorScorerFloat32Benchmark {@Param({"1024"})@@ -92,8 +100,8 @@ public class VectorScorerFloat32Benchmark {Directory dir;IndexInput in;KnnVectorValues values;- UpdateableRandomVectorScorer defDotScorer;- UpdateableRandomVectorScorer optDotScorer;+ UpdateableRandomVectorScorer defDotScorer, defCosScorer, defEucScorer, defMipScorer;+ UpdateableRandomVectorScorer optDotScorer, optCosScorer, optEucScorer, optMipScorer;@Setup(Level.Trial)public void setup() throws IOException {@@ -121,12 +129,24 @@ public class VectorScorerFloat32Benchmark {values = vectorValues(size, numVectors, in, DOT_PRODUCT);var def = DefaultFlatVectorScorer.INSTANCE;defDotScorer = def.getRandomVectorScorerSupplier(DOT_PRODUCT, values.copy()).scorer();+ defCosScorer = def.getRandomVectorScorerSupplier(COSINE, values.copy()).scorer();+ defEucScorer = def.getRandomVectorScorerSupplier(EUCLIDEAN, values.copy()).scorer();+ defMipScorer = def.getRandomVectorScorerSupplier(MAXIMUM_INNER_PRODUCT, values.copy()).scorer();defDotScorer.setScoringOrdinal(targetOrd);+ defCosScorer.setScoringOrdinal(targetOrd);+ defEucScorer.setScoringOrdinal(targetOrd);+ defMipScorer.setScoringOrdinal(targetOrd);// optimized scorervar opt = FlatVectorScorerUtil.getLucene99FlatVectorsScorer();optDotScorer = opt.getRandomVectorScorerSupplier(DOT_PRODUCT, values.copy()).scorer();+ optCosScorer = opt.getRandomVectorScorerSupplier(COSINE, values.copy()).scorer();+ optEucScorer = opt.getRandomVectorScorerSupplier(EUCLIDEAN, values.copy()).scorer();+ optMipScorer = opt.getRandomVectorScorerSupplier(MAXIMUM_INNER_PRODUCT, values.copy()).scorer();optDotScorer.setScoringOrdinal(targetOrd);+ optCosScorer.setScoringOrdinal(targetOrd);+ optEucScorer.setScoringOrdinal(targetOrd);+ optMipScorer.setScoringOrdinal(targetOrd);List<Integer> list = IntStream.range(0, numVectors).boxed().collect(Collectors.toList());Collections.shuffle(list, random);@@ -146,10 +166,21 @@ public class VectorScorerFloat32Benchmark {}public void pollute(Random random) throws IOException {+ // exercise various similarities to ensure they don't have negative effects, e.g.,+ // type pollution on virtual calls, etc.float[] vec = randomVector(size, random);var opt = FlatVectorScorerUtil.getLucene99FlatVectorsScorer();var scorer = opt.getRandomVectorScorer(DOT_PRODUCT, values.copy(), vec);+for (int i = 0; i < 2; i++) {+ dotProductOptScorer();+ dotProductOptBulkScore();+ cosineOptScorer();+ cosineDefaultBulk();+ euclideanOptScorer();+ euclideanOptBulkScore();+ mipOptScorer();+ mipOptBulkScore();for (int v = 0; v < numVectorsToScore; v++) {scores[v] = scorer.score(indices[v]);}@@ -157,6 +188,8 @@ public class VectorScorerFloat32Benchmark {}}+ // -- dot product+@Benchmarkpublic float[] dotProductDefault() throws IOException {for (int v = 0; v < numVectorsToScore; v++) {@@ -185,6 +218,96 @@ public class VectorScorerFloat32Benchmark {return scores;}+ // -- euclidean++ @Benchmark+ public float[] euclideanDefault() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = defEucScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] euclideanDefaultBulk() throws IOException {+ defEucScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }++ @Benchmark+ public float[] euclideanOptScorer() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = optEucScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] euclideanOptBulkScore() throws IOException {+ optEucScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }++ // -- euclidean++ @Benchmark+ public float[] cosineDefault() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = defCosScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] cosineDefaultBulk() throws IOException {+ defCosScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }++ @Benchmark+ public float[] cosineOptScorer() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = optCosScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] cosineOptBulkScore() throws IOException {+ optCosScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }++ // -- max inner product++ @Benchmark+ public float[] mipDefault() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = defMipScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] mipDefaultBulk() throws IOException {+ defMipScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }++ @Benchmark+ public float[] mipOptScorer() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = optMipScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] mipOptBulkScore() throws IOException {+ optMipScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }+static float[] randomVector(int dims, Random random) {float[] fa = new float[dims];for (int i = 0; i < dims; ++i) {diff --git a/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.javaindex 765ef0455a3..2417159e183 100644--- a/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java+++ b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java@@ -65,4 +65,67 @@ public class TestVectorScorerFloat32Benchmark extends LuceneTestCase {actualScores = ArrayUtil.copyArray(bench.scores);assertArrayEquals(expectedScores, actualScores, delta);}++ public void testCosine() throws IOException {+ Arrays.fill(bench.scores, 0.0f);+ bench.cosineDefault();+ var expectedScores = ArrayUtil.copyArray(bench.scores);++ Arrays.fill(bench.scores, 0.0f);+ bench.cosineDefaultBulk();+ var bulkScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, bulkScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.cosineOptScorer();+ var actualScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, actualScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.cosineOptBulkScore();+ actualScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, actualScores, delta);+ }++ public void testEuclidean() throws IOException {+ Arrays.fill(bench.scores, 0.0f);+ bench.euclideanDefault();+ var expectedScores = ArrayUtil.copyArray(bench.scores);++ Arrays.fill(bench.scores, 0.0f);+ bench.euclideanDefaultBulk();+ var bulkScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, bulkScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.euclideanOptScorer();+ var actualScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, actualScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.euclideanOptBulkScore();+ actualScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, actualScores, delta);+ }++ public void testMip() throws IOException {+ Arrays.fill(bench.scores, 0.0f);+ bench.mipDefault();+ var expectedScores = ArrayUtil.copyArray(bench.scores);++ Arrays.fill(bench.scores, 0.0f);+ bench.mipDefaultBulk();+ var bulkScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, bulkScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.mipOptScorer();+ var actualScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, actualScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.mipOptBulkScore();+ actualScores = ArrayUtil.copyArray(bench.scores);+ assertArrayEquals(expectedScores, actualScores, delta);+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java b/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.javaindex f905eeae24c..a692d917e60 100644--- a/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java+++ b/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java@@ -19,6 +19,7 @@ package org.apache.lucene.index;import static org.apache.lucene.util.VectorUtil.cosine;import static org.apache.lucene.util.VectorUtil.dotProduct;import static org.apache.lucene.util.VectorUtil.dotProductScore;+import static org.apache.lucene.util.VectorUtil.normalizeDistanceToUnitInterval;import static org.apache.lucene.util.VectorUtil.normalizeToUnitInterval;import static org.apache.lucene.util.VectorUtil.scaleMaxInnerProductScore;import static org.apache.lucene.util.VectorUtil.squareDistance;@@ -34,7 +35,7 @@ public enum VectorSimilarityFunction {EUCLIDEAN {@Overridepublic float compare(float[] v1, float[] v2) {- return 1 / (1 + squareDistance(v1, v2));+ return normalizeDistanceToUnitInterval(squareDistance(v1, v2));}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.javaindex 04a3ddeffaf..4f64aa8b816 100644--- a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java@@ -322,6 +322,19 @@ public final class VectorUtil {return Math.max((1 + value) / 2, 0);}+ /**+ * Maps a non-negative squared distance to a similarity score in the range (0, 1].+ *+ * <p>Uses the transformation: {@code similarity = 1 / (1 + squaredDistance)}. Smaller distances+ * yield scores closer to 1; larger distances approach 0.+ *+ * @param squaredDistance squared Euclidean distance (must be ≥ 0)+ * @return similarity score in (0, 1]+ */+ public static float normalizeDistanceToUnitInterval(float squaredDistance) {+ return 1.0f / (1.0f + squaredDistance);+ }+/*** Checks if a float vector only has finite components.*diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.javaindex 456747e563f..8f60de6cda1 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java@@ -50,15 +50,13 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorerprivate RandomVectorScorerSupplier getFloatScoringSupplier(FloatVectorValues vectorValues, VectorSimilarityFunction similarityType) throws IOException {- if (similarityType == VectorSimilarityFunction.DOT_PRODUCT) { // dot product for now- if (vectorValues instanceof HasIndexSlice sliceableValues- && sliceableValues.getSlice() != null) {- var scorer =- Lucene99MemorySegmentFloatVectorScorerSupplier.create(- similarityType, sliceableValues.getSlice(), vectorValues);- if (scorer.isPresent()) {- return scorer.get();- }+ if (vectorValues instanceof HasIndexSlice sliceableValues+ && sliceableValues.getSlice() != null) {+ var scorer =+ Lucene99MemorySegmentFloatVectorScorerSupplier.create(+ similarityType, sliceableValues.getSlice(), vectorValues);+ if (scorer.isPresent()) {+ return scorer.get();}}return delegate.getRandomVectorScorerSupplier(similarityType, vectorValues);@@ -87,16 +85,14 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorerVectorSimilarityFunction similarityType, KnnVectorValues vectorValues, float[] target)throws IOException {checkDimensions(target.length, vectorValues.dimension());- if (similarityType == VectorSimilarityFunction.DOT_PRODUCT) { // just for now- if (vectorValues instanceof FloatVectorValues fvv- && fvv instanceof HasIndexSlice floatVectorValues- && floatVectorValues.getSlice() != null) {- var scorer =- Lucene99MemorySegmentFloatVectorScorer.create(- similarityType, floatVectorValues.getSlice(), fvv, target);- if (scorer.isPresent()) {- return scorer.get();- }+ if (vectorValues instanceof FloatVectorValues fvv+ && fvv instanceof HasIndexSlice floatVectorValues+ && floatVectorValues.getSlice() != null) {+ var scorer =+ Lucene99MemorySegmentFloatVectorScorer.create(+ similarityType, floatVectorValues.getSlice(), fvv, target);+ if (scorer.isPresent()) {+ return scorer.get();}}return delegate.getRandomVectorScorer(similarityType, vectorValues, target);diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.javaindex 49e07f36a64..96d84b40bc9 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java@@ -16,8 +16,6 @@*/package org.apache.lucene.internal.vectorization;-import static org.apache.lucene.util.VectorUtil.normalizeToUnitInterval;-import java.io.IOException;import java.lang.foreign.MemorySegment;import java.util.Optional;@@ -26,6 +24,7 @@ import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.FilterIndexInput;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.MemorySegmentAccessInput;+import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.hnsw.RandomVectorScorer;abstract sealed class Lucene99MemorySegmentFloatVectorScorer@@ -35,16 +34,14 @@ abstract sealed class Lucene99MemorySegmentFloatVectorScorerfinal int vectorByteSize;final MemorySegment seg;final float[] query;+ final float[] scratchScores = new float[4];/*** Return an optional whose value, if present, is the scorer. Otherwise, an empty optional is* returned.*/public static Optional<Lucene99MemorySegmentFloatVectorScorer> create(- VectorSimilarityFunction type,- IndexInput input,- FloatVectorValues values,- float[] queryVector)+ VectorSimilarityFunction type, IndexInput input, FloatVectorValues values, float[] query)throws IOException {input = FilterIndexInput.unwrapOnlyTest(input);MemorySegment seg;@@ -54,21 +51,20 @@ abstract sealed class Lucene99MemorySegmentFloatVectorScorer}checkInvariants(values.size(), values.getVectorByteLength(), input);return switch (type) {- case COSINE -> Optional.empty(); // of(new CosineScorer(msInput, values, queryVector));- case DOT_PRODUCT -> Optional.of(new DotProductScorer(seg, values, queryVector));- case EUCLIDEAN -> Optional.empty(); // of(new EuclideanScorer(msInput, values, queryVector));- case MAXIMUM_INNER_PRODUCT ->- Optional.empty(); // of(new MaxInnerProductScorer(msInput, values, queryVector));+ case COSINE -> Optional.of(new CosineScorer(seg, values, query));+ case DOT_PRODUCT -> Optional.of(new DotProductScorer(seg, values, query));+ case EUCLIDEAN -> Optional.of(new EuclideanScorer(seg, values, query));+ case MAXIMUM_INNER_PRODUCT -> Optional.of(new MaxInnerProductScorer(seg, values, query));};}Lucene99MemorySegmentFloatVectorScorer(- MemorySegment seg, FloatVectorValues values, float[] queryVector) {+ MemorySegment seg, FloatVectorValues values, float[] query) {super(values);this.values = values;this.seg = seg;this.vectorByteSize = values.getVectorByteLength();- this.query = queryVector;+ this.query = query;}static void checkInvariants(int maxOrd, int vectorByteLength, IndexInput input) {@@ -83,13 +79,85 @@ abstract sealed class Lucene99MemorySegmentFloatVectorScorer}}+ @Override+ public void bulkScore(int[] nodes, float[] scores, int numNodes) throws IOException {+ int i = 0;+ final int limit = numNodes & ~3;+ for (; i < limit; i += 4) {+ long offset1 = (long) nodes[i] * vectorByteSize;+ long offset2 = (long) nodes[i + 1] * vectorByteSize;+ long offset3 = (long) nodes[i + 2] * vectorByteSize;+ long offset4 = (long) nodes[i + 3] * vectorByteSize;+ vectorOp(seg, scratchScores, offset1, offset2, offset3, offset4, query.length);+ scores[i + 0] = normalizeRawScore(scratchScores[0]);+ scores[i + 1] = normalizeRawScore(scratchScores[1]);+ scores[i + 2] = normalizeRawScore(scratchScores[2]);+ scores[i + 3] = normalizeRawScore(scratchScores[3]);+ }+ // Handle remaining 1–3 nodes in bulk (if any)+ int remaining = numNodes - i;+ if (remaining > 0) {+ long addr1 = (long) nodes[i] * vectorByteSize;+ long addr2 = (remaining > 1) ? (long) nodes[i + 1] * vectorByteSize : addr1;+ long addr3 = (remaining > 2) ? (long) nodes[i + 2] * vectorByteSize : addr1;+ vectorOp(seg, scratchScores, addr1, addr2, addr3, addr3, query.length);+ scores[i] = normalizeRawScore(scratchScores[0]);+ if (remaining > 1) scores[i + 1] = normalizeRawScore(scratchScores[1]);+ if (remaining > 2) scores[i + 2] = normalizeRawScore(scratchScores[2]);+ }+ }++ abstract void vectorOp(+ MemorySegment seg,+ float[] scores,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount);++ abstract float normalizeRawScore(float value);++ static final class CosineScorer extends Lucene99MemorySegmentFloatVectorScorer {++ static final MemorySegmentBulkVectorOps.Cosine COS_OPS =+ MemorySegmentBulkVectorOps.COS_INSTANCE;++ CosineScorer(MemorySegment seg, FloatVectorValues values, float[] query) {+ super(seg, values, query);+ }++ @Override+ public float score(int node) throws IOException {+ checkOrdinal(node);+ // just delegates to existing scorer that copies on-heap+ return VectorSimilarityFunction.COSINE.compare(query, values.vectorValue(node));+ }++ @Override+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ COS_OPS.cosineBulk(+ seg, scores, query, node1Offset, node2Offset, node3Offset, node4Offset, elementCount);+ }++ @Override+ float normalizeRawScore(float rawScore) {+ return VectorUtil.normalizeToUnitInterval(rawScore);+ }+ }+static final class DotProductScorer extends Lucene99MemorySegmentFloatVectorScorer {static final MemorySegmentBulkVectorOps.DotProduct DOT_OPS =MemorySegmentBulkVectorOps.DOT_INSTANCE;- final float[] scratchScores = new float[4];-DotProductScorer(MemorySegment input, FloatVectorValues values, float[] query) {super(input, values, query);}@@ -102,32 +170,92 @@ abstract sealed class Lucene99MemorySegmentFloatVectorScorer}@Override- public void bulkScore(int[] nodes, float[] scores, int numNodes) {- int i = 0;- final int limit = numNodes & ~3;- for (; i < limit; i += 4) {- long offset1 = (long) nodes[i] * vectorByteSize;- long offset2 = (long) nodes[i + 1] * vectorByteSize;- long offset3 = (long) nodes[i + 2] * vectorByteSize;- long offset4 = (long) nodes[i + 3] * vectorByteSize;- DOT_OPS.dotProductBulk(- seg, scratchScores, query, offset1, offset2, offset3, offset4, query.length);- scores[i + 0] = normalizeToUnitInterval(scratchScores[0]);- scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);- scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);- scores[i + 3] = normalizeToUnitInterval(scratchScores[3]);- }- // Handle remaining 1–3 nodes in bulk (if any)- int remaining = numNodes - i;- if (remaining > 0) {- long addr1 = (long) nodes[i] * vectorByteSize;- long addr2 = (remaining > 1) ? (long) nodes[i + 1] * vectorByteSize : addr1;- long addr3 = (remaining > 2) ? (long) nodes[i + 2] * vectorByteSize : addr1;- DOT_OPS.dotProductBulk(seg, scratchScores, query, addr1, addr2, addr3, addr1, query.length);- scores[i] = normalizeToUnitInterval(scratchScores[0]);- if (remaining > 1) scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);- if (remaining > 2) scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);- }+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ DOT_OPS.dotProductBulk(+ seg, scores, query, node1Offset, node2Offset, node3Offset, node4Offset, elementCount);+ }++ @Override+ float normalizeRawScore(float rawScore) {+ return VectorUtil.normalizeToUnitInterval(rawScore);+ }+ }++ static final class EuclideanScorer extends Lucene99MemorySegmentFloatVectorScorer {++ static final MemorySegmentBulkVectorOps.SqrDistance SQR_OPS =+ MemorySegmentBulkVectorOps.SQR_INSTANCE;++ EuclideanScorer(MemorySegment seg, FloatVectorValues values, float[] query) {+ super(seg, values, query);+ }++ @Override+ public float score(int node) throws IOException {+ checkOrdinal(node);+ // just delegates to existing scorer that copies on-heap+ return VectorSimilarityFunction.EUCLIDEAN.compare(query, values.vectorValue(node));+ }++ @Override+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ SQR_OPS.sqrDistanceBulk(+ seg, scores, query, node1Offset, node2Offset, node3Offset, node4Offset, elementCount);+ }++ @Override+ float normalizeRawScore(float rawScore) {+ return VectorUtil.normalizeDistanceToUnitInterval(rawScore);+ }+ }++ static final class MaxInnerProductScorer extends Lucene99MemorySegmentFloatVectorScorer {++ static final MemorySegmentBulkVectorOps.DotProduct DOT_OPS =+ MemorySegmentBulkVectorOps.DOT_INSTANCE;++ MaxInnerProductScorer(MemorySegment seg, FloatVectorValues values, float[] query) {+ super(seg, values, query);+ }++ @Override+ public float score(int node) throws IOException {+ checkOrdinal(node);+ // just delegates to existing scorer that copies on-heap+ return VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(+ query, values.vectorValue(node));+ }++ @Override+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ DOT_OPS.dotProductBulk(+ seg, scores, query, node1Offset, node2Offset, node3Offset, node4Offset, elementCount);+ }++ @Override+ float normalizeRawScore(float rawScore) {+ return VectorUtil.scaleMaxInnerProductScore(rawScore);}}}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.javaindex 61417452a1e..48290fb7abb 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java@@ -16,8 +16,6 @@*/package org.apache.lucene.internal.vectorization;-import static org.apache.lucene.util.VectorUtil.normalizeToUnitInterval;-import java.io.IOException;import java.lang.foreign.MemorySegment;import java.util.Optional;@@ -26,6 +24,7 @@ import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.FilterIndexInput;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.MemorySegmentAccessInput;+import org.apache.lucene.util.VectorUtil;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;@@ -53,11 +52,10 @@ public abstract sealed class Lucene99MemorySegmentFloatVectorScorerSupplier}checkInvariants(values.size(), values.getVectorByteLength(), input);return switch (type) {- case COSINE -> Optional.empty(); // of(new CosineSupplier(msInput, values));+ case COSINE -> Optional.of(new CosineSupplier(seg, values));case DOT_PRODUCT -> Optional.of(new DotProductSupplier(seg, values));- case EUCLIDEAN -> Optional.empty(); // of(new EuclideanSupplier(msInput, values));- case MAXIMUM_INNER_PRODUCT ->- Optional.empty(); // of(new MaxInnerProductSupplier(msInput, values));+ case EUCLIDEAN -> Optional.of(new EuclideanSupplier(seg, values));+ case MAXIMUM_INNER_PRODUCT -> Optional.of(new MaxInnerProductSupplier(seg, values));};}@@ -75,9 +73,47 @@ public abstract sealed class Lucene99MemorySegmentFloatVectorScorerSupplier}}- final void checkOrdinal(int ord) {- if (ord < 0 || ord >= maxOrd) {- throw new IllegalArgumentException("illegal ordinal: " + ord);+ static final class CosineSupplier extends Lucene99MemorySegmentFloatVectorScorerSupplier {++ static final MemorySegmentBulkVectorOps.Cosine COS_OPS =+ MemorySegmentBulkVectorOps.COS_INSTANCE;++ CosineSupplier(MemorySegment seg, FloatVectorValues values) {+ super(seg, values);+ }++ @Override+ public UpdateableRandomVectorScorer scorer() {+ return new AbstractBulkScorer(values) {+ @Override+ float vectorOp(MemorySegment seg, long q, long d, int elementCount) {+ return COS_OPS.cosine(seg, q, d, dims);+ }++ @Override+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long queryOffset,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ COS_OPS.cosineBulk(+ seg, scores, queryOffset, node1Offset, node2Offset, node3Offset, node4Offset, dims);+ }++ @Override+ float normalizeRawScore(float rawScore) {+ return VectorUtil.normalizeToUnitInterval(rawScore);+ }+ };+ }++ @Override+ public CosineSupplier copy() throws IOException {+ return new CosineSupplier(seg, values.copy()); // TODO: check copy}}@@ -86,61 +122,35 @@ public abstract sealed class Lucene99MemorySegmentFloatVectorScorerSupplierstatic final MemorySegmentBulkVectorOps.DotProduct DOT_OPS =MemorySegmentBulkVectorOps.DOT_INSTANCE;- final float[] scratchScores = new float[4];-DotProductSupplier(MemorySegment seg, FloatVectorValues values) {super(seg, values);}@Overridepublic UpdateableRandomVectorScorer scorer() {- return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(values) {- private int queryOrd;-+ return new AbstractBulkScorer(values) {@Override- public float score(int node) throws IOException {- checkOrdinal(node);- long queryAddr = (long) queryOrd * vectorByteSize;- long addr = (long) node * vectorByteSize;- var raw = DOT_OPS.dotProduct(seg, queryAddr, addr, dims);- return normalizeToUnitInterval(raw);+ float vectorOp(MemorySegment seg, long q, long d, int elementCount) {+ return DOT_OPS.dotProduct(seg, q, d, dims);}@Override- public void bulkScore(int[] nodes, float[] scores, int numNodes) {- // TODO checkOrdinal(node1 ....);- int i = 0;- long queryAddr = (long) queryOrd * vectorByteSize;- final int limit = numNodes & ~3;- for (; i < limit; i += 4) {- long offset1 = (long) nodes[i + 0] * vectorByteSize;- long offset2 = (long) nodes[i + 1] * vectorByteSize;- long offset3 = (long) nodes[i + 2] * vectorByteSize;- long offset4 = (long) nodes[i + 3] * vectorByteSize;- DOT_OPS.dotProductBulk(- seg, scratchScores, queryAddr, offset1, offset2, offset3, offset4, dims);- scores[i + 0] = normalizeToUnitInterval(scratchScores[0]);- scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);- scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);- scores[i + 3] = normalizeToUnitInterval(scratchScores[3]);- }- // Handle remaining 1–3 nodes in bulk (if any)- int remaining = numNodes - i;- if (remaining > 0) {- long addr1 = (long) nodes[i] * vectorByteSize;- long addr2 = (remaining > 1) ? (long) nodes[i + 1] * vectorByteSize : addr1;- long addr3 = (remaining > 2) ? (long) nodes[i + 2] * vectorByteSize : addr1;- DOT_OPS.dotProductBulk(seg, scratchScores, queryAddr, addr1, addr2, addr3, addr3, dims);- scores[i] = normalizeToUnitInterval(scratchScores[0]);- if (remaining > 1) scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);- if (remaining > 2) scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);- }+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long queryOffset,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ DOT_OPS.dotProductBulk(+ seg, scores, queryOffset, node1Offset, node2Offset, node3Offset, node4Offset, dims);}@Override- public void setScoringOrdinal(int node) {- checkOrdinal(node);- queryOrd = node;+ float normalizeRawScore(float rawScore) {+ return VectorUtil.normalizeToUnitInterval(rawScore);}};}@@ -150,4 +160,167 @@ public abstract sealed class Lucene99MemorySegmentFloatVectorScorerSupplierreturn new DotProductSupplier(seg, values);}}++ static final class EuclideanSupplier extends Lucene99MemorySegmentFloatVectorScorerSupplier {++ static final MemorySegmentBulkVectorOps.SqrDistance SQR_OPS =+ MemorySegmentBulkVectorOps.SQR_INSTANCE;++ EuclideanSupplier(MemorySegment seg, FloatVectorValues values) {+ super(seg, values);+ }++ @Override+ public UpdateableRandomVectorScorer scorer() {+ return new AbstractBulkScorer(values) {+ @Override+ float vectorOp(MemorySegment seg, long q, long d, int elementCount) {+ return SQR_OPS.sqrDistance(seg, q, d, dims);+ }++ @Override+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long queryOffset,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ SQR_OPS.sqrDistanceBulk(+ seg, scores, queryOffset, node1Offset, node2Offset, node3Offset, node4Offset, dims);+ }++ @Override+ float normalizeRawScore(float rawScore) {+ return VectorUtil.normalizeDistanceToUnitInterval(rawScore);+ }+ };+ }++ @Override+ public EuclideanSupplier copy() throws IOException {+ return new EuclideanSupplier(seg, values); // TODO: need to copy ?+ }+ }++ static final class MaxInnerProductSupplier+ extends Lucene99MemorySegmentFloatVectorScorerSupplier {++ static final MemorySegmentBulkVectorOps.DotProduct DOT_OPS =+ MemorySegmentBulkVectorOps.DOT_INSTANCE;++ MaxInnerProductSupplier(MemorySegment seg, FloatVectorValues values) {+ super(seg, values);+ }++ @Override+ public UpdateableRandomVectorScorer scorer() {+ return new AbstractBulkScorer(values) {+ @Override+ float vectorOp(MemorySegment seg, long q, long d, int elementCount) {+ return DOT_OPS.dotProduct(seg, q, d, dims);+ }++ @Override+ void vectorOp(+ MemorySegment seg,+ float[] scores,+ long queryOffset,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount) {+ DOT_OPS.dotProductBulk(+ seg, scores, queryOffset, node1Offset, node2Offset, node3Offset, node4Offset, dims);+ }++ @Override+ float normalizeRawScore(float rawScore) {+ return VectorUtil.scaleMaxInnerProductScore(rawScore);+ }+ };+ }++ @Override+ public MaxInnerProductSupplier copy() throws IOException {+ return new MaxInnerProductSupplier(seg, values);+ }+ }++ abstract class AbstractBulkScorer+ extends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {+ private int queryOrd;+ final float[] scratchScores = new float[4];++ AbstractBulkScorer(FloatVectorValues values) {+ super(values);+ }++ final void checkOrdinal(int ord) {+ if (ord < 0 || ord >= maxOrd) {+ throw new IllegalArgumentException("illegal ordinal: " + ord);+ }+ }++ abstract float vectorOp(MemorySegment seg, long q, long d, int elementCount);++ abstract void vectorOp(+ MemorySegment seg,+ float[] scores,+ long queryOffset,+ long node1Offset,+ long node2Offset,+ long node3Offset,+ long node4Offset,+ int elementCount);++ abstract float normalizeRawScore(float rawScore);++ @Override+ public float score(int node) {+ checkOrdinal(node);+ long queryAddr = (long) queryOrd * vectorByteSize;+ long addr = (long) node * vectorByteSize;+ var raw = vectorOp(seg, queryAddr, addr, dims);+ return normalizeRawScore(raw);+ }++ @Override+ public void bulkScore(int[] nodes, float[] scores, int numNodes) {+ int i = 0;+ long queryAddr = (long) queryOrd * vectorByteSize;+ final int limit = numNodes & ~3;+ for (; i < limit; i += 4) {+ long offset1 = (long) nodes[i] * vectorByteSize;+ long offset2 = (long) nodes[i + 1] * vectorByteSize;+ long offset3 = (long) nodes[i + 2] * vectorByteSize;+ long offset4 = (long) nodes[i + 3] * vectorByteSize;+ vectorOp(seg, scratchScores, queryAddr, offset1, offset2, offset3, offset4, dims);+ scores[i + 0] = normalizeRawScore(scratchScores[0]);+ scores[i + 1] = normalizeRawScore(scratchScores[1]);+ scores[i + 2] = normalizeRawScore(scratchScores[2]);+ scores[i + 3] = normalizeRawScore(scratchScores[3]);+ }+ // Handle remaining 1–3 nodes in bulk (if any)+ int remaining = numNodes - i;+ if (remaining > 0) {+ long addr1 = (long) nodes[i] * vectorByteSize;+ long addr2 = (remaining > 1) ? (long) nodes[i + 1] * vectorByteSize : addr1;+ long addr3 = (remaining > 2) ? (long) nodes[i + 2] * vectorByteSize : addr1;+ vectorOp(seg, scratchScores, queryAddr, addr1, addr2, addr3, addr1, dims);+ scores[i] = normalizeRawScore(scratchScores[0]);+ if (remaining > 1) scores[i + 1] = normalizeRawScore(scratchScores[1]);+ if (remaining > 2) scores[i + 2] = normalizeRawScore(scratchScores[2]);+ }+ }++ @Override+ public void setScoringOrdinal(int node) {+ checkOrdinal(node);+ queryOrd = node;+ }+ }}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.javaindex 20aefce4831..0447389d7b8 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java@@ -36,6 +36,8 @@ public final class MemorySegmentBulkVectorOps {static final ValueLayout.OfFloat LAYOUT_LE_FLOAT = ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(LE);public static final DotProduct DOT_INSTANCE = new DotProduct();+ public static final Cosine COS_INSTANCE = new Cosine();+ public static final SqrDistance SQR_INSTANCE = new SqrDistance();private MemorySegmentBulkVectorOps() {}@@ -144,4 +146,265 @@ public final class MemorySegmentBulkVectorOps {scores[3] = sum4;}}++ // -- cosine++ public static final class Cosine {++ private Cosine() {}++ public void cosineBulk(+ MemorySegment dataSeg,+ float[] scores,+ float[] q,+ long d1,+ long d2,+ long d3,+ long d4,+ int elementCount) {+ cosineBulkImpl(dataSeg, scores, q, -1L, d1, d2, d3, d4, elementCount);+ }++ public void cosineBulk(+ MemorySegment seg,+ float[] scores,+ long q,+ long d1,+ long d2,+ long d3,+ long d4,+ int elementCount) {+ cosineBulkImpl(seg, scores, null, q, d1, d2, d3, d4, elementCount);+ }++ public float cosine(MemorySegment seg, long q, long d, int elementCount) {+ int i = 0;+ FloatVector sv = FloatVector.zero(FLOAT_SPECIES);+ FloatVector qvNorm = FloatVector.zero(FLOAT_SPECIES);+ FloatVector dvNorm = FloatVector.zero(FLOAT_SPECIES);+ final int limit = FLOAT_SPECIES.loopBound(elementCount);+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ final long offset = (long) i * Float.BYTES;+ FloatVector qv = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, q + offset, LE);+ FloatVector dv = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d + offset, LE);+ sv = fma(qv, dv, sv);+ qvNorm = fma(qv, qv, qvNorm);+ dvNorm = fma(dv, dv, dvNorm);+ }+ float sum = sv.reduceLanes(VectorOperators.ADD);+ float qNorm = qvNorm.reduceLanes(VectorOperators.ADD);+ float dNorm = dvNorm.reduceLanes(VectorOperators.ADD);++ for (; i < elementCount; i++) {+ final long offset = (long) i * Float.BYTES;+ final float qValue = seg.get(LAYOUT_LE_FLOAT, q + offset);+ final float dValue = seg.get(LAYOUT_LE_FLOAT, d + offset);+ sum = fma(qValue, dValue, sum);+ qNorm = fma(qValue, qValue, qNorm);+ dNorm = fma(dValue, dValue, dNorm);+ }+ return (float) (sum / Math.sqrt((double) qNorm * (double) dNorm));+ }++ private void cosineBulkImpl(+ MemorySegment seg,+ float[] scores,+ float[] qArray,+ long qOffset,+ long d1,+ long d2,+ long d3,+ long d4,+ int elementCount) {+ int i = 0;+ FloatVector sv1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv4 = FloatVector.zero(FLOAT_SPECIES);++ FloatVector qvNorm = FloatVector.zero(FLOAT_SPECIES);+ FloatVector dv1Norm = FloatVector.zero(FLOAT_SPECIES);+ FloatVector dv2Norm = FloatVector.zero(FLOAT_SPECIES);+ FloatVector dv3Norm = FloatVector.zero(FLOAT_SPECIES);+ FloatVector dv4Norm = FloatVector.zero(FLOAT_SPECIES);++ final int limit = FLOAT_SPECIES.loopBound(elementCount);+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ final long offset = (long) i * Float.BYTES;+ FloatVector dv1 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d1 + offset, LE);+ FloatVector dv2 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d2 + offset, LE);+ FloatVector dv3 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d3 + offset, LE);+ FloatVector dv4 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d4 + offset, LE);+ FloatVector qv;+ if (qOffset == -1L) {+ qv = FloatVector.fromArray(FLOAT_SPECIES, qArray, i);+ } else {+ qv = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, qOffset + offset, LE);+ }+ qvNorm = fma(qv, qv, qvNorm);+ dv1Norm = fma(dv1, dv1, dv1Norm);+ sv1 = fma(qv, dv1, sv1);+ dv2Norm = fma(dv2, dv2, dv2Norm);+ sv2 = fma(qv, dv2, sv2);+ dv3Norm = fma(dv3, dv3, dv3Norm);+ sv3 = fma(qv, dv3, sv3);+ dv4Norm = fma(dv4, dv4, dv4Norm);+ sv4 = fma(qv, dv4, sv4);+ }+ float sum1 = sv1.reduceLanes(VectorOperators.ADD);+ float sum2 = sv2.reduceLanes(VectorOperators.ADD);+ float sum3 = sv3.reduceLanes(VectorOperators.ADD);+ float sum4 = sv4.reduceLanes(VectorOperators.ADD);+ float qNorm = qvNorm.reduceLanes(VectorOperators.ADD);+ float d1Norm = dv1Norm.reduceLanes(VectorOperators.ADD);+ float d2Norm = dv2Norm.reduceLanes(VectorOperators.ADD);+ float d3Norm = dv3Norm.reduceLanes(VectorOperators.ADD);+ float d4Norm = dv4Norm.reduceLanes(VectorOperators.ADD);++ for (; i < elementCount; i++) {+ final long offset = (long) i * Float.BYTES;+ final float qValue;+ if (qOffset == -1L) {+ qValue = qArray[i];+ } else {+ qValue = seg.get(LAYOUT_LE_FLOAT, qOffset + offset);+ }+ final float d1Value = seg.get(LAYOUT_LE_FLOAT, d1 + offset);+ final float d2Value = seg.get(LAYOUT_LE_FLOAT, d2 + offset);+ final float d3Value = seg.get(LAYOUT_LE_FLOAT, d3 + offset);+ final float d4Value = seg.get(LAYOUT_LE_FLOAT, d4 + offset);+ sum1 = fma(qValue, d1Value, sum1);+ sum2 = fma(qValue, d2Value, sum2);+ sum3 = fma(qValue, d3Value, sum3);+ sum4 = fma(qValue, d4Value, sum4);+ qNorm = fma(qValue, qValue, qNorm);+ d1Norm = fma(d1Value, d1Value, d1Norm);+ d2Norm = fma(d2Value, d2Value, d2Norm);+ d3Norm = fma(d3Value, d3Value, d3Norm);+ d4Norm = fma(d4Value, d4Value, d4Norm);+ }+ scores[0] = (float) (sum1 / Math.sqrt((double) qNorm * (double) d1Norm));+ scores[1] = (float) (sum2 / Math.sqrt((double) qNorm * (double) d2Norm));+ scores[2] = (float) (sum3 / Math.sqrt((double) qNorm * (double) d3Norm));+ scores[3] = (float) (sum4 / Math.sqrt((double) qNorm * (double) d4Norm));+ }+ }++ // -- square distance++ public static final class SqrDistance {++ private SqrDistance() {}++ public void sqrDistanceBulk(+ MemorySegment dataSeg,+ float[] scores,+ float[] q,+ long d1,+ long d2,+ long d3,+ long d4,+ int elementCount) {+ sqrDistanceBulkImpl(dataSeg, scores, q, -1L, d1, d2, d3, d4, elementCount);+ }++ public void sqrDistanceBulk(+ MemorySegment seg,+ float[] scores,+ long q,+ long d1,+ long d2,+ long d3,+ long d4,+ int elementCount) {+ sqrDistanceBulkImpl(seg, scores, null, q, d1, d2, d3, d4, elementCount);+ }++ public float sqrDistance(MemorySegment seg, long q, long d, int elementCount) {+ int i = 0;+ FloatVector sv = FloatVector.zero(FLOAT_SPECIES);+ final int limit = FLOAT_SPECIES.loopBound(elementCount);+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ final long offset = (long) i * Float.BYTES;+ FloatVector qv = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, q + offset, LE);+ FloatVector dv = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d + offset, LE);+ FloatVector diff = qv.sub(dv);+ sv = fma(diff, diff, sv);+ }+ float score = sv.reduceLanes(VectorOperators.ADD);++ for (; i < elementCount; i++) {+ final long offset = (long) i * Float.BYTES;+ float diff = seg.get(LAYOUT_LE_FLOAT, q + offset) - seg.get(LAYOUT_LE_FLOAT, d + offset);+ score = fma(diff, diff, score);+ }+ return score;+ }++ private void sqrDistanceBulkImpl(+ MemorySegment seg,+ float[] scores,+ float[] qArray,+ long qOffset,+ long d1,+ long d2,+ long d3,+ long d4,+ int elementCount) {+ int i = 0;+ FloatVector sv1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv4 = FloatVector.zero(FLOAT_SPECIES);++ final int limit = FLOAT_SPECIES.loopBound(elementCount);+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ final long offset = (long) i * Float.BYTES;+ FloatVector dv1 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d1 + offset, LE);+ FloatVector dv2 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d2 + offset, LE);+ FloatVector dv3 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d3 + offset, LE);+ FloatVector dv4 = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, d4 + offset, LE);+ FloatVector qv;+ if (qOffset == -1L) {+ qv = FloatVector.fromArray(FLOAT_SPECIES, qArray, i);+ } else {+ qv = FloatVector.fromMemorySegment(FLOAT_SPECIES, seg, qOffset + offset, LE);+ }+ FloatVector diff1 = qv.sub(dv1);+ FloatVector diff2 = qv.sub(dv2);+ FloatVector diff3 = qv.sub(dv3);+ FloatVector diff4 = qv.sub(dv4);+ sv1 = fma(diff1, diff1, sv1);+ sv2 = fma(diff2, diff2, sv2);+ sv3 = fma(diff3, diff3, sv3);+ sv4 = fma(diff4, diff4, sv4);+ }+ float sum1 = sv1.reduceLanes(VectorOperators.ADD);+ float sum2 = sv2.reduceLanes(VectorOperators.ADD);+ float sum3 = sv3.reduceLanes(VectorOperators.ADD);+ float sum4 = sv4.reduceLanes(VectorOperators.ADD);++ for (; i < elementCount; i++) {+ final long offset = (long) i * Float.BYTES;+ final float qValue;+ if (qOffset == -1L) {+ qValue = qArray[i];+ } else {+ qValue = seg.get(LAYOUT_LE_FLOAT, qOffset + offset);+ }+ float diff1 = qValue - seg.get(LAYOUT_LE_FLOAT, d1 + offset);+ float diff2 = qValue - seg.get(LAYOUT_LE_FLOAT, d2 + offset);+ float diff3 = qValue - seg.get(LAYOUT_LE_FLOAT, d3 + offset);+ float diff4 = qValue - seg.get(LAYOUT_LE_FLOAT, d4 + offset);+ sum1 = fma(diff1, diff1, sum1);+ sum2 = fma(diff2, diff2, sum2);+ sum3 = fma(diff3, diff3, sum3);+ sum4 = fma(diff4, diff4, sum4);+ }+ scores[0] = sum1;+ scores[1] = sum2;+ scores[2] = sum3;+ scores[3] = sum4;+ }+ }}