Skip to content

Apache Lucene open-source search software

Add bulk off-heap scoring for float32 vectors (#14980)

Chris Hegarty84e5df34e83c
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d0512811482..7b44e8368bf 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -233,6 +233,8 @@ Optimizations
* GITHUB#15001: Remove full integrity check from SortingStoredFieldsConsumer (Martijn van Groningen)
+* GITHUB#14980: Add bulk off-heap scoring for float32 vectors (Chris Hegarty)
+
Changes in Runtime Behavior
---------------------
* GITHUB#14823: Decrease TieredMergePolicy's default number of segments per
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java
new file mode 100644
index 00000000000..5ad753e7fbf
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED;
+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
+
+import java.io.IOException;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+// first iteration is complete garbage, so make sure we really warmup
+@Warmup(iterations = 4, time = 1)
+// real iterations. not useful to spend tons of time here, better to fork more
+@Measurement(iterations = 5, time = 1)
+// engage some noise reduction
+@Fork(
+ value = 3,
+ jvmArgsAppend = {
+ "-Xmx2g",
+ "-Xms2g",
+ "-XX:+AlwaysPreTouch",
+ "--add-modules=jdk.incubator.vector"
+ })
+public class VectorScorerFloat32Benchmark {
+
+ @Param({"1024"})
+ public int size;
+
+ public int numVectors = 128_000;
+ public int numVectorsToScore = 20_000;
+
+ float[] scores;
+ int[] indices;
+ Path path;
+ Directory dir;
+ IndexInput in;
+ KnnVectorValues values;
+ UpdateableRandomVectorScorer defDotScorer;
+ UpdateableRandomVectorScorer optDotScorer;
+
+ static final ValueLayout.OfFloat JAVA_FLOAT_LE =
+ ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+
+ @Setup(Level.Trial)
+ public void setup() throws IOException {
+ var random = ThreadLocalRandom.current();
+ path = Files.createTempDirectory("VectorScorerFloat32Benchmark");
+ dir = new MMapDirectory(path);
+ try (IndexOutput out = dir.createOutput("vector.data", IOContext.DEFAULT)) {
+ var ba = new byte[size * Float.BYTES];
+ var seg = MemorySegment.ofArray(ba);
+ for (int v = 0; v < numVectors; v++) {
+ var src = MemorySegment.ofArray(randomVector(size, random));
+ MemorySegment.copy(src, JAVA_FLOAT_UNALIGNED, 0L, seg, JAVA_FLOAT_LE, 0L, size);
+ out.writeBytes(ba, 0, ba.length);
+ }
+ }
+ perIterationInit();
+ }
+
+ @Setup(Level.Iteration)
+ public void perIterationInit() throws IOException {
+ var random = ThreadLocalRandom.current();
+ scores = new float[numVectorsToScore];
+ in = dir.openInput("vector.data", IOContext.DEFAULT);
+ int targetOrd = random.nextInt(numVectors);
+
+ // default scorer
+ values = vectorValues(size, numVectors, in, DOT_PRODUCT);
+ var def = DefaultFlatVectorScorer.INSTANCE;
+ defDotScorer = def.getRandomVectorScorerSupplier(DOT_PRODUCT, values.copy()).scorer();
+ defDotScorer.setScoringOrdinal(targetOrd);
+
+ // optimized scorer
+ var opt = FlatVectorScorerUtil.getLucene99FlatVectorsScorer();
+ optDotScorer = opt.getRandomVectorScorerSupplier(DOT_PRODUCT, values.copy()).scorer();
+ optDotScorer.setScoringOrdinal(targetOrd);
+
+ List<Integer> list = IntStream.range(0, numVectors).boxed().collect(Collectors.toList());
+ Collections.shuffle(list, random);
+ indices = list.stream().limit(numVectorsToScore).mapToInt(i -> i).toArray();
+ }
+
+ @TearDown
+ public void teardown() throws IOException {
+ IOUtils.close(in);
+ dir.deleteFile("vector.data");
+ IOUtils.close(dir);
+ Files.delete(path);
+ }
+
+ @Benchmark
+ public float[] dotProductDefault() throws IOException {
+ for (int v = 0; v < numVectorsToScore; v++) {
+ scores[v] = defDotScorer.score(indices[v]);
+ }
+ return scores;
+ }
+
+ @Benchmark
+ public float[] dotProductDefaultBulk() throws IOException {
+ defDotScorer.bulkScore(indices, scores, indices.length);
+ return scores;
+ }
+
+ @Benchmark
+ public float[] dotProductOptScorer() throws IOException {
+ for (int v = 0; v < numVectorsToScore; v++) {
+ scores[v] = optDotScorer.score(indices[v]);
+ }
+ return scores;
+ }
+
+ @Benchmark
+ public float[] dotProductOptBulkScore() throws IOException {
+ optDotScorer.bulkScore(indices, scores, indices.length);
+ return scores;
+ }
+
+ static float[] randomVector(int dims, Random random) {
+ float[] fa = new float[dims];
+ for (int i = 0; i < dims; ++i) {
+ fa[i] = random.nextFloat();
+ }
+ return fa;
+ }
+
+ static KnnVectorValues vectorValues(
+ int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
+ int byteSize = dims * Float.BYTES;
+ return new OffHeapFloatVectorValues.DenseOffHeapVectorValues(
+ dims,
+ size,
+ in.slice("test", 0, in.length()),
+ byteSize,
+ new ThrowingFlatVectorScorer(),
+ sim);
+ }
+
+ static final class ThrowingFlatVectorScorer implements FlatVectorsScorer {
+
+ @Override
+ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
+ VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public RandomVectorScorer getRandomVectorScorer(
+ VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public RandomVectorScorer getRandomVectorScorer(
+ VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, byte[] target) {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java
new file mode 100644
index 00000000000..9d6d9db6ce1
--- /dev/null
+++ b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+public class TestVectorScorerFloat32Benchmark extends LuceneTestCase {
+
+ VectorScorerFloat32Benchmark bench;
+ float delta;
+
+ @Before
+ public void setup() throws IOException {
+ bench = new VectorScorerFloat32Benchmark();
+ bench.size = 1024;
+ bench.numVectors = random().nextInt(1, 256);
+ bench.numVectorsToScore = random().nextInt(bench.numVectors);
+ delta = 1e-3f * bench.size;
+ bench.setup();
+ }
+
+ @After
+ public void teardown() throws IOException {
+ bench.teardown();
+ }
+
+ public void testDotProduct() throws IOException {
+ Arrays.fill(bench.scores, 0.0f);
+ bench.dotProductDefault();
+ var expectedScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);
+
+ Arrays.fill(bench.scores, 0.0f);
+ bench.dotProductDefaultBulk();
+ var bulkScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);
+ assertArrayEquals(expectedScores, bulkScores, delta);
+
+ Arrays.fill(bench.scores, 0.0f);
+ bench.dotProductOptScorer();
+ var actualScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);
+ assertArrayEquals(expectedScores, actualScores, delta);
+
+ Arrays.fill(bench.scores, 0.0f);
+ bench.dotProductOptBulkScore();
+ actualScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);
+ assertArrayEquals(expectedScores, actualScores, delta);
+ }
+}
diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
index bd8cbb2c388..456747e563f 100644
--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
@@ -21,6 +21,7 @@ import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
import org.apache.lucene.codecs.lucene95.HasIndexSlice;
import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
@@ -41,6 +42,30 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorer
@Override
public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityType, KnnVectorValues vectorValues) throws IOException {
+ return switch (vectorValues.getEncoding()) {
+ case FLOAT32 -> getFloatScoringSupplier((FloatVectorValues) vectorValues, similarityType);
+ case BYTE -> getByteScorerSupplier((ByteVectorValues) vectorValues, similarityType);
+ };
+ }
+
+ private RandomVectorScorerSupplier getFloatScoringSupplier(
+ FloatVectorValues vectorValues, VectorSimilarityFunction similarityType) throws IOException {
+ if (similarityType == VectorSimilarityFunction.DOT_PRODUCT) { // dot product for now
+ if (vectorValues instanceof HasIndexSlice sliceableValues
+ && sliceableValues.getSlice() != null) {
+ var scorer =
+ Lucene99MemorySegmentFloatVectorScorerSupplier.create(
+ similarityType, sliceableValues.getSlice(), vectorValues);
+ if (scorer.isPresent()) {
+ return scorer.get();
+ }
+ }
+ }
+ return delegate.getRandomVectorScorerSupplier(similarityType, vectorValues);
+ }
+
+ private RandomVectorScorerSupplier getByteScorerSupplier(
+ ByteVectorValues vectorValues, VectorSimilarityFunction similarityType) throws IOException {
// a quantized values here is a wrapping or delegation issue
assert !(vectorValues instanceof QuantizedByteVectorValues);
// currently only supports binary vectors
@@ -61,7 +86,19 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorer
public RandomVectorScorer getRandomVectorScorer(
VectorSimilarityFunction similarityType, KnnVectorValues vectorValues, float[] target)
throws IOException {
- // currently only supports binary vectors, so always delegate
+ checkDimensions(target.length, vectorValues.dimension());
+ if (similarityType == VectorSimilarityFunction.DOT_PRODUCT) { // just for now
+ if (vectorValues instanceof FloatVectorValues fvv
+ && fvv instanceof HasIndexSlice floatVectorValues
+ && floatVectorValues.getSlice() != null) {
+ var scorer =
+ Lucene99MemorySegmentFloatVectorScorer.create(
+ similarityType, floatVectorValues.getSlice(), fvv, target);
+ if (scorer.isPresent()) {
+ return scorer.get();
+ }
+ }
+ }
return delegate.getRandomVectorScorer(similarityType, vectorValues, target);
}
diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java
new file mode 100644
index 00000000000..c1fd989aa64
--- /dev/null
+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.internal.vectorization;
+
+import static org.apache.lucene.util.VectorUtil.normalizeToUnitInterval;
+
+import java.io.IOException;
+import java.lang.foreign.MemorySegment;
+import java.util.Optional;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MemorySegmentAccessInput;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+
+abstract sealed class Lucene99MemorySegmentFloatVectorScorer
+ extends RandomVectorScorer.AbstractRandomVectorScorer {
+
+ final FloatVectorValues values;
+ final int vectorByteSize;
+ final MemorySegmentAccessInput input;
+ final float[] query;
+ byte[] scratch;
+
+ /**
+ * Return an optional whose value, if present, is the scorer. Otherwise, an empty optional is
+ * returned.
+ */
+ public static Optional<Lucene99MemorySegmentFloatVectorScorer> create(
+ VectorSimilarityFunction type,
+ IndexInput input,
+ FloatVectorValues values,
+ float[] queryVector) {
+ input = FilterIndexInput.unwrapOnlyTest(input);
+ if (!(input instanceof MemorySegmentAccessInput msInput)) {
+ return Optional.empty();
+ }
+ checkInvariants(values.size(), values.getVectorByteLength(), input);
+ return switch (type) {
+ case COSINE -> Optional.empty(); // of(new CosineScorer(msInput, values, queryVector));
+ case DOT_PRODUCT -> Optional.of(new DotProductScorer(msInput, values, queryVector));
+ case EUCLIDEAN -> Optional.empty(); // of(new EuclideanScorer(msInput, values, queryVector));
+ case MAXIMUM_INNER_PRODUCT ->
+ Optional.empty(); // of(new MaxInnerProductScorer(msInput, values, queryVector));
+ };
+ }
+
+ Lucene99MemorySegmentFloatVectorScorer(
+ MemorySegmentAccessInput input, FloatVectorValues values, float[] queryVector) {
+ super(values);
+ this.values = values;
+ this.input = input;
+ this.vectorByteSize = values.getVectorByteLength();
+ this.query = queryVector;
+ }
+
+ final MemorySegment getSegment(int ord) throws IOException {
+ checkOrdinal(ord);
+ long byteOffset = (long) ord * vectorByteSize;
+ MemorySegment seg = input.segmentSliceOrNull(byteOffset, vectorByteSize);
+ if (seg == null) {
+ if (scratch == null) {
+ scratch = new byte[vectorByteSize];
+ }
+ input.readBytes(byteOffset, scratch, 0, vectorByteSize);
+ seg = MemorySegment.ofArray(scratch);
+ }
+ return seg;
+ }
+
+ static void checkInvariants(int maxOrd, int vectorByteLength, IndexInput input) {
+ if (input.length() < (long) vectorByteLength * maxOrd) {
+ throw new IllegalArgumentException("input length is less than expected vector data");
+ }
+ }
+
+ final void checkOrdinal(int ord) {
+ if (ord < 0 || ord >= maxOrd()) {
+ throw new IllegalArgumentException("illegal ordinal: " + ord);
+ }
+ }
+
+ static final class DotProductScorer extends Lucene99MemorySegmentFloatVectorScorer {
+
+ static final MemorySegmentBulkVectorOps.DotFromQueryArray DOT_OPS =
+ new MemorySegmentBulkVectorOps.DotFromQueryArray();
+
+ DotProductScorer(MemorySegmentAccessInput input, FloatVectorValues values, float[] query) {
+ super(input, values, query);
+ }
+
+ @Override
+ public float score(int node) throws IOException {
+ checkOrdinal(node);
+ // just delegates to existing scorer that copies on-heap
+ return VectorSimilarityFunction.DOT_PRODUCT.compare(query, values.vectorValue(node));
+ }
+
+ @Override
+ public void bulkScore(int[] nodes, float[] scores, int numNodes) throws IOException {
+ float[] scratchScores = new float[4];
+ int i = 0;
+ final int limit = numNodes & ~3;
+ for (; i < limit; i += 4) {
+ MemorySegment ms1 = getSegment(nodes[i]);
+ MemorySegment ms2 = getSegment(nodes[i + 1]);
+ MemorySegment ms3 = getSegment(nodes[i + 2]);
+ MemorySegment ms4 = getSegment(nodes[i + 3]);
+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms4, query.length);
+ scores[i + 0] = normalizeToUnitInterval(scratchScores[0]);
+ scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);
+ scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);
+ scores[i + 3] = normalizeToUnitInterval(scratchScores[3]);
+ }
+ // Handle remaining 1–3 nodes in bulk (if any)
+ int remaining = numNodes - i;
+ if (remaining > 0) {
+ MemorySegment ms1 = getSegment(nodes[i]);
+ MemorySegment ms2 = (remaining > 1) ? getSegment(nodes[i + 1]) : ms1;
+ MemorySegment ms3 = (remaining > 2) ? getSegment(nodes[i + 2]) : ms1;
+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms1, query.length);
+ scores[i] = normalizeToUnitInterval(scratchScores[0]);
+ if (remaining > 1) scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);
+ if (remaining > 2) scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java
new file mode 100644
index 00000000000..33a3a0e33dd
--- /dev/null
+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.internal.vectorization;
+
+import static org.apache.lucene.util.VectorUtil.normalizeToUnitInterval;
+
+import java.io.IOException;
+import java.lang.foreign.MemorySegment;
+import java.util.Optional;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MemorySegmentAccessInput;
+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
+
+/** A score supplier of vectors whose element size is byte. */
+public abstract sealed class Lucene99MemorySegmentFloatVectorScorerSupplier
+ implements RandomVectorScorerSupplier {
+ final int vectorByteSize;
+ final int maxOrd;
+ final int dims;
+ final MemorySegmentAccessInput input;
+ final FloatVectorValues values; // to support ordToDoc/getAcceptOrds
+ byte[] queryScratch, scratch1, scratch2, scratch3, scratch4;
+
+ /**
+ * Return an optional whose value, if present, is the scorer supplier. Otherwise, an empty
+ * optional is returned.
+ */
+ static Optional<RandomVectorScorerSupplier> create(
+ VectorSimilarityFunction type, IndexInput input, FloatVectorValues values) {
+ input = FilterIndexInput.unwrapOnlyTest(input);
+ if (!(input instanceof MemorySegmentAccessInput msInput)) {
+ return Optional.empty();
+ }
+ checkInvariants(values.size(), values.getVectorByteLength(), input);
+ return switch (type) {
+ case COSINE -> Optional.empty(); // of(new CosineSupplier(msInput, values));
+ case DOT_PRODUCT -> Optional.of(new DotProductSupplier(msInput, values));
+ case EUCLIDEAN -> Optional.empty(); // of(new EuclideanSupplier(msInput, values));
+ case MAXIMUM_INNER_PRODUCT ->
+ Optional.empty(); // of(new MaxInnerProductSupplier(msInput, values));
+ };
+ }
+
+ Lucene99MemorySegmentFloatVectorScorerSupplier(
+ MemorySegmentAccessInput input, FloatVectorValues values) {
+ this.input = input;
+ this.values = values;
+ this.vectorByteSize = values.getVectorByteLength();
+ this.maxOrd = values.size();
+ this.dims = values.dimension();
+ }
+
+ static void checkInvariants(int maxOrd, int vectorByteLength, IndexInput input) {
+ if (input.length() < (long) vectorByteLength * maxOrd) {
+ throw new IllegalArgumentException("input length is less than expected vector data");
+ }
+ }
+
+ final void checkOrdinal(int ord) {
+ if (ord < 0 || ord >= maxOrd) {
+ throw new IllegalArgumentException("illegal ordinal: " + ord);
+ }
+ }
+
+ final MemorySegment getSegment(int ord, byte[] scratch) throws IOException {
+ long byteOffset = (long) ord * vectorByteSize;
+ MemorySegment seg = input.segmentSliceOrNull(byteOffset, vectorByteSize);
+ if (seg == null) {
+ if (scratch == null) {
+ scratch = new byte[vectorByteSize];
+ }
+ input.readBytes(byteOffset, scratch, 0, vectorByteSize);
+ seg = MemorySegment.ofArray(scratch);
+ }
+ return seg;
+ }
+
+ static final class DotProductSupplier extends Lucene99MemorySegmentFloatVectorScorerSupplier {
+
+ static final MemorySegmentBulkVectorOps.DotFromQuerySegment DOT_OPS =
+ new MemorySegmentBulkVectorOps.DotFromQuerySegment();
+
+ DotProductSupplier(MemorySegmentAccessInput input, FloatVectorValues values) {
+ super(input, values);
+ }
+
+ @Override
+ public UpdateableRandomVectorScorer scorer() {
+ return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(values) {
+ private int queryOrd;
+
+ @Override
+ public float score(int node) throws IOException {
+ checkOrdinal(node);
+ MemorySegment query = getSegment(queryOrd, queryScratch);
+ MemorySegment ms = getSegment(node, scratch1);
+ var raw = DOT_OPS.dotProduct(query, ms, dims);
+ return normalizeToUnitInterval(raw);
+ }
+
+ @Override
+ public void bulkScore(int[] nodes, float[] scores, int numNodes) throws IOException {
+ // TODO checkOrdinal(node1 ....);
+ float[] scratchScores = new float[4];
+ int i = 0;
+ MemorySegment query = getSegment(queryOrd, queryScratch);
+ final int limit = numNodes & ~3;
+ for (; i < limit; i += 4) {
+ MemorySegment ms1 = getSegment(nodes[i], scratch1);
+ MemorySegment ms2 = getSegment(nodes[i + 1], scratch2);
+ MemorySegment ms3 = getSegment(nodes[i + 2], scratch3);
+ MemorySegment ms4 = getSegment(nodes[i + 3], scratch4);
+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms4, dims);
+ scores[i + 0] = normalizeToUnitInterval(scratchScores[0]);
+ scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);
+ scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);
+ scores[i + 3] = normalizeToUnitInterval(scratchScores[3]);
+ }
+ // Handle remaining 1–3 nodes in bulk (if any)
+ int remaining = numNodes - i;
+ if (remaining > 0) {
+ MemorySegment ms1 = getSegment(nodes[i], scratch1);
+ MemorySegment ms2 = (remaining > 1) ? getSegment(nodes[i + 1], scratch2) : ms1;
+ MemorySegment ms3 = (remaining > 2) ? getSegment(nodes[i + 2], scratch3) : ms1;
+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms1, dims);
+ scores[i] = normalizeToUnitInterval(scratchScores[0]);
+ if (remaining > 1) scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);
+ if (remaining > 2) scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);
+ }
+ }
+
+ @Override
+ public void setScoringOrdinal(int node) {
+ checkOrdinal(node);
+ queryOrd = node;
+ }
+ };
+ }
+
+ @Override
+ public DotProductSupplier copy() throws IOException {
+ return new DotProductSupplier(input.clone(), values);
+ }
+ }
+}
diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java
new file mode 100644
index 00000000000..8c6e5b096b7
--- /dev/null
+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.internal.vectorization;
+
+import static org.apache.lucene.internal.vectorization.PanamaVectorConstants.PREFERRED_VECTOR_BITSIZE;
+import static org.apache.lucene.internal.vectorization.PanamaVectorUtilSupport.fma;
+
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+import java.nio.ByteOrder;
+import jdk.incubator.vector.FloatVector;
+import jdk.incubator.vector.VectorOperators;
+import jdk.incubator.vector.VectorShape;
+import jdk.incubator.vector.VectorSpecies;
+
+/** Implementations of bulk vector comparison operations. Currently only supports float32. */
+public final class MemorySegmentBulkVectorOps {
+
+ static final VectorSpecies<Float> FLOAT_SPECIES =
+ VectorSpecies.of(float.class, VectorShape.forBitSize(PREFERRED_VECTOR_BITSIZE));
+ static final ByteOrder LE = ByteOrder.LITTLE_ENDIAN;
+ static final ValueLayout.OfFloat LAYOUT_LE_FLOAT = ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(LE);
+
+ private MemorySegmentBulkVectorOps() {}
+
+ sealed interface FloatVectorLoader {
+ /** Returns the number of float elements. */
+ int length();
+
+ FloatVector load(VectorSpecies<Float> species, int index);
+
+ float tail(int index);
+ }
+
+ record FloatArrayLoader(float[] arr) implements FloatVectorLoader {
+ @Override
+ public int length() {
+ return arr.length;
+ }
+
+ @Override
+ public FloatVector load(VectorSpecies<Float> species, int index) {
+ // assert index + species.length() <= length();
+ return FloatVector.fromArray(species, arr, index);
+ }
+
+ @Override
+ public float tail(int index) {
+ // assert index <= length();
+ return arr[index];
+ }
+ }
+
+ record FloatMemorySegmentLoader(MemorySegment segment) implements FloatVectorLoader {
+ @Override
+ public int length() {
+ return Math.toIntExact(segment.byteSize() / Float.BYTES);
+ }
+
+ @Override
+ public FloatVector load(VectorSpecies<Float> species, int index) {
+ // assert index + species.length() <= length();
+ return FloatVector.fromMemorySegment(species, segment, (long) index * Float.BYTES, LE);
+ }
+
+ @Override
+ public float tail(int index) {
+ // assert index <= length();
+ return segment.get(LAYOUT_LE_FLOAT, (long) index * Float.BYTES);
+ }
+ }
+
+ public static final class DotFromQueryArray extends AbstractDotProduct {
+
+ public void dotProductBulk(
+ float[] scores,
+ float[] q,
+ MemorySegment d1,
+ MemorySegment d2,
+ MemorySegment d3,
+ MemorySegment d4,
+ int elementCount) {
+ super.dotProductBulk(scores, new FloatArrayLoader(q), d1, d2, d3, d4, elementCount);
+ }
+ }
+
+ public static final class DotFromQuerySegment extends AbstractDotProduct {
+
+ public void dotProductBulk(
+ float[] scores,
+ MemorySegment q,
+ MemorySegment d1,
+ MemorySegment d2,
+ MemorySegment d3,
+ MemorySegment d4,
+ int elementCount) {
+ super.dotProductBulk(scores, new FloatMemorySegmentLoader(q), d1, d2, d3, d4, elementCount);
+ }
+
+ public float dotProduct(MemorySegment q, MemorySegment d, int elementCount) {
+ int i = 0;
+ FloatVector sv = FloatVector.zero(FLOAT_SPECIES);
+ final int limit = FLOAT_SPECIES.loopBound(elementCount);
+ for (; i < limit; i += FLOAT_SPECIES.length()) {
+ final long offset = (long) i * Float.BYTES;
+ FloatVector qv = FloatVector.fromMemorySegment(FLOAT_SPECIES, q, offset, LE);
+ FloatVector dv = FloatVector.fromMemorySegment(FLOAT_SPECIES, d, offset, LE);
+ sv = fma(qv, dv, sv);
+ }
+ float score = sv.reduceLanes(VectorOperators.ADD);
+
+ for (; i < elementCount; i++) {
+ final long offset = (long) i * Float.BYTES;
+ score += q.get(LAYOUT_LE_FLOAT, offset) * d.get(LAYOUT_LE_FLOAT, offset);
+ }
+ return score;
+ }
+ }
+
+ abstract static class AbstractDotProduct {
+
+ protected void dotProductBulk(
+ float[] scores,
+ FloatVectorLoader q,
+ MemorySegment d1,
+ MemorySegment d2,
+ MemorySegment d3,
+ MemorySegment d4,
+ int elementCount) {
+ int i = 0;
+ FloatVector sv1 = FloatVector.zero(FLOAT_SPECIES);
+ FloatVector sv2 = FloatVector.zero(FLOAT_SPECIES);
+ FloatVector sv3 = FloatVector.zero(FLOAT_SPECIES);
+ FloatVector sv4 = FloatVector.zero(FLOAT_SPECIES);
+
+ final int limit = FLOAT_SPECIES.loopBound(elementCount);
+ for (; i < limit; i += FLOAT_SPECIES.length()) {
+ final long offset = (long) i * Float.BYTES;
+ FloatVector dv1 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d1, offset, LE);
+ FloatVector dv2 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d2, offset, LE);
+ FloatVector dv3 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d3, offset, LE);
+ FloatVector dv4 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d4, offset, LE);
+ FloatVector qv = q.load(FLOAT_SPECIES, i);
+ sv1 = fma(qv, dv1, sv1);
+ sv2 = fma(qv, dv2, sv2);
+ sv3 = fma(qv, dv3, sv3);
+ sv4 = fma(qv, dv4, sv4);
+ }
+ float sum1 = sv1.reduceLanes(VectorOperators.ADD);
+ float sum2 = sv2.reduceLanes(VectorOperators.ADD);
+ float sum3 = sv3.reduceLanes(VectorOperators.ADD);
+ float sum4 = sv4.reduceLanes(VectorOperators.ADD);
+
+ for (; i < elementCount; i++) {
+ final long offset = (long) i * Float.BYTES;
+ final float qValue = q.tail(i);
+ sum1 = fma(qValue, d1.get(LAYOUT_LE_FLOAT, offset), sum1);
+ sum2 = fma(qValue, d2.get(LAYOUT_LE_FLOAT, offset), sum2);
+ sum3 = fma(qValue, d3.get(LAYOUT_LE_FLOAT, offset), sum3);
+ sum4 = fma(qValue, d4.get(LAYOUT_LE_FLOAT, offset), sum4);
+ }
+ scores[0] = sum1;
+ scores[1] = sum2;
+ scores[2] = sum3;
+ scores[3] = sum4;
+ }
+ }
+}
diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
index 7572560923b..731aa8ab539 100644
--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
@@ -85,7 +85,7 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {
// the way FMA should work! if available use it, otherwise fall back to mul/add
@SuppressForbidden(reason = "Uses FMA only where fast and carefully contained")
- private static FloatVector fma(FloatVector a, FloatVector b, FloatVector c) {
+ static FloatVector fma(FloatVector a, FloatVector b, FloatVector c) {
if (Constants.HAS_FAST_VECTOR_FMA) {
return a.fma(b, c);
} else {
@@ -94,7 +94,7 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {
}
@SuppressForbidden(reason = "Uses FMA only where fast and carefully contained")
- private static float fma(float a, float b, float c) {
+ static float fma(float a, float b, float c) {
if (Constants.HAS_FAST_SCALAR_FMA) {
return Math.fma(a, b, c);
} else {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java
index 1dab736945d..54dc8015dde 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java
@@ -204,6 +204,7 @@ public class TestFlatVectorScorer extends BaseVectorizationTestCase {
var values = byteVectorValues(dims, size, in, sim);
assertBulkEqualsNonBulk(values, sim);
assertBulkEqualsNonBulkSupplier(values, sim);
+ assertScoresAgainstDefaultFlatScorer(values, sim);
}
}
}
@@ -226,11 +227,27 @@ public class TestFlatVectorScorer extends BaseVectorizationTestCase {
var values = floatVectorValues(dims, size, in, sim);
assertBulkEqualsNonBulk(values, sim);
assertBulkEqualsNonBulkSupplier(values, sim);
+ assertScoresAgainstDefaultFlatScorer(values, sim);
}
}
}
}
+ public void testOnHeapBulkScorerFloats() throws IOException {
+ int dims = random().nextInt(1, 1024);
+ int size = random().nextInt(2, 255);
+ List<float[]> vectors = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ vectors.add(randomFloatVector(dims));
+ }
+ var values = FloatVectorValues.fromFloats(vectors, dims);
+ for (var sim : List.of(COSINE, DOT_PRODUCT, EUCLIDEAN, MAXIMUM_INNER_PRODUCT)) {
+ assertBulkEqualsNonBulk(values, sim);
+ assertBulkEqualsNonBulkSupplier(values, sim);
+ assertScoresAgainstDefaultFlatScorer(values, sim);
+ }
+ }
+
void assertBulkEqualsNonBulk(KnnVectorValues values, VectorSimilarityFunction sim)
throws IOException {
final int dims = values.dimension();
@@ -256,19 +273,46 @@ public class TestFlatVectorScorer extends BaseVectorizationTestCase {
throws IOException {
final int size = values.size();
final float delta = 1e-3f * size;
- var ss = flatVectorsScorer.getRandomVectorScorerSupplier(sim, values);
- var updatableScorer = ss.scorer();
+ var supplier = flatVectorsScorer.getRandomVectorScorerSupplier(sim, values);
+ for (var ss : List.of(supplier, supplier.copy())) {
+ var updatableScorer = ss.scorer();
+ var targetNode = random().nextInt(size);
+ updatableScorer.setScoringOrdinal(targetNode);
+ int[] indices = randomIndices(size);
+ float[] expectedScores = new float[size];
+ for (int i = 0; i < size; i++) {
+ expectedScores[i] = updatableScorer.score(indices[i]);
+ }
+ float[] bulkScores = new float[size];
+ updatableScorer.bulkScore(indices, bulkScores, size);
+ assertArrayEquals(expectedScores, bulkScores, delta);
+ assertNoScoreBeyondNumNodes(updatableScorer, size);
+ }
+ }
+
+ // asserts scores against the default scorer.
+ void assertScoresAgainstDefaultFlatScorer(KnnVectorValues values, VectorSimilarityFunction sim)
+ throws IOException {
+ final int size = values.size();
+ final float delta = 1e-3f * size;
var targetNode = random().nextInt(size);
- updatableScorer.setScoringOrdinal(targetNode);
int[] indices = randomIndices(size);
+ var defaultScorer =
+ DefaultFlatVectorScorer.INSTANCE.getRandomVectorScorerSupplier(sim, values).scorer();
+ defaultScorer.setScoringOrdinal(targetNode);
float[] expectedScores = new float[size];
for (int i = 0; i < size; i++) {
- expectedScores[i] = updatableScorer.score(indices[i]);
+ expectedScores[i] = defaultScorer.score(indices[i]);
+ }
+
+ var supplier = flatVectorsScorer.getRandomVectorScorerSupplier(sim, values);
+ for (var ss : List.of(supplier, supplier.copy())) {
+ var updatableScorer = ss.scorer();
+ updatableScorer.setScoringOrdinal(targetNode);
+ float[] bulkScores = new float[size];
+ updatableScorer.bulkScore(indices, bulkScores, size);
+ assertArrayEquals(expectedScores, bulkScores, delta);
}
- float[] bulkScores = new float[size];
- updatableScorer.bulkScore(indices, bulkScores, size);
- assertArrayEquals(expectedScores, bulkScores, delta);
- assertNoScoreBeyondNumNodes(updatableScorer, size);
}
void assertNoScoreBeyondNumNodes(RandomVectorScorer scorer, int maxSize) throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
index 233be3a139f..be099398ba6 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
@@ -91,7 +91,7 @@ import org.apache.lucene.util.hnsw.HnswGraph.NodesIterator;
abstract class HnswGraphTestCase<T> extends LuceneTestCase {
VectorSimilarityFunction similarityFunction;
- DefaultFlatVectorScorer flatVectorScorer = new DefaultFlatVectorScorer();
+ DefaultFlatVectorScorer flatVectorScorer = DefaultFlatVectorScorer.INSTANCE;
abstract VectorEncoding getVectorEncoding();