Apache Lucene open-source search software
Add bulk off-heap scoring for float32 vectors (#14980)
Chris Hegarty84e5df34e83c
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex d0512811482..7b44e8368bf 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -233,6 +233,8 @@ Optimizations* GITHUB#15001: Remove full integrity check from SortingStoredFieldsConsumer (Martijn van Groningen)+* GITHUB#14980: Add bulk off-heap scoring for float32 vectors (Chris Hegarty)+Changes in Runtime Behavior---------------------* GITHUB#14823: Decrease TieredMergePolicy's default number of segments perdiff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.javanew file mode 100644index 00000000000..5ad753e7fbf--- /dev/null+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerFloat32Benchmark.java@@ -0,0 +1,216 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.benchmark.jmh;++import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED;+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;++import java.io.IOException;+import java.lang.foreign.MemorySegment;+import java.lang.foreign.ValueLayout;+import java.nio.ByteOrder;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Collections;+import java.util.List;+import java.util.Random;+import java.util.concurrent.ThreadLocalRandom;+import java.util.concurrent.TimeUnit;+import java.util.stream.Collectors;+import java.util.stream.IntStream;+import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;+import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;+import org.apache.lucene.index.KnnVectorValues;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.store.MMapDirectory;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.hnsw.RandomVectorScorer;+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;+import org.openjdk.jmh.annotations.Benchmark;+import org.openjdk.jmh.annotations.BenchmarkMode;+import org.openjdk.jmh.annotations.Fork;+import org.openjdk.jmh.annotations.Level;+import org.openjdk.jmh.annotations.Measurement;+import org.openjdk.jmh.annotations.Mode;+import org.openjdk.jmh.annotations.OutputTimeUnit;+import org.openjdk.jmh.annotations.Param;+import org.openjdk.jmh.annotations.Scope;+import org.openjdk.jmh.annotations.Setup;+import org.openjdk.jmh.annotations.State;+import org.openjdk.jmh.annotations.TearDown;+import org.openjdk.jmh.annotations.Warmup;++@BenchmarkMode(Mode.AverageTime)+@OutputTimeUnit(TimeUnit.MILLISECONDS)+@State(Scope.Benchmark)+// first iteration is complete garbage, so make sure we really warmup+@Warmup(iterations = 4, time = 1)+// real iterations. not useful to spend tons of time here, better to fork more+@Measurement(iterations = 5, time = 1)+// engage some noise reduction+@Fork(+ value = 3,+ jvmArgsAppend = {+ "-Xmx2g",+ "-Xms2g",+ "-XX:+AlwaysPreTouch",+ "--add-modules=jdk.incubator.vector"+ })+public class VectorScorerFloat32Benchmark {++ @Param({"1024"})+ public int size;++ public int numVectors = 128_000;+ public int numVectorsToScore = 20_000;++ float[] scores;+ int[] indices;+ Path path;+ Directory dir;+ IndexInput in;+ KnnVectorValues values;+ UpdateableRandomVectorScorer defDotScorer;+ UpdateableRandomVectorScorer optDotScorer;++ static final ValueLayout.OfFloat JAVA_FLOAT_LE =+ ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);++ @Setup(Level.Trial)+ public void setup() throws IOException {+ var random = ThreadLocalRandom.current();+ path = Files.createTempDirectory("VectorScorerFloat32Benchmark");+ dir = new MMapDirectory(path);+ try (IndexOutput out = dir.createOutput("vector.data", IOContext.DEFAULT)) {+ var ba = new byte[size * Float.BYTES];+ var seg = MemorySegment.ofArray(ba);+ for (int v = 0; v < numVectors; v++) {+ var src = MemorySegment.ofArray(randomVector(size, random));+ MemorySegment.copy(src, JAVA_FLOAT_UNALIGNED, 0L, seg, JAVA_FLOAT_LE, 0L, size);+ out.writeBytes(ba, 0, ba.length);+ }+ }+ perIterationInit();+ }++ @Setup(Level.Iteration)+ public void perIterationInit() throws IOException {+ var random = ThreadLocalRandom.current();+ scores = new float[numVectorsToScore];+ in = dir.openInput("vector.data", IOContext.DEFAULT);+ int targetOrd = random.nextInt(numVectors);++ // default scorer+ values = vectorValues(size, numVectors, in, DOT_PRODUCT);+ var def = DefaultFlatVectorScorer.INSTANCE;+ defDotScorer = def.getRandomVectorScorerSupplier(DOT_PRODUCT, values.copy()).scorer();+ defDotScorer.setScoringOrdinal(targetOrd);++ // optimized scorer+ var opt = FlatVectorScorerUtil.getLucene99FlatVectorsScorer();+ optDotScorer = opt.getRandomVectorScorerSupplier(DOT_PRODUCT, values.copy()).scorer();+ optDotScorer.setScoringOrdinal(targetOrd);++ List<Integer> list = IntStream.range(0, numVectors).boxed().collect(Collectors.toList());+ Collections.shuffle(list, random);+ indices = list.stream().limit(numVectorsToScore).mapToInt(i -> i).toArray();+ }++ @TearDown+ public void teardown() throws IOException {+ IOUtils.close(in);+ dir.deleteFile("vector.data");+ IOUtils.close(dir);+ Files.delete(path);+ }++ @Benchmark+ public float[] dotProductDefault() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = defDotScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] dotProductDefaultBulk() throws IOException {+ defDotScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }++ @Benchmark+ public float[] dotProductOptScorer() throws IOException {+ for (int v = 0; v < numVectorsToScore; v++) {+ scores[v] = optDotScorer.score(indices[v]);+ }+ return scores;+ }++ @Benchmark+ public float[] dotProductOptBulkScore() throws IOException {+ optDotScorer.bulkScore(indices, scores, indices.length);+ return scores;+ }++ static float[] randomVector(int dims, Random random) {+ float[] fa = new float[dims];+ for (int i = 0; i < dims; ++i) {+ fa[i] = random.nextFloat();+ }+ return fa;+ }++ static KnnVectorValues vectorValues(+ int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {+ int byteSize = dims * Float.BYTES;+ return new OffHeapFloatVectorValues.DenseOffHeapVectorValues(+ dims,+ size,+ in.slice("test", 0, in.length()),+ byteSize,+ new ThrowingFlatVectorScorer(),+ sim);+ }++ static final class ThrowingFlatVectorScorer implements FlatVectorsScorer {++ @Override+ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(+ VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues) {+ throw new UnsupportedOperationException();+ }++ @Override+ public RandomVectorScorer getRandomVectorScorer(+ VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target) {+ throw new UnsupportedOperationException();+ }++ @Override+ public RandomVectorScorer getRandomVectorScorer(+ VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, byte[] target) {+ throw new UnsupportedOperationException();+ }+ }+}diff --git a/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.javanew file mode 100644index 00000000000..9d6d9db6ce1--- /dev/null+++ b/lucene/benchmark-jmh/src/test/org/apache/lucene/benchmark/jmh/TestVectorScorerFloat32Benchmark.java@@ -0,0 +1,65 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.benchmark.jmh;++import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.junit.After;+import org.junit.Before;++public class TestVectorScorerFloat32Benchmark extends LuceneTestCase {++ VectorScorerFloat32Benchmark bench;+ float delta;++ @Before+ public void setup() throws IOException {+ bench = new VectorScorerFloat32Benchmark();+ bench.size = 1024;+ bench.numVectors = random().nextInt(1, 256);+ bench.numVectorsToScore = random().nextInt(bench.numVectors);+ delta = 1e-3f * bench.size;+ bench.setup();+ }++ @After+ public void teardown() throws IOException {+ bench.teardown();+ }++ public void testDotProduct() throws IOException {+ Arrays.fill(bench.scores, 0.0f);+ bench.dotProductDefault();+ var expectedScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);++ Arrays.fill(bench.scores, 0.0f);+ bench.dotProductDefaultBulk();+ var bulkScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);+ assertArrayEquals(expectedScores, bulkScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.dotProductOptScorer();+ var actualScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);+ assertArrayEquals(expectedScores, actualScores, delta);++ Arrays.fill(bench.scores, 0.0f);+ bench.dotProductOptBulkScore();+ actualScores = Arrays.copyOfRange(bench.scores, 0, bench.scores.length);+ assertArrayEquals(expectedScores, actualScores, delta);+ }+}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.javaindex bd8cbb2c388..456747e563f 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java@@ -21,6 +21,7 @@ import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;import org.apache.lucene.codecs.lucene95.HasIndexSlice;import org.apache.lucene.index.ByteVectorValues;+import org.apache.lucene.index.FloatVectorValues;import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.util.hnsw.RandomVectorScorer;@@ -41,6 +42,30 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorer@Overridepublic RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction similarityType, KnnVectorValues vectorValues) throws IOException {+ return switch (vectorValues.getEncoding()) {+ case FLOAT32 -> getFloatScoringSupplier((FloatVectorValues) vectorValues, similarityType);+ case BYTE -> getByteScorerSupplier((ByteVectorValues) vectorValues, similarityType);+ };+ }++ private RandomVectorScorerSupplier getFloatScoringSupplier(+ FloatVectorValues vectorValues, VectorSimilarityFunction similarityType) throws IOException {+ if (similarityType == VectorSimilarityFunction.DOT_PRODUCT) { // dot product for now+ if (vectorValues instanceof HasIndexSlice sliceableValues+ && sliceableValues.getSlice() != null) {+ var scorer =+ Lucene99MemorySegmentFloatVectorScorerSupplier.create(+ similarityType, sliceableValues.getSlice(), vectorValues);+ if (scorer.isPresent()) {+ return scorer.get();+ }+ }+ }+ return delegate.getRandomVectorScorerSupplier(similarityType, vectorValues);+ }++ private RandomVectorScorerSupplier getByteScorerSupplier(+ ByteVectorValues vectorValues, VectorSimilarityFunction similarityType) throws IOException {// a quantized values here is a wrapping or delegation issueassert !(vectorValues instanceof QuantizedByteVectorValues);// currently only supports binary vectors@@ -61,7 +86,19 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorerpublic RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction similarityType, KnnVectorValues vectorValues, float[] target)throws IOException {- // currently only supports binary vectors, so always delegate+ checkDimensions(target.length, vectorValues.dimension());+ if (similarityType == VectorSimilarityFunction.DOT_PRODUCT) { // just for now+ if (vectorValues instanceof FloatVectorValues fvv+ && fvv instanceof HasIndexSlice floatVectorValues+ && floatVectorValues.getSlice() != null) {+ var scorer =+ Lucene99MemorySegmentFloatVectorScorer.create(+ similarityType, floatVectorValues.getSlice(), fvv, target);+ if (scorer.isPresent()) {+ return scorer.get();+ }+ }+ }return delegate.getRandomVectorScorer(similarityType, vectorValues, target);}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.javanew file mode 100644index 00000000000..c1fd989aa64--- /dev/null+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorer.java@@ -0,0 +1,143 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.internal.vectorization;++import static org.apache.lucene.util.VectorUtil.normalizeToUnitInterval;++import java.io.IOException;+import java.lang.foreign.MemorySegment;+import java.util.Optional;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.FilterIndexInput;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.MemorySegmentAccessInput;+import org.apache.lucene.util.hnsw.RandomVectorScorer;++abstract sealed class Lucene99MemorySegmentFloatVectorScorer+ extends RandomVectorScorer.AbstractRandomVectorScorer {++ final FloatVectorValues values;+ final int vectorByteSize;+ final MemorySegmentAccessInput input;+ final float[] query;+ byte[] scratch;++ /**+ * Return an optional whose value, if present, is the scorer. Otherwise, an empty optional is+ * returned.+ */+ public static Optional<Lucene99MemorySegmentFloatVectorScorer> create(+ VectorSimilarityFunction type,+ IndexInput input,+ FloatVectorValues values,+ float[] queryVector) {+ input = FilterIndexInput.unwrapOnlyTest(input);+ if (!(input instanceof MemorySegmentAccessInput msInput)) {+ return Optional.empty();+ }+ checkInvariants(values.size(), values.getVectorByteLength(), input);+ return switch (type) {+ case COSINE -> Optional.empty(); // of(new CosineScorer(msInput, values, queryVector));+ case DOT_PRODUCT -> Optional.of(new DotProductScorer(msInput, values, queryVector));+ case EUCLIDEAN -> Optional.empty(); // of(new EuclideanScorer(msInput, values, queryVector));+ case MAXIMUM_INNER_PRODUCT ->+ Optional.empty(); // of(new MaxInnerProductScorer(msInput, values, queryVector));+ };+ }++ Lucene99MemorySegmentFloatVectorScorer(+ MemorySegmentAccessInput input, FloatVectorValues values, float[] queryVector) {+ super(values);+ this.values = values;+ this.input = input;+ this.vectorByteSize = values.getVectorByteLength();+ this.query = queryVector;+ }++ final MemorySegment getSegment(int ord) throws IOException {+ checkOrdinal(ord);+ long byteOffset = (long) ord * vectorByteSize;+ MemorySegment seg = input.segmentSliceOrNull(byteOffset, vectorByteSize);+ if (seg == null) {+ if (scratch == null) {+ scratch = new byte[vectorByteSize];+ }+ input.readBytes(byteOffset, scratch, 0, vectorByteSize);+ seg = MemorySegment.ofArray(scratch);+ }+ return seg;+ }++ static void checkInvariants(int maxOrd, int vectorByteLength, IndexInput input) {+ if (input.length() < (long) vectorByteLength * maxOrd) {+ throw new IllegalArgumentException("input length is less than expected vector data");+ }+ }++ final void checkOrdinal(int ord) {+ if (ord < 0 || ord >= maxOrd()) {+ throw new IllegalArgumentException("illegal ordinal: " + ord);+ }+ }++ static final class DotProductScorer extends Lucene99MemorySegmentFloatVectorScorer {++ static final MemorySegmentBulkVectorOps.DotFromQueryArray DOT_OPS =+ new MemorySegmentBulkVectorOps.DotFromQueryArray();++ DotProductScorer(MemorySegmentAccessInput input, FloatVectorValues values, float[] query) {+ super(input, values, query);+ }++ @Override+ public float score(int node) throws IOException {+ checkOrdinal(node);+ // just delegates to existing scorer that copies on-heap+ return VectorSimilarityFunction.DOT_PRODUCT.compare(query, values.vectorValue(node));+ }++ @Override+ public void bulkScore(int[] nodes, float[] scores, int numNodes) throws IOException {+ float[] scratchScores = new float[4];+ int i = 0;+ final int limit = numNodes & ~3;+ for (; i < limit; i += 4) {+ MemorySegment ms1 = getSegment(nodes[i]);+ MemorySegment ms2 = getSegment(nodes[i + 1]);+ MemorySegment ms3 = getSegment(nodes[i + 2]);+ MemorySegment ms4 = getSegment(nodes[i + 3]);+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms4, query.length);+ scores[i + 0] = normalizeToUnitInterval(scratchScores[0]);+ scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);+ scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);+ scores[i + 3] = normalizeToUnitInterval(scratchScores[3]);+ }+ // Handle remaining 1–3 nodes in bulk (if any)+ int remaining = numNodes - i;+ if (remaining > 0) {+ MemorySegment ms1 = getSegment(nodes[i]);+ MemorySegment ms2 = (remaining > 1) ? getSegment(nodes[i + 1]) : ms1;+ MemorySegment ms3 = (remaining > 2) ? getSegment(nodes[i + 2]) : ms1;+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms1, query.length);+ scores[i] = normalizeToUnitInterval(scratchScores[0]);+ if (remaining > 1) scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);+ if (remaining > 2) scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);+ }+ }+ }+}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.javanew file mode 100644index 00000000000..33a3a0e33dd--- /dev/null+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFloatVectorScorerSupplier.java@@ -0,0 +1,163 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.internal.vectorization;++import static org.apache.lucene.util.VectorUtil.normalizeToUnitInterval;++import java.io.IOException;+import java.lang.foreign.MemorySegment;+import java.util.Optional;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.FilterIndexInput;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.MemorySegmentAccessInput;+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;++/** A score supplier of vectors whose element size is byte. */+public abstract sealed class Lucene99MemorySegmentFloatVectorScorerSupplier+ implements RandomVectorScorerSupplier {+ final int vectorByteSize;+ final int maxOrd;+ final int dims;+ final MemorySegmentAccessInput input;+ final FloatVectorValues values; // to support ordToDoc/getAcceptOrds+ byte[] queryScratch, scratch1, scratch2, scratch3, scratch4;++ /**+ * Return an optional whose value, if present, is the scorer supplier. Otherwise, an empty+ * optional is returned.+ */+ static Optional<RandomVectorScorerSupplier> create(+ VectorSimilarityFunction type, IndexInput input, FloatVectorValues values) {+ input = FilterIndexInput.unwrapOnlyTest(input);+ if (!(input instanceof MemorySegmentAccessInput msInput)) {+ return Optional.empty();+ }+ checkInvariants(values.size(), values.getVectorByteLength(), input);+ return switch (type) {+ case COSINE -> Optional.empty(); // of(new CosineSupplier(msInput, values));+ case DOT_PRODUCT -> Optional.of(new DotProductSupplier(msInput, values));+ case EUCLIDEAN -> Optional.empty(); // of(new EuclideanSupplier(msInput, values));+ case MAXIMUM_INNER_PRODUCT ->+ Optional.empty(); // of(new MaxInnerProductSupplier(msInput, values));+ };+ }++ Lucene99MemorySegmentFloatVectorScorerSupplier(+ MemorySegmentAccessInput input, FloatVectorValues values) {+ this.input = input;+ this.values = values;+ this.vectorByteSize = values.getVectorByteLength();+ this.maxOrd = values.size();+ this.dims = values.dimension();+ }++ static void checkInvariants(int maxOrd, int vectorByteLength, IndexInput input) {+ if (input.length() < (long) vectorByteLength * maxOrd) {+ throw new IllegalArgumentException("input length is less than expected vector data");+ }+ }++ final void checkOrdinal(int ord) {+ if (ord < 0 || ord >= maxOrd) {+ throw new IllegalArgumentException("illegal ordinal: " + ord);+ }+ }++ final MemorySegment getSegment(int ord, byte[] scratch) throws IOException {+ long byteOffset = (long) ord * vectorByteSize;+ MemorySegment seg = input.segmentSliceOrNull(byteOffset, vectorByteSize);+ if (seg == null) {+ if (scratch == null) {+ scratch = new byte[vectorByteSize];+ }+ input.readBytes(byteOffset, scratch, 0, vectorByteSize);+ seg = MemorySegment.ofArray(scratch);+ }+ return seg;+ }++ static final class DotProductSupplier extends Lucene99MemorySegmentFloatVectorScorerSupplier {++ static final MemorySegmentBulkVectorOps.DotFromQuerySegment DOT_OPS =+ new MemorySegmentBulkVectorOps.DotFromQuerySegment();++ DotProductSupplier(MemorySegmentAccessInput input, FloatVectorValues values) {+ super(input, values);+ }++ @Override+ public UpdateableRandomVectorScorer scorer() {+ return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(values) {+ private int queryOrd;++ @Override+ public float score(int node) throws IOException {+ checkOrdinal(node);+ MemorySegment query = getSegment(queryOrd, queryScratch);+ MemorySegment ms = getSegment(node, scratch1);+ var raw = DOT_OPS.dotProduct(query, ms, dims);+ return normalizeToUnitInterval(raw);+ }++ @Override+ public void bulkScore(int[] nodes, float[] scores, int numNodes) throws IOException {+ // TODO checkOrdinal(node1 ....);+ float[] scratchScores = new float[4];+ int i = 0;+ MemorySegment query = getSegment(queryOrd, queryScratch);+ final int limit = numNodes & ~3;+ for (; i < limit; i += 4) {+ MemorySegment ms1 = getSegment(nodes[i], scratch1);+ MemorySegment ms2 = getSegment(nodes[i + 1], scratch2);+ MemorySegment ms3 = getSegment(nodes[i + 2], scratch3);+ MemorySegment ms4 = getSegment(nodes[i + 3], scratch4);+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms4, dims);+ scores[i + 0] = normalizeToUnitInterval(scratchScores[0]);+ scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);+ scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);+ scores[i + 3] = normalizeToUnitInterval(scratchScores[3]);+ }+ // Handle remaining 1–3 nodes in bulk (if any)+ int remaining = numNodes - i;+ if (remaining > 0) {+ MemorySegment ms1 = getSegment(nodes[i], scratch1);+ MemorySegment ms2 = (remaining > 1) ? getSegment(nodes[i + 1], scratch2) : ms1;+ MemorySegment ms3 = (remaining > 2) ? getSegment(nodes[i + 2], scratch3) : ms1;+ DOT_OPS.dotProductBulk(scratchScores, query, ms1, ms2, ms3, ms1, dims);+ scores[i] = normalizeToUnitInterval(scratchScores[0]);+ if (remaining > 1) scores[i + 1] = normalizeToUnitInterval(scratchScores[1]);+ if (remaining > 2) scores[i + 2] = normalizeToUnitInterval(scratchScores[2]);+ }+ }++ @Override+ public void setScoringOrdinal(int node) {+ checkOrdinal(node);+ queryOrd = node;+ }+ };+ }++ @Override+ public DotProductSupplier copy() throws IOException {+ return new DotProductSupplier(input.clone(), values);+ }+ }+}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.javanew file mode 100644index 00000000000..8c6e5b096b7--- /dev/null+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentBulkVectorOps.java@@ -0,0 +1,182 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.internal.vectorization;++import static org.apache.lucene.internal.vectorization.PanamaVectorConstants.PREFERRED_VECTOR_BITSIZE;+import static org.apache.lucene.internal.vectorization.PanamaVectorUtilSupport.fma;++import java.lang.foreign.MemorySegment;+import java.lang.foreign.ValueLayout;+import java.nio.ByteOrder;+import jdk.incubator.vector.FloatVector;+import jdk.incubator.vector.VectorOperators;+import jdk.incubator.vector.VectorShape;+import jdk.incubator.vector.VectorSpecies;++/** Implementations of bulk vector comparison operations. Currently only supports float32. */+public final class MemorySegmentBulkVectorOps {++ static final VectorSpecies<Float> FLOAT_SPECIES =+ VectorSpecies.of(float.class, VectorShape.forBitSize(PREFERRED_VECTOR_BITSIZE));+ static final ByteOrder LE = ByteOrder.LITTLE_ENDIAN;+ static final ValueLayout.OfFloat LAYOUT_LE_FLOAT = ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(LE);++ private MemorySegmentBulkVectorOps() {}++ sealed interface FloatVectorLoader {+ /** Returns the number of float elements. */+ int length();++ FloatVector load(VectorSpecies<Float> species, int index);++ float tail(int index);+ }++ record FloatArrayLoader(float[] arr) implements FloatVectorLoader {+ @Override+ public int length() {+ return arr.length;+ }++ @Override+ public FloatVector load(VectorSpecies<Float> species, int index) {+ // assert index + species.length() <= length();+ return FloatVector.fromArray(species, arr, index);+ }++ @Override+ public float tail(int index) {+ // assert index <= length();+ return arr[index];+ }+ }++ record FloatMemorySegmentLoader(MemorySegment segment) implements FloatVectorLoader {+ @Override+ public int length() {+ return Math.toIntExact(segment.byteSize() / Float.BYTES);+ }++ @Override+ public FloatVector load(VectorSpecies<Float> species, int index) {+ // assert index + species.length() <= length();+ return FloatVector.fromMemorySegment(species, segment, (long) index * Float.BYTES, LE);+ }++ @Override+ public float tail(int index) {+ // assert index <= length();+ return segment.get(LAYOUT_LE_FLOAT, (long) index * Float.BYTES);+ }+ }++ public static final class DotFromQueryArray extends AbstractDotProduct {++ public void dotProductBulk(+ float[] scores,+ float[] q,+ MemorySegment d1,+ MemorySegment d2,+ MemorySegment d3,+ MemorySegment d4,+ int elementCount) {+ super.dotProductBulk(scores, new FloatArrayLoader(q), d1, d2, d3, d4, elementCount);+ }+ }++ public static final class DotFromQuerySegment extends AbstractDotProduct {++ public void dotProductBulk(+ float[] scores,+ MemorySegment q,+ MemorySegment d1,+ MemorySegment d2,+ MemorySegment d3,+ MemorySegment d4,+ int elementCount) {+ super.dotProductBulk(scores, new FloatMemorySegmentLoader(q), d1, d2, d3, d4, elementCount);+ }++ public float dotProduct(MemorySegment q, MemorySegment d, int elementCount) {+ int i = 0;+ FloatVector sv = FloatVector.zero(FLOAT_SPECIES);+ final int limit = FLOAT_SPECIES.loopBound(elementCount);+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ final long offset = (long) i * Float.BYTES;+ FloatVector qv = FloatVector.fromMemorySegment(FLOAT_SPECIES, q, offset, LE);+ FloatVector dv = FloatVector.fromMemorySegment(FLOAT_SPECIES, d, offset, LE);+ sv = fma(qv, dv, sv);+ }+ float score = sv.reduceLanes(VectorOperators.ADD);++ for (; i < elementCount; i++) {+ final long offset = (long) i * Float.BYTES;+ score += q.get(LAYOUT_LE_FLOAT, offset) * d.get(LAYOUT_LE_FLOAT, offset);+ }+ return score;+ }+ }++ abstract static class AbstractDotProduct {++ protected void dotProductBulk(+ float[] scores,+ FloatVectorLoader q,+ MemorySegment d1,+ MemorySegment d2,+ MemorySegment d3,+ MemorySegment d4,+ int elementCount) {+ int i = 0;+ FloatVector sv1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sv4 = FloatVector.zero(FLOAT_SPECIES);++ final int limit = FLOAT_SPECIES.loopBound(elementCount);+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ final long offset = (long) i * Float.BYTES;+ FloatVector dv1 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d1, offset, LE);+ FloatVector dv2 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d2, offset, LE);+ FloatVector dv3 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d3, offset, LE);+ FloatVector dv4 = FloatVector.fromMemorySegment(FLOAT_SPECIES, d4, offset, LE);+ FloatVector qv = q.load(FLOAT_SPECIES, i);+ sv1 = fma(qv, dv1, sv1);+ sv2 = fma(qv, dv2, sv2);+ sv3 = fma(qv, dv3, sv3);+ sv4 = fma(qv, dv4, sv4);+ }+ float sum1 = sv1.reduceLanes(VectorOperators.ADD);+ float sum2 = sv2.reduceLanes(VectorOperators.ADD);+ float sum3 = sv3.reduceLanes(VectorOperators.ADD);+ float sum4 = sv4.reduceLanes(VectorOperators.ADD);++ for (; i < elementCount; i++) {+ final long offset = (long) i * Float.BYTES;+ final float qValue = q.tail(i);+ sum1 = fma(qValue, d1.get(LAYOUT_LE_FLOAT, offset), sum1);+ sum2 = fma(qValue, d2.get(LAYOUT_LE_FLOAT, offset), sum2);+ sum3 = fma(qValue, d3.get(LAYOUT_LE_FLOAT, offset), sum3);+ sum4 = fma(qValue, d4.get(LAYOUT_LE_FLOAT, offset), sum4);+ }+ scores[0] = sum1;+ scores[1] = sum2;+ scores[2] = sum3;+ scores[3] = sum4;+ }+ }+}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javaindex 7572560923b..731aa8ab539 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java@@ -85,7 +85,7 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {// the way FMA should work! if available use it, otherwise fall back to mul/add@SuppressForbidden(reason = "Uses FMA only where fast and carefully contained")- private static FloatVector fma(FloatVector a, FloatVector b, FloatVector c) {+ static FloatVector fma(FloatVector a, FloatVector b, FloatVector c) {if (Constants.HAS_FAST_VECTOR_FMA) {return a.fma(b, c);} else {@@ -94,7 +94,7 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {}@SuppressForbidden(reason = "Uses FMA only where fast and carefully contained")- private static float fma(float a, float b, float c) {+ static float fma(float a, float b, float c) {if (Constants.HAS_FAST_SCALAR_FMA) {return Math.fma(a, b, c);} else {diff --git a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.javaindex 1dab736945d..54dc8015dde 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java@@ -204,6 +204,7 @@ public class TestFlatVectorScorer extends BaseVectorizationTestCase {var values = byteVectorValues(dims, size, in, sim);assertBulkEqualsNonBulk(values, sim);assertBulkEqualsNonBulkSupplier(values, sim);+ assertScoresAgainstDefaultFlatScorer(values, sim);}}}@@ -226,11 +227,27 @@ public class TestFlatVectorScorer extends BaseVectorizationTestCase {var values = floatVectorValues(dims, size, in, sim);assertBulkEqualsNonBulk(values, sim);assertBulkEqualsNonBulkSupplier(values, sim);+ assertScoresAgainstDefaultFlatScorer(values, sim);}}}}+ public void testOnHeapBulkScorerFloats() throws IOException {+ int dims = random().nextInt(1, 1024);+ int size = random().nextInt(2, 255);+ List<float[]> vectors = new ArrayList<>();+ for (int i = 0; i < size; i++) {+ vectors.add(randomFloatVector(dims));+ }+ var values = FloatVectorValues.fromFloats(vectors, dims);+ for (var sim : List.of(COSINE, DOT_PRODUCT, EUCLIDEAN, MAXIMUM_INNER_PRODUCT)) {+ assertBulkEqualsNonBulk(values, sim);+ assertBulkEqualsNonBulkSupplier(values, sim);+ assertScoresAgainstDefaultFlatScorer(values, sim);+ }+ }+void assertBulkEqualsNonBulk(KnnVectorValues values, VectorSimilarityFunction sim)throws IOException {final int dims = values.dimension();@@ -256,19 +273,46 @@ public class TestFlatVectorScorer extends BaseVectorizationTestCase {throws IOException {final int size = values.size();final float delta = 1e-3f * size;- var ss = flatVectorsScorer.getRandomVectorScorerSupplier(sim, values);- var updatableScorer = ss.scorer();+ var supplier = flatVectorsScorer.getRandomVectorScorerSupplier(sim, values);+ for (var ss : List.of(supplier, supplier.copy())) {+ var updatableScorer = ss.scorer();+ var targetNode = random().nextInt(size);+ updatableScorer.setScoringOrdinal(targetNode);+ int[] indices = randomIndices(size);+ float[] expectedScores = new float[size];+ for (int i = 0; i < size; i++) {+ expectedScores[i] = updatableScorer.score(indices[i]);+ }+ float[] bulkScores = new float[size];+ updatableScorer.bulkScore(indices, bulkScores, size);+ assertArrayEquals(expectedScores, bulkScores, delta);+ assertNoScoreBeyondNumNodes(updatableScorer, size);+ }+ }++ // asserts scores against the default scorer.+ void assertScoresAgainstDefaultFlatScorer(KnnVectorValues values, VectorSimilarityFunction sim)+ throws IOException {+ final int size = values.size();+ final float delta = 1e-3f * size;var targetNode = random().nextInt(size);- updatableScorer.setScoringOrdinal(targetNode);int[] indices = randomIndices(size);+ var defaultScorer =+ DefaultFlatVectorScorer.INSTANCE.getRandomVectorScorerSupplier(sim, values).scorer();+ defaultScorer.setScoringOrdinal(targetNode);float[] expectedScores = new float[size];for (int i = 0; i < size; i++) {- expectedScores[i] = updatableScorer.score(indices[i]);+ expectedScores[i] = defaultScorer.score(indices[i]);+ }++ var supplier = flatVectorsScorer.getRandomVectorScorerSupplier(sim, values);+ for (var ss : List.of(supplier, supplier.copy())) {+ var updatableScorer = ss.scorer();+ updatableScorer.setScoringOrdinal(targetNode);+ float[] bulkScores = new float[size];+ updatableScorer.bulkScore(indices, bulkScores, size);+ assertArrayEquals(expectedScores, bulkScores, delta);}- float[] bulkScores = new float[size];- updatableScorer.bulkScore(indices, bulkScores, size);- assertArrayEquals(expectedScores, bulkScores, delta);- assertNoScoreBeyondNumNodes(updatableScorer, size);}void assertNoScoreBeyondNumNodes(RandomVectorScorer scorer, int maxSize) throws IOException {diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.javaindex 233be3a139f..be099398ba6 100644--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java@@ -91,7 +91,7 @@ import org.apache.lucene.util.hnsw.HnswGraph.NodesIterator;abstract class HnswGraphTestCase<T> extends LuceneTestCase {VectorSimilarityFunction similarityFunction;- DefaultFlatVectorScorer flatVectorScorer = new DefaultFlatVectorScorer();+ DefaultFlatVectorScorer flatVectorScorer = DefaultFlatVectorScorer.INSTANCE;abstract VectorEncoding getVectorEncoding();