Skip to content

Apache Lucene open-source search software

Fix default flat vector scorer supplier sharing backing array (#13355)

Chris Hegarty25f1efd8eb34
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/DefaultFlatVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/DefaultFlatVectorScorer.java
index e5496f3e10e..7fd500e85a2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/DefaultFlatVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/DefaultFlatVectorScorer.java
@@ -100,8 +100,13 @@ public class DefaultFlatVectorScorer implements FlatVectorsScorer {
}
@Override
- public RandomVectorScorer scorer(int ord) throws IOException {
- return new ByteVectorScorer(vectors2, vectors1.vectorValue(ord), similarityFunction);
+ public RandomVectorScorer scorer(int ord) {
+ return new RandomVectorScorer.AbstractRandomVectorScorer(vectors) {
+ @Override
+ public float score(int node) throws IOException {
+ return similarityFunction.compare(vectors1.vectorValue(ord), vectors2.vectorValue(node));
+ }
+ };
}
@Override
@@ -127,8 +132,13 @@ public class DefaultFlatVectorScorer implements FlatVectorsScorer {
}
@Override
- public RandomVectorScorer scorer(int ord) throws IOException {
- return new FloatVectorScorer(vectors2, vectors1.vectorValue(ord), similarityFunction);
+ public RandomVectorScorer scorer(int ord) {
+ return new RandomVectorScorer.AbstractRandomVectorScorer(vectors) {
+ @Override
+ public float score(int node) throws IOException {
+ return similarityFunction.compare(vectors1.vectorValue(ord), vectors2.vectorValue(node));
+ }
+ };
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java
new file mode 100644
index 00000000000..5dbebf08df5
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.hnsw;
+
+import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
+import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;
+import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT;
+import static org.hamcrest.Matchers.equalTo;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.codecs.lucene95.OffHeapByteVectorValues;
+import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;
+import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorScorer;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
+import org.hamcrest.Matcher;
+import org.hamcrest.MatcherAssert;
+
+public class TestFlatVectorScorer extends LuceneTestCase {
+
+ static volatile AtomicInteger count = new AtomicInteger();
+ final FlatVectorsScorer flatVectorsScorer;
+ final ThrowingSupplier<Directory> newDirectory;
+
+ public TestFlatVectorScorer(
+ FlatVectorsScorer flatVectorsScorer, ThrowingSupplier<Directory> newDirectory) {
+ this.flatVectorsScorer = flatVectorsScorer;
+ this.newDirectory = newDirectory;
+ }
+
+ @ParametersFactory
+ public static Iterable<Object[]> parametersFactory() {
+ var scorers =
+ List.of(
+ new DefaultFlatVectorScorer(),
+ new Lucene99ScalarQuantizedVectorScorer(new DefaultFlatVectorScorer()));
+ var dirs =
+ List.<ThrowingSupplier<Directory>>of(
+ TestFlatVectorScorer::newDirectory,
+ () -> new MMapDirectory(createTempDir(count.getAndIncrement() + "-")));
+
+ List<Object[]> objs = new ArrayList<>();
+ for (var scorer : scorers) {
+ for (var dir : dirs) {
+ objs.add(new Object[] {scorer, dir});
+ }
+ }
+ return objs;
+ }
+
+ // Tests that the creation of another scorer does not perturb previous scorers
+ public void testMultipleByteScorers() throws IOException {
+ byte[] vec0 = new byte[] {0, 0, 0, 0};
+ byte[] vec1 = new byte[] {1, 1, 1, 1};
+ byte[] vec2 = new byte[] {15, 15, 15, 15};
+
+ String fileName = "testMultipleByteScorers";
+ try (Directory dir = newDirectory.get()) {
+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ out.writeBytes(concat(vec0, vec1, vec2), 0, vec0.length * 3);
+ }
+ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+ var vectorValues = byteVectorValues(4, 3, in, EUCLIDEAN);
+ var ss = flatVectorsScorer.getRandomVectorScorerSupplier(EUCLIDEAN, vectorValues);
+
+ var scorerAgainstOrd0 = ss.scorer(0);
+ var firstScore = scorerAgainstOrd0.score(1);
+ @SuppressWarnings("unused")
+ var scorerAgainstOrd2 = ss.scorer(2);
+ var scoreAgain = scorerAgainstOrd0.score(1);
+
+ assertThat(scoreAgain, equalTo(firstScore));
+ }
+ }
+ }
+
+ // Tests that the creation of another scorer does not perturb previous scorers
+ public void testMultipleFloatScorers() throws IOException {
+ float[] vec0 = new float[] {0, 0, 0, 0};
+ float[] vec1 = new float[] {1, 1, 1, 1};
+ float[] vec2 = new float[] {15, 15, 15, 15};
+
+ String fileName = "testMultipleFloatScorers";
+ try (Directory dir = newDirectory.get()) {
+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ out.writeBytes(concat(vec0, vec1, vec2), 0, vec0.length * Float.BYTES * 3);
+ }
+ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+ var vectorValues = floatVectorValues(4, 3, in, EUCLIDEAN);
+ var ss = flatVectorsScorer.getRandomVectorScorerSupplier(EUCLIDEAN, vectorValues);
+
+ var scorerAgainstOrd0 = ss.scorer(0);
+ var firstScore = scorerAgainstOrd0.score(1);
+ @SuppressWarnings("unused")
+ var scorerAgainstOrd2 = ss.scorer(2);
+ var scoreAgain = scorerAgainstOrd0.score(1);
+
+ assertThat(scoreAgain, equalTo(firstScore));
+ }
+ }
+ }
+
+ public void testCheckByteDimensions() throws IOException {
+ byte[] vec0 = new byte[4];
+ String fileName = "testCheckByteDimensions";
+ try (Directory dir = newDirectory.get()) {
+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ out.writeBytes(vec0, 0, vec0.length);
+ }
+ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+ for (var sim : List.of(COSINE, DOT_PRODUCT, EUCLIDEAN, MAXIMUM_INNER_PRODUCT)) {
+ var vectorValues = byteVectorValues(4, 1, in, sim);
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> flatVectorsScorer.getRandomVectorScorer(sim, vectorValues, new byte[5]));
+ }
+ }
+ }
+ }
+
+ public void testCheckFloatDimensions() throws IOException {
+ float[] vec0 = new float[4];
+ String fileName = "testCheckFloatDimensions";
+ try (Directory dir = newDirectory.get()) {
+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ out.writeBytes(concat(vec0), 0, vec0.length * Float.BYTES);
+ }
+ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+ for (var sim : List.of(COSINE, DOT_PRODUCT, EUCLIDEAN, MAXIMUM_INNER_PRODUCT)) {
+ var vectorValues = floatVectorValues(4, 1, in, sim);
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> flatVectorsScorer.getRandomVectorScorer(sim, vectorValues, new float[5]));
+ }
+ }
+ }
+ }
+
+ RandomAccessVectorValues byteVectorValues(
+ int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
+ return new OffHeapByteVectorValues.DenseOffHeapVectorValues(
+ dims, size, in.slice("byteValues", 0, in.length()), dims, flatVectorsScorer, sim);
+ }
+
+ RandomAccessVectorValues floatVectorValues(
+ int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
+ return new OffHeapFloatVectorValues.DenseOffHeapVectorValues(
+ dims,
+ size,
+ in.slice("floatValues", 0, in.length()),
+ dims * Float.BYTES,
+ flatVectorsScorer,
+ sim);
+ }
+
+ /** Concatenates float arrays as byte[]. */
+ public static byte[] concat(float[]... arrays) throws IOException {
+ var bb = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ for (var fa : arrays) {
+ for (var f : fa) {
+ bb.putFloat(0, f);
+ baos.write(bb.array());
+ }
+ }
+ return baos.toByteArray();
+ }
+ }
+
+ /** Concatenates byte arrays. */
+ public static byte[] concat(byte[]... arrays) throws IOException {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ for (var ba : arrays) {
+ baos.write(ba);
+ }
+ return baos.toByteArray();
+ }
+ }
+
+ public static <T> void assertThat(T actual, Matcher<? super T> matcher) {
+ MatcherAssert.assertThat("", actual, matcher);
+ }
+
+ @FunctionalInterface
+ public interface ThrowingSupplier<T> {
+ T get() throws IOException;
+ }
+}