Skip to content

Apache Lucene open-source search software

Fix the COSINE merge scorer in Lucene104ScalarQuantizedVectorsReader (#16665)

John Maurice8ece74d1e210
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c702f80ea5d..0346c940703 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -607,6 +607,14 @@ API Changes
Lucene 10.4 behavior and is deprecated, and [Byte|Float]VectorSimilarityQuery.Adaptive exposes the 10.5 behavior.
(Kaival Parikh, Chris Hostetter, Michael Sokolov)
+Bug Fixes
+---------------------
+
+* GITHUB#16665: Fix Lucene104HnswScalarQuantizedVectorsFormat merges with an asymmetric encoding
+ (1 and 2 bit quantization today) and COSINE similarity: the merge scorer's query side was
+ quantized from un-normalized vectors while the index side was normalized, so the merge trips an
+ assertion and otherwise builds the graph from wrong scores. (John Maurice)
+
======================= Lucene 10.5.1 =======================
Bug Fixes
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index 3e6065ff663..bd4d6c874d9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -532,6 +532,11 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
return CloseableRandomVectorScorerSupplier.create(supplier, vectorValues.size(), () -> {});
}
FloatVectorValues floatVectorValues = getFloatVectorValues(fieldInfo.name);
+ if (fieldInfo.getVectorSimilarityFunction() == VectorSimilarityFunction.COSINE) {
+ // the index side of this segment was quantized from normalized vectors, the query side must
+ // be too
+ floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues);
+ }
OptimizedScalarQuantizer quantizer =
new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction());
String tempScoreQuantizedVectorName = null;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
index 433b0643910..2525d16bca6 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
@@ -907,46 +907,4 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
return new Float16AsFloatVectorValues(values.copy());
}
}
-
- static final class NormalizedFloatVectorValues extends FloatVectorValues {
- private final FloatVectorValues values;
- private final float[] normalizedVector;
-
- NormalizedFloatVectorValues(FloatVectorValues values) {
- this.values = values;
- this.normalizedVector = new float[values.dimension()];
- }
-
- @Override
- public int dimension() {
- return values.dimension();
- }
-
- @Override
- public int size() {
- return values.size();
- }
-
- @Override
- public int ordToDoc(int ord) {
- return values.ordToDoc(ord);
- }
-
- @Override
- public float[] vectorValue(int ord) throws IOException {
- System.arraycopy(values.vectorValue(ord), 0, normalizedVector, 0, normalizedVector.length);
- VectorUtil.l2normalize(normalizedVector);
- return normalizedVector;
- }
-
- @Override
- public DocIndexIterator iterator() {
- return values.iterator();
- }
-
- @Override
- public NormalizedFloatVectorValues copy() throws IOException {
- return new NormalizedFloatVectorValues(values.copy());
- }
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java
new file mode 100644
index 00000000000..b2be57c5b31
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.io.IOException;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.util.VectorUtil;
+
+/** A view of {@link FloatVectorValues} that returns every vector normalized to unit length. */
+final class NormalizedFloatVectorValues extends FloatVectorValues {
+ private final FloatVectorValues values;
+ private final float[] normalizedVector;
+
+ NormalizedFloatVectorValues(FloatVectorValues values) {
+ this.values = values;
+ this.normalizedVector = new float[values.dimension()];
+ }
+
+ @Override
+ public int dimension() {
+ return values.dimension();
+ }
+
+ @Override
+ public int size() {
+ return values.size();
+ }
+
+ @Override
+ public int ordToDoc(int ord) {
+ return values.ordToDoc(ord);
+ }
+
+ @Override
+ public float[] vectorValue(int ord) throws IOException {
+ System.arraycopy(values.vectorValue(ord), 0, normalizedVector, 0, normalizedVector.length);
+ VectorUtil.l2normalize(normalizedVector);
+ return normalizedVector;
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return values.iterator();
+ }
+
+ @Override
+ public NormalizedFloatVectorValues copy() throws IOException {
+ return new NormalizedFloatVectorValues(values.copy());
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
index 0ec1689c36e..fb0f88e2290 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
@@ -29,27 +29,40 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.AcceptDocs;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.SameThreadExecutorService;
import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+import org.apache.lucene.util.quantization.QuantizedVectorsReader;
import org.junit.Before;
public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
@@ -105,6 +118,97 @@ public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVector
assertThat(customCodec.knnVectorsFormat().toString(), is(oneOf(defaultScorer, memSegScorer)));
}
+ /**
+ * A merge that builds a graph over an asymmetric encoding quantizes the merged vectors again for
+ * the query side of the merge scorer. For COSINE those vectors must be normalized first, like the
+ * index side was and like a search-time query is, so the merge scorer must score a pair exactly
+ * like the search-time scorer does for the same vector. The base test case only feeds unit
+ * vectors, for which the two agree by accident; this uses vectors that are not.
+ */
+ public void testMergeCosineWithNonUnitVectors() throws Exception {
+ for (ScalarEncoding asymmetric : ScalarEncoding.values()) {
+ if (asymmetric.isAsymmetric() == false) {
+ continue;
+ }
+ // a threshold of 0 makes every merge build a graph, so the merge scorer is always requested
+ KnnVectorsFormat graphFormat =
+ new Lucene104HnswScalarQuantizedVectorsFormat(
+ asymmetric,
+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+ 1,
+ null,
+ 0);
+ IndexWriterConfig config =
+ newIndexWriterConfig()
+ .setCodec(TestUtil.alwaysKnnVectorsFormat(graphFormat))
+ .setMergePolicy(newLogMergePolicy());
+ int dim = random().nextInt(8, 64);
+ int numDocs = atLeast(50);
+ float[][] vectors = new float[numDocs][];
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config)) {
+ for (int i = 0; i < numDocs; i++) {
+ float[] vector = randomVector(dim);
+ // scale away from the unit sphere
+ float scale = random().nextFloat(2f, 10f);
+ for (int j = 0; j < dim; j++) {
+ vector[j] *= scale;
+ }
+ vectors[i] = vector;
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField("f", vector, VectorSimilarityFunction.COSINE));
+ w.addDocument(doc);
+ if (i == numDocs / 2) {
+ w.commit(); // two segments, so the merge has something to merge
+ }
+ }
+ w.commit();
+ w.forceMerge(1);
+ try (IndexReader reader = DirectoryReader.open(w)) {
+ LeafReader r = getOnlyLeafReader(reader);
+ SegmentReader segmentReader = (SegmentReader) FilterLeafReader.unwrap(r);
+ FieldInfo fieldInfo = r.getFieldInfos().fieldInfo("f");
+ Lucene99HnswVectorsReader hnswReader =
+ (Lucene99HnswVectorsReader) segmentReader.getVectorReader().unwrapReaderForField("f");
+ FlatVectorsReader flatReader = hnswReader.getFlatVectorsReader();
+ QuantizedVectorsReader quantizedReader = (QuantizedVectorsReader) flatReader;
+ SegmentWriteState writeState =
+ new SegmentWriteState(
+ InfoStream.getDefault(),
+ dir,
+ segmentReader.getSegmentInfo().info,
+ r.getFieldInfos(),
+ null,
+ IOContext.DEFAULT);
+ KnnVectorValues quantizedValues = quantizedReader.getQuantizedVectorValues("f");
+ FlatVectorsScorer searchScorer = flatReader.getFlatVectorScorer("f");
+ try (CloseableRandomVectorScorerSupplier mergeScorers =
+ quantizedReader.getRandomVectorScorerSupplierForMerge(fieldInfo, writeState)) {
+ FloatVectorValues rawValues = r.getFloatVectorValues("f");
+ UpdateableRandomVectorScorer mergeScorer = mergeScorers.scorer();
+ for (int i = 0; i < 5; i++) {
+ int queryOrd = random().nextInt(numDocs);
+ int targetOrd = random().nextInt(numDocs);
+ // the search-time scorer normalizes a COSINE query before quantizing it
+ RandomVectorScorer expected =
+ searchScorer.getRandomVectorScorer(
+ VectorSimilarityFunction.COSINE,
+ quantizedValues,
+ rawValues.vectorValue(queryOrd));
+ mergeScorer.setScoringOrdinal(queryOrd);
+ assertEquals(
+ "encoding " + asymmetric + " query ord " + queryOrd + " target ord " + targetOrd,
+ expected.score(targetOrd),
+ mergeScorer.score(targetOrd),
+ 1e-5f);
+ }
+ }
+ }
+ }
+ }
+ }
+
public void testSingleVectorCase() throws Exception {
float[] vector = randomVector(random().nextInt(12, 500));
for (VectorSimilarityFunction similarityFunction : VectorSimilarityFunction.values()) {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
index e326e926ab7..f253adb05d3 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
@@ -223,8 +223,7 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsFor
byte[] scratch = new byte[encoding.getDiscreteDimensions(dims)];
byte[] expectedVector = new byte[encoding.getDocPackedLength(scratch.length)];
if (similarityFunction == VectorSimilarityFunction.COSINE) {
- vectorValues =
- new Lucene104ScalarQuantizedVectorsWriter.NormalizedFloatVectorValues(vectorValues);
+ vectorValues = new NormalizedFloatVectorValues(vectorValues);
}
KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator();