Apache Lucene open-source search software
Fix the COSINE merge scorer in Lucene104ScalarQuantizedVectorsReader (#16665)
John Maurice8ece74d1e210
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c702f80ea5d..0346c940703 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -607,6 +607,14 @@ API ChangesLucene 10.4 behavior and is deprecated, and [Byte|Float]VectorSimilarityQuery.Adaptive exposes the 10.5 behavior.(Kaival Parikh, Chris Hostetter, Michael Sokolov)+Bug Fixes+---------------------++* GITHUB#16665: Fix Lucene104HnswScalarQuantizedVectorsFormat merges with an asymmetric encoding+ (1 and 2 bit quantization today) and COSINE similarity: the merge scorer's query side was+ quantized from un-normalized vectors while the index side was normalized, so the merge trips an+ assertion and otherwise builds the graph from wrong scores. (John Maurice)+======================= Lucene 10.5.1 =======================Bug Fixesdiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.javaindex 3e6065ff663..bd4d6c874d9 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java@@ -532,6 +532,11 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReaderreturn CloseableRandomVectorScorerSupplier.create(supplier, vectorValues.size(), () -> {});}FloatVectorValues floatVectorValues = getFloatVectorValues(fieldInfo.name);+ if (fieldInfo.getVectorSimilarityFunction() == VectorSimilarityFunction.COSINE) {+ // the index side of this segment was quantized from normalized vectors, the query side must+ // be too+ floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues);+ }OptimizedScalarQuantizer quantizer =new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction());String tempScoreQuantizedVectorName = null;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.javaindex 433b0643910..2525d16bca6 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java@@ -907,46 +907,4 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {return new Float16AsFloatVectorValues(values.copy());}}-- static final class NormalizedFloatVectorValues extends FloatVectorValues {- private final FloatVectorValues values;- private final float[] normalizedVector;-- NormalizedFloatVectorValues(FloatVectorValues values) {- this.values = values;- this.normalizedVector = new float[values.dimension()];- }-- @Override- public int dimension() {- return values.dimension();- }-- @Override- public int size() {- return values.size();- }-- @Override- public int ordToDoc(int ord) {- return values.ordToDoc(ord);- }-- @Override- public float[] vectorValue(int ord) throws IOException {- System.arraycopy(values.vectorValue(ord), 0, normalizedVector, 0, normalizedVector.length);- VectorUtil.l2normalize(normalizedVector);- return normalizedVector;- }-- @Override- public DocIndexIterator iterator() {- return values.iterator();- }-- @Override- public NormalizedFloatVectorValues copy() throws IOException {- return new NormalizedFloatVectorValues(values.copy());- }- }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.javanew file mode 100644index 00000000000..b2be57c5b31--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java@@ -0,0 +1,64 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.io.IOException;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.util.VectorUtil;++/** A view of {@link FloatVectorValues} that returns every vector normalized to unit length. */+final class NormalizedFloatVectorValues extends FloatVectorValues {+ private final FloatVectorValues values;+ private final float[] normalizedVector;++ NormalizedFloatVectorValues(FloatVectorValues values) {+ this.values = values;+ this.normalizedVector = new float[values.dimension()];+ }++ @Override+ public int dimension() {+ return values.dimension();+ }++ @Override+ public int size() {+ return values.size();+ }++ @Override+ public int ordToDoc(int ord) {+ return values.ordToDoc(ord);+ }++ @Override+ public float[] vectorValue(int ord) throws IOException {+ System.arraycopy(values.vectorValue(ord), 0, normalizedVector, 0, normalizedVector.length);+ VectorUtil.l2normalize(normalizedVector);+ return normalizedVector;+ }++ @Override+ public DocIndexIterator iterator() {+ return values.iterator();+ }++ @Override+ public NormalizedFloatVectorValues copy() throws IOException {+ return new NormalizedFloatVectorValues(values.copy());+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.javaindex 0ec1689c36e..fb0f88e2290 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java@@ -29,27 +29,40 @@ import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.FilterCodec;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnFloatVectorField;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FilterLeafReader;import org.apache.lucene.index.FloatVectorValues;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.SegmentReader;+import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.search.AcceptDocs;import org.apache.lucene.search.TopDocs;import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.InfoStream;import org.apache.lucene.util.SameThreadExecutorService;import org.apache.lucene.util.VectorUtil;+import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;+import org.apache.lucene.util.hnsw.RandomVectorScorer;+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;+import org.apache.lucene.util.quantization.QuantizedVectorsReader;import org.junit.Before;public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {@@ -105,6 +118,97 @@ public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorassertThat(customCodec.knnVectorsFormat().toString(), is(oneOf(defaultScorer, memSegScorer)));}+ /**+ * A merge that builds a graph over an asymmetric encoding quantizes the merged vectors again for+ * the query side of the merge scorer. For COSINE those vectors must be normalized first, like the+ * index side was and like a search-time query is, so the merge scorer must score a pair exactly+ * like the search-time scorer does for the same vector. The base test case only feeds unit+ * vectors, for which the two agree by accident; this uses vectors that are not.+ */+ public void testMergeCosineWithNonUnitVectors() throws Exception {+ for (ScalarEncoding asymmetric : ScalarEncoding.values()) {+ if (asymmetric.isAsymmetric() == false) {+ continue;+ }+ // a threshold of 0 makes every merge build a graph, so the merge scorer is always requested+ KnnVectorsFormat graphFormat =+ new Lucene104HnswScalarQuantizedVectorsFormat(+ asymmetric,+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,+ 1,+ null,+ 0);+ IndexWriterConfig config =+ newIndexWriterConfig()+ .setCodec(TestUtil.alwaysKnnVectorsFormat(graphFormat))+ .setMergePolicy(newLogMergePolicy());+ int dim = random().nextInt(8, 64);+ int numDocs = atLeast(50);+ float[][] vectors = new float[numDocs][];+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config)) {+ for (int i = 0; i < numDocs; i++) {+ float[] vector = randomVector(dim);+ // scale away from the unit sphere+ float scale = random().nextFloat(2f, 10f);+ for (int j = 0; j < dim; j++) {+ vector[j] *= scale;+ }+ vectors[i] = vector;+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("f", vector, VectorSimilarityFunction.COSINE));+ w.addDocument(doc);+ if (i == numDocs / 2) {+ w.commit(); // two segments, so the merge has something to merge+ }+ }+ w.commit();+ w.forceMerge(1);+ try (IndexReader reader = DirectoryReader.open(w)) {+ LeafReader r = getOnlyLeafReader(reader);+ SegmentReader segmentReader = (SegmentReader) FilterLeafReader.unwrap(r);+ FieldInfo fieldInfo = r.getFieldInfos().fieldInfo("f");+ Lucene99HnswVectorsReader hnswReader =+ (Lucene99HnswVectorsReader) segmentReader.getVectorReader().unwrapReaderForField("f");+ FlatVectorsReader flatReader = hnswReader.getFlatVectorsReader();+ QuantizedVectorsReader quantizedReader = (QuantizedVectorsReader) flatReader;+ SegmentWriteState writeState =+ new SegmentWriteState(+ InfoStream.getDefault(),+ dir,+ segmentReader.getSegmentInfo().info,+ r.getFieldInfos(),+ null,+ IOContext.DEFAULT);+ KnnVectorValues quantizedValues = quantizedReader.getQuantizedVectorValues("f");+ FlatVectorsScorer searchScorer = flatReader.getFlatVectorScorer("f");+ try (CloseableRandomVectorScorerSupplier mergeScorers =+ quantizedReader.getRandomVectorScorerSupplierForMerge(fieldInfo, writeState)) {+ FloatVectorValues rawValues = r.getFloatVectorValues("f");+ UpdateableRandomVectorScorer mergeScorer = mergeScorers.scorer();+ for (int i = 0; i < 5; i++) {+ int queryOrd = random().nextInt(numDocs);+ int targetOrd = random().nextInt(numDocs);+ // the search-time scorer normalizes a COSINE query before quantizing it+ RandomVectorScorer expected =+ searchScorer.getRandomVectorScorer(+ VectorSimilarityFunction.COSINE,+ quantizedValues,+ rawValues.vectorValue(queryOrd));+ mergeScorer.setScoringOrdinal(queryOrd);+ assertEquals(+ "encoding " + asymmetric + " query ord " + queryOrd + " target ord " + targetOrd,+ expected.score(targetOrd),+ mergeScorer.score(targetOrd),+ 1e-5f);+ }+ }+ }+ }+ }+ }+public void testSingleVectorCase() throws Exception {float[] vector = randomVector(random().nextInt(12, 500));for (VectorSimilarityFunction similarityFunction : VectorSimilarityFunction.values()) {diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.javaindex e326e926ab7..f253adb05d3 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java@@ -223,8 +223,7 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsForbyte[] scratch = new byte[encoding.getDiscreteDimensions(dims)];byte[] expectedVector = new byte[encoding.getDocPackedLength(scratch.length)];if (similarityFunction == VectorSimilarityFunction.COSINE) {- vectorValues =- new Lucene104ScalarQuantizedVectorsWriter.NormalizedFloatVectorValues(vectorValues);+ vectorValues = new NormalizedFloatVectorValues(vectorValues);}KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator();