Skip to content

Apache Lucene open-source search software

Remove vector merge scorer tests that check exact byte equality (#16752)

Simon Cooperaae7dc11cace
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedMergeScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedMergeScorer.java
index d75318c34a1..2cf580a05cd 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedMergeScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedMergeScorer.java
@@ -16,10 +16,8 @@
*/
package org.apache.lucene.codecs.lucene104;
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -27,7 +25,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.IntPredicate;
import java.util.function.LongConsumer;
-import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
@@ -44,7 +41,6 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnFloatVectorField;
-import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CodecReader;
@@ -64,10 +60,7 @@ import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SerialMergeScheduler;
-import org.apache.lucene.index.Term;
import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.FilterIndexInput;
@@ -78,10 +71,8 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.hnsw.HnswGraph;
-import org.apache.lucene.util.hnsw.HnswGraphBuilder;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
@@ -198,79 +189,6 @@ public class TestLucene104ScalarQuantizedMergeScorer extends LuceneTestCase {
counts.mergedRawBytesRead() < (long) DIM * Float.BYTES);
}
- /**
- * Verifies that writer and reader paths produce identical quantized vector data and HNSW graphs.
- * One {@code multiScalarQuantize} call must match separate index-side and query-side {@code
- * scalarQuantize} calls.
- *
- * <p>The sorted index and sparse second vector field exercise non-dense merged order. Non-unit
- * COSINE vectors exercise query-side normalization.
- */
- public void testGraphIsIdenticalToTheReaderFallback() throws IOException {
- for (ScalarEncoding encoding : asymmetricEncodings()) {
- for (VectorSimilarityFunction similarity : VectorSimilarityFunction.values()) {
- assertBothPathsWriteTheSameFiles(encoding, similarity, List.of(), VECTOR_EXTENSIONS);
- }
- }
- }
-
- /**
- * Verifies that writer and reader paths produce identical vector records when deletions change
- * merged ordinals.
- *
- * <p>The graph is excluded because HNSW output is not reproducible across otherwise identical
- * merges that drop documents.
- */
- public void testMergedRecordsAreIdenticalWithDeletions() throws IOException {
- for (ScalarEncoding encoding : asymmetricEncodings()) {
- VectorSimilarityFunction similarity =
- RandomPicks.randomFrom(random(), VectorSimilarityFunction.values());
- List<String> deleted = new ArrayList<>();
- for (int i = 0; i < 2 * DOCS_PER_SEGMENT; i++) {
- if (random().nextInt(10) == 0) {
- deleted.add(Integer.toString(i));
- }
- }
- assertBothPathsWriteTheSameFiles(encoding, similarity, deleted, RECORD_EXTENSIONS);
- }
- }
-
- private void assertBothPathsWriteTheSameFiles(
- ScalarEncoding encoding,
- VectorSimilarityFunction similarity,
- List<String> deleted,
- Set<String> compared)
- throws IOException {
- long savedSeed = HnswGraphBuilder.randSeed;
- try {
- float[][] vectors = randomVectors(2 * DOCS_PER_SEGMENT, similarity);
- long seed = random().nextLong();
- HnswGraphBuilder.randSeed = seed;
- Map<String, byte[]> writerPath =
- mergedVectorFiles(writerPathFormat(encoding, ALWAYS_GRAPH), vectors, similarity, deleted);
- HnswGraphBuilder.randSeed = seed;
- Map<String, byte[]> readerPath =
- mergedVectorFiles(readerPathFormat(encoding, ALWAYS_GRAPH), vectors, similarity, deleted);
- assertEquals(
- "different files were written for " + encoding + "/" + similarity,
- writerPath.keySet(),
- readerPath.keySet());
- for (String extension : compared) {
- assertArrayEquals(
- "the body of the merged ."
- + extension
- + " file, header and footer aside, differs for "
- + encoding
- + "/"
- + similarity,
- writerPath.get(extension),
- readerPath.get(extension));
- }
- } finally {
- HnswGraphBuilder.randSeed = savedSeed;
- }
- }
-
/**
* Verifies that a merge with no graph never creates a handoff file. This covers an HNSW merge
* rejected by the graph-size predicate and a flat merge that supplies no predicate to the
@@ -707,88 +625,6 @@ public class TestLucene104ScalarQuantizedMergeScorer extends LuceneTestCase {
}
}
- /**
- * Merges two segments and returns merged vector-file bodies keyed by extension. Headers and
- * footers are omitted because random segment IDs and their checksums differ between equivalent
- * indexes.
- *
- * <p>The index is sorted, a second vector field is sparse, and documents named by {@code deleted}
- * are removed before the merge.
- */
- private Map<String, byte[]> mergedVectorFiles(
- KnnVectorsFormat format,
- float[][] vectors,
- VectorSimilarityFunction similarity,
- List<String> deleted)
- throws IOException {
- try (Directory dir = newDirectory()) {
- IndexWriterConfig config =
- new IndexWriterConfig()
- .setCodec(TestUtil.alwaysKnnVectorsFormat(format))
- .setIndexSort(new Sort(new SortField("sort", SortField.Type.LONG)))
- // the two arms have to meet the same segments, so nothing may merge in the background
- .setMergeScheduler(new SerialMergeScheduler())
- .setUseCompoundFile(false);
- config.getCodec().compoundFormat().setShouldUseCompoundFile(false);
- try (IndexWriter writer = new IndexWriter(dir, config)) {
- for (int i = 0; i < vectors.length; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));
- doc.add(new NumericDocValuesField("sort", (i * 7919L) % 1000));
- doc.add(new KnnFloatVectorField("v", vectors[i], similarity));
- if (i % 3 == 0) {
- doc.add(new KnnFloatVectorField("w", vectors[vectors.length - 1 - i], similarity));
- }
- writer.addDocument(doc);
- if (i == vectors.length / 2 - 1) {
- writer.commit();
- }
- }
- writer.commit();
- for (String id : deleted) {
- writer.deleteDocuments(new Term("id", id));
- }
- writer.commit();
- writer.forceMerge(1);
- writer.commit();
- }
- SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
- assertEquals(1, infos.size());
- Map<String, byte[]> files = new HashMap<>();
- for (String file : infos.info(0).files()) {
- String extension = file.substring(file.lastIndexOf('.') + 1);
- if (VECTOR_EXTENSIONS.contains(extension)) {
- files.put(extension, fileBody(dir, file));
- }
- }
- assertTrue("no raw vectors were written", files.containsKey("vec"));
- assertTrue("no quantized vectors were written", files.containsKey("veq"));
- assertTrue("no graph was written, so this compares nothing", files.get("vex").length > 0);
- return files;
- }
- }
-
- /** Vector record files, excluding the HNSW graph. */
- private static final Set<String> RECORD_EXTENSIONS = Set.of("vec", "veq", "vemq");
-
- private static final Set<String> VECTOR_EXTENSIONS = Set.of("vec", "veq", "vemq", "vex");
-
- /** The bytes of a codec file between its index header and its footer. */
- private static byte[] fileBody(Directory dir, String file) throws IOException {
- try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
- in.readInt(); // magic
- in.readString(); // codec name
- in.readInt(); // version
- in.skipBytes(StringHelper.ID_LENGTH);
- in.skipBytes(in.readByte() & 0xFF); // segment suffix
- long start = in.getFilePointer();
- long length = in.length() - CodecUtil.footerLength() - start;
- byte[] body = new byte[Math.toIntExact(length)];
- in.readBytes(body, 0, body.length);
- return body;
- }
- }
-
private float[][] randomVectors(int count, VectorSimilarityFunction similarity) {
float[][] vectors = new float[count][];
for (int i = 0; i < count; i++) {