Apache Lucene open-source search software
Add a sandboxed scalar-quantized version of the de-duplicating vector format (#16506)
Kaival Parikhfb0ca7cd29d6
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 6134ad5759e..9aa30587797 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -385,9 +385,10 @@ New Featuresrequiring the ICU analysis plugin. Uses a generated switch for fold exceptions withCharacter.toLowerCase() as the default fallback. (Robert Muir, Costin Leau)-* GITHUB#15979: Add a sandboxed de-duplicating HNSW vector format (DedupHnswVectorsFormat) that stores- each distinct full-precision vector once, shared across all documents and fields that reference it.- (Kaival Parikh)+* GITHUB#15979, GITHUB#16506: Add sandboxed de-duplicating HNSW vector formats (DedupHnswVectorsFormat,+ DedupHnswScalarQuantizedVectorsFormat) that store each distinct full-precision and quantized vector+ once, shared across all documents and fields that reference it. Quantization is data-blind (assumes+ vectors are centered at zero) for effective de-duplication across fields. (Kaival Parikh)* GITHUB#16484: Add SpanMultiTermQueryWrapper.FrequentTermsSpanBooleanQueryRewrite. It's a rewrite method thattranslates each term into a SpanTermQuery within a {@link SpanOrQuery}, but retains only the most frequent terms so itdiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.javaindex f16ec3726ca..4ab62c49cf6 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java@@ -177,7 +177,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVereturn vectorValue.length;}- static void packNibbles(byte[] unpacked, byte[] packed) {+ /** Packs two 4-bit values per output byte: the upper half holds {@code unpacked[i]}. */+ public static void packNibbles(byte[] unpacked, byte[] packed) {assert unpacked.length == packed.length * 2;for (int i = 0; i < packed.length; i++) {int x = unpacked[i] << 4 | unpacked[packed.length + i];@@ -185,7 +186,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe}}- static void unpackNibbles(byte[] packed, byte[] unpacked) {+ /** Unpacks two 4-bit values per input byte, the reverse of {@link #packNibbles}. */+ public static void unpackNibbles(byte[] packed, byte[] unpacked) {assert unpacked.length == packed.length * 2;for (int i = 0; i < packed.length; i++) {unpacked[i] = (byte) ((packed[i] >> 4) & 0x0F);@@ -242,7 +244,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe}/** Dense off-heap scalar quantized vector values */- static class DenseOffHeapVectorValues extends OffHeapScalarQuantizedVectorValues {+ public static class DenseOffHeapVectorValues extends OffHeapScalarQuantizedVectorValues {DenseOffHeapVectorValues(int dimension,int size,@@ -265,7 +267,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVeslice);}- DenseOffHeapVectorValues(+ /** Creates dense off-heap scalar quantized vector values over the given slice. */+ public DenseOffHeapVectorValues(boolean isQuerySide,int dimension,int size,diff --git a/lucene/sandbox/src/java/module-info.java b/lucene/sandbox/src/java/module-info.javaindex 8a3a60d7c54..3a45ccb74b1 100644--- a/lucene/sandbox/src/java/module-info.java+++ b/lucene/sandbox/src/java/module-info.java@@ -43,5 +43,6 @@ module org.apache.lucene.sandbox {org.apache.lucene.sandbox.codecs.idversion.IDVersionPostingsFormat;provides org.apache.lucene.codecs.KnnVectorsFormat withorg.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormat,- org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat;+ org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat,+ org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormat;}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.javaindex 220578920b3..f2485b2f721 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java@@ -110,14 +110,14 @@ import org.apache.lucene.index.SegmentWriteState;final class DedupFlatVectorsFormat extends FlatVectorsFormat {static final String NAME = "DedupFlatVectorsFormat";- static final String META_CODEC_NAME = "DedupFlatVectorsFormatMeta";- static final String META_EXTENSION = "vdm";+ private static final String META_CODEC_NAME = "DedupFlatVectorsFormatMeta";+ private static final String META_EXTENSION = "vdm";- static final String VECTOR_DATA_CODEC_NAME = "DedupFlatVectorsFormatVectorData";- static final String VECTOR_DATA_EXTENSION = "vdd";+ private static final String VECTOR_DATA_CODEC_NAME = "DedupFlatVectorsFormatVectorData";+ private static final String VECTOR_DATA_EXTENSION = "vdd";- static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;+ private static final int VERSION_START = 0;+ private static final int VERSION_CURRENT = VERSION_START;private static final DedupFlatVectorsScorer FLAT_VECTORS_SCORER = new DedupFlatVectorsScorer();@@ -127,11 +127,29 @@ final class DedupFlatVectorsFormat extends FlatVectorsFormat {@Overridepublic FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {- return new DedupFlatVectorsWriter(state, FLAT_VECTORS_SCORER);+ return new DedupFlatVectorsWriter(+ state,+ FLAT_VECTORS_SCORER,+ null,+ META_CODEC_NAME,+ META_EXTENSION,+ VECTOR_DATA_CODEC_NAME,+ VECTOR_DATA_EXTENSION,+ null,+ null,+ VERSION_CURRENT);}@Overridepublic FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new DedupFlatVectorsReader(state, FLAT_VECTORS_SCORER);+ return new DedupFlatVectorsReader(+ state,+ FLAT_VECTORS_SCORER,+ META_CODEC_NAME,+ META_EXTENSION,+ VECTOR_DATA_CODEC_NAME,+ VECTOR_DATA_EXTENSION,+ VERSION_START,+ VERSION_CURRENT);}}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.javaindex dfe5454cd0d..a54424abd09 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java@@ -19,12 +19,6 @@ package org.apache.lucene.sandbox.codecs.dedup;import static org.apache.lucene.index.VectorEncoding.BYTE;import static org.apache.lucene.index.VectorEncoding.FLOAT16;import static org.apache.lucene.index.VectorEncoding.FLOAT32;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_CODEC_NAME;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_EXTENSION;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_CODEC_NAME;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_EXTENSION;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VERSION_CURRENT;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VERSION_START;import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupBytes;import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloat16s;import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloats;@@ -72,15 +66,25 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {private final FlatVectorsScorer vectorsScorer;private final Map<String, FieldEntry> fields;private final IndexInput vectorData;-- DedupFlatVectorsReader(SegmentReadState state, FlatVectorsScorer vectorsScorer)+ private final String vectorDataExtension;++ DedupFlatVectorsReader(+ SegmentReadState state,+ FlatVectorsScorer vectorsScorer,+ String metaCodecName,+ String metaExtension,+ String vectorDataCodecName,+ String vectorDataExtension,+ int versionStart,+ int versionCurrent)throws IOException {this.vectorsScorer = vectorsScorer;this.fields = new HashMap<>();+ this.vectorDataExtension = vectorDataExtension;String metaFileName =- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, META_EXTENSION);+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);int versionMeta;try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {@@ -89,9 +93,9 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {versionMeta =CodecUtil.checkIndexHeader(meta,- META_CODEC_NAME,- VERSION_START,- VERSION_CURRENT,+ metaCodecName,+ versionStart,+ versionCurrent,state.segmentInfo.getId(),state.segmentSuffix);readMetaBody(meta, state.fieldInfos);@@ -103,7 +107,14 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {}}- this.vectorData = openDataInput(state, versionMeta);+ this.vectorData =+ openDataInput(+ state,+ versionMeta,+ vectorDataCodecName,+ vectorDataExtension,+ versionStart,+ versionCurrent);}private void readMetaBody(ChecksumIndexInput meta, FieldInfos fieldInfos) throws IOException {@@ -174,12 +185,18 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {}}- private static IndexInput openDataInput(SegmentReadState state, int versionMeta)+ private static IndexInput openDataInput(+ SegmentReadState state,+ int versionMeta,+ String vectorDataCodecName,+ String vectorDataExtension,+ int versionStart,+ int versionCurrent)throws IOException {String fileName =IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, VECTOR_DATA_EXTENSION);+ state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);IOContext.FileOpenHint[] hints = {FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM@@ -193,9 +210,9 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {int versionVectorData =CodecUtil.checkIndexHeader(in,- VECTOR_DATA_CODEC_NAME,- VERSION_START,- VERSION_CURRENT,+ vectorDataCodecName,+ versionStart,+ versionCurrent,state.segmentInfo.getId(),state.segmentSuffix);if (versionMeta != versionVectorData) {@@ -203,7 +220,7 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {"Format versions mismatch: meta="+ versionMeta+ ", "- + VECTOR_DATA_CODEC_NAME+ + vectorDataCodecName+ "="+ versionVectorData,in);@@ -348,10 +365,11 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {}// TODO: This is an over-estimation.return Map.of(- VECTOR_DATA_EXTENSION,+ vectorDataExtension,entry.fieldInfo.fieldOrdToGroupOrdSize() + entry.groupInfo.vectorDataSize());}+ // package-private for testingrecord FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo) {private static final long SHALLOW_SIZE =RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class)diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.javaindex 63da8b2d950..1526c0ce29b 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.java@@ -25,7 +25,6 @@ import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.Bits;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;@@ -37,112 +36,123 @@ import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;** @lucene.experimental*/-final class DedupFlatVectorsScorer implements FlatVectorsScorer {- private static final FlatVectorsScorer SCORER =+sealed class DedupFlatVectorsScorer implements FlatVectorsScorer+ permits DedupScalarQuantizedVectorsScorer {++ private static final FlatVectorsScorer FLAT_SCORER =FlatVectorScorerUtil.getLucene99FlatVectorsScorer();+ private final FlatVectorsScorer delegate;++ DedupFlatVectorsScorer() {+ this(FLAT_SCORER);+ }++ protected DedupFlatVectorsScorer(FlatVectorsScorer delegate) {+ this.delegate = delegate;+ }++ /** Resolves the values to score; subclasses may unwrap composite values. */+ protected KnnVectorValues unwrap(KnnVectorValues vectorValues) {+ return vectorValues;+ }+@Overridepublic RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)throws IOException {+ vectorValues = unwrap(vectorValues);if (vectorValues instanceof DedupVectorValues dedupValues) {- RandomVectorScorerSupplier fieldView =- SCORER.getRandomVectorScorerSupplier(similarityFunction, vectorValues);RandomVectorScorerSupplier groupView =- SCORER.getRandomVectorScorerSupplier(similarityFunction, dedupValues.getGroupView());+ delegate.getRandomVectorScorerSupplier(similarityFunction, dedupValues.getGroupView());return new RandomVectorScorerSupplierImpl(- fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());}- return SCORER.getRandomVectorScorerSupplier(similarityFunction, vectorValues);+ return delegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues);}@Overridepublic RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)throws IOException {+ vectorValues = unwrap(vectorValues);if (vectorValues instanceof DedupVectorValues dedupValues) {- RandomVectorScorer fieldView =- SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);RandomVectorScorer groupView =- SCORER.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);- return new RandomVectorScorerImpl(fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());+ delegate.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);+ return new RandomVectorScorerImpl(+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());}- return SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);+ return delegate.getRandomVectorScorer(similarityFunction, vectorValues, target);}@Overridepublic RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, byte[] target)throws IOException {+ vectorValues = unwrap(vectorValues);if (vectorValues instanceof DedupVectorValues dedupValues) {- RandomVectorScorer fieldView =- SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);RandomVectorScorer groupView =- SCORER.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);- return new RandomVectorScorerImpl(fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());+ delegate.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);+ return new RandomVectorScorerImpl(+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());}- return SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);+ return delegate.getRandomVectorScorer(similarityFunction, vectorValues, target);}@Overridepublic RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, short[] target)throws IOException {+ vectorValues = unwrap(vectorValues);if (vectorValues instanceof DedupVectorValues dedupValues) {- RandomVectorScorer fieldView =- SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);RandomVectorScorer groupView =- SCORER.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);- return new RandomVectorScorerImpl(fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());+ delegate.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);+ return new RandomVectorScorerImpl(+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());}- return SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);+ return delegate.getRandomVectorScorer(similarityFunction, vectorValues, target);}- private record RandomVectorScorerSupplierImpl(- RandomVectorScorerSupplier fieldView,+ /**+ * Supplies scorers whose scoring and target ordinals are both translated to group ordinals, with+ * doc operations on the original values.+ */+ record RandomVectorScorerSupplierImpl(+ KnnVectorValues values,RandomVectorScorerSupplier groupView,FieldOrdToGroupOrd fieldOrdToGroupOrd)implements RandomVectorScorerSupplier {@Overridepublic UpdateableRandomVectorScorer scorer() throws IOException {- return new UpdateableRandomVectorScorerImpl(- fieldView.scorer(), groupView.scorer(), fieldOrdToGroupOrd);+ return new UpdateableRandomVectorScorerImpl(values, groupView.scorer(), fieldOrdToGroupOrd);}@Overridepublic RandomVectorScorerSupplier copy() throws IOException {return new RandomVectorScorerSupplierImpl(- fieldView.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());+ values.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());}}- private static class RandomVectorScorerImpl implements RandomVectorScorer {- private final RandomVectorScorer fieldView;+ private static class RandomVectorScorerImpl+ extends RandomVectorScorer.AbstractRandomVectorScorer {+private final RandomVectorScorer groupView;private final FieldOrdToGroupOrd fieldOrdToGroupOrd;private int[] scratch;RandomVectorScorerImpl(- RandomVectorScorer fieldView,+ KnnVectorValues values,RandomVectorScorer groupView,FieldOrdToGroupOrd fieldOrdToGroupOrd) {- this.fieldView = fieldView;++ super(values);this.groupView = groupView;this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;this.scratch = new int[SCRATCH_INITIAL_SIZE];}- @Override- public int ordToDoc(int ord) {- return fieldView.ordToDoc(ord);- }-- @Override- public Bits getAcceptOrds(Bits acceptDocs) {- return fieldView.getAcceptOrds(acceptDocs);- }-@Overridepublic float score(int node) throws IOException {return groupView.score(fieldOrdToGroupOrd.get(node));@@ -158,23 +168,20 @@ final class DedupFlatVectorsScorer implements FlatVectorsScorer {}return groupView.bulkScore(scratch, scores, numNodes);}-- @Override- public int maxOrd() {- return fieldView.maxOrd();- }}private static final class UpdateableRandomVectorScorerImpl extends RandomVectorScorerImplimplements UpdateableRandomVectorScorer {+private final UpdateableRandomVectorScorer groupView;private final FieldOrdToGroupOrd fieldOrdToGroupOrd;UpdateableRandomVectorScorerImpl(- UpdateableRandomVectorScorer fieldView,+ KnnVectorValues values,UpdateableRandomVectorScorer groupView,FieldOrdToGroupOrd fieldOrdToGroupOrd) {- super(fieldView, groupView, fieldOrdToGroupOrd);++ super(values, groupView, fieldOrdToGroupOrd);this.groupView = groupView;this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.javaindex d8020fb2e85..1e3b6418cd7 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.java@@ -16,12 +16,6 @@*/package org.apache.lucene.sandbox.codecs.dedup;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_CODEC_NAME;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_EXTENSION;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_CODEC_NAME;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_EXTENSION;-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VERSION_CURRENT;-import java.io.IOException;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;@@ -41,6 +35,10 @@ import org.apache.lucene.util.RamUsageEstimator;* or merging existing segments (never both), delegating to {@link DedupFlushContext} or {@link* DedupMergeContext} accordingly.*+ * <p>Also used by {@link DedupScalarQuantizedVectorsFormat} (with a non-null {@link+ * DedupQuantizer}) to additionally write a quantized copy of each applicable group, into a separate+ * quantized vector data file.+ ** @lucene.experimental*/final class DedupFlatVectorsWriter extends FlatVectorsWriter {@@ -49,42 +47,73 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {private final IndexOutput meta;private final IndexOutput vectorData;+ private final IndexOutput quantizedVectorData;private boolean finished;+ private final DedupQuantizer quantizer;+private final DedupFlushContext flushContext;private boolean usedForFlush;private final DedupMergeContext mergeContext;private boolean usedForMerge;- DedupFlatVectorsWriter(SegmentWriteState state, FlatVectorsScorer vectorsScorer)+ DedupFlatVectorsWriter(+ SegmentWriteState state,+ FlatVectorsScorer vectorsScorer,+ DedupQuantizer quantizer,+ String metaCodecName,+ String metaExtension,+ String vectorDataCodecName,+ String vectorDataExtension,+ String quantizedVectorDataCodecName,+ String quantizedVectorDataExtension,+ int versionCurrent)throws IOException {super(vectorsScorer);this.finished = false;+ this.quantizer = quantizer;this.flushContext = new DedupFlushContext();this.usedForFlush = false;this.mergeContext = new DedupMergeContext();this.usedForMerge = false;String metaFileName =- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, META_EXTENSION);+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);String vectorDataFileName =IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, VECTOR_DATA_EXTENSION);+ state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);boolean success = false;try {this.meta = state.directory.createOutput(metaFileName, state.context);- this.vectorData = state.directory.createOutput(vectorDataFileName, state.context);CodecUtil.writeIndexHeader(- meta, META_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);+ meta, metaCodecName, versionCurrent, state.segmentInfo.getId(), state.segmentSuffix);++ this.vectorData = state.directory.createOutput(vectorDataFileName, state.context);CodecUtil.writeIndexHeader(vectorData,- VECTOR_DATA_CODEC_NAME,- VERSION_CURRENT,+ vectorDataCodecName,+ versionCurrent,state.segmentInfo.getId(),state.segmentSuffix);++ if (quantizer != null) {+ String quantizedVectorDataFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, quantizedVectorDataExtension);+ this.quantizedVectorData =+ state.directory.createOutput(quantizedVectorDataFileName, state.context);+ CodecUtil.writeIndexHeader(+ quantizedVectorData,+ quantizedVectorDataCodecName,+ versionCurrent,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ } else {+ this.quantizedVectorData = null;+ }success = true;} finally {if (success == false) {@@ -110,7 +139,7 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {}usedForFlush = true;- flushContext.flush(meta, vectorData, maxDoc, sortMap);+ flushContext.flush(meta, vectorData, quantizedVectorData, maxDoc, sortMap, quantizer);}@Override@@ -126,6 +155,10 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {CodecUtil.writeFooter(meta);CodecUtil.writeFooter(vectorData);++ if (quantizedVectorData != null) {+ CodecUtil.writeFooter(quantizedVectorData);+ }}@Override@@ -140,12 +173,12 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {}private void finishMerge() throws IOException {- mergeContext.finish(meta, vectorData);+ mergeContext.finish(meta, vectorData, quantizedVectorData, quantizer);}@Overridepublic void close() throws IOException {- IOUtils.close(meta, vectorData);+ IOUtils.close(meta, vectorData, quantizedVectorData);}@Overridediff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.javaindex 59ad5465b83..d29645500c5 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java@@ -28,9 +28,11 @@ import java.nio.FloatBuffer;import java.nio.ShortBuffer;import java.util.ArrayList;import java.util.Arrays;+import java.util.EnumSet;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Set;import org.apache.lucene.codecs.KnnVectorsWriter;import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;import org.apache.lucene.index.DocsWithFieldSet;@@ -94,11 +96,34 @@ final class DedupFlushContext implements Accountable {return fieldVectorsWriter;}- void flush(IndexOutput meta, IndexOutput vectorData, int maxDoc, Sorter.DocMap sortMap)+ /**+ * Writes each group's distinct vectors once, followed by per-field metadata. When {@code+ * quantizer} is non-null, a quantized copy of each applicable group is also written and its block+ * location appended to the group metadata.+ */+ void flush(+ IndexOutput meta,+ IndexOutput vectorData,+ IndexOutput quantizedVectorData,+ int maxDoc,+ Sorter.DocMap sortMap,+ DedupQuantizer quantizer)throws IOException {Map<GroupKey, Integer> groupOrds = new HashMap<>();+ // quantization flavors referencing each group, derived from its fields' similarity functions+ Map<GroupKey, Set<DedupQuantizer.Flavor>> groupFlavors = new HashMap<>();+ if (quantizer != null) {+ for (FieldData fieldData : fieldDataList) {+ if (fieldData.fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {+ groupFlavors+ .computeIfAbsent(fieldData.groupKey, _ -> EnumSet.noneOf(DedupQuantizer.Flavor.class))+ .add(DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));+ }+ }+ }+int groupOrd = 0;for (Map.Entry<GroupKey, DedupGroup<?>> entry : groups.entrySet()) {GroupKey groupKey = entry.getKey();@@ -121,6 +146,22 @@ final class DedupFlushContext implements Accountable {groupOrd, dimension, encoding, groupNumVectors, vectorDataOffset, vectorDataSize);groupInfo.write(meta);+ if (quantizer != null) {+ if (group instanceof FloatGroup floatGroup) {+ quantizer.writeGroup(+ meta,+ quantizedVectorData,+ encoding,+ dimension,+ groupNumVectors,+ groupFlavors.get(groupKey),+ floatGroup::get,+ null);+ } else {+ DedupQuantizer.writeEmptyGroup(meta);+ }+ }+groupOrds.put(groupKey, groupOrd);groupOrd++;}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.javanew file mode 100644index 00000000000..a39816de55d--- /dev/null+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java@@ -0,0 +1,246 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.HNSW_GRAPH_THRESHOLD;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN;++import java.io.IOException;+import java.util.concurrent.ExecutorService;+import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.KnnVectorsWriter;+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;+import org.apache.lucene.index.MergePolicy;+import org.apache.lucene.index.MergeScheduler;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.util.hnsw.HnswGraph;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;++/**+ * An HNSW vector format that de-duplicates vectors, storing each distinct vector once in both raw+ * and scalar quantized form.+ *+ * <p>Graph construction and search are identical to {@link+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat}, with searches scoring against the+ * quantized vectors. A {@link DedupScalarQuantizedVectorsFormat} is used for the flat vector+ * storage, which stores each distinct vector exactly once, shared across all documents that+ * reference it. See {@link DedupHnswVectorsFormat} for the intended multi-field usage pattern.+ *+ * <p>If you customize this format, be sure to <b>share the same instance</b> of the underlying+ * {@link DedupScalarQuantizedVectorsFormat} to de-duplicate vectors correctly.+ *+ * @lucene.experimental+ */+public final class DedupHnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {+ private static final String NAME = "DedupHnswScalarQuantizedVectorsFormat";++ /**+ * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to+ * {@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See+ * {@link HnswGraph} for more details.+ */+ private final int maxConn;++ /**+ * The number of candidate neighbors to track while searching the graph for each newly inserted+ * node. Defaults to {@link+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link+ * HnswGraph} for details.+ */+ private final int beamWidth;++ /** The scalar encoding for quantization. Defaults to {@link ScalarEncoding#UNSIGNED_BYTE}. */+ private final ScalarEncoding encoding;++ /** The format for storing, reading, and merging vectors on disk. */+ private final FlatVectorsFormat flatVectorsFormat;++ private final int numMergeWorkers;+ private final TaskExecutor mergeExec;++ /**+ * The threshold to use to bypass HNSW graph building for tiny segments in terms of k for a graph+ * i.e. number of docs to match the query (default is {@link+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#HNSW_GRAPH_THRESHOLD}).+ *+ * <ul>+ * <li>0 indicates that the graph is always built.+ * <li>Positive values require that many estimated visited nodes before a graph is built.+ * <li>Negative values aren't allowed.+ * </ul>+ */+ private final int tinySegmentsThreshold;++ /** Constructs a format with UNSIGNED_BYTE encoding and default graph construction parameters */+ public DedupHnswScalarQuantizedVectorsFormat() {+ this(+ ScalarEncoding.UNSIGNED_BYTE,+ DEFAULT_MAX_CONN,+ DEFAULT_BEAM_WIDTH,+ DEFAULT_NUM_MERGE_WORKER,+ null,+ HNSW_GRAPH_THRESHOLD);+ }++ /**+ * Constructs a format with UNSIGNED_BYTE encoding, using the given graph construction parameters.+ *+ * @param maxConn the maximum number of connections to a node in the HNSW graph+ * @param beamWidth the size of the queue maintained during graph construction.+ */+ public DedupHnswScalarQuantizedVectorsFormat(int maxConn, int beamWidth) {+ this(+ ScalarEncoding.UNSIGNED_BYTE,+ maxConn,+ beamWidth,+ DEFAULT_NUM_MERGE_WORKER,+ null,+ HNSW_GRAPH_THRESHOLD);+ }++ /**+ * Constructs a format using the given quantization encoding and graph construction parameters.+ *+ * @param encoding the quantization encoding used to encode the vectors+ * @param maxConn the maximum number of connections to a node in the HNSW graph+ * @param beamWidth the size of the queue maintained during graph construction.+ */+ public DedupHnswScalarQuantizedVectorsFormat(+ ScalarEncoding encoding, int maxConn, int beamWidth) {+ this(encoding, maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null, HNSW_GRAPH_THRESHOLD);+ }++ /**+ * Constructs a format using the given quantization encoding and graph construction parameters.+ *+ * @param encoding the quantization encoding used to encode the vectors+ * @param maxConn the maximum number of connections to a node in the HNSW graph+ * @param beamWidth the size of the queue maintained during graph construction.+ * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If+ * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec+ * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are+ * generated by this format to do the merge. If null, the configured {@link+ * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used.+ */+ public DedupHnswScalarQuantizedVectorsFormat(+ ScalarEncoding encoding,+ int maxConn,+ int beamWidth,+ int numMergeWorkers,+ ExecutorService mergeExec) {+ this(encoding, maxConn, beamWidth, numMergeWorkers, mergeExec, HNSW_GRAPH_THRESHOLD);+ }++ /**+ * Constructs a format using the given quantization encoding and graph construction parameters.+ *+ * @param encoding the quantization encoding used to encode the vectors+ * @param maxConn the maximum number of connections to a node in the HNSW graph+ * @param beamWidth the size of the queue maintained during graph construction.+ * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If+ * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec+ * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are+ * generated by this format to do the merge. If null, the configured {@link+ * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used.+ * @param tinySegmentsThreshold the expected number of vector operations to return k nearest+ * neighbors of the current graph size+ */+ public DedupHnswScalarQuantizedVectorsFormat(+ ScalarEncoding encoding,+ int maxConn,+ int beamWidth,+ int numMergeWorkers,+ ExecutorService mergeExec,+ int tinySegmentsThreshold) {+ super(NAME);+ if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) {+ throw new IllegalArgumentException(+ "maxConn must be positive and less than or equal to "+ + MAXIMUM_MAX_CONN+ + "; maxConn="+ + maxConn);+ }+ if (beamWidth <= 0 || beamWidth > MAXIMUM_BEAM_WIDTH) {+ throw new IllegalArgumentException(+ "beamWidth must be positive and less than or equal to "+ + MAXIMUM_BEAM_WIDTH+ + "; beamWidth="+ + beamWidth);+ }+ this.maxConn = maxConn;+ this.beamWidth = beamWidth;+ this.encoding = encoding;+ this.flatVectorsFormat = new DedupScalarQuantizedVectorsFormat(encoding);+ this.tinySegmentsThreshold = tinySegmentsThreshold;+ if (numMergeWorkers == 1 && mergeExec != null) {+ throw new IllegalArgumentException(+ "No executor service is needed as we'll use single thread to merge");+ }+ this.numMergeWorkers = numMergeWorkers;+ if (mergeExec != null) {+ this.mergeExec = new TaskExecutor(mergeExec);+ } else {+ this.mergeExec = null;+ }+ }++ @Override+ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {+ return new Lucene99HnswVectorsWriter(+ state,+ maxConn,+ beamWidth,+ flatVectorsFormat,+ flatVectorsFormat.fieldsWriter(state),+ numMergeWorkers,+ mergeExec,+ tinySegmentsThreshold);+ }++ @Override+ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {+ return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));+ }++ @Override+ public int getMaxDimensions(String fieldName) {+ return DEFAULT_MAX_DIMENSIONS;+ }++ @Override+ public String toString() {+ return NAME+ + "(maxConn="+ + maxConn+ + ", beamWidth="+ + beamWidth+ + ", encoding="+ + encoding+ + ", tinySegmentsThreshold="+ + tinySegmentsThreshold+ + ")";+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.javaindex bff40f47055..2c568638b5b 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java@@ -219,16 +219,12 @@ public final class DedupHnswVectorsFormat extends KnnVectorsFormat {@Overridepublic String toString() {return NAME- + "(name="- + NAME- + ", maxConn="+ + "(maxConn="+ maxConn+ ", beamWidth="+ beamWidth+ ", tinySegmentsThreshold="+ tinySegmentsThreshold- + ", flatVectorFormat="- + FORMAT+ ")";}}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.javaindex 55ca0e2cb1a..9b99cc490da 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java@@ -28,9 +28,11 @@ import java.nio.FloatBuffer;import java.nio.ShortBuffer;import java.util.ArrayList;import java.util.Arrays;+import java.util.EnumSet;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Set;import java.util.stream.Collectors;import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.DocIDMerger;@@ -86,7 +88,18 @@ final class DedupMergeContext implements Accountable {mergeState.segmentInfo.maxDoc()));}- void finish(IndexOutput meta, IndexOutput vectorData) throws IOException {+ /**+ * Merges each group's distinct vectors, followed by per-field metadata. When {@code quantizer} is+ * non-null, a quantized copy of each applicable group is also written and its block location+ * appended to the group metadata — copying records from source segments in this format, and+ * re-reading a distinct vector through its handle to quantize it otherwise.+ */+ void finish(+ IndexOutput meta,+ IndexOutput vectorData,+ IndexOutput quantizedVectorData,+ DedupQuantizer quantizer)+ throws IOException {// Evaluate compatible fields together for correct de-duplicationMap<GroupKey, List<FieldData>> fieldGroups =@@ -120,6 +133,27 @@ final class DedupMergeContext implements Accountable {groupOrd, dimension, encoding, groupNumVectors, vectorDataOffset, vectorDataSize);groupInfo.write(meta);+ if (quantizer != null) {+ if (mergeGroup instanceof FloatGroup floatGroup) {+ Set<DedupQuantizer.Flavor> flavors = EnumSet.noneOf(DedupQuantizer.Flavor.class);+ for (FieldData fieldData : entry.getValue()) {+ flavors.add(+ DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));+ }+ quantizer.writeGroup(+ meta,+ quantizedVectorData,+ encoding,+ dimension,+ groupNumVectors,+ flavors,+ ord -> floatGroup.get(ord).get(),+ floatGroup::preQuantized);+ } else {+ DedupQuantizer.writeEmptyGroup(meta);+ }+ }+groupOrds.put(groupKey, groupOrd);groupOrd++;}@@ -254,6 +288,22 @@ final class DedupMergeContext implements Accountable {return new FloatVector(sub.values, sub.iterator.index());}+ /**+ * The already-quantized record of the distinct vector at a group ordinal, when its source+ * segment is in this format (data-blind quantization is a pure function of the raw vector, so+ * the record can be copied on merge instead of re-quantizing), or {@code null}.+ */+ DedupQuantizer.PreQuantized preQuantized(int ord) {+ FloatVector handle = get(ord);+ if (handle.values()+ instanceof DedupScalarQuantizedVectorValues.RawAndQuantizedValues rawAndQuantized) {+ DedupScalarQuantizedVectorValues.FieldValues quantized =+ rawAndQuantized.getQuantizedValues();+ return new DedupQuantizer.PreQuantized(quantized, quantized.flavor(), handle.ord());+ }+ return null;+ }+@Overridepublic long hash(FloatVector vector) throws IOException {buffer.put(0, vector.get());diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.javanew file mode 100644index 00000000000..f764f0ef4b7--- /dev/null+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java@@ -0,0 +1,295 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import java.io.IOException;+import java.util.EnumMap;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import org.apache.lucene.codecs.lucene104.OffHeapScalarQuantizedVectorValues;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.VectorEncoding;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.VectorUtil;+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;++/**+ * Write-side helper performing <i>data-blind</i> scalar quantization of de-duplicated vectors.+ *+ * <p>Quantization assumes input vectors are evenly distributed around the origin, i.e. the centroid+ * is a zero vector. The quantized record of a vector (packed bytes, optimized intervals, an+ * additional correction and the component sum) is then a pure function of the raw vector and the+ * {@link Flavor} derived from the field's similarity function: independent of the other vectors in+ * the segment. Identical vectors thus quantize identically and the quantized data de-duplicates+ * along with the raw data, shared across all documents and fields of a group with the same flavor.+ *+ * <p>For the same reason, a record already computed by a source segment does not change on merge:+ * when a distinct vector originates from a segment in this format (with the same encoding and+ * flavor), its quantized record is <b>copied</b> instead of re-reading the raw vector and+ * re-quantizing it.+ *+ * <p>Records follow the {@link+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat} conventions exactly+ * (per flavor), so they are scored by the stock {@link+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer}.+ *+ * @lucene.experimental+ */+record DedupQuantizer(ScalarEncoding encoding) {++ /**+ * How a vector is prepared and quantized, derived from the field's {@link+ * VectorSimilarityFunction}. Fields of a group whose similarity functions map to the same flavor+ * share one quantized record per distinct vector; a group stores one block per flavor in use.+ */+ enum Flavor {++ /** Vectors quantized as-is; the additional correction holds the squared norm. */+ EUCLIDEAN(0, VectorSimilarityFunction.EUCLIDEAN, false),++ /**+ * Vectors quantized as-is; the additional correction holds the dot product with the (zero)+ * centroid, i.e. zero.+ */+ DOT_PRODUCT(1, VectorSimilarityFunction.DOT_PRODUCT, false),++ /**+ * Vectors are l2-normalized before quantization (for cosine similarity, scored as a dot+ * product); the additional correction is zero as for {@link #DOT_PRODUCT}.+ */+ NORMALIZED(2, VectorSimilarityFunction.DOT_PRODUCT, true);++ /** The number used to identify this flavor on the wire, rather than relying on ordinal. */+ private final int wireNumber;++ private final VectorSimilarityFunction quantizerFunction;+ private final boolean normalized;++ Flavor(int wireNumber, VectorSimilarityFunction quantizerFunction, boolean normalized) {+ this.wireNumber = wireNumber;+ this.quantizerFunction = quantizerFunction;+ this.normalized = normalized;+ }++ static Flavor of(VectorSimilarityFunction function) {+ return switch (function) {+ case EUCLIDEAN -> EUCLIDEAN;+ case DOT_PRODUCT, MAXIMUM_INNER_PRODUCT -> DOT_PRODUCT;+ case COSINE -> NORMALIZED;+ };+ }++ int getWireNumber() {+ return wireNumber;+ }++ /** Returns the flavor for the given wire number, or empty if unknown. */+ static Optional<Flavor> fromWireNumber(int wireNumber) {+ for (Flavor flavor : values()) {+ if (flavor.wireNumber == wireNumber) {+ return Optional.of(flavor);+ }+ }+ return Optional.empty();+ }++ /**+ * The quantizer producing this flavor's records against a zero centroid. NOTE: {@link+ * #NORMALIZED} quantizes with {@link VectorSimilarityFunction#DOT_PRODUCT} (identical bytes and+ * corrections, without the unit-centroid requirement of a cosine-configured quantizer).+ */+ OptimizedScalarQuantizer quantizer() {+ return new OptimizedScalarQuantizer(quantizerFunction);+ }++ boolean normalized() {+ return normalized;+ }+ }++ /** Supplies the distinct (raw) float vector at a group ordinal. */+ interface FloatVectorSupplier {+ float[] get(int ord) throws IOException;+ }++ /**+ * The already-quantized record of a distinct vector, held by a source segment at {@code ord} with+ * the given flavor.+ */+ record PreQuantized(QuantizedByteVectorValues values, Flavor flavor, int ord) {}++ /**+ * Supplies the {@link PreQuantized} record of the distinct vector at a group ordinal, or {@code+ * null} when its source segment does not hold one.+ */+ interface PreQuantizedSupplier {+ PreQuantized get(int ord) throws IOException;+ }++ /** The encoding, offset and size of one flavor's quantized data block. */+ record QuantizedBlock(+ ScalarEncoding encoding, long quantizedDataOffset, long quantizedDataSize) {}++ /** Writes an empty flavor-block list, for groups without quantized data. */+ static void writeEmptyGroup(IndexOutput meta) throws IOException {+ meta.writeVInt(0);+ }++ /** Reads the flavor-block list of a group, the counterpart of {@link #writeGroup}. */+ static Map<Flavor, QuantizedBlock> readGroup(IndexInput meta) throws IOException {+ int numFlavors = meta.readVInt();+ Map<Flavor, QuantizedBlock> blocks = new EnumMap<>(Flavor.class);+ for (int i = 0; i < numFlavors; i++) {+ int flavorWireNumber = meta.readVInt();+ Flavor flavor =+ Flavor.fromWireNumber(flavorWireNumber)+ .orElseThrow(+ () ->+ new CorruptIndexException(+ "Invalid flavor wire number: " + flavorWireNumber, meta));+ int wireNumber = meta.readVInt();+ ScalarEncoding encoding =+ ScalarEncoding.fromWireNumber(wireNumber)+ .orElseThrow(+ () ->+ new CorruptIndexException(+ "Invalid scalar encoding wire number: " + wireNumber, meta));+ long offset = meta.readLong();+ long size = meta.readLong();+ if (blocks.put(flavor, new QuantizedBlock(encoding, offset, size)) != null) {+ throw new CorruptIndexException("Duplicate flavor: " + flavor, meta);+ }+ }+ return blocks;+ }++ /**+ * Quantizes and writes a group's distinct vectors: one block per flavor, one record per group+ * ordinal within each block. Wherever a {@code preQuantized} record with a matching encoding and+ * flavor is available (i.e. the vector originates from a segment in this format), it is copied+ * as-is; otherwise the raw vector is quantized. The block locations are written to {@code meta};+ * non-FLOAT32 groups (stored raw only) record an empty list.+ */+ void writeGroup(+ IndexOutput meta,+ IndexOutput quantizedVectorData,+ VectorEncoding groupEncoding,+ int dimension,+ int numVectors,+ Set<Flavor> flavors,+ FloatVectorSupplier vectors,+ PreQuantizedSupplier preQuantized)+ throws IOException {++ if (groupEncoding != VectorEncoding.FLOAT32) {+ writeEmptyGroup(meta);+ return;+ }++ meta.writeVInt(flavors.size());+ for (Flavor flavor : Flavor.values()) {+ if (flavors.contains(flavor) == false) {+ continue;+ }+ QuantizedBlock block =+ writeFlavorBlock(+ quantizedVectorData, flavor, dimension, numVectors, vectors, preQuantized);+ meta.writeVInt(flavor.getWireNumber());+ meta.writeVInt(block.encoding().getWireNumber());+ meta.writeLong(block.quantizedDataOffset());+ meta.writeLong(block.quantizedDataSize());+ }+ }++ private QuantizedBlock writeFlavorBlock(+ IndexOutput quantizedVectorData,+ Flavor flavor,+ int dimension,+ int numVectors,+ FloatVectorSupplier vectors,+ PreQuantizedSupplier preQuantized)+ throws IOException {++ OptimizedScalarQuantizer quantizer = flavor.quantizer();+ float[] zeroCentroid = new float[dimension];+ float[] normalized = flavor.normalized() ? new float[dimension] : null;+ byte[] scratch = new byte[encoding.getDiscreteDimensions(dimension)];+ byte[] packed =+ switch (encoding) {+ case UNSIGNED_BYTE, SEVEN_BIT -> scratch;+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE, DIBIT_QUERY_NIBBLE ->+ new byte[encoding.getDocPackedLength(scratch.length)];+ };++ long quantizedDataOffset = quantizedVectorData.alignFilePointer(Float.BYTES);+ for (int ord = 0; ord < numVectors; ord++) {+ // The quantized record is a pure function of the raw vector and the flavor: a record+ // already computed by a source segment can be copied instead of re-quantizing.+ if (preQuantized != null) {+ PreQuantized pre = preQuantized.get(ord);+ if (pre != null && pre.flavor() == flavor && pre.values().getScalarEncoding() == encoding) {+ // NOTE: read the packed bytes before the corrective terms, which are then served from the+ // record cached by the read of the packed bytes+ writeRecord(+ quantizedVectorData,+ pre.values().vectorValue(pre.ord()),+ pre.values().getCorrectiveTerms(pre.ord()));+ continue;+ }+ }++ float[] vector = vectors.get(ord);+ if (flavor.normalized()) {+ // normalize a copy: the source buffer is shared / owned by the group+ System.arraycopy(vector, 0, normalized, 0, dimension);+ VectorUtil.l2normalize(normalized);+ vector = normalized;+ }+ // NOTE: scalarQuantize subtracts the centroid from the input in place, but with a zero+ // centroid the values are unchanged, so shared / owned buffers are safe to pass directly.+ OptimizedScalarQuantizer.QuantizationResult corrections =+ quantizer.scalarQuantize(vector, scratch, encoding.getBits(), zeroCentroid);+ switch (encoding) {+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(scratch, packed);+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(scratch, packed);+ case DIBIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.transposeDibit(scratch, packed);+ case UNSIGNED_BYTE, SEVEN_BIT -> {}+ }+ writeRecord(quantizedVectorData, packed, corrections);+ }+ long quantizedDataSize = quantizedVectorData.getFilePointer() - quantizedDataOffset;++ return new QuantizedBlock(encoding, quantizedDataOffset, quantizedDataSize);+ }++ private static void writeRecord(+ IndexOutput quantizedVectorData,+ byte[] packed,+ OptimizedScalarQuantizer.QuantizationResult corrections)+ throws IOException {+ quantizedVectorData.writeBytes(packed, packed.length);+ quantizedVectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval()));+ quantizedVectorData.writeInt(Float.floatToIntBits(corrections.upperInterval()));+ quantizedVectorData.writeInt(Float.floatToIntBits(corrections.additionalCorrection()));+ quantizedVectorData.writeInt(corrections.quantizedComponentSum());+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.javanew file mode 100644index 00000000000..8504a4c6d0f--- /dev/null+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java@@ -0,0 +1,331 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import static org.apache.lucene.index.VectorEncoding.FLOAT32;++import java.io.IOException;+import org.apache.lucene.codecs.lucene104.OffHeapScalarQuantizedVectorValues;+import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdOffHeap;+import org.apache.lucene.search.VectorScorer;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.hnsw.RandomVectorScorer;+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;++/**+ * Vector values over data-blind scalar quantized, de-duplicated vectors. Mirrors the raw {@link+ * DedupVectorValues} structure: a dense group view (an {@link OffHeapScalarQuantizedVectorValues}+ * with a zero centroid) holds one quantized record per distinct vector, and a {@link FieldValues}+ * view maps a field's per-document ordinals onto it via the shared {@code fieldOrdToGroupOrd}+ * translation.+ *+ * @lucene.experimental+ */+final class DedupScalarQuantizedVectorValues {++ private DedupScalarQuantizedVectorValues() {}++ /** Number of corrective bytes stored per quantized vector. */+ static final int CORRECTIVE_BYTES = 3 * Float.BYTES + Integer.BYTES;++ /**+ * A dense view over a group's quantized records of one {@link DedupQuantizer.Flavor}, indexed by+ * group ordinal. Quantization is data-blind, i.e. against a zero centroid (see {@link+ * DedupQuantizer}). With {@code isQuerySide}, records hold query-encoded vectors instead (only+ * different for asymmetric encodings, used for merge-time graph construction).+ */+ static QuantizedByteVectorValues groupValues(+ boolean isQuerySide,+ DedupScalarQuantizedVectorsScorer vectorsScorer,+ VectorSimilarityFunction function,+ DedupQuantizer.Flavor flavor,+ ScalarEncoding encoding,+ int dimension,+ int groupNumVectors,+ IndexInput slice) {++ return new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(+ isQuerySide,+ dimension,+ groupNumVectors,+ new float[dimension], // always a zero centroid, quantization is data-blind+ 0,+ flavor.quantizer(),+ encoding,+ function,+ vectorsScorer,+ slice);+ }++ static FieldValues loadQuantized(+ DedupScalarQuantizedVectorsScorer vectorsScorer,+ VectorSimilarityFunction function,+ ScalarEncoding encoding,+ OrdToDocDISIReaderConfiguration configuration,+ int dimension,+ int groupNumVectors,+ IndexInput vectorData,+ IndexInput quantizedVectorData,+ long quantizedDataOffset,+ long quantizedDataSize,+ long fieldOrdToGroupOrdOffset,+ long fieldOrdToGroupOrdSize)+ throws IOException {++ final OffHeapFloatVectorValues fieldView =+ OffHeapFloatVectorValues.load(+ function, vectorsScorer, configuration, FLOAT32, dimension, 0, 0, vectorData);++ final QuantizedByteVectorValues groupView =+ groupValues(+ false,+ vectorsScorer,+ function,+ DedupQuantizer.Flavor.of(function),+ encoding,+ dimension,+ groupNumVectors,+ quantizedVectorData.slice(+ "quantized-group-slice", quantizedDataOffset, quantizedDataSize));++ final FieldOrdToGroupOrd fieldOrdToGroupOrd =+ new FieldOrdToGroupOrdOffHeap(vectorData, fieldOrdToGroupOrdOffset, fieldOrdToGroupOrdSize);++ return new FieldValues(vectorsScorer, function, fieldView, groupView, fieldOrdToGroupOrd);+ }++ /**+ * A field's quantized vector values: doc operations delegate to the field's ordinal-to-doc+ * mapping while vector operations resolve through {@code fieldOrdToGroupOrd} into the shared+ * group view.+ */+ static final class FieldValues extends QuantizedByteVectorValues implements DedupVectorValues {+ private final DedupScalarQuantizedVectorsScorer vectorsScorer;+ private final VectorSimilarityFunction function;+ private final FloatVectorValues fieldView;+ private final QuantizedByteVectorValues groupView;+ private final FieldOrdToGroupOrd fieldOrdToGroupOrd;+ private int[] scratch;++ FieldValues(+ DedupScalarQuantizedVectorsScorer vectorsScorer,+ VectorSimilarityFunction function,+ FloatVectorValues fieldView,+ QuantizedByteVectorValues groupView,+ FieldOrdToGroupOrd fieldOrdToGroupOrd) {+ this.vectorsScorer = vectorsScorer;+ this.function = function;+ this.fieldView = fieldView;+ this.groupView = groupView;+ this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;+ this.scratch = new int[DedupUtil.SCRATCH_INITIAL_SIZE];+ }++ @Override+ public QuantizedByteVectorValues getGroupView() {+ return groupView;+ }++ @Override+ public FieldOrdToGroupOrd getFieldOrdToGroupOrd() {+ return fieldOrdToGroupOrd;+ }++ /** The quantization flavor of this field's records, derived from its similarity function. */+ DedupQuantizer.Flavor flavor() {+ return DedupQuantizer.Flavor.of(function);+ }++ @Override+ public int dimension() {+ return groupView.dimension();+ }++ @Override+ public int size() {+ return fieldView.size();+ }++ @Override+ public int ordToDoc(int ord) {+ return fieldView.ordToDoc(ord);+ }++ @Override+ public Bits getAcceptOrds(Bits acceptDocs) {+ return fieldView.getAcceptOrds(acceptDocs);+ }++ @Override+ public DocIndexIterator iterator() {+ return fieldView.iterator();+ }++ @Override+ public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {+ if (scratch.length < ordsToPrefetch.length) { // grow if needed+ scratch = ArrayUtil.grow(scratch, ordsToPrefetch.length);+ }+ for (int i = 0; i < numOrds; i++) {+ scratch[i] = fieldOrdToGroupOrd.get(ordsToPrefetch[i]);+ }+ groupView.prefetch(scratch, numOrds);+ }++ @Override+ public byte[] vectorValue(int ord) throws IOException {+ return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));+ }++ @Override+ public OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int ord)+ throws IOException {+ return groupView.getCorrectiveTerms(fieldOrdToGroupOrd.get(ord));+ }++ @Override+ public OptimizedScalarQuantizer getQuantizer() {+ return groupView.getQuantizer();+ }++ @Override+ public ScalarEncoding getScalarEncoding() {+ return groupView.getScalarEncoding();+ }++ @Override+ public float[] getCentroid() throws IOException {+ return groupView.getCentroid();+ }++ @Override+ public float getCentroidDP() {+ return 0;+ }++ @Override+ public int getVectorByteLength() {+ return groupView.getVectorByteLength();+ }++ @Override+ public FieldValues copy() throws IOException {+ return new FieldValues(+ vectorsScorer, function, fieldView.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());+ }++ @Override+ public VectorScorer scorer(float[] target) throws IOException {+ if (size() == 0) {+ return null;+ }+ FieldValues copy = copy();+ DocIndexIterator indexIterator = copy.iterator();+ RandomVectorScorer vectorScorer = vectorsScorer.getRandomVectorScorer(function, copy, target);+ boolean isDense = copy.fieldView instanceof OffHeapFloatVectorValues.DenseOffHeapVectorValues;+ return new DedupVectorScorer(indexIterator, vectorScorer, isDense);+ }+ }++ /**+ * Full-precision view of a field backed by the raw de-duplicated vectors, whose {@link+ * #scorer(float[])} scores against the quantized values instead. Mirrors {@code+ * Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedVectorValues}.+ */+ static final class RawAndQuantizedValues extends FloatVectorValues implements DedupVectorValues {+ private final DedupVectorValues.FloatImpl rawValues;+ private final FieldValues quantizedValues;++ RawAndQuantizedValues(DedupVectorValues.FloatImpl rawValues, FieldValues quantizedValues) {+ this.rawValues = rawValues;+ this.quantizedValues = quantizedValues;+ }++ FieldValues getQuantizedValues() {+ return quantizedValues;+ }++ @Override+ public FloatVectorValues getGroupView() {+ return rawValues.getGroupView();+ }++ @Override+ public FieldOrdToGroupOrd getFieldOrdToGroupOrd() {+ return rawValues.getFieldOrdToGroupOrd();+ }++ @Override+ public int dimension() {+ return rawValues.dimension();+ }++ @Override+ public int size() {+ return rawValues.size();+ }++ @Override+ public int ordToDoc(int ord) {+ return rawValues.ordToDoc(ord);+ }++ @Override+ public Bits getAcceptOrds(Bits acceptDocs) {+ return rawValues.getAcceptOrds(acceptDocs);+ }++ @Override+ public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {+ rawValues.prefetch(ordsToPrefetch, numOrds);+ }++ @Override+ public float[] vectorValue(int ord) throws IOException {+ return rawValues.vectorValue(ord);+ }++ @Override+ public DocIndexIterator iterator() {+ return rawValues.iterator();+ }++ @Override+ public RawAndQuantizedValues copy() throws IOException {+ return new RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());+ }++ @Override+ public VectorScorer scorer(float[] target) throws IOException {+ return quantizedValues.scorer(target);+ }++ @Override+ public VectorScorer rescorer(float[] target) throws IOException {+ return rawValues.rescorer(target);+ }+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.javanew file mode 100644index 00000000000..5ecfb0cac55--- /dev/null+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java@@ -0,0 +1,136 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import java.io.IOException;+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;+import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;++/**+ * A scalar quantized version of {@link DedupFlatVectorsFormat} that stores each distinct vector+ * once, in both raw and quantized form.+ *+ * <p>Quantization is <i>data-blind</i>: input vectors are assumed to be evenly distributed, i.e.+ * centered on a zero vector (see {@link DedupQuantizer}). The quantized record of a vector is then+ * a pure function of the raw vector and the {@link DedupQuantizer.Flavor} derived from the field's+ * similarity function — independent of the other vectors in the segment — so quantized records+ * de-duplicate like raw vectors: identical vectors across documents and fields of a group whose+ * similarity functions map to the same flavor share one quantized record, resolved through the same+ * {@code fieldOrdToGroupOrd} translation map. Records follow the {@link+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat} conventions per flavor,+ * and are scored by the stock {@link+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer}. Note the accuracy+ * tradeoff relative to that format, which centers vectors on a per-field centroid before+ * quantizing.+ *+ * <p>Only {@link org.apache.lucene.index.VectorEncoding#FLOAT32} vectors are quantized; BYTE and+ * FLOAT16 vectors are stored raw only, identical to {@link DedupFlatVectorsFormat}.+ *+ * <h2>.vdd (vector de-dup data) file</h2>+ *+ * <p>The raw vectors and per-field data, laid out exactly as the {@link DedupFlatVectorsFormat}+ * {@code .vdd} file.+ *+ * <h2>.vdqd (vector de-dup quantized data) file</h2>+ *+ * <p>One block per FLOAT32 group and {@link DedupQuantizer.Flavor} in use (aligned to {@code+ * Float.BYTES}), holding one quantized record per distinct vector:+ *+ * <ul>+ * <li><b>[byte]</b> the packed quantized values (layout depends on the {@link ScalarEncoding})+ * <li><b>[float]</b> the lower optimized interval+ * <li><b>[float]</b> the upper optimized interval+ * <li><b>[float]</b> the additional correction (the squared norm for the EUCLIDEAN flavor, zero+ * otherwise)+ * <li><b>[int]</b> the sum of the quantized components+ * </ul>+ *+ * <h2>.vdqm (vector de-dup quantized metadata) file</h2>+ *+ * <p>The layout of the {@link DedupFlatVectorsFormat} {@code .vdm} file, with each group+ * additionally recording its flavor blocks:+ *+ * <ul>+ * <li><b>[vint]</b> the number of flavor blocks ({@code 0} for non-FLOAT32 groups)+ * <li>per flavor block: <b>[vint]</b> the flavor wire number, <b>[vint]</b> the {@link+ * ScalarEncoding} wire number of its records, <b>[int64]</b> offset and <b>[int64]</b> length+ * of its quantized records in the .vdqd file+ * </ul>+ *+ * @lucene.experimental+ */+final class DedupScalarQuantizedVectorsFormat extends FlatVectorsFormat {+ static final String NAME = "DedupScalarQuantizedVectorsFormat";++ private static final String META_CODEC_NAME = "DedupScalarQuantizedVectorsFormatMeta";+ private static final String META_EXTENSION = "vdqm";++ private static final String VECTOR_DATA_CODEC_NAME =+ "DedupScalarQuantizedVectorsFormatVectorData";+ private static final String VECTOR_DATA_EXTENSION = "vdd";++ private static final String QUANTIZED_VECTOR_DATA_CODEC_NAME =+ "DedupScalarQuantizedVectorsFormatQuantizedVectorData";+ private static final String QUANTIZED_VECTOR_DATA_EXTENSION = "vdqd";++ private static final int VERSION_START = 0;+ private static final int VERSION_CURRENT = VERSION_START;++ private static final DedupScalarQuantizedVectorsScorer FLAT_VECTORS_SCORER =+ new DedupScalarQuantizedVectorsScorer();++ private final ScalarEncoding encoding;++ DedupScalarQuantizedVectorsFormat(ScalarEncoding encoding) {+ super(NAME);+ this.encoding = encoding;+ }++ @Override+ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {+ return new DedupFlatVectorsWriter(+ state,+ FLAT_VECTORS_SCORER,+ new DedupQuantizer(encoding),+ META_CODEC_NAME,+ META_EXTENSION,+ VECTOR_DATA_CODEC_NAME,+ VECTOR_DATA_EXTENSION,+ QUANTIZED_VECTOR_DATA_CODEC_NAME,+ QUANTIZED_VECTOR_DATA_EXTENSION,+ VERSION_CURRENT);+ }++ @Override+ public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {+ return new DedupScalarQuantizedVectorsReader(+ state,+ FLAT_VECTORS_SCORER,+ META_CODEC_NAME,+ META_EXTENSION,+ VECTOR_DATA_CODEC_NAME,+ VECTOR_DATA_EXTENSION,+ QUANTIZED_VECTOR_DATA_CODEC_NAME,+ QUANTIZED_VECTOR_DATA_EXTENSION,+ VERSION_START,+ VERSION_CURRENT);+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.javanew file mode 100644index 00000000000..5551e11e15a--- /dev/null+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java@@ -0,0 +1,582 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import static org.apache.lucene.index.VectorEncoding.BYTE;+import static org.apache.lucene.index.VectorEncoding.FLOAT16;+import static org.apache.lucene.index.VectorEncoding.FLOAT32;+import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupBytes;+import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloat16s;+import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloats;++import java.io.IOException;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;+import org.apache.lucene.index.ByteVectorValues;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.Float16VectorValues;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.MergePolicy;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.index.VectorEncoding;+import org.apache.lucene.sandbox.codecs.dedup.DedupQuantizer.QuantizedBlock;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;+import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;+import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;+import org.apache.lucene.store.ChecksumIndexInput;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.RamUsageEstimator;+import org.apache.lucene.util.VectorUtil;+import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;+import org.apache.lucene.util.hnsw.RandomVectorScorer;+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;+import org.apache.lucene.util.quantization.QuantizedVectorsReader;+import org.apache.lucene.util.quantization.ScalarQuantizer;++/**+ * Reads de-duplicated flat vectors written by {@link DedupScalarQuantizedVectorsFormat}. Each+ * FLOAT32 field exposes a full-precision view backed by the raw de-duplicated vectors, scored+ * against a data-blind quantized view sharing the same {@code fieldOrdToGroupOrd} translation map.+ * BYTE and FLOAT16 fields are stored and read raw only.+ *+ * @lucene.experimental+ */+final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader+ implements QuantizedVectorsReader {+ private static final long SHALLOW_SIZE =+ RamUsageEstimator.shallowSizeOfInstance(DedupScalarQuantizedVectorsReader.class);++ private final DedupScalarQuantizedVectorsScorer vectorsScorer;+ private final Map<String, FieldEntry> fields;+ private final IndexInput vectorData;+ private final IndexInput quantizedVectorData;+ private final String vectorDataExtension;+ private final String quantizedVectorDataExtension;++ DedupScalarQuantizedVectorsReader(+ SegmentReadState state,+ DedupScalarQuantizedVectorsScorer vectorsScorer,+ String metaCodecName,+ String metaExtension,+ String vectorDataCodecName,+ String vectorDataExtension,+ String quantizedVectorDataCodecName,+ String quantizedVectorDataExtension,+ int versionStart,+ int versionCurrent)+ throws IOException {++ this.vectorsScorer = vectorsScorer;+ this.fields = new HashMap<>();+ this.vectorDataExtension = vectorDataExtension;+ this.quantizedVectorDataExtension = quantizedVectorDataExtension;++ String metaFileName =+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);++ int versionMeta;+ try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {+ Throwable priorE = null;+ try {+ versionMeta =+ CodecUtil.checkIndexHeader(+ meta,+ metaCodecName,+ versionStart,+ versionCurrent,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ readMetaBody(meta, state.fieldInfos);+ } catch (Throwable e) {+ priorE = e;+ throw e;+ } finally {+ CodecUtil.checkFooter(meta, priorE);+ }+ }++ this.vectorData =+ openDataInput(+ state,+ versionMeta,+ vectorDataCodecName,+ vectorDataExtension,+ versionStart,+ versionCurrent);++ this.quantizedVectorData =+ openDataInput(+ state,+ versionMeta,+ quantizedVectorDataCodecName,+ quantizedVectorDataExtension,+ versionStart,+ versionCurrent);+ }++ private void readMetaBody(ChecksumIndexInput meta, FieldInfos fieldInfos) throws IOException {+ List<QuantizedGroupInfo> groupInfos = new ArrayList<>();+ while (true) {+ GroupInfo groupInfo = GroupInfo.readFromMeta(meta);+ if (groupInfo == null) {+ break;+ }+ Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks = DedupQuantizer.readGroup(meta);+ validateQuantizedBlocks(meta, groupInfo, quantizedBlocks);+ groupInfos.add(new QuantizedGroupInfo(groupInfo, quantizedBlocks));+ }++ while (true) {+ ReadFieldInfo fieldInfo = ReadFieldInfo.read(meta);+ if (fieldInfo == null) {+ break;+ }++ FieldInfo info = fieldInfos.fieldInfo(fieldInfo.fieldNumber());+ if (info == null) {+ throw new CorruptIndexException("Invalid field number: " + fieldInfo.fieldNumber(), meta);+ } else if (fieldInfo.function() != info.getVectorSimilarityFunction()) {+ throw new CorruptIndexException(+ "Inconsistent vector function: indexed="+ + fieldInfo.function()+ + ", actual="+ + info.getVectorSimilarityFunction(),+ meta);+ } else if (fieldInfo.dimension() != info.getVectorDimension()) {+ throw new CorruptIndexException(+ "Inconsistent vector dimension: indexed="+ + fieldInfo.dimension()+ + ", actual="+ + info.getVectorDimension(),+ meta);+ } else if (fieldInfo.encoding() != info.getVectorEncoding()) {+ throw new CorruptIndexException(+ "Inconsistent vector encoding: indexed="+ + fieldInfo.encoding()+ + ", actual="+ + info.getVectorEncoding(),+ meta);+ }++ if (fieldInfo.groupOrd() < 0 || fieldInfo.groupOrd() >= groupInfos.size()) {+ throw new CorruptIndexException(+ "Invalid groupId=" + fieldInfo.groupOrd() + ", numGroups=" + groupInfos.size(), meta);+ }++ QuantizedGroupInfo quantizedGroupInfo = groupInfos.get(fieldInfo.groupOrd());+ GroupInfo groupInfo = quantizedGroupInfo.groupInfo();+ QuantizedBlock quantizedBlock = null;+ if (fieldInfo.encoding() == FLOAT32) {+ DedupQuantizer.Flavor flavor = DedupQuantizer.Flavor.of(fieldInfo.function());+ quantizedBlock = quantizedGroupInfo.quantizedBlocks().get(flavor);+ if (quantizedBlock == null) {+ throw new CorruptIndexException(+ "Missing quantized block for flavor=" + flavor + ", field=" + info.name, meta);+ }+ }+ if (fieldInfo.dimension() != groupInfo.dimension()) {+ throw new CorruptIndexException(+ "Vector dimension mismatch: field="+ + fieldInfo.dimension()+ + ", group="+ + groupInfo.dimension(),+ meta);+ } else if (fieldInfo.encoding() != groupInfo.encoding()) {+ throw new CorruptIndexException(+ "Vector encoding mismatch: field="+ + fieldInfo.encoding()+ + ", group="+ + groupInfo.encoding(),+ meta);+ }++ fields.put(info.name, new FieldEntry(fieldInfo, groupInfo, quantizedBlock));+ }+ }++ private static void validateQuantizedBlocks(+ ChecksumIndexInput meta,+ GroupInfo groupInfo,+ Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks)+ throws IOException {+ if (groupInfo.encoding() != FLOAT32) {+ if (quantizedBlocks.isEmpty() == false) {+ throw new CorruptIndexException(+ "Unexpected quantized data for encoding=" + groupInfo.encoding(), meta);+ }+ return;+ }+ for (Map.Entry<DedupQuantizer.Flavor, QuantizedBlock> entry : quantizedBlocks.entrySet()) {+ long expectedSize =+ (long) groupInfo.groupNumVectors()+ * (entry.getValue().encoding().getDocPackedLength(groupInfo.dimension())+ + DedupScalarQuantizedVectorValues.CORRECTIVE_BYTES);+ if (entry.getValue().quantizedDataSize() != expectedSize) {+ throw new CorruptIndexException(+ "Quantized data size mismatch: flavor="+ + entry.getKey()+ + ", expected="+ + expectedSize+ + ", actual="+ + entry.getValue().quantizedDataSize()+ + ", numVectors="+ + groupInfo.groupNumVectors()+ + ", dimension="+ + groupInfo.dimension(),+ meta);+ }+ }+ }++ private static IndexInput openDataInput(+ SegmentReadState state,+ int versionMeta,+ String vectorDataCodecName,+ String vectorDataExtension,+ int versionStart,+ int versionCurrent)+ throws IOException {++ String fileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);++ IOContext.FileOpenHint[] hints = {+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM+ };+ IOContext context = state.context.withHints(hints);++ IndexInput in = null;+ boolean success = false;+ try {+ in = state.directory.openInput(fileName, context);+ int versionVectorData =+ CodecUtil.checkIndexHeader(+ in,+ vectorDataCodecName,+ versionStart,+ versionCurrent,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ if (versionMeta != versionVectorData) {+ throw new CorruptIndexException(+ "Format versions mismatch: meta="+ + versionMeta+ + ", "+ + vectorDataCodecName+ + "="+ + versionVectorData,+ in);+ }+ CodecUtil.retrieveChecksum(in);+ success = true;+ return in;+ } finally {+ if (success == false) {+ IOUtils.closeWhileHandlingException(in);+ }+ }+ }++ @Override+ public FlatVectorsScorer getFlatVectorScorer(String field) {+ return vectorsScorer;+ }++ // package-private for testing+ FieldEntry getEntry(String field, VectorEncoding expected) {+ FieldEntry entry = fields.get(field);+ if (entry == null) {+ throw new IllegalArgumentException("field=" + field + " not found");+ } else if (entry.fieldInfo().encoding() != expected) {+ throw new IllegalArgumentException("field=" + field + " not indexed as " + expected);+ }+ return entry;+ }++ @Override+ public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException {+ FieldEntry entry = getEntry(field, FLOAT32);+ FieldValues quantizedValues = getQuantizedVectorValues(entry);+ return vectorsScorer.getRandomVectorScorer(+ entry.fieldInfo().function(), quantizedValues, target);+ }++ @Override+ public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException {+ FieldEntry entry = getEntry(field, BYTE);+ ByteVectorValues vectorValues = getByteVectorValues(entry);+ return vectorsScorer.getRandomVectorScorer(entry.fieldInfo().function(), vectorValues, target);+ }++ @Override+ public RandomVectorScorer getRandomVectorScorer(String field, short[] target) throws IOException {+ FieldEntry entry = getEntry(field, FLOAT16);+ Float16VectorValues vectorValues = getFloat16VectorValues(entry);+ return vectorsScorer.getRandomVectorScorer(entry.fieldInfo().function(), vectorValues, target);+ }++ @Override+ public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {+ CodecUtil.checksumEntireFile(vectorData, merge);+ CodecUtil.checksumEntireFile(quantizedVectorData, merge);+ }++ private DedupVectorValues.FloatImpl getRawFloatVectorValues(FieldEntry entry) throws IOException {+ return loadDedupFloats(+ vectorsScorer,+ entry.fieldInfo().function(),+ entry.fieldInfo().ordToDoc(),+ entry.fieldInfo().dimension(),+ entry.groupInfo().groupNumVectors(),+ vectorData,+ entry.groupInfo().vectorDataOffset(),+ entry.groupInfo().vectorDataSize(),+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),+ entry.fieldInfo().fieldOrdToGroupOrdSize());+ }++ private FieldValues getQuantizedVectorValues(FieldEntry entry) throws IOException {+ return DedupScalarQuantizedVectorValues.loadQuantized(+ vectorsScorer,+ entry.fieldInfo().function(),+ entry.quantizedBlock().encoding(),+ entry.fieldInfo().ordToDoc(),+ entry.fieldInfo().dimension(),+ entry.groupInfo().groupNumVectors(),+ vectorData,+ quantizedVectorData,+ entry.quantizedBlock().quantizedDataOffset(),+ entry.quantizedBlock().quantizedDataSize(),+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),+ entry.fieldInfo().fieldOrdToGroupOrdSize());+ }++ @Override+ public FloatVectorValues getFloatVectorValues(String field) throws IOException {+ FieldEntry entry = getEntry(field, FLOAT32);+ return new RawAndQuantizedValues(+ getRawFloatVectorValues(entry), getQuantizedVectorValues(entry));+ }++ private ByteVectorValues getByteVectorValues(FieldEntry entry) throws IOException {+ return loadDedupBytes(+ vectorsScorer,+ entry.fieldInfo().function(),+ entry.fieldInfo().ordToDoc(),+ entry.fieldInfo().dimension(),+ entry.groupInfo().groupNumVectors(),+ vectorData,+ entry.groupInfo().vectorDataOffset(),+ entry.groupInfo().vectorDataSize(),+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),+ entry.fieldInfo().fieldOrdToGroupOrdSize());+ }++ @Override+ public ByteVectorValues getByteVectorValues(String field) throws IOException {+ return getByteVectorValues(getEntry(field, BYTE));+ }++ private Float16VectorValues getFloat16VectorValues(FieldEntry entry) throws IOException {+ return loadDedupFloat16s(+ vectorsScorer,+ entry.fieldInfo().function(),+ entry.fieldInfo().ordToDoc(),+ entry.fieldInfo().dimension(),+ entry.groupInfo().groupNumVectors(),+ vectorData,+ entry.groupInfo().vectorDataOffset(),+ entry.groupInfo().vectorDataSize(),+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),+ entry.fieldInfo().fieldOrdToGroupOrdSize());+ }++ @Override+ public Float16VectorValues getFloat16VectorValues(String field) throws IOException {+ return getFloat16VectorValues(getEntry(field, FLOAT16));+ }++ @Override+ public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {+ FieldEntry entry = getEntry(field, FLOAT32);+ return getQuantizedVectorValues(entry);+ }++ @Override+ public ScalarQuantizer getQuantizationState(String fieldName) {+ return null;+ }++ @Override+ public CloseableRandomVectorScorerSupplier getRandomVectorScorerSupplierForMerge(+ FieldInfo fieldInfo, SegmentWriteState segmentWriteState) throws IOException {+ FieldEntry entry = fields.get(fieldInfo.name);+ if (entry == null || entry.fieldInfo().encoding() != FLOAT32) {+ // BYTE and FLOAT16 fields are stored raw only+ return null;+ }+ FieldValues quantizedValues = getQuantizedVectorValues(entry);+ ScalarEncoding encoding = entry.quantizedBlock().encoding();+ if (encoding.isAsymmetric() == false) {+ RandomVectorScorerSupplier supplier =+ vectorsScorer.getRandomVectorScorerSupplier(+ fieldInfo.getVectorSimilarityFunction(), quantizedValues);+ return CloseableRandomVectorScorerSupplier.create(supplier, quantizedValues.size(), () -> {});+ }++ // Asymmetric encodings compare query-encoded vectors against the stored doc-encoded vectors.+ // Write a query-encoded record per distinct vector of the group into a temporary file; both+ // sides then resolve through the shared fieldOrdToGroupOrd translation.+ DedupVectorValues.FloatImpl rawValues = getRawFloatVectorValues(entry);+ FloatVectorValues groupView = rawValues.getGroupView();+ int dimension = entry.fieldInfo().dimension();+ int groupNumVectors = entry.groupInfo().groupNumVectors();+ DedupQuantizer.Flavor flavor =+ DedupQuantizer.Flavor.of(fieldInfo.getVectorSimilarityFunction());++ String tempQueryVectorsName = null;+ try (IndexOutput tempQueryVectors =+ segmentWriteState.directory.createTempOutput(+ segmentWriteState.segmentInfo.name, "dedup_queries", segmentWriteState.context)) {+ tempQueryVectorsName = tempQueryVectors.getName();++ OptimizedScalarQuantizer quantizer = flavor.quantizer();+ float[] zeroCentroid = new float[dimension];+ float[] vector = new float[dimension];+ byte[] scratch = new byte[encoding.getDiscreteDimensions(dimension)];+ byte[] toQuery = new byte[encoding.getQueryPackedLength(dimension)];+ for (int groupOrd = 0; groupOrd < groupNumVectors; groupOrd++) {+ // copy: normalization and quantization mutate the input, which is a shared buffer+ System.arraycopy(groupView.vectorValue(groupOrd), 0, vector, 0, dimension);+ if (flavor.normalized()) {+ VectorUtil.l2normalize(vector);+ }+ OptimizedScalarQuantizer.QuantizationResult corrections =+ quantizer.scalarQuantize(vector, scratch, encoding.getQueryBits(), zeroCentroid);+ OptimizedScalarQuantizer.transposeHalfByte(scratch, toQuery);+ tempQueryVectors.writeBytes(toQuery, toQuery.length);+ tempQueryVectors.writeInt(Float.floatToIntBits(corrections.lowerInterval()));+ tempQueryVectors.writeInt(Float.floatToIntBits(corrections.upperInterval()));+ tempQueryVectors.writeInt(Float.floatToIntBits(corrections.additionalCorrection()));+ tempQueryVectors.writeInt(corrections.quantizedComponentSum());+ }+ CodecUtil.writeFooter(tempQueryVectors);+ } catch (Throwable t) {+ if (tempQueryVectorsName != null) {+ IOUtils.deleteFilesSuppressingExceptions(+ t, segmentWriteState.directory, tempQueryVectorsName);+ }+ throw t;+ }++ IndexInput queryVectorsInput =+ segmentWriteState.directory.openInput(tempQueryVectorsName, segmentWriteState.context);+ final String finalTempQueryVectorsName = tempQueryVectorsName;+ try {+ QuantizedByteVectorValues queryValues =+ DedupScalarQuantizedVectorValues.groupValues(+ true,+ vectorsScorer,+ fieldInfo.getVectorSimilarityFunction(),+ flavor,+ encoding,+ dimension,+ groupNumVectors,+ queryVectorsInput);+ RandomVectorScorerSupplier supplier =+ DedupScalarQuantizedVectorsScorer.asymmetricMergeSupplier(+ fieldInfo.getVectorSimilarityFunction(), queryValues, quantizedValues);+ return CloseableRandomVectorScorerSupplier.create(+ supplier,+ quantizedValues.size(),+ () -> {+ IOUtils.close(queryVectorsInput);+ IOUtils.deleteFilesIgnoringExceptions(+ segmentWriteState.directory, finalTempQueryVectorsName);+ });+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, queryVectorsInput);+ IOUtils.deleteFilesSuppressingExceptions(+ t, segmentWriteState.directory, finalTempQueryVectorsName);+ throw t;+ }+ }++ @Override+ public void close() throws IOException {+ IOUtils.close(vectorData, quantizedVectorData);+ }++ @Override+ public long ramBytesUsed() {+ return SHALLOW_SIZE + fields.size() * FieldEntry.SHALLOW_SIZE;+ }++ @Override+ public Map<String, Long> getOffHeapByteSize(FieldInfo fieldInfo) {+ FieldEntry entry = fields.get(fieldInfo.name);+ if (entry == null) {+ return Map.of();+ }++ // TODO: This is an over-estimation.+ long vectorDataSize =+ entry.fieldInfo().fieldOrdToGroupOrdSize() + entry.groupInfo().vectorDataSize();++ if (entry.quantizedBlock() == null) {+ return Map.of(vectorDataExtension, vectorDataSize);+ }++ return Map.of(+ vectorDataExtension,+ vectorDataSize,+ quantizedVectorDataExtension,+ entry.quantizedBlock().quantizedDataSize());+ }++ // package-private for testing+ record FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo, QuantizedBlock quantizedBlock) {+ private static final long SHALLOW_SIZE =+ RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class)+ + RamUsageEstimator.shallowSizeOfInstance(ReadFieldInfo.class)+ + RamUsageEstimator.shallowSizeOfInstance(GroupInfo.class)+ + RamUsageEstimator.shallowSizeOfInstance(QuantizedBlock.class);+ }++ private record QuantizedGroupInfo(+ GroupInfo groupInfo, Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks) {}+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.javanew file mode 100644index 00000000000..b4aa648a873--- /dev/null+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java@@ -0,0 +1,69 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer;+import org.apache.lucene.index.KnnVectorValues;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;++/**+ * Scorer for scalar quantized, de-duplicated vectors: a {@link DedupFlatVectorsScorer} delegating+ * vector operations to the {@link Lucene104ScalarQuantizedVectorScorer}, whose record and scoring+ * conventions the quantized group views follow exactly (see {@link DedupQuantizer.Flavor}).+ *+ * @lucene.experimental+ */+final class DedupScalarQuantizedVectorsScorer extends DedupFlatVectorsScorer {++ private static final Lucene104ScalarQuantizedVectorScorer QUANTIZED_SCORER =+ new Lucene104ScalarQuantizedVectorScorer(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());++ DedupScalarQuantizedVectorsScorer() {+ super(QUANTIZED_SCORER);+ }++ /** Full-precision views resolve to their quantized values for scoring. */+ @Override+ protected KnnVectorValues unwrap(KnnVectorValues vectorValues) {+ if (vectorValues instanceof RawAndQuantizedValues rawAndQuantized) {+ return rawAndQuantized.getQuantizedValues();+ }+ return vectorValues;+ }++ /**+ * Supplier for merge-time graph construction with asymmetric encodings: query-encoded distinct+ * vectors (one per <b>group</b> ordinal, in a temporary file) are compared against the stored+ * doc-encoded vectors, both sides resolving through {@code fieldOrdToGroupOrd} (see {@link+ * DedupScalarQuantizedVectorsReader#getRandomVectorScorerSupplierForMerge}).+ */+ static RandomVectorScorerSupplier asymmetricMergeSupplier(+ VectorSimilarityFunction similarityFunction,+ QuantizedByteVectorValues queryValues,+ FieldValues targetValues) {+ RandomVectorScorerSupplier groupView =+ QUANTIZED_SCORER.getRandomVectorScorerSupplier(+ similarityFunction, queryValues, targetValues.getGroupView());+ return new RandomVectorScorerSupplierImpl(+ targetValues, groupView, targetValues.getFieldOrdToGroupOrd());+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.javaindex 802f15d7240..b2102fa2e5a 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java@@ -54,7 +54,13 @@ import org.apache.lucene.util.packed.DirectReader;* (one entry per distinct vector). {@code fieldOrdToGroupOrd} translates a document ordinal in the* field into its group ordinal.*/-sealed interface DedupVectorValues {+sealed interface DedupVectorValues+ permits DedupVectorValues.ByteImpl,+ DedupVectorValues.FloatImpl,+ DedupVectorValues.Float16Impl,+ DedupScalarQuantizedVectorValues.FieldValues,+ DedupScalarQuantizedVectorValues.RawAndQuantizedValues {+/** The dense view over distinct vectors, indexed by group ordinal. */KnnVectorValues getGroupView();@@ -71,7 +77,7 @@ sealed interface DedupVectorValues {FieldOrdToGroupOrd copy() throws IOException;}- static ByteVectorValues loadDedupBytes(+ static ByteImpl loadDedupBytes(FlatVectorsScorer vectorsScorer,VectorSimilarityFunction function,OrdToDocDISIReaderConfiguration configuration,@@ -191,7 +197,7 @@ sealed interface DedupVectorValues {}}- static FloatVectorValues loadDedupFloats(+ static FloatImpl loadDedupFloats(FlatVectorsScorer vectorsScorer,VectorSimilarityFunction function,OrdToDocDISIReaderConfiguration configuration,@@ -311,7 +317,7 @@ sealed interface DedupVectorValues {}}- static Float16VectorValues loadDedupFloat16s(+ static Float16Impl loadDedupFloat16s(FlatVectorsScorer vectorsScorer,VectorSimilarityFunction function,OrdToDocDISIReaderConfiguration configuration,@@ -469,7 +475,7 @@ sealed interface DedupVectorValues {@Overridepublic FieldOrdToGroupOrd copy() {- return new FieldOrdToGroupOrdArrayList(fieldOrdToGroupOrd);+ throw new UnsupportedOperationException("not meant for copying");}}@@ -484,7 +490,7 @@ sealed interface DedupVectorValues {@Overridepublic FieldOrdToGroupOrd copy() {- return new FieldOrdToGroupOrdMappedArrayList(map, fieldOrdToGroupOrd);+ throw new UnsupportedOperationException("not meant for copying");}}@@ -513,7 +519,7 @@ sealed interface DedupVectorValues {@Overridepublic FieldOrdToGroupOrd copy() throws IOException {return new FieldOrdToGroupOrdOffHeap(- vectorData, fieldOrdToGroupOrdOffset, fieldOrdToGroupOrdSize);+ vectorData.clone(), fieldOrdToGroupOrdOffset, fieldOrdToGroupOrdSize);}}}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.javaindex 31892094183..0752b9e2d13 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java@@ -16,11 +16,20 @@*//**- * De-duplicating HNSW vector format.+ * De-duplicating HNSW vector formats.** <p>Stores each distinct vector once and shares it across the documents and fields that reference- * it, while reusing the Lucene 9.9 HNSW graph. See {@link- * org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat} for the entry point and {@link- * org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat} for the on-disk layout.+ * it, while reusing the Lucene 9.9 HNSW graph. Two formats are provided:+ *+ * <ul>+ * <li>{@link org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat} stores and scores raw+ * vectors; see {@link org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat} for the+ * on-disk layout.+ * <li>{@link org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormat}+ * additionally stores a scalar quantized copy of each distinct FLOAT32 vector, used for+ * scoring; see {@link+ * org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorsFormat} for the on-disk+ * layout.+ * </ul>*/package org.apache.lucene.sandbox.codecs.dedup;diff --git a/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormatindex 4e3e3fc1fca..4cacd83f5fa 100644--- a/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat+++ b/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat@@ -15,3 +15,4 @@org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormatorg.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat+org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormatdiff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.javaindex bf8b1d4e9fc..24f484c65bd 100644--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.java+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.java@@ -44,7 +44,11 @@ import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.internal.hppc.IntArrayList;import org.apache.lucene.internal.hppc.LongArrayList;+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdArrayList;+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdMappedArrayList;import org.apache.lucene.search.Query;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.util.LuceneTestCase;@@ -418,6 +422,17 @@ public class TestDedupFlatVectorsFormat extends LuceneTestCase {}}+ /** Tests specific to on-heap versions of the field ord -> group ord mapping. */+ public void testOnHeapFieldOrdToGroupOrd() {+ FieldOrdToGroupOrd arrayBacked = new FieldOrdToGroupOrdArrayList(new IntArrayList());+ expectThrows(UnsupportedOperationException.class, arrayBacked::copy); // not meant for copying.++ FieldOrdToGroupOrd mappedArrayBacked =+ new FieldOrdToGroupOrdMappedArrayList(new int[0], new IntArrayList());+ expectThrows(+ UnsupportedOperationException.class, mappedArrayBacked::copy); // not meant for copying.+ }+/** Number of distinct vectors physically stored for a field's group. */private static int groupNumVectors(KnnVectorValues values) {assertThat(values, instanceOf(DedupVectorValues.class));diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.javanew file mode 100644index 00000000000..113a95b66a4--- /dev/null+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java@@ -0,0 +1,112 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;++import java.io.IOException;+import org.apache.lucene.codecs.Codec;+import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;+import org.apache.lucene.index.CodecReader;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.VectorEncoding;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.SameThreadExecutorService;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;+import org.junit.Before;++/**+ * Runs the standard KNN vectors format suite against the de-duplicating scalar quantized HNSW+ * format, with a randomized quantization encoding. Quantization-specific de-duplication behavior is+ * covered by {@link TestDedupScalarQuantizedVectorsFormat}.+ */+public class TestDedupHnswScalarQuantizedVectorsFormat extends TestDedupHnswVectorsFormat {++ private KnnVectorsFormat format;++ @Before+ @Override+ public void setUp() throws Exception {+ ScalarEncoding[] encodingValues = ScalarEncoding.values();+ ScalarEncoding encoding = encodingValues[random().nextInt(encodingValues.length)];+ format =+ new DedupHnswScalarQuantizedVectorsFormat(encoding, DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH);+ super.setUp();+ }++ @Override+ protected Codec getCodec() {+ return TestUtil.alwaysKnnVectorsFormat(format);+ }++ public void testLimits() {+ expectThrows(+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(-1, 20));+ expectThrows(+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(0, 20));+ expectThrows(+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(20, 0));+ expectThrows(+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(20, -1));+ expectThrows(+ IllegalArgumentException.class,+ () -> new DedupHnswScalarQuantizedVectorsFormat(512 + 1, 20));+ expectThrows(+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(20, 3201));+ expectThrows(+ IllegalArgumentException.class,+ () ->+ new DedupHnswScalarQuantizedVectorsFormat(+ ScalarEncoding.UNSIGNED_BYTE, 20, 100, 1, new SameThreadExecutorService()));+ }++ @Override+ protected void assertOffHeapByteSize(LeafReader r, String fieldName) throws IOException {+ var fieldInfo = r.getFieldInfos().fieldInfo(fieldName);++ if (r instanceof CodecReader codecReader) {+ KnnVectorsReader knnVectorsReader = codecReader.getVectorReader();+ knnVectorsReader = knnVectorsReader.unwrapReaderForField(fieldName);+ var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo);+ long totalByteSize = offHeap.values().stream().mapToLong(Long::longValue).sum();+ if (knnVectorsReader instanceof Lucene99HnswVectorsReader) {+ if (getNumVectors(knnVectorsReader, fieldInfo) == 0) {+ assertEquals(0L, totalByteSize);+ } else {+ assertTrue(totalByteSize > 0);+ assertTrue(offHeap.get("vdd") > 0L); // NOTE: different from vec+ if (fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {+ assertTrue(offHeap.get("vdqd") > 0L); // NOTE: different from veq+ } else {+ assertNull(offHeap.get("vdqd")); // BYTE and FLOAT16 fields are stored raw only+ }++ if (hasHNSW(knnVectorsReader, fieldInfo)) {+ assertTrue(offHeap.get("vex") > 0L);+ } else {+ assertTrue(offHeap.get("vex") == null || offHeap.get("vex") == 0);+ }+ }+ }+ } else {+ throw new AssertionError("unexpected:" + r.getClass());+ }+ }+}diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.javanew file mode 100644index 00000000000..f9da057ae88--- /dev/null+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java@@ -0,0 +1,430 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.sandbox.codecs.dedup;++import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;+import static org.apache.lucene.index.VectorEncoding.BYTE;+import static org.apache.lucene.index.VectorEncoding.FLOAT32;+import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;+import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;+import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT;+import static org.hamcrest.Matchers.instanceOf;++import java.util.Arrays;+import java.util.List;+import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.KnnByteVectorField;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.index.ByteVectorValues;+import org.apache.lucene.index.CodecReader;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.KnnVectorValues;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.search.AcceptDocs;+import org.apache.lucene.search.ScoreDoc;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.VectorUtil;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;++/**+ * Tests that {@link DedupHnswScalarQuantizedVectorsFormat} stores each distinct vector once, in+ * both raw and quantized form. General de-duplication behavior is covered by {@link+ * TestDedupFlatVectorsFormat}; this test focuses on the quantized side.+ */+public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {++ private static final ScalarEncoding ENCODING = ScalarEncoding.UNSIGNED_BYTE;++ private static IndexWriterConfig config() {+ return newIndexWriterConfig()+ .setCodec(TestUtil.alwaysKnnVectorsFormat(new DedupHnswScalarQuantizedVectorsFormat()));+ }++ /** Size in bytes of one quantized record for the given dimension. */+ private static long quantizedRecordSize(int dimension) {+ return ENCODING.getDocPackedLength(dimension)+ + DedupScalarQuantizedVectorValues.CORRECTIVE_BYTES;+ }++ /** Repeated vectors are stored once, raw and quantized, but still read back per document. */+ public void testDuplicatesStoredOnce() throws Exception {+ float[] a = {1, 2, 3, 4};+ float[] b = {5, 6, 7, 8};+ float[][] docVectors = {a, b, a, b, a, b}; // 3 copies each of 2 vectors+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ for (int ord = 0; ord < docVectors.length; ord++) {+ Document doc = new Document();+ doc.add(new NumericDocValuesField("id", ord));+ doc.add(new KnnFloatVectorField("f", docVectors[ord], EUCLIDEAN));+ w.addDocument(doc);+ }+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ FloatVectorValues values = leafReader.getFloatVectorValues("f");+ assertEquals(docVectors.length, values.size()); // one entry per document+ assertEquals(2, groupNumVectors(values)); // only two distinct vectors stored++ // raw vectors read back exactly, resolved per document+ for (int ord = 0; ord < values.size(); ord++) {+ float[] vector = values.vectorValue(ord);+ assertTrue(Arrays.equals(a, vector) || Arrays.equals(b, vector));+ }++ // the quantized block also holds exactly two records+ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT32);+ assertEquals(2 * quantizedRecordSize(a.length), entry.quantizedBlock().quantizedDataSize());+ }+ }+ }++ /**+ * Fields with the same dimension and encoding share one raw copy of a vector; their quantized+ * records are stored per {@link DedupQuantizer.Flavor}, shared across fields whose similarity+ * functions map to the same flavor.+ */+ public void testDuplicatesAcrossFieldsShareQuantizedGroup() throws Exception {+ float[] shared = {9, 8, 7, 6};+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("f1", shared, EUCLIDEAN));+ doc.add(new KnnFloatVectorField("f2", shared, DOT_PRODUCT)); // different function+ doc.add(new KnnFloatVectorField("f3", shared, COSINE)); // different function+ doc.add(new KnnFloatVectorField("f4", shared, MAXIMUM_INNER_PRODUCT)); // same flavor as f2+ w.addDocument(doc);+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leaf = getOnlyLeafReader(reader);++ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leaf, "f1");+ assertEquals(sqReader, getQuantizedReader(leaf, "f2"));+ assertEquals(sqReader, getQuantizedReader(leaf, "f3"));+ assertEquals(sqReader, getQuantizedReader(leaf, "f4"));++ // all fields resolve to the same raw group, with one distinct vector+ DedupScalarQuantizedVectorsReader.FieldEntry e1 = sqReader.getEntry("f1", FLOAT32);+ DedupScalarQuantizedVectorsReader.FieldEntry e2 = sqReader.getEntry("f2", FLOAT32);+ DedupScalarQuantizedVectorsReader.FieldEntry e3 = sqReader.getEntry("f3", FLOAT32);+ DedupScalarQuantizedVectorsReader.FieldEntry e4 = sqReader.getEntry("f4", FLOAT32);+ assertEquals(e1.groupInfo(), e2.groupInfo());+ assertEquals(e1.groupInfo(), e3.groupInfo());+ assertEquals(e1.groupInfo(), e4.groupInfo());+ assertEquals(1, e1.groupInfo().groupNumVectors());++ // quantized data is split by flavor: EUCLIDEAN (f1), DOT_PRODUCT (f2 + f4, shared) and+ // NORMALIZED (f3), one record each+ assertNotEquals(e1.quantizedBlock(), e2.quantizedBlock());+ assertNotEquals(e1.quantizedBlock(), e3.quantizedBlock());+ assertNotEquals(e2.quantizedBlock(), e3.quantizedBlock());+ assertEquals(e2.quantizedBlock(), e4.quantizedBlock());+ for (DedupScalarQuantizedVectorsReader.FieldEntry entry : List.of(e1, e2, e3, e4)) {+ assertEquals(+ quantizedRecordSize(shared.length), entry.quantizedBlock().quantizedDataSize());+ }++ for (String field : new String[] {"f1", "f2", "f3", "f4"}) {+ FloatVectorValues values = leaf.getFloatVectorValues(field);+ assertEquals(1, groupNumVectors(values));+ assertArrayEquals(shared, values.vectorValue(0), 0f);+ }+ }+ }+ }++ /** Duplicates spanning multiple segments collapse to a single quantized copy when merged. */+ public void testDuplicatesAcrossSegmentsDedupOnMerge() throws Exception {+ float[] a = {1, 1, 1, 1};+ float[] b = {2, 2, 2, 2};+ float[][] docVectors = {a, b, a}; // 3 docs across 3 segments, 2 distinct+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ for (float[] docVector : docVectors) {+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("f", docVector, EUCLIDEAN));+ w.addDocument(doc);+ w.commit(); // one segment per document+ }+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ FloatVectorValues values = leafReader.getFloatVectorValues("f");+ assertEquals(docVectors.length, values.size());+ assertEquals(2, groupNumVectors(values)); // a's duplicate collapsed across segments++ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT32);+ assertEquals(2 * quantizedRecordSize(a.length), entry.quantizedBlock().quantizedDataSize());+ }+ }+ }++ /** Documents sharing a vector get identical scores, ranking above unrelated documents. */+ public void testSearchDuplicatesShareScores() throws Exception {+ int dimension = 16;+ float[] a = randomVector(dimension);+ float[] b = randomVector(dimension);+ for (int i = 0; i < dimension; i++) {+ b[i] = -a[i]; // far away from a+ }+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ int numA = 3, numB = 3;+ for (int i = 0; i < numA; i++) {+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("f", a, function));+ w.addDocument(doc);+ }+ for (int i = 0; i < numB; i++) {+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("f", b, function));+ w.addDocument(doc);+ }+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ TopDocs topDocs =+ leafReader.searchNearestVectors(+ "f",+ a,+ numA + numB,+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),+ Integer.MAX_VALUE);+ assertEquals(numA + numB, topDocs.scoreDocs.length);+ // duplicates of a rank first with identical scores+ float scoreA = topDocs.scoreDocs[0].score;+ for (int i = 0; i < numA; i++) {+ assertEquals("function=" + function, scoreA, topDocs.scoreDocs[i].score, 0f);+ }+ for (int i = numA; i < numA + numB; i++) {+ ScoreDoc scoreDoc = topDocs.scoreDocs[i];+ assertTrue("function=" + function, scoreDoc.score < scoreA);+ }+ }+ }+ }+ }++ /** Quantized scores are close to true scores for every similarity function. */+ public void testScoreAccuracy() throws Exception {+ int dimension = 16;+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {+ float[] docVector = randomVector(dimension);+ float[] query = randomVector(dimension);+ if (function == DOT_PRODUCT) { // requires normalized vectors+ VectorUtil.l2normalize(docVector);+ VectorUtil.l2normalize(query);+ }+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("f", docVector, function));+ w.addDocument(doc);+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ float trueScore = function.compare(docVector, query);+ TopDocs topDocs =+ leafReader.searchNearestVectors(+ "f",+ query,+ 1,+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),+ Integer.MAX_VALUE);+ assertEquals(1, topDocs.totalHits.value());+ // NOTE: data-blind quantization is less accurate than centroid-centered quantization,+ // so allow a small relative tolerance+ float tolerance = Math.max(0.05f, 0.05f * Math.abs(trueScore));+ assertEquals("function=" + function, trueScore, topDocs.scoreDocs[0].score, tolerance);+ }+ }+ }+ }++ /**+ * Cosine similarity ignores vector magnitude: since vectors are stored unnormalized (to share+ * quantized bytes across fields), the norm-based correction must equalize scaled duplicates.+ */+ public void testCosineIgnoresMagnitude() throws Exception {+ float[] small = {1, 2, 3, 4};+ float[] large = new float[small.length];+ for (int i = 0; i < small.length; i++) {+ large[i] = small[i] * 1000;+ }+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ for (float[] docVector : new float[][] {small, large}) {+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("f", docVector, COSINE));+ w.addDocument(doc);+ }+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ TopDocs topDocs =+ leafReader.searchNearestVectors(+ "f",+ small,+ 2,+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),+ Integer.MAX_VALUE);+ assertEquals(2, topDocs.scoreDocs.length);+ // both docs point in the same direction: cosine similarity ~1 -> score ~1+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {+ assertEquals(1f, scoreDoc.score, 0.01f);+ }+ }+ }+ }++ /** BYTE fields pass through to raw de-duplicated storage, without a quantized block. */+ public void testByteVectorsStoredRawOnly() throws Exception {+ byte[] a = {1, 2, 3, 4};+ byte[] b = {5, 6, 7, 8};+ byte[][] docVectors = {a, b, a, b}; // 2 copies each of 2 vectors+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ for (byte[] docVector : docVectors) {+ Document doc = new Document();+ doc.add(new KnnByteVectorField("f", docVector, EUCLIDEAN));+ w.addDocument(doc);+ }+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ ByteVectorValues values = leafReader.getByteVectorValues("f");+ assertEquals(docVectors.length, values.size());+ assertEquals(2, groupNumVectors(values)); // still de-duplicated++ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", BYTE);+ assertNull(entry.quantizedBlock()); // no quantized data+ }+ }+ }++ /**+ * Every encoding survives a merge with graph building forced (exercising the merge-time scorer+ * suppliers, including the temporary query-vectors file of asymmetric encodings) and searches+ * sanely afterwards.+ */+ public void testAllEncodingsMergeAndSearch() throws Exception {+ int dimension = 32;+ int numDistinct = 20;+ float[][] distinct = new float[numDistinct][];+ for (int i = 0; i < numDistinct; i++) {+ distinct[i] = randomVector(dimension);+ }+ for (ScalarEncoding encoding : ScalarEncoding.values()) {+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {+ IndexWriterConfig config =+ newIndexWriterConfig()+ .setCodec(+ TestUtil.alwaysKnnVectorsFormat(+ new DedupHnswScalarQuantizedVectorsFormat(+ encoding,+ DEFAULT_MAX_CONN,+ DEFAULT_BEAM_WIDTH,+ 1,+ null,+ 0))); // always build graphs+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config)) {+ for (int segment = 0; segment < 2; segment++) { // duplicates across segments+ for (int i = 0; i < numDistinct; i++) {+ Document doc = new Document();+ float[] vector =+ function == DOT_PRODUCT+ ? VectorUtil.l2normalize(distinct[i].clone())+ : distinct[i];+ doc.add(new KnnFloatVectorField("f", vector, function));+ w.addDocument(doc);+ }+ w.commit();+ }+ w.forceMerge(1); // exercises the merge-time graph construction scorer++ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ FloatVectorValues values = leafReader.getFloatVectorValues("f");+ assertEquals(2 * numDistinct, values.size());+ assertEquals(numDistinct, groupNumVectors(values)); // de-duplicated across segments++ float[] query =+ function == DOT_PRODUCT ? VectorUtil.l2normalize(distinct[0].clone()) : distinct[0];+ TopDocs topDocs =+ leafReader.searchNearestVectors(+ "f",+ query,+ 2,+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),+ Integer.MAX_VALUE);+ String context = "encoding=" + encoding + ", function=" + function;+ assertEquals(context, 2, topDocs.scoreDocs.length);+ // the two duplicates of the query vector share the (top) score+ assertEquals(context, topDocs.scoreDocs[0].score, topDocs.scoreDocs[1].score, 0f);+ }+ }+ }+ }+ }++ private float[] randomVector(int dimension) {+ float[] vector = new float[dimension];+ for (int i = 0; i < dimension; i++) {+ vector[i] = random().nextFloat(-1, 1);+ }+ return vector;+ }++ /** Number of distinct vectors physically stored for a field's group. */+ private static int groupNumVectors(KnnVectorValues values) {+ assertThat(values, instanceOf(DedupVectorValues.class));+ return ((DedupVectorValues) values).getGroupView().size();+ }++ /** Get underlying quantized dedup vector reader instance. */+ private static DedupScalarQuantizedVectorsReader getQuantizedReader(+ LeafReader leafReader, String fieldName) {+ assertThat(leafReader, instanceOf(CodecReader.class));+ KnnVectorsReader knnVectorsReader = ((CodecReader) leafReader).getVectorReader();+ knnVectorsReader = knnVectorsReader.unwrapReaderForField(fieldName);++ assertThat(knnVectorsReader, instanceOf(Lucene99HnswVectorsReader.class));+ FlatVectorsReader flatReader =+ ((Lucene99HnswVectorsReader) knnVectorsReader).getFlatVectorsReader();++ assertThat(flatReader, instanceOf(DedupScalarQuantizedVectorsReader.class));+ return (DedupScalarQuantizedVectorsReader) flatReader;+ }+}