Skip to content

Apache Lucene open-source search software

Add a sandboxed scalar-quantized version of the de-duplicating vector format (#16506)

Kaival Parikhfb0ca7cd29d6
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6134ad5759e..9aa30587797 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -385,9 +385,10 @@ New Features
requiring the ICU analysis plugin. Uses a generated switch for fold exceptions with
Character.toLowerCase() as the default fallback. (Robert Muir, Costin Leau)
-* GITHUB#15979: Add a sandboxed de-duplicating HNSW vector format (DedupHnswVectorsFormat) that stores
- each distinct full-precision vector once, shared across all documents and fields that reference it.
- (Kaival Parikh)
+* GITHUB#15979, GITHUB#16506: Add sandboxed de-duplicating HNSW vector formats (DedupHnswVectorsFormat,
+ DedupHnswScalarQuantizedVectorsFormat) that store each distinct full-precision and quantized vector
+ once, shared across all documents and fields that reference it. Quantization is data-blind (assumes
+ vectors are centered at zero) for effective de-duplication across fields. (Kaival Parikh)
* GITHUB#16484: Add SpanMultiTermQueryWrapper.FrequentTermsSpanBooleanQueryRewrite. It's a rewrite method that
translates each term into a SpanTermQuery within a {@link SpanOrQuery}, but retains only the most frequent terms so it
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
index f16ec3726ca..4ab62c49cf6 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
@@ -177,7 +177,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
return vectorValue.length;
}
- static void packNibbles(byte[] unpacked, byte[] packed) {
+ /** Packs two 4-bit values per output byte: the upper half holds {@code unpacked[i]}. */
+ public static void packNibbles(byte[] unpacked, byte[] packed) {
assert unpacked.length == packed.length * 2;
for (int i = 0; i < packed.length; i++) {
int x = unpacked[i] << 4 | unpacked[packed.length + i];
@@ -185,7 +186,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
}
}
- static void unpackNibbles(byte[] packed, byte[] unpacked) {
+ /** Unpacks two 4-bit values per input byte, the reverse of {@link #packNibbles}. */
+ public static void unpackNibbles(byte[] packed, byte[] unpacked) {
assert unpacked.length == packed.length * 2;
for (int i = 0; i < packed.length; i++) {
unpacked[i] = (byte) ((packed[i] >> 4) & 0x0F);
@@ -242,7 +244,7 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
}
/** Dense off-heap scalar quantized vector values */
- static class DenseOffHeapVectorValues extends OffHeapScalarQuantizedVectorValues {
+ public static class DenseOffHeapVectorValues extends OffHeapScalarQuantizedVectorValues {
DenseOffHeapVectorValues(
int dimension,
int size,
@@ -265,7 +267,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
slice);
}
- DenseOffHeapVectorValues(
+ /** Creates dense off-heap scalar quantized vector values over the given slice. */
+ public DenseOffHeapVectorValues(
boolean isQuerySide,
int dimension,
int size,
diff --git a/lucene/sandbox/src/java/module-info.java b/lucene/sandbox/src/java/module-info.java
index 8a3a60d7c54..3a45ccb74b1 100644
--- a/lucene/sandbox/src/java/module-info.java
+++ b/lucene/sandbox/src/java/module-info.java
@@ -43,5 +43,6 @@ module org.apache.lucene.sandbox {
org.apache.lucene.sandbox.codecs.idversion.IDVersionPostingsFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormat,
- org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat;
+ org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat,
+ org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormat;
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java
index 220578920b3..f2485b2f721 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsFormat.java
@@ -110,14 +110,14 @@ import org.apache.lucene.index.SegmentWriteState;
final class DedupFlatVectorsFormat extends FlatVectorsFormat {
static final String NAME = "DedupFlatVectorsFormat";
- static final String META_CODEC_NAME = "DedupFlatVectorsFormatMeta";
- static final String META_EXTENSION = "vdm";
+ private static final String META_CODEC_NAME = "DedupFlatVectorsFormatMeta";
+ private static final String META_EXTENSION = "vdm";
- static final String VECTOR_DATA_CODEC_NAME = "DedupFlatVectorsFormatVectorData";
- static final String VECTOR_DATA_EXTENSION = "vdd";
+ private static final String VECTOR_DATA_CODEC_NAME = "DedupFlatVectorsFormatVectorData";
+ private static final String VECTOR_DATA_EXTENSION = "vdd";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ private static final int VERSION_START = 0;
+ private static final int VERSION_CURRENT = VERSION_START;
private static final DedupFlatVectorsScorer FLAT_VECTORS_SCORER = new DedupFlatVectorsScorer();
@@ -127,11 +127,29 @@ final class DedupFlatVectorsFormat extends FlatVectorsFormat {
@Override
public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new DedupFlatVectorsWriter(state, FLAT_VECTORS_SCORER);
+ return new DedupFlatVectorsWriter(
+ state,
+ FLAT_VECTORS_SCORER,
+ null,
+ META_CODEC_NAME,
+ META_EXTENSION,
+ VECTOR_DATA_CODEC_NAME,
+ VECTOR_DATA_EXTENSION,
+ null,
+ null,
+ VERSION_CURRENT);
}
@Override
public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new DedupFlatVectorsReader(state, FLAT_VECTORS_SCORER);
+ return new DedupFlatVectorsReader(
+ state,
+ FLAT_VECTORS_SCORER,
+ META_CODEC_NAME,
+ META_EXTENSION,
+ VECTOR_DATA_CODEC_NAME,
+ VECTOR_DATA_EXTENSION,
+ VERSION_START,
+ VERSION_CURRENT);
}
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
index dfe5454cd0d..a54424abd09 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
@@ -19,12 +19,6 @@ package org.apache.lucene.sandbox.codecs.dedup;
import static org.apache.lucene.index.VectorEncoding.BYTE;
import static org.apache.lucene.index.VectorEncoding.FLOAT16;
import static org.apache.lucene.index.VectorEncoding.FLOAT32;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_CODEC_NAME;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_EXTENSION;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_CODEC_NAME;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_EXTENSION;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VERSION_CURRENT;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VERSION_START;
import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupBytes;
import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloat16s;
import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloats;
@@ -72,15 +66,25 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
private final FlatVectorsScorer vectorsScorer;
private final Map<String, FieldEntry> fields;
private final IndexInput vectorData;
-
- DedupFlatVectorsReader(SegmentReadState state, FlatVectorsScorer vectorsScorer)
+ private final String vectorDataExtension;
+
+ DedupFlatVectorsReader(
+ SegmentReadState state,
+ FlatVectorsScorer vectorsScorer,
+ String metaCodecName,
+ String metaExtension,
+ String vectorDataCodecName,
+ String vectorDataExtension,
+ int versionStart,
+ int versionCurrent)
throws IOException {
this.vectorsScorer = vectorsScorer;
this.fields = new HashMap<>();
+ this.vectorDataExtension = vectorDataExtension;
String metaFileName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, META_EXTENSION);
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
int versionMeta;
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
@@ -89,9 +93,9 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
versionMeta =
CodecUtil.checkIndexHeader(
meta,
- META_CODEC_NAME,
- VERSION_START,
- VERSION_CURRENT,
+ metaCodecName,
+ versionStart,
+ versionCurrent,
state.segmentInfo.getId(),
state.segmentSuffix);
readMetaBody(meta, state.fieldInfos);
@@ -103,7 +107,14 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
}
}
- this.vectorData = openDataInput(state, versionMeta);
+ this.vectorData =
+ openDataInput(
+ state,
+ versionMeta,
+ vectorDataCodecName,
+ vectorDataExtension,
+ versionStart,
+ versionCurrent);
}
private void readMetaBody(ChecksumIndexInput meta, FieldInfos fieldInfos) throws IOException {
@@ -174,12 +185,18 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
}
}
- private static IndexInput openDataInput(SegmentReadState state, int versionMeta)
+ private static IndexInput openDataInput(
+ SegmentReadState state,
+ int versionMeta,
+ String vectorDataCodecName,
+ String vectorDataExtension,
+ int versionStart,
+ int versionCurrent)
throws IOException {
String fileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, VECTOR_DATA_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);
IOContext.FileOpenHint[] hints = {
FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM
@@ -193,9 +210,9 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
int versionVectorData =
CodecUtil.checkIndexHeader(
in,
- VECTOR_DATA_CODEC_NAME,
- VERSION_START,
- VERSION_CURRENT,
+ vectorDataCodecName,
+ versionStart,
+ versionCurrent,
state.segmentInfo.getId(),
state.segmentSuffix);
if (versionMeta != versionVectorData) {
@@ -203,7 +220,7 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
"Format versions mismatch: meta="
+ versionMeta
+ ", "
- + VECTOR_DATA_CODEC_NAME
+ + vectorDataCodecName
+ "="
+ versionVectorData,
in);
@@ -348,10 +365,11 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
}
// TODO: This is an over-estimation.
return Map.of(
- VECTOR_DATA_EXTENSION,
+ vectorDataExtension,
entry.fieldInfo.fieldOrdToGroupOrdSize() + entry.groupInfo.vectorDataSize());
}
+ // package-private for testing
record FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo) {
private static final long SHALLOW_SIZE =
RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class)
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.java
index 63da8b2d950..1526c0ce29b 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsScorer.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
@@ -37,112 +36,123 @@ import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
*
* @lucene.experimental
*/
-final class DedupFlatVectorsScorer implements FlatVectorsScorer {
- private static final FlatVectorsScorer SCORER =
+sealed class DedupFlatVectorsScorer implements FlatVectorsScorer
+ permits DedupScalarQuantizedVectorsScorer {
+
+ private static final FlatVectorsScorer FLAT_SCORER =
FlatVectorScorerUtil.getLucene99FlatVectorsScorer();
+ private final FlatVectorsScorer delegate;
+
+ DedupFlatVectorsScorer() {
+ this(FLAT_SCORER);
+ }
+
+ protected DedupFlatVectorsScorer(FlatVectorsScorer delegate) {
+ this.delegate = delegate;
+ }
+
+ /** Resolves the values to score; subclasses may unwrap composite values. */
+ protected KnnVectorValues unwrap(KnnVectorValues vectorValues) {
+ return vectorValues;
+ }
+
@Override
public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)
throws IOException {
+ vectorValues = unwrap(vectorValues);
if (vectorValues instanceof DedupVectorValues dedupValues) {
- RandomVectorScorerSupplier fieldView =
- SCORER.getRandomVectorScorerSupplier(similarityFunction, vectorValues);
RandomVectorScorerSupplier groupView =
- SCORER.getRandomVectorScorerSupplier(similarityFunction, dedupValues.getGroupView());
+ delegate.getRandomVectorScorerSupplier(similarityFunction, dedupValues.getGroupView());
return new RandomVectorScorerSupplierImpl(
- fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());
+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());
}
- return SCORER.getRandomVectorScorerSupplier(similarityFunction, vectorValues);
+ return delegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues);
}
@Override
public RandomVectorScorer getRandomVectorScorer(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)
throws IOException {
+ vectorValues = unwrap(vectorValues);
if (vectorValues instanceof DedupVectorValues dedupValues) {
- RandomVectorScorer fieldView =
- SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);
RandomVectorScorer groupView =
- SCORER.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);
- return new RandomVectorScorerImpl(fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());
+ delegate.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);
+ return new RandomVectorScorerImpl(
+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());
}
- return SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);
+ return delegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
}
@Override
public RandomVectorScorer getRandomVectorScorer(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, byte[] target)
throws IOException {
+ vectorValues = unwrap(vectorValues);
if (vectorValues instanceof DedupVectorValues dedupValues) {
- RandomVectorScorer fieldView =
- SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);
RandomVectorScorer groupView =
- SCORER.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);
- return new RandomVectorScorerImpl(fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());
+ delegate.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);
+ return new RandomVectorScorerImpl(
+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());
}
- return SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);
+ return delegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
}
@Override
public RandomVectorScorer getRandomVectorScorer(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, short[] target)
throws IOException {
+ vectorValues = unwrap(vectorValues);
if (vectorValues instanceof DedupVectorValues dedupValues) {
- RandomVectorScorer fieldView =
- SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);
RandomVectorScorer groupView =
- SCORER.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);
- return new RandomVectorScorerImpl(fieldView, groupView, dedupValues.getFieldOrdToGroupOrd());
+ delegate.getRandomVectorScorer(similarityFunction, dedupValues.getGroupView(), target);
+ return new RandomVectorScorerImpl(
+ vectorValues, groupView, dedupValues.getFieldOrdToGroupOrd());
}
- return SCORER.getRandomVectorScorer(similarityFunction, vectorValues, target);
+ return delegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
}
- private record RandomVectorScorerSupplierImpl(
- RandomVectorScorerSupplier fieldView,
+ /**
+ * Supplies scorers whose scoring and target ordinals are both translated to group ordinals, with
+ * doc operations on the original values.
+ */
+ record RandomVectorScorerSupplierImpl(
+ KnnVectorValues values,
RandomVectorScorerSupplier groupView,
FieldOrdToGroupOrd fieldOrdToGroupOrd)
implements RandomVectorScorerSupplier {
@Override
public UpdateableRandomVectorScorer scorer() throws IOException {
- return new UpdateableRandomVectorScorerImpl(
- fieldView.scorer(), groupView.scorer(), fieldOrdToGroupOrd);
+ return new UpdateableRandomVectorScorerImpl(values, groupView.scorer(), fieldOrdToGroupOrd);
}
@Override
public RandomVectorScorerSupplier copy() throws IOException {
return new RandomVectorScorerSupplierImpl(
- fieldView.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());
+ values.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());
}
}
- private static class RandomVectorScorerImpl implements RandomVectorScorer {
- private final RandomVectorScorer fieldView;
+ private static class RandomVectorScorerImpl
+ extends RandomVectorScorer.AbstractRandomVectorScorer {
+
private final RandomVectorScorer groupView;
private final FieldOrdToGroupOrd fieldOrdToGroupOrd;
private int[] scratch;
RandomVectorScorerImpl(
- RandomVectorScorer fieldView,
+ KnnVectorValues values,
RandomVectorScorer groupView,
FieldOrdToGroupOrd fieldOrdToGroupOrd) {
- this.fieldView = fieldView;
+
+ super(values);
this.groupView = groupView;
this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;
this.scratch = new int[SCRATCH_INITIAL_SIZE];
}
- @Override
- public int ordToDoc(int ord) {
- return fieldView.ordToDoc(ord);
- }
-
- @Override
- public Bits getAcceptOrds(Bits acceptDocs) {
- return fieldView.getAcceptOrds(acceptDocs);
- }
-
@Override
public float score(int node) throws IOException {
return groupView.score(fieldOrdToGroupOrd.get(node));
@@ -158,23 +168,20 @@ final class DedupFlatVectorsScorer implements FlatVectorsScorer {
}
return groupView.bulkScore(scratch, scores, numNodes);
}
-
- @Override
- public int maxOrd() {
- return fieldView.maxOrd();
- }
}
private static final class UpdateableRandomVectorScorerImpl extends RandomVectorScorerImpl
implements UpdateableRandomVectorScorer {
+
private final UpdateableRandomVectorScorer groupView;
private final FieldOrdToGroupOrd fieldOrdToGroupOrd;
UpdateableRandomVectorScorerImpl(
- UpdateableRandomVectorScorer fieldView,
+ KnnVectorValues values,
UpdateableRandomVectorScorer groupView,
FieldOrdToGroupOrd fieldOrdToGroupOrd) {
- super(fieldView, groupView, fieldOrdToGroupOrd);
+
+ super(values, groupView, fieldOrdToGroupOrd);
this.groupView = groupView;
this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.java
index d8020fb2e85..1e3b6418cd7 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsWriter.java
@@ -16,12 +16,6 @@
*/
package org.apache.lucene.sandbox.codecs.dedup;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_CODEC_NAME;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.META_EXTENSION;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_CODEC_NAME;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VECTOR_DATA_EXTENSION;
-import static org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat.VERSION_CURRENT;
-
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;
@@ -41,6 +35,10 @@ import org.apache.lucene.util.RamUsageEstimator;
* or merging existing segments (never both), delegating to {@link DedupFlushContext} or {@link
* DedupMergeContext} accordingly.
*
+ * <p>Also used by {@link DedupScalarQuantizedVectorsFormat} (with a non-null {@link
+ * DedupQuantizer}) to additionally write a quantized copy of each applicable group, into a separate
+ * quantized vector data file.
+ *
* @lucene.experimental
*/
final class DedupFlatVectorsWriter extends FlatVectorsWriter {
@@ -49,42 +47,73 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {
private final IndexOutput meta;
private final IndexOutput vectorData;
+ private final IndexOutput quantizedVectorData;
private boolean finished;
+ private final DedupQuantizer quantizer;
+
private final DedupFlushContext flushContext;
private boolean usedForFlush;
private final DedupMergeContext mergeContext;
private boolean usedForMerge;
- DedupFlatVectorsWriter(SegmentWriteState state, FlatVectorsScorer vectorsScorer)
+ DedupFlatVectorsWriter(
+ SegmentWriteState state,
+ FlatVectorsScorer vectorsScorer,
+ DedupQuantizer quantizer,
+ String metaCodecName,
+ String metaExtension,
+ String vectorDataCodecName,
+ String vectorDataExtension,
+ String quantizedVectorDataCodecName,
+ String quantizedVectorDataExtension,
+ int versionCurrent)
throws IOException {
super(vectorsScorer);
this.finished = false;
+ this.quantizer = quantizer;
this.flushContext = new DedupFlushContext();
this.usedForFlush = false;
this.mergeContext = new DedupMergeContext();
this.usedForMerge = false;
String metaFileName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, META_EXTENSION);
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
String vectorDataFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, VECTOR_DATA_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);
boolean success = false;
try {
this.meta = state.directory.createOutput(metaFileName, state.context);
- this.vectorData = state.directory.createOutput(vectorDataFileName, state.context);
CodecUtil.writeIndexHeader(
- meta, META_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ meta, metaCodecName, versionCurrent, state.segmentInfo.getId(), state.segmentSuffix);
+
+ this.vectorData = state.directory.createOutput(vectorDataFileName, state.context);
CodecUtil.writeIndexHeader(
vectorData,
- VECTOR_DATA_CODEC_NAME,
- VERSION_CURRENT,
+ vectorDataCodecName,
+ versionCurrent,
state.segmentInfo.getId(),
state.segmentSuffix);
+
+ if (quantizer != null) {
+ String quantizedVectorDataFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, quantizedVectorDataExtension);
+ this.quantizedVectorData =
+ state.directory.createOutput(quantizedVectorDataFileName, state.context);
+ CodecUtil.writeIndexHeader(
+ quantizedVectorData,
+ quantizedVectorDataCodecName,
+ versionCurrent,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ } else {
+ this.quantizedVectorData = null;
+ }
success = true;
} finally {
if (success == false) {
@@ -110,7 +139,7 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {
}
usedForFlush = true;
- flushContext.flush(meta, vectorData, maxDoc, sortMap);
+ flushContext.flush(meta, vectorData, quantizedVectorData, maxDoc, sortMap, quantizer);
}
@Override
@@ -126,6 +155,10 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {
CodecUtil.writeFooter(meta);
CodecUtil.writeFooter(vectorData);
+
+ if (quantizedVectorData != null) {
+ CodecUtil.writeFooter(quantizedVectorData);
+ }
}
@Override
@@ -140,12 +173,12 @@ final class DedupFlatVectorsWriter extends FlatVectorsWriter {
}
private void finishMerge() throws IOException {
- mergeContext.finish(meta, vectorData);
+ mergeContext.finish(meta, vectorData, quantizedVectorData, quantizer);
}
@Override
public void close() throws IOException {
- IOUtils.close(meta, vectorData);
+ IOUtils.close(meta, vectorData, quantizedVectorData);
}
@Override
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java
index 59ad5465b83..d29645500c5 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java
@@ -28,9 +28,11 @@ import java.nio.FloatBuffer;
import java.nio.ShortBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;
import org.apache.lucene.index.DocsWithFieldSet;
@@ -94,11 +96,34 @@ final class DedupFlushContext implements Accountable {
return fieldVectorsWriter;
}
- void flush(IndexOutput meta, IndexOutput vectorData, int maxDoc, Sorter.DocMap sortMap)
+ /**
+ * Writes each group's distinct vectors once, followed by per-field metadata. When {@code
+ * quantizer} is non-null, a quantized copy of each applicable group is also written and its block
+ * location appended to the group metadata.
+ */
+ void flush(
+ IndexOutput meta,
+ IndexOutput vectorData,
+ IndexOutput quantizedVectorData,
+ int maxDoc,
+ Sorter.DocMap sortMap,
+ DedupQuantizer quantizer)
throws IOException {
Map<GroupKey, Integer> groupOrds = new HashMap<>();
+ // quantization flavors referencing each group, derived from its fields' similarity functions
+ Map<GroupKey, Set<DedupQuantizer.Flavor>> groupFlavors = new HashMap<>();
+ if (quantizer != null) {
+ for (FieldData fieldData : fieldDataList) {
+ if (fieldData.fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {
+ groupFlavors
+ .computeIfAbsent(fieldData.groupKey, _ -> EnumSet.noneOf(DedupQuantizer.Flavor.class))
+ .add(DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));
+ }
+ }
+ }
+
int groupOrd = 0;
for (Map.Entry<GroupKey, DedupGroup<?>> entry : groups.entrySet()) {
GroupKey groupKey = entry.getKey();
@@ -121,6 +146,22 @@ final class DedupFlushContext implements Accountable {
groupOrd, dimension, encoding, groupNumVectors, vectorDataOffset, vectorDataSize);
groupInfo.write(meta);
+ if (quantizer != null) {
+ if (group instanceof FloatGroup floatGroup) {
+ quantizer.writeGroup(
+ meta,
+ quantizedVectorData,
+ encoding,
+ dimension,
+ groupNumVectors,
+ groupFlavors.get(groupKey),
+ floatGroup::get,
+ null);
+ } else {
+ DedupQuantizer.writeEmptyGroup(meta);
+ }
+ }
+
groupOrds.put(groupKey, groupOrd);
groupOrd++;
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java
new file mode 100644
index 00000000000..a39816de55d
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.HNSW_GRAPH_THRESHOLD;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.util.hnsw.HnswGraph;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+
+/**
+ * An HNSW vector format that de-duplicates vectors, storing each distinct vector once in both raw
+ * and scalar quantized form.
+ *
+ * <p>Graph construction and search are identical to {@link
+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat}, with searches scoring against the
+ * quantized vectors. A {@link DedupScalarQuantizedVectorsFormat} is used for the flat vector
+ * storage, which stores each distinct vector exactly once, shared across all documents that
+ * reference it. See {@link DedupHnswVectorsFormat} for the intended multi-field usage pattern.
+ *
+ * <p>If you customize this format, be sure to <b>share the same instance</b> of the underlying
+ * {@link DedupScalarQuantizedVectorsFormat} to de-duplicate vectors correctly.
+ *
+ * @lucene.experimental
+ */
+public final class DedupHnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {
+ private static final String NAME = "DedupHnswScalarQuantizedVectorsFormat";
+
+ /**
+ * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to
+ * {@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See
+ * {@link HnswGraph} for more details.
+ */
+ private final int maxConn;
+
+ /**
+ * The number of candidate neighbors to track while searching the graph for each newly inserted
+ * node. Defaults to {@link
+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link
+ * HnswGraph} for details.
+ */
+ private final int beamWidth;
+
+ /** The scalar encoding for quantization. Defaults to {@link ScalarEncoding#UNSIGNED_BYTE}. */
+ private final ScalarEncoding encoding;
+
+ /** The format for storing, reading, and merging vectors on disk. */
+ private final FlatVectorsFormat flatVectorsFormat;
+
+ private final int numMergeWorkers;
+ private final TaskExecutor mergeExec;
+
+ /**
+ * The threshold to use to bypass HNSW graph building for tiny segments in terms of k for a graph
+ * i.e. number of docs to match the query (default is {@link
+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#HNSW_GRAPH_THRESHOLD}).
+ *
+ * <ul>
+ * <li>0 indicates that the graph is always built.
+ * <li>Positive values require that many estimated visited nodes before a graph is built.
+ * <li>Negative values aren't allowed.
+ * </ul>
+ */
+ private final int tinySegmentsThreshold;
+
+ /** Constructs a format with UNSIGNED_BYTE encoding and default graph construction parameters */
+ public DedupHnswScalarQuantizedVectorsFormat() {
+ this(
+ ScalarEncoding.UNSIGNED_BYTE,
+ DEFAULT_MAX_CONN,
+ DEFAULT_BEAM_WIDTH,
+ DEFAULT_NUM_MERGE_WORKER,
+ null,
+ HNSW_GRAPH_THRESHOLD);
+ }
+
+ /**
+ * Constructs a format with UNSIGNED_BYTE encoding, using the given graph construction parameters.
+ *
+ * @param maxConn the maximum number of connections to a node in the HNSW graph
+ * @param beamWidth the size of the queue maintained during graph construction.
+ */
+ public DedupHnswScalarQuantizedVectorsFormat(int maxConn, int beamWidth) {
+ this(
+ ScalarEncoding.UNSIGNED_BYTE,
+ maxConn,
+ beamWidth,
+ DEFAULT_NUM_MERGE_WORKER,
+ null,
+ HNSW_GRAPH_THRESHOLD);
+ }
+
+ /**
+ * Constructs a format using the given quantization encoding and graph construction parameters.
+ *
+ * @param encoding the quantization encoding used to encode the vectors
+ * @param maxConn the maximum number of connections to a node in the HNSW graph
+ * @param beamWidth the size of the queue maintained during graph construction.
+ */
+ public DedupHnswScalarQuantizedVectorsFormat(
+ ScalarEncoding encoding, int maxConn, int beamWidth) {
+ this(encoding, maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null, HNSW_GRAPH_THRESHOLD);
+ }
+
+ /**
+ * Constructs a format using the given quantization encoding and graph construction parameters.
+ *
+ * @param encoding the quantization encoding used to encode the vectors
+ * @param maxConn the maximum number of connections to a node in the HNSW graph
+ * @param beamWidth the size of the queue maintained during graph construction.
+ * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If
+ * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec
+ * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are
+ * generated by this format to do the merge. If null, the configured {@link
+ * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used.
+ */
+ public DedupHnswScalarQuantizedVectorsFormat(
+ ScalarEncoding encoding,
+ int maxConn,
+ int beamWidth,
+ int numMergeWorkers,
+ ExecutorService mergeExec) {
+ this(encoding, maxConn, beamWidth, numMergeWorkers, mergeExec, HNSW_GRAPH_THRESHOLD);
+ }
+
+ /**
+ * Constructs a format using the given quantization encoding and graph construction parameters.
+ *
+ * @param encoding the quantization encoding used to encode the vectors
+ * @param maxConn the maximum number of connections to a node in the HNSW graph
+ * @param beamWidth the size of the queue maintained during graph construction.
+ * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If
+ * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec
+ * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are
+ * generated by this format to do the merge. If null, the configured {@link
+ * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used.
+ * @param tinySegmentsThreshold the expected number of vector operations to return k nearest
+ * neighbors of the current graph size
+ */
+ public DedupHnswScalarQuantizedVectorsFormat(
+ ScalarEncoding encoding,
+ int maxConn,
+ int beamWidth,
+ int numMergeWorkers,
+ ExecutorService mergeExec,
+ int tinySegmentsThreshold) {
+ super(NAME);
+ if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) {
+ throw new IllegalArgumentException(
+ "maxConn must be positive and less than or equal to "
+ + MAXIMUM_MAX_CONN
+ + "; maxConn="
+ + maxConn);
+ }
+ if (beamWidth <= 0 || beamWidth > MAXIMUM_BEAM_WIDTH) {
+ throw new IllegalArgumentException(
+ "beamWidth must be positive and less than or equal to "
+ + MAXIMUM_BEAM_WIDTH
+ + "; beamWidth="
+ + beamWidth);
+ }
+ this.maxConn = maxConn;
+ this.beamWidth = beamWidth;
+ this.encoding = encoding;
+ this.flatVectorsFormat = new DedupScalarQuantizedVectorsFormat(encoding);
+ this.tinySegmentsThreshold = tinySegmentsThreshold;
+ if (numMergeWorkers == 1 && mergeExec != null) {
+ throw new IllegalArgumentException(
+ "No executor service is needed as we'll use single thread to merge");
+ }
+ this.numMergeWorkers = numMergeWorkers;
+ if (mergeExec != null) {
+ this.mergeExec = new TaskExecutor(mergeExec);
+ } else {
+ this.mergeExec = null;
+ }
+ }
+
+ @Override
+ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new Lucene99HnswVectorsWriter(
+ state,
+ maxConn,
+ beamWidth,
+ flatVectorsFormat,
+ flatVectorsFormat.fieldsWriter(state),
+ numMergeWorkers,
+ mergeExec,
+ tinySegmentsThreshold);
+ }
+
+ @Override
+ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+ return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));
+ }
+
+ @Override
+ public int getMaxDimensions(String fieldName) {
+ return DEFAULT_MAX_DIMENSIONS;
+ }
+
+ @Override
+ public String toString() {
+ return NAME
+ + "(maxConn="
+ + maxConn
+ + ", beamWidth="
+ + beamWidth
+ + ", encoding="
+ + encoding
+ + ", tinySegmentsThreshold="
+ + tinySegmentsThreshold
+ + ")";
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java
index bff40f47055..2c568638b5b 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java
@@ -219,16 +219,12 @@ public final class DedupHnswVectorsFormat extends KnnVectorsFormat {
@Override
public String toString() {
return NAME
- + "(name="
- + NAME
- + ", maxConn="
+ + "(maxConn="
+ maxConn
+ ", beamWidth="
+ beamWidth
+ ", tinySegmentsThreshold="
+ tinySegmentsThreshold
- + ", flatVectorFormat="
- + FORMAT
+ ")";
}
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java
index 55ca0e2cb1a..9b99cc490da 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java
@@ -28,9 +28,11 @@ import java.nio.FloatBuffer;
import java.nio.ShortBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocIDMerger;
@@ -86,7 +88,18 @@ final class DedupMergeContext implements Accountable {
mergeState.segmentInfo.maxDoc()));
}
- void finish(IndexOutput meta, IndexOutput vectorData) throws IOException {
+ /**
+ * Merges each group's distinct vectors, followed by per-field metadata. When {@code quantizer} is
+ * non-null, a quantized copy of each applicable group is also written and its block location
+ * appended to the group metadata — copying records from source segments in this format, and
+ * re-reading a distinct vector through its handle to quantize it otherwise.
+ */
+ void finish(
+ IndexOutput meta,
+ IndexOutput vectorData,
+ IndexOutput quantizedVectorData,
+ DedupQuantizer quantizer)
+ throws IOException {
// Evaluate compatible fields together for correct de-duplication
Map<GroupKey, List<FieldData>> fieldGroups =
@@ -120,6 +133,27 @@ final class DedupMergeContext implements Accountable {
groupOrd, dimension, encoding, groupNumVectors, vectorDataOffset, vectorDataSize);
groupInfo.write(meta);
+ if (quantizer != null) {
+ if (mergeGroup instanceof FloatGroup floatGroup) {
+ Set<DedupQuantizer.Flavor> flavors = EnumSet.noneOf(DedupQuantizer.Flavor.class);
+ for (FieldData fieldData : entry.getValue()) {
+ flavors.add(
+ DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));
+ }
+ quantizer.writeGroup(
+ meta,
+ quantizedVectorData,
+ encoding,
+ dimension,
+ groupNumVectors,
+ flavors,
+ ord -> floatGroup.get(ord).get(),
+ floatGroup::preQuantized);
+ } else {
+ DedupQuantizer.writeEmptyGroup(meta);
+ }
+ }
+
groupOrds.put(groupKey, groupOrd);
groupOrd++;
}
@@ -254,6 +288,22 @@ final class DedupMergeContext implements Accountable {
return new FloatVector(sub.values, sub.iterator.index());
}
+ /**
+ * The already-quantized record of the distinct vector at a group ordinal, when its source
+ * segment is in this format (data-blind quantization is a pure function of the raw vector, so
+ * the record can be copied on merge instead of re-quantizing), or {@code null}.
+ */
+ DedupQuantizer.PreQuantized preQuantized(int ord) {
+ FloatVector handle = get(ord);
+ if (handle.values()
+ instanceof DedupScalarQuantizedVectorValues.RawAndQuantizedValues rawAndQuantized) {
+ DedupScalarQuantizedVectorValues.FieldValues quantized =
+ rawAndQuantized.getQuantizedValues();
+ return new DedupQuantizer.PreQuantized(quantized, quantized.flavor(), handle.ord());
+ }
+ return null;
+ }
+
@Override
public long hash(FloatVector vector) throws IOException {
buffer.put(0, vector.get());
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java
new file mode 100644
index 00000000000..f764f0ef4b7
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import java.io.IOException;
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.lucene.codecs.lucene104.OffHeapScalarQuantizedVectorValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+
+/**
+ * Write-side helper performing <i>data-blind</i> scalar quantization of de-duplicated vectors.
+ *
+ * <p>Quantization assumes input vectors are evenly distributed around the origin, i.e. the centroid
+ * is a zero vector. The quantized record of a vector (packed bytes, optimized intervals, an
+ * additional correction and the component sum) is then a pure function of the raw vector and the
+ * {@link Flavor} derived from the field's similarity function: independent of the other vectors in
+ * the segment. Identical vectors thus quantize identically and the quantized data de-duplicates
+ * along with the raw data, shared across all documents and fields of a group with the same flavor.
+ *
+ * <p>For the same reason, a record already computed by a source segment does not change on merge:
+ * when a distinct vector originates from a segment in this format (with the same encoding and
+ * flavor), its quantized record is <b>copied</b> instead of re-reading the raw vector and
+ * re-quantizing it.
+ *
+ * <p>Records follow the {@link
+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat} conventions exactly
+ * (per flavor), so they are scored by the stock {@link
+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer}.
+ *
+ * @lucene.experimental
+ */
+record DedupQuantizer(ScalarEncoding encoding) {
+
+ /**
+ * How a vector is prepared and quantized, derived from the field's {@link
+ * VectorSimilarityFunction}. Fields of a group whose similarity functions map to the same flavor
+ * share one quantized record per distinct vector; a group stores one block per flavor in use.
+ */
+ enum Flavor {
+
+ /** Vectors quantized as-is; the additional correction holds the squared norm. */
+ EUCLIDEAN(0, VectorSimilarityFunction.EUCLIDEAN, false),
+
+ /**
+ * Vectors quantized as-is; the additional correction holds the dot product with the (zero)
+ * centroid, i.e. zero.
+ */
+ DOT_PRODUCT(1, VectorSimilarityFunction.DOT_PRODUCT, false),
+
+ /**
+ * Vectors are l2-normalized before quantization (for cosine similarity, scored as a dot
+ * product); the additional correction is zero as for {@link #DOT_PRODUCT}.
+ */
+ NORMALIZED(2, VectorSimilarityFunction.DOT_PRODUCT, true);
+
+ /** The number used to identify this flavor on the wire, rather than relying on ordinal. */
+ private final int wireNumber;
+
+ private final VectorSimilarityFunction quantizerFunction;
+ private final boolean normalized;
+
+ Flavor(int wireNumber, VectorSimilarityFunction quantizerFunction, boolean normalized) {
+ this.wireNumber = wireNumber;
+ this.quantizerFunction = quantizerFunction;
+ this.normalized = normalized;
+ }
+
+ static Flavor of(VectorSimilarityFunction function) {
+ return switch (function) {
+ case EUCLIDEAN -> EUCLIDEAN;
+ case DOT_PRODUCT, MAXIMUM_INNER_PRODUCT -> DOT_PRODUCT;
+ case COSINE -> NORMALIZED;
+ };
+ }
+
+ int getWireNumber() {
+ return wireNumber;
+ }
+
+ /** Returns the flavor for the given wire number, or empty if unknown. */
+ static Optional<Flavor> fromWireNumber(int wireNumber) {
+ for (Flavor flavor : values()) {
+ if (flavor.wireNumber == wireNumber) {
+ return Optional.of(flavor);
+ }
+ }
+ return Optional.empty();
+ }
+
+ /**
+ * The quantizer producing this flavor's records against a zero centroid. NOTE: {@link
+ * #NORMALIZED} quantizes with {@link VectorSimilarityFunction#DOT_PRODUCT} (identical bytes and
+ * corrections, without the unit-centroid requirement of a cosine-configured quantizer).
+ */
+ OptimizedScalarQuantizer quantizer() {
+ return new OptimizedScalarQuantizer(quantizerFunction);
+ }
+
+ boolean normalized() {
+ return normalized;
+ }
+ }
+
+ /** Supplies the distinct (raw) float vector at a group ordinal. */
+ interface FloatVectorSupplier {
+ float[] get(int ord) throws IOException;
+ }
+
+ /**
+ * The already-quantized record of a distinct vector, held by a source segment at {@code ord} with
+ * the given flavor.
+ */
+ record PreQuantized(QuantizedByteVectorValues values, Flavor flavor, int ord) {}
+
+ /**
+ * Supplies the {@link PreQuantized} record of the distinct vector at a group ordinal, or {@code
+ * null} when its source segment does not hold one.
+ */
+ interface PreQuantizedSupplier {
+ PreQuantized get(int ord) throws IOException;
+ }
+
+ /** The encoding, offset and size of one flavor's quantized data block. */
+ record QuantizedBlock(
+ ScalarEncoding encoding, long quantizedDataOffset, long quantizedDataSize) {}
+
+ /** Writes an empty flavor-block list, for groups without quantized data. */
+ static void writeEmptyGroup(IndexOutput meta) throws IOException {
+ meta.writeVInt(0);
+ }
+
+ /** Reads the flavor-block list of a group, the counterpart of {@link #writeGroup}. */
+ static Map<Flavor, QuantizedBlock> readGroup(IndexInput meta) throws IOException {
+ int numFlavors = meta.readVInt();
+ Map<Flavor, QuantizedBlock> blocks = new EnumMap<>(Flavor.class);
+ for (int i = 0; i < numFlavors; i++) {
+ int flavorWireNumber = meta.readVInt();
+ Flavor flavor =
+ Flavor.fromWireNumber(flavorWireNumber)
+ .orElseThrow(
+ () ->
+ new CorruptIndexException(
+ "Invalid flavor wire number: " + flavorWireNumber, meta));
+ int wireNumber = meta.readVInt();
+ ScalarEncoding encoding =
+ ScalarEncoding.fromWireNumber(wireNumber)
+ .orElseThrow(
+ () ->
+ new CorruptIndexException(
+ "Invalid scalar encoding wire number: " + wireNumber, meta));
+ long offset = meta.readLong();
+ long size = meta.readLong();
+ if (blocks.put(flavor, new QuantizedBlock(encoding, offset, size)) != null) {
+ throw new CorruptIndexException("Duplicate flavor: " + flavor, meta);
+ }
+ }
+ return blocks;
+ }
+
+ /**
+ * Quantizes and writes a group's distinct vectors: one block per flavor, one record per group
+ * ordinal within each block. Wherever a {@code preQuantized} record with a matching encoding and
+ * flavor is available (i.e. the vector originates from a segment in this format), it is copied
+ * as-is; otherwise the raw vector is quantized. The block locations are written to {@code meta};
+ * non-FLOAT32 groups (stored raw only) record an empty list.
+ */
+ void writeGroup(
+ IndexOutput meta,
+ IndexOutput quantizedVectorData,
+ VectorEncoding groupEncoding,
+ int dimension,
+ int numVectors,
+ Set<Flavor> flavors,
+ FloatVectorSupplier vectors,
+ PreQuantizedSupplier preQuantized)
+ throws IOException {
+
+ if (groupEncoding != VectorEncoding.FLOAT32) {
+ writeEmptyGroup(meta);
+ return;
+ }
+
+ meta.writeVInt(flavors.size());
+ for (Flavor flavor : Flavor.values()) {
+ if (flavors.contains(flavor) == false) {
+ continue;
+ }
+ QuantizedBlock block =
+ writeFlavorBlock(
+ quantizedVectorData, flavor, dimension, numVectors, vectors, preQuantized);
+ meta.writeVInt(flavor.getWireNumber());
+ meta.writeVInt(block.encoding().getWireNumber());
+ meta.writeLong(block.quantizedDataOffset());
+ meta.writeLong(block.quantizedDataSize());
+ }
+ }
+
+ private QuantizedBlock writeFlavorBlock(
+ IndexOutput quantizedVectorData,
+ Flavor flavor,
+ int dimension,
+ int numVectors,
+ FloatVectorSupplier vectors,
+ PreQuantizedSupplier preQuantized)
+ throws IOException {
+
+ OptimizedScalarQuantizer quantizer = flavor.quantizer();
+ float[] zeroCentroid = new float[dimension];
+ float[] normalized = flavor.normalized() ? new float[dimension] : null;
+ byte[] scratch = new byte[encoding.getDiscreteDimensions(dimension)];
+ byte[] packed =
+ switch (encoding) {
+ case UNSIGNED_BYTE, SEVEN_BIT -> scratch;
+ case PACKED_NIBBLE, SINGLE_BIT_QUERY_NIBBLE, DIBIT_QUERY_NIBBLE ->
+ new byte[encoding.getDocPackedLength(scratch.length)];
+ };
+
+ long quantizedDataOffset = quantizedVectorData.alignFilePointer(Float.BYTES);
+ for (int ord = 0; ord < numVectors; ord++) {
+ // The quantized record is a pure function of the raw vector and the flavor: a record
+ // already computed by a source segment can be copied instead of re-quantizing.
+ if (preQuantized != null) {
+ PreQuantized pre = preQuantized.get(ord);
+ if (pre != null && pre.flavor() == flavor && pre.values().getScalarEncoding() == encoding) {
+ // NOTE: read the packed bytes before the corrective terms, which are then served from the
+ // record cached by the read of the packed bytes
+ writeRecord(
+ quantizedVectorData,
+ pre.values().vectorValue(pre.ord()),
+ pre.values().getCorrectiveTerms(pre.ord()));
+ continue;
+ }
+ }
+
+ float[] vector = vectors.get(ord);
+ if (flavor.normalized()) {
+ // normalize a copy: the source buffer is shared / owned by the group
+ System.arraycopy(vector, 0, normalized, 0, dimension);
+ VectorUtil.l2normalize(normalized);
+ vector = normalized;
+ }
+ // NOTE: scalarQuantize subtracts the centroid from the input in place, but with a zero
+ // centroid the values are unchanged, so shared / owned buffers are safe to pass directly.
+ OptimizedScalarQuantizer.QuantizationResult corrections =
+ quantizer.scalarQuantize(vector, scratch, encoding.getBits(), zeroCentroid);
+ switch (encoding) {
+ case PACKED_NIBBLE -> OffHeapScalarQuantizedVectorValues.packNibbles(scratch, packed);
+ case SINGLE_BIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.packAsBinary(scratch, packed);
+ case DIBIT_QUERY_NIBBLE -> OptimizedScalarQuantizer.transposeDibit(scratch, packed);
+ case UNSIGNED_BYTE, SEVEN_BIT -> {}
+ }
+ writeRecord(quantizedVectorData, packed, corrections);
+ }
+ long quantizedDataSize = quantizedVectorData.getFilePointer() - quantizedDataOffset;
+
+ return new QuantizedBlock(encoding, quantizedDataOffset, quantizedDataSize);
+ }
+
+ private static void writeRecord(
+ IndexOutput quantizedVectorData,
+ byte[] packed,
+ OptimizedScalarQuantizer.QuantizationResult corrections)
+ throws IOException {
+ quantizedVectorData.writeBytes(packed, packed.length);
+ quantizedVectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval()));
+ quantizedVectorData.writeInt(Float.floatToIntBits(corrections.upperInterval()));
+ quantizedVectorData.writeInt(Float.floatToIntBits(corrections.additionalCorrection()));
+ quantizedVectorData.writeInt(corrections.quantizedComponentSum());
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
new file mode 100644
index 00000000000..8504a4c6d0f
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import static org.apache.lucene.index.VectorEncoding.FLOAT32;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.lucene104.OffHeapScalarQuantizedVectorValues;
+import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;
+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;
+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdOffHeap;
+import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+
+/**
+ * Vector values over data-blind scalar quantized, de-duplicated vectors. Mirrors the raw {@link
+ * DedupVectorValues} structure: a dense group view (an {@link OffHeapScalarQuantizedVectorValues}
+ * with a zero centroid) holds one quantized record per distinct vector, and a {@link FieldValues}
+ * view maps a field's per-document ordinals onto it via the shared {@code fieldOrdToGroupOrd}
+ * translation.
+ *
+ * @lucene.experimental
+ */
+final class DedupScalarQuantizedVectorValues {
+
+ private DedupScalarQuantizedVectorValues() {}
+
+ /** Number of corrective bytes stored per quantized vector. */
+ static final int CORRECTIVE_BYTES = 3 * Float.BYTES + Integer.BYTES;
+
+ /**
+ * A dense view over a group's quantized records of one {@link DedupQuantizer.Flavor}, indexed by
+ * group ordinal. Quantization is data-blind, i.e. against a zero centroid (see {@link
+ * DedupQuantizer}). With {@code isQuerySide}, records hold query-encoded vectors instead (only
+ * different for asymmetric encodings, used for merge-time graph construction).
+ */
+ static QuantizedByteVectorValues groupValues(
+ boolean isQuerySide,
+ DedupScalarQuantizedVectorsScorer vectorsScorer,
+ VectorSimilarityFunction function,
+ DedupQuantizer.Flavor flavor,
+ ScalarEncoding encoding,
+ int dimension,
+ int groupNumVectors,
+ IndexInput slice) {
+
+ return new OffHeapScalarQuantizedVectorValues.DenseOffHeapVectorValues(
+ isQuerySide,
+ dimension,
+ groupNumVectors,
+ new float[dimension], // always a zero centroid, quantization is data-blind
+ 0,
+ flavor.quantizer(),
+ encoding,
+ function,
+ vectorsScorer,
+ slice);
+ }
+
+ static FieldValues loadQuantized(
+ DedupScalarQuantizedVectorsScorer vectorsScorer,
+ VectorSimilarityFunction function,
+ ScalarEncoding encoding,
+ OrdToDocDISIReaderConfiguration configuration,
+ int dimension,
+ int groupNumVectors,
+ IndexInput vectorData,
+ IndexInput quantizedVectorData,
+ long quantizedDataOffset,
+ long quantizedDataSize,
+ long fieldOrdToGroupOrdOffset,
+ long fieldOrdToGroupOrdSize)
+ throws IOException {
+
+ final OffHeapFloatVectorValues fieldView =
+ OffHeapFloatVectorValues.load(
+ function, vectorsScorer, configuration, FLOAT32, dimension, 0, 0, vectorData);
+
+ final QuantizedByteVectorValues groupView =
+ groupValues(
+ false,
+ vectorsScorer,
+ function,
+ DedupQuantizer.Flavor.of(function),
+ encoding,
+ dimension,
+ groupNumVectors,
+ quantizedVectorData.slice(
+ "quantized-group-slice", quantizedDataOffset, quantizedDataSize));
+
+ final FieldOrdToGroupOrd fieldOrdToGroupOrd =
+ new FieldOrdToGroupOrdOffHeap(vectorData, fieldOrdToGroupOrdOffset, fieldOrdToGroupOrdSize);
+
+ return new FieldValues(vectorsScorer, function, fieldView, groupView, fieldOrdToGroupOrd);
+ }
+
+ /**
+ * A field's quantized vector values: doc operations delegate to the field's ordinal-to-doc
+ * mapping while vector operations resolve through {@code fieldOrdToGroupOrd} into the shared
+ * group view.
+ */
+ static final class FieldValues extends QuantizedByteVectorValues implements DedupVectorValues {
+ private final DedupScalarQuantizedVectorsScorer vectorsScorer;
+ private final VectorSimilarityFunction function;
+ private final FloatVectorValues fieldView;
+ private final QuantizedByteVectorValues groupView;
+ private final FieldOrdToGroupOrd fieldOrdToGroupOrd;
+ private int[] scratch;
+
+ FieldValues(
+ DedupScalarQuantizedVectorsScorer vectorsScorer,
+ VectorSimilarityFunction function,
+ FloatVectorValues fieldView,
+ QuantizedByteVectorValues groupView,
+ FieldOrdToGroupOrd fieldOrdToGroupOrd) {
+ this.vectorsScorer = vectorsScorer;
+ this.function = function;
+ this.fieldView = fieldView;
+ this.groupView = groupView;
+ this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;
+ this.scratch = new int[DedupUtil.SCRATCH_INITIAL_SIZE];
+ }
+
+ @Override
+ public QuantizedByteVectorValues getGroupView() {
+ return groupView;
+ }
+
+ @Override
+ public FieldOrdToGroupOrd getFieldOrdToGroupOrd() {
+ return fieldOrdToGroupOrd;
+ }
+
+ /** The quantization flavor of this field's records, derived from its similarity function. */
+ DedupQuantizer.Flavor flavor() {
+ return DedupQuantizer.Flavor.of(function);
+ }
+
+ @Override
+ public int dimension() {
+ return groupView.dimension();
+ }
+
+ @Override
+ public int size() {
+ return fieldView.size();
+ }
+
+ @Override
+ public int ordToDoc(int ord) {
+ return fieldView.ordToDoc(ord);
+ }
+
+ @Override
+ public Bits getAcceptOrds(Bits acceptDocs) {
+ return fieldView.getAcceptOrds(acceptDocs);
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return fieldView.iterator();
+ }
+
+ @Override
+ public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {
+ if (scratch.length < ordsToPrefetch.length) { // grow if needed
+ scratch = ArrayUtil.grow(scratch, ordsToPrefetch.length);
+ }
+ for (int i = 0; i < numOrds; i++) {
+ scratch[i] = fieldOrdToGroupOrd.get(ordsToPrefetch[i]);
+ }
+ groupView.prefetch(scratch, numOrds);
+ }
+
+ @Override
+ public byte[] vectorValue(int ord) throws IOException {
+ return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));
+ }
+
+ @Override
+ public OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int ord)
+ throws IOException {
+ return groupView.getCorrectiveTerms(fieldOrdToGroupOrd.get(ord));
+ }
+
+ @Override
+ public OptimizedScalarQuantizer getQuantizer() {
+ return groupView.getQuantizer();
+ }
+
+ @Override
+ public ScalarEncoding getScalarEncoding() {
+ return groupView.getScalarEncoding();
+ }
+
+ @Override
+ public float[] getCentroid() throws IOException {
+ return groupView.getCentroid();
+ }
+
+ @Override
+ public float getCentroidDP() {
+ return 0;
+ }
+
+ @Override
+ public int getVectorByteLength() {
+ return groupView.getVectorByteLength();
+ }
+
+ @Override
+ public FieldValues copy() throws IOException {
+ return new FieldValues(
+ vectorsScorer, function, fieldView.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());
+ }
+
+ @Override
+ public VectorScorer scorer(float[] target) throws IOException {
+ if (size() == 0) {
+ return null;
+ }
+ FieldValues copy = copy();
+ DocIndexIterator indexIterator = copy.iterator();
+ RandomVectorScorer vectorScorer = vectorsScorer.getRandomVectorScorer(function, copy, target);
+ boolean isDense = copy.fieldView instanceof OffHeapFloatVectorValues.DenseOffHeapVectorValues;
+ return new DedupVectorScorer(indexIterator, vectorScorer, isDense);
+ }
+ }
+
+ /**
+ * Full-precision view of a field backed by the raw de-duplicated vectors, whose {@link
+ * #scorer(float[])} scores against the quantized values instead. Mirrors {@code
+ * Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedVectorValues}.
+ */
+ static final class RawAndQuantizedValues extends FloatVectorValues implements DedupVectorValues {
+ private final DedupVectorValues.FloatImpl rawValues;
+ private final FieldValues quantizedValues;
+
+ RawAndQuantizedValues(DedupVectorValues.FloatImpl rawValues, FieldValues quantizedValues) {
+ this.rawValues = rawValues;
+ this.quantizedValues = quantizedValues;
+ }
+
+ FieldValues getQuantizedValues() {
+ return quantizedValues;
+ }
+
+ @Override
+ public FloatVectorValues getGroupView() {
+ return rawValues.getGroupView();
+ }
+
+ @Override
+ public FieldOrdToGroupOrd getFieldOrdToGroupOrd() {
+ return rawValues.getFieldOrdToGroupOrd();
+ }
+
+ @Override
+ public int dimension() {
+ return rawValues.dimension();
+ }
+
+ @Override
+ public int size() {
+ return rawValues.size();
+ }
+
+ @Override
+ public int ordToDoc(int ord) {
+ return rawValues.ordToDoc(ord);
+ }
+
+ @Override
+ public Bits getAcceptOrds(Bits acceptDocs) {
+ return rawValues.getAcceptOrds(acceptDocs);
+ }
+
+ @Override
+ public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {
+ rawValues.prefetch(ordsToPrefetch, numOrds);
+ }
+
+ @Override
+ public float[] vectorValue(int ord) throws IOException {
+ return rawValues.vectorValue(ord);
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return rawValues.iterator();
+ }
+
+ @Override
+ public RawAndQuantizedValues copy() throws IOException {
+ return new RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());
+ }
+
+ @Override
+ public VectorScorer scorer(float[] target) throws IOException {
+ return quantizedValues.scorer(target);
+ }
+
+ @Override
+ public VectorScorer rescorer(float[] target) throws IOException {
+ return rawValues.rescorer(target);
+ }
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java
new file mode 100644
index 00000000000..5ecfb0cac55
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
+import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+
+/**
+ * A scalar quantized version of {@link DedupFlatVectorsFormat} that stores each distinct vector
+ * once, in both raw and quantized form.
+ *
+ * <p>Quantization is <i>data-blind</i>: input vectors are assumed to be evenly distributed, i.e.
+ * centered on a zero vector (see {@link DedupQuantizer}). The quantized record of a vector is then
+ * a pure function of the raw vector and the {@link DedupQuantizer.Flavor} derived from the field's
+ * similarity function — independent of the other vectors in the segment — so quantized records
+ * de-duplicate like raw vectors: identical vectors across documents and fields of a group whose
+ * similarity functions map to the same flavor share one quantized record, resolved through the same
+ * {@code fieldOrdToGroupOrd} translation map. Records follow the {@link
+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat} conventions per flavor,
+ * and are scored by the stock {@link
+ * org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer}. Note the accuracy
+ * tradeoff relative to that format, which centers vectors on a per-field centroid before
+ * quantizing.
+ *
+ * <p>Only {@link org.apache.lucene.index.VectorEncoding#FLOAT32} vectors are quantized; BYTE and
+ * FLOAT16 vectors are stored raw only, identical to {@link DedupFlatVectorsFormat}.
+ *
+ * <h2>.vdd (vector de-dup data) file</h2>
+ *
+ * <p>The raw vectors and per-field data, laid out exactly as the {@link DedupFlatVectorsFormat}
+ * {@code .vdd} file.
+ *
+ * <h2>.vdqd (vector de-dup quantized data) file</h2>
+ *
+ * <p>One block per FLOAT32 group and {@link DedupQuantizer.Flavor} in use (aligned to {@code
+ * Float.BYTES}), holding one quantized record per distinct vector:
+ *
+ * <ul>
+ * <li><b>[byte]</b> the packed quantized values (layout depends on the {@link ScalarEncoding})
+ * <li><b>[float]</b> the lower optimized interval
+ * <li><b>[float]</b> the upper optimized interval
+ * <li><b>[float]</b> the additional correction (the squared norm for the EUCLIDEAN flavor, zero
+ * otherwise)
+ * <li><b>[int]</b> the sum of the quantized components
+ * </ul>
+ *
+ * <h2>.vdqm (vector de-dup quantized metadata) file</h2>
+ *
+ * <p>The layout of the {@link DedupFlatVectorsFormat} {@code .vdm} file, with each group
+ * additionally recording its flavor blocks:
+ *
+ * <ul>
+ * <li><b>[vint]</b> the number of flavor blocks ({@code 0} for non-FLOAT32 groups)
+ * <li>per flavor block: <b>[vint]</b> the flavor wire number, <b>[vint]</b> the {@link
+ * ScalarEncoding} wire number of its records, <b>[int64]</b> offset and <b>[int64]</b> length
+ * of its quantized records in the .vdqd file
+ * </ul>
+ *
+ * @lucene.experimental
+ */
+final class DedupScalarQuantizedVectorsFormat extends FlatVectorsFormat {
+ static final String NAME = "DedupScalarQuantizedVectorsFormat";
+
+ private static final String META_CODEC_NAME = "DedupScalarQuantizedVectorsFormatMeta";
+ private static final String META_EXTENSION = "vdqm";
+
+ private static final String VECTOR_DATA_CODEC_NAME =
+ "DedupScalarQuantizedVectorsFormatVectorData";
+ private static final String VECTOR_DATA_EXTENSION = "vdd";
+
+ private static final String QUANTIZED_VECTOR_DATA_CODEC_NAME =
+ "DedupScalarQuantizedVectorsFormatQuantizedVectorData";
+ private static final String QUANTIZED_VECTOR_DATA_EXTENSION = "vdqd";
+
+ private static final int VERSION_START = 0;
+ private static final int VERSION_CURRENT = VERSION_START;
+
+ private static final DedupScalarQuantizedVectorsScorer FLAT_VECTORS_SCORER =
+ new DedupScalarQuantizedVectorsScorer();
+
+ private final ScalarEncoding encoding;
+
+ DedupScalarQuantizedVectorsFormat(ScalarEncoding encoding) {
+ super(NAME);
+ this.encoding = encoding;
+ }
+
+ @Override
+ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new DedupFlatVectorsWriter(
+ state,
+ FLAT_VECTORS_SCORER,
+ new DedupQuantizer(encoding),
+ META_CODEC_NAME,
+ META_EXTENSION,
+ VECTOR_DATA_CODEC_NAME,
+ VECTOR_DATA_EXTENSION,
+ QUANTIZED_VECTOR_DATA_CODEC_NAME,
+ QUANTIZED_VECTOR_DATA_EXTENSION,
+ VERSION_CURRENT);
+ }
+
+ @Override
+ public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+ return new DedupScalarQuantizedVectorsReader(
+ state,
+ FLAT_VECTORS_SCORER,
+ META_CODEC_NAME,
+ META_EXTENSION,
+ VECTOR_DATA_CODEC_NAME,
+ VECTOR_DATA_EXTENSION,
+ QUANTIZED_VECTOR_DATA_CODEC_NAME,
+ QUANTIZED_VECTOR_DATA_EXTENSION,
+ VERSION_START,
+ VERSION_CURRENT);
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
new file mode 100644
index 00000000000..5551e11e15a
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
@@ -0,0 +1,582 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import static org.apache.lucene.index.VectorEncoding.BYTE;
+import static org.apache.lucene.index.VectorEncoding.FLOAT16;
+import static org.apache.lucene.index.VectorEncoding.FLOAT32;
+import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupBytes;
+import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloat16s;
+import static org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.loadDedupFloats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.sandbox.codecs.dedup.DedupQuantizer.QuantizedBlock;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;
+import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;
+import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+import org.apache.lucene.util.quantization.QuantizedVectorsReader;
+import org.apache.lucene.util.quantization.ScalarQuantizer;
+
+/**
+ * Reads de-duplicated flat vectors written by {@link DedupScalarQuantizedVectorsFormat}. Each
+ * FLOAT32 field exposes a full-precision view backed by the raw de-duplicated vectors, scored
+ * against a data-blind quantized view sharing the same {@code fieldOrdToGroupOrd} translation map.
+ * BYTE and FLOAT16 fields are stored and read raw only.
+ *
+ * @lucene.experimental
+ */
+final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
+ implements QuantizedVectorsReader {
+ private static final long SHALLOW_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(DedupScalarQuantizedVectorsReader.class);
+
+ private final DedupScalarQuantizedVectorsScorer vectorsScorer;
+ private final Map<String, FieldEntry> fields;
+ private final IndexInput vectorData;
+ private final IndexInput quantizedVectorData;
+ private final String vectorDataExtension;
+ private final String quantizedVectorDataExtension;
+
+ DedupScalarQuantizedVectorsReader(
+ SegmentReadState state,
+ DedupScalarQuantizedVectorsScorer vectorsScorer,
+ String metaCodecName,
+ String metaExtension,
+ String vectorDataCodecName,
+ String vectorDataExtension,
+ String quantizedVectorDataCodecName,
+ String quantizedVectorDataExtension,
+ int versionStart,
+ int versionCurrent)
+ throws IOException {
+
+ this.vectorsScorer = vectorsScorer;
+ this.fields = new HashMap<>();
+ this.vectorDataExtension = vectorDataExtension;
+ this.quantizedVectorDataExtension = quantizedVectorDataExtension;
+
+ String metaFileName =
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+
+ int versionMeta;
+ try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) {
+ Throwable priorE = null;
+ try {
+ versionMeta =
+ CodecUtil.checkIndexHeader(
+ meta,
+ metaCodecName,
+ versionStart,
+ versionCurrent,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ readMetaBody(meta, state.fieldInfos);
+ } catch (Throwable e) {
+ priorE = e;
+ throw e;
+ } finally {
+ CodecUtil.checkFooter(meta, priorE);
+ }
+ }
+
+ this.vectorData =
+ openDataInput(
+ state,
+ versionMeta,
+ vectorDataCodecName,
+ vectorDataExtension,
+ versionStart,
+ versionCurrent);
+
+ this.quantizedVectorData =
+ openDataInput(
+ state,
+ versionMeta,
+ quantizedVectorDataCodecName,
+ quantizedVectorDataExtension,
+ versionStart,
+ versionCurrent);
+ }
+
+ private void readMetaBody(ChecksumIndexInput meta, FieldInfos fieldInfos) throws IOException {
+ List<QuantizedGroupInfo> groupInfos = new ArrayList<>();
+ while (true) {
+ GroupInfo groupInfo = GroupInfo.readFromMeta(meta);
+ if (groupInfo == null) {
+ break;
+ }
+ Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks = DedupQuantizer.readGroup(meta);
+ validateQuantizedBlocks(meta, groupInfo, quantizedBlocks);
+ groupInfos.add(new QuantizedGroupInfo(groupInfo, quantizedBlocks));
+ }
+
+ while (true) {
+ ReadFieldInfo fieldInfo = ReadFieldInfo.read(meta);
+ if (fieldInfo == null) {
+ break;
+ }
+
+ FieldInfo info = fieldInfos.fieldInfo(fieldInfo.fieldNumber());
+ if (info == null) {
+ throw new CorruptIndexException("Invalid field number: " + fieldInfo.fieldNumber(), meta);
+ } else if (fieldInfo.function() != info.getVectorSimilarityFunction()) {
+ throw new CorruptIndexException(
+ "Inconsistent vector function: indexed="
+ + fieldInfo.function()
+ + ", actual="
+ + info.getVectorSimilarityFunction(),
+ meta);
+ } else if (fieldInfo.dimension() != info.getVectorDimension()) {
+ throw new CorruptIndexException(
+ "Inconsistent vector dimension: indexed="
+ + fieldInfo.dimension()
+ + ", actual="
+ + info.getVectorDimension(),
+ meta);
+ } else if (fieldInfo.encoding() != info.getVectorEncoding()) {
+ throw new CorruptIndexException(
+ "Inconsistent vector encoding: indexed="
+ + fieldInfo.encoding()
+ + ", actual="
+ + info.getVectorEncoding(),
+ meta);
+ }
+
+ if (fieldInfo.groupOrd() < 0 || fieldInfo.groupOrd() >= groupInfos.size()) {
+ throw new CorruptIndexException(
+ "Invalid groupId=" + fieldInfo.groupOrd() + ", numGroups=" + groupInfos.size(), meta);
+ }
+
+ QuantizedGroupInfo quantizedGroupInfo = groupInfos.get(fieldInfo.groupOrd());
+ GroupInfo groupInfo = quantizedGroupInfo.groupInfo();
+ QuantizedBlock quantizedBlock = null;
+ if (fieldInfo.encoding() == FLOAT32) {
+ DedupQuantizer.Flavor flavor = DedupQuantizer.Flavor.of(fieldInfo.function());
+ quantizedBlock = quantizedGroupInfo.quantizedBlocks().get(flavor);
+ if (quantizedBlock == null) {
+ throw new CorruptIndexException(
+ "Missing quantized block for flavor=" + flavor + ", field=" + info.name, meta);
+ }
+ }
+ if (fieldInfo.dimension() != groupInfo.dimension()) {
+ throw new CorruptIndexException(
+ "Vector dimension mismatch: field="
+ + fieldInfo.dimension()
+ + ", group="
+ + groupInfo.dimension(),
+ meta);
+ } else if (fieldInfo.encoding() != groupInfo.encoding()) {
+ throw new CorruptIndexException(
+ "Vector encoding mismatch: field="
+ + fieldInfo.encoding()
+ + ", group="
+ + groupInfo.encoding(),
+ meta);
+ }
+
+ fields.put(info.name, new FieldEntry(fieldInfo, groupInfo, quantizedBlock));
+ }
+ }
+
+ private static void validateQuantizedBlocks(
+ ChecksumIndexInput meta,
+ GroupInfo groupInfo,
+ Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks)
+ throws IOException {
+ if (groupInfo.encoding() != FLOAT32) {
+ if (quantizedBlocks.isEmpty() == false) {
+ throw new CorruptIndexException(
+ "Unexpected quantized data for encoding=" + groupInfo.encoding(), meta);
+ }
+ return;
+ }
+ for (Map.Entry<DedupQuantizer.Flavor, QuantizedBlock> entry : quantizedBlocks.entrySet()) {
+ long expectedSize =
+ (long) groupInfo.groupNumVectors()
+ * (entry.getValue().encoding().getDocPackedLength(groupInfo.dimension())
+ + DedupScalarQuantizedVectorValues.CORRECTIVE_BYTES);
+ if (entry.getValue().quantizedDataSize() != expectedSize) {
+ throw new CorruptIndexException(
+ "Quantized data size mismatch: flavor="
+ + entry.getKey()
+ + ", expected="
+ + expectedSize
+ + ", actual="
+ + entry.getValue().quantizedDataSize()
+ + ", numVectors="
+ + groupInfo.groupNumVectors()
+ + ", dimension="
+ + groupInfo.dimension(),
+ meta);
+ }
+ }
+ }
+
+ private static IndexInput openDataInput(
+ SegmentReadState state,
+ int versionMeta,
+ String vectorDataCodecName,
+ String vectorDataExtension,
+ int versionStart,
+ int versionCurrent)
+ throws IOException {
+
+ String fileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);
+
+ IOContext.FileOpenHint[] hints = {
+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM
+ };
+ IOContext context = state.context.withHints(hints);
+
+ IndexInput in = null;
+ boolean success = false;
+ try {
+ in = state.directory.openInput(fileName, context);
+ int versionVectorData =
+ CodecUtil.checkIndexHeader(
+ in,
+ vectorDataCodecName,
+ versionStart,
+ versionCurrent,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ if (versionMeta != versionVectorData) {
+ throw new CorruptIndexException(
+ "Format versions mismatch: meta="
+ + versionMeta
+ + ", "
+ + vectorDataCodecName
+ + "="
+ + versionVectorData,
+ in);
+ }
+ CodecUtil.retrieveChecksum(in);
+ success = true;
+ return in;
+ } finally {
+ if (success == false) {
+ IOUtils.closeWhileHandlingException(in);
+ }
+ }
+ }
+
+ @Override
+ public FlatVectorsScorer getFlatVectorScorer(String field) {
+ return vectorsScorer;
+ }
+
+ // package-private for testing
+ FieldEntry getEntry(String field, VectorEncoding expected) {
+ FieldEntry entry = fields.get(field);
+ if (entry == null) {
+ throw new IllegalArgumentException("field=" + field + " not found");
+ } else if (entry.fieldInfo().encoding() != expected) {
+ throw new IllegalArgumentException("field=" + field + " not indexed as " + expected);
+ }
+ return entry;
+ }
+
+ @Override
+ public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException {
+ FieldEntry entry = getEntry(field, FLOAT32);
+ FieldValues quantizedValues = getQuantizedVectorValues(entry);
+ return vectorsScorer.getRandomVectorScorer(
+ entry.fieldInfo().function(), quantizedValues, target);
+ }
+
+ @Override
+ public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException {
+ FieldEntry entry = getEntry(field, BYTE);
+ ByteVectorValues vectorValues = getByteVectorValues(entry);
+ return vectorsScorer.getRandomVectorScorer(entry.fieldInfo().function(), vectorValues, target);
+ }
+
+ @Override
+ public RandomVectorScorer getRandomVectorScorer(String field, short[] target) throws IOException {
+ FieldEntry entry = getEntry(field, FLOAT16);
+ Float16VectorValues vectorValues = getFloat16VectorValues(entry);
+ return vectorsScorer.getRandomVectorScorer(entry.fieldInfo().function(), vectorValues, target);
+ }
+
+ @Override
+ public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
+ CodecUtil.checksumEntireFile(vectorData, merge);
+ CodecUtil.checksumEntireFile(quantizedVectorData, merge);
+ }
+
+ private DedupVectorValues.FloatImpl getRawFloatVectorValues(FieldEntry entry) throws IOException {
+ return loadDedupFloats(
+ vectorsScorer,
+ entry.fieldInfo().function(),
+ entry.fieldInfo().ordToDoc(),
+ entry.fieldInfo().dimension(),
+ entry.groupInfo().groupNumVectors(),
+ vectorData,
+ entry.groupInfo().vectorDataOffset(),
+ entry.groupInfo().vectorDataSize(),
+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),
+ entry.fieldInfo().fieldOrdToGroupOrdSize());
+ }
+
+ private FieldValues getQuantizedVectorValues(FieldEntry entry) throws IOException {
+ return DedupScalarQuantizedVectorValues.loadQuantized(
+ vectorsScorer,
+ entry.fieldInfo().function(),
+ entry.quantizedBlock().encoding(),
+ entry.fieldInfo().ordToDoc(),
+ entry.fieldInfo().dimension(),
+ entry.groupInfo().groupNumVectors(),
+ vectorData,
+ quantizedVectorData,
+ entry.quantizedBlock().quantizedDataOffset(),
+ entry.quantizedBlock().quantizedDataSize(),
+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),
+ entry.fieldInfo().fieldOrdToGroupOrdSize());
+ }
+
+ @Override
+ public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+ FieldEntry entry = getEntry(field, FLOAT32);
+ return new RawAndQuantizedValues(
+ getRawFloatVectorValues(entry), getQuantizedVectorValues(entry));
+ }
+
+ private ByteVectorValues getByteVectorValues(FieldEntry entry) throws IOException {
+ return loadDedupBytes(
+ vectorsScorer,
+ entry.fieldInfo().function(),
+ entry.fieldInfo().ordToDoc(),
+ entry.fieldInfo().dimension(),
+ entry.groupInfo().groupNumVectors(),
+ vectorData,
+ entry.groupInfo().vectorDataOffset(),
+ entry.groupInfo().vectorDataSize(),
+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),
+ entry.fieldInfo().fieldOrdToGroupOrdSize());
+ }
+
+ @Override
+ public ByteVectorValues getByteVectorValues(String field) throws IOException {
+ return getByteVectorValues(getEntry(field, BYTE));
+ }
+
+ private Float16VectorValues getFloat16VectorValues(FieldEntry entry) throws IOException {
+ return loadDedupFloat16s(
+ vectorsScorer,
+ entry.fieldInfo().function(),
+ entry.fieldInfo().ordToDoc(),
+ entry.fieldInfo().dimension(),
+ entry.groupInfo().groupNumVectors(),
+ vectorData,
+ entry.groupInfo().vectorDataOffset(),
+ entry.groupInfo().vectorDataSize(),
+ entry.fieldInfo().fieldOrdToGroupOrdOffset(),
+ entry.fieldInfo().fieldOrdToGroupOrdSize());
+ }
+
+ @Override
+ public Float16VectorValues getFloat16VectorValues(String field) throws IOException {
+ return getFloat16VectorValues(getEntry(field, FLOAT16));
+ }
+
+ @Override
+ public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {
+ FieldEntry entry = getEntry(field, FLOAT32);
+ return getQuantizedVectorValues(entry);
+ }
+
+ @Override
+ public ScalarQuantizer getQuantizationState(String fieldName) {
+ return null;
+ }
+
+ @Override
+ public CloseableRandomVectorScorerSupplier getRandomVectorScorerSupplierForMerge(
+ FieldInfo fieldInfo, SegmentWriteState segmentWriteState) throws IOException {
+ FieldEntry entry = fields.get(fieldInfo.name);
+ if (entry == null || entry.fieldInfo().encoding() != FLOAT32) {
+ // BYTE and FLOAT16 fields are stored raw only
+ return null;
+ }
+ FieldValues quantizedValues = getQuantizedVectorValues(entry);
+ ScalarEncoding encoding = entry.quantizedBlock().encoding();
+ if (encoding.isAsymmetric() == false) {
+ RandomVectorScorerSupplier supplier =
+ vectorsScorer.getRandomVectorScorerSupplier(
+ fieldInfo.getVectorSimilarityFunction(), quantizedValues);
+ return CloseableRandomVectorScorerSupplier.create(supplier, quantizedValues.size(), () -> {});
+ }
+
+ // Asymmetric encodings compare query-encoded vectors against the stored doc-encoded vectors.
+ // Write a query-encoded record per distinct vector of the group into a temporary file; both
+ // sides then resolve through the shared fieldOrdToGroupOrd translation.
+ DedupVectorValues.FloatImpl rawValues = getRawFloatVectorValues(entry);
+ FloatVectorValues groupView = rawValues.getGroupView();
+ int dimension = entry.fieldInfo().dimension();
+ int groupNumVectors = entry.groupInfo().groupNumVectors();
+ DedupQuantizer.Flavor flavor =
+ DedupQuantizer.Flavor.of(fieldInfo.getVectorSimilarityFunction());
+
+ String tempQueryVectorsName = null;
+ try (IndexOutput tempQueryVectors =
+ segmentWriteState.directory.createTempOutput(
+ segmentWriteState.segmentInfo.name, "dedup_queries", segmentWriteState.context)) {
+ tempQueryVectorsName = tempQueryVectors.getName();
+
+ OptimizedScalarQuantizer quantizer = flavor.quantizer();
+ float[] zeroCentroid = new float[dimension];
+ float[] vector = new float[dimension];
+ byte[] scratch = new byte[encoding.getDiscreteDimensions(dimension)];
+ byte[] toQuery = new byte[encoding.getQueryPackedLength(dimension)];
+ for (int groupOrd = 0; groupOrd < groupNumVectors; groupOrd++) {
+ // copy: normalization and quantization mutate the input, which is a shared buffer
+ System.arraycopy(groupView.vectorValue(groupOrd), 0, vector, 0, dimension);
+ if (flavor.normalized()) {
+ VectorUtil.l2normalize(vector);
+ }
+ OptimizedScalarQuantizer.QuantizationResult corrections =
+ quantizer.scalarQuantize(vector, scratch, encoding.getQueryBits(), zeroCentroid);
+ OptimizedScalarQuantizer.transposeHalfByte(scratch, toQuery);
+ tempQueryVectors.writeBytes(toQuery, toQuery.length);
+ tempQueryVectors.writeInt(Float.floatToIntBits(corrections.lowerInterval()));
+ tempQueryVectors.writeInt(Float.floatToIntBits(corrections.upperInterval()));
+ tempQueryVectors.writeInt(Float.floatToIntBits(corrections.additionalCorrection()));
+ tempQueryVectors.writeInt(corrections.quantizedComponentSum());
+ }
+ CodecUtil.writeFooter(tempQueryVectors);
+ } catch (Throwable t) {
+ if (tempQueryVectorsName != null) {
+ IOUtils.deleteFilesSuppressingExceptions(
+ t, segmentWriteState.directory, tempQueryVectorsName);
+ }
+ throw t;
+ }
+
+ IndexInput queryVectorsInput =
+ segmentWriteState.directory.openInput(tempQueryVectorsName, segmentWriteState.context);
+ final String finalTempQueryVectorsName = tempQueryVectorsName;
+ try {
+ QuantizedByteVectorValues queryValues =
+ DedupScalarQuantizedVectorValues.groupValues(
+ true,
+ vectorsScorer,
+ fieldInfo.getVectorSimilarityFunction(),
+ flavor,
+ encoding,
+ dimension,
+ groupNumVectors,
+ queryVectorsInput);
+ RandomVectorScorerSupplier supplier =
+ DedupScalarQuantizedVectorsScorer.asymmetricMergeSupplier(
+ fieldInfo.getVectorSimilarityFunction(), queryValues, quantizedValues);
+ return CloseableRandomVectorScorerSupplier.create(
+ supplier,
+ quantizedValues.size(),
+ () -> {
+ IOUtils.close(queryVectorsInput);
+ IOUtils.deleteFilesIgnoringExceptions(
+ segmentWriteState.directory, finalTempQueryVectorsName);
+ });
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, queryVectorsInput);
+ IOUtils.deleteFilesSuppressingExceptions(
+ t, segmentWriteState.directory, finalTempQueryVectorsName);
+ throw t;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(vectorData, quantizedVectorData);
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return SHALLOW_SIZE + fields.size() * FieldEntry.SHALLOW_SIZE;
+ }
+
+ @Override
+ public Map<String, Long> getOffHeapByteSize(FieldInfo fieldInfo) {
+ FieldEntry entry = fields.get(fieldInfo.name);
+ if (entry == null) {
+ return Map.of();
+ }
+
+ // TODO: This is an over-estimation.
+ long vectorDataSize =
+ entry.fieldInfo().fieldOrdToGroupOrdSize() + entry.groupInfo().vectorDataSize();
+
+ if (entry.quantizedBlock() == null) {
+ return Map.of(vectorDataExtension, vectorDataSize);
+ }
+
+ return Map.of(
+ vectorDataExtension,
+ vectorDataSize,
+ quantizedVectorDataExtension,
+ entry.quantizedBlock().quantizedDataSize());
+ }
+
+ // package-private for testing
+ record FieldEntry(ReadFieldInfo fieldInfo, GroupInfo groupInfo, QuantizedBlock quantizedBlock) {
+ private static final long SHALLOW_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class)
+ + RamUsageEstimator.shallowSizeOfInstance(ReadFieldInfo.class)
+ + RamUsageEstimator.shallowSizeOfInstance(GroupInfo.class)
+ + RamUsageEstimator.shallowSizeOfInstance(QuantizedBlock.class);
+ }
+
+ private record QuantizedGroupInfo(
+ GroupInfo groupInfo, Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks) {}
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java
new file mode 100644
index 00000000000..b4aa648a873
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer;
+import org.apache.lucene.index.KnnVectorValues;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;
+import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+
+/**
+ * Scorer for scalar quantized, de-duplicated vectors: a {@link DedupFlatVectorsScorer} delegating
+ * vector operations to the {@link Lucene104ScalarQuantizedVectorScorer}, whose record and scoring
+ * conventions the quantized group views follow exactly (see {@link DedupQuantizer.Flavor}).
+ *
+ * @lucene.experimental
+ */
+final class DedupScalarQuantizedVectorsScorer extends DedupFlatVectorsScorer {
+
+ private static final Lucene104ScalarQuantizedVectorScorer QUANTIZED_SCORER =
+ new Lucene104ScalarQuantizedVectorScorer(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());
+
+ DedupScalarQuantizedVectorsScorer() {
+ super(QUANTIZED_SCORER);
+ }
+
+ /** Full-precision views resolve to their quantized values for scoring. */
+ @Override
+ protected KnnVectorValues unwrap(KnnVectorValues vectorValues) {
+ if (vectorValues instanceof RawAndQuantizedValues rawAndQuantized) {
+ return rawAndQuantized.getQuantizedValues();
+ }
+ return vectorValues;
+ }
+
+ /**
+ * Supplier for merge-time graph construction with asymmetric encodings: query-encoded distinct
+ * vectors (one per <b>group</b> ordinal, in a temporary file) are compared against the stored
+ * doc-encoded vectors, both sides resolving through {@code fieldOrdToGroupOrd} (see {@link
+ * DedupScalarQuantizedVectorsReader#getRandomVectorScorerSupplierForMerge}).
+ */
+ static RandomVectorScorerSupplier asymmetricMergeSupplier(
+ VectorSimilarityFunction similarityFunction,
+ QuantizedByteVectorValues queryValues,
+ FieldValues targetValues) {
+ RandomVectorScorerSupplier groupView =
+ QUANTIZED_SCORER.getRandomVectorScorerSupplier(
+ similarityFunction, queryValues, targetValues.getGroupView());
+ return new RandomVectorScorerSupplierImpl(
+ targetValues, groupView, targetValues.getFieldOrdToGroupOrd());
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
index 802f15d7240..b2102fa2e5a 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
@@ -54,7 +54,13 @@ import org.apache.lucene.util.packed.DirectReader;
* (one entry per distinct vector). {@code fieldOrdToGroupOrd} translates a document ordinal in the
* field into its group ordinal.
*/
-sealed interface DedupVectorValues {
+sealed interface DedupVectorValues
+ permits DedupVectorValues.ByteImpl,
+ DedupVectorValues.FloatImpl,
+ DedupVectorValues.Float16Impl,
+ DedupScalarQuantizedVectorValues.FieldValues,
+ DedupScalarQuantizedVectorValues.RawAndQuantizedValues {
+
/** The dense view over distinct vectors, indexed by group ordinal. */
KnnVectorValues getGroupView();
@@ -71,7 +77,7 @@ sealed interface DedupVectorValues {
FieldOrdToGroupOrd copy() throws IOException;
}
- static ByteVectorValues loadDedupBytes(
+ static ByteImpl loadDedupBytes(
FlatVectorsScorer vectorsScorer,
VectorSimilarityFunction function,
OrdToDocDISIReaderConfiguration configuration,
@@ -191,7 +197,7 @@ sealed interface DedupVectorValues {
}
}
- static FloatVectorValues loadDedupFloats(
+ static FloatImpl loadDedupFloats(
FlatVectorsScorer vectorsScorer,
VectorSimilarityFunction function,
OrdToDocDISIReaderConfiguration configuration,
@@ -311,7 +317,7 @@ sealed interface DedupVectorValues {
}
}
- static Float16VectorValues loadDedupFloat16s(
+ static Float16Impl loadDedupFloat16s(
FlatVectorsScorer vectorsScorer,
VectorSimilarityFunction function,
OrdToDocDISIReaderConfiguration configuration,
@@ -469,7 +475,7 @@ sealed interface DedupVectorValues {
@Override
public FieldOrdToGroupOrd copy() {
- return new FieldOrdToGroupOrdArrayList(fieldOrdToGroupOrd);
+ throw new UnsupportedOperationException("not meant for copying");
}
}
@@ -484,7 +490,7 @@ sealed interface DedupVectorValues {
@Override
public FieldOrdToGroupOrd copy() {
- return new FieldOrdToGroupOrdMappedArrayList(map, fieldOrdToGroupOrd);
+ throw new UnsupportedOperationException("not meant for copying");
}
}
@@ -513,7 +519,7 @@ sealed interface DedupVectorValues {
@Override
public FieldOrdToGroupOrd copy() throws IOException {
return new FieldOrdToGroupOrdOffHeap(
- vectorData, fieldOrdToGroupOrdOffset, fieldOrdToGroupOrdSize);
+ vectorData.clone(), fieldOrdToGroupOrdOffset, fieldOrdToGroupOrdSize);
}
}
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java
index 31892094183..0752b9e2d13 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java
@@ -16,11 +16,20 @@
*/
/**
- * De-duplicating HNSW vector format.
+ * De-duplicating HNSW vector formats.
*
* <p>Stores each distinct vector once and shares it across the documents and fields that reference
- * it, while reusing the Lucene 9.9 HNSW graph. See {@link
- * org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat} for the entry point and {@link
- * org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat} for the on-disk layout.
+ * it, while reusing the Lucene 9.9 HNSW graph. Two formats are provided:
+ *
+ * <ul>
+ * <li>{@link org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat} stores and scores raw
+ * vectors; see {@link org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat} for the
+ * on-disk layout.
+ * <li>{@link org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormat}
+ * additionally stores a scalar quantized copy of each distinct FLOAT32 vector, used for
+ * scoring; see {@link
+ * org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorsFormat} for the on-disk
+ * layout.
+ * </ul>
*/
package org.apache.lucene.sandbox.codecs.dedup;
diff --git a/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
index 4e3e3fc1fca..4cacd83f5fa 100644
--- a/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
+++ b/lucene/sandbox/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
@@ -15,3 +15,4 @@
org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormat
org.apache.lucene.sandbox.codecs.dedup.DedupHnswVectorsFormat
+org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormat
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.java
index bf8b1d4e9fc..24f484c65bd 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupFlatVectorsFormat.java
@@ -44,7 +44,11 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.LongArrayList;
+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;
+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdArrayList;
+import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdMappedArrayList;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.util.LuceneTestCase;
@@ -418,6 +422,17 @@ public class TestDedupFlatVectorsFormat extends LuceneTestCase {
}
}
+ /** Tests specific to on-heap versions of the field ord -> group ord mapping. */
+ public void testOnHeapFieldOrdToGroupOrd() {
+ FieldOrdToGroupOrd arrayBacked = new FieldOrdToGroupOrdArrayList(new IntArrayList());
+ expectThrows(UnsupportedOperationException.class, arrayBacked::copy); // not meant for copying.
+
+ FieldOrdToGroupOrd mappedArrayBacked =
+ new FieldOrdToGroupOrdMappedArrayList(new int[0], new IntArrayList());
+ expectThrows(
+ UnsupportedOperationException.class, mappedArrayBacked::copy); // not meant for copying.
+ }
+
/** Number of distinct vectors physically stored for a field's group. */
private static int groupNumVectors(KnnVectorValues values) {
assertThat(values, instanceOf(DedupVectorValues.class));
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java
new file mode 100644
index 00000000000..113a95b66a4
--- /dev/null
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.SameThreadExecutorService;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+import org.junit.Before;
+
+/**
+ * Runs the standard KNN vectors format suite against the de-duplicating scalar quantized HNSW
+ * format, with a randomized quantization encoding. Quantization-specific de-duplication behavior is
+ * covered by {@link TestDedupScalarQuantizedVectorsFormat}.
+ */
+public class TestDedupHnswScalarQuantizedVectorsFormat extends TestDedupHnswVectorsFormat {
+
+ private KnnVectorsFormat format;
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ ScalarEncoding[] encodingValues = ScalarEncoding.values();
+ ScalarEncoding encoding = encodingValues[random().nextInt(encodingValues.length)];
+ format =
+ new DedupHnswScalarQuantizedVectorsFormat(encoding, DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH);
+ super.setUp();
+ }
+
+ @Override
+ protected Codec getCodec() {
+ return TestUtil.alwaysKnnVectorsFormat(format);
+ }
+
+ public void testLimits() {
+ expectThrows(
+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(-1, 20));
+ expectThrows(
+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(0, 20));
+ expectThrows(
+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(20, 0));
+ expectThrows(
+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(20, -1));
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> new DedupHnswScalarQuantizedVectorsFormat(512 + 1, 20));
+ expectThrows(
+ IllegalArgumentException.class, () -> new DedupHnswScalarQuantizedVectorsFormat(20, 3201));
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ new DedupHnswScalarQuantizedVectorsFormat(
+ ScalarEncoding.UNSIGNED_BYTE, 20, 100, 1, new SameThreadExecutorService()));
+ }
+
+ @Override
+ protected void assertOffHeapByteSize(LeafReader r, String fieldName) throws IOException {
+ var fieldInfo = r.getFieldInfos().fieldInfo(fieldName);
+
+ if (r instanceof CodecReader codecReader) {
+ KnnVectorsReader knnVectorsReader = codecReader.getVectorReader();
+ knnVectorsReader = knnVectorsReader.unwrapReaderForField(fieldName);
+ var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo);
+ long totalByteSize = offHeap.values().stream().mapToLong(Long::longValue).sum();
+ if (knnVectorsReader instanceof Lucene99HnswVectorsReader) {
+ if (getNumVectors(knnVectorsReader, fieldInfo) == 0) {
+ assertEquals(0L, totalByteSize);
+ } else {
+ assertTrue(totalByteSize > 0);
+ assertTrue(offHeap.get("vdd") > 0L); // NOTE: different from vec
+ if (fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {
+ assertTrue(offHeap.get("vdqd") > 0L); // NOTE: different from veq
+ } else {
+ assertNull(offHeap.get("vdqd")); // BYTE and FLOAT16 fields are stored raw only
+ }
+
+ if (hasHNSW(knnVectorsReader, fieldInfo)) {
+ assertTrue(offHeap.get("vex") > 0L);
+ } else {
+ assertTrue(offHeap.get("vex") == null || offHeap.get("vex") == 0);
+ }
+ }
+ }
+ } else {
+ throw new AssertionError("unexpected:" + r.getClass());
+ }
+ }
+}
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java
new file mode 100644
index 00000000000..f9da057ae88
--- /dev/null
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.dedup;
+
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
+import static org.apache.lucene.index.VectorEncoding.BYTE;
+import static org.apache.lucene.index.VectorEncoding.FLOAT32;
+import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
+import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;
+import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT;
+import static org.hamcrest.Matchers.instanceOf;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnByteVectorField;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KnnVectorValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.AcceptDocs;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+
+/**
+ * Tests that {@link DedupHnswScalarQuantizedVectorsFormat} stores each distinct vector once, in
+ * both raw and quantized form. General de-duplication behavior is covered by {@link
+ * TestDedupFlatVectorsFormat}; this test focuses on the quantized side.
+ */
+public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
+
+ private static final ScalarEncoding ENCODING = ScalarEncoding.UNSIGNED_BYTE;
+
+ private static IndexWriterConfig config() {
+ return newIndexWriterConfig()
+ .setCodec(TestUtil.alwaysKnnVectorsFormat(new DedupHnswScalarQuantizedVectorsFormat()));
+ }
+
+ /** Size in bytes of one quantized record for the given dimension. */
+ private static long quantizedRecordSize(int dimension) {
+ return ENCODING.getDocPackedLength(dimension)
+ + DedupScalarQuantizedVectorValues.CORRECTIVE_BYTES;
+ }
+
+ /** Repeated vectors are stored once, raw and quantized, but still read back per document. */
+ public void testDuplicatesStoredOnce() throws Exception {
+ float[] a = {1, 2, 3, 4};
+ float[] b = {5, 6, 7, 8};
+ float[][] docVectors = {a, b, a, b, a, b}; // 3 copies each of 2 vectors
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ for (int ord = 0; ord < docVectors.length; ord++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", ord));
+ doc.add(new KnnFloatVectorField("f", docVectors[ord], EUCLIDEAN));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ FloatVectorValues values = leafReader.getFloatVectorValues("f");
+ assertEquals(docVectors.length, values.size()); // one entry per document
+ assertEquals(2, groupNumVectors(values)); // only two distinct vectors stored
+
+ // raw vectors read back exactly, resolved per document
+ for (int ord = 0; ord < values.size(); ord++) {
+ float[] vector = values.vectorValue(ord);
+ assertTrue(Arrays.equals(a, vector) || Arrays.equals(b, vector));
+ }
+
+ // the quantized block also holds exactly two records
+ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");
+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT32);
+ assertEquals(2 * quantizedRecordSize(a.length), entry.quantizedBlock().quantizedDataSize());
+ }
+ }
+ }
+
+ /**
+ * Fields with the same dimension and encoding share one raw copy of a vector; their quantized
+ * records are stored per {@link DedupQuantizer.Flavor}, shared across fields whose similarity
+ * functions map to the same flavor.
+ */
+ public void testDuplicatesAcrossFieldsShareQuantizedGroup() throws Exception {
+ float[] shared = {9, 8, 7, 6};
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField("f1", shared, EUCLIDEAN));
+ doc.add(new KnnFloatVectorField("f2", shared, DOT_PRODUCT)); // different function
+ doc.add(new KnnFloatVectorField("f3", shared, COSINE)); // different function
+ doc.add(new KnnFloatVectorField("f4", shared, MAXIMUM_INNER_PRODUCT)); // same flavor as f2
+ w.addDocument(doc);
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leaf = getOnlyLeafReader(reader);
+
+ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leaf, "f1");
+ assertEquals(sqReader, getQuantizedReader(leaf, "f2"));
+ assertEquals(sqReader, getQuantizedReader(leaf, "f3"));
+ assertEquals(sqReader, getQuantizedReader(leaf, "f4"));
+
+ // all fields resolve to the same raw group, with one distinct vector
+ DedupScalarQuantizedVectorsReader.FieldEntry e1 = sqReader.getEntry("f1", FLOAT32);
+ DedupScalarQuantizedVectorsReader.FieldEntry e2 = sqReader.getEntry("f2", FLOAT32);
+ DedupScalarQuantizedVectorsReader.FieldEntry e3 = sqReader.getEntry("f3", FLOAT32);
+ DedupScalarQuantizedVectorsReader.FieldEntry e4 = sqReader.getEntry("f4", FLOAT32);
+ assertEquals(e1.groupInfo(), e2.groupInfo());
+ assertEquals(e1.groupInfo(), e3.groupInfo());
+ assertEquals(e1.groupInfo(), e4.groupInfo());
+ assertEquals(1, e1.groupInfo().groupNumVectors());
+
+ // quantized data is split by flavor: EUCLIDEAN (f1), DOT_PRODUCT (f2 + f4, shared) and
+ // NORMALIZED (f3), one record each
+ assertNotEquals(e1.quantizedBlock(), e2.quantizedBlock());
+ assertNotEquals(e1.quantizedBlock(), e3.quantizedBlock());
+ assertNotEquals(e2.quantizedBlock(), e3.quantizedBlock());
+ assertEquals(e2.quantizedBlock(), e4.quantizedBlock());
+ for (DedupScalarQuantizedVectorsReader.FieldEntry entry : List.of(e1, e2, e3, e4)) {
+ assertEquals(
+ quantizedRecordSize(shared.length), entry.quantizedBlock().quantizedDataSize());
+ }
+
+ for (String field : new String[] {"f1", "f2", "f3", "f4"}) {
+ FloatVectorValues values = leaf.getFloatVectorValues(field);
+ assertEquals(1, groupNumVectors(values));
+ assertArrayEquals(shared, values.vectorValue(0), 0f);
+ }
+ }
+ }
+ }
+
+ /** Duplicates spanning multiple segments collapse to a single quantized copy when merged. */
+ public void testDuplicatesAcrossSegmentsDedupOnMerge() throws Exception {
+ float[] a = {1, 1, 1, 1};
+ float[] b = {2, 2, 2, 2};
+ float[][] docVectors = {a, b, a}; // 3 docs across 3 segments, 2 distinct
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ for (float[] docVector : docVectors) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField("f", docVector, EUCLIDEAN));
+ w.addDocument(doc);
+ w.commit(); // one segment per document
+ }
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ FloatVectorValues values = leafReader.getFloatVectorValues("f");
+ assertEquals(docVectors.length, values.size());
+ assertEquals(2, groupNumVectors(values)); // a's duplicate collapsed across segments
+
+ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");
+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT32);
+ assertEquals(2 * quantizedRecordSize(a.length), entry.quantizedBlock().quantizedDataSize());
+ }
+ }
+ }
+
+ /** Documents sharing a vector get identical scores, ranking above unrelated documents. */
+ public void testSearchDuplicatesShareScores() throws Exception {
+ int dimension = 16;
+ float[] a = randomVector(dimension);
+ float[] b = randomVector(dimension);
+ for (int i = 0; i < dimension; i++) {
+ b[i] = -a[i]; // far away from a
+ }
+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ int numA = 3, numB = 3;
+ for (int i = 0; i < numA; i++) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField("f", a, function));
+ w.addDocument(doc);
+ }
+ for (int i = 0; i < numB; i++) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField("f", b, function));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ TopDocs topDocs =
+ leafReader.searchNearestVectors(
+ "f",
+ a,
+ numA + numB,
+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
+ Integer.MAX_VALUE);
+ assertEquals(numA + numB, topDocs.scoreDocs.length);
+ // duplicates of a rank first with identical scores
+ float scoreA = topDocs.scoreDocs[0].score;
+ for (int i = 0; i < numA; i++) {
+ assertEquals("function=" + function, scoreA, topDocs.scoreDocs[i].score, 0f);
+ }
+ for (int i = numA; i < numA + numB; i++) {
+ ScoreDoc scoreDoc = topDocs.scoreDocs[i];
+ assertTrue("function=" + function, scoreDoc.score < scoreA);
+ }
+ }
+ }
+ }
+ }
+
+ /** Quantized scores are close to true scores for every similarity function. */
+ public void testScoreAccuracy() throws Exception {
+ int dimension = 16;
+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {
+ float[] docVector = randomVector(dimension);
+ float[] query = randomVector(dimension);
+ if (function == DOT_PRODUCT) { // requires normalized vectors
+ VectorUtil.l2normalize(docVector);
+ VectorUtil.l2normalize(query);
+ }
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField("f", docVector, function));
+ w.addDocument(doc);
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ float trueScore = function.compare(docVector, query);
+ TopDocs topDocs =
+ leafReader.searchNearestVectors(
+ "f",
+ query,
+ 1,
+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
+ Integer.MAX_VALUE);
+ assertEquals(1, topDocs.totalHits.value());
+ // NOTE: data-blind quantization is less accurate than centroid-centered quantization,
+ // so allow a small relative tolerance
+ float tolerance = Math.max(0.05f, 0.05f * Math.abs(trueScore));
+ assertEquals("function=" + function, trueScore, topDocs.scoreDocs[0].score, tolerance);
+ }
+ }
+ }
+ }
+
+ /**
+ * Cosine similarity ignores vector magnitude: since vectors are stored unnormalized (to share
+ * quantized bytes across fields), the norm-based correction must equalize scaled duplicates.
+ */
+ public void testCosineIgnoresMagnitude() throws Exception {
+ float[] small = {1, 2, 3, 4};
+ float[] large = new float[small.length];
+ for (int i = 0; i < small.length; i++) {
+ large[i] = small[i] * 1000;
+ }
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ for (float[] docVector : new float[][] {small, large}) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField("f", docVector, COSINE));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ TopDocs topDocs =
+ leafReader.searchNearestVectors(
+ "f",
+ small,
+ 2,
+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
+ Integer.MAX_VALUE);
+ assertEquals(2, topDocs.scoreDocs.length);
+ // both docs point in the same direction: cosine similarity ~1 -> score ~1
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ assertEquals(1f, scoreDoc.score, 0.01f);
+ }
+ }
+ }
+ }
+
+ /** BYTE fields pass through to raw de-duplicated storage, without a quantized block. */
+ public void testByteVectorsStoredRawOnly() throws Exception {
+ byte[] a = {1, 2, 3, 4};
+ byte[] b = {5, 6, 7, 8};
+ byte[][] docVectors = {a, b, a, b}; // 2 copies each of 2 vectors
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ for (byte[] docVector : docVectors) {
+ Document doc = new Document();
+ doc.add(new KnnByteVectorField("f", docVector, EUCLIDEAN));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ ByteVectorValues values = leafReader.getByteVectorValues("f");
+ assertEquals(docVectors.length, values.size());
+ assertEquals(2, groupNumVectors(values)); // still de-duplicated
+
+ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");
+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", BYTE);
+ assertNull(entry.quantizedBlock()); // no quantized data
+ }
+ }
+ }
+
+ /**
+ * Every encoding survives a merge with graph building forced (exercising the merge-time scorer
+ * suppliers, including the temporary query-vectors file of asymmetric encodings) and searches
+ * sanely afterwards.
+ */
+ public void testAllEncodingsMergeAndSearch() throws Exception {
+ int dimension = 32;
+ int numDistinct = 20;
+ float[][] distinct = new float[numDistinct][];
+ for (int i = 0; i < numDistinct; i++) {
+ distinct[i] = randomVector(dimension);
+ }
+ for (ScalarEncoding encoding : ScalarEncoding.values()) {
+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {
+ IndexWriterConfig config =
+ newIndexWriterConfig()
+ .setCodec(
+ TestUtil.alwaysKnnVectorsFormat(
+ new DedupHnswScalarQuantizedVectorsFormat(
+ encoding,
+ DEFAULT_MAX_CONN,
+ DEFAULT_BEAM_WIDTH,
+ 1,
+ null,
+ 0))); // always build graphs
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config)) {
+ for (int segment = 0; segment < 2; segment++) { // duplicates across segments
+ for (int i = 0; i < numDistinct; i++) {
+ Document doc = new Document();
+ float[] vector =
+ function == DOT_PRODUCT
+ ? VectorUtil.l2normalize(distinct[i].clone())
+ : distinct[i];
+ doc.add(new KnnFloatVectorField("f", vector, function));
+ w.addDocument(doc);
+ }
+ w.commit();
+ }
+ w.forceMerge(1); // exercises the merge-time graph construction scorer
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ FloatVectorValues values = leafReader.getFloatVectorValues("f");
+ assertEquals(2 * numDistinct, values.size());
+ assertEquals(numDistinct, groupNumVectors(values)); // de-duplicated across segments
+
+ float[] query =
+ function == DOT_PRODUCT ? VectorUtil.l2normalize(distinct[0].clone()) : distinct[0];
+ TopDocs topDocs =
+ leafReader.searchNearestVectors(
+ "f",
+ query,
+ 2,
+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
+ Integer.MAX_VALUE);
+ String context = "encoding=" + encoding + ", function=" + function;
+ assertEquals(context, 2, topDocs.scoreDocs.length);
+ // the two duplicates of the query vector share the (top) score
+ assertEquals(context, topDocs.scoreDocs[0].score, topDocs.scoreDocs[1].score, 0f);
+ }
+ }
+ }
+ }
+ }
+
+ private float[] randomVector(int dimension) {
+ float[] vector = new float[dimension];
+ for (int i = 0; i < dimension; i++) {
+ vector[i] = random().nextFloat(-1, 1);
+ }
+ return vector;
+ }
+
+ /** Number of distinct vectors physically stored for a field's group. */
+ private static int groupNumVectors(KnnVectorValues values) {
+ assertThat(values, instanceOf(DedupVectorValues.class));
+ return ((DedupVectorValues) values).getGroupView().size();
+ }
+
+ /** Get underlying quantized dedup vector reader instance. */
+ private static DedupScalarQuantizedVectorsReader getQuantizedReader(
+ LeafReader leafReader, String fieldName) {
+ assertThat(leafReader, instanceOf(CodecReader.class));
+ KnnVectorsReader knnVectorsReader = ((CodecReader) leafReader).getVectorReader();
+ knnVectorsReader = knnVectorsReader.unwrapReaderForField(fieldName);
+
+ assertThat(knnVectorsReader, instanceOf(Lucene99HnswVectorsReader.class));
+ FlatVectorsReader flatReader =
+ ((Lucene99HnswVectorsReader) knnVectorsReader).getFlatVectorsReader();
+
+ assertThat(flatReader, instanceOf(DedupScalarQuantizedVectorsReader.class));
+ return (DedupScalarQuantizedVectorsReader) flatReader;
+ }
+}