Apache Lucene open-source search software
Add scalar quantization support for FLOAT16 vectors in the dedup KNN codec (#16710)
Pranshu Shuklaaf9d10f5f1d0
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex d5415f70b59..a85c1f7b75c 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -172,6 +172,8 @@ New Features* GITHUB#16628: Add re-ranking and prefetch support for fp16 vectors. (Pulkit Gupta)+* GITHUB#16710: Add scalar quantization support for FLOAT16 vectors in the dedup KNN codec. (Pranshu Shukla)+Improvements---------------------* GITHUB#15704: Replace LinkedList with more efficient data structure. (Renato Haeberli)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.javaindex 011a62c1e07..e6f5aa4a16d 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java@@ -192,7 +192,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVeVectorUtil.int4Unpack(packed, unpacked);}- static OffHeapScalarQuantizedVectorValues load(+ /** Load off-heap scalar quantized vectors. */+ public static OffHeapScalarQuantizedVectorValues load(OrdToDocDISIReaderConfiguration configuration,int dimension,int size,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.javaindex e4c921ddee2..deaa1093b45 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.java@@ -226,4 +226,11 @@ public class OrdToDocDISIReaderConfiguration {public boolean isDense() {return docsWithFieldOffset == -1;}++ /**+ * @return the number of documents that have a value for the field+ */+ public int size() {+ return size;+ }}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.javaindex d92a474d325..7b05c202cb2 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java@@ -116,7 +116,8 @@ final class DedupFlushContext implements Accountable {Map<GroupKey, Set<DedupQuantizer.Flavor>> groupFlavors = new HashMap<>();if (quantizer != null) {for (FieldData fieldData : fieldDataList) {- if (fieldData.fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {+ VectorEncoding fieldEncoding = fieldData.fieldInfo.getVectorEncoding();+ if (fieldEncoding.isFloatingPoint()) {groupFlavors.computeIfAbsent(fieldData.groupKey, _ -> EnumSet.noneOf(DedupQuantizer.Flavor.class)).add(DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));@@ -157,6 +158,20 @@ final class DedupFlushContext implements Accountable {groupFlavors.get(groupKey),floatGroup::get,null);+ } else if (group instanceof Float16Group float16Group) {+ // FLOAT16 is stored raw as short[]; inflate to float[] for data-blind quantization. The+ // quantized record is a pure function of the float values and flavor, so it matches the+ // record a FLOAT32 vector of the same values would produce.+ float[] inflated = new float[dimension];+ quantizer.writeGroup(+ meta,+ quantizedVectorData,+ encoding,+ dimension,+ groupNumVectors,+ groupFlavors.get(groupKey),+ ord -> DedupUtil.inflateFloat16(float16Group.get(ord), inflated),+ null);} else {DedupQuantizer.writeEmptyGroup(meta);}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.javaindex 827cdba6003..fa0747d2f7f 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java@@ -135,11 +135,13 @@ final class DedupMergeContext implements Accountable {if (quantizer != null) {if (mergeGroup instanceof FloatGroup floatGroup) {+ // Flavors referenced by this group's fieldsSet<DedupQuantizer.Flavor> flavors = EnumSet.noneOf(DedupQuantizer.Flavor.class);for (FieldData fieldData : entry.getValue()) {flavors.add(DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));}+quantizer.writeGroup(meta,quantizedVectorData,@@ -149,6 +151,24 @@ final class DedupMergeContext implements Accountable {flavors,ord -> floatGroup.get(ord).get(),floatGroup::preQuantized);+ } else if (mergeGroup instanceof Float16Group float16Group) {+ // Flavors referenced by this group's fields+ Set<DedupQuantizer.Flavor> flavors = EnumSet.noneOf(DedupQuantizer.Flavor.class);+ for (FieldData fieldData : entry.getValue()) {+ flavors.add(+ DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));+ }+ // FLOAT16 is stored raw as short[]; inflate to float[] for data-blind quantization.+ float[] inflated = new float[dimension];+ quantizer.writeGroup(+ meta,+ quantizedVectorData,+ encoding,+ dimension,+ groupNumVectors,+ flavors,+ ord -> DedupUtil.inflateFloat16(float16Group.get(ord).get(), inflated),+ float16Group::preQuantized);} else {DedupQuantizer.writeEmptyGroup(meta);}@@ -298,7 +318,8 @@ final class DedupMergeContext implements Accountable {DedupQuantizer.PreQuantized preQuantized(int ord) {FloatVector handle = get(ord);if (handle.values()- instanceof DedupScalarQuantizedVectorValues.RawAndQuantizedValues rawAndQuantized) {+ instanceof+ DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues rawAndQuantized) {DedupScalarQuantizedVectorValues.FieldValues quantized =rawAndQuantized.getQuantizedValues();return new DedupQuantizer.PreQuantized(quantized, quantized.flavor(), handle.ord());@@ -373,6 +394,23 @@ final class DedupMergeContext implements Accountable {return new Float16Vector(sub.values, sub.iterator.index());}+ /**+ * The already-quantized record of the distinct vector at a group ordinal, when its source+ * segment is in this format (data-blind quantization is a pure function of the inflated raw+ * vector, so the record can be copied on merge instead of re-quantizing), or {@code null}.+ */+ DedupQuantizer.PreQuantized preQuantized(int ord) {+ Float16Vector handle = get(ord);+ if (handle.values()+ instanceof+ DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues rawAndQuantized) {+ DedupScalarQuantizedVectorValues.FieldValues quantized =+ rawAndQuantized.getQuantizedValues();+ return new DedupQuantizer.PreQuantized(quantized, quantized.flavor(), handle.ord());+ }+ return null;+ }+@Overridepublic long hash(Float16Vector vector) throws IOException {buffer.put(0, vector.get());diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.javaindex f764f0ef4b7..66065250d26 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java@@ -187,7 +187,10 @@ record DedupQuantizer(ScalarEncoding encoding) {* ordinal within each block. Wherever a {@code preQuantized} record with a matching encoding and* flavor is available (i.e. the vector originates from a segment in this format), it is copied* as-is; otherwise the raw vector is quantized. The block locations are written to {@code meta};- * non-FLOAT32 groups (stored raw only) record an empty list.+ * BYTE groups (stored raw only) record an empty list. FLOAT32 and FLOAT16 groups are both+ * quantized; callers supply {@code vectors} as {@code float[]} (FLOAT16 inflates its {@code+ * short[]} storage first), since the quantized record is a pure function of the {@code float[]}+ * values and the flavor, independent of the source encoding.*/void writeGroup(IndexOutput meta,@@ -200,7 +203,7 @@ record DedupQuantizer(ScalarEncoding encoding) {PreQuantizedSupplier preQuantized)throws IOException {- if (groupEncoding != VectorEncoding.FLOAT32) {+ if (groupEncoding.isFloatingPoint() == false) {writeEmptyGroup(meta);return;}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.javaindex b4acc025f82..e988ad6f5e1 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java@@ -16,13 +16,12 @@*/package org.apache.lucene.sandbox.codecs.dedup;-import static org.apache.lucene.index.VectorEncoding.FLOAT32;-import java.io.IOException;import org.apache.lucene.codecs.lucene104.OffHeapScalarQuantizedVectorValues;-import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;+import org.apache.lucene.index.Float16VectorValues;import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdOffHeap;@@ -80,6 +79,11 @@ final class DedupScalarQuantizedVectorValues {slice);}+ /**+ * Loads a field's quantized vector values. The {@code fieldView} only maps field ords to docs;+ * all vector reads, corrective terms, centroid and quantizer resolve through the shared group+ * view.+ */static FieldValues loadQuantized(DedupScalarQuantizedVectorsScorer vectorsScorer,VectorSimilarityFunction function,@@ -96,16 +100,29 @@ final class DedupScalarQuantizedVectorValues {int fieldOrdToGroupOrdBitsPerValue)throws IOException {- final OffHeapFloatVectorValues fieldView =- OffHeapFloatVectorValues.load(- function, vectorsScorer, configuration, FLOAT32, dimension, 0, 0, vectorData);+ final DedupQuantizer.Flavor flavor = DedupQuantizer.Flavor.of(function);++ final OffHeapScalarQuantizedVectorValues fieldView =+ OffHeapScalarQuantizedVectorValues.load(+ configuration,+ dimension,+ configuration.size(),+ flavor.quantizer(),+ encoding,+ function,+ vectorsScorer,+ new float[dimension], // always a zero centroid, quantization is data-blind+ 0,+ 0,+ 0,+ vectorData);final QuantizedByteVectorValues groupView =groupValues(false,vectorsScorer,function,- DedupQuantizer.Flavor.of(function),+ flavor,encoding,dimension,groupNumVectors,@@ -119,7 +136,8 @@ final class DedupScalarQuantizedVectorValues {fieldOrdToGroupOrdSize,fieldOrdToGroupOrdBitsPerValue);- return new FieldValues(vectorsScorer, function, fieldView, groupView, fieldOrdToGroupOrd);+ return new FieldValues(+ vectorsScorer, function, fieldView, groupView, fieldOrdToGroupOrd, configuration.isDense());}/**@@ -130,22 +148,25 @@ final class DedupScalarQuantizedVectorValues {static final class FieldValues extends QuantizedByteVectorValues implements DedupVectorValues {private final DedupScalarQuantizedVectorsScorer vectorsScorer;private final VectorSimilarityFunction function;- private final FloatVectorValues fieldView;+ private final KnnVectorValues fieldView;private final QuantizedByteVectorValues groupView;private final FieldOrdToGroupOrd fieldOrdToGroupOrd;+ private final boolean isDense;private int[] scratch;FieldValues(DedupScalarQuantizedVectorsScorer vectorsScorer,VectorSimilarityFunction function,- FloatVectorValues fieldView,+ KnnVectorValues fieldView,QuantizedByteVectorValues groupView,- FieldOrdToGroupOrd fieldOrdToGroupOrd) {+ FieldOrdToGroupOrd fieldOrdToGroupOrd,+ boolean isDense) {this.vectorsScorer = vectorsScorer;this.function = function;this.fieldView = fieldView;this.groupView = groupView;this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;+ this.isDense = isDense;this.scratch = new int[DedupUtil.SCRATCH_INITIAL_SIZE];}@@ -244,7 +265,12 @@ final class DedupScalarQuantizedVectorValues {@Overridepublic FieldValues copy() throws IOException {return new FieldValues(- vectorsScorer, function, fieldView.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());+ vectorsScorer,+ function,+ fieldView.copy(),+ groupView.copy(),+ fieldOrdToGroupOrd.copy(),+ isDense);}@Override@@ -255,7 +281,17 @@ final class DedupScalarQuantizedVectorValues {FieldValues copy = copy();DocIndexIterator indexIterator = copy.iterator();RandomVectorScorer vectorScorer = vectorsScorer.getRandomVectorScorer(function, copy, target);- boolean isDense = copy.fieldView instanceof OffHeapFloatVectorValues.DenseOffHeapVectorValues;+ return new DedupVectorScorer(indexIterator, vectorScorer, isDense);+ }++ @Override+ public VectorScorer scorer(short[] target) throws IOException {+ if (size() == 0) {+ return null;+ }+ FieldValues copy = copy();+ DocIndexIterator indexIterator = copy.iterator();+ RandomVectorScorer vectorScorer = vectorsScorer.getRandomVectorScorer(function, copy, target);return new DedupVectorScorer(indexIterator, vectorScorer, isDense);}}@@ -265,11 +301,13 @@ final class DedupScalarQuantizedVectorValues {* #scorer(float[])} scores against the quantized values instead. Mirrors {@code* Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedVectorValues}.*/- static final class RawAndQuantizedValues extends FloatVectorValues implements DedupVectorValues {+ static final class Float32RawAndQuantizedValues extends FloatVectorValues+ implements DedupVectorValues {private final DedupVectorValues.FloatImpl rawValues;private final FieldValues quantizedValues;- RawAndQuantizedValues(DedupVectorValues.FloatImpl rawValues, FieldValues quantizedValues) {+ Float32RawAndQuantizedValues(+ DedupVectorValues.FloatImpl rawValues, FieldValues quantizedValues) {this.rawValues = rawValues;this.quantizedValues = quantizedValues;}@@ -324,8 +362,8 @@ final class DedupScalarQuantizedVectorValues {}@Override- public RawAndQuantizedValues copy() throws IOException {- return new RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());+ public Float32RawAndQuantizedValues copy() throws IOException {+ return new Float32RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());}@Override@@ -338,4 +376,85 @@ final class DedupScalarQuantizedVectorValues {return rawValues.rescorer(target);}}++ /**+ * Full-precision view of a field backed by the raw de-duplicated vectors, whose {@link+ * #scorer(short[])} scores against the quantized values instead. Mirrors {@code+ * Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedFloat16VectorValues}.+ */+ static final class Float16RawAndQuantizedValues extends Float16VectorValues+ implements DedupVectorValues {+ private final DedupVectorValues.Float16Impl rawValues;+ private final FieldValues quantizedValues;++ Float16RawAndQuantizedValues(+ DedupVectorValues.Float16Impl rawValues, FieldValues quantizedValues) {+ this.rawValues = rawValues;+ this.quantizedValues = quantizedValues;+ }++ FieldValues getQuantizedValues() {+ return quantizedValues;+ }++ @Override+ public KnnVectorValues getGroupView() {+ return rawValues.getGroupView();+ }++ @Override+ public FieldOrdToGroupOrd getFieldOrdToGroupOrd() {+ return rawValues.getFieldOrdToGroupOrd();+ }++ @Override+ public int dimension() {+ return rawValues.dimension();+ }++ @Override+ public int size() {+ return rawValues.size();+ }++ @Override+ public int ordToDoc(int ord) {+ return rawValues.ordToDoc(ord);+ }++ @Override+ public Bits getAcceptOrds(Bits acceptDocs) {+ return rawValues.getAcceptOrds(acceptDocs);+ }++ @Override+ public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {+ rawValues.prefetch(ordsToPrefetch, numOrds);+ }++ @Override+ public short[] vectorValue(int ord) throws IOException {+ return rawValues.vectorValue(ord);+ }++ @Override+ public DocIndexIterator iterator() {+ return rawValues.iterator();+ }++ @Override+ public Float16RawAndQuantizedValues copy() throws IOException {+ return new Float16RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());+ }++ @Override+ public VectorScorer scorer(short[] target) throws IOException {+ return quantizedValues.scorer(target);+ }++ @Override+ public VectorScorer rescorer(short[] target) throws IOException {+ return rawValues.rescorer(target);+ }+ }}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.javaindex 5ecfb0cac55..6cdec559f1f 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java@@ -41,8 +41,11 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncod* tradeoff relative to that format, which centers vectors on a per-field centroid before* quantizing.*- * <p>Only {@link org.apache.lucene.index.VectorEncoding#FLOAT32} vectors are quantized; BYTE and- * FLOAT16 vectors are stored raw only, identical to {@link DedupFlatVectorsFormat}.+ * <p>Both {@link org.apache.lucene.index.VectorEncoding#FLOAT32} and {@link+ * org.apache.lucene.index.VectorEncoding#FLOAT16} vectors are quantized (FLOAT16 vectors are+ * inflated to {@code float} for the data-blind quantizer, and kept raw as {@code short[]} for+ * full-fidelity readback); BYTE vectors are stored raw only, identical to {@link+ * DedupFlatVectorsFormat}.** <h2>.vdd (vector de-dup data) file</h2>*@@ -51,8 +54,8 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncod** <h2>.vdqd (vector de-dup quantized data) file</h2>*- * <p>One block per FLOAT32 group and {@link DedupQuantizer.Flavor} in use (aligned to {@code- * Float.BYTES}), holding one quantized record per distinct vector:+ * <p>One block per quantized (FLOAT32 or FLOAT16) group and {@link DedupQuantizer.Flavor} in use+ * (aligned to {@code Float.BYTES}), holding one quantized record per distinct vector:** <ul>* <li><b>[byte]</b> the packed quantized values (layout depends on the {@link ScalarEncoding})@@ -69,7 +72,7 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncod* additionally recording its flavor blocks:** <ul>- * <li><b>[vint]</b> the number of flavor blocks ({@code 0} for non-FLOAT32 groups)+ * <li><b>[vint]</b> the number of flavor blocks ({@code 0} for BYTE groups)* <li>per flavor block: <b>[vint]</b> the flavor wire number, <b>[vint]</b> the {@link* ScalarEncoding} wire number of its records, <b>[int64]</b> offset and <b>[int64]</b> length* of its quantized records in the .vdqd filediff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.javaindex 66ed85c7208..ca22c1c8a31 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java@@ -44,7 +44,8 @@ import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.sandbox.codecs.dedup.DedupQuantizer.QuantizedBlock;import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;-import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues;import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;import org.apache.lucene.store.ChecksumIndexInput;@@ -66,10 +67,10 @@ import org.apache.lucene.util.quantization.QuantizedVectorsReader;import org.apache.lucene.util.quantization.ScalarQuantizer;/**- * Reads de-duplicated flat vectors written by {@link DedupScalarQuantizedVectorsFormat}. Each- * FLOAT32 field exposes a full-precision view backed by the raw de-duplicated vectors, scored- * against a data-blind quantized view sharing the same {@code fieldOrdToGroupOrd} translation map.- * BYTE and FLOAT16 fields are stored and read raw only.+ * Reads de-duplicated flat vectors written by {@link DedupScalarQuantizedVectorsFormat}. FLOAT32+ * and FLOAT16 fields each expose a full-precision view backed by the raw de-duplicated vectors,+ * scored against a data-blind quantized view sharing the same {@code fieldOrdToGroupOrd}+ * translation map. BYTE fields are stored and read raw only.** @lucene.experimental*/@@ -199,7 +200,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderQuantizedGroupInfo quantizedGroupInfo = groupInfos.get(fieldInfo.groupOrd());GroupInfo groupInfo = quantizedGroupInfo.groupInfo();QuantizedBlock quantizedBlock = null;- if (fieldInfo.encoding() == FLOAT32) {+ if (fieldInfo.encoding().isFloatingPoint()) {DedupQuantizer.Flavor flavor = DedupQuantizer.Flavor.of(fieldInfo.function());quantizedBlock = quantizedGroupInfo.quantizedBlocks().get(flavor);if (quantizedBlock == null) {@@ -232,7 +233,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderGroupInfo groupInfo,Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks)throws IOException {- if (groupInfo.encoding() != FLOAT32) {+ if (groupInfo.encoding().isFloatingPoint() == false) {if (quantizedBlocks.isEmpty() == false) {throw new CorruptIndexException("Unexpected quantized data for encoding=" + groupInfo.encoding(), meta);@@ -343,8 +344,9 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader@Overridepublic RandomVectorScorer getRandomVectorScorer(String field, short[] target) throws IOException {FieldEntry entry = getEntry(field, FLOAT16);- Float16VectorValues vectorValues = getFloat16VectorValues(entry);- return vectorsScorer.getRandomVectorScorer(entry.fieldInfo().function(), vectorValues, target);+ FieldValues quantizedValues = getQuantizedVectorValues(entry);+ return vectorsScorer.getRandomVectorScorer(+ entry.fieldInfo().function(), quantizedValues, target);}@Override@@ -388,7 +390,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader@Overridepublic FloatVectorValues getFloatVectorValues(String field) throws IOException {FieldEntry entry = getEntry(field, FLOAT32);- return new RawAndQuantizedValues(+ return new Float32RawAndQuantizedValues(getRawFloatVectorValues(entry), getQuantizedVectorValues(entry));}@@ -412,7 +414,8 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderreturn getByteVectorValues(getEntry(field, BYTE));}- private Float16VectorValues getFloat16VectorValues(FieldEntry entry) throws IOException {+ private DedupVectorValues.Float16Impl getRawFloat16VectorValues(FieldEntry entry)+ throws IOException {return loadDedupFloat16s(vectorsScorer,entry.fieldInfo().function(),@@ -429,12 +432,19 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader@Overridepublic Float16VectorValues getFloat16VectorValues(String field) throws IOException {- return getFloat16VectorValues(getEntry(field, FLOAT16));+ FieldEntry entry = getEntry(field, FLOAT16);+ return new Float16RawAndQuantizedValues(+ getRawFloat16VectorValues(entry), getQuantizedVectorValues(entry));}@Override- public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {- FieldEntry entry = getEntry(field, FLOAT32);+ public FieldValues getQuantizedVectorValues(String field) throws IOException {+ FieldEntry entry = fields.get(field);+ if (entry == null) {+ throw new IllegalArgumentException("field=" + field + " not found");+ } else if (entry.fieldInfo().encoding().isFloatingPoint() == false) {+ throw new IllegalArgumentException("field=" + field + " not indexed as floating point");+ }return getQuantizedVectorValues(entry);}@@ -447,8 +457,8 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderpublic CloseableRandomVectorScorerSupplier getRandomVectorScorerSupplierForMerge(FieldInfo fieldInfo, SegmentWriteState segmentWriteState) throws IOException {FieldEntry entry = fields.get(fieldInfo.name);- if (entry == null || entry.fieldInfo().encoding() != FLOAT32) {- // BYTE and FLOAT16 fields are stored raw only+ if (entry == null || entry.fieldInfo().encoding().isFloatingPoint() == false) {+ // Not a quantized field (missing, or e.g. BYTE stored raw only)return null;}FieldValues quantizedValues = getQuantizedVectorValues(entry);@@ -462,9 +472,18 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader// Asymmetric encodings compare query-encoded vectors against the stored doc-encoded vectors.// Write a query-encoded record per distinct vector of the group into a temporary file; both- // sides then resolve through the shared fieldOrdToGroupOrd translation.- DedupVectorValues.FloatImpl rawValues = getRawFloatVectorValues(entry);- FloatVectorValues groupView = rawValues.getGroupView();+ // sides then resolve through the shared fieldOrdToGroupOrd translation. The group's raw vectors+ // are read directly from the group view (indexed by group ordinal, not field ordinal) as+ // float[] (FLOAT16 groups are inflated from short[]).+ FloatVectorValues floatVectorValues;+ if (fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT16) {+ DedupVectorValues.Float16Impl float16RawValues = getRawFloat16VectorValues(entry);+ floatVectorValues = new Float16AsFloatVectorValues(float16RawValues.getGroupView());+ } else {+ DedupVectorValues.FloatImpl float32RawValues = getRawFloatVectorValues(entry);+ floatVectorValues = float32RawValues.getGroupView();+ }+int dimension = entry.fieldInfo().dimension();int groupNumVectors = entry.groupInfo().groupNumVectors();DedupQuantizer.Flavor flavor =@@ -483,7 +502,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderbyte[] toQuery = new byte[encoding.getQueryPackedLength(dimension)];for (int groupOrd = 0; groupOrd < groupNumVectors; groupOrd++) {// copy: normalization and quantization mutate the input, which is a shared buffer- System.arraycopy(groupView.vectorValue(groupOrd), 0, vector, 0, dimension);+ System.arraycopy(floatVectorValues.vectorValue(groupOrd), 0, vector, 0, dimension);if (flavor.normalized()) {VectorUtil.l2normalize(vector);}@@ -590,4 +609,48 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderprivate record QuantizedGroupInfo(GroupInfo groupInfo, Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks) {}++ /**+ * Exposes a {@link Float16VectorValues} as {@link FloatVectorValues}, inflating fp16 to fp32 on+ * read, so the merge path can reuse the fp32 quantization classes.+ */+ private static final class Float16AsFloatVectorValues extends FloatVectorValues {+ private final Float16VectorValues values;+ private final float[] floatVector;++ Float16AsFloatVectorValues(Float16VectorValues values) {+ this.values = values;+ this.floatVector = new float[values.dimension()];+ }++ @Override+ public int dimension() {+ return values.dimension();+ }++ @Override+ public int size() {+ return values.size();+ }++ @Override+ public int ordToDoc(int ord) {+ return values.ordToDoc(ord);+ }++ @Override+ public float[] vectorValue(int ord) throws IOException {+ return DedupUtil.inflateFloat16(values.vectorValue(ord), floatVector);+ }++ @Override+ public DocIndexIterator iterator() {+ return values.iterator();+ }++ @Override+ public Float16AsFloatVectorValues copy() throws IOException {+ return new Float16AsFloatVectorValues(values.copy());+ }+ }}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.javaindex b4aa648a873..94117f55a20 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java@@ -21,7 +21,8 @@ import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer;import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;-import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues;+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues;import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;import org.apache.lucene.util.quantization.QuantizedByteVectorValues;@@ -41,10 +42,13 @@ final class DedupScalarQuantizedVectorsScorer extends DedupFlatVectorsScorer {super(QUANTIZED_SCORER);}- /** Full-precision views resolve to their quantized values for scoring. */+ /** Full-precision views (float32 or float16) resolve to their quantized values for scoring. */@Overrideprotected KnnVectorValues unwrap(KnnVectorValues vectorValues) {- if (vectorValues instanceof RawAndQuantizedValues rawAndQuantized) {+ if (vectorValues instanceof Float32RawAndQuantizedValues rawAndQuantized) {+ return rawAndQuantized.getQuantizedValues();+ }+ if (vectorValues instanceof Float16RawAndQuantizedValues rawAndQuantized) {return rawAndQuantized.getQuantizedValues();}return vectorValues;diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.javaindex 70981af05c5..5bf14670164 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java@@ -212,6 +212,19 @@ final class DedupUtil {return murmurhash3_x64_128(bytes, 0, bytes.length, GOOD_FAST_HASH_SEED)[0];}+ /**+ * Inflates a float16 vector ({@code short[]}) into the reused {@code float[]} buffer for the+ * fp32-based scalar quantizer.+ */+ static float[] inflateFloat16(short[] float16Vector, float[] dest) {+ assert float16Vector.length == dest.length+ : "length mismatch: float16Vector=" + float16Vector.length + " dest=" + dest.length;+ for (int i = 0; i < float16Vector.length; i++) {+ dest[i] = Float.float16ToFloat(float16Vector[i]);+ }+ return dest;+ }+static long alignBytes(IndexOutput output, VectorEncoding encoding) throws IOException {int alignBytes =switch (encoding) {diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.javaindex fffe388ff6d..bf7526ba42c 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java@@ -58,7 +58,8 @@ sealed interface DedupVectorValuesDedupVectorValues.FloatImpl,DedupVectorValues.Float16Impl,DedupScalarQuantizedVectorValues.FieldValues,- DedupScalarQuantizedVectorValues.RawAndQuantizedValues {+ DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues,+ DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues {/** The dense view over distinct vectors, indexed by group ordinal. */KnnVectorValues getGroupView();diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.javaindex 0752b9e2d13..66b5fd6503b 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java@@ -26,8 +26,8 @@* vectors; see {@link org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat} for the* on-disk layout.* <li>{@link org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormat}- * additionally stores a scalar quantized copy of each distinct FLOAT32 vector, used for- * scoring; see {@link+ * additionally stores a scalar quantized copy of each distinct FLOAT32 or FLOAT16 vector,+ * used for scoring; see {@link* org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorsFormat} for the on-disk* layout.* </ul>diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.javaindex 113a95b66a4..10ec9fba2d9 100644--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java@@ -26,7 +26,6 @@ import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.LeafReader;-import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.SameThreadExecutorService;import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;@@ -92,10 +91,10 @@ public class TestDedupHnswScalarQuantizedVectorsFormat extends TestDedupHnswVect} else {assertTrue(totalByteSize > 0);assertTrue(offHeap.get("vdd") > 0L); // NOTE: different from vec- if (fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {+ if (fieldInfo.getVectorEncoding().isFloatingPoint()) {assertTrue(offHeap.get("vdqd") > 0L); // NOTE: different from veq} else {- assertNull(offHeap.get("vdqd")); // BYTE and FLOAT16 fields are stored raw only+ assertNull(offHeap.get("vdqd")); // BYTE fields are stored raw only}if (hasHNSW(knnVectorsReader, fieldInfo)) {diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.javaindex f9da057ae88..53b459442f3 100644--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java@@ -19,6 +19,7 @@ package org.apache.lucene.sandbox.codecs.dedup;import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;import static org.apache.lucene.index.VectorEncoding.BYTE;+import static org.apache.lucene.index.VectorEncoding.FLOAT16;import static org.apache.lucene.index.VectorEncoding.FLOAT32;import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;@@ -33,11 +34,13 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsReader;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnByteVectorField;+import org.apache.lucene.document.KnnFloat16VectorField;import org.apache.lucene.document.KnnFloatVectorField;import org.apache.lucene.document.NumericDocValuesField;import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.Float16VectorValues;import org.apache.lucene.index.FloatVectorValues;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;@@ -51,6 +54,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.VectorUtil;+import org.apache.lucene.util.hnsw.HnswGraph;import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;/**@@ -62,9 +66,27 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {private static final ScalarEncoding ENCODING = ScalarEncoding.UNSIGNED_BYTE;+ /**+ * Build an index writer config whose dedup format always builds the HNSW graph.+ *+ * <p>The default {@code tinySegmentsThreshold} ({@link+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#HNSW_GRAPH_THRESHOLD} = 100) lets+ * small segments skip graph construction and store vectors flat only. These tests use tiny+ * deterministic datasets, so we pin the threshold to {@code 0} ("always build the graph") rather+ * than inflating document counts, guaranteeing the merge-time graph construction and search paths+ * are exercised.+ */private static IndexWriterConfig config() {return newIndexWriterConfig()- .setCodec(TestUtil.alwaysKnnVectorsFormat(new DedupHnswScalarQuantizedVectorsFormat()));+ .setCodec(+ TestUtil.alwaysKnnVectorsFormat(+ new DedupHnswScalarQuantizedVectorsFormat(+ ENCODING,+ DEFAULT_MAX_CONN,+ DEFAULT_BEAM_WIDTH,+ 1,+ null,+ 0))); // tinySegmentsThreshold=0 => always build the HNSW graph}/** Size in bytes of one quantized record for the given dimension. */@@ -180,6 +202,7 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {FloatVectorValues values = leafReader.getFloatVectorValues("f");assertEquals(docVectors.length, values.size());assertEquals(2, groupNumVectors(values)); // a's duplicate collapsed across segments+ assertGraphBuilt(leafReader, "f"); // merge built the HNSW graph, not the tiny shortcutDedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT32);@@ -221,6 +244,7 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),Integer.MAX_VALUE);assertEquals(numA + numB, topDocs.scoreDocs.length);+ assertGraphBuilt(leafReader, "f"); // search ran over a real HNSW graph// duplicates of a rank first with identical scoresfloat scoreA = topDocs.scoreDocs[0].score;for (int i = 0; i < numA; i++) {@@ -333,6 +357,107 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {}}+ /**+ * FLOAT16 fields are quantized like FLOAT32: repeated vectors are stored once (raw and+ * quantized), the raw {@code short[]} still reads back per document, and a quantized block is+ * present.+ */+ public void testFloat16DuplicatesStoredOnceQuantized() throws Exception {+ short[] a = toFloat16(new float[] {1, 2, 3, 4});+ short[] b = toFloat16(new float[] {5, 6, 7, 8});+ short[][] docVectors = {a, b, a, b, a, b}; // 3 copies each of 2 vectors+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ for (short[] docVector : docVectors) {+ Document doc = new Document();+ doc.add(new KnnFloat16VectorField("f", docVector, EUCLIDEAN));+ w.addDocument(doc);+ }+ w.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ Float16VectorValues values = leafReader.getFloat16VectorValues("f");+ assertEquals(docVectors.length, values.size()); // one entry per document+ assertEquals(2, groupNumVectors(values)); // only two distinct vectors stored++ // raw float16 vectors read back exactly, resolved per document+ for (int ord = 0; ord < values.size(); ord++) {+ short[] vector = values.vectorValue(ord);+ assertTrue(Arrays.equals(a, vector) || Arrays.equals(b, vector));+ }++ // the quantized block holds exactly two records+ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT16);+ assertNotNull(entry.quantizedBlock());+ assertEquals(2 * quantizedRecordSize(a.length), entry.quantizedBlock().quantizedDataSize());+ }+ }+ }++ /**+ * FLOAT16 duplicates that span multiple segments collapse to a single quantized copy on merge,+ * and searches rank the query's duplicates together at the top.+ */+ public void testFloat16DuplicatesAcrossSegmentsAndSearch() throws Exception {+ int dimension = 16;+ float[] af = randomVector(dimension);+ float[] bf = new float[dimension];+ for (int i = 0; i < dimension; i++) {+ bf[i] = -af[i]; // far away from a+ }+ short[] a = toFloat16(af);+ short[] b = toFloat16(bf);+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config())) {+ int numA = 3, numB = 3;+ for (int i = 0; i < numA; i++) {+ Document doc = new Document();+ doc.add(new KnnFloat16VectorField("f", a, EUCLIDEAN));+ w.addDocument(doc);+ w.commit(); // one segment per document+ }+ for (int i = 0; i < numB; i++) {+ Document doc = new Document();+ doc.add(new KnnFloat16VectorField("f", b, EUCLIDEAN));+ w.addDocument(doc);+ w.commit();+ }+ w.forceMerge(1); // exercises FP16 merge-time quantized dedup + scorer supplier+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ Float16VectorValues values = leafReader.getFloat16VectorValues("f");+ assertEquals(numA + numB, values.size());+ assertEquals(2, groupNumVectors(values)); // collapsed across segments+ assertGraphBuilt(leafReader, "f"); // merge built the HNSW graph, not the tiny shortcut++ TopDocs topDocs =+ leafReader.searchNearestVectors(+ "f",+ a,+ numA + numB,+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),+ Integer.MAX_VALUE);+ assertEquals(numA + numB, topDocs.scoreDocs.length);+ float scoreA = topDocs.scoreDocs[0].score;+ for (int i = 0; i < numA; i++) {+ assertEquals(scoreA, topDocs.scoreDocs[i].score, 0f); // duplicates share a score+ }+ for (int i = numA; i < numA + numB; i++) {+ assertTrue(topDocs.scoreDocs[i].score < scoreA);+ }+ }+ }+ }++ private static short[] toFloat16(float[] vector) {+ short[] out = new short[vector.length];+ for (int i = 0; i < vector.length; i++) {+ out[i] = Float.floatToFloat16(vector[i]);+ }+ return out;+ }+/*** Every encoding survives a merge with graph building forced (exercising the merge-time scorer* suppliers, including the temporary query-vectors file of asymmetric encodings) and searches@@ -379,6 +504,7 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {FloatVectorValues values = leafReader.getFloatVectorValues("f");assertEquals(2 * numDistinct, values.size());assertEquals(numDistinct, groupNumVectors(values)); // de-duplicated across segments+ assertGraphBuilt(leafReader, "f"); // threshold=0 forced the merge-time graph buildfloat[] query =function == DOT_PRODUCT ? VectorUtil.l2normalize(distinct[0].clone()) : distinct[0];@@ -399,6 +525,148 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {}}+ /**+ * Every encoding survives a merge of FLOAT16 fields with graph building forced (exercising the+ * merge-time scorer suppliers, including the temporary query-vectors file of asymmetric encodings+ * and the FLOAT16 {@code short[]} -> {@code float[]} inflation on the raw side) and searches+ * sanely afterwards.+ */+ public void testFloat16AllEncodingsMergeAndSearch() throws Exception {+ int dimension = 32;+ int numDistinct = 20;+ float[][] distinct = new float[numDistinct][];+ for (int i = 0; i < numDistinct; i++) {+ distinct[i] = randomVector(dimension);+ }+ for (ScalarEncoding encoding : ScalarEncoding.values()) {+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {+ IndexWriterConfig config =+ newIndexWriterConfig()+ .setCodec(+ TestUtil.alwaysKnnVectorsFormat(+ new DedupHnswScalarQuantizedVectorsFormat(+ encoding,+ DEFAULT_MAX_CONN,+ DEFAULT_BEAM_WIDTH,+ 1,+ null,+ 0))); // always build graphs+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config)) {+ for (int segment = 0; segment < 2; segment++) { // duplicates across segments+ for (int i = 0; i < numDistinct; i++) {+ Document doc = new Document();+ float[] vector =+ function == DOT_PRODUCT+ ? VectorUtil.l2normalize(distinct[i].clone())+ : distinct[i];+ doc.add(new KnnFloat16VectorField("f", toFloat16(vector), function));+ w.addDocument(doc);+ }+ w.commit();+ }+ w.forceMerge(1); // exercises the merge-time graph construction scorer++ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ Float16VectorValues values = leafReader.getFloat16VectorValues("f");+ assertEquals(2 * numDistinct, values.size());+ assertEquals(numDistinct, groupNumVectors(values)); // de-duplicated across segments+ assertGraphBuilt(leafReader, "f"); // threshold=0 forced the merge-time graph build++ float[] queryFloat =+ function == DOT_PRODUCT ? VectorUtil.l2normalize(distinct[0].clone()) : distinct[0];+ short[] query = toFloat16(queryFloat);+ TopDocs topDocs =+ leafReader.searchNearestVectors(+ "f",+ query,+ 2,+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),+ Integer.MAX_VALUE);+ String context = "encoding=" + encoding + ", function=" + function;+ assertEquals(context, 2, topDocs.scoreDocs.length);+ // the two duplicates of the query vector share the (top) score+ assertEquals(context, topDocs.scoreDocs[0].score, topDocs.scoreDocs[1].score, 0f);+ }+ }+ }+ }+ }++ public void testFloat16SharedGroupAsymmetricMerge() throws Exception {+ int dimension = 32;+ int smallCount = 2;+ int bigCount = 20;+ float[][] small = new float[smallCount][];+ float[][] big = new float[bigCount][];+ for (int i = 0; i < smallCount; i++) {+ small[i] = randomVector(dimension);+ }+ for (int i = 0; i < bigCount; i++) {+ big[i] = randomVector(dimension);+ }+ for (ScalarEncoding encoding : ScalarEncoding.values()) {+ if (encoding.isAsymmetric() == false) {+ continue; // the EOF only reproduces on the asymmetric merge path+ }+ IndexWriterConfig config =+ newIndexWriterConfig()+ .setCodec(+ TestUtil.alwaysKnnVectorsFormat(+ new DedupHnswScalarQuantizedVectorsFormat(+ encoding,+ DEFAULT_MAX_CONN,+ DEFAULT_BEAM_WIDTH,+ 1,+ null,+ 0))); // always build graphs+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config)) {+ for (int segment = 0; segment < 2; segment++) { // duplicates across segments+ for (int i = 0; i < smallCount; i++) {+ Document d = new Document();+ d.add(new KnnFloat16VectorField("small", toFloat16(small[i]), EUCLIDEAN));+ w.addDocument(d);+ }+ for (int i = 0; i < bigCount; i++) {+ Document d = new Document();+ d.add(new KnnFloat16VectorField("big", toFloat16(big[i]), EUCLIDEAN));+ w.addDocument(d);+ }+ w.commit();+ }+ // "small" has smallCount*2 doc-entries but shares a group of smallCount+bigCount distinct+ // vectors; merging must map its field ords into the larger group without overrun.+ w.forceMerge(1);++ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader leafReader = getOnlyLeafReader(reader);+ String context = "encoding=" + encoding;++ Float16VectorValues smallValues = leafReader.getFloat16VectorValues("small");+ assertEquals(context, 2 * smallCount, smallValues.size());+ Float16VectorValues bigValues = leafReader.getFloat16VectorValues("big");+ assertEquals(context, 2 * bigCount, bigValues.size());+ assertGraphBuilt(leafReader, "small"); // threshold=0 forced the merge-time graph build+ assertGraphBuilt(leafReader, "big");++ // search the small field: its two duplicates of the query share the top score+ short[] q = toFloat16(small[0]);+ TopDocs topDocs =+ leafReader.searchNearestVectors(+ "small",+ q,+ 2,+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),+ Integer.MAX_VALUE);+ assertEquals(context, 2, topDocs.scoreDocs.length);+ assertEquals(context, topDocs.scoreDocs[0].score, topDocs.scoreDocs[1].score, 0f);+ }+ }+ }+ }+private float[] randomVector(int dimension) {float[] vector = new float[dimension];for (int i = 0; i < dimension; i++) {@@ -413,6 +681,22 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {return ((DedupVectorValues) values).getGroupView().size();}+ /**+ * Asserts an HNSW graph was actually built for the field (i.e. the tiny-segment shortcut was not+ * taken). A skipped graph yields {@link org.apache.lucene.util.hnsw.HnswGraph#EMPTY} with size 0.+ */+ private static void assertGraphBuilt(LeafReader leafReader, String field) throws Exception {+ assertThat(leafReader, instanceOf(CodecReader.class));+ KnnVectorsReader knnVectorsReader = ((CodecReader) leafReader).getVectorReader();+ KnnVectorsReader perField = knnVectorsReader.unwrapReaderForField(field);+ assertThat(perField, instanceOf(Lucene99HnswVectorsReader.class));+ HnswGraph graph = ((Lucene99HnswVectorsReader) perField).getGraph(field);+ assertNotNull("no graph for field=" + field, graph);+ assertTrue(+ "expected an HNSW graph to be built for field=" + field + " but it was empty",+ graph.size() > 0);+ }+/** Get underlying quantized dedup vector reader instance. */private static DedupScalarQuantizedVectorsReader getQuantizedReader(LeafReader leafReader, String fieldName) {