Skip to content

Apache Lucene open-source search software

Add scalar quantization support for FLOAT16 vectors in the dedup KNN codec (#16710)

Pranshu Shuklaaf9d10f5f1d0
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d5415f70b59..a85c1f7b75c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -172,6 +172,8 @@ New Features
* GITHUB#16628: Add re-ranking and prefetch support for fp16 vectors. (Pulkit Gupta)
+* GITHUB#16710: Add scalar quantization support for FLOAT16 vectors in the dedup KNN codec. (Pranshu Shukla)
+
Improvements
---------------------
* GITHUB#15704: Replace LinkedList with more efficient data structure. (Renato Haeberli)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
index 011a62c1e07..e6f5aa4a16d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
@@ -192,7 +192,8 @@ public abstract class OffHeapScalarQuantizedVectorValues extends QuantizedByteVe
VectorUtil.int4Unpack(packed, unpacked);
}
- static OffHeapScalarQuantizedVectorValues load(
+ /** Load off-heap scalar quantized vectors. */
+ public static OffHeapScalarQuantizedVectorValues load(
OrdToDocDISIReaderConfiguration configuration,
int dimension,
int size,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.java
index e4c921ddee2..deaa1093b45 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OrdToDocDISIReaderConfiguration.java
@@ -226,4 +226,11 @@ public class OrdToDocDISIReaderConfiguration {
public boolean isDense() {
return docsWithFieldOffset == -1;
}
+
+ /**
+ * @return the number of documents that have a value for the field
+ */
+ public int size() {
+ return size;
+ }
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java
index d92a474d325..7b05c202cb2 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlushContext.java
@@ -116,7 +116,8 @@ final class DedupFlushContext implements Accountable {
Map<GroupKey, Set<DedupQuantizer.Flavor>> groupFlavors = new HashMap<>();
if (quantizer != null) {
for (FieldData fieldData : fieldDataList) {
- if (fieldData.fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {
+ VectorEncoding fieldEncoding = fieldData.fieldInfo.getVectorEncoding();
+ if (fieldEncoding.isFloatingPoint()) {
groupFlavors
.computeIfAbsent(fieldData.groupKey, _ -> EnumSet.noneOf(DedupQuantizer.Flavor.class))
.add(DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));
@@ -157,6 +158,20 @@ final class DedupFlushContext implements Accountable {
groupFlavors.get(groupKey),
floatGroup::get,
null);
+ } else if (group instanceof Float16Group float16Group) {
+ // FLOAT16 is stored raw as short[]; inflate to float[] for data-blind quantization. The
+ // quantized record is a pure function of the float values and flavor, so it matches the
+ // record a FLOAT32 vector of the same values would produce.
+ float[] inflated = new float[dimension];
+ quantizer.writeGroup(
+ meta,
+ quantizedVectorData,
+ encoding,
+ dimension,
+ groupNumVectors,
+ groupFlavors.get(groupKey),
+ ord -> DedupUtil.inflateFloat16(float16Group.get(ord), inflated),
+ null);
} else {
DedupQuantizer.writeEmptyGroup(meta);
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java
index 827cdba6003..fa0747d2f7f 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupMergeContext.java
@@ -135,11 +135,13 @@ final class DedupMergeContext implements Accountable {
if (quantizer != null) {
if (mergeGroup instanceof FloatGroup floatGroup) {
+ // Flavors referenced by this group's fields
Set<DedupQuantizer.Flavor> flavors = EnumSet.noneOf(DedupQuantizer.Flavor.class);
for (FieldData fieldData : entry.getValue()) {
flavors.add(
DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));
}
+
quantizer.writeGroup(
meta,
quantizedVectorData,
@@ -149,6 +151,24 @@ final class DedupMergeContext implements Accountable {
flavors,
ord -> floatGroup.get(ord).get(),
floatGroup::preQuantized);
+ } else if (mergeGroup instanceof Float16Group float16Group) {
+ // Flavors referenced by this group's fields
+ Set<DedupQuantizer.Flavor> flavors = EnumSet.noneOf(DedupQuantizer.Flavor.class);
+ for (FieldData fieldData : entry.getValue()) {
+ flavors.add(
+ DedupQuantizer.Flavor.of(fieldData.fieldInfo.getVectorSimilarityFunction()));
+ }
+ // FLOAT16 is stored raw as short[]; inflate to float[] for data-blind quantization.
+ float[] inflated = new float[dimension];
+ quantizer.writeGroup(
+ meta,
+ quantizedVectorData,
+ encoding,
+ dimension,
+ groupNumVectors,
+ flavors,
+ ord -> DedupUtil.inflateFloat16(float16Group.get(ord).get(), inflated),
+ float16Group::preQuantized);
} else {
DedupQuantizer.writeEmptyGroup(meta);
}
@@ -298,7 +318,8 @@ final class DedupMergeContext implements Accountable {
DedupQuantizer.PreQuantized preQuantized(int ord) {
FloatVector handle = get(ord);
if (handle.values()
- instanceof DedupScalarQuantizedVectorValues.RawAndQuantizedValues rawAndQuantized) {
+ instanceof
+ DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues rawAndQuantized) {
DedupScalarQuantizedVectorValues.FieldValues quantized =
rawAndQuantized.getQuantizedValues();
return new DedupQuantizer.PreQuantized(quantized, quantized.flavor(), handle.ord());
@@ -373,6 +394,23 @@ final class DedupMergeContext implements Accountable {
return new Float16Vector(sub.values, sub.iterator.index());
}
+ /**
+ * The already-quantized record of the distinct vector at a group ordinal, when its source
+ * segment is in this format (data-blind quantization is a pure function of the inflated raw
+ * vector, so the record can be copied on merge instead of re-quantizing), or {@code null}.
+ */
+ DedupQuantizer.PreQuantized preQuantized(int ord) {
+ Float16Vector handle = get(ord);
+ if (handle.values()
+ instanceof
+ DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues rawAndQuantized) {
+ DedupScalarQuantizedVectorValues.FieldValues quantized =
+ rawAndQuantized.getQuantizedValues();
+ return new DedupQuantizer.PreQuantized(quantized, quantized.flavor(), handle.ord());
+ }
+ return null;
+ }
+
@Override
public long hash(Float16Vector vector) throws IOException {
buffer.put(0, vector.get());
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java
index f764f0ef4b7..66065250d26 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupQuantizer.java
@@ -187,7 +187,10 @@ record DedupQuantizer(ScalarEncoding encoding) {
* ordinal within each block. Wherever a {@code preQuantized} record with a matching encoding and
* flavor is available (i.e. the vector originates from a segment in this format), it is copied
* as-is; otherwise the raw vector is quantized. The block locations are written to {@code meta};
- * non-FLOAT32 groups (stored raw only) record an empty list.
+ * BYTE groups (stored raw only) record an empty list. FLOAT32 and FLOAT16 groups are both
+ * quantized; callers supply {@code vectors} as {@code float[]} (FLOAT16 inflates its {@code
+ * short[]} storage first), since the quantized record is a pure function of the {@code float[]}
+ * values and the flavor, independent of the source encoding.
*/
void writeGroup(
IndexOutput meta,
@@ -200,7 +203,7 @@ record DedupQuantizer(ScalarEncoding encoding) {
PreQuantizedSupplier preQuantized)
throws IOException {
- if (groupEncoding != VectorEncoding.FLOAT32) {
+ if (groupEncoding.isFloatingPoint() == false) {
writeEmptyGroup(meta);
return;
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
index b4acc025f82..e988ad6f5e1 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
@@ -16,13 +16,12 @@
*/
package org.apache.lucene.sandbox.codecs.dedup;
-import static org.apache.lucene.index.VectorEncoding.FLOAT32;
-
import java.io.IOException;
import org.apache.lucene.codecs.lucene104.OffHeapScalarQuantizedVectorValues;
-import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues;
import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
+import org.apache.lucene.index.Float16VectorValues;
import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;
import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrdOffHeap;
@@ -80,6 +79,11 @@ final class DedupScalarQuantizedVectorValues {
slice);
}
+ /**
+ * Loads a field's quantized vector values. The {@code fieldView} only maps field ords to docs;
+ * all vector reads, corrective terms, centroid and quantizer resolve through the shared group
+ * view.
+ */
static FieldValues loadQuantized(
DedupScalarQuantizedVectorsScorer vectorsScorer,
VectorSimilarityFunction function,
@@ -96,16 +100,29 @@ final class DedupScalarQuantizedVectorValues {
int fieldOrdToGroupOrdBitsPerValue)
throws IOException {
- final OffHeapFloatVectorValues fieldView =
- OffHeapFloatVectorValues.load(
- function, vectorsScorer, configuration, FLOAT32, dimension, 0, 0, vectorData);
+ final DedupQuantizer.Flavor flavor = DedupQuantizer.Flavor.of(function);
+
+ final OffHeapScalarQuantizedVectorValues fieldView =
+ OffHeapScalarQuantizedVectorValues.load(
+ configuration,
+ dimension,
+ configuration.size(),
+ flavor.quantizer(),
+ encoding,
+ function,
+ vectorsScorer,
+ new float[dimension], // always a zero centroid, quantization is data-blind
+ 0,
+ 0,
+ 0,
+ vectorData);
final QuantizedByteVectorValues groupView =
groupValues(
false,
vectorsScorer,
function,
- DedupQuantizer.Flavor.of(function),
+ flavor,
encoding,
dimension,
groupNumVectors,
@@ -119,7 +136,8 @@ final class DedupScalarQuantizedVectorValues {
fieldOrdToGroupOrdSize,
fieldOrdToGroupOrdBitsPerValue);
- return new FieldValues(vectorsScorer, function, fieldView, groupView, fieldOrdToGroupOrd);
+ return new FieldValues(
+ vectorsScorer, function, fieldView, groupView, fieldOrdToGroupOrd, configuration.isDense());
}
/**
@@ -130,22 +148,25 @@ final class DedupScalarQuantizedVectorValues {
static final class FieldValues extends QuantizedByteVectorValues implements DedupVectorValues {
private final DedupScalarQuantizedVectorsScorer vectorsScorer;
private final VectorSimilarityFunction function;
- private final FloatVectorValues fieldView;
+ private final KnnVectorValues fieldView;
private final QuantizedByteVectorValues groupView;
private final FieldOrdToGroupOrd fieldOrdToGroupOrd;
+ private final boolean isDense;
private int[] scratch;
FieldValues(
DedupScalarQuantizedVectorsScorer vectorsScorer,
VectorSimilarityFunction function,
- FloatVectorValues fieldView,
+ KnnVectorValues fieldView,
QuantizedByteVectorValues groupView,
- FieldOrdToGroupOrd fieldOrdToGroupOrd) {
+ FieldOrdToGroupOrd fieldOrdToGroupOrd,
+ boolean isDense) {
this.vectorsScorer = vectorsScorer;
this.function = function;
this.fieldView = fieldView;
this.groupView = groupView;
this.fieldOrdToGroupOrd = fieldOrdToGroupOrd;
+ this.isDense = isDense;
this.scratch = new int[DedupUtil.SCRATCH_INITIAL_SIZE];
}
@@ -244,7 +265,12 @@ final class DedupScalarQuantizedVectorValues {
@Override
public FieldValues copy() throws IOException {
return new FieldValues(
- vectorsScorer, function, fieldView.copy(), groupView.copy(), fieldOrdToGroupOrd.copy());
+ vectorsScorer,
+ function,
+ fieldView.copy(),
+ groupView.copy(),
+ fieldOrdToGroupOrd.copy(),
+ isDense);
}
@Override
@@ -255,7 +281,17 @@ final class DedupScalarQuantizedVectorValues {
FieldValues copy = copy();
DocIndexIterator indexIterator = copy.iterator();
RandomVectorScorer vectorScorer = vectorsScorer.getRandomVectorScorer(function, copy, target);
- boolean isDense = copy.fieldView instanceof OffHeapFloatVectorValues.DenseOffHeapVectorValues;
+ return new DedupVectorScorer(indexIterator, vectorScorer, isDense);
+ }
+
+ @Override
+ public VectorScorer scorer(short[] target) throws IOException {
+ if (size() == 0) {
+ return null;
+ }
+ FieldValues copy = copy();
+ DocIndexIterator indexIterator = copy.iterator();
+ RandomVectorScorer vectorScorer = vectorsScorer.getRandomVectorScorer(function, copy, target);
return new DedupVectorScorer(indexIterator, vectorScorer, isDense);
}
}
@@ -265,11 +301,13 @@ final class DedupScalarQuantizedVectorValues {
* #scorer(float[])} scores against the quantized values instead. Mirrors {@code
* Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedVectorValues}.
*/
- static final class RawAndQuantizedValues extends FloatVectorValues implements DedupVectorValues {
+ static final class Float32RawAndQuantizedValues extends FloatVectorValues
+ implements DedupVectorValues {
private final DedupVectorValues.FloatImpl rawValues;
private final FieldValues quantizedValues;
- RawAndQuantizedValues(DedupVectorValues.FloatImpl rawValues, FieldValues quantizedValues) {
+ Float32RawAndQuantizedValues(
+ DedupVectorValues.FloatImpl rawValues, FieldValues quantizedValues) {
this.rawValues = rawValues;
this.quantizedValues = quantizedValues;
}
@@ -324,8 +362,8 @@ final class DedupScalarQuantizedVectorValues {
}
@Override
- public RawAndQuantizedValues copy() throws IOException {
- return new RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());
+ public Float32RawAndQuantizedValues copy() throws IOException {
+ return new Float32RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());
}
@Override
@@ -338,4 +376,85 @@ final class DedupScalarQuantizedVectorValues {
return rawValues.rescorer(target);
}
}
+
+ /**
+ * Full-precision view of a field backed by the raw de-duplicated vectors, whose {@link
+ * #scorer(short[])} scores against the quantized values instead. Mirrors {@code
+ * Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedFloat16VectorValues}.
+ */
+ static final class Float16RawAndQuantizedValues extends Float16VectorValues
+ implements DedupVectorValues {
+ private final DedupVectorValues.Float16Impl rawValues;
+ private final FieldValues quantizedValues;
+
+ Float16RawAndQuantizedValues(
+ DedupVectorValues.Float16Impl rawValues, FieldValues quantizedValues) {
+ this.rawValues = rawValues;
+ this.quantizedValues = quantizedValues;
+ }
+
+ FieldValues getQuantizedValues() {
+ return quantizedValues;
+ }
+
+ @Override
+ public KnnVectorValues getGroupView() {
+ return rawValues.getGroupView();
+ }
+
+ @Override
+ public FieldOrdToGroupOrd getFieldOrdToGroupOrd() {
+ return rawValues.getFieldOrdToGroupOrd();
+ }
+
+ @Override
+ public int dimension() {
+ return rawValues.dimension();
+ }
+
+ @Override
+ public int size() {
+ return rawValues.size();
+ }
+
+ @Override
+ public int ordToDoc(int ord) {
+ return rawValues.ordToDoc(ord);
+ }
+
+ @Override
+ public Bits getAcceptOrds(Bits acceptDocs) {
+ return rawValues.getAcceptOrds(acceptDocs);
+ }
+
+ @Override
+ public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {
+ rawValues.prefetch(ordsToPrefetch, numOrds);
+ }
+
+ @Override
+ public short[] vectorValue(int ord) throws IOException {
+ return rawValues.vectorValue(ord);
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return rawValues.iterator();
+ }
+
+ @Override
+ public Float16RawAndQuantizedValues copy() throws IOException {
+ return new Float16RawAndQuantizedValues(rawValues.copy(), quantizedValues.copy());
+ }
+
+ @Override
+ public VectorScorer scorer(short[] target) throws IOException {
+ return quantizedValues.scorer(target);
+ }
+
+ @Override
+ public VectorScorer rescorer(short[] target) throws IOException {
+ return rawValues.rescorer(target);
+ }
+ }
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java
index 5ecfb0cac55..6cdec559f1f 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsFormat.java
@@ -41,8 +41,11 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncod
* tradeoff relative to that format, which centers vectors on a per-field centroid before
* quantizing.
*
- * <p>Only {@link org.apache.lucene.index.VectorEncoding#FLOAT32} vectors are quantized; BYTE and
- * FLOAT16 vectors are stored raw only, identical to {@link DedupFlatVectorsFormat}.
+ * <p>Both {@link org.apache.lucene.index.VectorEncoding#FLOAT32} and {@link
+ * org.apache.lucene.index.VectorEncoding#FLOAT16} vectors are quantized (FLOAT16 vectors are
+ * inflated to {@code float} for the data-blind quantizer, and kept raw as {@code short[]} for
+ * full-fidelity readback); BYTE vectors are stored raw only, identical to {@link
+ * DedupFlatVectorsFormat}.
*
* <h2>.vdd (vector de-dup data) file</h2>
*
@@ -51,8 +54,8 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncod
*
* <h2>.vdqd (vector de-dup quantized data) file</h2>
*
- * <p>One block per FLOAT32 group and {@link DedupQuantizer.Flavor} in use (aligned to {@code
- * Float.BYTES}), holding one quantized record per distinct vector:
+ * <p>One block per quantized (FLOAT32 or FLOAT16) group and {@link DedupQuantizer.Flavor} in use
+ * (aligned to {@code Float.BYTES}), holding one quantized record per distinct vector:
*
* <ul>
* <li><b>[byte]</b> the packed quantized values (layout depends on the {@link ScalarEncoding})
@@ -69,7 +72,7 @@ import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncod
* additionally recording its flavor blocks:
*
* <ul>
- * <li><b>[vint]</b> the number of flavor blocks ({@code 0} for non-FLOAT32 groups)
+ * <li><b>[vint]</b> the number of flavor blocks ({@code 0} for BYTE groups)
* <li>per flavor block: <b>[vint]</b> the flavor wire number, <b>[vint]</b> the {@link
* ScalarEncoding} wire number of its records, <b>[int64]</b> offset and <b>[int64]</b> length
* of its quantized records in the .vdqd file
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
index 66ed85c7208..ca22c1c8a31 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
@@ -44,7 +44,8 @@ import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.sandbox.codecs.dedup.DedupQuantizer.QuantizedBlock;
import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;
-import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues;
import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;
import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;
import org.apache.lucene.store.ChecksumIndexInput;
@@ -66,10 +67,10 @@ import org.apache.lucene.util.quantization.QuantizedVectorsReader;
import org.apache.lucene.util.quantization.ScalarQuantizer;
/**
- * Reads de-duplicated flat vectors written by {@link DedupScalarQuantizedVectorsFormat}. Each
- * FLOAT32 field exposes a full-precision view backed by the raw de-duplicated vectors, scored
- * against a data-blind quantized view sharing the same {@code fieldOrdToGroupOrd} translation map.
- * BYTE and FLOAT16 fields are stored and read raw only.
+ * Reads de-duplicated flat vectors written by {@link DedupScalarQuantizedVectorsFormat}. FLOAT32
+ * and FLOAT16 fields each expose a full-precision view backed by the raw de-duplicated vectors,
+ * scored against a data-blind quantized view sharing the same {@code fieldOrdToGroupOrd}
+ * translation map. BYTE fields are stored and read raw only.
*
* @lucene.experimental
*/
@@ -199,7 +200,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
QuantizedGroupInfo quantizedGroupInfo = groupInfos.get(fieldInfo.groupOrd());
GroupInfo groupInfo = quantizedGroupInfo.groupInfo();
QuantizedBlock quantizedBlock = null;
- if (fieldInfo.encoding() == FLOAT32) {
+ if (fieldInfo.encoding().isFloatingPoint()) {
DedupQuantizer.Flavor flavor = DedupQuantizer.Flavor.of(fieldInfo.function());
quantizedBlock = quantizedGroupInfo.quantizedBlocks().get(flavor);
if (quantizedBlock == null) {
@@ -232,7 +233,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
GroupInfo groupInfo,
Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks)
throws IOException {
- if (groupInfo.encoding() != FLOAT32) {
+ if (groupInfo.encoding().isFloatingPoint() == false) {
if (quantizedBlocks.isEmpty() == false) {
throw new CorruptIndexException(
"Unexpected quantized data for encoding=" + groupInfo.encoding(), meta);
@@ -343,8 +344,9 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
@Override
public RandomVectorScorer getRandomVectorScorer(String field, short[] target) throws IOException {
FieldEntry entry = getEntry(field, FLOAT16);
- Float16VectorValues vectorValues = getFloat16VectorValues(entry);
- return vectorsScorer.getRandomVectorScorer(entry.fieldInfo().function(), vectorValues, target);
+ FieldValues quantizedValues = getQuantizedVectorValues(entry);
+ return vectorsScorer.getRandomVectorScorer(
+ entry.fieldInfo().function(), quantizedValues, target);
}
@Override
@@ -388,7 +390,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
@Override
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldEntry entry = getEntry(field, FLOAT32);
- return new RawAndQuantizedValues(
+ return new Float32RawAndQuantizedValues(
getRawFloatVectorValues(entry), getQuantizedVectorValues(entry));
}
@@ -412,7 +414,8 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
return getByteVectorValues(getEntry(field, BYTE));
}
- private Float16VectorValues getFloat16VectorValues(FieldEntry entry) throws IOException {
+ private DedupVectorValues.Float16Impl getRawFloat16VectorValues(FieldEntry entry)
+ throws IOException {
return loadDedupFloat16s(
vectorsScorer,
entry.fieldInfo().function(),
@@ -429,12 +432,19 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
@Override
public Float16VectorValues getFloat16VectorValues(String field) throws IOException {
- return getFloat16VectorValues(getEntry(field, FLOAT16));
+ FieldEntry entry = getEntry(field, FLOAT16);
+ return new Float16RawAndQuantizedValues(
+ getRawFloat16VectorValues(entry), getQuantizedVectorValues(entry));
}
@Override
- public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {
- FieldEntry entry = getEntry(field, FLOAT32);
+ public FieldValues getQuantizedVectorValues(String field) throws IOException {
+ FieldEntry entry = fields.get(field);
+ if (entry == null) {
+ throw new IllegalArgumentException("field=" + field + " not found");
+ } else if (entry.fieldInfo().encoding().isFloatingPoint() == false) {
+ throw new IllegalArgumentException("field=" + field + " not indexed as floating point");
+ }
return getQuantizedVectorValues(entry);
}
@@ -447,8 +457,8 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
public CloseableRandomVectorScorerSupplier getRandomVectorScorerSupplierForMerge(
FieldInfo fieldInfo, SegmentWriteState segmentWriteState) throws IOException {
FieldEntry entry = fields.get(fieldInfo.name);
- if (entry == null || entry.fieldInfo().encoding() != FLOAT32) {
- // BYTE and FLOAT16 fields are stored raw only
+ if (entry == null || entry.fieldInfo().encoding().isFloatingPoint() == false) {
+ // Not a quantized field (missing, or e.g. BYTE stored raw only)
return null;
}
FieldValues quantizedValues = getQuantizedVectorValues(entry);
@@ -462,9 +472,18 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
// Asymmetric encodings compare query-encoded vectors against the stored doc-encoded vectors.
// Write a query-encoded record per distinct vector of the group into a temporary file; both
- // sides then resolve through the shared fieldOrdToGroupOrd translation.
- DedupVectorValues.FloatImpl rawValues = getRawFloatVectorValues(entry);
- FloatVectorValues groupView = rawValues.getGroupView();
+ // sides then resolve through the shared fieldOrdToGroupOrd translation. The group's raw vectors
+ // are read directly from the group view (indexed by group ordinal, not field ordinal) as
+ // float[] (FLOAT16 groups are inflated from short[]).
+ FloatVectorValues floatVectorValues;
+ if (fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT16) {
+ DedupVectorValues.Float16Impl float16RawValues = getRawFloat16VectorValues(entry);
+ floatVectorValues = new Float16AsFloatVectorValues(float16RawValues.getGroupView());
+ } else {
+ DedupVectorValues.FloatImpl float32RawValues = getRawFloatVectorValues(entry);
+ floatVectorValues = float32RawValues.getGroupView();
+ }
+
int dimension = entry.fieldInfo().dimension();
int groupNumVectors = entry.groupInfo().groupNumVectors();
DedupQuantizer.Flavor flavor =
@@ -483,7 +502,7 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
byte[] toQuery = new byte[encoding.getQueryPackedLength(dimension)];
for (int groupOrd = 0; groupOrd < groupNumVectors; groupOrd++) {
// copy: normalization and quantization mutate the input, which is a shared buffer
- System.arraycopy(groupView.vectorValue(groupOrd), 0, vector, 0, dimension);
+ System.arraycopy(floatVectorValues.vectorValue(groupOrd), 0, vector, 0, dimension);
if (flavor.normalized()) {
VectorUtil.l2normalize(vector);
}
@@ -590,4 +609,48 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
private record QuantizedGroupInfo(
GroupInfo groupInfo, Map<DedupQuantizer.Flavor, QuantizedBlock> quantizedBlocks) {}
+
+ /**
+ * Exposes a {@link Float16VectorValues} as {@link FloatVectorValues}, inflating fp16 to fp32 on
+ * read, so the merge path can reuse the fp32 quantization classes.
+ */
+ private static final class Float16AsFloatVectorValues extends FloatVectorValues {
+ private final Float16VectorValues values;
+ private final float[] floatVector;
+
+ Float16AsFloatVectorValues(Float16VectorValues values) {
+ this.values = values;
+ this.floatVector = new float[values.dimension()];
+ }
+
+ @Override
+ public int dimension() {
+ return values.dimension();
+ }
+
+ @Override
+ public int size() {
+ return values.size();
+ }
+
+ @Override
+ public int ordToDoc(int ord) {
+ return values.ordToDoc(ord);
+ }
+
+ @Override
+ public float[] vectorValue(int ord) throws IOException {
+ return DedupUtil.inflateFloat16(values.vectorValue(ord), floatVector);
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return values.iterator();
+ }
+
+ @Override
+ public Float16AsFloatVectorValues copy() throws IOException {
+ return new Float16AsFloatVectorValues(values.copy());
+ }
+ }
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java
index b4aa648a873..94117f55a20 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsScorer.java
@@ -21,7 +21,8 @@ import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.FieldValues;
-import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.RawAndQuantizedValues;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues;
+import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
@@ -41,10 +42,13 @@ final class DedupScalarQuantizedVectorsScorer extends DedupFlatVectorsScorer {
super(QUANTIZED_SCORER);
}
- /** Full-precision views resolve to their quantized values for scoring. */
+ /** Full-precision views (float32 or float16) resolve to their quantized values for scoring. */
@Override
protected KnnVectorValues unwrap(KnnVectorValues vectorValues) {
- if (vectorValues instanceof RawAndQuantizedValues rawAndQuantized) {
+ if (vectorValues instanceof Float32RawAndQuantizedValues rawAndQuantized) {
+ return rawAndQuantized.getQuantizedValues();
+ }
+ if (vectorValues instanceof Float16RawAndQuantizedValues rawAndQuantized) {
return rawAndQuantized.getQuantizedValues();
}
return vectorValues;
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java
index 70981af05c5..5bf14670164 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java
@@ -212,6 +212,19 @@ final class DedupUtil {
return murmurhash3_x64_128(bytes, 0, bytes.length, GOOD_FAST_HASH_SEED)[0];
}
+ /**
+ * Inflates a float16 vector ({@code short[]}) into the reused {@code float[]} buffer for the
+ * fp32-based scalar quantizer.
+ */
+ static float[] inflateFloat16(short[] float16Vector, float[] dest) {
+ assert float16Vector.length == dest.length
+ : "length mismatch: float16Vector=" + float16Vector.length + " dest=" + dest.length;
+ for (int i = 0; i < float16Vector.length; i++) {
+ dest[i] = Float.float16ToFloat(float16Vector[i]);
+ }
+ return dest;
+ }
+
static long alignBytes(IndexOutput output, VectorEncoding encoding) throws IOException {
int alignBytes =
switch (encoding) {
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
index fffe388ff6d..bf7526ba42c 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
@@ -58,7 +58,8 @@ sealed interface DedupVectorValues
DedupVectorValues.FloatImpl,
DedupVectorValues.Float16Impl,
DedupScalarQuantizedVectorValues.FieldValues,
- DedupScalarQuantizedVectorValues.RawAndQuantizedValues {
+ DedupScalarQuantizedVectorValues.Float32RawAndQuantizedValues,
+ DedupScalarQuantizedVectorValues.Float16RawAndQuantizedValues {
/** The dense view over distinct vectors, indexed by group ordinal. */
KnnVectorValues getGroupView();
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java
index 0752b9e2d13..66b5fd6503b 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/package-info.java
@@ -26,8 +26,8 @@
* vectors; see {@link org.apache.lucene.sandbox.codecs.dedup.DedupFlatVectorsFormat} for the
* on-disk layout.
* <li>{@link org.apache.lucene.sandbox.codecs.dedup.DedupHnswScalarQuantizedVectorsFormat}
- * additionally stores a scalar quantized copy of each distinct FLOAT32 vector, used for
- * scoring; see {@link
+ * additionally stores a scalar quantized copy of each distinct FLOAT32 or FLOAT16 vector,
+ * used for scoring; see {@link
* org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorsFormat} for the on-disk
* layout.
* </ul>
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java
index 113a95b66a4..10ec9fba2d9 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupHnswScalarQuantizedVectorsFormat.java
@@ -26,7 +26,6 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.SameThreadExecutorService;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
@@ -92,10 +91,10 @@ public class TestDedupHnswScalarQuantizedVectorsFormat extends TestDedupHnswVect
} else {
assertTrue(totalByteSize > 0);
assertTrue(offHeap.get("vdd") > 0L); // NOTE: different from vec
- if (fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) {
+ if (fieldInfo.getVectorEncoding().isFloatingPoint()) {
assertTrue(offHeap.get("vdqd") > 0L); // NOTE: different from veq
} else {
- assertNull(offHeap.get("vdqd")); // BYTE and FLOAT16 fields are stored raw only
+ assertNull(offHeap.get("vdqd")); // BYTE fields are stored raw only
}
if (hasHNSW(knnVectorsReader, fieldInfo)) {
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java
index f9da057ae88..53b459442f3 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/dedup/TestDedupScalarQuantizedVectorsFormat.java
@@ -19,6 +19,7 @@ package org.apache.lucene.sandbox.codecs.dedup;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
import static org.apache.lucene.index.VectorEncoding.BYTE;
+import static org.apache.lucene.index.VectorEncoding.FLOAT16;
import static org.apache.lucene.index.VectorEncoding.FLOAT32;
import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
@@ -33,11 +34,13 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnByteVectorField;
+import org.apache.lucene.document.KnnFloat16VectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.Float16VectorValues;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -51,6 +54,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.util.hnsw.HnswGraph;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
/**
@@ -62,9 +66,27 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
private static final ScalarEncoding ENCODING = ScalarEncoding.UNSIGNED_BYTE;
+ /**
+ * Build an index writer config whose dedup format always builds the HNSW graph.
+ *
+ * <p>The default {@code tinySegmentsThreshold} ({@link
+ * org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat#HNSW_GRAPH_THRESHOLD} = 100) lets
+ * small segments skip graph construction and store vectors flat only. These tests use tiny
+ * deterministic datasets, so we pin the threshold to {@code 0} ("always build the graph") rather
+ * than inflating document counts, guaranteeing the merge-time graph construction and search paths
+ * are exercised.
+ */
private static IndexWriterConfig config() {
return newIndexWriterConfig()
- .setCodec(TestUtil.alwaysKnnVectorsFormat(new DedupHnswScalarQuantizedVectorsFormat()));
+ .setCodec(
+ TestUtil.alwaysKnnVectorsFormat(
+ new DedupHnswScalarQuantizedVectorsFormat(
+ ENCODING,
+ DEFAULT_MAX_CONN,
+ DEFAULT_BEAM_WIDTH,
+ 1,
+ null,
+ 0))); // tinySegmentsThreshold=0 => always build the HNSW graph
}
/** Size in bytes of one quantized record for the given dimension. */
@@ -180,6 +202,7 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
FloatVectorValues values = leafReader.getFloatVectorValues("f");
assertEquals(docVectors.length, values.size());
assertEquals(2, groupNumVectors(values)); // a's duplicate collapsed across segments
+ assertGraphBuilt(leafReader, "f"); // merge built the HNSW graph, not the tiny shortcut
DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");
DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT32);
@@ -221,6 +244,7 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
Integer.MAX_VALUE);
assertEquals(numA + numB, topDocs.scoreDocs.length);
+ assertGraphBuilt(leafReader, "f"); // search ran over a real HNSW graph
// duplicates of a rank first with identical scores
float scoreA = topDocs.scoreDocs[0].score;
for (int i = 0; i < numA; i++) {
@@ -333,6 +357,107 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
}
}
+ /**
+ * FLOAT16 fields are quantized like FLOAT32: repeated vectors are stored once (raw and
+ * quantized), the raw {@code short[]} still reads back per document, and a quantized block is
+ * present.
+ */
+ public void testFloat16DuplicatesStoredOnceQuantized() throws Exception {
+ short[] a = toFloat16(new float[] {1, 2, 3, 4});
+ short[] b = toFloat16(new float[] {5, 6, 7, 8});
+ short[][] docVectors = {a, b, a, b, a, b}; // 3 copies each of 2 vectors
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ for (short[] docVector : docVectors) {
+ Document doc = new Document();
+ doc.add(new KnnFloat16VectorField("f", docVector, EUCLIDEAN));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ Float16VectorValues values = leafReader.getFloat16VectorValues("f");
+ assertEquals(docVectors.length, values.size()); // one entry per document
+ assertEquals(2, groupNumVectors(values)); // only two distinct vectors stored
+
+ // raw float16 vectors read back exactly, resolved per document
+ for (int ord = 0; ord < values.size(); ord++) {
+ short[] vector = values.vectorValue(ord);
+ assertTrue(Arrays.equals(a, vector) || Arrays.equals(b, vector));
+ }
+
+ // the quantized block holds exactly two records
+ DedupScalarQuantizedVectorsReader sqReader = getQuantizedReader(leafReader, "f");
+ DedupScalarQuantizedVectorsReader.FieldEntry entry = sqReader.getEntry("f", FLOAT16);
+ assertNotNull(entry.quantizedBlock());
+ assertEquals(2 * quantizedRecordSize(a.length), entry.quantizedBlock().quantizedDataSize());
+ }
+ }
+ }
+
+ /**
+ * FLOAT16 duplicates that span multiple segments collapse to a single quantized copy on merge,
+ * and searches rank the query's duplicates together at the top.
+ */
+ public void testFloat16DuplicatesAcrossSegmentsAndSearch() throws Exception {
+ int dimension = 16;
+ float[] af = randomVector(dimension);
+ float[] bf = new float[dimension];
+ for (int i = 0; i < dimension; i++) {
+ bf[i] = -af[i]; // far away from a
+ }
+ short[] a = toFloat16(af);
+ short[] b = toFloat16(bf);
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config())) {
+ int numA = 3, numB = 3;
+ for (int i = 0; i < numA; i++) {
+ Document doc = new Document();
+ doc.add(new KnnFloat16VectorField("f", a, EUCLIDEAN));
+ w.addDocument(doc);
+ w.commit(); // one segment per document
+ }
+ for (int i = 0; i < numB; i++) {
+ Document doc = new Document();
+ doc.add(new KnnFloat16VectorField("f", b, EUCLIDEAN));
+ w.addDocument(doc);
+ w.commit();
+ }
+ w.forceMerge(1); // exercises FP16 merge-time quantized dedup + scorer supplier
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ Float16VectorValues values = leafReader.getFloat16VectorValues("f");
+ assertEquals(numA + numB, values.size());
+ assertEquals(2, groupNumVectors(values)); // collapsed across segments
+ assertGraphBuilt(leafReader, "f"); // merge built the HNSW graph, not the tiny shortcut
+
+ TopDocs topDocs =
+ leafReader.searchNearestVectors(
+ "f",
+ a,
+ numA + numB,
+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
+ Integer.MAX_VALUE);
+ assertEquals(numA + numB, topDocs.scoreDocs.length);
+ float scoreA = topDocs.scoreDocs[0].score;
+ for (int i = 0; i < numA; i++) {
+ assertEquals(scoreA, topDocs.scoreDocs[i].score, 0f); // duplicates share a score
+ }
+ for (int i = numA; i < numA + numB; i++) {
+ assertTrue(topDocs.scoreDocs[i].score < scoreA);
+ }
+ }
+ }
+ }
+
+ private static short[] toFloat16(float[] vector) {
+ short[] out = new short[vector.length];
+ for (int i = 0; i < vector.length; i++) {
+ out[i] = Float.floatToFloat16(vector[i]);
+ }
+ return out;
+ }
+
/**
* Every encoding survives a merge with graph building forced (exercising the merge-time scorer
* suppliers, including the temporary query-vectors file of asymmetric encodings) and searches
@@ -379,6 +504,7 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
FloatVectorValues values = leafReader.getFloatVectorValues("f");
assertEquals(2 * numDistinct, values.size());
assertEquals(numDistinct, groupNumVectors(values)); // de-duplicated across segments
+ assertGraphBuilt(leafReader, "f"); // threshold=0 forced the merge-time graph build
float[] query =
function == DOT_PRODUCT ? VectorUtil.l2normalize(distinct[0].clone()) : distinct[0];
@@ -399,6 +525,148 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
}
}
+ /**
+ * Every encoding survives a merge of FLOAT16 fields with graph building forced (exercising the
+ * merge-time scorer suppliers, including the temporary query-vectors file of asymmetric encodings
+ * and the FLOAT16 {@code short[]} -> {@code float[]} inflation on the raw side) and searches
+ * sanely afterwards.
+ */
+ public void testFloat16AllEncodingsMergeAndSearch() throws Exception {
+ int dimension = 32;
+ int numDistinct = 20;
+ float[][] distinct = new float[numDistinct][];
+ for (int i = 0; i < numDistinct; i++) {
+ distinct[i] = randomVector(dimension);
+ }
+ for (ScalarEncoding encoding : ScalarEncoding.values()) {
+ for (VectorSimilarityFunction function : VectorSimilarityFunction.values()) {
+ IndexWriterConfig config =
+ newIndexWriterConfig()
+ .setCodec(
+ TestUtil.alwaysKnnVectorsFormat(
+ new DedupHnswScalarQuantizedVectorsFormat(
+ encoding,
+ DEFAULT_MAX_CONN,
+ DEFAULT_BEAM_WIDTH,
+ 1,
+ null,
+ 0))); // always build graphs
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config)) {
+ for (int segment = 0; segment < 2; segment++) { // duplicates across segments
+ for (int i = 0; i < numDistinct; i++) {
+ Document doc = new Document();
+ float[] vector =
+ function == DOT_PRODUCT
+ ? VectorUtil.l2normalize(distinct[i].clone())
+ : distinct[i];
+ doc.add(new KnnFloat16VectorField("f", toFloat16(vector), function));
+ w.addDocument(doc);
+ }
+ w.commit();
+ }
+ w.forceMerge(1); // exercises the merge-time graph construction scorer
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ Float16VectorValues values = leafReader.getFloat16VectorValues("f");
+ assertEquals(2 * numDistinct, values.size());
+ assertEquals(numDistinct, groupNumVectors(values)); // de-duplicated across segments
+ assertGraphBuilt(leafReader, "f"); // threshold=0 forced the merge-time graph build
+
+ float[] queryFloat =
+ function == DOT_PRODUCT ? VectorUtil.l2normalize(distinct[0].clone()) : distinct[0];
+ short[] query = toFloat16(queryFloat);
+ TopDocs topDocs =
+ leafReader.searchNearestVectors(
+ "f",
+ query,
+ 2,
+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
+ Integer.MAX_VALUE);
+ String context = "encoding=" + encoding + ", function=" + function;
+ assertEquals(context, 2, topDocs.scoreDocs.length);
+ // the two duplicates of the query vector share the (top) score
+ assertEquals(context, topDocs.scoreDocs[0].score, topDocs.scoreDocs[1].score, 0f);
+ }
+ }
+ }
+ }
+ }
+
+ public void testFloat16SharedGroupAsymmetricMerge() throws Exception {
+ int dimension = 32;
+ int smallCount = 2;
+ int bigCount = 20;
+ float[][] small = new float[smallCount][];
+ float[][] big = new float[bigCount][];
+ for (int i = 0; i < smallCount; i++) {
+ small[i] = randomVector(dimension);
+ }
+ for (int i = 0; i < bigCount; i++) {
+ big[i] = randomVector(dimension);
+ }
+ for (ScalarEncoding encoding : ScalarEncoding.values()) {
+ if (encoding.isAsymmetric() == false) {
+ continue; // the EOF only reproduces on the asymmetric merge path
+ }
+ IndexWriterConfig config =
+ newIndexWriterConfig()
+ .setCodec(
+ TestUtil.alwaysKnnVectorsFormat(
+ new DedupHnswScalarQuantizedVectorsFormat(
+ encoding,
+ DEFAULT_MAX_CONN,
+ DEFAULT_BEAM_WIDTH,
+ 1,
+ null,
+ 0))); // always build graphs
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config)) {
+ for (int segment = 0; segment < 2; segment++) { // duplicates across segments
+ for (int i = 0; i < smallCount; i++) {
+ Document d = new Document();
+ d.add(new KnnFloat16VectorField("small", toFloat16(small[i]), EUCLIDEAN));
+ w.addDocument(d);
+ }
+ for (int i = 0; i < bigCount; i++) {
+ Document d = new Document();
+ d.add(new KnnFloat16VectorField("big", toFloat16(big[i]), EUCLIDEAN));
+ w.addDocument(d);
+ }
+ w.commit();
+ }
+ // "small" has smallCount*2 doc-entries but shares a group of smallCount+bigCount distinct
+ // vectors; merging must map its field ords into the larger group without overrun.
+ w.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ String context = "encoding=" + encoding;
+
+ Float16VectorValues smallValues = leafReader.getFloat16VectorValues("small");
+ assertEquals(context, 2 * smallCount, smallValues.size());
+ Float16VectorValues bigValues = leafReader.getFloat16VectorValues("big");
+ assertEquals(context, 2 * bigCount, bigValues.size());
+ assertGraphBuilt(leafReader, "small"); // threshold=0 forced the merge-time graph build
+ assertGraphBuilt(leafReader, "big");
+
+ // search the small field: its two duplicates of the query share the top score
+ short[] q = toFloat16(small[0]);
+ TopDocs topDocs =
+ leafReader.searchNearestVectors(
+ "small",
+ q,
+ 2,
+ AcceptDocs.fromLiveDocs(null, leafReader.maxDoc()),
+ Integer.MAX_VALUE);
+ assertEquals(context, 2, topDocs.scoreDocs.length);
+ assertEquals(context, topDocs.scoreDocs[0].score, topDocs.scoreDocs[1].score, 0f);
+ }
+ }
+ }
+ }
+
private float[] randomVector(int dimension) {
float[] vector = new float[dimension];
for (int i = 0; i < dimension; i++) {
@@ -413,6 +681,22 @@ public class TestDedupScalarQuantizedVectorsFormat extends LuceneTestCase {
return ((DedupVectorValues) values).getGroupView().size();
}
+ /**
+ * Asserts an HNSW graph was actually built for the field (i.e. the tiny-segment shortcut was not
+ * taken). A skipped graph yields {@link org.apache.lucene.util.hnsw.HnswGraph#EMPTY} with size 0.
+ */
+ private static void assertGraphBuilt(LeafReader leafReader, String field) throws Exception {
+ assertThat(leafReader, instanceOf(CodecReader.class));
+ KnnVectorsReader knnVectorsReader = ((CodecReader) leafReader).getVectorReader();
+ KnnVectorsReader perField = knnVectorsReader.unwrapReaderForField(field);
+ assertThat(perField, instanceOf(Lucene99HnswVectorsReader.class));
+ HnswGraph graph = ((Lucene99HnswVectorsReader) perField).getGraph(field);
+ assertNotNull("no graph for field=" + field, graph);
+ assertTrue(
+ "expected an HNSW graph to be built for field=" + field + " but it was empty",
+ graph.size() > 0);
+ }
+
/** Get underlying quantized dedup vector reader instance. */
private static DedupScalarQuantizedVectorsReader getQuantizedReader(
LeafReader leafReader, String fieldName) {