Skip to content

Apache Lucene open-source search software

Add FlatFieldVectorsWriter#asKnnVectorValues (#16062)

Lorenzo Dematté1ca725763e91
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index e6b774085f7..87ebb03bee7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -290,6 +290,9 @@ API Changes
* GITHUB#15990: Add experimental api to IndexWriter for columnar indexing. (Tim Brooks)
+* GITHUB#16062: Add FlatFieldVectorsWriter.asKnnVectorValues to return directly a KnnVectorValues "view" over
+ vectors to be written. (Lorenzo Dematte)
+
New Features
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.java
index fc71bb729db..d9126cca055 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.java
@@ -20,7 +20,11 @@ package org.apache.lucene.codecs.hnsw;
import java.io.IOException;
import java.util.List;
import org.apache.lucene.codecs.KnnFieldVectorsWriter;
+import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocsWithFieldSet;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
+import org.apache.lucene.index.VectorEncoding;
/**
* Vectors' writer for a field
@@ -34,6 +38,25 @@ public abstract class FlatFieldVectorsWriter<T> extends KnnFieldVectorsWriter<T>
*/
public abstract List<T> getVectors();
+ /**
+ * Returns a {@link KnnVectorValues} view over the vectors to be written.
+ *
+ * <p>This default implementation uses {@link #getVectors()} and wraps them via {@link
+ * FloatVectorValues#fromFloats} / {@link ByteVectorValues#fromBytes}. Subclasses backed by
+ * alternative storage may override to return a richer view (e.g. one that exposes capability
+ * information to scorers, or avoids heap materialization).
+ *
+ * @param encoding the vector encoding/element type
+ * @param dim the field's declared vector dimension
+ */
+ @SuppressWarnings("unchecked")
+ public KnnVectorValues asKnnVectorValues(VectorEncoding encoding, int dim) throws IOException {
+ return switch (encoding) {
+ case FLOAT32 -> FloatVectorValues.fromFloats((List<float[]>) getVectors(), dim);
+ case BYTE -> ByteVectorValues.fromBytes((List<byte[]>) getVectors(), dim);
+ };
+ }
+
/**
* @return the docsWithFieldSet for the field writer
*/
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
index 4f06b06321b..1e330fee21b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
@@ -38,10 +38,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
-import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocsWithFieldSet;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.MergeState;
@@ -736,20 +734,10 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {
this.flatFieldVectorsWriter = Objects.requireNonNull(flatFieldVectorsWriter);
this.graphThreshold = tinySegmentsThreshold;
this.scorerSupplier =
- switch (fieldInfo.getVectorEncoding()) {
- case BYTE ->
- scorer.getRandomVectorScorerSupplier(
- fieldInfo.getVectorSimilarityFunction(),
- ByteVectorValues.fromBytes(
- (List<byte[]>) flatFieldVectorsWriter.getVectors(),
- fieldInfo.getVectorDimension()));
- case FLOAT32 ->
- scorer.getRandomVectorScorerSupplier(
- fieldInfo.getVectorSimilarityFunction(),
- FloatVectorValues.fromFloats(
- (List<float[]>) flatFieldVectorsWriter.getVectors(),
- fieldInfo.getVectorDimension()));
- };
+ scorer.getRandomVectorScorerSupplier(
+ fieldInfo.getVectorSimilarityFunction(),
+ flatFieldVectorsWriter.asKnnVectorValues(
+ fieldInfo.getVectorEncoding(), fieldInfo.getVectorDimension()));
if (graphThreshold <= 0) {
// Initialize graph builder if optimization is disabled