Apache Lucene open-source search software
Add FlatFieldVectorsWriter#asKnnVectorValues (#16062)
Lorenzo Dematté1ca725763e91
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex e6b774085f7..87ebb03bee7 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -290,6 +290,9 @@ API Changes* GITHUB#15990: Add experimental api to IndexWriter for columnar indexing. (Tim Brooks)+* GITHUB#16062: Add FlatFieldVectorsWriter.asKnnVectorValues to return directly a KnnVectorValues "view" over+ vectors to be written. (Lorenzo Dematte)+New Features---------------------diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.javaindex fc71bb729db..d9126cca055 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatFieldVectorsWriter.java@@ -20,7 +20,11 @@ package org.apache.lucene.codecs.hnsw;import java.io.IOException;import java.util.List;import org.apache.lucene.codecs.KnnFieldVectorsWriter;+import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.DocsWithFieldSet;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.KnnVectorValues;+import org.apache.lucene.index.VectorEncoding;/*** Vectors' writer for a field@@ -34,6 +38,25 @@ public abstract class FlatFieldVectorsWriter<T> extends KnnFieldVectorsWriter<T>*/public abstract List<T> getVectors();+ /**+ * Returns a {@link KnnVectorValues} view over the vectors to be written.+ *+ * <p>This default implementation uses {@link #getVectors()} and wraps them via {@link+ * FloatVectorValues#fromFloats} / {@link ByteVectorValues#fromBytes}. Subclasses backed by+ * alternative storage may override to return a richer view (e.g. one that exposes capability+ * information to scorers, or avoids heap materialization).+ *+ * @param encoding the vector encoding/element type+ * @param dim the field's declared vector dimension+ */+ @SuppressWarnings("unchecked")+ public KnnVectorValues asKnnVectorValues(VectorEncoding encoding, int dim) throws IOException {+ return switch (encoding) {+ case FLOAT32 -> FloatVectorValues.fromFloats((List<float[]>) getVectors(), dim);+ case BYTE -> ByteVectorValues.fromBytes((List<byte[]>) getVectors(), dim);+ };+ }+/*** @return the docsWithFieldSet for the field writer*/diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.javaindex 4f06b06321b..1e330fee21b 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java@@ -38,10 +38,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;-import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.DocsWithFieldSet;import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FloatVectorValues;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.MergeState;@@ -736,20 +734,10 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {this.flatFieldVectorsWriter = Objects.requireNonNull(flatFieldVectorsWriter);this.graphThreshold = tinySegmentsThreshold;this.scorerSupplier =- switch (fieldInfo.getVectorEncoding()) {- case BYTE ->- scorer.getRandomVectorScorerSupplier(- fieldInfo.getVectorSimilarityFunction(),- ByteVectorValues.fromBytes(- (List<byte[]>) flatFieldVectorsWriter.getVectors(),- fieldInfo.getVectorDimension()));- case FLOAT32 ->- scorer.getRandomVectorScorerSupplier(- fieldInfo.getVectorSimilarityFunction(),- FloatVectorValues.fromFloats(- (List<float[]>) flatFieldVectorsWriter.getVectors(),- fieldInfo.getVectorDimension()));- };+ scorer.getRandomVectorScorerSupplier(+ fieldInfo.getVectorSimilarityFunction(),+ flatFieldVectorsWriter.asKnnVectorValues(+ fieldInfo.getVectorEncoding(), fieldInfo.getVectorDimension()));if (graphThreshold <= 0) {// Initialize graph builder if optimization is disabled