Skip to content

Apache Lucene open-source search software

SimpleText[Float|Byte]VectorValues::scorer should return null when the vector values is empty (#13444)

Chris Hegartyf3c2b91630cb
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
index 6518b4aeab7..ab2486f4518 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
@@ -445,6 +445,9 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(float[] target) {
+ if (size() == 0) {
+ return null;
+ }
OffHeapFloatVectorValues values = this.copy();
return new VectorScorer() {
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
index 19675597c0a..048280466d4 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
@@ -494,6 +494,9 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(float[] target) {
+ if (size == 0) {
+ return null;
+ }
OffHeapFloatVectorValues values = this.copy();
return new VectorScorer() {
@Override
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java
index b914acf3fbb..720e1f56468 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java
@@ -73,4 +73,9 @@ public class TestLucene90HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
public void testByteVectorScorerIteration() {
// unimplemented
}
+
+ @Override
+ public void testEmptyByteVectorData() {
+ // unimplemented
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java
index b27a42700cb..09b5a50b4bc 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java
@@ -72,4 +72,9 @@ public class TestLucene91HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
public void testByteVectorScorerIteration() {
// unimplemented
}
+
+ @Override
+ public void testEmptyByteVectorData() {
+ // unimplemented
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java
index aaee5abe4ad..5189791ef17 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java
@@ -62,4 +62,9 @@ public class TestLucene92HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
public void testByteVectorScorerIteration() {
// unimplemented
}
+
+ @Override
+ public void testEmptyByteVectorData() {
+ // unimplemented
+ }
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
index b8d2ad5702c..faba629715b 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
@@ -360,6 +360,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(float[] target) {
+ if (size() == 0) {
+ return null;
+ }
SimpleTextFloatVectorValues simpleTextFloatVectorValues =
new SimpleTextFloatVectorValues(this);
return new VectorScorer() {
@@ -470,6 +473,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(byte[] target) {
+ if (size() == 0) {
+ return null;
+ }
SimpleTextByteVectorValues simpleTextByteVectorValues = new SimpleTextByteVectorValues(this);
return new VectorScorer() {
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index bfafd19a75c..7a1f8b232c9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -17,6 +17,7 @@
package org.apache.lucene.tests.index;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.ByteArrayOutputStream;
@@ -838,6 +839,58 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
}
+ public void testEmptyFloatVectorData() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
+ var doc1 = new Document();
+ doc1.add(new StringField("id", "0", Field.Store.NO));
+ doc1.add(new KnnFloatVectorField("v", new float[] {2, 3, 5, 6}, DOT_PRODUCT));
+ w.addDocument(doc1);
+
+ var doc2 = new Document();
+ doc2.add(new StringField("id", "1", Field.Store.NO));
+ w.addDocument(doc2);
+
+ w.deleteDocuments(new Term("id", Integer.toString(0)));
+ w.commit();
+ w.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader r = getOnlyLeafReader(reader);
+ FloatVectorValues values = r.getFloatVectorValues("v");
+ assertNotNull(values);
+ assertEquals(0, values.size());
+ assertNull(values.scorer(new float[] {2, 3, 5, 6}));
+ }
+ }
+ }
+
+ public void testEmptyByteVectorData() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
+ var doc1 = new Document();
+ doc1.add(new StringField("id", "0", Field.Store.NO));
+ doc1.add(new KnnByteVectorField("v", new byte[] {2, 3, 5, 6}, DOT_PRODUCT));
+ w.addDocument(doc1);
+
+ var doc2 = new Document();
+ doc2.add(new StringField("id", "1", Field.Store.NO));
+ w.addDocument(doc2);
+
+ w.deleteDocuments(new Term("id", Integer.toString(0)));
+ w.commit();
+ w.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader r = getOnlyLeafReader(reader);
+ ByteVectorValues values = r.getByteVectorValues("v");
+ assertNotNull(values);
+ assertEquals(0, values.size());
+ assertNull(values.scorer(new byte[] {2, 3, 5, 6}));
+ }
+ }
+ }
+
protected VectorSimilarityFunction randomSimilarity() {
return VectorSimilarityFunction.values()[
random().nextInt(VectorSimilarityFunction.values().length)];