Apache Lucene open-source search software
SimpleText[Float|Byte]VectorValues::scorer should return null when the vector values is empty (#13444)
Chris Hegartyf3c2b91630cb
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.javaindex 6518b4aeab7..ab2486f4518 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java@@ -445,6 +445,9 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {@Overridepublic VectorScorer scorer(float[] target) {+ if (size() == 0) {+ return null;+ }OffHeapFloatVectorValues values = this.copy();return new VectorScorer() {@Overridediff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.javaindex 19675597c0a..048280466d4 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java@@ -494,6 +494,9 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {@Overridepublic VectorScorer scorer(float[] target) {+ if (size == 0) {+ return null;+ }OffHeapFloatVectorValues values = this.copy();return new VectorScorer() {@Overridediff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.javaindex b914acf3fbb..720e1f56468 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java@@ -73,4 +73,9 @@ public class TestLucene90HnswVectorsFormat extends BaseKnnVectorsFormatTestCasepublic void testByteVectorScorerIteration() {// unimplemented}++ @Override+ public void testEmptyByteVectorData() {+ // unimplemented+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.javaindex b27a42700cb..09b5a50b4bc 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java@@ -72,4 +72,9 @@ public class TestLucene91HnswVectorsFormat extends BaseKnnVectorsFormatTestCasepublic void testByteVectorScorerIteration() {// unimplemented}++ @Override+ public void testEmptyByteVectorData() {+ // unimplemented+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.javaindex aaee5abe4ad..5189791ef17 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java@@ -62,4 +62,9 @@ public class TestLucene92HnswVectorsFormat extends BaseKnnVectorsFormatTestCasepublic void testByteVectorScorerIteration() {// unimplemented}++ @Override+ public void testEmptyByteVectorData() {+ // unimplemented+ }}diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.javaindex b8d2ad5702c..faba629715b 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java@@ -360,6 +360,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {@Overridepublic VectorScorer scorer(float[] target) {+ if (size() == 0) {+ return null;+ }SimpleTextFloatVectorValues simpleTextFloatVectorValues =new SimpleTextFloatVectorValues(this);return new VectorScorer() {@@ -470,6 +473,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {@Overridepublic VectorScorer scorer(byte[] target) {+ if (size() == 0) {+ return null;+ }SimpleTextByteVectorValues simpleTextByteVectorValues = new SimpleTextByteVectorValues(this);return new VectorScorer() {@Overridediff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.javaindex bfafd19a75c..7a1f8b232c9 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java@@ -17,6 +17,7 @@package org.apache.lucene.tests.index;import static java.nio.charset.StandardCharsets.UTF_8;+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import java.io.ByteArrayOutputStream;@@ -838,6 +839,58 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe}}+ public void testEmptyFloatVectorData() throws Exception {+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {+ var doc1 = new Document();+ doc1.add(new StringField("id", "0", Field.Store.NO));+ doc1.add(new KnnFloatVectorField("v", new float[] {2, 3, 5, 6}, DOT_PRODUCT));+ w.addDocument(doc1);++ var doc2 = new Document();+ doc2.add(new StringField("id", "1", Field.Store.NO));+ w.addDocument(doc2);++ w.deleteDocuments(new Term("id", Integer.toString(0)));+ w.commit();+ w.forceMerge(1);++ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader r = getOnlyLeafReader(reader);+ FloatVectorValues values = r.getFloatVectorValues("v");+ assertNotNull(values);+ assertEquals(0, values.size());+ assertNull(values.scorer(new float[] {2, 3, 5, 6}));+ }+ }+ }++ public void testEmptyByteVectorData() throws Exception {+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {+ var doc1 = new Document();+ doc1.add(new StringField("id", "0", Field.Store.NO));+ doc1.add(new KnnByteVectorField("v", new byte[] {2, 3, 5, 6}, DOT_PRODUCT));+ w.addDocument(doc1);++ var doc2 = new Document();+ doc2.add(new StringField("id", "1", Field.Store.NO));+ w.addDocument(doc2);++ w.deleteDocuments(new Term("id", Integer.toString(0)));+ w.commit();+ w.forceMerge(1);++ try (DirectoryReader reader = DirectoryReader.open(w)) {+ LeafReader r = getOnlyLeafReader(reader);+ ByteVectorValues values = r.getByteVectorValues("v");+ assertNotNull(values);+ assertEquals(0, values.size());+ assertNull(values.scorer(new byte[] {2, 3, 5, 6}));+ }+ }+ }+protected VectorSimilarityFunction randomSimilarity() {return VectorSimilarityFunction.values()[random().nextInt(VectorSimilarityFunction.values().length)];