Apache Lucene open-source search software
Prevent zero vectors on cosine similarity fields (#15751)
Vigya Sharmac592f384aa55
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 219cfda4c6c..aedb0d49601 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -197,6 +197,8 @@ API Changes* GITHUB#15723: Add hook for LRUQueryCache to allow custom population strategies (Nhat Nguyen)+* GITHUB#15751: Prevent zero vectors in knn fields with cosine similarity configured (Vigya Sharma)+New Features---------------------diff --git a/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java b/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.javaindex ddeef44c72a..570ef43c56f 100644--- a/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java+++ b/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java@@ -23,6 +23,7 @@ import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.search.KnnByteVectorQuery;import org.apache.lucene.search.Query;+import org.apache.lucene.util.VectorUtil;/*** A field that contains a single byte numeric vector (or none) for each document. Vectors are dense@@ -49,6 +50,10 @@ public class KnnByteVectorField extends Field {if (similarityFunction == null) {throw new IllegalArgumentException("similarity function must not be null");}+ if (similarityFunction == VectorSimilarityFunction.COSINE+ && VectorUtil.isZeroVector(v) == true) {+ throw new IllegalArgumentException("zero vector not allowed with cosine similarity function");+ }FieldType type = new FieldType();type.setVectorAttributes(dimension, VectorEncoding.BYTE, similarityFunction);type.freeze();@@ -138,6 +143,10 @@ public class KnnByteVectorField extends Field {throw new IllegalArgumentException("The number of vector dimensions does not match the field type");}+ if (fieldType.vectorSimilarityFunction() == VectorSimilarityFunction.COSINE+ && VectorUtil.isZeroVector(vector) == true) {+ throw new IllegalArgumentException("zero vector not allowed with cosine similarity function");+ }fieldsData = vector;}diff --git a/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java b/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.javaindex 63a55ddc669..dcdb8a1735d 100644--- a/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java+++ b/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java@@ -50,6 +50,10 @@ public class KnnFloatVectorField extends Field {if (similarityFunction == null) {throw new IllegalArgumentException("similarity function must not be null");}+ if (similarityFunction == VectorSimilarityFunction.COSINE+ && VectorUtil.isZeroVector(v) == true) {+ throw new IllegalArgumentException("zero vector not allowed with cosine similarity function");+ }FieldType type = new FieldType();type.setVectorAttributes(dimension, VectorEncoding.FLOAT32, similarityFunction);type.freeze();@@ -139,6 +143,11 @@ public class KnnFloatVectorField extends Field {throw new IllegalArgumentException("The number of vector dimensions does not match the field type");}+ if (fieldType.vectorSimilarityFunction() == VectorSimilarityFunction.COSINE+ && VectorUtil.isZeroVector(vector) == true) {+ throw new IllegalArgumentException(+ "zero vector not allowed with cosine similarity configured in field type");+ }fieldsData = VectorUtil.checkFinite(vector);}diff --git a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.javaindex e10fb528cba..54246a3f533 100644--- a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java@@ -441,6 +441,26 @@ public final class VectorUtil {return v;}+ /** Returns true if all dimensions of provided vector are zero, false otherwise. */+ public static boolean isZeroVector(float[] v) {+ for (float value : v) {+ if (value != 0) {+ return false;+ }+ }+ return true;+ }++ /** Returns true if all dimensions of provided vector are zero, false otherwise. */+ public static boolean isZeroVector(byte[] v) {+ for (float value : v) {+ if (value != 0) {+ return false;+ }+ }+ return true;+ }+/*** Given an array {@code buffer} that is sorted between indexes {@code 0} inclusive and {@code to}* exclusive, find the first array index whose value is greater than or equal to {@code target}.diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.javaindex 5c1b8f17294..8eeecaf0fa6 100644--- a/lucene/core/src/test/org/apache/lucene/document/TestField.java+++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java@@ -734,6 +734,37 @@ public class TestField extends LuceneTestCase {}}+ public void testKnnFieldZeroVectors() throws Exception {+ float[] v = new float[5];+ IllegalArgumentException zeroError =+ expectThrows(+ IllegalArgumentException.class,+ () -> new KnnFloatVectorField("knnFloats", v, VectorSimilarityFunction.COSINE));+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));++ FieldType fieldType = KnnFloatVectorField.createFieldType(5, VectorSimilarityFunction.COSINE);+ zeroError =+ expectThrows(+ IllegalArgumentException.class,+ () -> new KnnFloatVectorField("knnFloats", v, fieldType));+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));++ byte[] bv = new byte[5];+ zeroError =+ expectThrows(+ IllegalArgumentException.class,+ () -> new KnnByteVectorField("knnBytes", bv, VectorSimilarityFunction.COSINE));+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));++ FieldType byteFieldType =+ KnnByteVectorField.createFieldType(5, VectorSimilarityFunction.COSINE);+ zeroError =+ expectThrows(+ IllegalArgumentException.class,+ () -> new KnnByteVectorField("knnBytes", bv, byteFieldType));+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));+ }+private void trySetByteValue(Field f) {expectThrows(IllegalArgumentException.class,