Skip to content

Apache Lucene open-source search software

Prevent zero vectors on cosine similarity fields (#15751)

Vigya Sharmac592f384aa55
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 219cfda4c6c..aedb0d49601 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -197,6 +197,8 @@ API Changes
* GITHUB#15723: Add hook for LRUQueryCache to allow custom population strategies (Nhat Nguyen)
+* GITHUB#15751: Prevent zero vectors in knn fields with cosine similarity configured (Vigya Sharma)
+
New Features
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java b/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java
index ddeef44c72a..570ef43c56f 100644
--- a/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java
@@ -23,6 +23,7 @@ import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.KnnByteVectorQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
/**
* A field that contains a single byte numeric vector (or none) for each document. Vectors are dense
@@ -49,6 +50,10 @@ public class KnnByteVectorField extends Field {
if (similarityFunction == null) {
throw new IllegalArgumentException("similarity function must not be null");
}
+ if (similarityFunction == VectorSimilarityFunction.COSINE
+ && VectorUtil.isZeroVector(v) == true) {
+ throw new IllegalArgumentException("zero vector not allowed with cosine similarity function");
+ }
FieldType type = new FieldType();
type.setVectorAttributes(dimension, VectorEncoding.BYTE, similarityFunction);
type.freeze();
@@ -138,6 +143,10 @@ public class KnnByteVectorField extends Field {
throw new IllegalArgumentException(
"The number of vector dimensions does not match the field type");
}
+ if (fieldType.vectorSimilarityFunction() == VectorSimilarityFunction.COSINE
+ && VectorUtil.isZeroVector(vector) == true) {
+ throw new IllegalArgumentException("zero vector not allowed with cosine similarity function");
+ }
fieldsData = vector;
}
diff --git a/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java b/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java
index 63a55ddc669..dcdb8a1735d 100644
--- a/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java
@@ -50,6 +50,10 @@ public class KnnFloatVectorField extends Field {
if (similarityFunction == null) {
throw new IllegalArgumentException("similarity function must not be null");
}
+ if (similarityFunction == VectorSimilarityFunction.COSINE
+ && VectorUtil.isZeroVector(v) == true) {
+ throw new IllegalArgumentException("zero vector not allowed with cosine similarity function");
+ }
FieldType type = new FieldType();
type.setVectorAttributes(dimension, VectorEncoding.FLOAT32, similarityFunction);
type.freeze();
@@ -139,6 +143,11 @@ public class KnnFloatVectorField extends Field {
throw new IllegalArgumentException(
"The number of vector dimensions does not match the field type");
}
+ if (fieldType.vectorSimilarityFunction() == VectorSimilarityFunction.COSINE
+ && VectorUtil.isZeroVector(vector) == true) {
+ throw new IllegalArgumentException(
+ "zero vector not allowed with cosine similarity configured in field type");
+ }
fieldsData = VectorUtil.checkFinite(vector);
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java
index e10fb528cba..54246a3f533 100644
--- a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java
@@ -441,6 +441,26 @@ public final class VectorUtil {
return v;
}
+ /** Returns true if all dimensions of provided vector are zero, false otherwise. */
+ public static boolean isZeroVector(float[] v) {
+ for (float value : v) {
+ if (value != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** Returns true if all dimensions of provided vector are zero, false otherwise. */
+ public static boolean isZeroVector(byte[] v) {
+ for (float value : v) {
+ if (value != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Given an array {@code buffer} that is sorted between indexes {@code 0} inclusive and {@code to}
* exclusive, find the first array index whose value is greater than or equal to {@code target}.
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java
index 5c1b8f17294..8eeecaf0fa6 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestField.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java
@@ -734,6 +734,37 @@ public class TestField extends LuceneTestCase {
}
}
+ public void testKnnFieldZeroVectors() throws Exception {
+ float[] v = new float[5];
+ IllegalArgumentException zeroError =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> new KnnFloatVectorField("knnFloats", v, VectorSimilarityFunction.COSINE));
+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));
+
+ FieldType fieldType = KnnFloatVectorField.createFieldType(5, VectorSimilarityFunction.COSINE);
+ zeroError =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> new KnnFloatVectorField("knnFloats", v, fieldType));
+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));
+
+ byte[] bv = new byte[5];
+ zeroError =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> new KnnByteVectorField("knnBytes", bv, VectorSimilarityFunction.COSINE));
+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));
+
+ FieldType byteFieldType =
+ KnnByteVectorField.createFieldType(5, VectorSimilarityFunction.COSINE);
+ zeroError =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> new KnnByteVectorField("knnBytes", bv, byteFieldType));
+ assertTrue(zeroError.getMessage().contains("zero vector not allowed"));
+ }
+
private void trySetByteValue(Field f) {
expectThrows(
IllegalArgumentException.class,