Skip to content

Apache DataFusion SQL Query Engine

fix: clamp cosine_distance to enforce metric non-negativity and boundedness [0.0, 2.0] (#25744)

Ignatius Pang059aaa217abf
diff --git a/datafusion/functions-nested/src/cosine_distance.rs b/datafusion/functions-nested/src/cosine_distance.rs
index 1637b2cea1..11e8447fb8 100644
--- a/datafusion/functions-nested/src/cosine_distance.rs
+++ b/datafusion/functions-nested/src/cosine_distance.rs
@@ -226,7 +226,8 @@ fn general_cosine_distance<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
if sq1 == 0.0 || sq2 == 0.0 {
builder.append_null();
} else {
- builder.append_value(1.0 - dot / (sq1.sqrt() * sq2.sqrt()));
+ let sim = (dot / (sq1.sqrt() * sq2.sqrt())).clamp(-1.0, 1.0);
+ builder.append_value((1.0 - sim).clamp(0.0, 2.0));
}
}
diff --git a/datafusion/sqllogictest/test_files/cosine_distance.slt b/datafusion/sqllogictest/test_files/cosine_distance.slt
index 12bc5621ed..97a7cbc67f 100644
--- a/datafusion/sqllogictest/test_files/cosine_distance.slt
+++ b/datafusion/sqllogictest/test_files/cosine_distance.slt
@@ -29,12 +29,36 @@ select cosine_distance([1.0, 2.0, 3.0], [1.0, 2.0, 3.0]);
----
0
+# Identical vectors susceptible to floating point sqrt roundoff: distance = 0.0
+query R
+select cosine_distance([1.0, 1.0, 1.0], [1.0, 1.0, 1.0]);
+----
+0
+
+# Identical vectors metric non-negativity: distance >= 0.0
+query B
+select cosine_distance([1.0, 1.0, 1.0], [1.0, 1.0, 1.0]) >= 0.0;
+----
+true
+
# Opposite vectors: distance = 2.0
query R
select cosine_distance([1.0, 0.0], [-1.0, 0.0]);
----
2
+# Opposite 3D vectors bounded: distance = 2.0
+query R
+select cosine_distance([1.0, 1.0, 1.0], [-1.0, -1.0, -1.0]);
+----
+2
+
+# Opposite vectors metric boundedness: distance <= 2.0
+query B
+select cosine_distance([1.0, 1.0, 1.0], [-1.0, -1.0, -1.0]) <= 2.0;
+----
+true
+
# 45-degree angle: distance ≈ 0.293
query R
select round(cosine_distance([1.0, 0.0], [1.0, 1.0]), 3);