Apache DataFusion SQL Query Engine
fix: reject nested arrays in array_distance (#23995)
Yongting You8c4638890378
diff --git a/datafusion/functions-nested/src/distance.rs b/datafusion/functions-nested/src/distance.rsindex edf1806b66..c9aec81667 100644--- a/datafusion/functions-nested/src/distance.rs+++ b/datafusion/functions-nested/src/distance.rs@@ -18,9 +18,7 @@//! [ScalarUDFImpl] definitions for array_distance function.use crate::utils::make_scalar_function;-use arrow::array::{- Array, ArrayRef, Float64Array, LargeListArray, ListArray, OffsetSizeTrait,-};+use arrow::array::{Array, ArrayRef, Float64Array, OffsetSizeTrait};use arrow::datatypes::{DataType,DataType::{FixedSizeList, LargeList, List, Null},@@ -35,7 +33,6 @@ use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,Volatility,};-use datafusion_functions::downcast_arg;use datafusion_macros::user_doc;use itertools::Itertools;use std::sync::Arc;@@ -44,13 +41,13 @@ make_udf_expr_and_func!(ArrayDistance,array_distance,array,- "returns the Euclidean distance between two numeric arrays.",+ "returns the Euclidean distance between two one-dimensional numeric arrays.",array_distance_udf);#[user_doc(doc_section(label = "Array Functions"),- description = "Returns the Euclidean distance between two input arrays of equal length.",+ description = "Returns the Euclidean distance between two one-dimensional input arrays of equal length.",syntax_example = "array_distance(array1, array2)",sql_example = r#"```sql> select array_distance([1, 2], [1, 4]);@@ -106,16 +103,30 @@ impl ScalarUDFImpl for ArrayDistance {fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {let [_, _] = take_function_args(self.name(), arg_types)?;let coercion = Some(&ListCoercion::FixedSizedListToList);- let arg_types = arg_types.iter().map(|arg_type| {- if matches!(arg_type, Null | List(_) | LargeList(_) | FixedSizeList(..)) {+ let arg_types = arg_types.iter().map(|arg_type| match arg_type {+ Null => Ok(coerced_type_with_base_type_only(+ arg_type,+ &DataType::Float64,+ coercion,+ )),+ List(field) | LargeList(field) | FixedSizeList(field, _) => {+ // Distance between nested lists is not supported+ if matches!(+ field.data_type(),+ List(_) | LargeList(_) | FixedSizeList(..)+ ) {+ return plan_err!(+ "{} only supports one-dimensional arrays, got {arg_type}",+ self.name()+ );+ }Ok(coerced_type_with_base_type_only(arg_type,&DataType::Float64,coercion,))- } else {- plan_err!("{} does not support type {arg_type}", self.name())}+ _ => plan_err!("{} does not support type {arg_type}", self.name()),});arg_types.try_collect()@@ -172,43 +183,6 @@ fn compute_array_distance(None => return Ok(None),};- let mut value1 = value1;- let mut value2 = value2;-- loop {- match value1.data_type() {- List(_) => {- if downcast_arg!(value1, ListArray).null_count() > 0 {- return Ok(None);- }- value1 = downcast_arg!(value1, ListArray).value(0);- }- LargeList(_) => {- if downcast_arg!(value1, LargeListArray).null_count() > 0 {- return Ok(None);- }- value1 = downcast_arg!(value1, LargeListArray).value(0);- }- _ => break,- }-- match value2.data_type() {- List(_) => {- if downcast_arg!(value2, ListArray).null_count() > 0 {- return Ok(None);- }- value2 = downcast_arg!(value2, ListArray).value(0);- }- LargeList(_) => {- if downcast_arg!(value2, LargeListArray).null_count() > 0 {- return Ok(None);- }- value2 = downcast_arg!(value2, LargeListArray).value(0);- }- _ => break,- }- }-// Check for NULL values inside the arraysif value1.null_count() != 0 || value2.null_count() != 0 {return Ok(None);diff --git a/datafusion/sqllogictest/test_files/array/array_length.slt b/datafusion/sqllogictest/test_files/array/array_length.sltindex 1bb5382339..7741d815bc 100644--- a/datafusion/sqllogictest/test_files/array/array_length.slt+++ b/datafusion/sqllogictest/test_files/array/array_length.slt@@ -159,21 +159,6 @@ select array_distance([2], [3]), list_distance([1], [2]), list_distance([1], [-2query errorselect list_distance([1], [1, 2]);-query R-select array_distance([[1, 1]], [1, 2]);------1--query R-select array_distance([[1, 1]], [[1, 2]]);------1--query R-select array_distance([[1, 1]], [[1, 2]]);------1-query RRselect array_distance([1, 1, 0, 0], [2, 2, 1, 1]), list_distance([1, 2, 3], [1, 2, 3]);----@@ -204,6 +189,40 @@ select list_distance([1, 2, 3], [1, 2, 3]) AS distance;----0+# array_distance with null outer arrays+query RR+select+ array_distance(arrow_cast(NULL, 'List(Float64)'), [1, 2]),+ array_distance([1, 2], arrow_cast(NULL, 'List(Float64)'));+----+NULL NULL++# invalid argument count and types+query error DataFusion error: Error during planning: Execution error: Function 'array_distance' user-defined coercion failed with: Execution error: array_distance function requires 2 arguments+select array_distance();++query error DataFusion error: Error during planning: Execution error: Function 'array_distance' user-defined coercion failed with: Execution error: array_distance function requires 2 arguments+select array_distance([1]);++query error DataFusion error: Error during planning: Execution error: Function 'array_distance' user-defined coercion failed with: Execution error: array_distance function requires 2 arguments+select array_distance([1], [2], [3]);++query error array_distance does not support type Int64+select array_distance(1, [1]);++query error array_distance does not support types+select array_distance([1], arrow_cast([1], 'LargeList(Float64)'));++query error array_distance only supports one-dimensional arrays+select array_distance([[1, 1]], [1, 2]);++query error array_distance only supports one-dimensional arrays+select array_distance([[1, 1]], [[1, 2]]);++query error array_distance only supports one-dimensional arrays+select array_distance([[1, 2], [100, 100]], [[1, 4], [0, 0]]);++# array_distance with columnsquery RRRselect array_distance(column1, column2), array_distance(column1, column3), array_distance(column1, column4) from arrays_distance_table;diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex d97082e881..803362d9d5 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -948,3 +948,20 @@ See [PR #23827](https://github.com/apache/datafusion/pull/23827) for details.The Minimum Supported Rust Version (MSRV) has been updated to [`1.94.0`].[`1.94.0`]: https://releases.rs/docs/1.94.0/++### `array_distance` scalar function now rejects multidimensional arrays++`array_distance` only supports one-dimensional arrays. Previously, when given+multidimensional arrays, it computed the distance using only the first+subarray and ignored the remaining subarrays. For example:++```sql+SELECT array_distance(+ [[1, 2], [100, 100]],+ [[1, 4], [0, 0]]+);+```++Previously, this query returned `2.0`, the distance between `[1, 2]` and+`[1, 4]`. It now returns a planning error stating that `array_distance` only+supports one-dimensional arrays.diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex a865a3d182..e63ec0654c 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -3611,7 +3611,7 @@ array_dims(array)### `array_distance`-Returns the Euclidean distance between two input arrays of equal length.+Returns the Euclidean distance between two one-dimensional input arrays of equal length.```sqlarray_distance(array1, array2)