Apache DataFusion SQL Query Engine
fix: make array null argument handling follow SQL semantics (#22508)
Kumar Ujjawale4e8f2328d85
diff --git a/datafusion/functions-nested/src/array_any_match.rs b/datafusion/functions-nested/src/array_any_match.rsindex 3ce43a23c2..e3e99ad063 100644--- a/datafusion/functions-nested/src/array_any_match.rs+++ b/datafusion/functions-nested/src/array_any_match.rs@@ -171,12 +171,12 @@ impl HigherOrderUDF for ArrayAnyMatch {&self,args: HigherOrderReturnFieldArgs,) -> Result<Arc<Field>> {- let [ValueOrLambda::Value(list), _] =+ let [ValueOrLambda::Value(list), ValueOrLambda::Lambda(lambda)] =take_function_args(self.name(), args.arg_fields)?else {return plan_err!("{} expects a value as first argument", self.name());};- let nullable = list.is_nullable();+ let nullable = list.is_nullable() || lambda.is_nullable();Ok(Arc::new(Field::new("", DataType::Boolean, nullable)))}@@ -272,14 +272,14 @@ mod tests {};use datafusion_common::{DFSchema, Result};use datafusion_expr::{- Expr, col,+ Expr, HigherOrderReturnFieldArgs, HigherOrderUDF, ValueOrLambda, col,execution_props::ExecutionProps,expr::{HigherOrderFunction, LambdaVariable},lambda, lit,};use datafusion_physical_expr::create_physical_expr;- use crate::array_any_match::array_any_match_higher_order_function;+ use crate::array_any_match::{ArrayAnyMatch, array_any_match_higher_order_function};fn run_any_match(list: impl arrow::array::Array + Clone + 'static,@@ -413,6 +413,44 @@ mod tests {Ok(())}+ #[test]+ fn test_any_match_return_field_nullability() -> Result<()> {+ for list_nullable in [true, false] {+ for lambda_nullable in [true, false] {+ let list = Arc::new(Field::new(+ "list",+ DataType::new_list(DataType::Int32, true),+ list_nullable,+ ));+ let lambda =+ Arc::new(Field::new("predicate", DataType::Boolean, lambda_nullable));+ let arg_fields = [+ ValueOrLambda::Value(Arc::clone(&list)),+ ValueOrLambda::Lambda(Arc::clone(&lambda)),+ ];+ let scalar_arguments = [None, None];++ let result = ArrayAnyMatch::new().return_field_from_args(+ HigherOrderReturnFieldArgs {+ arg_fields: &arg_fields,+ scalar_arguments: &scalar_arguments,+ },+ )?;++ assert_eq!(+ result,+ Arc::new(Field::new(+ "",+ DataType::Boolean,+ list_nullable || lambda_nullable,+ ))+ );+ }+ }++ Ok(())+ }+// Predicate must not be evaluated on elements belonging to null rows.// The 10 in the null row would satisfy x > 5, but the row result must be None.#[test]diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rsindex fe1e31e8d5..202a76bd0b 100644--- a/datafusion/functions-nested/src/extract.rs+++ b/datafusion/functions-nested/src/extract.rs@@ -256,8 +256,8 @@ wherelet end = offset_window[1];let len = end - start;- // array is null- if array.is_null(row_index) {+ // array or index is null+ if array.is_null(row_index) || indexes.is_null(row_index) {mutable.extend_nulls(1);continue;}@@ -1107,7 +1107,7 @@ mod tests {};use arrow::array::{ListArray, RecordBatch};use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};- use arrow::datatypes::{DataType, Field};+ use arrow::datatypes::{DataType, Field, Int32Type};use datafusion_common::{Column, DFSchema, Result, assert_batches_eq};use datafusion_expr::expr::ScalarFunction;use datafusion_expr::{Expr, ExprSchemable};@@ -1198,6 +1198,26 @@ mod tests {Ok(())}+ #[test]+ fn test_array_element_null_index_with_non_zero_buffer_returns_null() -> Result<()> {+ let list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ Some(vec![Some(4)]),+ Some(vec![Some(5)]),+ ]);+ let indexes = Int64Array::new(+ ScalarBuffer::from(vec![1, 1, 1]),+ Some(NullBuffer::from(vec![true, false, true])),+ );++ let result = general_array_element(&list_array, &indexes)?;+ let expected = Int32Array::from(vec![Some(1), None, Some(5)]);++ assert_eq!(result.as_primitive::<Int32Type>(), &expected);++ Ok(())+ }+#[test]fn test_array_any_null_handling() -> Result<()> {let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rsindex 1dde2aa762..44ef56c039 100644--- a/datafusion/functions-nested/src/remove.rs+++ b/datafusion/functions-nested/src/remove.rs@@ -19,10 +19,11 @@use crate::utils;use arrow::array::{- Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, OffsetBufferBuilder,- OffsetSizeTrait, Scalar, cast::AsArray, make_array,+ Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, NullBufferBuilder,+ OffsetBufferBuilder, OffsetSizeTrait, Scalar, cast::AsArray, make_array,+ new_null_array,};-use arrow::buffer::{NullBuffer, OffsetBuffer};+use arrow::buffer::OffsetBuffer;use arrow::datatypes::{DataType, FieldRef};use datafusion_common::cast::as_int64_array;use datafusion_common::utils::ListCoercion;@@ -110,7 +111,9 @@ impl ScalarUDFImpl for ArrayRemove {&self,args: datafusion_expr::ReturnFieldArgs,) -> Result<FieldRef> {- Ok(Arc::clone(&args.arg_fields[0]))+ let array_field = args.arg_fields[0].as_ref().clone();+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());+ Ok(Arc::new(array_field.with_nullable(nullable)))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -128,7 +131,8 @@ impl ScalarUDFImpl for ArrayRemove {}element_arg => {let element_array = element_arg.to_array(num_rows)?;- let result = array_remove_internal(&list_array, &element_array, &[1])?;+ let result =+ array_remove_internal(&list_array, &element_array, &[Some(1)])?;Ok(ColumnarValue::Array(result))}}@@ -228,7 +232,9 @@ impl ScalarUDFImpl for ArrayRemoveN {&self,args: datafusion_expr::ReturnFieldArgs,) -> Result<FieldRef> {- Ok(Arc::clone(&args.arg_fields[0]))+ let array_field = args.arg_fields[0].as_ref().clone();+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());+ Ok(Arc::new(array_field.with_nullable(nullable)))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -242,8 +248,10 @@ impl ScalarUDFImpl for ArrayRemoveN {ColumnarValue::Scalar(scalar_max),) if !scalar_element.is_null() && !scalar_element.data_type().is_nested() => {let ScalarValue::Int64(Some(n)) = scalar_max else {- // null max means no remove- return Ok(ColumnarValue::Array(list_array));+ return Ok(ColumnarValue::Array(new_null_array(+ list_array.data_type(),+ num_rows,+ )));};let result =array_remove_with_scalar_args(&list_array, scalar_element, *n)?;@@ -252,16 +260,7 @@ impl ScalarUDFImpl for ArrayRemoveN {(element_arg, max_arg) => {let element_array = element_arg.to_array(num_rows)?;let max_array = max_arg.to_array(num_rows)?;- let max_array = as_int64_array(&max_array)?;- let arr_n = (0..max_array.len())- .map(|i| {- if max_array.is_null(i) {- 0- } else {- max_array.value(i)- }- })- .collect::<Vec<_>>();+ let arr_n = as_int64_array(&max_array)?.iter().collect::<Vec<_>>();let result = array_remove_internal(&list_array, &element_array, &arr_n)?;Ok(ColumnarValue::Array(result))}@@ -351,7 +350,9 @@ impl ScalarUDFImpl for ArrayRemoveAll {&self,args: datafusion_expr::ReturnFieldArgs,) -> Result<FieldRef> {- Ok(Arc::clone(&args.arg_fields[0]))+ let array_field = args.arg_fields[0].as_ref().clone();+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());+ Ok(Arc::new(array_field.with_nullable(nullable)))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -369,8 +370,11 @@ impl ScalarUDFImpl for ArrayRemoveAll {}element_arg => {let element_array = element_arg.to_array(num_rows)?;- let result =- array_remove_internal(&list_array, &element_array, &[i64::MAX])?;+ let result = array_remove_internal(+ &list_array,+ &element_array,+ &[Some(i64::MAX)],+ )?;Ok(ColumnarValue::Array(result))}}@@ -388,7 +392,7 @@ impl ScalarUDFImpl for ArrayRemoveAll {fn array_remove_internal(array: &ArrayRef,element_array: &ArrayRef,- arr_n: &[i64],+ arr_n: &[Option<i64>],) -> Result<ArrayRef> {match array.data_type() {DataType::List(_) => {@@ -447,7 +451,7 @@ fn array_remove_with_scalar_args(fn general_remove<OffsetSize: OffsetSizeTrait>(list_array: &GenericListArray<OffsetSize>,element_array: &ArrayRef,- arr_n: &[i64],+ arr_n: &[Option<i64>],) -> Result<ArrayRef> {let list_field = match list_array.data_type() {DataType::List(field) | DataType::LargeList(field) => field,@@ -468,24 +472,28 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(false,Capacities::Array(original_data.len()),);-- // Pre-compute combined null bitmap- let nulls = NullBuffer::union(list_array.nulls(), element_array.nulls());+ let mut valid = NullBufferBuilder::new(list_array.len());for (row_index, offset_window) in list_array.offsets().windows(2).enumerate() {- if nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)) {+ if list_array.is_null(row_index) || element_array.is_null(row_index) {offsets.push(offsets[row_index]);+ valid.append_null();continue;}- let start = offset_window[0].to_usize().unwrap();- let end = offset_window[1].to_usize().unwrap();- // n is the number of elements to remove in this rowlet n = if arr_n.len() == 1 {arr_n[0]} else {arr_n[row_index]};+ let Some(n) = n else {+ offsets.push(offsets[row_index]);+ valid.append_null();+ continue;+ };++ let start = offset_window[0].to_usize().unwrap();+ let end = offset_window[1].to_usize().unwrap();// compare each element in the list, `false` means the element matches and should be removedlet eq_array = utils::compare_element_to_list(@@ -501,6 +509,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(if num_to_remove == 0 {mutable.extend(0, start, end);offsets.push(offsets[row_index] + OffsetSize::usize_as(end - start));+ valid.append_non_null();continue;}@@ -531,6 +540,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(}offsets.push(offsets[row_index] + OffsetSize::usize_as(copied));+ valid.append_non_null();}let new_values = make_array(mutable.freeze());@@ -538,7 +548,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(Arc::clone(list_field),OffsetBuffer::new(offsets.into()),new_values,- nulls,+ valid.finish(),)?))}@@ -645,8 +655,10 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(mod tests {use crate::remove::{ArrayRemove, ArrayRemoveAll, ArrayRemoveN};use arrow::array::{- Array, ArrayRef, AsArray, GenericListArray, ListArray, OffsetSizeTrait,+ Array, ArrayRef, AsArray, GenericListArray, Int32Array, Int64Array, ListArray,+ OffsetSizeTrait,};+ use arrow::buffer::{NullBuffer, ScalarBuffer};use arrow::datatypes::{DataType, Field, Int32Type};use datafusion_common::ScalarValue;use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl};@@ -658,25 +670,34 @@ mod tests {fn test_array_remove_nullability() {for nullability in [true, false] {for item_nullability in [true, false] {- let input_field = Arc::new(Field::new(- "num",- DataType::new_list(DataType::Int32, item_nullability),- nullability,- ));- let args_fields = vec![- Arc::clone(&input_field),- Arc::new(Field::new("a", DataType::Int32, false)),- ];- let scalar_args = vec![None, Some(&ScalarValue::Int32(Some(1)))];-- let result = ArrayRemove::new()- .return_field_from_args(ReturnFieldArgs {- arg_fields: &args_fields,- scalar_arguments: &scalar_args,- })- .unwrap();-- assert_eq!(result, input_field);+ for element_nullability in [true, false] {+ let input_field = Arc::new(Field::new(+ "num",+ DataType::new_list(DataType::Int32, item_nullability),+ nullability,+ ));+ let args_fields = vec![+ Arc::clone(&input_field),+ Arc::new(Field::new("a", DataType::Int32, element_nullability)),+ ];+ let scalar_args = vec![None, Some(&ScalarValue::Int32(Some(1)))];++ let result = ArrayRemove::new()+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &args_fields,+ scalar_arguments: &scalar_args,+ })+ .unwrap();++ let expected = Arc::new(+ input_field+ .as_ref()+ .clone()+ .with_nullable(nullability || element_nullability),+ );++ assert_eq!(result, expected);+ }}}}@@ -685,30 +706,47 @@ mod tests {fn test_array_remove_n_nullability() {for nullability in [true, false] {for item_nullability in [true, false] {- let input_field = Arc::new(Field::new(- "num",- DataType::new_list(DataType::Int32, item_nullability),- nullability,- ));- let args_fields = vec![- Arc::clone(&input_field),- Arc::new(Field::new("a", DataType::Int32, false)),- Arc::new(Field::new("b", DataType::Int64, false)),- ];- let scalar_args = vec![- None,- Some(&ScalarValue::Int32(Some(1))),- Some(&ScalarValue::Int64(Some(1))),- ];-- let result = ArrayRemoveN::new()- .return_field_from_args(ReturnFieldArgs {- arg_fields: &args_fields,- scalar_arguments: &scalar_args,- })- .unwrap();-- assert_eq!(result, input_field);+ for element_nullability in [true, false] {+ for count_nullability in [true, false] {+ let input_field = Arc::new(Field::new(+ "num",+ DataType::new_list(DataType::Int32, item_nullability),+ nullability,+ ));+ let args_fields = vec![+ Arc::clone(&input_field),+ Arc::new(Field::new(+ "a",+ DataType::Int32,+ element_nullability,+ )),+ Arc::new(Field::new("b", DataType::Int64, count_nullability)),+ ];+ let scalar_args = vec![+ None,+ Some(&ScalarValue::Int32(Some(1))),+ Some(&ScalarValue::Int64(Some(1))),+ ];++ let result = ArrayRemoveN::new()+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &args_fields,+ scalar_arguments: &scalar_args,+ })+ .unwrap();++ let expected_nullable =+ nullability || element_nullability || count_nullability;+ let expected = Arc::new(+ input_field+ .as_ref()+ .clone()+ .with_nullable(expected_nullable),+ );++ assert_eq!(result, expected);+ }+ }}}}@@ -717,19 +755,33 @@ mod tests {fn test_array_remove_all_nullability() {for nullability in [true, false] {for item_nullability in [true, false] {- let input_field = Arc::new(Field::new(- "num",- DataType::new_list(DataType::Int32, item_nullability),- nullability,- ));- let result = ArrayRemoveAll::new()- .return_field_from_args(ReturnFieldArgs {- arg_fields: &[Arc::clone(&input_field)],- scalar_arguments: &[None],- })- .unwrap();-- assert_eq!(result, input_field);+ for element_nullability in [true, false] {+ let input_field = Arc::new(Field::new(+ "num",+ DataType::new_list(DataType::Int32, item_nullability),+ nullability,+ ));+ let args_fields = vec![+ Arc::clone(&input_field),+ Arc::new(Field::new("a", DataType::Int32, element_nullability)),+ ];+ let scalar_args = vec![None, Some(&ScalarValue::Int32(Some(1)))];+ let result = ArrayRemoveAll::new()+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &args_fields,+ scalar_arguments: &scalar_args,+ })+ .unwrap();++ let expected = Arc::new(+ input_field+ .as_ref()+ .clone()+ .with_nullable(nullability || element_nullability),+ );++ assert_eq!(result, expected);+ }}}}@@ -907,6 +959,58 @@ mod tests {assert_array_remove_n(input_list, expected_list, element_to_remove, 2);}+ #[test]+ fn test_array_remove_n_null_count_returns_null() {+ let array: ArrayRef =+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(2)]),+ Some(vec![Some(4), Some(2)]),+ ]));+ let element: ArrayRef = Arc::new(Int32Array::from(vec![2, 2]));+ let max: ArrayRef = Arc::new(Int64Array::new(+ ScalarBuffer::from(vec![1, 1]),+ Some(NullBuffer::from(vec![true, false])),+ ));++ let udf = ArrayRemoveN::new();+ let args_fields = vec![+ Arc::new(Field::new("num", array.data_type().clone(), false)),+ Arc::new(Field::new("el", DataType::Int32, false)),+ Arc::new(Field::new("count", DataType::Int64, true)),+ ];+ let scalar_args = vec![None, None, None];+ let return_field = udf+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &args_fields,+ scalar_arguments: &scalar_args,+ })+ .unwrap();+ let result = udf+ .invoke_with_args(ScalarFunctionArgs {+ args: vec![+ ColumnarValue::Array(array),+ ColumnarValue::Array(element),+ ColumnarValue::Array(max),+ ],+ arg_fields: args_fields,+ number_rows: 2,+ return_field,+ config_options: Arc::new(Default::default()),+ })+ .unwrap();+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2)]),+ None,+ ]);++ match result {+ ColumnarValue::Array(array) => {+ assert_eq!(array.as_list::<i32>(), &expected);+ }+ _ => panic!("Expected ColumnarValue::Array"),+ }+ }+fn assert_array_remove_n(input_list: ArrayRef,expected_list: GenericListArray<i32>,diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rsindex 825530923f..878ed04e6f 100644--- a/datafusion/functions-nested/src/repeat.rs+++ b/datafusion/functions-nested/src/repeat.rs@@ -184,7 +184,9 @@ fn general_repeat<O: OffsetSizeTrait>(let mut take_indices = Vec::with_capacity(total_repeated_values);for idx in 0..count_array.len() {- let count = get_count_with_validity(count_array, idx);+ let Some(count) = repeat_count(count_array, idx) else {+ continue;+ };take_indices.extend(std::iter::repeat_n(idx as u64, count));}@@ -224,7 +226,9 @@ fn general_list_repeat<O: OffsetSizeTrait>(// calculate capacities for pre-allocationlet mut inner_total = 0usize;for i in 0..count_array.len() {- let count = get_count_with_validity(count_array, i);+ let Some(count) = repeat_count(count_array, i) else {+ continue;+ };if count > 0 && list_array.is_valid(i) {let len = list_offsets[i + 1].to_usize().unwrap()- list_offsets[i].to_usize().unwrap();@@ -243,7 +247,9 @@ fn general_list_repeat<O: OffsetSizeTrait>(inner_offsets.push(O::zero());for row_idx in 0..count_array.len() {- let count = get_count_with_validity(count_array, row_idx);+ let Some(count) = repeat_count(count_array, row_idx) else {+ continue;+ };let list_is_valid = list_array.is_valid(row_idx);let start = list_offsets[row_idx].to_usize().unwrap();let end = list_offsets[row_idx + 1].to_usize().unwrap();@@ -278,7 +284,6 @@ fn general_list_repeat<O: OffsetSizeTrait>(Some(NullBuffer::new(inner_nulls.finish())),)?;- // Build outer ListArrayOk(Arc::new(GenericListArray::<O>::try_new(Arc::new(Field::new_list_field(list_array.data_type().to_owned(),@@ -298,7 +303,10 @@ fn build_repeat_offsets<O: OffsetSizeTrait>(let mut running_offset = 0usize;for idx in 0..count_array.len() {- let count = get_count_with_validity(count_array, idx);+ let Some(count) = repeat_count(count_array, idx) else {+ offsets.push(*offsets.last().unwrap());+ continue;+ };running_offset = checked_repeat_len_add(running_offset, count)?;ensure_array_repeat_output_len::<O>(running_offset)?;let offset = O::from_usize(running_offset).ok_or_else(|| {@@ -363,24 +371,80 @@ fn max_vec_elements<T>() -> usize {.unwrap_or(usize::MAX)}-/// Helper function to get count from count_array at given index-/// Return 0 for null values or non-positive count.+/// Helper function to get count from count_array at given index.+/// Returns `None` for NULL values and `Some(0)` for non-positive counts.#[inline]-fn get_count_with_validity(count_array: &Int64Array, idx: usize) -> usize {+fn repeat_count(count_array: &Int64Array, idx: usize) -> Option<usize> {if count_array.is_null(idx) {- 0+ None} else {let c = count_array.value(idx);- if c > 0 { c as usize } else { 0 }+ Some(if c > 0 { c as usize } else { 0 })}}#[cfg(test)]mod tests {- use super::array_repeat_inner;- use arrow::array::{ArrayRef, Int64Array};+ use super::{array_repeat_inner, general_list_repeat, general_repeat};+ use arrow::array::{Array, ArrayRef, AsArray, Int32Array, Int64Array, ListArray};+ use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};+ use arrow::datatypes::{Field, Int32Type};+ use datafusion_common::Result;use std::sync::Arc;+ #[test]+ fn test_array_repeat_null_count_stays_null() -> Result<()> {+ let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));+ let counts = Int64Array::new(+ ScalarBuffer::from(vec![2, 1, 1]),+ Some(NullBuffer::from(vec![true, false, true])),+ );++ let result = general_repeat::<i32>(&array, &counts)?;+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(1)]),+ None,+ Some(vec![Some(3)]),+ ]);++ assert_eq!(result.as_list::<i32>(), &expected);++ Ok(())+ }++ #[test]+ fn test_array_repeat_nested_null_count_stays_null() -> Result<()> {+ let list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2)]),+ Some(vec![Some(3), Some(4)]),+ Some(vec![Some(5)]),+ ]);+ let counts = Int64Array::new(+ ScalarBuffer::from(vec![2, 1, 1]),+ Some(NullBuffer::from(vec![true, false, true])),+ );++ let result = general_list_repeat::<i32>(&list_array, &counts)?;+ let repeated_values = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2)]),+ Some(vec![Some(1), Some(2)]),+ Some(vec![Some(5)]),+ ]);+ let expected = ListArray::new(+ Arc::new(Field::new_list_field(+ repeated_values.data_type().clone(),+ true,+ )),+ OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 2, 3])),+ Arc::new(repeated_values),+ Some(NullBuffer::from(vec![true, false, true])),+ );++ assert_eq!(result.as_list::<i32>(), &expected);++ Ok(())+ }+#[test]fn scalar_count_exceeding_max_array_size_returns_error() {let element: ArrayRef = Arc::new(Int64Array::from(vec![1]));diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rsindex 908218f536..f129972fc7 100644--- a/datafusion/functions-nested/src/replace.rs+++ b/datafusion/functions-nested/src/replace.rs@@ -139,8 +139,12 @@ impl ScalarUDFImpl for ArrayReplace {(from_arg, to_arg) => {let from_array = from_arg.to_array(num_rows)?;let to_array = to_arg.to_array(num_rows)?;- let result =- array_replace_internal(&list_array, &from_array, &to_array, &[1])?;+ let result = array_replace_internal(+ &list_array,+ &from_array,+ &to_array,+ &[Some(1)],+ )?;Ok(ColumnarValue::Array(result))}}@@ -229,8 +233,10 @@ impl ScalarUDFImpl for ArrayReplaceN {ColumnarValue::Scalar(scalar_max),) => {let ScalarValue::Int64(Some(n)) = scalar_max else {- // null max means no replacements- return Ok(ColumnarValue::Array(list_array));+ return Ok(ColumnarValue::Array(new_null_array(+ list_array.data_type(),+ num_rows,+ )));};let result = array_replace_with_scalar_args(&list_array,@@ -244,18 +250,12 @@ impl ScalarUDFImpl for ArrayReplaceN {let from_array = from_arg.to_array(num_rows)?;let to_array = to_arg.to_array(num_rows)?;let max_array = max_arg.to_array(num_rows)?;- let max_array = as_int64_array(&max_array)?;- let arr_n = (0..max_array.len())- .map(|i| {- if max_array.is_null(i) {- 0- } else {- max_array.value(i)- }- })- .collect::<Vec<_>>();- let result =- array_replace_internal(&list_array, &from_array, &to_array, &arr_n)?;+ let result = array_replace_n_inner(+ &list_array,+ &from_array,+ &to_array,+ &max_array,+ )?;Ok(ColumnarValue::Array(result))}}@@ -351,7 +351,7 @@ impl ScalarUDFImpl for ArrayReplaceAll {&list_array,&from_array,&to_array,- &[i64::MAX],+ &[Some(i64::MAX)],)?;Ok(ColumnarValue::Array(result))}@@ -388,7 +388,7 @@ fn general_replace<O: OffsetSizeTrait>(list_array: &GenericListArray<O>,from_array: &ArrayRef,to_array: &ArrayRef,- arr_n: &[i64],+ arr_n: &[Option<i64>],) -> Result<ArrayRef> {// Build up the offsets for the final output arraylet mut offsets: Vec<O> = vec![O::usize_as(0)];@@ -413,6 +413,17 @@ fn general_replace<O: OffsetSizeTrait>(continue;}+ let n = if arr_n.len() == 1 {+ arr_n[0]+ } else {+ arr_n[row_index]+ };+ let Some(n) = n else {+ offsets.push(offsets[row_index]);+ valid.append_null();+ continue;+ };+let start = offset_window[0];let end = offset_window[1];@@ -425,11 +436,6 @@ fn general_replace<O: OffsetSizeTrait>(let original_idx = O::usize_as(0);let replace_idx = O::usize_as(1);- let n = if arr_n.len() == 1 {- arr_n[0]- } else {- arr_n[row_index]- };let mut counter = 0;// All elements are false, no need to replace, just copy original data@@ -611,7 +617,7 @@ fn array_replace_with_scalar_args(list_array,&from_array,&to_array,- &vec![max_replacements; num_rows],+ &vec![Some(max_replacements); num_rows],);}@@ -634,7 +640,7 @@ fn array_replace_internal(array: &ArrayRef,from: &ArrayRef,to: &ArrayRef,- arr_n: &[i64],+ arr_n: &[Option<i64>],) -> Result<ArrayRef> {match array.data_type() {DataType::List(_) => {@@ -649,3 +655,87 @@ fn array_replace_internal(array_type => exec_err!("array_replace does not support type '{array_type}'."),}}++fn array_replace_n_inner(+ array: &ArrayRef,+ from: &ArrayRef,+ to: &ArrayRef,+ max: &ArrayRef,+) -> Result<ArrayRef> {+ let arr_n = as_int64_array(max)?.iter().collect::<Vec<_>>();+ array_replace_internal(array, from, to, &arr_n)+}++#[cfg(test)]+mod tests {+ use super::{ArrayReplaceN, array_replace_n_inner};+ use arrow::array::{ArrayRef, AsArray, Int32Array, Int64Array, ListArray};+ use arrow::buffer::{NullBuffer, ScalarBuffer};+ use arrow::datatypes::{DataType, Field, Int32Type};+ use datafusion_common::{Result, ScalarValue, config::ConfigOptions};+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};+ use std::sync::Arc;++ #[test]+ fn test_array_replace_n_null_max_returns_null() -> Result<()> {+ let array: ArrayRef =+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ Some(vec![Some(4), Some(2)]),+ ]));+ let from: ArrayRef = Arc::new(Int32Array::from(vec![2, 2]));+ let to: ArrayRef = Arc::new(Int32Array::from(vec![9, 9]));+ let max: ArrayRef = Arc::new(Int64Array::new(+ ScalarBuffer::from(vec![1, 1]),+ Some(NullBuffer::from(vec![true, false])),+ ));++ let result = array_replace_n_inner(&array, &from, &to, &max)?;+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(9), Some(3)]),+ None,+ ]);++ assert_eq!(result.as_list::<i32>(), &expected);++ Ok(())+ }++ #[test]+ fn test_array_replace_n_scalar_null_max_returns_null() -> Result<()> {+ let array: ArrayRef =+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ Some(vec![Some(4), Some(2)]),+ ]));+ let array_field = Arc::new(Field::new("array", array.data_type().clone(), true));++ let result = ArrayReplaceN::new().invoke_with_args(ScalarFunctionArgs {+ args: vec![+ ColumnarValue::Array(Arc::clone(&array)),+ ColumnarValue::Scalar(ScalarValue::Int32(Some(2))),+ ColumnarValue::Scalar(ScalarValue::Int32(Some(9))),+ ColumnarValue::Scalar(ScalarValue::Int64(None)),+ ],+ arg_fields: vec![+ Arc::clone(&array_field),+ Arc::new(Field::new("from", DataType::Int32, false)),+ Arc::new(Field::new("to", DataType::Int32, false)),+ Arc::new(Field::new("max", DataType::Int64, true)),+ ],+ number_rows: array.len(),+ return_field: Arc::clone(&array_field),+ config_options: Arc::new(ConfigOptions::default()),+ })?;++ let result = result.into_array(array.len())?;+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Option::<Vec<Option<i32>>>::None,+ Option::<Vec<Option<i32>>>::None,+ ]);++ assert_eq!(result.as_list::<i32>(), &expected);++ Ok(())+ }+}diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rsindex 243f3531f9..d11064bf7e 100644--- a/datafusion/functions-nested/src/resize.rs+++ b/datafusion/functions-nested/src/resize.rs@@ -203,7 +203,7 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(let mut max_extra: usize = 0;let mut output_values_len: usize = 0;for (row_index, offset_window) in array.offsets().windows(2).enumerate() {- if array.is_null(row_index) {+ if array.is_null(row_index) || count_array.is_null(row_index) {continue;}let target_count = count_array.value(row_index).to_usize().ok_or_else(|| {@@ -308,7 +308,7 @@ wherelet mut null_builder = NullBufferBuilder::new(array.len());for (row_index, offset_window) in array.offsets().windows(2).enumerate() {- if array.is_null(row_index) {+ if array.is_null(row_index) || count_array.is_null(row_index) {null_builder.append_null();offsets.push(offsets[row_index]);continue;@@ -341,3 +341,36 @@ wherenull_builder.finish(),)?))}++#[cfg(test)]+mod tests {+ use super::array_resize_inner;+ use arrow::array::{ArrayRef, AsArray, Int64Array, ListArray};+ use arrow::buffer::{NullBuffer, ScalarBuffer};+ use arrow::datatypes::Int32Type;+ use datafusion_common::Result;+ use std::sync::Arc;++ #[test]+ fn test_array_resize_null_size_returns_null() -> Result<()> {+ let array: ArrayRef =+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ Some(vec![Some(4), Some(5)]),+ ]));+ let size: ArrayRef = Arc::new(Int64Array::new(+ ScalarBuffer::from(vec![2, 1]),+ Some(NullBuffer::from(vec![true, false])),+ ));++ let result = array_resize_inner(&[array, size])?;+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2)]),+ None,+ ]);++ assert_eq!(result.as_list::<i32>(), &expected);++ Ok(())+ }+}diff --git a/datafusion/sqllogictest/test_files/array/array_element.slt b/datafusion/sqllogictest/test_files/array/array_element.sltindex 7c960653ed..846d869ec6 100644--- a/datafusion/sqllogictest/test_files/array/array_element.slt+++ b/datafusion/sqllogictest/test_files/array/array_element.slt@@ -231,6 +231,22 @@ NULLNULL55+# array_element with null index from column+query I+select array_element(column1, column2) from slices where column2 is NULL;+----+NULL++query I+select array_element(arrow_cast(column1, 'LargeList(Int64)'), column2) from slices where column2 is NULL;+----+NULL++query I+select array_element(column1, column2) from fixed_slices where column2 is NULL;+----+NULL+# array_element with columns and scalarsquery IIselect array_element(make_array(1, 2, 3, 4, 5), column2), array_element(column1, 3) from slices;diff --git a/datafusion/sqllogictest/test_files/array/array_remove.slt b/datafusion/sqllogictest/test_files/array/array_remove.sltindex 195f7a0f33..23ebf00239 100644--- a/datafusion/sqllogictest/test_files/array/array_remove.slt+++ b/datafusion/sqllogictest/test_files/array/array_remove.slt@@ -271,7 +271,7 @@ query ??select array_remove_n(make_array(1, 2, 2, 1, 1), 2, NULL),array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, NULL);-----[1, 2, 2, 1, 1] [1, 2, 2, 1, 1]+NULL NULL# array_remove_n with null element scalar (LargeList)query ??@@ -280,6 +280,20 @@ select array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'),----NULL [1, 1, 1]+# array_remove_n with null max scalar+query ??+select array_remove_n(make_array(1, 2, 2, 1, 1), 2, NULL),+ array_remove_n(make_array(1, 2, 2, 1, 1), 2, 2);+----+NULL [1, 1, 1]++# array_remove_n with null max scalar (LargeList)+query ??+select array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, NULL),+ array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, 2);+----+NULL [1, 1, 1]+# array_remove_n with null element from columnquery ?select array_remove_n(column1, column2, column3) from (values@@ -293,7 +307,7 @@ select array_remove_n(column1, column2, column3) from (values[1, 1, 1]NULL[5, 6, 5, 5]-[7, 8, 8, 7, 7]+NULLNULL# array_remove_n with null element from column (LargeList)@@ -308,7 +322,7 @@ select array_remove_n(column1, column2, column3) from (values[1, 1, 1]NULL[5, 6, 5, 5]-[7, 8, 8, 7, 7]+NULL# array_remove_n scalar function #1query ???diff --git a/datafusion/sqllogictest/test_files/array/array_replace.slt b/datafusion/sqllogictest/test_files/array/array_replace.sltindex f83e3b4d75..77793228c9 100644--- a/datafusion/sqllogictest/test_files/array/array_replace.slt+++ b/datafusion/sqllogictest/test_files/array/array_replace.slt@@ -356,19 +356,19 @@ selectquery ?select array_replace_n(make_array(1, 2, 3, 4, 5), NULL, NULL, NULL);-----[1, 2, 3, 4, 5]+NULLquery ?select array_replace_n(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), NULL, NULL, NULL);-----[1, 2, 3, 4, 5]+NULLquery ??selectarray_replace_n(make_array(1, 2, 2), 2, 9, NULL),array_replace_n(arrow_cast(make_array(1, 2, 2), 'LargeList(Int64)'), 2, 9, NULL);-----[1, 2, 2] [1, 2, 2]+NULL NULL# array_replace_n with null max from columnquery ?@@ -378,7 +378,7 @@ select array_replace_n(column1, column2, column3, column4) from (values) as t(column1, column2, column3, column4);----[1, 9, 9]-[3, 4, 4]+NULL# array_replace_n scalar function with columns #1query ?diff --git a/datafusion/sqllogictest/test_files/array/array_resize.slt b/datafusion/sqllogictest/test_files/array/array_resize.sltindex 91febb76ac..cb2ffa3a7e 100644--- a/datafusion/sqllogictest/test_files/array/array_resize.slt+++ b/datafusion/sqllogictest/test_files/array/array_resize.slt@@ -75,6 +75,12 @@ select array_resize(arrow_cast(make_array(1.1, 2.2, 3.3), 'LargeList(Float64)'),----[1.1, 2.2, 3.3, 9.9, 9.9, 9.9, 9.9, 9.9, 9.9, 9.9]+# array_resize null size+query ??+select array_resize(make_array(1, 2, 3), NULL), array_resize(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), NULL);+----+NULL NULL+# array_resize scalar function #5query ?select array_resize(column1, column2, column3) from arrays_values;@@ -84,7 +90,7 @@ select array_resize(column1, column2, column3) from arrays_values;[21, 22, 23, NULL, 25, 26, 27, 28, 29, 30, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3][31, 32, 33, 34, 35, NULL, 37, 38, 39, 40, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]NULL-[]+NULL[51, 52, NULL, 54, 55, 56, 57, 58, 59, 60, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL][61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7]@@ -96,7 +102,7 @@ select array_resize(arrow_cast(column1, 'LargeList(Int64)'), column2, column3) f[21, 22, 23, NULL, 25, 26, 27, 28, 29, 30, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3][31, 32, 33, 34, 35, NULL, 37, 38, 39, 40, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]NULL-[]+NULL[51, 52, NULL, 54, 55, 56, 57, 58, 59, 60, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL][61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7]@@ -139,7 +145,7 @@ select array_resize(column1, column2, column3) from array_resize_values;[21, 22, 23, 24, NULL, 26, 27, 28][31, 32, 33, 34, 35, 36, NULL, 38, 39, 40, 4, 4]NULL-[]+NULL[51, 52, 53, 54, 55, NULL, 57, 58, 59, 60, NULL, NULL, NULL][61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7]@@ -152,7 +158,7 @@ select array_resize(arrow_cast(column1, 'LargeList(Int64)'), column2, column3) f[21, 22, 23, 24, NULL, 26, 27, 28][31, 32, 33, 34, 35, 36, NULL, 38, 39, 40, 4, 4]NULL-[]+NULL[51, 52, 53, 54, 55, NULL, 57, 58, 59, 60, NULL, NULL, NULL][61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7]@@ -165,7 +171,7 @@ select array_resize(column1, column2, 9) from array_resize_values;[21, 22, 23, 24, NULL, 26, 27, 28][31, 32, 33, 34, 35, 36, NULL, 38, 39, 40, 9, 9]NULL-[]+NULL[51, 52, 53, 54, 55, NULL, 57, 58, 59, 60, 9, 9, 9][61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 9, 9, 9, 9, 9]