Skip to content

Apache DataFusion SQL Query Engine

fix: make array null argument handling follow SQL semantics (#22508)

Kumar Ujjawale4e8f2328d85
diff --git a/datafusion/functions-nested/src/array_any_match.rs b/datafusion/functions-nested/src/array_any_match.rs
index 3ce43a23c2..e3e99ad063 100644
--- a/datafusion/functions-nested/src/array_any_match.rs
+++ b/datafusion/functions-nested/src/array_any_match.rs
@@ -171,12 +171,12 @@ impl HigherOrderUDF for ArrayAnyMatch {
&self,
args: HigherOrderReturnFieldArgs,
) -> Result<Arc<Field>> {
- let [ValueOrLambda::Value(list), _] =
+ let [ValueOrLambda::Value(list), ValueOrLambda::Lambda(lambda)] =
take_function_args(self.name(), args.arg_fields)?
else {
return plan_err!("{} expects a value as first argument", self.name());
};
- let nullable = list.is_nullable();
+ let nullable = list.is_nullable() || lambda.is_nullable();
Ok(Arc::new(Field::new("", DataType::Boolean, nullable)))
}
@@ -272,14 +272,14 @@ mod tests {
};
use datafusion_common::{DFSchema, Result};
use datafusion_expr::{
- Expr, col,
+ Expr, HigherOrderReturnFieldArgs, HigherOrderUDF, ValueOrLambda, col,
execution_props::ExecutionProps,
expr::{HigherOrderFunction, LambdaVariable},
lambda, lit,
};
use datafusion_physical_expr::create_physical_expr;
- use crate::array_any_match::array_any_match_higher_order_function;
+ use crate::array_any_match::{ArrayAnyMatch, array_any_match_higher_order_function};
fn run_any_match(
list: impl arrow::array::Array + Clone + 'static,
@@ -413,6 +413,44 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_any_match_return_field_nullability() -> Result<()> {
+ for list_nullable in [true, false] {
+ for lambda_nullable in [true, false] {
+ let list = Arc::new(Field::new(
+ "list",
+ DataType::new_list(DataType::Int32, true),
+ list_nullable,
+ ));
+ let lambda =
+ Arc::new(Field::new("predicate", DataType::Boolean, lambda_nullable));
+ let arg_fields = [
+ ValueOrLambda::Value(Arc::clone(&list)),
+ ValueOrLambda::Lambda(Arc::clone(&lambda)),
+ ];
+ let scalar_arguments = [None, None];
+
+ let result = ArrayAnyMatch::new().return_field_from_args(
+ HigherOrderReturnFieldArgs {
+ arg_fields: &arg_fields,
+ scalar_arguments: &scalar_arguments,
+ },
+ )?;
+
+ assert_eq!(
+ result,
+ Arc::new(Field::new(
+ "",
+ DataType::Boolean,
+ list_nullable || lambda_nullable,
+ ))
+ );
+ }
+ }
+
+ Ok(())
+ }
+
// Predicate must not be evaluated on elements belonging to null rows.
// The 10 in the null row would satisfy x > 5, but the row result must be None.
#[test]
diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs
index fe1e31e8d5..202a76bd0b 100644
--- a/datafusion/functions-nested/src/extract.rs
+++ b/datafusion/functions-nested/src/extract.rs
@@ -256,8 +256,8 @@ where
let end = offset_window[1];
let len = end - start;
- // array is null
- if array.is_null(row_index) {
+ // array or index is null
+ if array.is_null(row_index) || indexes.is_null(row_index) {
mutable.extend_nulls(1);
continue;
}
@@ -1107,7 +1107,7 @@ mod tests {
};
use arrow::array::{ListArray, RecordBatch};
use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
- use arrow::datatypes::{DataType, Field};
+ use arrow::datatypes::{DataType, Field, Int32Type};
use datafusion_common::{Column, DFSchema, Result, assert_batches_eq};
use datafusion_expr::expr::ScalarFunction;
use datafusion_expr::{Expr, ExprSchemable};
@@ -1198,6 +1198,26 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_array_element_null_index_with_non_zero_buffer_returns_null() -> Result<()> {
+ let list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2), Some(3)]),
+ Some(vec![Some(4)]),
+ Some(vec![Some(5)]),
+ ]);
+ let indexes = Int64Array::new(
+ ScalarBuffer::from(vec![1, 1, 1]),
+ Some(NullBuffer::from(vec![true, false, true])),
+ );
+
+ let result = general_array_element(&list_array, &indexes)?;
+ let expected = Int32Array::from(vec![Some(1), None, Some(5)]);
+
+ assert_eq!(result.as_primitive::<Int32Type>(), &expected);
+
+ Ok(())
+ }
+
#[test]
fn test_array_any_null_handling() -> Result<()> {
let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rs
index 1dde2aa762..44ef56c039 100644
--- a/datafusion/functions-nested/src/remove.rs
+++ b/datafusion/functions-nested/src/remove.rs
@@ -19,10 +19,11 @@
use crate::utils;
use arrow::array::{
- Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, OffsetBufferBuilder,
- OffsetSizeTrait, Scalar, cast::AsArray, make_array,
+ Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, NullBufferBuilder,
+ OffsetBufferBuilder, OffsetSizeTrait, Scalar, cast::AsArray, make_array,
+ new_null_array,
};
-use arrow::buffer::{NullBuffer, OffsetBuffer};
+use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{DataType, FieldRef};
use datafusion_common::cast::as_int64_array;
use datafusion_common::utils::ListCoercion;
@@ -110,7 +111,9 @@ impl ScalarUDFImpl for ArrayRemove {
&self,
args: datafusion_expr::ReturnFieldArgs,
) -> Result<FieldRef> {
- Ok(Arc::clone(&args.arg_fields[0]))
+ let array_field = args.arg_fields[0].as_ref().clone();
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+ Ok(Arc::new(array_field.with_nullable(nullable)))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -128,7 +131,8 @@ impl ScalarUDFImpl for ArrayRemove {
}
element_arg => {
let element_array = element_arg.to_array(num_rows)?;
- let result = array_remove_internal(&list_array, &element_array, &[1])?;
+ let result =
+ array_remove_internal(&list_array, &element_array, &[Some(1)])?;
Ok(ColumnarValue::Array(result))
}
}
@@ -228,7 +232,9 @@ impl ScalarUDFImpl for ArrayRemoveN {
&self,
args: datafusion_expr::ReturnFieldArgs,
) -> Result<FieldRef> {
- Ok(Arc::clone(&args.arg_fields[0]))
+ let array_field = args.arg_fields[0].as_ref().clone();
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+ Ok(Arc::new(array_field.with_nullable(nullable)))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -242,8 +248,10 @@ impl ScalarUDFImpl for ArrayRemoveN {
ColumnarValue::Scalar(scalar_max),
) if !scalar_element.is_null() && !scalar_element.data_type().is_nested() => {
let ScalarValue::Int64(Some(n)) = scalar_max else {
- // null max means no remove
- return Ok(ColumnarValue::Array(list_array));
+ return Ok(ColumnarValue::Array(new_null_array(
+ list_array.data_type(),
+ num_rows,
+ )));
};
let result =
array_remove_with_scalar_args(&list_array, scalar_element, *n)?;
@@ -252,16 +260,7 @@ impl ScalarUDFImpl for ArrayRemoveN {
(element_arg, max_arg) => {
let element_array = element_arg.to_array(num_rows)?;
let max_array = max_arg.to_array(num_rows)?;
- let max_array = as_int64_array(&max_array)?;
- let arr_n = (0..max_array.len())
- .map(|i| {
- if max_array.is_null(i) {
- 0
- } else {
- max_array.value(i)
- }
- })
- .collect::<Vec<_>>();
+ let arr_n = as_int64_array(&max_array)?.iter().collect::<Vec<_>>();
let result = array_remove_internal(&list_array, &element_array, &arr_n)?;
Ok(ColumnarValue::Array(result))
}
@@ -351,7 +350,9 @@ impl ScalarUDFImpl for ArrayRemoveAll {
&self,
args: datafusion_expr::ReturnFieldArgs,
) -> Result<FieldRef> {
- Ok(Arc::clone(&args.arg_fields[0]))
+ let array_field = args.arg_fields[0].as_ref().clone();
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+ Ok(Arc::new(array_field.with_nullable(nullable)))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -369,8 +370,11 @@ impl ScalarUDFImpl for ArrayRemoveAll {
}
element_arg => {
let element_array = element_arg.to_array(num_rows)?;
- let result =
- array_remove_internal(&list_array, &element_array, &[i64::MAX])?;
+ let result = array_remove_internal(
+ &list_array,
+ &element_array,
+ &[Some(i64::MAX)],
+ )?;
Ok(ColumnarValue::Array(result))
}
}
@@ -388,7 +392,7 @@ impl ScalarUDFImpl for ArrayRemoveAll {
fn array_remove_internal(
array: &ArrayRef,
element_array: &ArrayRef,
- arr_n: &[i64],
+ arr_n: &[Option<i64>],
) -> Result<ArrayRef> {
match array.data_type() {
DataType::List(_) => {
@@ -447,7 +451,7 @@ fn array_remove_with_scalar_args(
fn general_remove<OffsetSize: OffsetSizeTrait>(
list_array: &GenericListArray<OffsetSize>,
element_array: &ArrayRef,
- arr_n: &[i64],
+ arr_n: &[Option<i64>],
) -> Result<ArrayRef> {
let list_field = match list_array.data_type() {
DataType::List(field) | DataType::LargeList(field) => field,
@@ -468,24 +472,28 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
false,
Capacities::Array(original_data.len()),
);
-
- // Pre-compute combined null bitmap
- let nulls = NullBuffer::union(list_array.nulls(), element_array.nulls());
+ let mut valid = NullBufferBuilder::new(list_array.len());
for (row_index, offset_window) in list_array.offsets().windows(2).enumerate() {
- if nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)) {
+ if list_array.is_null(row_index) || element_array.is_null(row_index) {
offsets.push(offsets[row_index]);
+ valid.append_null();
continue;
}
- let start = offset_window[0].to_usize().unwrap();
- let end = offset_window[1].to_usize().unwrap();
- // n is the number of elements to remove in this row
let n = if arr_n.len() == 1 {
arr_n[0]
} else {
arr_n[row_index]
};
+ let Some(n) = n else {
+ offsets.push(offsets[row_index]);
+ valid.append_null();
+ continue;
+ };
+
+ let start = offset_window[0].to_usize().unwrap();
+ let end = offset_window[1].to_usize().unwrap();
// compare each element in the list, `false` means the element matches and should be removed
let eq_array = utils::compare_element_to_list(
@@ -501,6 +509,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
if num_to_remove == 0 {
mutable.extend(0, start, end);
offsets.push(offsets[row_index] + OffsetSize::usize_as(end - start));
+ valid.append_non_null();
continue;
}
@@ -531,6 +540,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
}
offsets.push(offsets[row_index] + OffsetSize::usize_as(copied));
+ valid.append_non_null();
}
let new_values = make_array(mutable.freeze());
@@ -538,7 +548,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
Arc::clone(list_field),
OffsetBuffer::new(offsets.into()),
new_values,
- nulls,
+ valid.finish(),
)?))
}
@@ -645,8 +655,10 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
mod tests {
use crate::remove::{ArrayRemove, ArrayRemoveAll, ArrayRemoveN};
use arrow::array::{
- Array, ArrayRef, AsArray, GenericListArray, ListArray, OffsetSizeTrait,
+ Array, ArrayRef, AsArray, GenericListArray, Int32Array, Int64Array, ListArray,
+ OffsetSizeTrait,
};
+ use arrow::buffer::{NullBuffer, ScalarBuffer};
use arrow::datatypes::{DataType, Field, Int32Type};
use datafusion_common::ScalarValue;
use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl};
@@ -658,25 +670,34 @@ mod tests {
fn test_array_remove_nullability() {
for nullability in [true, false] {
for item_nullability in [true, false] {
- let input_field = Arc::new(Field::new(
- "num",
- DataType::new_list(DataType::Int32, item_nullability),
- nullability,
- ));
- let args_fields = vec![
- Arc::clone(&input_field),
- Arc::new(Field::new("a", DataType::Int32, false)),
- ];
- let scalar_args = vec![None, Some(&ScalarValue::Int32(Some(1)))];
-
- let result = ArrayRemove::new()
- .return_field_from_args(ReturnFieldArgs {
- arg_fields: &args_fields,
- scalar_arguments: &scalar_args,
- })
- .unwrap();
-
- assert_eq!(result, input_field);
+ for element_nullability in [true, false] {
+ let input_field = Arc::new(Field::new(
+ "num",
+ DataType::new_list(DataType::Int32, item_nullability),
+ nullability,
+ ));
+ let args_fields = vec![
+ Arc::clone(&input_field),
+ Arc::new(Field::new("a", DataType::Int32, element_nullability)),
+ ];
+ let scalar_args = vec![None, Some(&ScalarValue::Int32(Some(1)))];
+
+ let result = ArrayRemove::new()
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &args_fields,
+ scalar_arguments: &scalar_args,
+ })
+ .unwrap();
+
+ let expected = Arc::new(
+ input_field
+ .as_ref()
+ .clone()
+ .with_nullable(nullability || element_nullability),
+ );
+
+ assert_eq!(result, expected);
+ }
}
}
}
@@ -685,30 +706,47 @@ mod tests {
fn test_array_remove_n_nullability() {
for nullability in [true, false] {
for item_nullability in [true, false] {
- let input_field = Arc::new(Field::new(
- "num",
- DataType::new_list(DataType::Int32, item_nullability),
- nullability,
- ));
- let args_fields = vec![
- Arc::clone(&input_field),
- Arc::new(Field::new("a", DataType::Int32, false)),
- Arc::new(Field::new("b", DataType::Int64, false)),
- ];
- let scalar_args = vec![
- None,
- Some(&ScalarValue::Int32(Some(1))),
- Some(&ScalarValue::Int64(Some(1))),
- ];
-
- let result = ArrayRemoveN::new()
- .return_field_from_args(ReturnFieldArgs {
- arg_fields: &args_fields,
- scalar_arguments: &scalar_args,
- })
- .unwrap();
-
- assert_eq!(result, input_field);
+ for element_nullability in [true, false] {
+ for count_nullability in [true, false] {
+ let input_field = Arc::new(Field::new(
+ "num",
+ DataType::new_list(DataType::Int32, item_nullability),
+ nullability,
+ ));
+ let args_fields = vec![
+ Arc::clone(&input_field),
+ Arc::new(Field::new(
+ "a",
+ DataType::Int32,
+ element_nullability,
+ )),
+ Arc::new(Field::new("b", DataType::Int64, count_nullability)),
+ ];
+ let scalar_args = vec![
+ None,
+ Some(&ScalarValue::Int32(Some(1))),
+ Some(&ScalarValue::Int64(Some(1))),
+ ];
+
+ let result = ArrayRemoveN::new()
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &args_fields,
+ scalar_arguments: &scalar_args,
+ })
+ .unwrap();
+
+ let expected_nullable =
+ nullability || element_nullability || count_nullability;
+ let expected = Arc::new(
+ input_field
+ .as_ref()
+ .clone()
+ .with_nullable(expected_nullable),
+ );
+
+ assert_eq!(result, expected);
+ }
+ }
}
}
}
@@ -717,19 +755,33 @@ mod tests {
fn test_array_remove_all_nullability() {
for nullability in [true, false] {
for item_nullability in [true, false] {
- let input_field = Arc::new(Field::new(
- "num",
- DataType::new_list(DataType::Int32, item_nullability),
- nullability,
- ));
- let result = ArrayRemoveAll::new()
- .return_field_from_args(ReturnFieldArgs {
- arg_fields: &[Arc::clone(&input_field)],
- scalar_arguments: &[None],
- })
- .unwrap();
-
- assert_eq!(result, input_field);
+ for element_nullability in [true, false] {
+ let input_field = Arc::new(Field::new(
+ "num",
+ DataType::new_list(DataType::Int32, item_nullability),
+ nullability,
+ ));
+ let args_fields = vec![
+ Arc::clone(&input_field),
+ Arc::new(Field::new("a", DataType::Int32, element_nullability)),
+ ];
+ let scalar_args = vec![None, Some(&ScalarValue::Int32(Some(1)))];
+ let result = ArrayRemoveAll::new()
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &args_fields,
+ scalar_arguments: &scalar_args,
+ })
+ .unwrap();
+
+ let expected = Arc::new(
+ input_field
+ .as_ref()
+ .clone()
+ .with_nullable(nullability || element_nullability),
+ );
+
+ assert_eq!(result, expected);
+ }
}
}
}
@@ -907,6 +959,58 @@ mod tests {
assert_array_remove_n(input_list, expected_list, element_to_remove, 2);
}
+ #[test]
+ fn test_array_remove_n_null_count_returns_null() {
+ let array: ArrayRef =
+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2), Some(2)]),
+ Some(vec![Some(4), Some(2)]),
+ ]));
+ let element: ArrayRef = Arc::new(Int32Array::from(vec![2, 2]));
+ let max: ArrayRef = Arc::new(Int64Array::new(
+ ScalarBuffer::from(vec![1, 1]),
+ Some(NullBuffer::from(vec![true, false])),
+ ));
+
+ let udf = ArrayRemoveN::new();
+ let args_fields = vec![
+ Arc::new(Field::new("num", array.data_type().clone(), false)),
+ Arc::new(Field::new("el", DataType::Int32, false)),
+ Arc::new(Field::new("count", DataType::Int64, true)),
+ ];
+ let scalar_args = vec![None, None, None];
+ let return_field = udf
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &args_fields,
+ scalar_arguments: &scalar_args,
+ })
+ .unwrap();
+ let result = udf
+ .invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(array),
+ ColumnarValue::Array(element),
+ ColumnarValue::Array(max),
+ ],
+ arg_fields: args_fields,
+ number_rows: 2,
+ return_field,
+ config_options: Arc::new(Default::default()),
+ })
+ .unwrap();
+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ None,
+ ]);
+
+ match result {
+ ColumnarValue::Array(array) => {
+ assert_eq!(array.as_list::<i32>(), &expected);
+ }
+ _ => panic!("Expected ColumnarValue::Array"),
+ }
+ }
+
fn assert_array_remove_n(
input_list: ArrayRef,
expected_list: GenericListArray<i32>,
diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rs
index 825530923f..878ed04e6f 100644
--- a/datafusion/functions-nested/src/repeat.rs
+++ b/datafusion/functions-nested/src/repeat.rs
@@ -184,7 +184,9 @@ fn general_repeat<O: OffsetSizeTrait>(
let mut take_indices = Vec::with_capacity(total_repeated_values);
for idx in 0..count_array.len() {
- let count = get_count_with_validity(count_array, idx);
+ let Some(count) = repeat_count(count_array, idx) else {
+ continue;
+ };
take_indices.extend(std::iter::repeat_n(idx as u64, count));
}
@@ -224,7 +226,9 @@ fn general_list_repeat<O: OffsetSizeTrait>(
// calculate capacities for pre-allocation
let mut inner_total = 0usize;
for i in 0..count_array.len() {
- let count = get_count_with_validity(count_array, i);
+ let Some(count) = repeat_count(count_array, i) else {
+ continue;
+ };
if count > 0 && list_array.is_valid(i) {
let len = list_offsets[i + 1].to_usize().unwrap()
- list_offsets[i].to_usize().unwrap();
@@ -243,7 +247,9 @@ fn general_list_repeat<O: OffsetSizeTrait>(
inner_offsets.push(O::zero());
for row_idx in 0..count_array.len() {
- let count = get_count_with_validity(count_array, row_idx);
+ let Some(count) = repeat_count(count_array, row_idx) else {
+ continue;
+ };
let list_is_valid = list_array.is_valid(row_idx);
let start = list_offsets[row_idx].to_usize().unwrap();
let end = list_offsets[row_idx + 1].to_usize().unwrap();
@@ -278,7 +284,6 @@ fn general_list_repeat<O: OffsetSizeTrait>(
Some(NullBuffer::new(inner_nulls.finish())),
)?;
- // Build outer ListArray
Ok(Arc::new(GenericListArray::<O>::try_new(
Arc::new(Field::new_list_field(
list_array.data_type().to_owned(),
@@ -298,7 +303,10 @@ fn build_repeat_offsets<O: OffsetSizeTrait>(
let mut running_offset = 0usize;
for idx in 0..count_array.len() {
- let count = get_count_with_validity(count_array, idx);
+ let Some(count) = repeat_count(count_array, idx) else {
+ offsets.push(*offsets.last().unwrap());
+ continue;
+ };
running_offset = checked_repeat_len_add(running_offset, count)?;
ensure_array_repeat_output_len::<O>(running_offset)?;
let offset = O::from_usize(running_offset).ok_or_else(|| {
@@ -363,24 +371,80 @@ fn max_vec_elements<T>() -> usize {
.unwrap_or(usize::MAX)
}
-/// Helper function to get count from count_array at given index
-/// Return 0 for null values or non-positive count.
+/// Helper function to get count from count_array at given index.
+/// Returns `None` for NULL values and `Some(0)` for non-positive counts.
#[inline]
-fn get_count_with_validity(count_array: &Int64Array, idx: usize) -> usize {
+fn repeat_count(count_array: &Int64Array, idx: usize) -> Option<usize> {
if count_array.is_null(idx) {
- 0
+ None
} else {
let c = count_array.value(idx);
- if c > 0 { c as usize } else { 0 }
+ Some(if c > 0 { c as usize } else { 0 })
}
}
#[cfg(test)]
mod tests {
- use super::array_repeat_inner;
- use arrow::array::{ArrayRef, Int64Array};
+ use super::{array_repeat_inner, general_list_repeat, general_repeat};
+ use arrow::array::{Array, ArrayRef, AsArray, Int32Array, Int64Array, ListArray};
+ use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
+ use arrow::datatypes::{Field, Int32Type};
+ use datafusion_common::Result;
use std::sync::Arc;
+ #[test]
+ fn test_array_repeat_null_count_stays_null() -> Result<()> {
+ let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let counts = Int64Array::new(
+ ScalarBuffer::from(vec![2, 1, 1]),
+ Some(NullBuffer::from(vec![true, false, true])),
+ );
+
+ let result = general_repeat::<i32>(&array, &counts)?;
+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(1)]),
+ None,
+ Some(vec![Some(3)]),
+ ]);
+
+ assert_eq!(result.as_list::<i32>(), &expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_array_repeat_nested_null_count_stays_null() -> Result<()> {
+ let list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(3), Some(4)]),
+ Some(vec![Some(5)]),
+ ]);
+ let counts = Int64Array::new(
+ ScalarBuffer::from(vec![2, 1, 1]),
+ Some(NullBuffer::from(vec![true, false, true])),
+ );
+
+ let result = general_list_repeat::<i32>(&list_array, &counts)?;
+ let repeated_values = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(5)]),
+ ]);
+ let expected = ListArray::new(
+ Arc::new(Field::new_list_field(
+ repeated_values.data_type().clone(),
+ true,
+ )),
+ OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 2, 3])),
+ Arc::new(repeated_values),
+ Some(NullBuffer::from(vec![true, false, true])),
+ );
+
+ assert_eq!(result.as_list::<i32>(), &expected);
+
+ Ok(())
+ }
+
#[test]
fn scalar_count_exceeding_max_array_size_returns_error() {
let element: ArrayRef = Arc::new(Int64Array::from(vec![1]));
diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rs
index 908218f536..f129972fc7 100644
--- a/datafusion/functions-nested/src/replace.rs
+++ b/datafusion/functions-nested/src/replace.rs
@@ -139,8 +139,12 @@ impl ScalarUDFImpl for ArrayReplace {
(from_arg, to_arg) => {
let from_array = from_arg.to_array(num_rows)?;
let to_array = to_arg.to_array(num_rows)?;
- let result =
- array_replace_internal(&list_array, &from_array, &to_array, &[1])?;
+ let result = array_replace_internal(
+ &list_array,
+ &from_array,
+ &to_array,
+ &[Some(1)],
+ )?;
Ok(ColumnarValue::Array(result))
}
}
@@ -229,8 +233,10 @@ impl ScalarUDFImpl for ArrayReplaceN {
ColumnarValue::Scalar(scalar_max),
) => {
let ScalarValue::Int64(Some(n)) = scalar_max else {
- // null max means no replacements
- return Ok(ColumnarValue::Array(list_array));
+ return Ok(ColumnarValue::Array(new_null_array(
+ list_array.data_type(),
+ num_rows,
+ )));
};
let result = array_replace_with_scalar_args(
&list_array,
@@ -244,18 +250,12 @@ impl ScalarUDFImpl for ArrayReplaceN {
let from_array = from_arg.to_array(num_rows)?;
let to_array = to_arg.to_array(num_rows)?;
let max_array = max_arg.to_array(num_rows)?;
- let max_array = as_int64_array(&max_array)?;
- let arr_n = (0..max_array.len())
- .map(|i| {
- if max_array.is_null(i) {
- 0
- } else {
- max_array.value(i)
- }
- })
- .collect::<Vec<_>>();
- let result =
- array_replace_internal(&list_array, &from_array, &to_array, &arr_n)?;
+ let result = array_replace_n_inner(
+ &list_array,
+ &from_array,
+ &to_array,
+ &max_array,
+ )?;
Ok(ColumnarValue::Array(result))
}
}
@@ -351,7 +351,7 @@ impl ScalarUDFImpl for ArrayReplaceAll {
&list_array,
&from_array,
&to_array,
- &[i64::MAX],
+ &[Some(i64::MAX)],
)?;
Ok(ColumnarValue::Array(result))
}
@@ -388,7 +388,7 @@ fn general_replace<O: OffsetSizeTrait>(
list_array: &GenericListArray<O>,
from_array: &ArrayRef,
to_array: &ArrayRef,
- arr_n: &[i64],
+ arr_n: &[Option<i64>],
) -> Result<ArrayRef> {
// Build up the offsets for the final output array
let mut offsets: Vec<O> = vec![O::usize_as(0)];
@@ -413,6 +413,17 @@ fn general_replace<O: OffsetSizeTrait>(
continue;
}
+ let n = if arr_n.len() == 1 {
+ arr_n[0]
+ } else {
+ arr_n[row_index]
+ };
+ let Some(n) = n else {
+ offsets.push(offsets[row_index]);
+ valid.append_null();
+ continue;
+ };
+
let start = offset_window[0];
let end = offset_window[1];
@@ -425,11 +436,6 @@ fn general_replace<O: OffsetSizeTrait>(
let original_idx = O::usize_as(0);
let replace_idx = O::usize_as(1);
- let n = if arr_n.len() == 1 {
- arr_n[0]
- } else {
- arr_n[row_index]
- };
let mut counter = 0;
// All elements are false, no need to replace, just copy original data
@@ -611,7 +617,7 @@ fn array_replace_with_scalar_args(
list_array,
&from_array,
&to_array,
- &vec![max_replacements; num_rows],
+ &vec![Some(max_replacements); num_rows],
);
}
@@ -634,7 +640,7 @@ fn array_replace_internal(
array: &ArrayRef,
from: &ArrayRef,
to: &ArrayRef,
- arr_n: &[i64],
+ arr_n: &[Option<i64>],
) -> Result<ArrayRef> {
match array.data_type() {
DataType::List(_) => {
@@ -649,3 +655,87 @@ fn array_replace_internal(
array_type => exec_err!("array_replace does not support type '{array_type}'."),
}
}
+
+fn array_replace_n_inner(
+ array: &ArrayRef,
+ from: &ArrayRef,
+ to: &ArrayRef,
+ max: &ArrayRef,
+) -> Result<ArrayRef> {
+ let arr_n = as_int64_array(max)?.iter().collect::<Vec<_>>();
+ array_replace_internal(array, from, to, &arr_n)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{ArrayReplaceN, array_replace_n_inner};
+ use arrow::array::{ArrayRef, AsArray, Int32Array, Int64Array, ListArray};
+ use arrow::buffer::{NullBuffer, ScalarBuffer};
+ use arrow::datatypes::{DataType, Field, Int32Type};
+ use datafusion_common::{Result, ScalarValue, config::ConfigOptions};
+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+ use std::sync::Arc;
+
+ #[test]
+ fn test_array_replace_n_null_max_returns_null() -> Result<()> {
+ let array: ArrayRef =
+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2), Some(3)]),
+ Some(vec![Some(4), Some(2)]),
+ ]));
+ let from: ArrayRef = Arc::new(Int32Array::from(vec![2, 2]));
+ let to: ArrayRef = Arc::new(Int32Array::from(vec![9, 9]));
+ let max: ArrayRef = Arc::new(Int64Array::new(
+ ScalarBuffer::from(vec![1, 1]),
+ Some(NullBuffer::from(vec![true, false])),
+ ));
+
+ let result = array_replace_n_inner(&array, &from, &to, &max)?;
+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(9), Some(3)]),
+ None,
+ ]);
+
+ assert_eq!(result.as_list::<i32>(), &expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_array_replace_n_scalar_null_max_returns_null() -> Result<()> {
+ let array: ArrayRef =
+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2), Some(3)]),
+ Some(vec![Some(4), Some(2)]),
+ ]));
+ let array_field = Arc::new(Field::new("array", array.data_type().clone(), true));
+
+ let result = ArrayReplaceN::new().invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(Arc::clone(&array)),
+ ColumnarValue::Scalar(ScalarValue::Int32(Some(2))),
+ ColumnarValue::Scalar(ScalarValue::Int32(Some(9))),
+ ColumnarValue::Scalar(ScalarValue::Int64(None)),
+ ],
+ arg_fields: vec![
+ Arc::clone(&array_field),
+ Arc::new(Field::new("from", DataType::Int32, false)),
+ Arc::new(Field::new("to", DataType::Int32, false)),
+ Arc::new(Field::new("max", DataType::Int64, true)),
+ ],
+ number_rows: array.len(),
+ return_field: Arc::clone(&array_field),
+ config_options: Arc::new(ConfigOptions::default()),
+ })?;
+
+ let result = result.into_array(array.len())?;
+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Option::<Vec<Option<i32>>>::None,
+ Option::<Vec<Option<i32>>>::None,
+ ]);
+
+ assert_eq!(result.as_list::<i32>(), &expected);
+
+ Ok(())
+ }
+}
diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rs
index 243f3531f9..d11064bf7e 100644
--- a/datafusion/functions-nested/src/resize.rs
+++ b/datafusion/functions-nested/src/resize.rs
@@ -203,7 +203,7 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(
let mut max_extra: usize = 0;
let mut output_values_len: usize = 0;
for (row_index, offset_window) in array.offsets().windows(2).enumerate() {
- if array.is_null(row_index) {
+ if array.is_null(row_index) || count_array.is_null(row_index) {
continue;
}
let target_count = count_array.value(row_index).to_usize().ok_or_else(|| {
@@ -308,7 +308,7 @@ where
let mut null_builder = NullBufferBuilder::new(array.len());
for (row_index, offset_window) in array.offsets().windows(2).enumerate() {
- if array.is_null(row_index) {
+ if array.is_null(row_index) || count_array.is_null(row_index) {
null_builder.append_null();
offsets.push(offsets[row_index]);
continue;
@@ -341,3 +341,36 @@ where
null_builder.finish(),
)?))
}
+
+#[cfg(test)]
+mod tests {
+ use super::array_resize_inner;
+ use arrow::array::{ArrayRef, AsArray, Int64Array, ListArray};
+ use arrow::buffer::{NullBuffer, ScalarBuffer};
+ use arrow::datatypes::Int32Type;
+ use datafusion_common::Result;
+ use std::sync::Arc;
+
+ #[test]
+ fn test_array_resize_null_size_returns_null() -> Result<()> {
+ let array: ArrayRef =
+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2), Some(3)]),
+ Some(vec![Some(4), Some(5)]),
+ ]));
+ let size: ArrayRef = Arc::new(Int64Array::new(
+ ScalarBuffer::from(vec![2, 1]),
+ Some(NullBuffer::from(vec![true, false])),
+ ));
+
+ let result = array_resize_inner(&[array, size])?;
+ let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ None,
+ ]);
+
+ assert_eq!(result.as_list::<i32>(), &expected);
+
+ Ok(())
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/array/array_element.slt b/datafusion/sqllogictest/test_files/array/array_element.slt
index 7c960653ed..846d869ec6 100644
--- a/datafusion/sqllogictest/test_files/array/array_element.slt
+++ b/datafusion/sqllogictest/test_files/array/array_element.slt
@@ -231,6 +231,22 @@ NULL
NULL
55
+# array_element with null index from column
+query I
+select array_element(column1, column2) from slices where column2 is NULL;
+----
+NULL
+
+query I
+select array_element(arrow_cast(column1, 'LargeList(Int64)'), column2) from slices where column2 is NULL;
+----
+NULL
+
+query I
+select array_element(column1, column2) from fixed_slices where column2 is NULL;
+----
+NULL
+
# array_element with columns and scalars
query II
select array_element(make_array(1, 2, 3, 4, 5), column2), array_element(column1, 3) from slices;
diff --git a/datafusion/sqllogictest/test_files/array/array_remove.slt b/datafusion/sqllogictest/test_files/array/array_remove.slt
index 195f7a0f33..23ebf00239 100644
--- a/datafusion/sqllogictest/test_files/array/array_remove.slt
+++ b/datafusion/sqllogictest/test_files/array/array_remove.slt
@@ -271,7 +271,7 @@ query ??
select array_remove_n(make_array(1, 2, 2, 1, 1), 2, NULL),
array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, NULL);
----
-[1, 2, 2, 1, 1] [1, 2, 2, 1, 1]
+NULL NULL
# array_remove_n with null element scalar (LargeList)
query ??
@@ -280,6 +280,20 @@ select array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'),
----
NULL [1, 1, 1]
+# array_remove_n with null max scalar
+query ??
+select array_remove_n(make_array(1, 2, 2, 1, 1), 2, NULL),
+ array_remove_n(make_array(1, 2, 2, 1, 1), 2, 2);
+----
+NULL [1, 1, 1]
+
+# array_remove_n with null max scalar (LargeList)
+query ??
+select array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, NULL),
+ array_remove_n(arrow_cast(make_array(1, 2, 2, 1, 1), 'LargeList(Int64)'), 2, 2);
+----
+NULL [1, 1, 1]
+
# array_remove_n with null element from column
query ?
select array_remove_n(column1, column2, column3) from (values
@@ -293,7 +307,7 @@ select array_remove_n(column1, column2, column3) from (values
[1, 1, 1]
NULL
[5, 6, 5, 5]
-[7, 8, 8, 7, 7]
+NULL
NULL
# array_remove_n with null element from column (LargeList)
@@ -308,7 +322,7 @@ select array_remove_n(column1, column2, column3) from (values
[1, 1, 1]
NULL
[5, 6, 5, 5]
-[7, 8, 8, 7, 7]
+NULL
# array_remove_n scalar function #1
query ???
diff --git a/datafusion/sqllogictest/test_files/array/array_replace.slt b/datafusion/sqllogictest/test_files/array/array_replace.slt
index f83e3b4d75..77793228c9 100644
--- a/datafusion/sqllogictest/test_files/array/array_replace.slt
+++ b/datafusion/sqllogictest/test_files/array/array_replace.slt
@@ -356,19 +356,19 @@ select
query ?
select array_replace_n(make_array(1, 2, 3, 4, 5), NULL, NULL, NULL);
----
-[1, 2, 3, 4, 5]
+NULL
query ?
select array_replace_n(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), NULL, NULL, NULL);
----
-[1, 2, 3, 4, 5]
+NULL
query ??
select
array_replace_n(make_array(1, 2, 2), 2, 9, NULL),
array_replace_n(arrow_cast(make_array(1, 2, 2), 'LargeList(Int64)'), 2, 9, NULL);
----
-[1, 2, 2] [1, 2, 2]
+NULL NULL
# array_replace_n with null max from column
query ?
@@ -378,7 +378,7 @@ select array_replace_n(column1, column2, column3, column4) from (values
) as t(column1, column2, column3, column4);
----
[1, 9, 9]
-[3, 4, 4]
+NULL
# array_replace_n scalar function with columns #1
query ?
diff --git a/datafusion/sqllogictest/test_files/array/array_resize.slt b/datafusion/sqllogictest/test_files/array/array_resize.slt
index 91febb76ac..cb2ffa3a7e 100644
--- a/datafusion/sqllogictest/test_files/array/array_resize.slt
+++ b/datafusion/sqllogictest/test_files/array/array_resize.slt
@@ -75,6 +75,12 @@ select array_resize(arrow_cast(make_array(1.1, 2.2, 3.3), 'LargeList(Float64)'),
----
[1.1, 2.2, 3.3, 9.9, 9.9, 9.9, 9.9, 9.9, 9.9, 9.9]
+# array_resize null size
+query ??
+select array_resize(make_array(1, 2, 3), NULL), array_resize(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), NULL);
+----
+NULL NULL
+
# array_resize scalar function #5
query ?
select array_resize(column1, column2, column3) from arrays_values;
@@ -84,7 +90,7 @@ select array_resize(column1, column2, column3) from arrays_values;
[21, 22, 23, NULL, 25, 26, 27, 28, 29, 30, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]
[31, 32, 33, 34, 35, NULL, 37, 38, 39, 40, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]
NULL
-[]
+NULL
[51, 52, NULL, 54, 55, 56, 57, 58, 59, 60, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL]
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7]
@@ -96,7 +102,7 @@ select array_resize(arrow_cast(column1, 'LargeList(Int64)'), column2, column3) f
[21, 22, 23, NULL, 25, 26, 27, 28, 29, 30, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]
[31, 32, 33, 34, 35, NULL, 37, 38, 39, 40, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]
NULL
-[]
+NULL
[51, 52, NULL, 54, 55, 56, 57, 58, 59, 60, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL]
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7]
@@ -139,7 +145,7 @@ select array_resize(column1, column2, column3) from array_resize_values;
[21, 22, 23, 24, NULL, 26, 27, 28]
[31, 32, 33, 34, 35, 36, NULL, 38, 39, 40, 4, 4]
NULL
-[]
+NULL
[51, 52, 53, 54, 55, NULL, 57, 58, 59, 60, NULL, NULL, NULL]
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7]
@@ -152,7 +158,7 @@ select array_resize(arrow_cast(column1, 'LargeList(Int64)'), column2, column3) f
[21, 22, 23, 24, NULL, 26, 27, 28]
[31, 32, 33, 34, 35, 36, NULL, 38, 39, 40, 4, 4]
NULL
-[]
+NULL
[51, 52, 53, 54, 55, NULL, 57, 58, 59, 60, NULL, NULL, NULL]
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 7, 7, 7, 7, 7]
@@ -165,7 +171,7 @@ select array_resize(column1, column2, 9) from array_resize_values;
[21, 22, 23, 24, NULL, 26, 27, 28]
[31, 32, 33, 34, 35, 36, NULL, 38, 39, 40, 9, 9]
NULL
-[]
+NULL
[51, 52, 53, 54, 55, NULL, 57, 58, 59, 60, 9, 9, 9]
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 9, 9, 9, 9, 9]