Apache DataFusion SQL Query Engine
fix: maintain inner list nullability for `array_sort` (#19948)
Jeffrey Vo8023947fadd1
diff --git a/datafusion/functions-nested/src/sort.rs b/datafusion/functions-nested/src/sort.rsindex ba2da0f760..cbe101f111 100644--- a/datafusion/functions-nested/src/sort.rs+++ b/datafusion/functions-nested/src/sort.rs@@ -18,16 +18,14 @@//! [`ScalarUDFImpl`] definitions for array_sort function.use crate::utils::make_scalar_function;-use arrow::array::{- Array, ArrayRef, GenericListArray, NullBufferBuilder, OffsetSizeTrait, new_null_array,-};+use arrow::array::{Array, ArrayRef, GenericListArray, OffsetSizeTrait, new_null_array};use arrow::buffer::OffsetBuffer;use arrow::compute::SortColumn;use arrow::datatypes::{DataType, FieldRef};use arrow::{compute, compute::SortOptions};use datafusion_common::cast::{as_large_list_array, as_list_array, as_string_array};use datafusion_common::utils::ListCoercion;-use datafusion_common::{Result, exec_err, plan_err};+use datafusion_common::{Result, exec_err};use datafusion_expr::{ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation,ScalarUDFImpl, Signature, TypeSignature, Volatility,@@ -134,18 +132,7 @@ impl ScalarUDFImpl for ArraySort {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- match &arg_types[0] {- DataType::Null => Ok(DataType::Null),- DataType::List(field) => {- Ok(DataType::new_list(field.data_type().clone(), true))- }- DataType::LargeList(field) => {- Ok(DataType::new_large_list(field.data_type().clone(), true))- }- arg_type => {- plan_err!("{} does not support type {arg_type}", self.name())- }- }+ Ok(arg_types[0].clone())}fn invoke_with_args(@@ -206,11 +193,11 @@ fn array_sort_inner(args: &[ArrayRef]) -> Result<ArrayRef> {}DataType::List(field) => {let array = as_list_array(&args[0])?;- array_sort_generic(array, field, sort_options)+ array_sort_generic(array, Arc::clone(field), sort_options)}DataType::LargeList(field) => {let array = as_large_list_array(&args[0])?;- array_sort_generic(array, field, sort_options)+ array_sort_generic(array, Arc::clone(field), sort_options)}// Signature should prevent this arm ever occurring_ => exec_err!("array_sort expects list for first argument"),@@ -219,18 +206,16 @@ fn array_sort_inner(args: &[ArrayRef]) -> Result<ArrayRef> {fn array_sort_generic<OffsetSize: OffsetSizeTrait>(list_array: &GenericListArray<OffsetSize>,- field: &FieldRef,+ field: FieldRef,sort_options: Option<SortOptions>,) -> Result<ArrayRef> {let row_count = list_array.len();let mut array_lengths = vec![];let mut arrays = vec![];- let mut valid = NullBufferBuilder::new(row_count);for i in 0..row_count {if list_array.is_null(i) {array_lengths.push(0);- valid.append_null();} else {let arr_ref = list_array.value(i);@@ -253,25 +238,22 @@ fn array_sort_generic<OffsetSize: OffsetSizeTrait>(};array_lengths.push(sorted_array.len());arrays.push(sorted_array);- valid.append_non_null();}}- let buffer = valid.finish();-let elements = arrays.iter().map(|a| a.as_ref()).collect::<Vec<&dyn Array>>();let list_arr = if elements.is_empty() {- GenericListArray::<OffsetSize>::new_null(Arc::clone(field), row_count)+ GenericListArray::<OffsetSize>::new_null(field, row_count)} else {GenericListArray::<OffsetSize>::new(- Arc::clone(field),+ field,OffsetBuffer::from_lengths(array_lengths),Arc::new(compute::concat(elements.as_slice())?),- buffer,+ list_array.nulls().cloned(),)};Ok(Arc::new(list_arr))diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.sltindex e0a9746793..590f6b1a9a 100644--- a/datafusion/sqllogictest/test_files/array.slt+++ b/datafusion/sqllogictest/test_files/array.slt@@ -2577,6 +2577,31 @@ NULL NULLNULL NULLNULL NULL+# maintains inner nullability+query ?T+select array_sort(column1), arrow_typeof(array_sort(column1))+from values+ (arrow_cast([], 'List(non-null Int32)')),+ (arrow_cast(NULL, 'List(non-null Int32)')),+ (arrow_cast([1, 3, 5, -5], 'List(non-null Int32)'))+;+----+[] List(non-null Int32)+NULL List(non-null Int32)+[-5, 1, 3, 5] List(non-null Int32)++query ?T+select column1, arrow_typeof(column1)+from values (array_sort(arrow_cast([1, 3, 5, -5], 'LargeList(non-null Int32)')));+----+[-5, 1, 3, 5] LargeList(non-null Int32)++query ?T+select column1, arrow_typeof(column1)+from values (array_sort(arrow_cast([1, 3, 5, -5], 'FixedSizeList(4 x non-null Int32)')));+----+[-5, 1, 3, 5] List(non-null Int32)+query ?select array_sort([struct('foo', 3), struct('foo', 1), struct('bar', 1)])----