Apache DataFusion SQL Query Engine
chore: Use `value_field` to fetch list element field (#26567)
Neil Conway139da57bdae4
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex 6727ab7dcc..0ff108ff36 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -1356,14 +1356,8 @@ fn truncate_list_nulls<O: OffsetSizeTrait>(let offsets = OffsetBuffer::from_lengths(lengths);let values = make_array(mutable_array_data.freeze());- let field = match list.data_type() {- DataType::List(field) => field,- DataType::LargeList(field) => field,- _ => unreachable!(),- };-return Ok(GenericListArray::try_new(- Arc::clone(field),+ Arc::clone(list.value_field()),offsets,values,list.nulls().cloned(),diff --git a/datafusion/functions-nested/src/array_compact.rs b/datafusion/functions-nested/src/array_compact.rsindex 12e3638294..4e61d1a837 100644--- a/datafusion/functions-nested/src/array_compact.rs+++ b/datafusion/functions-nested/src/array_compact.rs@@ -26,8 +26,8 @@ use arrow::array::{};use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};use arrow::compute::filter;+use arrow::datatypes::DataType;use arrow::datatypes::DataType::{LargeList, List, Null};-use arrow::datatypes::{DataType, FieldRef};use datafusion_common::cast::as_generic_list_array;use datafusion_common::utils::offset_span_len;use datafusion_common::{Result, exec_err, utils::take_function_args};@@ -115,8 +115,8 @@ fn array_compact_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {let [input_array] = take_function_args("array_compact", arg)?;match &input_array.data_type() {- List(field) => compact_list::<i32>(input_array, field),- LargeList(field) => compact_list::<i64>(input_array, field),+ List(_) => compact_list::<i32>(input_array),+ LargeList(_) => compact_list::<i64>(input_array),Null => Ok(Arc::clone(input_array)),array_type => exec_err!("array_compact does not support type '{array_type}'."),}@@ -126,10 +126,7 @@ fn array_compact_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {////// Each row is a range in a shared child array. Compaction removes null child/// values and rebuilds the row offsets, preserving null list rows.-fn compact_list<O: OffsetSizeTrait>(- input_array: &ArrayRef,- field: &FieldRef,-) -> Result<ArrayRef> {+fn compact_list<O: OffsetSizeTrait>(input_array: &ArrayRef) -> Result<ArrayRef> {let list_array = as_generic_list_array::<O>(input_array.as_ref())?;let visible_len = offset_span_len(list_array.offsets());if visible_len == 0 || list_array.null_count() == list_array.len() {@@ -152,10 +149,13 @@ fn compact_list<O: OffsetSizeTrait>(return Ok(Arc::clone(input_array));};if values_nulls.null_count() == values_nulls.len() {- return Ok(empty_list_values(list_array, Arc::clone(field)));+ return Ok(empty_list_values(+ list_array,+ Arc::clone(list_array.value_field()),+ ));}- build_compacted_list(list_array, field, values.as_ref(), &values_nulls)+ build_compacted_list(list_array, values.as_ref(), &values_nulls)}/// Copy primitive values while computing result row offsets. For other types,@@ -167,7 +167,6 @@ fn compact_list<O: OffsetSizeTrait>(/// subtract the first list offset when indexing `values`.fn build_compacted_list<O: OffsetSizeTrait>(list_array: &GenericListArray<O>,- field: &FieldRef,values: &dyn Array,values_nulls: &NullBuffer,) -> Result<ArrayRef> {@@ -176,7 +175,7 @@ fn build_compacted_list<O: OffsetSizeTrait>(_ => compact_non_primitive(values, list_array, values_nulls),}?;Ok(Arc::new(GenericListArray::<O>::try_new(- Arc::clone(field),+ Arc::clone(list_array.value_field()),offsets,new_values,list_array.nulls().cloned(),diff --git a/datafusion/functions-nested/src/array_scale.rs b/datafusion/functions-nested/src/array_scale.rsindex 5dad7cb93b..6a283ee3a3 100644--- a/datafusion/functions-nested/src/array_scale.rs+++ b/datafusion/functions-nested/src/array_scale.rs@@ -205,15 +205,8 @@ fn general_array_scale<O: OffsetSizeTrait>(// Preserve the inner field from the input array (including any user// metadata). After `coerce_types` the inner type is Float64, but the// input may still carry field-level annotations worth keeping.- let field = match list_array.data_type() {- List(f) | LargeList(f) => Arc::clone(f),- other => {- return internal_err!("array_scale unexpected list type: {other}");- }- };-Ok(Arc::new(GenericListArray::<O>::try_new(- field,+ Arc::clone(list_array.value_field()),OffsetBuffer::new(new_offsets.into()),values_array,row_nulls,diff --git a/datafusion/functions-nested/src/except.rs b/datafusion/functions-nested/src/except.rsindex fd2b6a0897..cf680199bf 100644--- a/datafusion/functions-nested/src/except.rs+++ b/datafusion/functions-nested/src/except.rs@@ -26,7 +26,7 @@ use arrow::array::{};use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::compute::take;-use arrow::datatypes::{DataType, FieldRef};+use arrow::datatypes::DataType;use arrow::row::{RowConverter, SortField};use datafusion_common::utils::{ListCoercion, take_function_args};use datafusion_common::{HashSet, Result, internal_err};@@ -143,18 +143,18 @@ fn array_except_inner(args: &[ArrayRef]) -> Result<ArrayRef> {| (DataType::Null, dt @ DataType::LargeList(_))| (dt @ DataType::List(_), DataType::Null)| (dt @ DataType::LargeList(_), DataType::Null) => Ok(new_null_array(dt, len)),- (DataType::List(field), DataType::List(_)) => {+ (DataType::List(_), DataType::List(_)) => {check_datatypes("array_except", &[array1, array2])?;let list1 = array1.as_list::<i32>();let list2 = array2.as_list::<i32>();- let result = general_except::<i32>(list1, list2, field)?;+ let result = general_except::<i32>(list1, list2)?;Ok(Arc::new(result))}- (DataType::LargeList(field), DataType::LargeList(_)) => {+ (DataType::LargeList(_), DataType::LargeList(_)) => {check_datatypes("array_except", &[array1, array2])?;let list1 = array1.as_list::<i64>();let list2 = array2.as_list::<i64>();- let result = general_except::<i64>(list1, list2, field)?;+ let result = general_except::<i64>(list1, list2)?;Ok(Arc::new(result))}(dt1, dt2) => {@@ -166,7 +166,6 @@ fn array_except_inner(args: &[ArrayRef]) -> Result<ArrayRef> {fn general_except<OffsetSize: OffsetSizeTrait>(l: &GenericListArray<OffsetSize>,r: &GenericListArray<OffsetSize>,- field: &FieldRef,) -> Result<GenericListArray<OffsetSize>> {let converter = RowConverter::new(vec![SortField::new(l.value_type())])?;@@ -231,7 +230,7 @@ fn general_except<OffsetSize: OffsetSizeTrait>(};Ok(GenericListArray::<OffsetSize>::new(- field.to_owned(),+ Arc::clone(l.value_field()),OffsetBuffer::new(offsets.into()),values,nulls,@@ -242,7 +241,7 @@ fn general_except<OffsetSize: OffsetSizeTrait>(mod tests {use super::{ArrayExcept, general_except};use arrow::array::{Array, AsArray, Int32Array, LargeListArray, ListArray};- use arrow::datatypes::{DataType, Field, Float64Type, Int32Type};+ use arrow::datatypes::{Field, Float64Type, Int32Type};use datafusion_common::{Result, config::ConfigOptions};use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};use std::sync::Arc;@@ -312,11 +311,7 @@ mod tests {Some(vec![Some(-98.0)]),]).slice(1, 1);- let DataType::LargeList(field) = l.data_type() else {- unreachable!()- };-- let result = general_except::<i64>(&l, &r, field)?;+ let result = general_except::<i64>(&l, &r)?;let values = result.value(0);let values = values.as_primitive::<Float64Type>();assert_eq!(values.len(), 1);diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rsindex 25ce9ecfeb..c2f9c22c75 100644--- a/datafusion/functions-nested/src/extract.rs+++ b/datafusion/functions-nested/src/extract.rs@@ -31,7 +31,6 @@ use datafusion_common::cast::as_list_array;use datafusion_common::cast::{as_int64_array, as_large_list_view_array, as_list_view_array,};-use datafusion_common::internal_err;use datafusion_common::utils::ListCoercion;use datafusion_common::utils::offset_span_len;use datafusion_common::{@@ -48,7 +47,7 @@ use datafusion_expr::{use datafusion_macros::user_doc;use std::sync::Arc;-use crate::utils::{list_inner_field, make_scalar_function};+use crate::utils::make_scalar_function;// Create static instances of ScalarUDFs for each functionmake_udf_expr_and_func!(@@ -608,7 +607,7 @@ where// Carry the input's list field through to the output so that the returned// type matches the one promised by `return_type` / `return_field_from_args`,// including the field name, nullability and metadata.- let field = list_inner_field("general_array_slice", array.data_type())?;+ let field = Arc::clone(array.value_field());// `use_nulls` is false because we never call `try_extend_nulls`: null rows are// emitted as empty slices. Arrow still allocates a validity buffer on its own@@ -696,14 +695,7 @@ wherelet values = array.values();let original_data = values.to_data();let capacity = Capacities::Array(original_data.len());- let field = match array.data_type() {- ListView(field) | LargeListView(field) => Arc::clone(field),- other => {- return internal_err!(- "general_list_view_array_slice got unexpected data type: {other}"- );- }- };+ let field = Arc::clone(array.value_field());// See the note on `use_nulls` in `general_array_slice`.let mut mutable =diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rsindex befcc09d22..20f36d3d33 100644--- a/datafusion/functions-nested/src/remove.rs+++ b/datafusion/functions-nested/src/remove.rs@@ -454,14 +454,6 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(element_array: &ArrayRef,arr_n: &[Option<i64>],) -> Result<ArrayRef> {- let (DataType::List(list_field) | DataType::LargeList(list_field)) =- list_array.data_type()- else {- return exec_err!(- "Expected List or LargeList data type, got {:?}",- list_array.data_type()- );- };let original_data = list_array.values().to_data();// Build up the offsets for the final output arraylet mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);@@ -545,7 +537,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(let new_values = make_array(mutable.freeze());Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(- Arc::clone(list_field),+ Arc::clone(list_array.value_field()),OffsetBuffer::new(offsets.into()),new_values,valid.finish(),@@ -566,15 +558,6 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(return Ok(Arc::new(list_array.clone()));}- let (DataType::List(list_field) | DataType::LargeList(list_field)) =- list_array.data_type()- else {- return exec_err!(- "Expected List or LargeList data type, got {:?}",- list_array.data_type()- );- };-let list_offsets = list_array.offsets();let (first_offset, values_range_len) = offset_span(list_offsets);let values_slice = list_array.values().slice(first_offset, values_range_len);@@ -642,7 +625,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(let new_values = make_array(mutable.freeze());Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(- Arc::clone(list_field),+ Arc::clone(list_array.value_field()),OffsetBuffer::new(offsets.into()),new_values,nulls,diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rsindex 6b682e3ce9..2a96bb8b5a 100644--- a/datafusion/functions-nested/src/resize.rs+++ b/datafusion/functions-nested/src/resize.rs@@ -24,11 +24,8 @@ use arrow::array::{};use arrow::buffer::OffsetBuffer;use arrow::datatypes::DataType;+use arrow::datatypes::DataType::{LargeList, List};use arrow::datatypes::Field;-use arrow::datatypes::{- DataType::{LargeList, List},- FieldRef,-};use datafusion_common::cast::{as_int64_array, as_large_list_array, as_list_array};use datafusion_common::utils::ListCoercion;use datafusion_common::{@@ -177,13 +174,13 @@ fn array_resize_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {};match &arg[0].data_type() {- List(field) => {+ List(_) => {let array = as_list_array(&arg[0])?;- general_list_resize::<i32>(array, new_len, field, new_element)+ general_list_resize::<i32>(array, new_len, new_element)}- LargeList(field) => {+ LargeList(_) => {let array = as_large_list_array(&arg[0])?;- general_list_resize::<i64>(array, new_len, field, new_element)+ general_list_resize::<i64>(array, new_len, new_element)}array_type => exec_err!("array_resize does not support type '{array_type}'."),}@@ -200,7 +197,6 @@ fn resize_count(count_array: &Int64Array, idx: usize) -> Result<usize> {fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(array: &GenericListArray<O>,count_array: &Int64Array,- field: &FieldRef,default_element: Option<ArrayRef>,) -> Result<ArrayRef> {let data_type = array.value_type();@@ -268,7 +264,6 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(build_resized_list(array,count_array,- field,&original_data,Some(&default_value_data),output_values_len,@@ -287,7 +282,6 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(build_resized_list(array,count_array,- field,&original_data,default_value_data.as_ref(),output_values_len,@@ -308,7 +302,6 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(fn build_resized_list<O, F>(array: &GenericListArray<O>,count_array: &Int64Array,- field: &FieldRef,original_data: &arrow::array::ArrayData,default_value_data: Option<&arrow::array::ArrayData>,output_values_len: usize,@@ -358,7 +351,7 @@ wherelet data = mutable.freeze();Ok(Arc::new(GenericListArray::<O>::try_new(- Arc::clone(field),+ Arc::clone(array.value_field()),OffsetBuffer::<O>::new(offsets.into()),arrow::array::make_array(data),null_builder.finish(),diff --git a/datafusion/functions-nested/src/reverse.rs b/datafusion/functions-nested/src/reverse.rsindex 69745c6ee1..79ca196037 100644--- a/datafusion/functions-nested/src/reverse.rs+++ b/datafusion/functions-nested/src/reverse.rs@@ -24,10 +24,10 @@ use arrow::array::{};use arrow::buffer::{OffsetBuffer, ScalarBuffer};use arrow::compute::take;+use arrow::datatypes::DataType;use arrow::datatypes::DataType::{FixedSizeList, LargeList, LargeListView, List, ListView, Null,};-use arrow::datatypes::{DataType, FieldRef};use datafusion_common::cast::{as_fixed_size_list_array, as_large_list_array, as_large_list_view_array,as_list_array, as_list_view_array,@@ -119,26 +119,26 @@ pub fn array_reverse_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {let [input_array] = take_function_args("array_reverse", arg)?;match &input_array.data_type() {- List(field) => {+ List(_) => {let array = as_list_array(input_array)?;- general_array_reverse::<i32>(array, field)+ general_array_reverse::<i32>(array)}- LargeList(field) => {+ LargeList(_) => {let array = as_large_list_array(input_array)?;- general_array_reverse::<i64>(array, field)+ general_array_reverse::<i64>(array)}- FixedSizeList(field, _) => {+ FixedSizeList(_, _) => {let array = as_fixed_size_list_array(input_array)?;- fixed_size_array_reverse(array, field)+ fixed_size_array_reverse(array)}Null => Ok(Arc::clone(input_array)),- ListView(field) => {+ ListView(_) => {let array = as_list_view_array(input_array)?;- list_view_reverse::<i32>(array, field)+ list_view_reverse::<i32>(array)}- LargeListView(field) => {+ LargeListView(_) => {let array = as_large_list_view_array(input_array)?;- list_view_reverse::<i64>(array, field)+ list_view_reverse::<i64>(array)}array_type => exec_err!("array_reverse does not support type '{array_type}'."),}@@ -146,7 +146,6 @@ pub fn array_reverse_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {fn general_array_reverse<O: OffsetSizeTrait>(array: &GenericListArray<O>,- field: &FieldRef,) -> Result<ArrayRef> {let values = array.values();let mut offsets = Vec::with_capacity(array.len() + 1);@@ -188,7 +187,7 @@ fn general_array_reverse<O: OffsetSizeTrait>(};let values = take(&values, &indices_array, None)?;Ok(Arc::new(GenericListArray::<O>::try_new(- Arc::clone(field),+ Arc::clone(array.value_field()),OffsetBuffer::<O>::new(offsets.into()),values,array.nulls().cloned(),@@ -203,7 +202,6 @@ fn general_array_reverse<O: OffsetSizeTrait>(/// even if the original array had elements out of order.fn list_view_reverse<O: OffsetSizeTrait>(array: &GenericListViewArray<O>,- field: &FieldRef,) -> Result<ArrayRef> {let offsets = array.offsets();let values = array.values();@@ -255,7 +253,7 @@ fn list_view_reverse<O: OffsetSizeTrait>(};let values = take(&values, &indices_array, None)?;Ok(Arc::new(GenericListViewArray::<O>::try_new(- Arc::clone(field),+ Arc::clone(array.value_field()),ScalarBuffer::from(new_offsets),ScalarBuffer::from(new_sizes),values,@@ -263,10 +261,7 @@ fn list_view_reverse<O: OffsetSizeTrait>()?))}-fn fixed_size_array_reverse(- array: &FixedSizeListArray,- field: &FieldRef,-) -> Result<ArrayRef> {+fn fixed_size_array_reverse(array: &FixedSizeListArray) -> Result<ArrayRef> {let values: &Arc<dyn Array> = array.values();// Since each fixed size list in the physical array is the same size and we keep the order@@ -281,7 +276,7 @@ fn fixed_size_array_reverse(let values = take(&values, &indices_array, None)?;Ok(Arc::new(FixedSizeListArray::try_new(- Arc::clone(field),+ Arc::clone(array.value_field()),array.value_length(),values,array.nulls().cloned(),@@ -326,10 +321,7 @@ mod tests {let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]));let nulls = Some(NullBuffer::from(vec![true, true, false, true]));let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);- let result = list_view_reverse(- &list_view,- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = list_view_reverse(&list_view)?;let reversed = list_view_values(result.as_list_view::<i32>());let expected = vec![Some(vec![1]),@@ -349,10 +341,7 @@ mod tests {let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]));let nulls = Some(NullBuffer::from(vec![true, true, false, true]));let list_view = LargeListViewArray::new(field, offsets, sizes, values, nulls);- let result = list_view_reverse(- &list_view,- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = list_view_reverse(&list_view)?;let reversed = list_view_values(result.as_list_view::<i64>());let expected = vec![Some(vec![1]),@@ -377,10 +366,7 @@ mod tests {]));let nulls = Some(NullBuffer::from(vec![true, true, false, true]));let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);- let result = list_view_reverse(- &list_view,- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = list_view_reverse(&list_view)?;let reversed = list_view_values(result.as_list_view::<i32>());let expected = vec![Some(vec![9, 8, 7]),@@ -405,10 +391,7 @@ mod tests {]));let nulls = Some(NullBuffer::from(vec![true, true, false, true]));let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);- let result = list_view_reverse(- &list_view,- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = list_view_reverse(&list_view)?;let reversed = list_view_values(result.as_list_view::<i32>());let expected = vec![Some(vec![9, 8, 7]),@@ -429,10 +412,7 @@ mod tests {let values = Arc::new(Int32Array::from(empty_array));let nulls = None;let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);- let result = list_view_reverse(- &list_view,- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = list_view_reverse(&list_view)?;let reversed = list_view_values(result.as_list_view::<i32>());let expected: Vec<Option<Vec<i32>>> = vec![];assert_eq!(expected, reversed);@@ -447,10 +427,7 @@ mod tests {let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));let nulls = Some(NullBuffer::from(vec![false, false, false, false]));let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);- let result = list_view_reverse(- &list_view,- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = list_view_reverse(&list_view)?;let reversed = list_view_values(result.as_list_view::<i32>());let expected: Vec<Option<Vec<i32>>> = vec![None, None, None, None];assert_eq!(expected, reversed);@@ -461,15 +438,12 @@ mod tests {fn test_reverse_fixed_size_list() -> Result<()> {let field = Arc::new(Field::new("a", DataType::Int32, false));let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]));- let result = fixed_size_array_reverse(- &FixedSizeListArray::new(- field,- 3,- values,- Some(NullBuffer::from(vec![true, false, true])),- ),- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = fixed_size_array_reverse(&FixedSizeListArray::new(+ field,+ 3,+ values,+ Some(NullBuffer::from(vec![true, false, true])),+ ))?;let reversed = fixed_size_list_values(result.as_fixed_size_list());let expected = vec![Some(vec![3, 2, 1]), None, Some(vec![9, 8, 7])];assert_eq!(expected, reversed);@@ -483,10 +457,7 @@ mod tests {let values = Arc::new(Int32Array::from(empty_array));let nulls = None;let fixed_size_list = FixedSizeListArray::new(field, 3, values, nulls);- let result = fixed_size_array_reverse(- &fixed_size_list,- &Arc::new(Field::new("test", DataType::Int32, true)),- )?;+ let result = fixed_size_array_reverse(&fixed_size_list)?;let reversed = fixed_size_list_values(result.as_fixed_size_list());let expected: Vec<Option<Vec<i32>>> = vec![];assert_eq!(expected, reversed);diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rsindex 01093555b3..5d7ae70ae7 100644--- a/datafusion/functions-nested/src/set_ops.rs+++ b/datafusion/functions-nested/src/set_ops.rs@@ -343,7 +343,6 @@ pub(crate) fn normalize_visible_values<OffsetSize: OffsetSizeTrait>(fn generic_set_lists<OffsetSize: OffsetSizeTrait>(l: &GenericListArray<OffsetSize>,r: &GenericListArray<OffsetSize>,- field: Arc<Field>,set_op: SetOp,) -> Result<ArrayRef> {if l.is_empty() || l.value_type().is_null() {@@ -383,7 +382,6 @@ fn generic_set_lists<OffsetSize: OffsetSizeTrait>(r,&rows_l,&rows_r,- field,&combined_values,r_offset,),@@ -392,7 +390,6 @@ fn generic_set_lists<OffsetSize: OffsetSizeTrait>(r,&rows_l,&rows_r,- field,&combined_values,r_offset,),@@ -406,7 +403,6 @@ fn generic_set_loop<OffsetSize: OffsetSizeTrait, const IS_UNION: bool>(r: &GenericListArray<OffsetSize>,rows_l: &arrow::row::Rows,rows_r: &arrow::row::Rows,- field: Arc<Field>,combined_values: &ArrayRef,r_offset: usize,) -> Result<ArrayRef> {@@ -504,7 +500,7 @@ fn generic_set_loop<OffsetSize: OffsetSizeTrait, const IS_UNION: bool>(};let arr = GenericListArray::<OffsetSize>::try_new(- field,+ Arc::clone(l.value_field()),OffsetBuffer::new(result_offsets.into()),final_values,NullBuffer::union(l.nulls(), r.nulls()),@@ -524,15 +520,15 @@ fn general_set_op(| (Null, dt @ LargeList(_))| (dt @ List(_), Null)| (dt @ LargeList(_), Null) => Ok(new_null_array(dt, len)),- (List(field), List(_)) => {+ (List(_), List(_)) => {let array1 = as_list_array(&array1)?;let array2 = as_list_array(&array2)?;- generic_set_lists::<i32>(array1, array2, Arc::clone(field), set_op)+ generic_set_lists::<i32>(array1, array2, set_op)}- (LargeList(field), LargeList(_)) => {+ (LargeList(_), LargeList(_)) => {let array1 = as_large_list_array(&array1)?;let array2 = as_large_list_array(&array2)?;- generic_set_lists::<i64>(array1, array2, Arc::clone(field), set_op)+ generic_set_lists::<i64>(array1, array2, set_op)}(data_type1, data_type2) => {internal_err!(@@ -794,11 +790,7 @@ mod tests {Some(vec![Some(-98.0)]),]).slice(1, 2);- let DataType::List(field) = l.data_type() else {- unreachable!()- };-- let distinct = general_array_distinct::<i32>(&l, field)?;+ let distinct = general_array_distinct::<i32>(&l, l.value_field())?;let distinct = distinct.as_list::<i32>();assert_eq!(collect_f64_bits(distinct, 0),@@ -811,7 +803,7 @@ mod tests {);assert!(distinct.is_null(1));- let union = generic_set_lists::<i32>(&l, &r, Arc::clone(field), SetOp::Union)?;+ let union = generic_set_lists::<i32>(&l, &r, SetOp::Union)?;let union = union.as_list::<i32>();assert_eq!(collect_f64_bits(union, 0),@@ -825,8 +817,7 @@ mod tests {);assert!(union.is_null(1));- let intersect =- generic_set_lists::<i32>(&l, &r, Arc::clone(field), SetOp::Intersect)?;+ let intersect = generic_set_lists::<i32>(&l, &r, SetOp::Intersect)?;let intersect = intersect.as_list::<i32>();assert_eq!(collect_f64_bits(intersect, 0),@@ -844,11 +835,7 @@ mod tests {Some(vec![Some(-99.0)]),]);let sliced = list.slice(1, 1);- let DataType::LargeList(field) = sliced.data_type() else {- unreachable!()- };-- let result = general_array_distinct::<i64>(&sliced, field)?;+ let result = general_array_distinct::<i64>(&sliced, sliced.value_field())?;let result = result.as_list::<i64>();assert_eq!(collect_f64_bits(result, 0),@@ -865,11 +852,7 @@ mod tests {Some(vec![Some(-0.0), Some(0.0)]),]);let sliced = list.slice(1, 1);- let DataType::List(field) = sliced.data_type() else {- unreachable!()- };-- let result = general_array_distinct::<i32>(&sliced, field)?;+ let result = general_array_distinct::<i32>(&sliced, sliced.value_field())?;let result = result.as_list::<i32>();assert_eq!(result.len(), 1);assert_eq!(result.value_length(0), 0);diff --git a/datafusion/functions-nested/src/sort.rs b/datafusion/functions-nested/src/sort.rsindex bccea76867..585f3445ce 100644--- a/datafusion/functions-nested/src/sort.rs+++ b/datafusion/functions-nested/src/sort.rs@@ -24,7 +24,7 @@ use arrow::array::{PrimitiveArray, UInt32Array, UInt64Array, new_empty_array, new_null_array,};use arrow::buffer::{NullBuffer, OffsetBuffer};-use arrow::datatypes::{ArrowNativeTypeOp, DataType, FieldRef};+use arrow::datatypes::{ArrowNativeTypeOp, DataType};use arrow::row::{RowConverter, Rows, SortField};use arrow::{compute, compute::SortOptions, downcast_primitive_array};use datafusion_common::cast::{as_large_list_array, as_list_array, as_string_array};@@ -188,13 +188,13 @@ fn array_sort_inner(args: &[ArrayRef]) -> Result<ArrayRef> {{Ok(Arc::clone(&args[0]))}- DataType::List(field) => {+ DataType::List(_) => {let array = as_list_array(&args[0])?;- array_sort_generic(array, Arc::clone(field), sort_options)+ array_sort_generic(array, sort_options)}- DataType::LargeList(field) => {+ DataType::LargeList(_) => {let array = as_large_list_array(&args[0])?;- array_sort_generic(array, Arc::clone(field), sort_options)+ array_sort_generic(array, sort_options)}// Signature should prevent this arm ever occurring_ => exec_err!("array_sort expects list for first argument"),@@ -203,15 +203,14 @@ fn array_sort_inner(args: &[ArrayRef]) -> Result<ArrayRef> {fn array_sort_generic<OffsetSize: OffsetSizeTrait>(list_array: &GenericListArray<OffsetSize>,- field: FieldRef,sort_options: Option<SortOptions>,) -> Result<ArrayRef> {let values = list_array.values();if values.data_type().is_primitive() {- array_sort_primitive(list_array, field, sort_options)+ array_sort_primitive(list_array, sort_options)} else {- array_sort_non_primitive(list_array, field, sort_options)+ array_sort_non_primitive(list_array, sort_options)}}@@ -219,12 +218,11 @@ fn array_sort_generic<OffsetSize: OffsetSizeTrait>(/// kernel.fn array_sort_primitive<OffsetSize: OffsetSizeTrait>(list_array: &GenericListArray<OffsetSize>,- field: FieldRef,sort_options: Option<SortOptions>,) -> Result<ArrayRef> {let values = list_array.values().as_ref();downcast_primitive_array! {- values => sort_primitive_list(values, list_array, field, sort_options),+ values => sort_primitive_list(values, list_array, sort_options),_ => exec_err!("array_sort: unsupported primitive type")}}@@ -232,7 +230,6 @@ fn array_sort_primitive<OffsetSize: OffsetSizeTrait>(fn sort_primitive_list<T: ArrowPrimitiveType, OffsetSize: OffsetSizeTrait>(prim_values: &PrimitiveArray<T>,list_array: &GenericListArray<OffsetSize>,- field: FieldRef,sort_options: Option<SortOptions>,) -> Result<ArrayRef>where@@ -267,7 +264,7 @@ where);Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(- field,+ Arc::clone(list_array.value_field()),rebase_offsets(offsets),sorted_values,list_nulls.cloned(),@@ -377,7 +374,6 @@ fn sort_rows_with_nulls<N: ArrowNativeTypeOp, O: OffsetSizeTrait>(/// the result with a single `take()`.fn array_sort_non_primitive<OffsetSize: OffsetSizeTrait>(list_array: &GenericListArray<OffsetSize>,- field: FieldRef,sort_options: Option<SortOptions>,) -> Result<ArrayRef> {let row_count = list_array.len();@@ -427,7 +423,7 @@ fn array_sort_non_primitive<OffsetSize: OffsetSizeTrait>(};Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(- field,+ Arc::clone(list_array.value_field()),OffsetBuffer::<OffsetSize>::new(new_offsets.into()),sorted_values,list_array.nulls().cloned(),diff --git a/datafusion/spark/src/function/array/shuffle.rs b/datafusion/spark/src/function/array/shuffle.rsindex c86433c763..c7f8333a58 100644--- a/datafusion/spark/src/function/array/shuffle.rs+++ b/datafusion/spark/src/function/array/shuffle.rs@@ -141,17 +141,17 @@ fn extract_seed(seed_arg: &ColumnarValue) -> Result<Option<u64>> {fn array_shuffle_with_seed(arg: &[ArrayRef], seed: Option<u64>) -> Result<ArrayRef> {let [input_array] = take_function_args("shuffle", arg)?;match &input_array.data_type() {- List(field) => {+ List(_) => {let array = as_list_array(input_array)?;- general_array_shuffle::<i32>(array, field, seed)+ general_array_shuffle::<i32>(array, seed)}- LargeList(field) => {+ LargeList(_) => {let array = as_large_list_array(input_array)?;- general_array_shuffle::<i64>(array, field, seed)+ general_array_shuffle::<i64>(array, seed)}- FixedSizeList(field, _) => {+ FixedSizeList(_, _) => {let array = as_fixed_size_list_array(input_array)?;- fixed_size_array_shuffle(array, field, seed)+ fixed_size_array_shuffle(array, seed)}Null => Ok(Arc::clone(input_array)),array_type => exec_err!(@@ -163,7 +163,6 @@ fn array_shuffle_with_seed(arg: &[ArrayRef], seed: Option<u64>) -> Result<ArrayRfn general_array_shuffle<O: OffsetSizeTrait>(array: &GenericListArray<O>,- field: &FieldRef,seed: Option<u64>,) -> Result<ArrayRef> {let values = array.values();@@ -212,7 +211,7 @@ fn general_array_shuffle<O: OffsetSizeTrait>(let data = mutable.freeze();Ok(Arc::new(GenericListArray::<O>::try_new(- Arc::clone(field),+ Arc::clone(array.value_field()),OffsetBuffer::<O>::new(offsets.into()),arrow::array::make_array(data),Some(nulls.into()),@@ -221,7 +220,6 @@ fn general_array_shuffle<O: OffsetSizeTrait>(fn fixed_size_array_shuffle(array: &FixedSizeListArray,- field: &FieldRef,seed: Option<u64>,) -> Result<ArrayRef> {let values = array.values();@@ -265,7 +263,7 @@ fn fixed_size_array_shuffle(let data = mutable.freeze();Ok(Arc::new(FixedSizeListArray::try_new(- Arc::clone(field),+ Arc::clone(array.value_field()),array.value_length(),arrow::array::make_array(data),Some(nulls.into()),