Skip to content

Apache DataFusion SQL Query Engine

chore: Use `value_field` to fetch list element field (#26567)

Neil Conway139da57bdae4
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index 6727ab7dcc..0ff108ff36 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -1356,14 +1356,8 @@ fn truncate_list_nulls<O: OffsetSizeTrait>(
let offsets = OffsetBuffer::from_lengths(lengths);
let values = make_array(mutable_array_data.freeze());
- let field = match list.data_type() {
- DataType::List(field) => field,
- DataType::LargeList(field) => field,
- _ => unreachable!(),
- };
-
return Ok(GenericListArray::try_new(
- Arc::clone(field),
+ Arc::clone(list.value_field()),
offsets,
values,
list.nulls().cloned(),
diff --git a/datafusion/functions-nested/src/array_compact.rs b/datafusion/functions-nested/src/array_compact.rs
index 12e3638294..4e61d1a837 100644
--- a/datafusion/functions-nested/src/array_compact.rs
+++ b/datafusion/functions-nested/src/array_compact.rs
@@ -26,8 +26,8 @@ use arrow::array::{
};
use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
use arrow::compute::filter;
+use arrow::datatypes::DataType;
use arrow::datatypes::DataType::{LargeList, List, Null};
-use arrow::datatypes::{DataType, FieldRef};
use datafusion_common::cast::as_generic_list_array;
use datafusion_common::utils::offset_span_len;
use datafusion_common::{Result, exec_err, utils::take_function_args};
@@ -115,8 +115,8 @@ fn array_compact_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
let [input_array] = take_function_args("array_compact", arg)?;
match &input_array.data_type() {
- List(field) => compact_list::<i32>(input_array, field),
- LargeList(field) => compact_list::<i64>(input_array, field),
+ List(_) => compact_list::<i32>(input_array),
+ LargeList(_) => compact_list::<i64>(input_array),
Null => Ok(Arc::clone(input_array)),
array_type => exec_err!("array_compact does not support type '{array_type}'."),
}
@@ -126,10 +126,7 @@ fn array_compact_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
///
/// Each row is a range in a shared child array. Compaction removes null child
/// values and rebuilds the row offsets, preserving null list rows.
-fn compact_list<O: OffsetSizeTrait>(
- input_array: &ArrayRef,
- field: &FieldRef,
-) -> Result<ArrayRef> {
+fn compact_list<O: OffsetSizeTrait>(input_array: &ArrayRef) -> Result<ArrayRef> {
let list_array = as_generic_list_array::<O>(input_array.as_ref())?;
let visible_len = offset_span_len(list_array.offsets());
if visible_len == 0 || list_array.null_count() == list_array.len() {
@@ -152,10 +149,13 @@ fn compact_list<O: OffsetSizeTrait>(
return Ok(Arc::clone(input_array));
};
if values_nulls.null_count() == values_nulls.len() {
- return Ok(empty_list_values(list_array, Arc::clone(field)));
+ return Ok(empty_list_values(
+ list_array,
+ Arc::clone(list_array.value_field()),
+ ));
}
- build_compacted_list(list_array, field, values.as_ref(), &values_nulls)
+ build_compacted_list(list_array, values.as_ref(), &values_nulls)
}
/// Copy primitive values while computing result row offsets. For other types,
@@ -167,7 +167,6 @@ fn compact_list<O: OffsetSizeTrait>(
/// subtract the first list offset when indexing `values`.
fn build_compacted_list<O: OffsetSizeTrait>(
list_array: &GenericListArray<O>,
- field: &FieldRef,
values: &dyn Array,
values_nulls: &NullBuffer,
) -> Result<ArrayRef> {
@@ -176,7 +175,7 @@ fn build_compacted_list<O: OffsetSizeTrait>(
_ => compact_non_primitive(values, list_array, values_nulls),
}?;
Ok(Arc::new(GenericListArray::<O>::try_new(
- Arc::clone(field),
+ Arc::clone(list_array.value_field()),
offsets,
new_values,
list_array.nulls().cloned(),
diff --git a/datafusion/functions-nested/src/array_scale.rs b/datafusion/functions-nested/src/array_scale.rs
index 5dad7cb93b..6a283ee3a3 100644
--- a/datafusion/functions-nested/src/array_scale.rs
+++ b/datafusion/functions-nested/src/array_scale.rs
@@ -205,15 +205,8 @@ fn general_array_scale<O: OffsetSizeTrait>(
// Preserve the inner field from the input array (including any user
// metadata). After `coerce_types` the inner type is Float64, but the
// input may still carry field-level annotations worth keeping.
- let field = match list_array.data_type() {
- List(f) | LargeList(f) => Arc::clone(f),
- other => {
- return internal_err!("array_scale unexpected list type: {other}");
- }
- };
-
Ok(Arc::new(GenericListArray::<O>::try_new(
- field,
+ Arc::clone(list_array.value_field()),
OffsetBuffer::new(new_offsets.into()),
values_array,
row_nulls,
diff --git a/datafusion/functions-nested/src/except.rs b/datafusion/functions-nested/src/except.rs
index fd2b6a0897..cf680199bf 100644
--- a/datafusion/functions-nested/src/except.rs
+++ b/datafusion/functions-nested/src/except.rs
@@ -26,7 +26,7 @@ use arrow::array::{
};
use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::compute::take;
-use arrow::datatypes::{DataType, FieldRef};
+use arrow::datatypes::DataType;
use arrow::row::{RowConverter, SortField};
use datafusion_common::utils::{ListCoercion, take_function_args};
use datafusion_common::{HashSet, Result, internal_err};
@@ -143,18 +143,18 @@ fn array_except_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
| (DataType::Null, dt @ DataType::LargeList(_))
| (dt @ DataType::List(_), DataType::Null)
| (dt @ DataType::LargeList(_), DataType::Null) => Ok(new_null_array(dt, len)),
- (DataType::List(field), DataType::List(_)) => {
+ (DataType::List(_), DataType::List(_)) => {
check_datatypes("array_except", &[array1, array2])?;
let list1 = array1.as_list::<i32>();
let list2 = array2.as_list::<i32>();
- let result = general_except::<i32>(list1, list2, field)?;
+ let result = general_except::<i32>(list1, list2)?;
Ok(Arc::new(result))
}
- (DataType::LargeList(field), DataType::LargeList(_)) => {
+ (DataType::LargeList(_), DataType::LargeList(_)) => {
check_datatypes("array_except", &[array1, array2])?;
let list1 = array1.as_list::<i64>();
let list2 = array2.as_list::<i64>();
- let result = general_except::<i64>(list1, list2, field)?;
+ let result = general_except::<i64>(list1, list2)?;
Ok(Arc::new(result))
}
(dt1, dt2) => {
@@ -166,7 +166,6 @@ fn array_except_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
fn general_except<OffsetSize: OffsetSizeTrait>(
l: &GenericListArray<OffsetSize>,
r: &GenericListArray<OffsetSize>,
- field: &FieldRef,
) -> Result<GenericListArray<OffsetSize>> {
let converter = RowConverter::new(vec![SortField::new(l.value_type())])?;
@@ -231,7 +230,7 @@ fn general_except<OffsetSize: OffsetSizeTrait>(
};
Ok(GenericListArray::<OffsetSize>::new(
- field.to_owned(),
+ Arc::clone(l.value_field()),
OffsetBuffer::new(offsets.into()),
values,
nulls,
@@ -242,7 +241,7 @@ fn general_except<OffsetSize: OffsetSizeTrait>(
mod tests {
use super::{ArrayExcept, general_except};
use arrow::array::{Array, AsArray, Int32Array, LargeListArray, ListArray};
- use arrow::datatypes::{DataType, Field, Float64Type, Int32Type};
+ use arrow::datatypes::{Field, Float64Type, Int32Type};
use datafusion_common::{Result, config::ConfigOptions};
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
use std::sync::Arc;
@@ -312,11 +311,7 @@ mod tests {
Some(vec![Some(-98.0)]),
])
.slice(1, 1);
- let DataType::LargeList(field) = l.data_type() else {
- unreachable!()
- };
-
- let result = general_except::<i64>(&l, &r, field)?;
+ let result = general_except::<i64>(&l, &r)?;
let values = result.value(0);
let values = values.as_primitive::<Float64Type>();
assert_eq!(values.len(), 1);
diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs
index 25ce9ecfeb..c2f9c22c75 100644
--- a/datafusion/functions-nested/src/extract.rs
+++ b/datafusion/functions-nested/src/extract.rs
@@ -31,7 +31,6 @@ use datafusion_common::cast::as_list_array;
use datafusion_common::cast::{
as_int64_array, as_large_list_view_array, as_list_view_array,
};
-use datafusion_common::internal_err;
use datafusion_common::utils::ListCoercion;
use datafusion_common::utils::offset_span_len;
use datafusion_common::{
@@ -48,7 +47,7 @@ use datafusion_expr::{
use datafusion_macros::user_doc;
use std::sync::Arc;
-use crate::utils::{list_inner_field, make_scalar_function};
+use crate::utils::make_scalar_function;
// Create static instances of ScalarUDFs for each function
make_udf_expr_and_func!(
@@ -608,7 +607,7 @@ where
// Carry the input's list field through to the output so that the returned
// type matches the one promised by `return_type` / `return_field_from_args`,
// including the field name, nullability and metadata.
- let field = list_inner_field("general_array_slice", array.data_type())?;
+ let field = Arc::clone(array.value_field());
// `use_nulls` is false because we never call `try_extend_nulls`: null rows are
// emitted as empty slices. Arrow still allocates a validity buffer on its own
@@ -696,14 +695,7 @@ where
let values = array.values();
let original_data = values.to_data();
let capacity = Capacities::Array(original_data.len());
- let field = match array.data_type() {
- ListView(field) | LargeListView(field) => Arc::clone(field),
- other => {
- return internal_err!(
- "general_list_view_array_slice got unexpected data type: {other}"
- );
- }
- };
+ let field = Arc::clone(array.value_field());
// See the note on `use_nulls` in `general_array_slice`.
let mut mutable =
diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rs
index befcc09d22..20f36d3d33 100644
--- a/datafusion/functions-nested/src/remove.rs
+++ b/datafusion/functions-nested/src/remove.rs
@@ -454,14 +454,6 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
element_array: &ArrayRef,
arr_n: &[Option<i64>],
) -> Result<ArrayRef> {
- let (DataType::List(list_field) | DataType::LargeList(list_field)) =
- list_array.data_type()
- else {
- return exec_err!(
- "Expected List or LargeList data type, got {:?}",
- list_array.data_type()
- );
- };
let original_data = list_array.values().to_data();
// Build up the offsets for the final output array
let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);
@@ -545,7 +537,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
let new_values = make_array(mutable.freeze());
Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(
- Arc::clone(list_field),
+ Arc::clone(list_array.value_field()),
OffsetBuffer::new(offsets.into()),
new_values,
valid.finish(),
@@ -566,15 +558,6 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
return Ok(Arc::new(list_array.clone()));
}
- let (DataType::List(list_field) | DataType::LargeList(list_field)) =
- list_array.data_type()
- else {
- return exec_err!(
- "Expected List or LargeList data type, got {:?}",
- list_array.data_type()
- );
- };
-
let list_offsets = list_array.offsets();
let (first_offset, values_range_len) = offset_span(list_offsets);
let values_slice = list_array.values().slice(first_offset, values_range_len);
@@ -642,7 +625,7 @@ fn general_remove_with_scalar<OffsetSize: OffsetSizeTrait>(
let new_values = make_array(mutable.freeze());
Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(
- Arc::clone(list_field),
+ Arc::clone(list_array.value_field()),
OffsetBuffer::new(offsets.into()),
new_values,
nulls,
diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rs
index 6b682e3ce9..2a96bb8b5a 100644
--- a/datafusion/functions-nested/src/resize.rs
+++ b/datafusion/functions-nested/src/resize.rs
@@ -24,11 +24,8 @@ use arrow::array::{
};
use arrow::buffer::OffsetBuffer;
use arrow::datatypes::DataType;
+use arrow::datatypes::DataType::{LargeList, List};
use arrow::datatypes::Field;
-use arrow::datatypes::{
- DataType::{LargeList, List},
- FieldRef,
-};
use datafusion_common::cast::{as_int64_array, as_large_list_array, as_list_array};
use datafusion_common::utils::ListCoercion;
use datafusion_common::{
@@ -177,13 +174,13 @@ fn array_resize_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
};
match &arg[0].data_type() {
- List(field) => {
+ List(_) => {
let array = as_list_array(&arg[0])?;
- general_list_resize::<i32>(array, new_len, field, new_element)
+ general_list_resize::<i32>(array, new_len, new_element)
}
- LargeList(field) => {
+ LargeList(_) => {
let array = as_large_list_array(&arg[0])?;
- general_list_resize::<i64>(array, new_len, field, new_element)
+ general_list_resize::<i64>(array, new_len, new_element)
}
array_type => exec_err!("array_resize does not support type '{array_type}'."),
}
@@ -200,7 +197,6 @@ fn resize_count(count_array: &Int64Array, idx: usize) -> Result<usize> {
fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(
array: &GenericListArray<O>,
count_array: &Int64Array,
- field: &FieldRef,
default_element: Option<ArrayRef>,
) -> Result<ArrayRef> {
let data_type = array.value_type();
@@ -268,7 +264,6 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(
build_resized_list(
array,
count_array,
- field,
&original_data,
Some(&default_value_data),
output_values_len,
@@ -287,7 +282,6 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(
build_resized_list(
array,
count_array,
- field,
&original_data,
default_value_data.as_ref(),
output_values_len,
@@ -308,7 +302,6 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(
fn build_resized_list<O, F>(
array: &GenericListArray<O>,
count_array: &Int64Array,
- field: &FieldRef,
original_data: &arrow::array::ArrayData,
default_value_data: Option<&arrow::array::ArrayData>,
output_values_len: usize,
@@ -358,7 +351,7 @@ where
let data = mutable.freeze();
Ok(Arc::new(GenericListArray::<O>::try_new(
- Arc::clone(field),
+ Arc::clone(array.value_field()),
OffsetBuffer::<O>::new(offsets.into()),
arrow::array::make_array(data),
null_builder.finish(),
diff --git a/datafusion/functions-nested/src/reverse.rs b/datafusion/functions-nested/src/reverse.rs
index 69745c6ee1..79ca196037 100644
--- a/datafusion/functions-nested/src/reverse.rs
+++ b/datafusion/functions-nested/src/reverse.rs
@@ -24,10 +24,10 @@ use arrow::array::{
};
use arrow::buffer::{OffsetBuffer, ScalarBuffer};
use arrow::compute::take;
+use arrow::datatypes::DataType;
use arrow::datatypes::DataType::{
FixedSizeList, LargeList, LargeListView, List, ListView, Null,
};
-use arrow::datatypes::{DataType, FieldRef};
use datafusion_common::cast::{
as_fixed_size_list_array, as_large_list_array, as_large_list_view_array,
as_list_array, as_list_view_array,
@@ -119,26 +119,26 @@ pub fn array_reverse_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
let [input_array] = take_function_args("array_reverse", arg)?;
match &input_array.data_type() {
- List(field) => {
+ List(_) => {
let array = as_list_array(input_array)?;
- general_array_reverse::<i32>(array, field)
+ general_array_reverse::<i32>(array)
}
- LargeList(field) => {
+ LargeList(_) => {
let array = as_large_list_array(input_array)?;
- general_array_reverse::<i64>(array, field)
+ general_array_reverse::<i64>(array)
}
- FixedSizeList(field, _) => {
+ FixedSizeList(_, _) => {
let array = as_fixed_size_list_array(input_array)?;
- fixed_size_array_reverse(array, field)
+ fixed_size_array_reverse(array)
}
Null => Ok(Arc::clone(input_array)),
- ListView(field) => {
+ ListView(_) => {
let array = as_list_view_array(input_array)?;
- list_view_reverse::<i32>(array, field)
+ list_view_reverse::<i32>(array)
}
- LargeListView(field) => {
+ LargeListView(_) => {
let array = as_large_list_view_array(input_array)?;
- list_view_reverse::<i64>(array, field)
+ list_view_reverse::<i64>(array)
}
array_type => exec_err!("array_reverse does not support type '{array_type}'."),
}
@@ -146,7 +146,6 @@ pub fn array_reverse_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
fn general_array_reverse<O: OffsetSizeTrait>(
array: &GenericListArray<O>,
- field: &FieldRef,
) -> Result<ArrayRef> {
let values = array.values();
let mut offsets = Vec::with_capacity(array.len() + 1);
@@ -188,7 +187,7 @@ fn general_array_reverse<O: OffsetSizeTrait>(
};
let values = take(&values, &indices_array, None)?;
Ok(Arc::new(GenericListArray::<O>::try_new(
- Arc::clone(field),
+ Arc::clone(array.value_field()),
OffsetBuffer::<O>::new(offsets.into()),
values,
array.nulls().cloned(),
@@ -203,7 +202,6 @@ fn general_array_reverse<O: OffsetSizeTrait>(
/// even if the original array had elements out of order.
fn list_view_reverse<O: OffsetSizeTrait>(
array: &GenericListViewArray<O>,
- field: &FieldRef,
) -> Result<ArrayRef> {
let offsets = array.offsets();
let values = array.values();
@@ -255,7 +253,7 @@ fn list_view_reverse<O: OffsetSizeTrait>(
};
let values = take(&values, &indices_array, None)?;
Ok(Arc::new(GenericListViewArray::<O>::try_new(
- Arc::clone(field),
+ Arc::clone(array.value_field()),
ScalarBuffer::from(new_offsets),
ScalarBuffer::from(new_sizes),
values,
@@ -263,10 +261,7 @@ fn list_view_reverse<O: OffsetSizeTrait>(
)?))
}
-fn fixed_size_array_reverse(
- array: &FixedSizeListArray,
- field: &FieldRef,
-) -> Result<ArrayRef> {
+fn fixed_size_array_reverse(array: &FixedSizeListArray) -> Result<ArrayRef> {
let values: &Arc<dyn Array> = array.values();
// Since each fixed size list in the physical array is the same size and we keep the order
@@ -281,7 +276,7 @@ fn fixed_size_array_reverse(
let values = take(&values, &indices_array, None)?;
Ok(Arc::new(FixedSizeListArray::try_new(
- Arc::clone(field),
+ Arc::clone(array.value_field()),
array.value_length(),
values,
array.nulls().cloned(),
@@ -326,10 +321,7 @@ mod tests {
let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]));
let nulls = Some(NullBuffer::from(vec![true, true, false, true]));
let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);
- let result = list_view_reverse(
- &list_view,
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = list_view_reverse(&list_view)?;
let reversed = list_view_values(result.as_list_view::<i32>());
let expected = vec![
Some(vec![1]),
@@ -349,10 +341,7 @@ mod tests {
let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]));
let nulls = Some(NullBuffer::from(vec![true, true, false, true]));
let list_view = LargeListViewArray::new(field, offsets, sizes, values, nulls);
- let result = list_view_reverse(
- &list_view,
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = list_view_reverse(&list_view)?;
let reversed = list_view_values(result.as_list_view::<i64>());
let expected = vec![
Some(vec![1]),
@@ -377,10 +366,7 @@ mod tests {
]));
let nulls = Some(NullBuffer::from(vec![true, true, false, true]));
let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);
- let result = list_view_reverse(
- &list_view,
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = list_view_reverse(&list_view)?;
let reversed = list_view_values(result.as_list_view::<i32>());
let expected = vec![
Some(vec![9, 8, 7]),
@@ -405,10 +391,7 @@ mod tests {
]));
let nulls = Some(NullBuffer::from(vec![true, true, false, true]));
let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);
- let result = list_view_reverse(
- &list_view,
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = list_view_reverse(&list_view)?;
let reversed = list_view_values(result.as_list_view::<i32>());
let expected = vec![
Some(vec![9, 8, 7]),
@@ -429,10 +412,7 @@ mod tests {
let values = Arc::new(Int32Array::from(empty_array));
let nulls = None;
let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);
- let result = list_view_reverse(
- &list_view,
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = list_view_reverse(&list_view)?;
let reversed = list_view_values(result.as_list_view::<i32>());
let expected: Vec<Option<Vec<i32>>> = vec![];
assert_eq!(expected, reversed);
@@ -447,10 +427,7 @@ mod tests {
let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
let nulls = Some(NullBuffer::from(vec![false, false, false, false]));
let list_view = ListViewArray::new(field, offsets, sizes, values, nulls);
- let result = list_view_reverse(
- &list_view,
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = list_view_reverse(&list_view)?;
let reversed = list_view_values(result.as_list_view::<i32>());
let expected: Vec<Option<Vec<i32>>> = vec![None, None, None, None];
assert_eq!(expected, reversed);
@@ -461,15 +438,12 @@ mod tests {
fn test_reverse_fixed_size_list() -> Result<()> {
let field = Arc::new(Field::new("a", DataType::Int32, false));
let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]));
- let result = fixed_size_array_reverse(
- &FixedSizeListArray::new(
- field,
- 3,
- values,
- Some(NullBuffer::from(vec![true, false, true])),
- ),
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = fixed_size_array_reverse(&FixedSizeListArray::new(
+ field,
+ 3,
+ values,
+ Some(NullBuffer::from(vec![true, false, true])),
+ ))?;
let reversed = fixed_size_list_values(result.as_fixed_size_list());
let expected = vec![Some(vec![3, 2, 1]), None, Some(vec![9, 8, 7])];
assert_eq!(expected, reversed);
@@ -483,10 +457,7 @@ mod tests {
let values = Arc::new(Int32Array::from(empty_array));
let nulls = None;
let fixed_size_list = FixedSizeListArray::new(field, 3, values, nulls);
- let result = fixed_size_array_reverse(
- &fixed_size_list,
- &Arc::new(Field::new("test", DataType::Int32, true)),
- )?;
+ let result = fixed_size_array_reverse(&fixed_size_list)?;
let reversed = fixed_size_list_values(result.as_fixed_size_list());
let expected: Vec<Option<Vec<i32>>> = vec![];
assert_eq!(expected, reversed);
diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rs
index 01093555b3..5d7ae70ae7 100644
--- a/datafusion/functions-nested/src/set_ops.rs
+++ b/datafusion/functions-nested/src/set_ops.rs
@@ -343,7 +343,6 @@ pub(crate) fn normalize_visible_values<OffsetSize: OffsetSizeTrait>(
fn generic_set_lists<OffsetSize: OffsetSizeTrait>(
l: &GenericListArray<OffsetSize>,
r: &GenericListArray<OffsetSize>,
- field: Arc<Field>,
set_op: SetOp,
) -> Result<ArrayRef> {
if l.is_empty() || l.value_type().is_null() {
@@ -383,7 +382,6 @@ fn generic_set_lists<OffsetSize: OffsetSizeTrait>(
r,
&rows_l,
&rows_r,
- field,
&combined_values,
r_offset,
),
@@ -392,7 +390,6 @@ fn generic_set_lists<OffsetSize: OffsetSizeTrait>(
r,
&rows_l,
&rows_r,
- field,
&combined_values,
r_offset,
),
@@ -406,7 +403,6 @@ fn generic_set_loop<OffsetSize: OffsetSizeTrait, const IS_UNION: bool>(
r: &GenericListArray<OffsetSize>,
rows_l: &arrow::row::Rows,
rows_r: &arrow::row::Rows,
- field: Arc<Field>,
combined_values: &ArrayRef,
r_offset: usize,
) -> Result<ArrayRef> {
@@ -504,7 +500,7 @@ fn generic_set_loop<OffsetSize: OffsetSizeTrait, const IS_UNION: bool>(
};
let arr = GenericListArray::<OffsetSize>::try_new(
- field,
+ Arc::clone(l.value_field()),
OffsetBuffer::new(result_offsets.into()),
final_values,
NullBuffer::union(l.nulls(), r.nulls()),
@@ -524,15 +520,15 @@ fn general_set_op(
| (Null, dt @ LargeList(_))
| (dt @ List(_), Null)
| (dt @ LargeList(_), Null) => Ok(new_null_array(dt, len)),
- (List(field), List(_)) => {
+ (List(_), List(_)) => {
let array1 = as_list_array(&array1)?;
let array2 = as_list_array(&array2)?;
- generic_set_lists::<i32>(array1, array2, Arc::clone(field), set_op)
+ generic_set_lists::<i32>(array1, array2, set_op)
}
- (LargeList(field), LargeList(_)) => {
+ (LargeList(_), LargeList(_)) => {
let array1 = as_large_list_array(&array1)?;
let array2 = as_large_list_array(&array2)?;
- generic_set_lists::<i64>(array1, array2, Arc::clone(field), set_op)
+ generic_set_lists::<i64>(array1, array2, set_op)
}
(data_type1, data_type2) => {
internal_err!(
@@ -794,11 +790,7 @@ mod tests {
Some(vec![Some(-98.0)]),
])
.slice(1, 2);
- let DataType::List(field) = l.data_type() else {
- unreachable!()
- };
-
- let distinct = general_array_distinct::<i32>(&l, field)?;
+ let distinct = general_array_distinct::<i32>(&l, l.value_field())?;
let distinct = distinct.as_list::<i32>();
assert_eq!(
collect_f64_bits(distinct, 0),
@@ -811,7 +803,7 @@ mod tests {
);
assert!(distinct.is_null(1));
- let union = generic_set_lists::<i32>(&l, &r, Arc::clone(field), SetOp::Union)?;
+ let union = generic_set_lists::<i32>(&l, &r, SetOp::Union)?;
let union = union.as_list::<i32>();
assert_eq!(
collect_f64_bits(union, 0),
@@ -825,8 +817,7 @@ mod tests {
);
assert!(union.is_null(1));
- let intersect =
- generic_set_lists::<i32>(&l, &r, Arc::clone(field), SetOp::Intersect)?;
+ let intersect = generic_set_lists::<i32>(&l, &r, SetOp::Intersect)?;
let intersect = intersect.as_list::<i32>();
assert_eq!(
collect_f64_bits(intersect, 0),
@@ -844,11 +835,7 @@ mod tests {
Some(vec![Some(-99.0)]),
]);
let sliced = list.slice(1, 1);
- let DataType::LargeList(field) = sliced.data_type() else {
- unreachable!()
- };
-
- let result = general_array_distinct::<i64>(&sliced, field)?;
+ let result = general_array_distinct::<i64>(&sliced, sliced.value_field())?;
let result = result.as_list::<i64>();
assert_eq!(
collect_f64_bits(result, 0),
@@ -865,11 +852,7 @@ mod tests {
Some(vec![Some(-0.0), Some(0.0)]),
]);
let sliced = list.slice(1, 1);
- let DataType::List(field) = sliced.data_type() else {
- unreachable!()
- };
-
- let result = general_array_distinct::<i32>(&sliced, field)?;
+ let result = general_array_distinct::<i32>(&sliced, sliced.value_field())?;
let result = result.as_list::<i32>();
assert_eq!(result.len(), 1);
assert_eq!(result.value_length(0), 0);
diff --git a/datafusion/functions-nested/src/sort.rs b/datafusion/functions-nested/src/sort.rs
index bccea76867..585f3445ce 100644
--- a/datafusion/functions-nested/src/sort.rs
+++ b/datafusion/functions-nested/src/sort.rs
@@ -24,7 +24,7 @@ use arrow::array::{
PrimitiveArray, UInt32Array, UInt64Array, new_empty_array, new_null_array,
};
use arrow::buffer::{NullBuffer, OffsetBuffer};
-use arrow::datatypes::{ArrowNativeTypeOp, DataType, FieldRef};
+use arrow::datatypes::{ArrowNativeTypeOp, DataType};
use arrow::row::{RowConverter, Rows, SortField};
use arrow::{compute, compute::SortOptions, downcast_primitive_array};
use datafusion_common::cast::{as_large_list_array, as_list_array, as_string_array};
@@ -188,13 +188,13 @@ fn array_sort_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
{
Ok(Arc::clone(&args[0]))
}
- DataType::List(field) => {
+ DataType::List(_) => {
let array = as_list_array(&args[0])?;
- array_sort_generic(array, Arc::clone(field), sort_options)
+ array_sort_generic(array, sort_options)
}
- DataType::LargeList(field) => {
+ DataType::LargeList(_) => {
let array = as_large_list_array(&args[0])?;
- array_sort_generic(array, Arc::clone(field), sort_options)
+ array_sort_generic(array, sort_options)
}
// Signature should prevent this arm ever occurring
_ => exec_err!("array_sort expects list for first argument"),
@@ -203,15 +203,14 @@ fn array_sort_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
fn array_sort_generic<OffsetSize: OffsetSizeTrait>(
list_array: &GenericListArray<OffsetSize>,
- field: FieldRef,
sort_options: Option<SortOptions>,
) -> Result<ArrayRef> {
let values = list_array.values();
if values.data_type().is_primitive() {
- array_sort_primitive(list_array, field, sort_options)
+ array_sort_primitive(list_array, sort_options)
} else {
- array_sort_non_primitive(list_array, field, sort_options)
+ array_sort_non_primitive(list_array, sort_options)
}
}
@@ -219,12 +218,11 @@ fn array_sort_generic<OffsetSize: OffsetSizeTrait>(
/// kernel.
fn array_sort_primitive<OffsetSize: OffsetSizeTrait>(
list_array: &GenericListArray<OffsetSize>,
- field: FieldRef,
sort_options: Option<SortOptions>,
) -> Result<ArrayRef> {
let values = list_array.values().as_ref();
downcast_primitive_array! {
- values => sort_primitive_list(values, list_array, field, sort_options),
+ values => sort_primitive_list(values, list_array, sort_options),
_ => exec_err!("array_sort: unsupported primitive type")
}
}
@@ -232,7 +230,6 @@ fn array_sort_primitive<OffsetSize: OffsetSizeTrait>(
fn sort_primitive_list<T: ArrowPrimitiveType, OffsetSize: OffsetSizeTrait>(
prim_values: &PrimitiveArray<T>,
list_array: &GenericListArray<OffsetSize>,
- field: FieldRef,
sort_options: Option<SortOptions>,
) -> Result<ArrayRef>
where
@@ -267,7 +264,7 @@ where
);
Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(
- field,
+ Arc::clone(list_array.value_field()),
rebase_offsets(offsets),
sorted_values,
list_nulls.cloned(),
@@ -377,7 +374,6 @@ fn sort_rows_with_nulls<N: ArrowNativeTypeOp, O: OffsetSizeTrait>(
/// the result with a single `take()`.
fn array_sort_non_primitive<OffsetSize: OffsetSizeTrait>(
list_array: &GenericListArray<OffsetSize>,
- field: FieldRef,
sort_options: Option<SortOptions>,
) -> Result<ArrayRef> {
let row_count = list_array.len();
@@ -427,7 +423,7 @@ fn array_sort_non_primitive<OffsetSize: OffsetSizeTrait>(
};
Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(
- field,
+ Arc::clone(list_array.value_field()),
OffsetBuffer::<OffsetSize>::new(new_offsets.into()),
sorted_values,
list_array.nulls().cloned(),
diff --git a/datafusion/spark/src/function/array/shuffle.rs b/datafusion/spark/src/function/array/shuffle.rs
index c86433c763..c7f8333a58 100644
--- a/datafusion/spark/src/function/array/shuffle.rs
+++ b/datafusion/spark/src/function/array/shuffle.rs
@@ -141,17 +141,17 @@ fn extract_seed(seed_arg: &ColumnarValue) -> Result<Option<u64>> {
fn array_shuffle_with_seed(arg: &[ArrayRef], seed: Option<u64>) -> Result<ArrayRef> {
let [input_array] = take_function_args("shuffle", arg)?;
match &input_array.data_type() {
- List(field) => {
+ List(_) => {
let array = as_list_array(input_array)?;
- general_array_shuffle::<i32>(array, field, seed)
+ general_array_shuffle::<i32>(array, seed)
}
- LargeList(field) => {
+ LargeList(_) => {
let array = as_large_list_array(input_array)?;
- general_array_shuffle::<i64>(array, field, seed)
+ general_array_shuffle::<i64>(array, seed)
}
- FixedSizeList(field, _) => {
+ FixedSizeList(_, _) => {
let array = as_fixed_size_list_array(input_array)?;
- fixed_size_array_shuffle(array, field, seed)
+ fixed_size_array_shuffle(array, seed)
}
Null => Ok(Arc::clone(input_array)),
array_type => exec_err!(
@@ -163,7 +163,6 @@ fn array_shuffle_with_seed(arg: &[ArrayRef], seed: Option<u64>) -> Result<ArrayR
fn general_array_shuffle<O: OffsetSizeTrait>(
array: &GenericListArray<O>,
- field: &FieldRef,
seed: Option<u64>,
) -> Result<ArrayRef> {
let values = array.values();
@@ -212,7 +211,7 @@ fn general_array_shuffle<O: OffsetSizeTrait>(
let data = mutable.freeze();
Ok(Arc::new(GenericListArray::<O>::try_new(
- Arc::clone(field),
+ Arc::clone(array.value_field()),
OffsetBuffer::<O>::new(offsets.into()),
arrow::array::make_array(data),
Some(nulls.into()),
@@ -221,7 +220,6 @@ fn general_array_shuffle<O: OffsetSizeTrait>(
fn fixed_size_array_shuffle(
array: &FixedSizeListArray,
- field: &FieldRef,
seed: Option<u64>,
) -> Result<ArrayRef> {
let values = array.values();
@@ -265,7 +263,7 @@ fn fixed_size_array_shuffle(
let data = mutable.freeze();
Ok(Arc::new(FixedSizeListArray::try_new(
- Arc::clone(field),
+ Arc::clone(array.value_field()),
array.value_length(),
arrow::array::make_array(data),
Some(nulls.into()),