Skip to content

Apache DataFusion SQL Query Engine

perf: Size builder capacity using visible array span, not total size (#25502)

Neil Conwayb8683f3d0ae3
diff --git a/datafusion/functions-nested/src/array_add.rs b/datafusion/functions-nested/src/array_add.rs
index dd170911fb..86b3fc7475 100644
--- a/datafusion/functions-nested/src/array_add.rs
+++ b/datafusion/functions-nested/src/array_add.rs
@@ -17,24 +17,20 @@
//! [`ScalarUDFImpl`] definitions for array_add function.
-use crate::utils::{coerce_array_math_arg_types, make_scalar_function};
-use arrow::array::{
- Array, ArrayRef, Float64Array, GenericListArray, NullBufferBuilder, OffsetSizeTrait,
+use crate::utils::{
+ array_math_binary_op, coerce_array_math_arg_types, make_scalar_function,
};
-use arrow::buffer::{NullBuffer, OffsetBuffer};
+use arrow::array::ArrayRef;
use arrow::datatypes::{
DataType,
DataType::{LargeList, List},
- Field,
};
-use datafusion_common::cast::{as_float64_array, as_generic_list_array};
use datafusion_common::{Result, exec_err, utils::take_function_args};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
use datafusion_macros::user_doc;
-use std::sync::Arc;
make_udf_expr_and_func!(
ArrayAdd,
@@ -120,84 +116,29 @@ impl ScalarUDFImpl for ArrayAdd {
fn array_add_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
let [array1, array2] = take_function_args("array_add", args)?;
+ let add = |a: f64, b: f64| a + b;
match (array1.data_type(), array2.data_type()) {
- (List(_), List(_)) => general_array_add::<i32>(array1, array2),
- (LargeList(_), LargeList(_)) => general_array_add::<i64>(array1, array2),
+ (List(_), List(_)) => {
+ array_math_binary_op::<i32, _>("array_add", array1, array2, add)
+ }
+ (LargeList(_), LargeList(_)) => {
+ array_math_binary_op::<i64, _>("array_add", array1, array2, add)
+ }
(arg_type1, arg_type2) => exec_err!(
"array_add received unexpected types after coercion: {arg_type1} and {arg_type2}"
),
}
}
-fn general_array_add<O: OffsetSizeTrait>(
- lhs: &ArrayRef,
- rhs: &ArrayRef,
-) -> Result<ArrayRef> {
- let lhs = as_generic_list_array::<O>(lhs)?;
- let rhs = as_generic_list_array::<O>(rhs)?;
-
- let lhs_values = as_float64_array(lhs.values())?;
- let rhs_values = as_float64_array(rhs.values())?;
- let lhs_offsets = lhs.value_offsets();
- let rhs_offsets = rhs.value_offsets();
-
- // Row-level validity: a row is valid iff both sides are valid at that row.
- let row_nulls = NullBuffer::union(lhs.nulls(), rhs.nulls());
-
- let mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());
- let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());
- let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);
- out_offsets.push(O::zero());
-
- for row in 0..lhs.len() {
- // Whole-row NULL on either side -> NULL output row, no elements.
- if row_nulls.as_ref().is_some_and(|nb| nb.is_null(row)) {
- out_offsets.push(out_offsets[row]);
- continue;
- }
-
- let start1 = lhs_offsets[row].as_usize();
- let len1 = lhs.value_length(row).as_usize();
- let start2 = rhs_offsets[row].as_usize();
- let len2 = rhs.value_length(row).as_usize();
-
- if len1 != len2 {
- return exec_err!(
- "array_add requires both list inputs to have the same length per row, got {len1} and {len2} at row {row}"
- );
- }
-
- let l_slice = lhs_values.slice(start1, len1);
- let r_slice = rhs_values.slice(start2, len2);
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::Arc;
- let l_vals = l_slice.values();
- let r_vals = r_slice.values();
-
- for i in 0..len1 {
- out_values.push(l_vals[i] + r_vals[i]);
- }
-
- // Per-element validity: position `i` is valid iff both lhs[i] and rhs[i]
- // are valid. `NullBuffer::union` returns `None` when both sides are
- // entirely valid.
- match NullBuffer::union(l_slice.nulls(), r_slice.nulls()) {
- Some(nb) => out_inner_nulls.append_buffer(&nb),
- None => out_inner_nulls.append_n_non_nulls(len1),
- }
-
- out_offsets.push(out_offsets[row] + O::usize_as(len1));
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ array_add_inner(&[Arc::clone(input), Arc::clone(input)])
+ })
}
-
- let values_array = Arc::new(Float64Array::new(
- out_values.into(),
- out_inner_nulls.finish(),
- ));
- let field = Arc::new(Field::new_list_field(DataType::Float64, true));
-
- Ok(Arc::new(GenericListArray::<O>::try_new(
- field,
- OffsetBuffer::new(out_offsets.into()),
- values_array,
- row_nulls,
- )?))
}
diff --git a/datafusion/functions-nested/src/array_compact.rs b/datafusion/functions-nested/src/array_compact.rs
index 4222d6264b..9815b96573 100644
--- a/datafusion/functions-nested/src/array_compact.rs
+++ b/datafusion/functions-nested/src/array_compact.rs
@@ -26,6 +26,7 @@ use arrow::buffer::OffsetBuffer;
use arrow::datatypes::DataType;
use arrow::datatypes::DataType::{LargeList, List, Null};
use datafusion_common::cast::{as_large_list_array, as_list_array};
+use datafusion_common::utils::offset_span;
use datafusion_common::{Result, exec_err, utils::take_function_args};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -136,8 +137,7 @@ fn compact_list<O: OffsetSizeTrait>(
// Fast path: no validity buffer, no nulls to remove
return Ok(Arc::new(list_array.clone()));
};
- let values_null_count = values_nulls.null_count();
- if values_null_count == 0 {
+ if values_nulls.null_count() == 0 {
// Fast path: validity buffer present but no nulls set
return Ok(Arc::new(list_array.clone()));
}
@@ -145,7 +145,9 @@ fn compact_list<O: OffsetSizeTrait>(
let list_nulls = list_array.nulls();
let list_offsets = list_array.offsets();
let original_data = values.to_data();
- let capacity = original_data.len() - values_null_count;
+ let (first_offset, visible_len) = offset_span(list_offsets);
+ let capacity =
+ visible_len - values_nulls.slice(first_offset, visible_len).null_count();
let mut offsets = Vec::<O>::with_capacity(list_array.len() + 1);
offsets.push(O::zero());
let mut mutable = MutableArrayData::with_capacities(
@@ -196,3 +198,15 @@ fn compact_list<O: OffsetSizeTrait>(
list_nulls.cloned(),
)?))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ array_compact_inner(std::slice::from_ref(input))
+ })
+ }
+}
diff --git a/datafusion/functions-nested/src/array_normalize.rs b/datafusion/functions-nested/src/array_normalize.rs
index 988e2679b9..386ea5d77b 100644
--- a/datafusion/functions-nested/src/array_normalize.rs
+++ b/datafusion/functions-nested/src/array_normalize.rs
@@ -28,7 +28,9 @@ use arrow::datatypes::{
Field,
};
use datafusion_common::cast::{as_float64_array, as_generic_list_array};
-use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only};
+use datafusion_common::utils::{
+ ListCoercion, coerced_type_with_base_type_only, offset_span_len,
+};
use datafusion_common::{Result, internal_err, plan_err, utils::take_function_args};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -141,9 +143,9 @@ fn array_normalize_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<ArrayRef> {
let list_array = as_generic_list_array::<O>(&arrays[0])?;
let values = as_float64_array(list_array.values())?;
- let offsets = list_array.value_offsets();
+ let offsets = list_array.offsets();
- let mut new_values: Vec<f64> = Vec::with_capacity(values.len());
+ let mut new_values: Vec<f64> = Vec::with_capacity(offset_span_len(offsets));
let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);
new_offsets.push(O::zero());
let mut nulls = NullBufferBuilder::new(list_array.len());
@@ -223,3 +225,15 @@ fn general_array_normalize<O: OffsetSizeTrait>(arrays: &[ArrayRef]) -> Result<Ar
nulls.finish(),
)?))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ array_normalize_inner(std::slice::from_ref(input))
+ })
+ }
+}
diff --git a/datafusion/functions-nested/src/array_scale.rs b/datafusion/functions-nested/src/array_scale.rs
index 5b9130e9f0..5dad7cb93b 100644
--- a/datafusion/functions-nested/src/array_scale.rs
+++ b/datafusion/functions-nested/src/array_scale.rs
@@ -26,7 +26,9 @@ use arrow::datatypes::{
Field,
};
use datafusion_common::cast::{as_float64_array, as_generic_list_array};
-use datafusion_common::utils::{ListCoercion, coerced_type_with_base_type_only};
+use datafusion_common::utils::{
+ ListCoercion, coerced_type_with_base_type_only, offset_span_len,
+};
use datafusion_common::{Result, internal_err, plan_err, utils::take_function_args};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -161,7 +163,7 @@ fn general_array_scale<O: OffsetSizeTrait>(
let scalar_array = as_float64_array(scalar)?;
let values = as_float64_array(list_array.values())?;
- let offsets = list_array.value_offsets();
+ let offsets = list_array.offsets();
// A row is null whenever either input row is null. The scalar applies
// uniformly across the array, so a null scalar makes the whole row
@@ -169,7 +171,7 @@ fn general_array_scale<O: OffsetSizeTrait>(
// rather than tracking row nulls inside the value loop.
let row_nulls = NullBuffer::union(list_array.nulls(), scalar_array.nulls());
- let mut value_builder = Float64Array::builder(values.len());
+ let mut value_builder = Float64Array::builder(offset_span_len(offsets));
let mut new_offsets = Vec::<O>::with_capacity(list_array.len() + 1);
new_offsets.push(O::zero());
@@ -217,3 +219,18 @@ fn general_array_scale<O: OffsetSizeTrait>(
row_nulls,
)?))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ array_scale_inner(&[
+ Arc::clone(input),
+ Arc::new(Float64Array::from(vec![2.0; input.len()])),
+ ])
+ })
+ }
+}
diff --git a/datafusion/functions-nested/src/array_subtract.rs b/datafusion/functions-nested/src/array_subtract.rs
index 24600da04f..628e6fc3a6 100644
--- a/datafusion/functions-nested/src/array_subtract.rs
+++ b/datafusion/functions-nested/src/array_subtract.rs
@@ -128,3 +128,16 @@ fn array_subtract_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
),
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::Arc;
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ array_subtract_inner(&[Arc::clone(input), Arc::clone(input)])
+ })
+ }
+}
diff --git a/datafusion/functions-nested/src/concat.rs b/datafusion/functions-nested/src/concat.rs
index 8b8d39574e..4f8c2e7975 100644
--- a/datafusion/functions-nested/src/concat.rs
+++ b/datafusion/functions-nested/src/concat.rs
@@ -32,7 +32,7 @@ use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::Result;
use datafusion_common::utils::{
- ListCoercion, base_type, coerced_type_with_base_type_only,
+ ListCoercion, base_type, coerced_type_with_base_type_only, offset_span_len,
};
use datafusion_common::{
cast::as_generic_list_array,
@@ -441,8 +441,11 @@ fn concat_internal<O: OffsetSizeTrait>(
list_arrays.iter().map(|la| la.values().to_data()).collect();
let values_data_refs: Vec<&ArrayData> = values_data.iter().collect();
- // Estimate capacity as the sum of all values arrays' lengths.
- let total_capacity: usize = values_data.iter().map(|d| d.len()).sum();
+ // Only reserve for values covered by the visible rows of each input.
+ let total_capacity: usize = list_arrays
+ .iter()
+ .map(|array| offset_span_len(array.offsets()))
+ .sum();
let mut mutable = MutableArrayData::with_capacities(
values_data_refs,
@@ -604,7 +607,8 @@ where
let values = list_array.values();
let original_data = values.to_data();
let element_data = element_array.to_data();
- let capacity = Capacities::Array(original_data.len() + element_data.len());
+ let capacity =
+ Capacities::Array(offset_span_len(list_array.offsets()) + element_array.len());
let mut mutable = MutableArrayData::with_capacities(
vec![&original_data, &element_data],
@@ -643,3 +647,28 @@ where
None,
)?))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_concat_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ array_concat_inner(&[Arc::clone(input), Arc::clone(input)])
+ })
+ }
+
+ #[test]
+ fn test_append_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ array_append_inner(
+ &[
+ Arc::clone(input),
+ Arc::new(arrow::array::Float64Array::from(vec![3.0; input.len()])),
+ ],
+ input.data_type(),
+ )
+ })
+ }
+}
diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs
index aa0d5f80d1..8f6cc2ce5d 100644
--- a/datafusion/functions-nested/src/extract.rs
+++ b/datafusion/functions-nested/src/extract.rs
@@ -33,6 +33,7 @@ use datafusion_common::cast::{
};
use datafusion_common::internal_err;
use datafusion_common::utils::ListCoercion;
+use datafusion_common::utils::offset_span_len;
use datafusion_common::{
Result, exec_datafusion_err, exec_err, internal_datafusion_err, plan_err,
utils::take_function_args,
@@ -213,7 +214,7 @@ where
}
let original_data = values.to_data();
- let capacity = Capacities::Array(original_data.len());
+ let capacity = Capacities::Array(array.len());
// use_nulls: true, we don't construct List for array_element, so we need explicit nulls.
let mut mutable =
@@ -602,7 +603,7 @@ where
{
let values = array.values();
let original_data = values.to_data();
- let capacity = Capacities::Array(original_data.len());
+ let capacity = Capacities::Array(offset_span_len(array.offsets()));
// Carry the input's list field through to the output so that the returned
// type matches the one promised by `return_type` / `return_field_from_args`,
// including the field name, nullability and metadata.
@@ -1396,4 +1397,25 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_element_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ super::array_element_inner(&[
+ Arc::clone(input),
+ Arc::new(Int64Array::from(vec![1; input.len()])),
+ ])
+ })
+ }
+
+ #[test]
+ fn test_slice_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ super::array_slice_inner(&[
+ Arc::clone(input),
+ Arc::new(Int64Array::from(vec![1; input.len()])),
+ Arc::new(Int64Array::from(vec![3; input.len()])),
+ ])
+ })
+ }
}
diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rs
index 7470c60b85..befcc09d22 100644
--- a/datafusion/functions-nested/src/remove.rs
+++ b/datafusion/functions-nested/src/remove.rs
@@ -25,7 +25,7 @@ use arrow::array::{
use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{DataType, FieldRef};
use datafusion_common::cast::as_int64_array;
-use datafusion_common::utils::{ListCoercion, offset_span};
+use datafusion_common::utils::{ListCoercion, offset_span, offset_span_len};
use datafusion_common::{
Result, ScalarValue, exec_err, internal_err, utils::take_function_args,
};
@@ -470,7 +470,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
let mut mutable = MutableArrayData::with_capacities(
vec![&original_data],
false,
- Capacities::Array(original_data.len()),
+ Capacities::Array(offset_span_len(list_array.offsets())),
);
let mut valid = NullBufferBuilder::new(list_array.len());
@@ -1167,4 +1167,13 @@ mod tests {
_ => panic!("Expected ColumnarValue::Array"),
}
}
+
+ #[test]
+ fn test_sliced_capacity() -> datafusion_common::Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ let element: ArrayRef =
+ Arc::new(arrow::array::Float64Array::from(vec![3.0; input.len()]));
+ super::array_remove_internal(input, &element, &vec![Some(1); input.len()])
+ })
+ }
}
diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rs
index af45058801..d86208da1c 100644
--- a/datafusion/functions-nested/src/replace.rs
+++ b/datafusion/functions-nested/src/replace.rs
@@ -24,7 +24,7 @@ use arrow::array::{
use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::cast::as_int64_array;
-use datafusion_common::utils::{ListCoercion, offset_span};
+use datafusion_common::utils::{ListCoercion, offset_span, offset_span_len};
use datafusion_common::{
Result, ScalarValue, exec_err, internal_err, utils::take_function_args,
};
@@ -445,7 +445,7 @@ fn general_replace<O: OffsetSizeTrait>(
let values = list_array.values();
let original_data = values.to_data();
let to_data = to_array.to_data();
- let capacity = Capacities::Array(original_data.len());
+ let capacity = Capacities::Array(offset_span_len(list_array.offsets()));
// First array is the original array, second array is the element to replace with.
let mut mutable = MutableArrayData::with_capacities(
@@ -824,4 +824,22 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ let from: ArrayRef =
+ Arc::new(arrow::array::Float64Array::from(vec![3.0; input.len()]));
+ let to: ArrayRef =
+ Arc::new(arrow::array::Float64Array::from(vec![7.0; input.len()]));
+ super::array_replace_internal(
+ "array_replace",
+ input,
+ &from,
+ &to,
+ &vec![Some(1); input.len()],
+ input.data_type(),
+ )
+ })
+ }
}
diff --git a/datafusion/functions-nested/src/reverse.rs b/datafusion/functions-nested/src/reverse.rs
index 587d0dd29f..2587a5ebff 100644
--- a/datafusion/functions-nested/src/reverse.rs
+++ b/datafusion/functions-nested/src/reverse.rs
@@ -32,6 +32,7 @@ use datafusion_common::cast::{
as_fixed_size_list_array, as_large_list_array, as_large_list_view_array,
as_list_array, as_list_view_array,
};
+use datafusion_common::utils::offset_span_len;
use datafusion_common::{Result, exec_err, utils::take_function_args};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -149,7 +150,7 @@ fn general_array_reverse<O: OffsetSizeTrait>(
) -> Result<ArrayRef> {
let values = array.values();
let mut offsets = vec![O::usize_as(0)];
- let mut indices: Vec<O> = Vec::with_capacity(values.len());
+ let mut indices: Vec<O> = Vec::with_capacity(offset_span_len(array.offsets()));
for (row_index, (&start, &end)) in array.offsets().iter().tuple_windows().enumerate()
{
diff --git a/datafusion/functions-nested/src/string.rs b/datafusion/functions-nested/src/string.rs
index 6d03e1cd9c..3c43408065 100644
--- a/datafusion/functions-nested/src/string.rs
+++ b/datafusion/functions-nested/src/string.rs
@@ -25,7 +25,7 @@ use arrow::array::{
};
use arrow::datatypes::{DataType, Field};
-use datafusion_common::utils::ListCoercion;
+use datafusion_common::utils::{ListCoercion, offset_span_len};
use datafusion_common::{DataFusionError, Result, ScalarValue, not_impl_err};
use std::fmt::{self, Write};
@@ -278,8 +278,10 @@ impl ScalarUDFImpl for StringToArray {
let result = match string_array.data_type() {
Utf8 => {
let arr = string_array.as_string::<i32>();
- let builder =
- StringBuilder::with_capacity(arr.len(), arr.get_buffer_memory_size());
+ let builder = StringBuilder::with_capacity(
+ arr.len(),
+ offset_span_len(arr.offsets()),
+ );
string_to_array_scalar_args(&arr, delimiter, null_value, builder)
}
Utf8View => {
@@ -291,7 +293,7 @@ impl ScalarUDFImpl for StringToArray {
let arr = string_array.as_string::<i64>();
let builder = LargeStringBuilder::with_capacity(
arr.len(),
- arr.get_buffer_memory_size(),
+ offset_span_len(arr.offsets()),
);
string_to_array_scalar_args(&arr, delimiter, null_value, builder)
}
@@ -423,7 +425,7 @@ fn string_to_array_fallback(args: &[ArrayRef]) -> Result<ArrayRef> {
Utf8 => {
let arr = args[0].as_string::<i32>();
let builder =
- StringBuilder::with_capacity(arr.len(), arr.get_buffer_memory_size());
+ StringBuilder::with_capacity(arr.len(), offset_span_len(arr.offsets()));
string_to_array_column_args(&arr, &args[1], null_value_array, builder)
}
Utf8View => {
@@ -435,7 +437,7 @@ fn string_to_array_fallback(args: &[ArrayRef]) -> Result<ArrayRef> {
let arr = args[0].as_string::<i64>();
let builder = LargeStringBuilder::with_capacity(
arr.len(),
- arr.get_buffer_memory_size(),
+ offset_span_len(arr.offsets()),
);
string_to_array_column_args(&arr, &args[1], null_value_array, builder)
}
@@ -815,3 +817,52 @@ impl StringArrayBuilderType for LargeStringBuilder {
LargeStringBuilder::append_null(self);
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ let padding = "x".repeat(65536);
+ let input = StringArray::from(vec![
+ Some(padding.as_str()),
+ Some("a,b"),
+ None,
+ Some(padding.as_str()),
+ ]);
+ for data_type in [Utf8, LargeUtf8] {
+ let input = cast(&input, &data_type)?;
+ for len in [0, 2] {
+ for delimiter in [
+ ColumnarValue::Scalar(ScalarValue::from(",")),
+ ColumnarValue::Array(Arc::new(StringArray::from(vec![","; len]))),
+ ] {
+ let result = StringToArray::new()
+ .invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(input.slice(1, len)),
+ delimiter,
+ ],
+ arg_fields: vec![
+ Field::new("input", data_type.clone(), true).into(),
+ Field::new("delimiter", Utf8, false).into(),
+ ],
+ number_rows: len,
+ return_field: Field::new(
+ "result",
+ DataType::new_list(data_type.clone(), true),
+ true,
+ )
+ .into(),
+ config_options: Default::default(),
+ })?
+ .into_array(len)?;
+ assert_eq!(result.len(), len);
+ assert!(result.get_buffer_memory_size() < 1024);
+ }
+ }
+ }
+ Ok(())
+ }
+}
diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs
index a6a93ed5df..b4fa39e2cc 100644
--- a/datafusion/functions-nested/src/utils.rs
+++ b/datafusion/functions-nested/src/utils.rs
@@ -30,6 +30,7 @@ use datafusion_common::cast::{
as_fixed_size_list_array, as_float64_array, as_generic_list_array,
as_large_list_array, as_large_list_view_array, as_list_array, as_list_view_array,
};
+use datafusion_common::utils::offset_span_len;
use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_err};
use datafusion_expr::ColumnarValue;
@@ -409,8 +410,9 @@ where
let row_nulls = NullBuffer::union(lhs.nulls(), rhs.nulls());
- let mut out_values: Vec<f64> = Vec::with_capacity(lhs_values.len());
- let mut out_inner_nulls = NullBufferBuilder::new(lhs_values.len());
+ let capacity = offset_span_len(lhs.offsets());
+ let mut out_values: Vec<f64> = Vec::with_capacity(capacity);
+ let mut out_inner_nulls = NullBufferBuilder::new(capacity);
let mut out_offsets = Vec::<O>::with_capacity(lhs.len() + 1);
out_offsets.push(O::zero());
@@ -507,7 +509,7 @@ pub(crate) fn needs_norm_scale(sum_of_squares: f64, len: usize) -> bool {
}
#[cfg(test)]
-mod tests {
+pub(crate) mod tests {
use super::*;
use arrow::array::ListArray;
use arrow::datatypes::Int64Type;
@@ -602,4 +604,52 @@ mod tests {
assert!(needs_norm_scale(f64::INFINITY, 1));
assert!(needs_norm_scale(f64::NAN, 1));
}
+
+ /// Tests the array function supplied via `run` on small slices of a large
+ /// list, checking two independent properties:
+ ///
+ /// 1. Correctness: results match those from equivalent inputs with compact
+ /// child storage.
+ /// 2. Capacity: output buffers retain little memory, catching reservations
+ /// based on the full backing child array rather than the visible slice.
+ pub(crate) fn check_sliced_list_behavior(
+ run: impl Fn(&ArrayRef) -> Result<ArrayRef>,
+ ) -> Result<()> {
+ let padding = 8192;
+ let visible = vec![Some(3.0), None, Some(4.0), Some(9.0), Some(9.0)];
+ let values = Float64Array::from_iter(
+ std::iter::repeat_n(Some(1.0), padding)
+ .chain(visible.iter().copied())
+ .chain(std::iter::repeat_n(Some(1.0), padding)),
+ );
+ let field = Arc::new(Field::new_list_field(DataType::Float64, true));
+ let input = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::from_lengths([padding, 3, 0, 2, padding]),
+ Arc::new(values),
+ Some(NullBuffer::from(vec![true, true, true, false, true])),
+ );
+ let compact = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::from_lengths([3, 0, 2]),
+ Arc::new(Float64Array::from(visible)),
+ Some(NullBuffer::from(vec![true, true, false])),
+ );
+ for data_type in [input.data_type().clone(), DataType::LargeList(field)] {
+ let input = arrow::compute::cast(&input, &data_type)?;
+ let compact = arrow::compute::cast(&compact, &data_type)?;
+ // Middle slice, empty slice, empty row, and null row with child data.
+ for (offset, len) in [(0, 3), (0, 0), (1, 1), (2, 1)] {
+ let result = run(&input.slice(1 + offset, len))?;
+ let expected = run(&compact.slice(offset, len))?;
+ assert_eq!(result.as_ref(), expected.as_ref());
+ assert!(
+ result.get_buffer_memory_size() < 1024,
+ "{data_type}: {} bytes for {len} rows",
+ result.get_buffer_memory_size()
+ );
+ }
+ }
+ Ok(())
+ }
}
diff --git a/datafusion/functions/src/strings.rs b/datafusion/functions/src/strings.rs
index 4e21f6c74f..9bdb858ba1 100644
--- a/datafusion/functions/src/strings.rs
+++ b/datafusion/functions/src/strings.rs
@@ -19,6 +19,7 @@ use std::marker::PhantomData;
use std::mem::size_of;
use std::sync::Arc;
+use datafusion_common::utils::offset_span_len;
use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,
plan_err,
@@ -1324,7 +1325,7 @@ impl ColumnarValueRef<'_> {
ColumnarValue::Array(array) => match array.data_type() {
DataType::Utf8 => {
let string_array = as_string_array(array)?;
- *data_size += string_array.values().len() * size_factor;
+ *data_size += offset_span_len(string_array.offsets()) * size_factor;
let column = if array.is_nullable() {
ColumnarValueRef::NullableArray(string_array)
} else {
@@ -1334,7 +1335,7 @@ impl ColumnarValueRef<'_> {
}
DataType::LargeUtf8 => {
let string_array = as_largestring_array(array);
- *data_size += string_array.values().len() * size_factor;
+ *data_size += offset_span_len(string_array.offsets()) * size_factor;
let column = if array.is_nullable() {
ColumnarValueRef::NullableLargeStringArray(string_array)
} else {
@@ -1354,7 +1355,7 @@ impl ColumnarValueRef<'_> {
}
DataType::Binary => {
let binary_array = as_binary_array(array)?;
- *data_size += binary_array.values().len() * size_factor;
+ *data_size += offset_span_len(binary_array.offsets()) * size_factor;
let column = if array.is_nullable() {
ColumnarValueRef::NullableBinaryArray(binary_array)
} else {
@@ -1364,7 +1365,7 @@ impl ColumnarValueRef<'_> {
}
DataType::LargeBinary => {
let binary_array = as_large_binary_array(array)?;
- *data_size += binary_array.values().len() * size_factor;
+ *data_size += offset_span_len(binary_array.offsets()) * size_factor;
let column = if array.is_nullable() {
ColumnarValueRef::NullableLargeBinaryArray(binary_array)
} else {
@@ -1427,6 +1428,32 @@ pub(crate) fn widest_string_type(types: &[DataType]) -> DataType {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::utils::test::sliced_byte_array;
+
+ #[test]
+ fn concat_capacity_uses_visible_bytes() -> Result<()> {
+ for data_type in [
+ DataType::Utf8,
+ DataType::LargeUtf8,
+ DataType::Binary,
+ DataType::LargeBinary,
+ ] {
+ let array = sliced_byte_array(&[Some("abc")], &data_type)?;
+ for len in [0, 1] {
+ let input = ColumnarValue::Array(array.slice(0, len));
+ let mut data_size = 0;
+ ColumnarValueRef::from_columnar_value(
+ &input,
+ &mut data_size,
+ len,
+ 2,
+ false,
+ )?;
+ assert_eq!(data_size, 6 * len, "{data_type}");
+ }
+ }
+ Ok(())
+ }
/// Run `scenario` against `builder`, finish with a null buffer derived
/// from `expected` (a bit is set wherever `expected[i].is_some()`), and
diff --git a/datafusion/functions/src/unicode/initcap.rs b/datafusion/functions/src/unicode/initcap.rs
index d521ca0411..d4f4f15323 100644
--- a/datafusion/functions/src/unicode/initcap.rs
+++ b/datafusion/functions/src/unicode/initcap.rs
@@ -170,7 +170,7 @@ fn initcap<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
let len = string_array.len();
let mut builder = GenericStringArrayBuilder::<T>::with_capacity(
len,
- string_array.value_data().len(),
+ offset_span_len(string_array.offsets()),
);
let mut container = String::new();
@@ -306,7 +306,7 @@ fn initcap_string(input: &str, container: &mut String) {
#[cfg(test)]
mod tests {
use crate::unicode::initcap::InitcapFunc;
- use crate::utils::test::test_function;
+ use crate::utils::test::{sliced_byte_array, test_function};
use arrow::array::{Array, ArrayRef, LargeStringArray, StringArray, StringViewArray};
use arrow::datatypes::DataType::{Utf8, Utf8View};
use datafusion_common::{Result, ScalarValue};
@@ -519,4 +519,17 @@ mod tests {
assert_eq!(result.value_data().len(), result.offsets().last() as usize);
Ok(())
}
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ for data_type in [Utf8, arrow::datatypes::DataType::LargeUtf8] {
+ let input = sliced_byte_array(&[Some("aé,b"), None], &data_type)?;
+ for len in [0, 2] {
+ let result = super::initcap_array(&input.slice(0, len))?;
+ assert_eq!(result.len(), len);
+ assert!(result.get_buffer_memory_size() < 1024);
+ }
+ }
+ Ok(())
+ }
}
diff --git a/datafusion/functions/src/unicode/translate.rs b/datafusion/functions/src/unicode/translate.rs
index 85e83897f4..e06e571c42 100644
--- a/datafusion/functions/src/unicode/translate.rs
+++ b/datafusion/functions/src/unicode/translate.rs
@@ -19,6 +19,7 @@ use arrow::array::{Array, ArrayRef, AsArray, GenericStringArray, StringArrayType
use arrow::buffer::NullBuffer;
use arrow::datatypes::DataType;
use datafusion_common::HashMap;
+use datafusion_common::utils::offset_span_len;
use super::common::try_as_scalar_str;
use crate::strings::{
@@ -114,7 +115,7 @@ impl ScalarUDFImpl for TranslateFunc {
let arr = string_array.as_string::<i32>();
let builder = GenericStringArrayBuilder::<i32>::with_capacity(
len,
- arr.value_data().len(),
+ offset_span_len(arr.offsets()),
);
translate_with_table(&arr, &table, builder)
}
@@ -122,7 +123,7 @@ impl ScalarUDFImpl for TranslateFunc {
let arr = string_array.as_string::<i64>();
let builder = GenericStringArrayBuilder::<i64>::with_capacity(
len,
- arr.value_data().len(),
+ offset_span_len(arr.offsets()),
);
translate_with_table(&arr, &table, builder)
}
@@ -160,7 +161,7 @@ fn invoke_translate(args: &[ArrayRef]) -> Result<ArrayRef> {
let to_array = args[2].as_string::<i32>();
let builder = GenericStringArrayBuilder::<i32>::with_capacity(
len,
- string_array.value_data().len(),
+ offset_span_len(string_array.offsets()),
);
translate(&string_array, from_array, to_array, builder)
}
@@ -170,7 +171,7 @@ fn invoke_translate(args: &[ArrayRef]) -> Result<ArrayRef> {
let to_array = args[2].as_string::<i32>();
let builder = GenericStringArrayBuilder::<i64>::with_capacity(
len,
- string_array.value_data().len(),
+ offset_span_len(string_array.offsets()),
);
translate(&string_array, from_array, to_array, builder)
}
@@ -442,7 +443,7 @@ mod tests {
use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
use crate::unicode::translate::TranslateFunc;
- use crate::utils::test::test_function;
+ use crate::utils::test::{sliced_byte_array, test_function};
#[test]
fn test_functions() -> Result<()> {
@@ -612,4 +613,48 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ use arrow::datatypes::Field;
+ use datafusion_expr::ScalarFunctionArgs;
+ for data_type in [Utf8, arrow::datatypes::DataType::LargeUtf8] {
+ let input = sliced_byte_array(&[Some("aé,b"), None], &data_type)?;
+ for len in [0, 2] {
+ for scalar in [false, true] {
+ let parameter = |s| {
+ if scalar {
+ ColumnarValue::Scalar(ScalarValue::from(s))
+ } else {
+ ColumnarValue::Array(Arc::new(StringArray::from(vec![
+ s;
+ len
+ ])))
+ }
+ };
+ let result = TranslateFunc::new()
+ .invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(input.slice(0, len)),
+ parameter("a"),
+ parameter("z"),
+ ],
+ arg_fields: vec![
+ Field::new("input", data_type.clone(), true).into(),
+ Field::new("from", Utf8, false).into(),
+ Field::new("to", Utf8, false).into(),
+ ],
+ number_rows: len,
+ return_field: Field::new("result", data_type.clone(), true)
+ .into(),
+ config_options: Default::default(),
+ })?
+ .into_array(len)?;
+ assert_eq!(result.len(), len);
+ assert!(result.get_buffer_memory_size() < 1024);
+ }
+ }
+ }
+ Ok(())
+ }
}
diff --git a/datafusion/functions/src/utils.rs b/datafusion/functions/src/utils.rs
index e64f3fc0f2..b6d9aacaad 100644
--- a/datafusion/functions/src/utils.rs
+++ b/datafusion/functions/src/utils.rs
@@ -650,7 +650,7 @@ pub mod test {
}
use arrow::{
- array::Int32Array,
+ array::{Int32Array, StringArray},
datatypes::{DataType, Int32Type},
};
use itertools::Either;
@@ -658,6 +658,21 @@ pub mod test {
use super::*;
+ /// Builds a string or binary array containing `visible`, with a nonzero
+ /// starting offset and a large retained values buffer for capacity tests.
+ pub(crate) fn sliced_byte_array(
+ visible: &[Option<&str>],
+ data_type: &DataType,
+ ) -> Result<ArrayRef> {
+ let padding = "x".repeat(65536);
+ let mut values = vec![Some(padding.as_str())];
+ values.extend_from_slice(visible);
+ values.push(Some(padding.as_str()));
+ let input = StringArray::from(values);
+ // Cast before slicing so every data type retains the padding.
+ Ok(cast(&input, data_type)?.slice(1, visible.len()))
+ }
+
#[test]
fn test_calculate_binary_math_scalar_null() {
let left = Int32Array::from(vec![1, 2]);
diff --git a/datafusion/spark/src/function/array/shuffle.rs b/datafusion/spark/src/function/array/shuffle.rs
index 2673c9155f..c05fcda77b 100644
--- a/datafusion/spark/src/function/array/shuffle.rs
+++ b/datafusion/spark/src/function/array/shuffle.rs
@@ -26,6 +26,7 @@ use arrow::datatypes::FieldRef;
use datafusion_common::cast::{
as_fixed_size_list_array, as_large_list_array, as_list_array,
};
+use datafusion_common::utils::offset_span_len;
use datafusion_common::{
Result, ScalarValue, exec_err, internal_err, utils::take_function_args,
};
@@ -167,7 +168,9 @@ fn general_array_shuffle<O: OffsetSizeTrait>(
) -> Result<ArrayRef> {
let values = array.values();
let original_data = values.to_data();
- let capacity = Capacities::Array(original_data.len());
+ // Each null row emits one placeholder child value.
+ let capacity =
+ Capacities::Array(offset_span_len(array.offsets()) + array.null_count());
let mut offsets = vec![O::usize_as(0)];
let mut nulls = vec![];
let mut mutable =
@@ -312,4 +315,26 @@ mod tests {
assert!(result.is_nullable());
assert_eq!(result.data_type(), nullable_field.data_type());
}
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ use arrow::array::{Int64Array, ListArray};
+ use arrow::buffer::NullBuffer;
+ let field = Arc::new(Field::new_list_field(DataType::Int64, true));
+ let input = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::new(vec![0, 8192, 8195, 8195, 16384].into()),
+ Arc::new(Int64Array::from_iter_values(0..16384)),
+ Some(NullBuffer::from(vec![true, true, false, true])),
+ );
+ for data_type in [input.data_type().clone(), LargeList(field)] {
+ let input = arrow::compute::cast(&input, &data_type)?;
+ for len in [0, 2] {
+ let result = array_shuffle_with_seed(&[input.slice(1, len)], Some(0))?;
+ assert_eq!(result.len(), len);
+ assert!(result.get_buffer_memory_size() < 1024);
+ }
+ }
+ Ok(())
+ }
}
diff --git a/datafusion/spark/src/function/math/unhex.rs b/datafusion/spark/src/function/math/unhex.rs
index 941dc4330f..e68c78ad15 100644
--- a/datafusion/spark/src/function/math/unhex.rs
+++ b/datafusion/spark/src/function/math/unhex.rs
@@ -21,7 +21,7 @@ use datafusion_common::cast::{
as_large_string_array, as_string_array, as_string_view_array,
};
use datafusion_common::types::logical_string;
-use datafusion_common::utils::take_function_args;
+use datafusion_common::utils::{offset_span_len, take_function_args};
use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err,
};
@@ -163,7 +163,7 @@ fn spark_unhex(args: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError>
ColumnarValue::Array(array) => match array.data_type() {
DataType::Utf8 => {
let array = as_string_array(array)?;
- let capacity = array.values().len().div_ceil(2);
+ let capacity = offset_span_len(array.offsets()).div_ceil(2);
Ok(ColumnarValue::Array(unhex_array(
array.iter(),
array.len(),
@@ -182,7 +182,7 @@ fn spark_unhex(args: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError>
}
DataType::LargeUtf8 => {
let array = as_large_string_array(array)?;
- let capacity = array.values().len().div_ceil(2);
+ let capacity = offset_span_len(array.offsets()).div_ceil(2);
Ok(ColumnarValue::Array(unhex_array(
array.iter(),
array.len(),
@@ -214,3 +214,17 @@ fn spark_unhex(args: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError>
},
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::function::utils::test::check_sliced_string_capacity;
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ check_sliced_string_capacity("ab", |input| {
+ let len = input.len();
+ spark_unhex(&[ColumnarValue::Array(input)])?.into_array(len)
+ })
+ }
+}
diff --git a/datafusion/spark/src/function/url/url_decode.rs b/datafusion/spark/src/function/url/url_decode.rs
index 0e527068b7..ce98febce6 100644
--- a/datafusion/spark/src/function/url/url_decode.rs
+++ b/datafusion/spark/src/function/url/url_decode.rs
@@ -25,6 +25,7 @@ use arrow::datatypes::DataType;
use datafusion_common::cast::{
as_large_string_array, as_string_array, as_string_view_array,
};
+use datafusion_common::utils::offset_span_len;
use datafusion_common::{Result, exec_datafusion_err, exec_err, plan_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
@@ -221,14 +222,18 @@ pub fn spark_handled_url_decode(
match &args[0].data_type() {
DataType::Utf8 => {
let array = as_string_array(&args[0])?;
- let builder =
- StringBuilder::with_capacity(array.len(), array.value_data().len());
+ let builder = StringBuilder::with_capacity(
+ array.len(),
+ offset_span_len(array.offsets()),
+ );
decode_all!(array, builder)
}
DataType::LargeUtf8 => {
let array = as_large_string_array(&args[0])?;
- let builder =
- LargeStringBuilder::with_capacity(array.len(), array.value_data().len());
+ let builder = LargeStringBuilder::with_capacity(
+ array.len(),
+ offset_span_len(array.offsets()),
+ );
decode_all!(array, builder)
}
DataType::Utf8View => {
@@ -244,6 +249,7 @@ pub fn spark_handled_url_decode(
mod tests {
use super::*;
+ use crate::function::utils::test::check_sliced_string_capacity;
use arrow::array::{LargeStringArray, StringArray, StringViewArray};
const INPUT: [Option<&str>; 7] = [
@@ -318,4 +324,9 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ check_sliced_string_capacity("ab", |input| spark_url_decode(&[input]))
+ }
}
diff --git a/datafusion/spark/src/function/url/url_encode.rs b/datafusion/spark/src/function/url/url_encode.rs
index 87a70af4ac..20c33f19be 100644
--- a/datafusion/spark/src/function/url/url_encode.rs
+++ b/datafusion/spark/src/function/url/url_encode.rs
@@ -24,6 +24,7 @@ use arrow::datatypes::DataType;
use datafusion_common::cast::{
as_large_string_array, as_string_array, as_string_view_array,
};
+use datafusion_common::utils::offset_span_len;
use datafusion_common::{Result, exec_err, plan_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
@@ -117,14 +118,18 @@ fn spark_url_encode(args: &[ArrayRef]) -> Result<ArrayRef> {
match &args[0].data_type() {
DataType::Utf8 => {
let array = as_string_array(&args[0])?;
- let builder =
- StringBuilder::with_capacity(array.len(), array.value_data().len());
+ let builder = StringBuilder::with_capacity(
+ array.len(),
+ offset_span_len(array.offsets()),
+ );
encode_all!(array, builder)
}
DataType::LargeUtf8 => {
let array = as_large_string_array(&args[0])?;
- let builder =
- LargeStringBuilder::with_capacity(array.len(), array.value_data().len());
+ let builder = LargeStringBuilder::with_capacity(
+ array.len(),
+ offset_span_len(array.offsets()),
+ );
encode_all!(array, builder)
}
DataType::Utf8View => {
@@ -139,6 +144,7 @@ fn spark_url_encode(args: &[ArrayRef]) -> Result<ArrayRef> {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::function::utils::test::check_sliced_string_capacity;
use arrow::array::{LargeStringArray, StringArray, StringViewArray};
const INPUT: [Option<&str>; 5] = [
@@ -183,4 +189,9 @@ mod tests {
assert_eq!(&StringViewArray::from(EXPECTED.to_vec()), result);
Ok(())
}
+
+ #[test]
+ fn test_sliced_capacity() -> Result<()> {
+ check_sliced_string_capacity("ab", |input| spark_url_encode(&[input]))
+ }
}
diff --git a/datafusion/spark/src/function/utils.rs b/datafusion/spark/src/function/utils.rs
index e272d91d8a..6c5b5d5d40 100644
--- a/datafusion/spark/src/function/utils.rs
+++ b/datafusion/spark/src/function/utils.rs
@@ -17,6 +17,10 @@
#[cfg(test)]
pub mod test {
+ use arrow::array::{Array, ArrayRef, StringArray};
+ use arrow::datatypes::DataType;
+ use datafusion_common::Result;
+
/// $FUNC ScalarUDFImpl to test
/// $ARGS arguments (vec) to pass to function
/// $EXPECTED a Result<ColumnarValue>
@@ -142,4 +146,28 @@ pub mod test {
}
pub(crate) use test_scalar_function;
+
+ /// Runs `run` on small slices of a string array that retains a large
+ /// values buffer, and checks that the output is sized for the visible rows.
+ pub(crate) fn check_sliced_string_capacity(
+ visible: &str,
+ run: impl Fn(ArrayRef) -> Result<ArrayRef>,
+ ) -> Result<()> {
+ let padding = "x".repeat(65536);
+ let input = StringArray::from(vec![
+ Some(padding.as_str()),
+ Some(visible),
+ None,
+ Some(padding.as_str()),
+ ]);
+ for data_type in [DataType::Utf8, DataType::LargeUtf8] {
+ let input = arrow::compute::cast(&input, &data_type)?;
+ for len in [0, 2] {
+ let result = run(input.slice(1, len))?;
+ assert_eq!(result.len(), len);
+ assert!(result.get_buffer_memory_size() < 1024);
+ }
+ }
+ Ok(())
+ }
}