Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize array set ops on sliced arrays (#20693)

Neil Conway538a2019f8e7
diff --git a/datafusion/functions-nested/benches/array_set_ops.rs b/datafusion/functions-nested/benches/array_set_ops.rs
index 087e48d076..d43bbdb577 100644
--- a/datafusion/functions-nested/benches/array_set_ops.rs
+++ b/datafusion/functions-nested/benches/array_set_ops.rs
@@ -35,12 +35,19 @@ use std::sync::Arc;
const NUM_ROWS: usize = 1000;
const ARRAY_SIZES: &[usize] = &[10, 50, 100];
const SEED: u64 = 42;
+/// Extra rows on each side when building sliced arrays, so the underlying
+/// values buffer is much larger than the visible portion.
+const SLICE_PADDING: usize = 5000;
fn criterion_benchmark(c: &mut Criterion) {
bench_array_union(c);
bench_array_intersect(c);
bench_array_except(c);
bench_array_distinct(c);
+ bench_array_union_sliced(c);
+ bench_array_intersect_sliced(c);
+ bench_array_distinct_sliced(c);
+ bench_array_except_sliced(c);
}
fn invoke_udf(udf: &impl ScalarUDFImpl, array1: &ArrayRef, array2: &ArrayRef) {
@@ -276,5 +283,107 @@ fn create_array_with_duplicates(
)
}
+/// Slice a pair of arrays to the middle `NUM_ROWS` rows from a larger array.
+fn slice_pair(arrays: &(ArrayRef, ArrayRef)) -> (ArrayRef, ArrayRef) {
+ let a1 = arrays.0.slice(SLICE_PADDING, NUM_ROWS);
+ let a2 = arrays.1.slice(SLICE_PADDING, NUM_ROWS);
+ (a1, a2)
+}
+
+fn bench_array_union_sliced(c: &mut Criterion) {
+ let mut group = c.benchmark_group("array_union_sliced");
+ let udf = ArrayUnion::new();
+
+ for &array_size in ARRAY_SIZES {
+ let (a1, a2) = slice_pair(&create_arrays_with_overlap(
+ NUM_ROWS + 2 * SLICE_PADDING,
+ array_size,
+ 0.5,
+ ));
+ group.bench_with_input(
+ BenchmarkId::from_parameter(array_size),
+ &array_size,
+ |b, _| b.iter(|| invoke_udf(&udf, &a1, &a2)),
+ );
+ }
+ group.finish();
+}
+
+fn bench_array_intersect_sliced(c: &mut Criterion) {
+ let mut group = c.benchmark_group("array_intersect_sliced");
+ let udf = ArrayIntersect::new();
+
+ for &array_size in ARRAY_SIZES {
+ let (a1, a2) = slice_pair(&create_arrays_with_overlap(
+ NUM_ROWS + 2 * SLICE_PADDING,
+ array_size,
+ 0.5,
+ ));
+ group.bench_with_input(
+ BenchmarkId::from_parameter(array_size),
+ &array_size,
+ |b, _| b.iter(|| invoke_udf(&udf, &a1, &a2)),
+ );
+ }
+ group.finish();
+}
+
+fn bench_array_except_sliced(c: &mut Criterion) {
+ let mut group = c.benchmark_group("array_except_sliced");
+ let udf = ArrayExcept::new();
+
+ for &array_size in ARRAY_SIZES {
+ let (a1, a2) = slice_pair(&create_arrays_with_overlap(
+ NUM_ROWS + 2 * SLICE_PADDING,
+ array_size,
+ 0.5,
+ ));
+ group.bench_with_input(
+ BenchmarkId::from_parameter(array_size),
+ &array_size,
+ |b, _| b.iter(|| invoke_udf(&udf, &a1, &a2)),
+ );
+ }
+ group.finish();
+}
+
+fn bench_array_distinct_sliced(c: &mut Criterion) {
+ let mut group = c.benchmark_group("array_distinct_sliced");
+ let udf = ArrayDistinct::new();
+
+ for &array_size in ARRAY_SIZES {
+ let array =
+ create_array_with_duplicates(NUM_ROWS + 2 * SLICE_PADDING, array_size, 0.5)
+ .slice(SLICE_PADDING, NUM_ROWS);
+ group.bench_with_input(
+ BenchmarkId::from_parameter(array_size),
+ &array_size,
+ |b, _| {
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: vec![ColumnarValue::Array(array.clone())],
+ arg_fields: vec![
+ Field::new("arr", array.data_type().clone(), false)
+ .into(),
+ ],
+ number_rows: NUM_ROWS,
+ return_field: Field::new(
+ "result",
+ array.data_type().clone(),
+ false,
+ )
+ .into(),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ },
+ );
+ }
+ group.finish();
+}
+
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
diff --git a/datafusion/functions-nested/src/except.rs b/datafusion/functions-nested/src/except.rs
index 932eecf4b8..136c6d6b7a 100644
--- a/datafusion/functions-nested/src/except.rs
+++ b/datafusion/functions-nested/src/except.rs
@@ -175,10 +175,16 @@ fn general_except<OffsetSize: OffsetSizeTrait>(
) -> Result<GenericListArray<OffsetSize>> {
let converter = RowConverter::new(vec![SortField::new(l.value_type())])?;
- let l_values = l.values().to_owned();
- let r_values = r.values().to_owned();
- let l_values = converter.convert_columns(&[l_values])?;
- let r_values = converter.convert_columns(&[r_values])?;
+ // Only convert the visible portion of the values array. For sliced
+ // ListArrays, values() returns the full underlying array but only
+ // elements between the first and last offset are referenced.
+ let l_first = l.offsets()[0].as_usize();
+ let l_len = l.offsets()[l.len()].as_usize() - l_first;
+ let l_values = converter.convert_columns(&[l.values().slice(l_first, l_len)])?;
+
+ let r_first = r.offsets()[0].as_usize();
+ let r_len = r.offsets()[r.len()].as_usize() - r_first;
+ let r_values = converter.convert_columns(&[r.values().slice(r_first, r_len)])?;
let mut offsets = Vec::<OffsetSize>::with_capacity(l.len() + 1);
offsets.push(OffsetSize::usize_as(0));
@@ -201,14 +207,14 @@ fn general_except<OffsetSize: OffsetSizeTrait>(
continue;
}
- for element_index in r_start.as_usize()..r_end.as_usize() {
+ for element_index in r_start.as_usize() - r_first..r_end.as_usize() - r_first {
let right_row = r_values.row(element_index);
dedup.insert(right_row);
}
- for element_index in l_start.as_usize()..l_end.as_usize() {
+ for element_index in l_start.as_usize() - l_first..l_end.as_usize() - l_first {
let left_row = l_values.row(element_index);
if dedup.insert(left_row) {
- indices.push(element_index);
+ indices.push(element_index + l_first);
}
}
@@ -237,3 +243,64 @@ fn general_except<OffsetSize: OffsetSizeTrait>(
nulls,
))
}
+
+#[cfg(test)]
+mod tests {
+ use super::ArrayExcept;
+ use arrow::array::{Array, AsArray, Int32Array, ListArray};
+ use arrow::datatypes::{Field, Int32Type};
+ use datafusion_common::{Result, config::ConfigOptions};
+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+ use std::sync::Arc;
+
+ #[test]
+ fn test_array_except_sliced_lists() -> Result<()> {
+ // l: [[1,2], [3,4], [5,6], [7,8]] → slice(1,2) → [[3,4], [5,6]]
+ // r: [[3], [5], [6], [8]] → slice(1,2) → [[5], [6]]
+ // except(l, r) should be [[3,4], [5]]
+ let l_full = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(3), Some(4)]),
+ Some(vec![Some(5), Some(6)]),
+ Some(vec![Some(7), Some(8)]),
+ ]);
+ let r_full = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(3)]),
+ Some(vec![Some(5)]),
+ Some(vec![Some(6)]),
+ Some(vec![Some(8)]),
+ ]);
+ let l_sliced = l_full.slice(1, 2);
+ let r_sliced = r_full.slice(1, 2);
+
+ let list_field = Arc::new(Field::new("item", l_sliced.data_type().clone(), true));
+ let return_field =
+ Arc::new(Field::new("return", l_sliced.data_type().clone(), true));
+
+ let result = ArrayExcept::new().invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(Arc::new(l_sliced)),
+ ColumnarValue::Array(Arc::new(r_sliced)),
+ ],
+ arg_fields: vec![Arc::clone(&list_field), Arc::clone(&list_field)],
+ number_rows: 2,
+ return_field,
+ config_options: Arc::new(ConfigOptions::default()),
+ })?;
+
+ let output = result.into_array(2)?;
+ let output = output.as_list::<i32>();
+
+ // Row 0: [3,4] except [5] = [3,4]
+ let row0 = output.value(0);
+ let row0 = row0.as_any().downcast_ref::<Int32Array>().unwrap();
+ assert_eq!(row0.values().as_ref(), &[3, 4]);
+
+ // Row 1: [5,6] except [6] = [5]
+ let row1 = output.value(1);
+ let row1 = row1.as_any().downcast_ref::<Int32Array>().unwrap();
+ assert_eq!(row1.values().as_ref(), &[5]);
+
+ Ok(())
+ }
+}
diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rs
index a3d2573747..5f6663e693 100644
--- a/datafusion/functions-nested/src/set_ops.rs
+++ b/datafusion/functions-nested/src/set_ops.rs
@@ -370,14 +370,25 @@ fn generic_set_lists<OffsetSize: OffsetSizeTrait>(
"{set_op:?} is not implemented for '{l:?}' and '{r:?}'"
);
- // Convert all values to rows in batch for performance.
let converter = RowConverter::new(vec![SortField::new(l.value_type())])?;
- let rows_l = converter.convert_columns(&[Arc::clone(l.values())])?;
- let rows_r = converter.convert_columns(&[Arc::clone(r.values())])?;
- // Combine value arrays so indices from both sides share a single index space.
- let combined_values = concat(&[l.values().as_ref(), r.values().as_ref()])?;
- let r_offset = l.values().len();
+ // Only convert the visible portion of the values array. For sliced
+ // ListArrays, values() returns the full underlying array but only
+ // elements between the first and last offset are referenced.
+ let l_first = l.offsets()[0].as_usize();
+ let l_len = l.offsets()[l.len()].as_usize() - l_first;
+ let rows_l = converter.convert_columns(&[l.values().slice(l_first, l_len)])?;
+
+ let r_first = r.offsets()[0].as_usize();
+ let r_len = r.offsets()[r.len()].as_usize() - r_first;
+ let rows_r = converter.convert_columns(&[r.values().slice(r_first, r_len)])?;
+
+ // Combine the *sliced* value arrays so 0-based indices from the row
+ // converter map directly into the concatenated array.
+ let l_values = l.values().slice(l_first, l_len);
+ let r_values = r.values().slice(r_first, r_len);
+ let combined_values = concat(&[l_values.as_ref(), r_values.as_ref()])?;
+ let r_offset = l_len;
match set_op {
SetOp::Union => generic_set_loop::<OffsetSize, true>(
@@ -414,6 +425,8 @@ fn generic_set_loop<OffsetSize: OffsetSizeTrait, const IS_UNION: bool>(
) -> Result<ArrayRef> {
let l_offsets = l.value_offsets();
let r_offsets = r.value_offsets();
+ let l_first = l.offsets()[0].as_usize();
+ let r_first = r.offsets()[0].as_usize();
let mut result_offsets = Vec::with_capacity(l.len() + 1);
result_offsets.push(OffsetSize::usize_as(0));
@@ -438,10 +451,10 @@ fn generic_set_loop<OffsetSize: OffsetSizeTrait, const IS_UNION: bool>(
continue;
}
- let l_start = l_offsets[i].as_usize();
- let l_end = l_offsets[i + 1].as_usize();
- let r_start = r_offsets[i].as_usize();
- let r_end = r_offsets[i + 1].as_usize();
+ let l_start = l_offsets[i].as_usize() - l_first;
+ let l_end = l_offsets[i + 1].as_usize() - l_first;
+ let r_start = r_offsets[i].as_usize() - r_first;
+ let r_end = r_offsets[i + 1].as_usize() - r_first;
seen.clear();
@@ -564,9 +577,16 @@ fn general_array_distinct<OffsetSize: OffsetSizeTrait>(
let mut offsets = Vec::with_capacity(array.len() + 1);
offsets.push(OffsetSize::usize_as(0));
- // Convert all values to row format in a single batch for performance
let converter = RowConverter::new(vec![SortField::new(dt.clone())])?;
- let rows = converter.convert_columns(&[Arc::clone(array.values())])?;
+
+ // Only convert the visible portion of the values array. For sliced
+ // ListArrays, values() returns the full underlying array but only
+ // elements between the first and last offset are referenced.
+ let first_offset = value_offsets[0].as_usize();
+ let visible_len = value_offsets[array.len()].as_usize() - first_offset;
+ let rows =
+ converter.convert_columns(&[array.values().slice(first_offset, visible_len)])?;
+
let mut indices: Vec<usize> = Vec::with_capacity(rows.num_rows());
let mut seen = HashSet::new();
for i in 0..array.len() {
@@ -578,8 +598,8 @@ fn general_array_distinct<OffsetSize: OffsetSizeTrait>(
continue;
}
- let start = value_offsets[i].as_usize();
- let end = value_offsets[i + 1].as_usize();
+ let start = value_offsets[i].as_usize() - first_offset;
+ let end = value_offsets[i + 1].as_usize() - first_offset;
seen.clear();
seen.reserve(end - start);
@@ -587,13 +607,15 @@ fn general_array_distinct<OffsetSize: OffsetSizeTrait>(
for idx in start..end {
let row = rows.row(idx);
if seen.insert(row) {
- indices.push(idx);
+ indices.push(idx + first_offset);
}
}
offsets.push(last_offset + OffsetSize::usize_as(seen.len()));
}
// Gather distinct values in a single pass, using the computed `indices`.
+ // Indices are absolute positions in array.values() (first_offset was added
+ // back when collecting them), so we can take directly from the full values.
// Use UInt64Array for LargeList to support values arrays exceeding u32::MAX.
let final_values = if indices.is_empty() {
new_empty_array(&dt)
@@ -621,14 +643,132 @@ mod tests {
use std::sync::Arc;
use arrow::{
- array::{Int32Array, ListArray},
+ array::{Array, AsArray, Int32Array, ListArray},
buffer::OffsetBuffer,
- datatypes::{DataType, Field},
+ datatypes::{DataType, Field, Int32Type},
};
- use datafusion_common::{DataFusionError, config::ConfigOptions};
- use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+ use datafusion_common::{DataFusionError, Result, config::ConfigOptions};
+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+
+ use crate::set_ops::{ArrayDistinct, ArrayIntersect, ArrayUnion, array_distinct_udf};
+
+ /// Build two sliced ListArrays and return them along with the shared list
+ /// field.
+ ///
+ /// l: [[1,2], [3,4], [5,6], [7,8]] → slice(1,2) → [[3,4], [5,6]]
+ /// r: [[1,3], [3,5], [5,7], [7,1]] → slice(1,2) → [[3,5], [5,7]]
+ fn make_sliced_pair() -> (ListArray, ListArray, Arc<Field>) {
+ let l = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(3), Some(4)]),
+ Some(vec![Some(5), Some(6)]),
+ Some(vec![Some(7), Some(8)]),
+ ]);
+ let r = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(3)]),
+ Some(vec![Some(3), Some(5)]),
+ Some(vec![Some(5), Some(7)]),
+ Some(vec![Some(7), Some(1)]),
+ ]);
+ let field = Arc::new(Field::new("item", l.data_type().clone(), true));
+ (l.slice(1, 2), r.slice(1, 2), field)
+ }
+
+ fn collect_i32_list(list: &ListArray) -> Vec<Vec<i32>> {
+ (0..list.len())
+ .map(|i| {
+ let arr = list.value(i);
+ arr.as_any()
+ .downcast_ref::<Int32Array>()
+ .unwrap()
+ .values()
+ .to_vec()
+ })
+ .collect()
+ }
+
+ #[test]
+ fn test_array_union_sliced_lists() -> Result<()> {
+ let (l, r, field) = make_sliced_pair();
+
+ let result = ArrayUnion::new().invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(Arc::new(l)),
+ ColumnarValue::Array(Arc::new(r)),
+ ],
+ arg_fields: vec![Arc::clone(&field), Arc::clone(&field)],
+ number_rows: 2,
+ return_field: Arc::clone(&field),
+ config_options: Arc::new(ConfigOptions::default()),
+ })?;
+
+ let output = result.into_array(2)?;
+ let output = output.as_list::<i32>();
+ let rows = collect_i32_list(output);
+
+ // Row 0: union([3,4], [3,5]) = [3,4,5]
+ assert_eq!(rows[0], vec![3, 4, 5]);
+ // Row 1: union([5,6], [5,7]) = [5,6,7]
+ assert_eq!(rows[1], vec![5, 6, 7]);
+ Ok(())
+ }
+
+ #[test]
+ fn test_array_intersect_sliced_lists() -> Result<()> {
+ let (l, r, field) = make_sliced_pair();
+
+ let result = ArrayIntersect::new().invoke_with_args(ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(Arc::new(l)),
+ ColumnarValue::Array(Arc::new(r)),
+ ],
+ arg_fields: vec![Arc::clone(&field), Arc::clone(&field)],
+ number_rows: 2,
+ return_field: Arc::clone(&field),
+ config_options: Arc::new(ConfigOptions::default()),
+ })?;
+
+ let output = result.into_array(2)?;
+ let output = output.as_list::<i32>();
+ let rows = collect_i32_list(output);
+
+ // Row 0: intersect([3,4], [3,5]) = [3]
+ assert_eq!(rows[0], vec![3]);
+ // Row 1: intersect([5,6], [5,7]) = [5]
+ assert_eq!(rows[1], vec![5]);
+ Ok(())
+ }
- use crate::set_ops::array_distinct_udf;
+ #[test]
+ fn test_array_distinct_sliced_list() -> Result<()> {
+ // [[1,1], [3,3,4], [5,5,6], [7,7]] → slice(1,2) → [[3,3,4], [5,5,6]]
+ let list = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(1)]),
+ Some(vec![Some(3), Some(3), Some(4)]),
+ Some(vec![Some(5), Some(5), Some(6)]),
+ Some(vec![Some(7), Some(7)]),
+ ]);
+ let sliced = list.slice(1, 2);
+ let field = Arc::new(Field::new("item", sliced.data_type().clone(), true));
+
+ let result = ArrayDistinct::new().invoke_with_args(ScalarFunctionArgs {
+ args: vec![ColumnarValue::Array(Arc::new(sliced))],
+ arg_fields: vec![Arc::clone(&field)],
+ number_rows: 2,
+ return_field: field,
+ config_options: Arc::new(ConfigOptions::default()),
+ })?;
+
+ let output = result.into_array(2)?;
+ let output = output.as_list::<i32>();
+ let rows = collect_i32_list(output);
+
+ // Row 0: distinct([3,3,4]) = [3,4]
+ assert_eq!(rows[0], vec![3, 4]);
+ // Row 1: distinct([5,5,6]) = [5,6]
+ assert_eq!(rows[1], vec![5, 6]);
+ Ok(())
+ }
#[test]
fn test_array_distinct_inner_nullability_result_type_match_return_type()