Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize `array_filter` (#25754)

Neil Conway8edd071fa277
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml
index 56bcde9836..80b94a0e12 100644
--- a/datafusion/functions-nested/Cargo.toml
+++ b/datafusion/functions-nested/Cargo.toml
@@ -82,6 +82,10 @@ name = "array_concat"
harness = false
name = "array_length"
+[[bench]]
+harness = false
+name = "array_filter"
+
[[bench]]
harness = false
name = "array_min_max"
diff --git a/datafusion/functions-nested/benches/array_filter.rs b/datafusion/functions-nested/benches/array_filter.rs
new file mode 100644
index 0000000000..c73adc6791
--- /dev/null
+++ b/datafusion/functions-nested/benches/array_filter.rs
@@ -0,0 +1,382 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{hint::black_box, sync::Arc};
+
+use arrow::array::{
+ Array, ArrayRef, Int32Array, ListArray, RecordBatch, StringArray, StringViewArray,
+ UInt64Array,
+};
+use arrow::buffer::{NullBuffer, OffsetBuffer};
+use arrow::compute::{cast, take};
+use arrow::datatypes::{DataType, Field, Schema};
+use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};
+use datafusion_common::{DFSchema, ScalarValue};
+use datafusion_expr::{
+ Expr, col, execution_props::ExecutionProps, expr::LambdaVariable, lambda, lit,
+ physical_planning_context::PhysicalPlanningContext,
+};
+use datafusion_functions_nested::expr_fn::array_filter;
+use datafusion_physical_expr::create_physical_expr;
+use rand::{Rng, SeedableRng, rngs::StdRng};
+
+// Inputs are built by hand rather than from SQL literals so the bench can cover
+// sliced arrays and elements stored under NULL list rows.
+// Defaults: 256 rows of 16 elements, seeded random values, no NULL list rows.
+struct Case {
+ kind: DataType,
+ rows: usize,
+ width: usize,
+ element_nulls: f64,
+ row_nulls: f64,
+ // NULL list rows keep their elements, which evaluation must strip.
+ hidden_storage: bool,
+ predicate: Predicate,
+ variable: bool,
+ sliced: bool,
+ large_list: bool,
+}
+
+#[derive(Clone, Copy)]
+enum Predicate {
+ LessThan(i32),
+ Scalar(Option<bool>),
+ IsNull,
+ Captured,
+}
+
+impl Default for Case {
+ fn default() -> Self {
+ Self {
+ kind: DataType::Int32,
+ rows: 256,
+ width: 16,
+ element_nulls: 0.0,
+ row_nulls: 0.0,
+ hidden_storage: false,
+ predicate: Predicate::LessThan(50),
+ variable: false,
+ sliced: false,
+ large_list: false,
+ }
+ }
+}
+
+fn string_value(value: i32) -> String {
+ format!("{value:03}-array-filter-value")
+}
+
+// Evaluate a planned projection, including lambda evaluation, on an in-memory
+// batch. Constructing inputs and planning are outside the timed loop.
+fn bench_case(c: &mut Criterion, name: &str, case: &Case) {
+ let mut rng = StdRng::seed_from_u64(42);
+ let row_valid = (0..case.rows)
+ .map(|_| !rng.random_bool(case.row_nulls))
+ .collect::<Vec<_>>();
+ let lengths = (0..case.rows)
+ .map(|row| {
+ if !row_valid[row] && !case.hidden_storage {
+ 0
+ } else if case.variable {
+ (row * 17) % (2 * case.width + 1)
+ } else {
+ case.width
+ }
+ })
+ .collect::<Vec<_>>();
+ let values = (0..lengths.iter().sum())
+ .map(|_| {
+ let value = rng.random_range(0..100);
+ (!rng.random_bool(case.element_nulls)).then_some(value)
+ })
+ .collect::<Vec<_>>();
+ let elements: ArrayRef = match case.kind {
+ DataType::Int32 => Arc::new(Int32Array::from(values.clone())),
+ DataType::Utf8 => Arc::new(StringArray::from_iter(
+ values.iter().map(|v| v.map(string_value)),
+ )),
+ DataType::Utf8View => Arc::new(StringViewArray::from_iter(
+ values.iter().map(|v| v.map(string_value)),
+ )),
+ _ => unreachable!(),
+ };
+ let row_nulls = (case.row_nulls != 0.0).then(|| NullBuffer::from(row_valid.clone()));
+ let field = Arc::new(Field::new_list_field(case.kind.clone(), true));
+ let list = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::from_lengths(lengths),
+ Arc::clone(&elements),
+ row_nulls,
+ );
+ let list = if case.sliced {
+ list.slice(1, case.rows - 2)
+ } else {
+ list
+ };
+ let first_row = usize::from(case.sliced);
+ let thresholds = (first_row..first_row + list.len())
+ .map(|row| (row % 100) as i32)
+ .collect::<Vec<_>>();
+
+ // Independently check logical output before timing, including NULL elements
+ // deliberately retained by IS NULL and values hidden by NULL list rows.
+ let mut selected = Vec::with_capacity(values.len());
+ let mut result_lengths = Vec::with_capacity(list.len());
+ for (row, offsets) in list.offsets().windows(2).enumerate() {
+ let before = selected.len();
+ if row_valid[first_row + row] {
+ let (start, end) = (offsets[0] as usize, offsets[1] as usize);
+ for (index, value) in (start..).zip(&values[start..end]) {
+ let keep = match case.predicate {
+ Predicate::LessThan(limit) => value.is_some_and(|v| v < limit),
+ Predicate::Scalar(value) => value == Some(true),
+ Predicate::IsNull => value.is_none(),
+ Predicate::Captured => value.is_some_and(|v| v < thresholds[row]),
+ };
+ if keep {
+ selected.push(index as u64);
+ }
+ }
+ }
+ result_lengths.push(selected.len() - before);
+ }
+ let expected = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::<i32>::from_lengths(result_lengths),
+ take(elements.as_ref(), &UInt64Array::from(selected), None).unwrap(),
+ list.nulls().cloned(),
+ );
+ let mut input = Arc::new(list) as ArrayRef;
+ let mut expected = Arc::new(expected) as ArrayRef;
+ if case.large_list {
+ let data_type = DataType::LargeList(Arc::clone(&field));
+ input = cast(&input, &data_type).unwrap();
+ expected = cast(&expected, &data_type).unwrap();
+ }
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("list", input.data_type().clone(), true),
+ Field::new("threshold", DataType::Int32, false),
+ ]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![input, Arc::new(Int32Array::from(thresholds))],
+ )
+ .unwrap();
+ let v = Expr::LambdaVariable(LambdaVariable::new("v".into(), Some(field)));
+ let body = match case.predicate {
+ Predicate::LessThan(limit) => {
+ let limit = if case.kind == DataType::Int32 {
+ lit(limit)
+ } else {
+ lit(ScalarValue::new_utf8(string_value(limit))
+ .cast_to(&case.kind)
+ .unwrap())
+ };
+ v.lt(limit)
+ }
+ Predicate::Scalar(value) => lit(ScalarValue::Boolean(value)),
+ Predicate::IsNull => v.is_null(),
+ Predicate::Captured => v.lt(col("threshold")),
+ };
+ let expression = create_physical_expr(
+ &array_filter(col("list"), lambda(["v"], body)),
+ &DFSchema::try_from(schema).unwrap(),
+ &ExecutionProps::new(),
+ &PhysicalPlanningContext::default(),
+ )
+ .unwrap();
+ let output = expression
+ .evaluate(&batch)
+ .unwrap()
+ .into_array(batch.num_rows())
+ .unwrap();
+ assert_eq!(output.as_ref(), expected.as_ref(), "{name}");
+ output.to_data().validate_full().unwrap();
+
+ let mut group = c.benchmark_group("array_filter");
+ group.sampling_mode(SamplingMode::Flat);
+ group.bench_function(name, |b| {
+ b.iter(|| black_box(expression.evaluate(black_box(&batch)).unwrap()))
+ });
+ group.finish();
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ for (kind, name) in [
+ (DataType::Int32, "int32"),
+ (DataType::Utf8, "utf8"),
+ (DataType::Utf8View, "utf8_view"),
+ ] {
+ for keep in [10, 50, 90] {
+ bench_case(
+ c,
+ &format!("{name}/keep{keep}/no_nulls"),
+ &Case {
+ kind: kind.clone(),
+ predicate: Predicate::LessThan(keep),
+ ..Case::default()
+ },
+ );
+ }
+ // 25% NULL elements make comparison results nullable. Half of the
+ // remaining elements pass, so approximately 37.5% of all slots survive.
+ bench_case(
+ c,
+ &format!("{name}/keep50/null_elements25"),
+ &Case {
+ kind: kind.clone(),
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ );
+ // IS NULL is cheap to evaluate, so list reconstruction dominates.
+ bench_case(
+ c,
+ &format!("{name}/keep_null_elements"),
+ &Case {
+ kind: kind.clone(),
+ predicate: Predicate::IsNull,
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ );
+ for percent in [50, 99] {
+ bench_case(
+ c,
+ &format!("{name}/keep50/null_rows{percent}"),
+ &Case {
+ kind: kind.clone(),
+ row_nulls: f64::from(percent) / 100.0,
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ );
+ }
+ }
+ for (name, case) in [
+ (
+ "utf8/wide_128",
+ Case {
+ kind: DataType::Utf8,
+ rows: 32,
+ width: 128,
+ ..Case::default()
+ },
+ ),
+ (
+ "utf8/variable_0_to_32",
+ Case {
+ kind: DataType::Utf8,
+ variable: true,
+ ..Case::default()
+ },
+ ),
+ (
+ "int32/large_1m/null_elements25",
+ Case {
+ rows: 65_536,
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ ),
+ // As many elements as the default, spread over 8x as many rows, so
+ // per-row work dominates.
+ (
+ "int32/narrow_2/null_elements25",
+ Case {
+ rows: 2048,
+ width: 2,
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ ),
+ (
+ "scalar/true",
+ Case {
+ predicate: Predicate::Scalar(Some(true)),
+ ..Case::default()
+ },
+ ),
+ (
+ "scalar/false",
+ Case {
+ predicate: Predicate::Scalar(Some(false)),
+ ..Case::default()
+ },
+ ),
+ (
+ "scalar/null",
+ Case {
+ predicate: Predicate::Scalar(None),
+ ..Case::default()
+ },
+ ),
+ (
+ "int32/keep_all",
+ Case {
+ predicate: Predicate::LessThan(100),
+ ..Case::default()
+ },
+ ),
+ (
+ "int32/keep_none",
+ Case {
+ predicate: Predicate::LessThan(0),
+ ..Case::default()
+ },
+ ),
+ (
+ "int32/captured_column",
+ Case {
+ predicate: Predicate::Captured,
+ ..Case::default()
+ },
+ ),
+ (
+ "int32/keep50/null_rows50/hidden_storage",
+ Case {
+ row_nulls: 0.5,
+ hidden_storage: true,
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ ),
+ (
+ "utf8/sliced",
+ Case {
+ kind: DataType::Utf8,
+ sliced: true,
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ ),
+ (
+ "utf8/large_list",
+ Case {
+ kind: DataType::Utf8,
+ large_list: true,
+ element_nulls: 0.25,
+ ..Case::default()
+ },
+ ),
+ ] {
+ bench_case(c, name, &case);
+ }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/functions-nested/src/array_any_match.rs b/datafusion/functions-nested/src/array_any_match.rs
index 97029eb422..b43c152f07 100644
--- a/datafusion/functions-nested/src/array_any_match.rs
+++ b/datafusion/functions-nested/src/array_any_match.rs
@@ -162,10 +162,11 @@ impl HigherOrderUDFImpl for ArrayAnyMatch {
};
let predicate = evaluated.boolean_predicate(self.name())?;
+ let row_offsets = evaluated.row_offsets()?;
let mut values = BooleanBuilder::with_capacity(evaluated.len());
- for i in 0..evaluated.len() {
- let (start, end) = evaluated.row_range(i);
+ for window in row_offsets.windows(2) {
+ let (start, end) = (window[0], window[1]);
// any_match_for_range returns None when nulls poison the result;
// null rows produce an empty range and return Some(false), but their
// null bit is preserved by attaching the original null bitmap below.
diff --git a/datafusion/functions-nested/src/array_filter.rs b/datafusion/functions-nested/src/array_filter.rs
index 9d59faaa0c..d016648c86 100644
--- a/datafusion/functions-nested/src/array_filter.rs
+++ b/datafusion/functions-nested/src/array_filter.rs
@@ -18,12 +18,8 @@
//! [`datafusion_expr::HigherOrderUDF`] definitions for array_filter function.
use arrow::{
- array::{
- Array, ArrayRef, AsArray, BooleanArray, LargeListArray, ListArray,
- OffsetSizeTrait,
- },
- buffer::OffsetBuffer,
- compute::filter as arrow_filter,
+ array::{Array, ArrayRef, AsArray, BooleanArray, GenericListArray, OffsetSizeTrait},
+ compute::{filter as arrow_filter, prep_null_mask_filter},
datatypes::{DataType, Field, FieldRef},
};
use datafusion_common::{Result, ScalarValue, exec_err};
@@ -35,11 +31,11 @@ use datafusion_expr::{
use datafusion_macros::user_doc;
use std::sync::Arc;
-use crate::utils::empty_list_values;
+use crate::utils::{empty_list_values, prepare_list_filter};
use crate::lambda_utils::{
- SingleListLambdaResult, coerce_single_list_arg, evaluate_single_list_predicate,
- single_list_lambda_parameters, value_lambda_pair,
+ EvaluatedListLambda, SingleListLambdaResult, coerce_single_list_arg,
+ evaluate_single_list_predicate, single_list_lambda_parameters, value_lambda_pair,
};
make_higher_order_function_expr_and_func!(
@@ -145,49 +141,10 @@ impl HigherOrderUDFImpl for ArrayFilter {
}
};
- // Scalar predicate short-circuit: x -> true or x -> false/null
- if let ColumnarValue::Scalar(ScalarValue::Boolean(b)) =
- &evaluated.evaluated_result
- {
- return match b {
- Some(true) => Ok(ColumnarValue::Array(evaluated.original_list)),
- _ => Ok(ColumnarValue::Array(empty_filtered_list(
- &evaluated.original_list,
- field,
- )?)),
- };
- }
-
- let predicate = evaluated.boolean_predicate(self.name())?;
-
// ListView and LargeListView are coerced to List/LargeList by coerce_value_types.
let filtered_list = match evaluated.original_list.data_type() {
- DataType::List(_) => {
- let (filtered_values, new_offsets) = filter_list_values(
- &evaluated.flattened_values,
- &predicate,
- &evaluated.adjusted_offsets::<i32>(),
- )?;
- Arc::new(ListArray::new(
- field,
- new_offsets,
- filtered_values,
- evaluated.nulls().cloned(),
- )) as ArrayRef
- }
- DataType::LargeList(_) => {
- let (filtered_values, new_offsets) = filter_list_values(
- &evaluated.flattened_values,
- &predicate,
- &evaluated.adjusted_offsets::<i64>(),
- )?;
- Arc::new(LargeListArray::new(
- field,
- new_offsets,
- filtered_values,
- evaluated.nulls().cloned(),
- ))
- }
+ DataType::List(_) => filter_list::<i32>(self.name(), evaluated, field)?,
+ DataType::LargeList(_) => filter_list::<i64>(self.name(), evaluated, field)?,
other => exec_err!("expected list, got {other}")?,
};
@@ -203,68 +160,84 @@ impl HigherOrderUDFImpl for ArrayFilter {
}
}
-/// Returns a list array with every non-null sublist emptied, preserving the null buffer.
-/// Used for the `x -> false` / `x -> null` scalar predicate short-circuit.
-fn empty_filtered_list(list_array: &ArrayRef, field: FieldRef) -> Result<ArrayRef> {
- Ok(match list_array.data_type() {
- DataType::List(_) => empty_list_values(list_array.as_list::<i32>(), field),
- DataType::LargeList(_) => empty_list_values(list_array.as_list::<i64>(), field),
- other => return exec_err!("expected list, got {other}"),
- })
-}
+/// Apply the evaluated predicate and rebuild the result's row offsets.
+fn filter_list<O: OffsetSizeTrait>(
+ name: &str,
+ evaluated: EvaluatedListLambda,
+ field: FieldRef,
+) -> Result<ArrayRef> {
+ let list = evaluated.original_list.as_list::<O>();
+ if let ColumnarValue::Scalar(ScalarValue::Boolean(b)) = &evaluated.evaluated_result {
+ return match b {
+ Some(true) => Ok(evaluated.original_list),
+ _ => Ok(empty_list_values(list, field)),
+ };
+ }
-/// Filters flat list values using a boolean predicate, returning filtered values and
-/// recomputed per-sublist offsets. Null predicate values are treated as false.
-fn filter_list_values<O: OffsetSizeTrait>(
- values: &ArrayRef,
- predicate: &BooleanArray,
- offsets: &OffsetBuffer<O>,
-) -> Result<(ArrayRef, OffsetBuffer<O>)> {
- let num_sublists = offsets.len().saturating_sub(1);
- let has_nulls = predicate.null_count() > 0;
- let new_offsets = OffsetBuffer::<O>::from_lengths((0..num_sublists).map(|i| {
- let start = offsets[i].as_usize();
- let end = offsets[i + 1].as_usize();
- if has_nulls {
- (start..end)
- .filter(|&j| predicate.is_valid(j) && predicate.value(j))
- .count()
- } else {
- predicate
- .values()
- .slice(start, end - start)
- .count_set_bits()
- }
- }));
+ let predicate = evaluated.boolean_predicate(name)?;
+ // Normalize NULL predicate results to false once, so row-offset calculation
+ // and filtering use the same keep bitmap. Selected elements may still be NULL.
+ let predicate = match predicate.null_count() {
+ 0 => predicate,
+ _ => prep_null_mask_filter(&predicate),
+ };
+ let (new_offsets, keep_mask) = prepare_list_filter(list, predicate.values());
- if new_offsets.last() == offsets.last() {
- return Ok((Arc::clone(values), offsets.clone()));
+ let kept_count = new_offsets.last().as_usize();
+ if kept_count == evaluated.flattened_values.len() {
+ return Ok(evaluated.original_list);
+ }
+ if kept_count == 0 {
+ return Ok(empty_list_values(list, field));
}
- // arrow_filter treats null predicate values as false
- let filtered_values = arrow_filter(values.as_ref(), predicate)?;
- Ok((filtered_values, new_offsets))
+ let filtered_values = arrow_filter(
+ evaluated.flattened_values.as_ref(),
+ &BooleanArray::new(keep_mask, None),
+ )?;
+ Ok(Arc::new(GenericListArray::<O>::new(
+ field,
+ new_offsets,
+ filtered_values,
+ list.nulls().cloned(),
+ )))
}
#[cfg(test)]
mod tests {
+ use std::{collections::HashMap, sync::Arc};
+
use arrow::{
- array::{Array, AsArray},
+ array::{Array, ArrayRef, AsArray, Int32Array, ListArray, OffsetSizeTrait},
buffer::{NullBuffer, OffsetBuffer},
+ compute::cast,
+ datatypes::{DataType, Field},
};
- use arrow::array::Int32Array;
-
use crate::array_filter::array_filter_higher_order_function;
use crate::lambda_utils::test_utils::{
create_i32_large_list, create_i32_list, eval_hof_on_i32_list,
eval_hof_on_i32_list_with_outer, v,
};
- use datafusion_expr::{col, lit};
+ use datafusion_expr::{Expr, col, expr::LambdaVariable, lit};
+
+ /// The result must share the input's element and offset buffers.
+ fn assert_reuses_input(actual: &ListArray, input: &ListArray) {
+ assert_eq!(actual, input);
+ assert!(Arc::ptr_eq(actual.values(), input.values()));
+ assert!(actual.offsets().ptr_eq(input.offsets()));
+ }
+
+ /// Compare physical layouts, which logical equality ignores.
+ fn assert_same_layout<O: OffsetSizeTrait>(actual: &dyn Array, expected: &dyn Array) {
+ let (actual, expected) = (actual.as_list::<O>(), expected.as_list::<O>());
+ assert_eq!(actual.offsets(), expected.offsets());
+ assert_eq!(actual.values(), expected.values());
+ }
fn keep_greater_than_two(
list: impl Array + Clone + 'static,
- ) -> datafusion_common::Result<arrow::array::ArrayRef> {
+ ) -> datafusion_common::Result<ArrayRef> {
eval_hof_on_i32_list(
array_filter_higher_order_function(),
list,
@@ -314,11 +287,11 @@ mod tests {
}
#[test]
- fn filter_on_sliced_list_should_not_evaluate_on_unreachable_values() {
+ fn filter_sliced_list() {
// First sublist [0] is sliced away; sliced array covers sublists [1..3]
let list = create_i32_list(
vec![
- 0, // unreachable after slice — if evaluated, it would appear in output
+ 0, // Outside the visible list rows.
1, 5, 2, 4, 3, 7,
],
OffsetBuffer::<i32>::from_lengths(vec![1, 3, 3]),
@@ -340,9 +313,9 @@ mod tests {
}
#[test]
- fn filter_should_not_be_evaluated_on_values_underlying_null() {
- // The null sublist (index 1) contains values that would pass the predicate
- // if evaluated. We verify they do NOT appear in the output.
+ fn filter_does_not_copy_values_underlying_null_rows() {
+ // The NULL list row contains values that pass the predicate. They must
+ // not occupy space in the result's element array.
let list = create_i32_list(
vec![1, 5, 99, 100, 3, 7],
OffsetBuffer::<i32>::from_lengths(vec![2, 2, 2]),
@@ -363,6 +336,39 @@ mod tests {
assert_eq!(actual.data_type(), expected.data_type());
assert_eq!(actual, &expected);
+ assert_same_layout::<i32>(actual, &expected);
+ }
+
+ #[test]
+ fn filter_passing_values_under_null_rows_are_not_kept() {
+ let nulls = Some(NullBuffer::from(vec![true, false, true]));
+ for (values, threshold, expected_values, expected_lengths) in [
+ // Only the elements stored under the NULL row pass, so the result
+ // keeps no elements.
+ (vec![1, 2, 99, 100, 3], 50, vec![], [0, 0, 0]),
+ // Every element passes, but the elements stored under the NULL row
+ // must still be dropped rather than returning the input as is.
+ (vec![3, 4, 98, 99, 5], 2, vec![3, 4, 5], [2, 0, 1]),
+ ] {
+ let list = create_i32_list(
+ values,
+ OffsetBuffer::<i32>::from_lengths([2, 2, 1]),
+ nulls.clone(),
+ );
+ let res = eval_hof_on_i32_list(
+ array_filter_higher_order_function(),
+ list,
+ v().gt(lit(threshold)),
+ )
+ .unwrap();
+ let expected = create_i32_list(
+ expected_values,
+ OffsetBuffer::<i32>::from_lengths(expected_lengths),
+ nulls.clone(),
+ );
+ assert_eq!(res.as_list::<i32>(), &expected);
+ assert_same_layout::<i32>(&res, &expected);
+ }
}
#[test]
@@ -391,7 +397,7 @@ mod tests {
);
// all elements > 2, so nothing is filtered — values buffer should be reused
let res = keep_greater_than_two(list.clone()).unwrap();
- assert_eq!(res.as_list::<i32>(), &list);
+ assert_reuses_input(res.as_list::<i32>(), &list);
}
#[test]
@@ -408,7 +414,103 @@ mod tests {
lit(true),
)
.unwrap();
- assert_eq!(res.as_list::<i32>(), &list);
+ assert_reuses_input(res.as_list::<i32>(), &list);
+ }
+
+ #[test]
+ fn filter_sliced_null_elements_and_rows() {
+ // Slice both the element array and the list rows. The visible input is
+ // [[NULL, 1, 4, NULL], NULL, [], [NULL, 3, 7]]. The NULL row retains 99,100.
+ let values = Int32Array::from(vec![
+ Some(-1),
+ Some(42),
+ None,
+ Some(1),
+ Some(4),
+ None,
+ Some(99),
+ Some(100),
+ None,
+ Some(3),
+ Some(7),
+ Some(88),
+ ])
+ .slice(1, 11);
+ let field = Arc::new(
+ Field::new("element", DataType::Int32, true)
+ .with_metadata(HashMap::from([("key".into(), "value".into())])),
+ );
+ let value = || {
+ Expr::LambdaVariable(LambdaVariable::new(
+ "v".into(),
+ Some(Arc::clone(&field)),
+ ))
+ };
+ let list = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::<i32>::from_lengths([1, 4, 2, 0, 3, 1]),
+ Arc::new(values),
+ Some(NullBuffer::from(vec![true, true, false, true, true, true])),
+ )
+ .slice(1, 4);
+
+ for large in [false, true] {
+ let data_type = if large {
+ DataType::LargeList(Arc::clone(&field))
+ } else {
+ DataType::List(Arc::clone(&field))
+ };
+ let input = cast(&list, &data_type).unwrap();
+ for (predicate, values, lengths) in [
+ (
+ value().gt(lit(2i32)),
+ vec![Some(4), Some(3), Some(7)],
+ [1, 0, 0, 2],
+ ),
+ (value().is_null(), vec![None, None, None], [2, 0, 0, 1]),
+ (
+ value().is_null().or(value().gt(lit(2i32))),
+ vec![None, Some(4), None, None, Some(3), Some(7)],
+ [3, 0, 0, 3],
+ ),
+ ] {
+ let result = eval_hof_on_i32_list(
+ array_filter_higher_order_function(),
+ Arc::clone(&input),
+ predicate,
+ )
+ .unwrap();
+ let expected = ListArray::new(
+ Arc::clone(&field),
+ OffsetBuffer::<i32>::from_lengths(lengths),
+ Arc::new(Int32Array::from(values)),
+ list.nulls().cloned(),
+ );
+ let expected = cast(&expected, &data_type).unwrap();
+ assert_eq!(result.as_ref(), expected.as_ref());
+ if large {
+ assert_same_layout::<i64>(&result, &expected);
+ } else {
+ assert_same_layout::<i32>(&result, &expected);
+ }
+ result.to_data().validate_full().unwrap();
+ }
+ }
+ }
+
+ #[test]
+ fn test_sliced_capacity() -> datafusion_common::Result<()> {
+ crate::utils::tests::check_sliced_list_behavior(|input| {
+ eval_hof_on_i32_list(
+ array_filter_higher_order_function(),
+ Arc::clone(input),
+ Expr::LambdaVariable(LambdaVariable::new(
+ "v".into(),
+ Some(Arc::new(Field::new("v", DataType::Float64, true))),
+ ))
+ .gt(lit(3.5)),
+ )
+ })
}
#[test]
diff --git a/datafusion/functions-nested/src/array_first.rs b/datafusion/functions-nested/src/array_first.rs
index 615dc47394..9a5e1db05f 100644
--- a/datafusion/functions-nested/src/array_first.rs
+++ b/datafusion/functions-nested/src/array_first.rs
@@ -32,8 +32,8 @@ use datafusion_macros::user_doc;
use std::sync::Arc;
use crate::lambda_utils::{
- EvaluatedListLambda, SingleListLambdaResult, coerce_single_list_arg,
- evaluate_single_list_predicate, single_list_lambda_parameters, value_lambda_pair,
+ SingleListLambdaResult, coerce_single_list_arg, evaluate_single_list_predicate,
+ single_list_lambda_parameters, value_lambda_pair,
};
make_higher_order_function_expr_and_func!(
@@ -150,7 +150,7 @@ impl HigherOrderUDFImpl for ArrayFirst {
let predicate = evaluated.boolean_predicate(self.name())?;
let indices = match evaluated.original_list.data_type() {
DataType::List(_) | DataType::LargeList(_) => {
- first_match_indices(&evaluated, &predicate)
+ first_match_indices(&evaluated.row_offsets()?, &predicate)
}
other => return exec_err!("expected list, got {other}"),
};
@@ -170,14 +170,11 @@ impl HigherOrderUDFImpl for ArrayFirst {
///
/// A null predicate value is treated as not matching. The matched element itself
/// may be null and is still returned.
-fn first_match_indices(
- evaluated: &EvaluatedListLambda,
- predicate: &BooleanArray,
-) -> UInt64Array {
- let mut builder = UInt64Builder::with_capacity(evaluated.len());
-
- for i in 0..evaluated.len() {
- let (start, end) = evaluated.row_range(i);
+fn first_match_indices(row_offsets: &[usize], predicate: &BooleanArray) -> UInt64Array {
+ let mut builder = UInt64Builder::with_capacity(row_offsets.len() - 1);
+
+ for window in row_offsets.windows(2) {
+ let (start, end) = (window[0], window[1]);
match (start..end).find(|&j| predicate.is_valid(j) && predicate.value(j)) {
Some(j) => builder.append_value(j as u64),
diff --git a/datafusion/functions-nested/src/lambda_utils.rs b/datafusion/functions-nested/src/lambda_utils.rs
index 89f2044213..59f88110fd 100644
--- a/datafusion/functions-nested/src/lambda_utils.rs
+++ b/datafusion/functions-nested/src/lambda_utils.rs
@@ -20,8 +20,8 @@
use arrow::array::{ArrayRef, AsArray, BooleanArray, OffsetSizeTrait, new_null_array};
use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::compute::take_arrays;
-use arrow::datatypes::{ArrowNativeType, DataType, FieldRef};
-use datafusion_common::utils::{adjust_offsets_for_slice, list_values_row_number};
+use arrow::datatypes::{DataType, FieldRef};
+use datafusion_common::utils::list_values_row_number;
use datafusion_common::{
Result, ScalarValue, plan_err,
utils::{list_values, take_function_args},
@@ -140,7 +140,6 @@ pub(crate) struct EvaluatedListLambda {
pub original_list: ArrayRef,
pub flattened_values: ArrayRef,
pub evaluated_result: ColumnarValue,
- row_offsets: Vec<usize>,
}
impl EvaluatedListLambda {
@@ -152,12 +151,10 @@ impl EvaluatedListLambda {
self.original_list.nulls()
}
- pub(crate) fn row_range(&self, i: usize) -> (usize, usize) {
- (self.row_offsets[i], self.row_offsets[i + 1])
- }
-
- pub(crate) fn adjusted_offsets<O: OffsetSizeTrait>(&self) -> OffsetBuffer<O> {
- OffsetBuffer::from_lengths(self.row_offsets.windows(2).map(|w| w[1] - w[0]))
+ /// Row boundaries within `flattened_values`, including the elements that
+ /// NULL rows store. Functions that walk elements row by row compute this once.
+ pub(crate) fn row_offsets(&self) -> Result<Vec<usize>> {
+ adjusted_row_offsets(&self.original_list)
}
pub(crate) fn boolean_predicate(&self, name: &str) -> Result<BooleanArray> {
@@ -176,18 +173,18 @@ impl EvaluatedListLambda {
fn adjusted_row_offsets(list: &ArrayRef) -> Result<Vec<usize>> {
Ok(match list.data_type() {
- DataType::List(_) => adjust_offsets_for_slice(list.as_list::<i32>())
- .iter()
- .map(|o| o.as_usize())
- .collect(),
- DataType::LargeList(_) => adjust_offsets_for_slice(list.as_list::<i64>())
- .iter()
- .map(|o| o.as_usize())
- .collect(),
+ DataType::List(_) => offsets_from_zero(list.as_list::<i32>().offsets()),
+ DataType::LargeList(_) => offsets_from_zero(list.as_list::<i64>().offsets()),
other => return exec_err!("expected list, got {other}"),
})
}
+/// Shift `offsets` so the first one is zero.
+fn offsets_from_zero<O: OffsetSizeTrait>(offsets: &OffsetBuffer<O>) -> Vec<usize> {
+ let first = offsets[0].as_usize();
+ offsets.iter().map(|o| o.as_usize() - first).collect()
+}
+
fn evaluate_single_list_lambda(
name: &str,
args: &HigherOrderFunctionArgs,
@@ -209,13 +206,10 @@ fn evaluate_single_list_lambda(
Ok(take_arrays(arrays, &indices, None)?)
})?;
- let row_offsets = adjusted_row_offsets(&original_list)?;
-
Ok(SingleListLambdaResult::Ready(EvaluatedListLambda {
original_list,
flattened_values,
evaluated_result,
- row_offsets,
}))
}
diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs
index cb4bea42aa..73bf3cbf4c 100644
--- a/datafusion/functions-nested/src/utils.rs
+++ b/datafusion/functions-nested/src/utils.rs
@@ -22,15 +22,15 @@ use std::sync::Arc;
use arrow::datatypes::{DataType, Field, FieldRef, Fields};
use arrow::array::{
- Array, ArrayRef, BooleanArray, Float64Array, GenericListArray, NullBufferBuilder,
- OffsetSizeTrait, Scalar, new_empty_array,
+ Array, ArrayRef, BooleanArray, BooleanBufferBuilder, Float64Array, GenericListArray,
+ NullBufferBuilder, OffsetSizeTrait, Scalar, new_empty_array,
};
-use arrow::buffer::{NullBuffer, OffsetBuffer};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
use datafusion_common::cast::{
as_fixed_size_list_array, as_float64_array, as_generic_list_array,
as_large_list_array, as_large_list_view_array, as_list_array, as_list_view_array,
};
-use datafusion_common::utils::offset_span_len;
+use datafusion_common::utils::{offset_span, offset_span_len};
use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_err};
use datafusion_expr::ColumnarValue;
@@ -50,6 +50,115 @@ pub(crate) fn empty_list_values<O: OffsetSizeTrait>(
))
}
+/// Build result row offsets and exclude elements stored under NULL list rows.
+///
+/// `keep` is a bitmap identifying which elements in the list's visible span
+/// should be preserved. The returned bitmap uses the same indexing, with bits
+/// for NULL list rows cleared. Selected elements may themselves be NULL; their
+/// validity is the caller's responsibility when copying values.
+pub(crate) fn prepare_list_filter<O: OffsetSizeTrait>(
+ list: &GenericListArray<O>,
+ keep: &BooleanBuffer,
+) -> (OffsetBuffer<O>, BooleanBuffer) {
+ let list_offsets = list.offsets();
+ let (first_offset, span_len) = offset_span(list_offsets);
+ debug_assert_eq!(keep.len(), span_len);
+ let mut offsets = Vec::with_capacity(list.len() + 1);
+ offsets.push(O::zero());
+ // Count the set bits before each row end in one pass over `keep`, which
+ // stays cheap for short rows.
+ let keep_mask = if list.null_count() == 0 {
+ // Without NULL rows, every set bit is kept.
+ let mut set_bits = SetBitCounter::new(keep.bit_chunks().iter_padded());
+ offsets.extend(list_offsets[1..].iter().map(|end| {
+ O::usize_as(set_bits.count_before(end.as_usize() - first_offset))
+ }));
+ keep.clone()
+ } else {
+ let mut set_bits = SetBitCounter::new(keep.bit_chunks().iter_padded());
+ let mut kept_count = 0;
+ // Set bits stored under NULL rows, which the result does not keep.
+ let mut hidden_count = 0;
+ // NULL list rows are usually empty, so reuse `keep` until a NULL row
+ // stores elements. Copy the bits of the preceding rows when that first
+ // happens.
+ let mut keep_mask: Option<BooleanBufferBuilder> = None;
+ for (row, window) in list_offsets.windows(2).enumerate() {
+ let start = window[0].as_usize() - first_offset;
+ let end = window[1].as_usize() - first_offset;
+ let len = end - start;
+ let bit_start = keep.offset() + start;
+ if list.is_valid(row) {
+ kept_count = set_bits.count_before(end) - hidden_count;
+ if let Some(keep_mask) = &mut keep_mask {
+ keep_mask
+ .append_packed_range(bit_start..bit_start + len, keep.values());
+ }
+ } else if len != 0 {
+ let set_before_row = set_bits.count_before(start);
+ hidden_count += set_bits.count_before(end) - set_before_row;
+ keep_mask
+ .get_or_insert_with(|| {
+ let mut builder = BooleanBufferBuilder::new(keep.len());
+ builder
+ .append_packed_range(keep.offset()..bit_start, keep.values());
+ builder
+ })
+ .append_n(len, false);
+ }
+ offsets.push(O::usize_as(kept_count));
+ }
+ keep_mask
+ .map(BooleanBufferBuilder::build)
+ .unwrap_or_else(|| keep.clone())
+ };
+ debug_assert_eq!(keep_mask.len(), keep.len());
+ // SAFETY: offsets start at zero and never decrease, since each one counts
+ // the kept elements up to a row's end. A count cannot exceed the input's
+ // element span, which fits in `O`, so converting it to `O` cannot overflow.
+ let offsets = unsafe { OffsetBuffer::new_unchecked(offsets.into()) };
+ (offsets, keep_mask)
+}
+
+/// Counts the set bits that precede a sequence of non-decreasing positions in a
+/// bitmap, reading each 64-bit word of the bitmap once.
+struct SetBitCounter<I> {
+ /// The bitmap's remaining words, with the final word padded with zeros.
+ words: I,
+ /// The word containing the most recent position.
+ word: u64,
+ /// Position of the first bit of `word`.
+ word_start: usize,
+ /// Set bits before `word_start`.
+ count_before_word: usize,
+}
+
+impl<I: Iterator<Item = u64>> SetBitCounter<I> {
+ fn new(mut words: I) -> Self {
+ let word = words.next().unwrap_or(0);
+ Self {
+ words,
+ word,
+ word_start: 0,
+ count_before_word: 0,
+ }
+ }
+
+ /// Returns the number of set bits before `end`. `end` must be no less than
+ /// in the previous call, and no greater than the bitmap's length.
+ #[inline]
+ fn count_before(&mut self, end: usize) -> usize {
+ debug_assert!(end >= self.word_start);
+ while end - self.word_start >= 64 {
+ self.count_before_word += self.word.count_ones() as usize;
+ self.word = self.words.next().unwrap_or(0);
+ self.word_start += 64;
+ }
+ let mask = (1u64 << (end - self.word_start)) - 1;
+ self.count_before_word + (self.word & mask).count_ones() as usize
+ }
+}
+
/// Computes the return type of a function that produces a list with the same
/// inner field as `array_type`, plus an element that may be null when
/// `element_nullable` is set.
@@ -528,6 +637,141 @@ pub(crate) mod tests {
use arrow::array::ListArray;
use arrow::datatypes::Int64Type;
use datafusion_common::utils::SingleRowListArrayBuilder;
+ use rand::{Rng, SeedableRng, rngs::StdRng};
+
+ /// Compare `prepare_list_filter` with a per-row reference, returning the
+ /// output bitmap so callers can check whether `keep` was reused.
+ fn assert_list_filter_matches_reference<O: OffsetSizeTrait>(
+ list: &GenericListArray<O>,
+ keep: &BooleanBuffer,
+ ) -> BooleanBuffer {
+ let first = list.offsets()[0].as_usize();
+ let mut expected = keep.iter().collect::<Vec<_>>();
+ let mut expected_offsets = vec![0];
+ for (row, window) in list.offsets().windows(2).enumerate() {
+ let start = window[0].as_usize() - first;
+ let end = window[1].as_usize() - first;
+ if list.is_null(row) {
+ expected[start..end].fill(false);
+ }
+ expected_offsets.push(expected[..end].iter().filter(|&&v| v).count());
+ }
+ let (offsets, mask) = prepare_list_filter(list, keep);
+ assert_eq!(mask.iter().collect::<Vec<_>>(), expected);
+ assert_eq!(
+ offsets.iter().map(|o| o.as_usize()).collect::<Vec<_>>(),
+ expected_offsets
+ );
+ mask
+ }
+
+ /// A list of NULL elements with the given row lengths and row validity,
+ /// sliced to `len` rows starting at row `offset`.
+ fn null_element_list<O: OffsetSizeTrait>(
+ lengths: &[usize],
+ nulls: Option<NullBuffer>,
+ offset: usize,
+ len: usize,
+ ) -> GenericListArray<O> {
+ GenericListArray::<O>::new(
+ Arc::new(Field::new_list_field(DataType::Int32, true)),
+ OffsetBuffer::from_lengths(lengths.iter().copied()),
+ // `prepare_list_filter` must permit selecting NULL elements.
+ arrow::array::new_null_array(&DataType::Int32, lengths.iter().sum()),
+ nulls,
+ )
+ .slice(offset, len)
+ }
+
+ /// Check a list sliced to its last seven rows, returning the input and output
+ /// bitmaps so callers can check whether the input was reused.
+ fn check_list_filter<O: OffsetSizeTrait>(
+ lengths: [usize; 8],
+ valid: [bool; 8],
+ ) -> (BooleanBuffer, BooleanBuffer) {
+ let nulls = Some(NullBuffer::from(valid.to_vec()));
+ let list = null_element_list::<O>(&lengths, nulls, 1, 7);
+ let len = offset_span(list.offsets()).1;
+ // Use an offset that is not a multiple of the bitmap's three-bit period.
+ let keep = BooleanBuffer::collect_bool(len + 1, |i| i % 3 != 0).slice(1, len);
+ let mask = assert_list_filter_matches_reference(&list, &keep);
+ (keep, mask)
+ }
+
+ #[test]
+ fn list_filter_mask_handles_slices_and_hidden_elements() {
+ let valid = [true, false, true, false, false, true, true, false];
+ // After slicing off the first row, the first lengths of each pair give an
+ // empty NULL row, a NULL row storing elements after a valid row,
+ // consecutive NULL rows, and a trailing one. The second lengths store no
+ // elements under NULL rows, which leaves the bitmap untouched.
+ for (hidden, empty) in [
+ ([2, 0, 2, 1, 2, 0, 3, 2], [2, 0, 2, 0, 0, 3, 3, 0]),
+ // Rows long enough that counting and copying bits spans 64-bit words.
+ ([2, 0, 63, 65, 64, 0, 65, 2], [2, 0, 63, 0, 0, 65, 65, 0]),
+ ] {
+ let (keep, mask) = check_list_filter::<i32>(hidden, valid);
+ assert!(!mask.inner().ptr_eq(keep.inner()));
+ let (keep, mask) = check_list_filter::<i32>(empty, valid);
+ assert!(mask.inner().ptr_eq(keep.inner()));
+ check_list_filter::<i64>(hidden, valid);
+ check_list_filter::<i64>(empty, valid);
+ }
+ }
+
+ #[test]
+ fn list_filter_matches_reference_on_random_lists() {
+ let mut rng = StdRng::seed_from_u64(42);
+ for _ in 0..500 {
+ let rows = rng.random_range(0..=24);
+ // Mostly short rows, plus some long enough to span 64-bit words.
+ let lengths = (0..rows)
+ .map(|_| {
+ let max_len = if rng.random_bool(0.2) { 130 } else { 4 };
+ rng.random_range(0..=max_len)
+ })
+ .collect::<Vec<_>>();
+ // Either no NULL buffer, or rows that are NULL with this probability.
+ let null_rows = [0.0, 0.3, 0.9][rng.random_range(0..3)];
+ let valid = (0..rows)
+ .map(|_| !rng.random_bool(null_rows))
+ .collect::<Vec<_>>();
+ let nulls = (null_rows > 0.0).then(|| NullBuffer::from(valid.clone()));
+ let offset = rng.random_range(0..=rows);
+ let len = rng.random_range(0..=rows - offset);
+ let span = lengths[offset..offset + len].iter().sum::<usize>();
+ let density = [0.0, 0.1, 0.5, 0.9, 1.0][rng.random_range(0..5)];
+ let keep_offset = rng.random_range(0..130);
+ let keep = BooleanBuffer::collect_bool(keep_offset + span, |_| {
+ rng.random_bool(density)
+ })
+ .slice(keep_offset, span);
+
+ let list = null_element_list::<i32>(&lengths, nulls.clone(), offset, len);
+ let mask = assert_list_filter_matches_reference(&list, &keep);
+ // `keep` is copied only when a visible NULL row stores elements.
+ let hides_elements =
+ (offset..offset + len).any(|row| !valid[row] && lengths[row] != 0);
+ assert_eq!(mask.inner().ptr_eq(keep.inner()), !hides_elements);
+ let list = null_element_list::<i64>(&lengths, nulls, offset, len);
+ assert_list_filter_matches_reference(&list, &keep);
+ }
+ }
+
+ #[test]
+ fn set_bit_counter_matches_per_position_counts() {
+ // A sliced bitmap whose length is a whole number of 64-bit words.
+ let bits =
+ BooleanBuffer::collect_bool(200, |i| i % 3 != 0 && i % 7 != 0).slice(5, 192);
+ let mut counter = SetBitCounter::new(bits.bit_chunks().iter_padded());
+ let mut expected = 0;
+ for end in 0..=bits.len() {
+ assert_eq!(counter.count_before(end), expected, "end {end}");
+ if end < bits.len() {
+ expected += usize::from(bits.value(end));
+ }
+ }
+ }
/// Only test internal functions, array-related sql functions will be tested in sqllogictest `array.slt`
#[test]
@@ -659,8 +903,9 @@ pub(crate) mod tests {
let result = run(&sliced)?;
let expected = run(&compact.slice(offset, len))?;
assert_eq!(result.as_ref(), expected.as_ref());
+ // Compare buffers, since evaluating through a plan re-wraps arrays.
assert!(
- Arc::ptr_eq(&result, &sliced)
+ result.to_data().ptr_eq(&sliced.to_data())
|| result.get_buffer_memory_size() < 1024,
"{data_type}: {} bytes for {len} rows",
result.get_buffer_memory_size()
diff --git a/datafusion/sqllogictest/test_files/array/array_filter.slt b/datafusion/sqllogictest/test_files/array/array_filter.slt
index b6d73fbe7d..6070c2993f 100644
--- a/datafusion/sqllogictest/test_files/array/array_filter.slt
+++ b/datafusion/sqllogictest/test_files/array/array_filter.slt
@@ -152,6 +152,18 @@ SELECT array_filter([1, 2, 3], v -> CAST(NULL AS BOOLEAN));
----
[]
+# A true predicate may retain a NULL element.
+query ?
+SELECT array_filter([NULL, 1, 3], v -> v IS NULL OR v > 2);
+----
+[NULL, 3]
+
+# Retaining a non-null nested array preserves its NULL elements.
+query ?
+SELECT array_filter([[NULL], [1], NULL], v -> v IS NOT NULL);
+----
+[[NULL], [1]]
+
query error DataFusion error: Error during planning: The function 'array_filter' expected 2 argument\(s\) but received 0
SELECT array_filter();