Apache DataFusion SQL Query Engine
perf: Optimize `array_filter` (#25754)
Neil Conway8edd071fa277
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.tomlindex 56bcde9836..80b94a0e12 100644--- a/datafusion/functions-nested/Cargo.toml+++ b/datafusion/functions-nested/Cargo.toml@@ -82,6 +82,10 @@ name = "array_concat"harness = falsename = "array_length"+[[bench]]+harness = false+name = "array_filter"+[[bench]]harness = falsename = "array_min_max"diff --git a/datafusion/functions-nested/benches/array_filter.rs b/datafusion/functions-nested/benches/array_filter.rsnew file mode 100644index 0000000000..c73adc6791--- /dev/null+++ b/datafusion/functions-nested/benches/array_filter.rs@@ -0,0 +1,382 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::{hint::black_box, sync::Arc};++use arrow::array::{+ Array, ArrayRef, Int32Array, ListArray, RecordBatch, StringArray, StringViewArray,+ UInt64Array,+};+use arrow::buffer::{NullBuffer, OffsetBuffer};+use arrow::compute::{cast, take};+use arrow::datatypes::{DataType, Field, Schema};+use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};+use datafusion_common::{DFSchema, ScalarValue};+use datafusion_expr::{+ Expr, col, execution_props::ExecutionProps, expr::LambdaVariable, lambda, lit,+ physical_planning_context::PhysicalPlanningContext,+};+use datafusion_functions_nested::expr_fn::array_filter;+use datafusion_physical_expr::create_physical_expr;+use rand::{Rng, SeedableRng, rngs::StdRng};++// Inputs are built by hand rather than from SQL literals so the bench can cover+// sliced arrays and elements stored under NULL list rows.+// Defaults: 256 rows of 16 elements, seeded random values, no NULL list rows.+struct Case {+ kind: DataType,+ rows: usize,+ width: usize,+ element_nulls: f64,+ row_nulls: f64,+ // NULL list rows keep their elements, which evaluation must strip.+ hidden_storage: bool,+ predicate: Predicate,+ variable: bool,+ sliced: bool,+ large_list: bool,+}++#[derive(Clone, Copy)]+enum Predicate {+ LessThan(i32),+ Scalar(Option<bool>),+ IsNull,+ Captured,+}++impl Default for Case {+ fn default() -> Self {+ Self {+ kind: DataType::Int32,+ rows: 256,+ width: 16,+ element_nulls: 0.0,+ row_nulls: 0.0,+ hidden_storage: false,+ predicate: Predicate::LessThan(50),+ variable: false,+ sliced: false,+ large_list: false,+ }+ }+}++fn string_value(value: i32) -> String {+ format!("{value:03}-array-filter-value")+}++// Evaluate a planned projection, including lambda evaluation, on an in-memory+// batch. Constructing inputs and planning are outside the timed loop.+fn bench_case(c: &mut Criterion, name: &str, case: &Case) {+ let mut rng = StdRng::seed_from_u64(42);+ let row_valid = (0..case.rows)+ .map(|_| !rng.random_bool(case.row_nulls))+ .collect::<Vec<_>>();+ let lengths = (0..case.rows)+ .map(|row| {+ if !row_valid[row] && !case.hidden_storage {+ 0+ } else if case.variable {+ (row * 17) % (2 * case.width + 1)+ } else {+ case.width+ }+ })+ .collect::<Vec<_>>();+ let values = (0..lengths.iter().sum())+ .map(|_| {+ let value = rng.random_range(0..100);+ (!rng.random_bool(case.element_nulls)).then_some(value)+ })+ .collect::<Vec<_>>();+ let elements: ArrayRef = match case.kind {+ DataType::Int32 => Arc::new(Int32Array::from(values.clone())),+ DataType::Utf8 => Arc::new(StringArray::from_iter(+ values.iter().map(|v| v.map(string_value)),+ )),+ DataType::Utf8View => Arc::new(StringViewArray::from_iter(+ values.iter().map(|v| v.map(string_value)),+ )),+ _ => unreachable!(),+ };+ let row_nulls = (case.row_nulls != 0.0).then(|| NullBuffer::from(row_valid.clone()));+ let field = Arc::new(Field::new_list_field(case.kind.clone(), true));+ let list = ListArray::new(+ Arc::clone(&field),+ OffsetBuffer::from_lengths(lengths),+ Arc::clone(&elements),+ row_nulls,+ );+ let list = if case.sliced {+ list.slice(1, case.rows - 2)+ } else {+ list+ };+ let first_row = usize::from(case.sliced);+ let thresholds = (first_row..first_row + list.len())+ .map(|row| (row % 100) as i32)+ .collect::<Vec<_>>();++ // Independently check logical output before timing, including NULL elements+ // deliberately retained by IS NULL and values hidden by NULL list rows.+ let mut selected = Vec::with_capacity(values.len());+ let mut result_lengths = Vec::with_capacity(list.len());+ for (row, offsets) in list.offsets().windows(2).enumerate() {+ let before = selected.len();+ if row_valid[first_row + row] {+ let (start, end) = (offsets[0] as usize, offsets[1] as usize);+ for (index, value) in (start..).zip(&values[start..end]) {+ let keep = match case.predicate {+ Predicate::LessThan(limit) => value.is_some_and(|v| v < limit),+ Predicate::Scalar(value) => value == Some(true),+ Predicate::IsNull => value.is_none(),+ Predicate::Captured => value.is_some_and(|v| v < thresholds[row]),+ };+ if keep {+ selected.push(index as u64);+ }+ }+ }+ result_lengths.push(selected.len() - before);+ }+ let expected = ListArray::new(+ Arc::clone(&field),+ OffsetBuffer::<i32>::from_lengths(result_lengths),+ take(elements.as_ref(), &UInt64Array::from(selected), None).unwrap(),+ list.nulls().cloned(),+ );+ let mut input = Arc::new(list) as ArrayRef;+ let mut expected = Arc::new(expected) as ArrayRef;+ if case.large_list {+ let data_type = DataType::LargeList(Arc::clone(&field));+ input = cast(&input, &data_type).unwrap();+ expected = cast(&expected, &data_type).unwrap();+ }+ let schema = Arc::new(Schema::new(vec![+ Field::new("list", input.data_type().clone(), true),+ Field::new("threshold", DataType::Int32, false),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![input, Arc::new(Int32Array::from(thresholds))],+ )+ .unwrap();+ let v = Expr::LambdaVariable(LambdaVariable::new("v".into(), Some(field)));+ let body = match case.predicate {+ Predicate::LessThan(limit) => {+ let limit = if case.kind == DataType::Int32 {+ lit(limit)+ } else {+ lit(ScalarValue::new_utf8(string_value(limit))+ .cast_to(&case.kind)+ .unwrap())+ };+ v.lt(limit)+ }+ Predicate::Scalar(value) => lit(ScalarValue::Boolean(value)),+ Predicate::IsNull => v.is_null(),+ Predicate::Captured => v.lt(col("threshold")),+ };+ let expression = create_physical_expr(+ &array_filter(col("list"), lambda(["v"], body)),+ &DFSchema::try_from(schema).unwrap(),+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )+ .unwrap();+ let output = expression+ .evaluate(&batch)+ .unwrap()+ .into_array(batch.num_rows())+ .unwrap();+ assert_eq!(output.as_ref(), expected.as_ref(), "{name}");+ output.to_data().validate_full().unwrap();++ let mut group = c.benchmark_group("array_filter");+ group.sampling_mode(SamplingMode::Flat);+ group.bench_function(name, |b| {+ b.iter(|| black_box(expression.evaluate(black_box(&batch)).unwrap()))+ });+ group.finish();+}++fn criterion_benchmark(c: &mut Criterion) {+ for (kind, name) in [+ (DataType::Int32, "int32"),+ (DataType::Utf8, "utf8"),+ (DataType::Utf8View, "utf8_view"),+ ] {+ for keep in [10, 50, 90] {+ bench_case(+ c,+ &format!("{name}/keep{keep}/no_nulls"),+ &Case {+ kind: kind.clone(),+ predicate: Predicate::LessThan(keep),+ ..Case::default()+ },+ );+ }+ // 25% NULL elements make comparison results nullable. Half of the+ // remaining elements pass, so approximately 37.5% of all slots survive.+ bench_case(+ c,+ &format!("{name}/keep50/null_elements25"),+ &Case {+ kind: kind.clone(),+ element_nulls: 0.25,+ ..Case::default()+ },+ );+ // IS NULL is cheap to evaluate, so list reconstruction dominates.+ bench_case(+ c,+ &format!("{name}/keep_null_elements"),+ &Case {+ kind: kind.clone(),+ predicate: Predicate::IsNull,+ element_nulls: 0.25,+ ..Case::default()+ },+ );+ for percent in [50, 99] {+ bench_case(+ c,+ &format!("{name}/keep50/null_rows{percent}"),+ &Case {+ kind: kind.clone(),+ row_nulls: f64::from(percent) / 100.0,+ element_nulls: 0.25,+ ..Case::default()+ },+ );+ }+ }+ for (name, case) in [+ (+ "utf8/wide_128",+ Case {+ kind: DataType::Utf8,+ rows: 32,+ width: 128,+ ..Case::default()+ },+ ),+ (+ "utf8/variable_0_to_32",+ Case {+ kind: DataType::Utf8,+ variable: true,+ ..Case::default()+ },+ ),+ (+ "int32/large_1m/null_elements25",+ Case {+ rows: 65_536,+ element_nulls: 0.25,+ ..Case::default()+ },+ ),+ // As many elements as the default, spread over 8x as many rows, so+ // per-row work dominates.+ (+ "int32/narrow_2/null_elements25",+ Case {+ rows: 2048,+ width: 2,+ element_nulls: 0.25,+ ..Case::default()+ },+ ),+ (+ "scalar/true",+ Case {+ predicate: Predicate::Scalar(Some(true)),+ ..Case::default()+ },+ ),+ (+ "scalar/false",+ Case {+ predicate: Predicate::Scalar(Some(false)),+ ..Case::default()+ },+ ),+ (+ "scalar/null",+ Case {+ predicate: Predicate::Scalar(None),+ ..Case::default()+ },+ ),+ (+ "int32/keep_all",+ Case {+ predicate: Predicate::LessThan(100),+ ..Case::default()+ },+ ),+ (+ "int32/keep_none",+ Case {+ predicate: Predicate::LessThan(0),+ ..Case::default()+ },+ ),+ (+ "int32/captured_column",+ Case {+ predicate: Predicate::Captured,+ ..Case::default()+ },+ ),+ (+ "int32/keep50/null_rows50/hidden_storage",+ Case {+ row_nulls: 0.5,+ hidden_storage: true,+ element_nulls: 0.25,+ ..Case::default()+ },+ ),+ (+ "utf8/sliced",+ Case {+ kind: DataType::Utf8,+ sliced: true,+ element_nulls: 0.25,+ ..Case::default()+ },+ ),+ (+ "utf8/large_list",+ Case {+ kind: DataType::Utf8,+ large_list: true,+ element_nulls: 0.25,+ ..Case::default()+ },+ ),+ ] {+ bench_case(c, name, &case);+ }+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/datafusion/functions-nested/src/array_any_match.rs b/datafusion/functions-nested/src/array_any_match.rsindex 97029eb422..b43c152f07 100644--- a/datafusion/functions-nested/src/array_any_match.rs+++ b/datafusion/functions-nested/src/array_any_match.rs@@ -162,10 +162,11 @@ impl HigherOrderUDFImpl for ArrayAnyMatch {};let predicate = evaluated.boolean_predicate(self.name())?;+ let row_offsets = evaluated.row_offsets()?;let mut values = BooleanBuilder::with_capacity(evaluated.len());- for i in 0..evaluated.len() {- let (start, end) = evaluated.row_range(i);+ for window in row_offsets.windows(2) {+ let (start, end) = (window[0], window[1]);// any_match_for_range returns None when nulls poison the result;// null rows produce an empty range and return Some(false), but their// null bit is preserved by attaching the original null bitmap below.diff --git a/datafusion/functions-nested/src/array_filter.rs b/datafusion/functions-nested/src/array_filter.rsindex 9d59faaa0c..d016648c86 100644--- a/datafusion/functions-nested/src/array_filter.rs+++ b/datafusion/functions-nested/src/array_filter.rs@@ -18,12 +18,8 @@//! [`datafusion_expr::HigherOrderUDF`] definitions for array_filter function.use arrow::{- array::{- Array, ArrayRef, AsArray, BooleanArray, LargeListArray, ListArray,- OffsetSizeTrait,- },- buffer::OffsetBuffer,- compute::filter as arrow_filter,+ array::{Array, ArrayRef, AsArray, BooleanArray, GenericListArray, OffsetSizeTrait},+ compute::{filter as arrow_filter, prep_null_mask_filter},datatypes::{DataType, Field, FieldRef},};use datafusion_common::{Result, ScalarValue, exec_err};@@ -35,11 +31,11 @@ use datafusion_expr::{use datafusion_macros::user_doc;use std::sync::Arc;-use crate::utils::empty_list_values;+use crate::utils::{empty_list_values, prepare_list_filter};use crate::lambda_utils::{- SingleListLambdaResult, coerce_single_list_arg, evaluate_single_list_predicate,- single_list_lambda_parameters, value_lambda_pair,+ EvaluatedListLambda, SingleListLambdaResult, coerce_single_list_arg,+ evaluate_single_list_predicate, single_list_lambda_parameters, value_lambda_pair,};make_higher_order_function_expr_and_func!(@@ -145,49 +141,10 @@ impl HigherOrderUDFImpl for ArrayFilter {}};- // Scalar predicate short-circuit: x -> true or x -> false/null- if let ColumnarValue::Scalar(ScalarValue::Boolean(b)) =- &evaluated.evaluated_result- {- return match b {- Some(true) => Ok(ColumnarValue::Array(evaluated.original_list)),- _ => Ok(ColumnarValue::Array(empty_filtered_list(- &evaluated.original_list,- field,- )?)),- };- }-- let predicate = evaluated.boolean_predicate(self.name())?;-// ListView and LargeListView are coerced to List/LargeList by coerce_value_types.let filtered_list = match evaluated.original_list.data_type() {- DataType::List(_) => {- let (filtered_values, new_offsets) = filter_list_values(- &evaluated.flattened_values,- &predicate,- &evaluated.adjusted_offsets::<i32>(),- )?;- Arc::new(ListArray::new(- field,- new_offsets,- filtered_values,- evaluated.nulls().cloned(),- )) as ArrayRef- }- DataType::LargeList(_) => {- let (filtered_values, new_offsets) = filter_list_values(- &evaluated.flattened_values,- &predicate,- &evaluated.adjusted_offsets::<i64>(),- )?;- Arc::new(LargeListArray::new(- field,- new_offsets,- filtered_values,- evaluated.nulls().cloned(),- ))- }+ DataType::List(_) => filter_list::<i32>(self.name(), evaluated, field)?,+ DataType::LargeList(_) => filter_list::<i64>(self.name(), evaluated, field)?,other => exec_err!("expected list, got {other}")?,};@@ -203,68 +160,84 @@ impl HigherOrderUDFImpl for ArrayFilter {}}-/// Returns a list array with every non-null sublist emptied, preserving the null buffer.-/// Used for the `x -> false` / `x -> null` scalar predicate short-circuit.-fn empty_filtered_list(list_array: &ArrayRef, field: FieldRef) -> Result<ArrayRef> {- Ok(match list_array.data_type() {- DataType::List(_) => empty_list_values(list_array.as_list::<i32>(), field),- DataType::LargeList(_) => empty_list_values(list_array.as_list::<i64>(), field),- other => return exec_err!("expected list, got {other}"),- })-}+/// Apply the evaluated predicate and rebuild the result's row offsets.+fn filter_list<O: OffsetSizeTrait>(+ name: &str,+ evaluated: EvaluatedListLambda,+ field: FieldRef,+) -> Result<ArrayRef> {+ let list = evaluated.original_list.as_list::<O>();+ if let ColumnarValue::Scalar(ScalarValue::Boolean(b)) = &evaluated.evaluated_result {+ return match b {+ Some(true) => Ok(evaluated.original_list),+ _ => Ok(empty_list_values(list, field)),+ };+ }-/// Filters flat list values using a boolean predicate, returning filtered values and-/// recomputed per-sublist offsets. Null predicate values are treated as false.-fn filter_list_values<O: OffsetSizeTrait>(- values: &ArrayRef,- predicate: &BooleanArray,- offsets: &OffsetBuffer<O>,-) -> Result<(ArrayRef, OffsetBuffer<O>)> {- let num_sublists = offsets.len().saturating_sub(1);- let has_nulls = predicate.null_count() > 0;- let new_offsets = OffsetBuffer::<O>::from_lengths((0..num_sublists).map(|i| {- let start = offsets[i].as_usize();- let end = offsets[i + 1].as_usize();- if has_nulls {- (start..end)- .filter(|&j| predicate.is_valid(j) && predicate.value(j))- .count()- } else {- predicate- .values()- .slice(start, end - start)- .count_set_bits()- }- }));+ let predicate = evaluated.boolean_predicate(name)?;+ // Normalize NULL predicate results to false once, so row-offset calculation+ // and filtering use the same keep bitmap. Selected elements may still be NULL.+ let predicate = match predicate.null_count() {+ 0 => predicate,+ _ => prep_null_mask_filter(&predicate),+ };+ let (new_offsets, keep_mask) = prepare_list_filter(list, predicate.values());- if new_offsets.last() == offsets.last() {- return Ok((Arc::clone(values), offsets.clone()));+ let kept_count = new_offsets.last().as_usize();+ if kept_count == evaluated.flattened_values.len() {+ return Ok(evaluated.original_list);+ }+ if kept_count == 0 {+ return Ok(empty_list_values(list, field));}- // arrow_filter treats null predicate values as false- let filtered_values = arrow_filter(values.as_ref(), predicate)?;- Ok((filtered_values, new_offsets))+ let filtered_values = arrow_filter(+ evaluated.flattened_values.as_ref(),+ &BooleanArray::new(keep_mask, None),+ )?;+ Ok(Arc::new(GenericListArray::<O>::new(+ field,+ new_offsets,+ filtered_values,+ list.nulls().cloned(),+ )))}#[cfg(test)]mod tests {+ use std::{collections::HashMap, sync::Arc};+use arrow::{- array::{Array, AsArray},+ array::{Array, ArrayRef, AsArray, Int32Array, ListArray, OffsetSizeTrait},buffer::{NullBuffer, OffsetBuffer},+ compute::cast,+ datatypes::{DataType, Field},};- use arrow::array::Int32Array;-use crate::array_filter::array_filter_higher_order_function;use crate::lambda_utils::test_utils::{create_i32_large_list, create_i32_list, eval_hof_on_i32_list,eval_hof_on_i32_list_with_outer, v,};- use datafusion_expr::{col, lit};+ use datafusion_expr::{Expr, col, expr::LambdaVariable, lit};++ /// The result must share the input's element and offset buffers.+ fn assert_reuses_input(actual: &ListArray, input: &ListArray) {+ assert_eq!(actual, input);+ assert!(Arc::ptr_eq(actual.values(), input.values()));+ assert!(actual.offsets().ptr_eq(input.offsets()));+ }++ /// Compare physical layouts, which logical equality ignores.+ fn assert_same_layout<O: OffsetSizeTrait>(actual: &dyn Array, expected: &dyn Array) {+ let (actual, expected) = (actual.as_list::<O>(), expected.as_list::<O>());+ assert_eq!(actual.offsets(), expected.offsets());+ assert_eq!(actual.values(), expected.values());+ }fn keep_greater_than_two(list: impl Array + Clone + 'static,- ) -> datafusion_common::Result<arrow::array::ArrayRef> {+ ) -> datafusion_common::Result<ArrayRef> {eval_hof_on_i32_list(array_filter_higher_order_function(),list,@@ -314,11 +287,11 @@ mod tests {}#[test]- fn filter_on_sliced_list_should_not_evaluate_on_unreachable_values() {+ fn filter_sliced_list() {// First sublist [0] is sliced away; sliced array covers sublists [1..3]let list = create_i32_list(vec![- 0, // unreachable after slice — if evaluated, it would appear in output+ 0, // Outside the visible list rows.1, 5, 2, 4, 3, 7,],OffsetBuffer::<i32>::from_lengths(vec![1, 3, 3]),@@ -340,9 +313,9 @@ mod tests {}#[test]- fn filter_should_not_be_evaluated_on_values_underlying_null() {- // The null sublist (index 1) contains values that would pass the predicate- // if evaluated. We verify they do NOT appear in the output.+ fn filter_does_not_copy_values_underlying_null_rows() {+ // The NULL list row contains values that pass the predicate. They must+ // not occupy space in the result's element array.let list = create_i32_list(vec![1, 5, 99, 100, 3, 7],OffsetBuffer::<i32>::from_lengths(vec![2, 2, 2]),@@ -363,6 +336,39 @@ mod tests {assert_eq!(actual.data_type(), expected.data_type());assert_eq!(actual, &expected);+ assert_same_layout::<i32>(actual, &expected);+ }++ #[test]+ fn filter_passing_values_under_null_rows_are_not_kept() {+ let nulls = Some(NullBuffer::from(vec![true, false, true]));+ for (values, threshold, expected_values, expected_lengths) in [+ // Only the elements stored under the NULL row pass, so the result+ // keeps no elements.+ (vec![1, 2, 99, 100, 3], 50, vec![], [0, 0, 0]),+ // Every element passes, but the elements stored under the NULL row+ // must still be dropped rather than returning the input as is.+ (vec![3, 4, 98, 99, 5], 2, vec![3, 4, 5], [2, 0, 1]),+ ] {+ let list = create_i32_list(+ values,+ OffsetBuffer::<i32>::from_lengths([2, 2, 1]),+ nulls.clone(),+ );+ let res = eval_hof_on_i32_list(+ array_filter_higher_order_function(),+ list,+ v().gt(lit(threshold)),+ )+ .unwrap();+ let expected = create_i32_list(+ expected_values,+ OffsetBuffer::<i32>::from_lengths(expected_lengths),+ nulls.clone(),+ );+ assert_eq!(res.as_list::<i32>(), &expected);+ assert_same_layout::<i32>(&res, &expected);+ }}#[test]@@ -391,7 +397,7 @@ mod tests {);// all elements > 2, so nothing is filtered — values buffer should be reusedlet res = keep_greater_than_two(list.clone()).unwrap();- assert_eq!(res.as_list::<i32>(), &list);+ assert_reuses_input(res.as_list::<i32>(), &list);}#[test]@@ -408,7 +414,103 @@ mod tests {lit(true),).unwrap();- assert_eq!(res.as_list::<i32>(), &list);+ assert_reuses_input(res.as_list::<i32>(), &list);+ }++ #[test]+ fn filter_sliced_null_elements_and_rows() {+ // Slice both the element array and the list rows. The visible input is+ // [[NULL, 1, 4, NULL], NULL, [], [NULL, 3, 7]]. The NULL row retains 99,100.+ let values = Int32Array::from(vec![+ Some(-1),+ Some(42),+ None,+ Some(1),+ Some(4),+ None,+ Some(99),+ Some(100),+ None,+ Some(3),+ Some(7),+ Some(88),+ ])+ .slice(1, 11);+ let field = Arc::new(+ Field::new("element", DataType::Int32, true)+ .with_metadata(HashMap::from([("key".into(), "value".into())])),+ );+ let value = || {+ Expr::LambdaVariable(LambdaVariable::new(+ "v".into(),+ Some(Arc::clone(&field)),+ ))+ };+ let list = ListArray::new(+ Arc::clone(&field),+ OffsetBuffer::<i32>::from_lengths([1, 4, 2, 0, 3, 1]),+ Arc::new(values),+ Some(NullBuffer::from(vec![true, true, false, true, true, true])),+ )+ .slice(1, 4);++ for large in [false, true] {+ let data_type = if large {+ DataType::LargeList(Arc::clone(&field))+ } else {+ DataType::List(Arc::clone(&field))+ };+ let input = cast(&list, &data_type).unwrap();+ for (predicate, values, lengths) in [+ (+ value().gt(lit(2i32)),+ vec![Some(4), Some(3), Some(7)],+ [1, 0, 0, 2],+ ),+ (value().is_null(), vec![None, None, None], [2, 0, 0, 1]),+ (+ value().is_null().or(value().gt(lit(2i32))),+ vec![None, Some(4), None, None, Some(3), Some(7)],+ [3, 0, 0, 3],+ ),+ ] {+ let result = eval_hof_on_i32_list(+ array_filter_higher_order_function(),+ Arc::clone(&input),+ predicate,+ )+ .unwrap();+ let expected = ListArray::new(+ Arc::clone(&field),+ OffsetBuffer::<i32>::from_lengths(lengths),+ Arc::new(Int32Array::from(values)),+ list.nulls().cloned(),+ );+ let expected = cast(&expected, &data_type).unwrap();+ assert_eq!(result.as_ref(), expected.as_ref());+ if large {+ assert_same_layout::<i64>(&result, &expected);+ } else {+ assert_same_layout::<i32>(&result, &expected);+ }+ result.to_data().validate_full().unwrap();+ }+ }+ }++ #[test]+ fn test_sliced_capacity() -> datafusion_common::Result<()> {+ crate::utils::tests::check_sliced_list_behavior(|input| {+ eval_hof_on_i32_list(+ array_filter_higher_order_function(),+ Arc::clone(input),+ Expr::LambdaVariable(LambdaVariable::new(+ "v".into(),+ Some(Arc::new(Field::new("v", DataType::Float64, true))),+ ))+ .gt(lit(3.5)),+ )+ })}#[test]diff --git a/datafusion/functions-nested/src/array_first.rs b/datafusion/functions-nested/src/array_first.rsindex 615dc47394..9a5e1db05f 100644--- a/datafusion/functions-nested/src/array_first.rs+++ b/datafusion/functions-nested/src/array_first.rs@@ -32,8 +32,8 @@ use datafusion_macros::user_doc;use std::sync::Arc;use crate::lambda_utils::{- EvaluatedListLambda, SingleListLambdaResult, coerce_single_list_arg,- evaluate_single_list_predicate, single_list_lambda_parameters, value_lambda_pair,+ SingleListLambdaResult, coerce_single_list_arg, evaluate_single_list_predicate,+ single_list_lambda_parameters, value_lambda_pair,};make_higher_order_function_expr_and_func!(@@ -150,7 +150,7 @@ impl HigherOrderUDFImpl for ArrayFirst {let predicate = evaluated.boolean_predicate(self.name())?;let indices = match evaluated.original_list.data_type() {DataType::List(_) | DataType::LargeList(_) => {- first_match_indices(&evaluated, &predicate)+ first_match_indices(&evaluated.row_offsets()?, &predicate)}other => return exec_err!("expected list, got {other}"),};@@ -170,14 +170,11 @@ impl HigherOrderUDFImpl for ArrayFirst {////// A null predicate value is treated as not matching. The matched element itself/// may be null and is still returned.-fn first_match_indices(- evaluated: &EvaluatedListLambda,- predicate: &BooleanArray,-) -> UInt64Array {- let mut builder = UInt64Builder::with_capacity(evaluated.len());-- for i in 0..evaluated.len() {- let (start, end) = evaluated.row_range(i);+fn first_match_indices(row_offsets: &[usize], predicate: &BooleanArray) -> UInt64Array {+ let mut builder = UInt64Builder::with_capacity(row_offsets.len() - 1);++ for window in row_offsets.windows(2) {+ let (start, end) = (window[0], window[1]);match (start..end).find(|&j| predicate.is_valid(j) && predicate.value(j)) {Some(j) => builder.append_value(j as u64),diff --git a/datafusion/functions-nested/src/lambda_utils.rs b/datafusion/functions-nested/src/lambda_utils.rsindex 89f2044213..59f88110fd 100644--- a/datafusion/functions-nested/src/lambda_utils.rs+++ b/datafusion/functions-nested/src/lambda_utils.rs@@ -20,8 +20,8 @@use arrow::array::{ArrayRef, AsArray, BooleanArray, OffsetSizeTrait, new_null_array};use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::compute::take_arrays;-use arrow::datatypes::{ArrowNativeType, DataType, FieldRef};-use datafusion_common::utils::{adjust_offsets_for_slice, list_values_row_number};+use arrow::datatypes::{DataType, FieldRef};+use datafusion_common::utils::list_values_row_number;use datafusion_common::{Result, ScalarValue, plan_err,utils::{list_values, take_function_args},@@ -140,7 +140,6 @@ pub(crate) struct EvaluatedListLambda {pub original_list: ArrayRef,pub flattened_values: ArrayRef,pub evaluated_result: ColumnarValue,- row_offsets: Vec<usize>,}impl EvaluatedListLambda {@@ -152,12 +151,10 @@ impl EvaluatedListLambda {self.original_list.nulls()}- pub(crate) fn row_range(&self, i: usize) -> (usize, usize) {- (self.row_offsets[i], self.row_offsets[i + 1])- }-- pub(crate) fn adjusted_offsets<O: OffsetSizeTrait>(&self) -> OffsetBuffer<O> {- OffsetBuffer::from_lengths(self.row_offsets.windows(2).map(|w| w[1] - w[0]))+ /// Row boundaries within `flattened_values`, including the elements that+ /// NULL rows store. Functions that walk elements row by row compute this once.+ pub(crate) fn row_offsets(&self) -> Result<Vec<usize>> {+ adjusted_row_offsets(&self.original_list)}pub(crate) fn boolean_predicate(&self, name: &str) -> Result<BooleanArray> {@@ -176,18 +173,18 @@ impl EvaluatedListLambda {fn adjusted_row_offsets(list: &ArrayRef) -> Result<Vec<usize>> {Ok(match list.data_type() {- DataType::List(_) => adjust_offsets_for_slice(list.as_list::<i32>())- .iter()- .map(|o| o.as_usize())- .collect(),- DataType::LargeList(_) => adjust_offsets_for_slice(list.as_list::<i64>())- .iter()- .map(|o| o.as_usize())- .collect(),+ DataType::List(_) => offsets_from_zero(list.as_list::<i32>().offsets()),+ DataType::LargeList(_) => offsets_from_zero(list.as_list::<i64>().offsets()),other => return exec_err!("expected list, got {other}"),})}+/// Shift `offsets` so the first one is zero.+fn offsets_from_zero<O: OffsetSizeTrait>(offsets: &OffsetBuffer<O>) -> Vec<usize> {+ let first = offsets[0].as_usize();+ offsets.iter().map(|o| o.as_usize() - first).collect()+}+fn evaluate_single_list_lambda(name: &str,args: &HigherOrderFunctionArgs,@@ -209,13 +206,10 @@ fn evaluate_single_list_lambda(Ok(take_arrays(arrays, &indices, None)?)})?;- let row_offsets = adjusted_row_offsets(&original_list)?;-Ok(SingleListLambdaResult::Ready(EvaluatedListLambda {original_list,flattened_values,evaluated_result,- row_offsets,}))}diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rsindex cb4bea42aa..73bf3cbf4c 100644--- a/datafusion/functions-nested/src/utils.rs+++ b/datafusion/functions-nested/src/utils.rs@@ -22,15 +22,15 @@ use std::sync::Arc;use arrow::datatypes::{DataType, Field, FieldRef, Fields};use arrow::array::{- Array, ArrayRef, BooleanArray, Float64Array, GenericListArray, NullBufferBuilder,- OffsetSizeTrait, Scalar, new_empty_array,+ Array, ArrayRef, BooleanArray, BooleanBufferBuilder, Float64Array, GenericListArray,+ NullBufferBuilder, OffsetSizeTrait, Scalar, new_empty_array,};-use arrow::buffer::{NullBuffer, OffsetBuffer};+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};use datafusion_common::cast::{as_fixed_size_list_array, as_float64_array, as_generic_list_array,as_large_list_array, as_large_list_view_array, as_list_array, as_list_view_array,};-use datafusion_common::utils::offset_span_len;+use datafusion_common::utils::{offset_span, offset_span_len};use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_err};use datafusion_expr::ColumnarValue;@@ -50,6 +50,115 @@ pub(crate) fn empty_list_values<O: OffsetSizeTrait>())}+/// Build result row offsets and exclude elements stored under NULL list rows.+///+/// `keep` is a bitmap identifying which elements in the list's visible span+/// should be preserved. The returned bitmap uses the same indexing, with bits+/// for NULL list rows cleared. Selected elements may themselves be NULL; their+/// validity is the caller's responsibility when copying values.+pub(crate) fn prepare_list_filter<O: OffsetSizeTrait>(+ list: &GenericListArray<O>,+ keep: &BooleanBuffer,+) -> (OffsetBuffer<O>, BooleanBuffer) {+ let list_offsets = list.offsets();+ let (first_offset, span_len) = offset_span(list_offsets);+ debug_assert_eq!(keep.len(), span_len);+ let mut offsets = Vec::with_capacity(list.len() + 1);+ offsets.push(O::zero());+ // Count the set bits before each row end in one pass over `keep`, which+ // stays cheap for short rows.+ let keep_mask = if list.null_count() == 0 {+ // Without NULL rows, every set bit is kept.+ let mut set_bits = SetBitCounter::new(keep.bit_chunks().iter_padded());+ offsets.extend(list_offsets[1..].iter().map(|end| {+ O::usize_as(set_bits.count_before(end.as_usize() - first_offset))+ }));+ keep.clone()+ } else {+ let mut set_bits = SetBitCounter::new(keep.bit_chunks().iter_padded());+ let mut kept_count = 0;+ // Set bits stored under NULL rows, which the result does not keep.+ let mut hidden_count = 0;+ // NULL list rows are usually empty, so reuse `keep` until a NULL row+ // stores elements. Copy the bits of the preceding rows when that first+ // happens.+ let mut keep_mask: Option<BooleanBufferBuilder> = None;+ for (row, window) in list_offsets.windows(2).enumerate() {+ let start = window[0].as_usize() - first_offset;+ let end = window[1].as_usize() - first_offset;+ let len = end - start;+ let bit_start = keep.offset() + start;+ if list.is_valid(row) {+ kept_count = set_bits.count_before(end) - hidden_count;+ if let Some(keep_mask) = &mut keep_mask {+ keep_mask+ .append_packed_range(bit_start..bit_start + len, keep.values());+ }+ } else if len != 0 {+ let set_before_row = set_bits.count_before(start);+ hidden_count += set_bits.count_before(end) - set_before_row;+ keep_mask+ .get_or_insert_with(|| {+ let mut builder = BooleanBufferBuilder::new(keep.len());+ builder+ .append_packed_range(keep.offset()..bit_start, keep.values());+ builder+ })+ .append_n(len, false);+ }+ offsets.push(O::usize_as(kept_count));+ }+ keep_mask+ .map(BooleanBufferBuilder::build)+ .unwrap_or_else(|| keep.clone())+ };+ debug_assert_eq!(keep_mask.len(), keep.len());+ // SAFETY: offsets start at zero and never decrease, since each one counts+ // the kept elements up to a row's end. A count cannot exceed the input's+ // element span, which fits in `O`, so converting it to `O` cannot overflow.+ let offsets = unsafe { OffsetBuffer::new_unchecked(offsets.into()) };+ (offsets, keep_mask)+}++/// Counts the set bits that precede a sequence of non-decreasing positions in a+/// bitmap, reading each 64-bit word of the bitmap once.+struct SetBitCounter<I> {+ /// The bitmap's remaining words, with the final word padded with zeros.+ words: I,+ /// The word containing the most recent position.+ word: u64,+ /// Position of the first bit of `word`.+ word_start: usize,+ /// Set bits before `word_start`.+ count_before_word: usize,+}++impl<I: Iterator<Item = u64>> SetBitCounter<I> {+ fn new(mut words: I) -> Self {+ let word = words.next().unwrap_or(0);+ Self {+ words,+ word,+ word_start: 0,+ count_before_word: 0,+ }+ }++ /// Returns the number of set bits before `end`. `end` must be no less than+ /// in the previous call, and no greater than the bitmap's length.+ #[inline]+ fn count_before(&mut self, end: usize) -> usize {+ debug_assert!(end >= self.word_start);+ while end - self.word_start >= 64 {+ self.count_before_word += self.word.count_ones() as usize;+ self.word = self.words.next().unwrap_or(0);+ self.word_start += 64;+ }+ let mask = (1u64 << (end - self.word_start)) - 1;+ self.count_before_word + (self.word & mask).count_ones() as usize+ }+}+/// Computes the return type of a function that produces a list with the same/// inner field as `array_type`, plus an element that may be null when/// `element_nullable` is set.@@ -528,6 +637,141 @@ pub(crate) mod tests {use arrow::array::ListArray;use arrow::datatypes::Int64Type;use datafusion_common::utils::SingleRowListArrayBuilder;+ use rand::{Rng, SeedableRng, rngs::StdRng};++ /// Compare `prepare_list_filter` with a per-row reference, returning the+ /// output bitmap so callers can check whether `keep` was reused.+ fn assert_list_filter_matches_reference<O: OffsetSizeTrait>(+ list: &GenericListArray<O>,+ keep: &BooleanBuffer,+ ) -> BooleanBuffer {+ let first = list.offsets()[0].as_usize();+ let mut expected = keep.iter().collect::<Vec<_>>();+ let mut expected_offsets = vec![0];+ for (row, window) in list.offsets().windows(2).enumerate() {+ let start = window[0].as_usize() - first;+ let end = window[1].as_usize() - first;+ if list.is_null(row) {+ expected[start..end].fill(false);+ }+ expected_offsets.push(expected[..end].iter().filter(|&&v| v).count());+ }+ let (offsets, mask) = prepare_list_filter(list, keep);+ assert_eq!(mask.iter().collect::<Vec<_>>(), expected);+ assert_eq!(+ offsets.iter().map(|o| o.as_usize()).collect::<Vec<_>>(),+ expected_offsets+ );+ mask+ }++ /// A list of NULL elements with the given row lengths and row validity,+ /// sliced to `len` rows starting at row `offset`.+ fn null_element_list<O: OffsetSizeTrait>(+ lengths: &[usize],+ nulls: Option<NullBuffer>,+ offset: usize,+ len: usize,+ ) -> GenericListArray<O> {+ GenericListArray::<O>::new(+ Arc::new(Field::new_list_field(DataType::Int32, true)),+ OffsetBuffer::from_lengths(lengths.iter().copied()),+ // `prepare_list_filter` must permit selecting NULL elements.+ arrow::array::new_null_array(&DataType::Int32, lengths.iter().sum()),+ nulls,+ )+ .slice(offset, len)+ }++ /// Check a list sliced to its last seven rows, returning the input and output+ /// bitmaps so callers can check whether the input was reused.+ fn check_list_filter<O: OffsetSizeTrait>(+ lengths: [usize; 8],+ valid: [bool; 8],+ ) -> (BooleanBuffer, BooleanBuffer) {+ let nulls = Some(NullBuffer::from(valid.to_vec()));+ let list = null_element_list::<O>(&lengths, nulls, 1, 7);+ let len = offset_span(list.offsets()).1;+ // Use an offset that is not a multiple of the bitmap's three-bit period.+ let keep = BooleanBuffer::collect_bool(len + 1, |i| i % 3 != 0).slice(1, len);+ let mask = assert_list_filter_matches_reference(&list, &keep);+ (keep, mask)+ }++ #[test]+ fn list_filter_mask_handles_slices_and_hidden_elements() {+ let valid = [true, false, true, false, false, true, true, false];+ // After slicing off the first row, the first lengths of each pair give an+ // empty NULL row, a NULL row storing elements after a valid row,+ // consecutive NULL rows, and a trailing one. The second lengths store no+ // elements under NULL rows, which leaves the bitmap untouched.+ for (hidden, empty) in [+ ([2, 0, 2, 1, 2, 0, 3, 2], [2, 0, 2, 0, 0, 3, 3, 0]),+ // Rows long enough that counting and copying bits spans 64-bit words.+ ([2, 0, 63, 65, 64, 0, 65, 2], [2, 0, 63, 0, 0, 65, 65, 0]),+ ] {+ let (keep, mask) = check_list_filter::<i32>(hidden, valid);+ assert!(!mask.inner().ptr_eq(keep.inner()));+ let (keep, mask) = check_list_filter::<i32>(empty, valid);+ assert!(mask.inner().ptr_eq(keep.inner()));+ check_list_filter::<i64>(hidden, valid);+ check_list_filter::<i64>(empty, valid);+ }+ }++ #[test]+ fn list_filter_matches_reference_on_random_lists() {+ let mut rng = StdRng::seed_from_u64(42);+ for _ in 0..500 {+ let rows = rng.random_range(0..=24);+ // Mostly short rows, plus some long enough to span 64-bit words.+ let lengths = (0..rows)+ .map(|_| {+ let max_len = if rng.random_bool(0.2) { 130 } else { 4 };+ rng.random_range(0..=max_len)+ })+ .collect::<Vec<_>>();+ // Either no NULL buffer, or rows that are NULL with this probability.+ let null_rows = [0.0, 0.3, 0.9][rng.random_range(0..3)];+ let valid = (0..rows)+ .map(|_| !rng.random_bool(null_rows))+ .collect::<Vec<_>>();+ let nulls = (null_rows > 0.0).then(|| NullBuffer::from(valid.clone()));+ let offset = rng.random_range(0..=rows);+ let len = rng.random_range(0..=rows - offset);+ let span = lengths[offset..offset + len].iter().sum::<usize>();+ let density = [0.0, 0.1, 0.5, 0.9, 1.0][rng.random_range(0..5)];+ let keep_offset = rng.random_range(0..130);+ let keep = BooleanBuffer::collect_bool(keep_offset + span, |_| {+ rng.random_bool(density)+ })+ .slice(keep_offset, span);++ let list = null_element_list::<i32>(&lengths, nulls.clone(), offset, len);+ let mask = assert_list_filter_matches_reference(&list, &keep);+ // `keep` is copied only when a visible NULL row stores elements.+ let hides_elements =+ (offset..offset + len).any(|row| !valid[row] && lengths[row] != 0);+ assert_eq!(mask.inner().ptr_eq(keep.inner()), !hides_elements);+ let list = null_element_list::<i64>(&lengths, nulls, offset, len);+ assert_list_filter_matches_reference(&list, &keep);+ }+ }++ #[test]+ fn set_bit_counter_matches_per_position_counts() {+ // A sliced bitmap whose length is a whole number of 64-bit words.+ let bits =+ BooleanBuffer::collect_bool(200, |i| i % 3 != 0 && i % 7 != 0).slice(5, 192);+ let mut counter = SetBitCounter::new(bits.bit_chunks().iter_padded());+ let mut expected = 0;+ for end in 0..=bits.len() {+ assert_eq!(counter.count_before(end), expected, "end {end}");+ if end < bits.len() {+ expected += usize::from(bits.value(end));+ }+ }+ }/// Only test internal functions, array-related sql functions will be tested in sqllogictest `array.slt`#[test]@@ -659,8 +903,9 @@ pub(crate) mod tests {let result = run(&sliced)?;let expected = run(&compact.slice(offset, len))?;assert_eq!(result.as_ref(), expected.as_ref());+ // Compare buffers, since evaluating through a plan re-wraps arrays.assert!(- Arc::ptr_eq(&result, &sliced)+ result.to_data().ptr_eq(&sliced.to_data())|| result.get_buffer_memory_size() < 1024,"{data_type}: {} bytes for {len} rows",result.get_buffer_memory_size()diff --git a/datafusion/sqllogictest/test_files/array/array_filter.slt b/datafusion/sqllogictest/test_files/array/array_filter.sltindex b6d73fbe7d..6070c2993f 100644--- a/datafusion/sqllogictest/test_files/array/array_filter.slt+++ b/datafusion/sqllogictest/test_files/array/array_filter.slt@@ -152,6 +152,18 @@ SELECT array_filter([1, 2, 3], v -> CAST(NULL AS BOOLEAN));----[]+# A true predicate may retain a NULL element.+query ?+SELECT array_filter([NULL, 1, 3], v -> v IS NULL OR v > 2);+----+[NULL, 3]++# Retaining a non-null nested array preserves its NULL elements.+query ?+SELECT array_filter([[NULL], [1], NULL], v -> v IS NOT NULL);+----+[[NULL], [1]]+query error DataFusion error: Error during planning: The function 'array_filter' expected 2 argument\(s\) but received 0SELECT array_filter();