Apache DataFusion SQL Query Engine
datafusion-physical-plan: handles nulls correctly in aggregate dynamic filters (#25579)
Jayant Shrivastava0af681889123
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs b/datafusion/physical-plan/src/aggregates/aggregate_stream.rsindex 0da8f0944e..5811b98d74 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_stream.rs@@ -85,7 +85,6 @@ struct AggregateStreamInner {}impl AggregateStreamInner {- // TODO: check if we get Null handling correct/// # Examples/// - Example 1/// Accumulators: min(c1)@@ -113,11 +112,12 @@ impl AggregateStreamInner {Vec::with_capacity(filter_state.accumulator_dyn_filter_info.len());for acc_info in &filter_state.accumulator_dyn_filter_info {- // Skip if we don't yet have a meaningful bound+ // Every aggregate needs a bound before we can prune rows. Otherwise,+ // another aggregate's predicate could discard its first non-NULL value.let bound = {let guard = acc_info.shared_bound.lock();if (*guard).is_null() {- continue;+ return Ok(lit(true));}guard.clone()};@@ -263,9 +263,9 @@ fn scalar_cmp_null_short_circuit(v1: &ScalarValue,v2: &ScalarValue,) -> Option<ScalarValue> {- match (v1, v2) {- (ScalarValue::Null, ScalarValue::Null) => Some(ScalarValue::Null),- (ScalarValue::Null, other) | (other, ScalarValue::Null) => Some(other.clone()),+ match (v1.is_null(), v2.is_null()) {+ (true, _) => Some(v2.clone()),+ (_, true) => Some(v1.clone()),_ => None,}}@@ -578,12 +578,17 @@ fn finalize_aggregation_with_metrics(mod tests {use super::*;use crate::aggregates::PhysicalGroupBy;+ use crate::common::collect as collect_stream;use crate::metrics::{MetricValue, MetricsSet};use crate::test::TestMemoryExec;use crate::{ExecutionPlan, collect};- use arrow::array::{Float64Array, UInt32Array};+ use arrow::array::{Float64Array, Int32Array, UInt32Array};use arrow::datatypes::{DataType, Field, Schema};- use datafusion_functions_aggregate::{array_agg::array_agg_udaf, sum::sum_udaf};+ use datafusion_functions_aggregate::{+ array_agg::array_agg_udaf,+ min_max::{max_udaf, min_udaf},+ sum::sum_udaf,+ };use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr,};@@ -677,6 +682,67 @@ mod tests {}}+ #[tokio::test]+ async fn aggregate_dynamic_filter_recovers_from_null_bounds() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, true),+ ]));+ let first = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 8])),+ Arc::new(Int32Array::from(vec![None, None])),+ ],+ )?;+ let second = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![2, 3])),+ Arc::new(Int32Array::from(vec![5, 7])),+ ],+ )?;+ let input = TestMemoryExec::try_new_exec(+ &[vec![first], vec![second]],+ Arc::clone(&schema),+ None,+ )?;+ let mut aggregates = Vec::new();+ for column in ["a", "b"] {+ for function in [min_udaf(), max_udaf()] {+ let alias = format!("{}({column})", function.name());+ aggregates.push(Arc::new(+ AggregateExprBuilder::new(function, vec![col(column, &schema)?])+ .schema(Arc::clone(&schema))+ .alias(alias)+ .build()?,+ ));+ }+ }+ let aggregate = AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::default(),+ aggregates,+ vec![None; 4],+ input,+ schema,+ )?;+ let filter = &aggregate.dynamic_filter.as_ref().unwrap().filter;+ let context = Arc::new(TaskContext::default());++ // Execute partitions sequentially to make filter publication deterministic.+ collect_stream(aggregate.execute(0, Arc::clone(&context))?).await?;+ assert_eq!(filter.current()?.to_string(), "true");++ // Typed NULL bounds must recover so filtering resumes for every aggregate.+ collect_stream(aggregate.execute(1, context)?).await?;+ assert_eq!(+ filter.current()?.to_string(),+ "a@0 < 1 OR a@0 > 8 OR b@1 < 5 OR b@1 > 7"+ );+ Ok(())+ }+#[tokio::test]async fn aggregate_stream_reports_per_aggregate_metrics() -> Result<()> {let schema = Arc::new(Schema::new(vec![@@ -841,10 +907,8 @@ mod tests {let context = Arc::new(TaskContext::default());for partition in 0..2 {- let _ = crate::common::collect(- aggregate.execute(partition, Arc::clone(&context))?,- )- .await?;+ let _ = collect_stream(aggregate.execute(partition, Arc::clone(&context))?)+ .await?;}let metrics = aggregate.metrics().unwrap();diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.sltindex c260efae95..20962e1548 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt@@ -443,6 +443,55 @@ drop table agg_dyn_mixed;statement okreset datafusion.execution.batch_size;+# --- one aggregate has a bound before another sees its first non-NULL value ---+# Both files establish MIN(a)=1 and MAX(a)=8 in their first row group. Until b+# has a non-NULL bound, filtering only on a would prune every remaining row and+# incorrectly leave MIN(b) and MAX(b) NULL, regardless of partition order.++statement ok+set datafusion.execution.batch_size = 2;++statement ok+COPY (+ SELECT * FROM (VALUES (1, NULL), (8, NULL), (2, 5), (3, 7)) AS v(a, b)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/file_0.parquet'+STORED AS PARQUET+OPTIONS ('format.max_row_group_size' '2');++statement ok+COPY (+ SELECT * FROM (VALUES (1, NULL), (8, NULL), (2, 5), (3, 7)) AS v(a, b)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/file_1.parquet'+STORED AS PARQUET+OPTIONS ('format.max_row_group_size' '2');++statement ok+CREATE EXTERNAL TABLE agg_dyn_late_bound (a INT, b INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/';++statement ok+set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = false;++query IIII+SELECT MIN(a), MAX(a), MIN(b), MAX(b) FROM agg_dyn_late_bound;+----+1 8 5 7++statement ok+set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = true;++query IIII+SELECT MIN(a), MAX(a), MIN(b), MAX(b) FROM agg_dyn_late_bound;+----+1 8 5 7++statement ok+drop table agg_dyn_late_bound;++statement ok+reset datafusion.execution.batch_size;+# --- all-NULLs input: filter should stay `true` (no meaningful bound) ---statement ok