Skip to content

Apache DataFusion SQL Query Engine

datafusion-physical-plan: handles nulls correctly in aggregate dynamic filters (#25579)

Jayant Shrivastava0af681889123
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs b/datafusion/physical-plan/src/aggregates/aggregate_stream.rs
index 0da8f0944e..5811b98d74 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_stream.rs
@@ -85,7 +85,6 @@ struct AggregateStreamInner {
}
impl AggregateStreamInner {
- // TODO: check if we get Null handling correct
/// # Examples
/// - Example 1
/// Accumulators: min(c1)
@@ -113,11 +112,12 @@ impl AggregateStreamInner {
Vec::with_capacity(filter_state.accumulator_dyn_filter_info.len());
for acc_info in &filter_state.accumulator_dyn_filter_info {
- // Skip if we don't yet have a meaningful bound
+ // Every aggregate needs a bound before we can prune rows. Otherwise,
+ // another aggregate's predicate could discard its first non-NULL value.
let bound = {
let guard = acc_info.shared_bound.lock();
if (*guard).is_null() {
- continue;
+ return Ok(lit(true));
}
guard.clone()
};
@@ -263,9 +263,9 @@ fn scalar_cmp_null_short_circuit(
v1: &ScalarValue,
v2: &ScalarValue,
) -> Option<ScalarValue> {
- match (v1, v2) {
- (ScalarValue::Null, ScalarValue::Null) => Some(ScalarValue::Null),
- (ScalarValue::Null, other) | (other, ScalarValue::Null) => Some(other.clone()),
+ match (v1.is_null(), v2.is_null()) {
+ (true, _) => Some(v2.clone()),
+ (_, true) => Some(v1.clone()),
_ => None,
}
}
@@ -578,12 +578,17 @@ fn finalize_aggregation_with_metrics(
mod tests {
use super::*;
use crate::aggregates::PhysicalGroupBy;
+ use crate::common::collect as collect_stream;
use crate::metrics::{MetricValue, MetricsSet};
use crate::test::TestMemoryExec;
use crate::{ExecutionPlan, collect};
- use arrow::array::{Float64Array, UInt32Array};
+ use arrow::array::{Float64Array, Int32Array, UInt32Array};
use arrow::datatypes::{DataType, Field, Schema};
- use datafusion_functions_aggregate::{array_agg::array_agg_udaf, sum::sum_udaf};
+ use datafusion_functions_aggregate::{
+ array_agg::array_agg_udaf,
+ min_max::{max_udaf, min_udaf},
+ sum::sum_udaf,
+ };
use datafusion_physical_expr::aggregate::{
AggregateExprBuilder, AggregateFunctionExpr,
};
@@ -677,6 +682,67 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn aggregate_dynamic_filter_recovers_from_null_bounds() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, true),
+ ]));
+ let first = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 8])),
+ Arc::new(Int32Array::from(vec![None, None])),
+ ],
+ )?;
+ let second = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![2, 3])),
+ Arc::new(Int32Array::from(vec![5, 7])),
+ ],
+ )?;
+ let input = TestMemoryExec::try_new_exec(
+ &[vec![first], vec![second]],
+ Arc::clone(&schema),
+ None,
+ )?;
+ let mut aggregates = Vec::new();
+ for column in ["a", "b"] {
+ for function in [min_udaf(), max_udaf()] {
+ let alias = format!("{}({column})", function.name());
+ aggregates.push(Arc::new(
+ AggregateExprBuilder::new(function, vec![col(column, &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias(alias)
+ .build()?,
+ ));
+ }
+ }
+ let aggregate = AggregateExec::try_new(
+ AggregateMode::Partial,
+ PhysicalGroupBy::default(),
+ aggregates,
+ vec![None; 4],
+ input,
+ schema,
+ )?;
+ let filter = &aggregate.dynamic_filter.as_ref().unwrap().filter;
+ let context = Arc::new(TaskContext::default());
+
+ // Execute partitions sequentially to make filter publication deterministic.
+ collect_stream(aggregate.execute(0, Arc::clone(&context))?).await?;
+ assert_eq!(filter.current()?.to_string(), "true");
+
+ // Typed NULL bounds must recover so filtering resumes for every aggregate.
+ collect_stream(aggregate.execute(1, context)?).await?;
+ assert_eq!(
+ filter.current()?.to_string(),
+ "a@0 < 1 OR a@0 > 8 OR b@1 < 5 OR b@1 > 7"
+ );
+ Ok(())
+ }
+
#[tokio::test]
async fn aggregate_stream_reports_per_aggregate_metrics() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
@@ -841,10 +907,8 @@ mod tests {
let context = Arc::new(TaskContext::default());
for partition in 0..2 {
- let _ = crate::common::collect(
- aggregate.execute(partition, Arc::clone(&context))?,
- )
- .await?;
+ let _ = collect_stream(aggregate.execute(partition, Arc::clone(&context))?)
+ .await?;
}
let metrics = aggregate.metrics().unwrap();
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
index c260efae95..20962e1548 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
@@ -443,6 +443,55 @@ drop table agg_dyn_mixed;
statement ok
reset datafusion.execution.batch_size;
+# --- one aggregate has a bound before another sees its first non-NULL value ---
+# Both files establish MIN(a)=1 and MAX(a)=8 in their first row group. Until b
+# has a non-NULL bound, filtering only on a would prune every remaining row and
+# incorrectly leave MIN(b) and MAX(b) NULL, regardless of partition order.
+
+statement ok
+set datafusion.execution.batch_size = 2;
+
+statement ok
+COPY (
+ SELECT * FROM (VALUES (1, NULL), (8, NULL), (2, 5), (3, 7)) AS v(a, b)
+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/file_0.parquet'
+STORED AS PARQUET
+OPTIONS ('format.max_row_group_size' '2');
+
+statement ok
+COPY (
+ SELECT * FROM (VALUES (1, NULL), (8, NULL), (2, 5), (3, 7)) AS v(a, b)
+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/file_1.parquet'
+STORED AS PARQUET
+OPTIONS ('format.max_row_group_size' '2');
+
+statement ok
+CREATE EXTERNAL TABLE agg_dyn_late_bound (a INT, b INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/';
+
+statement ok
+set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = false;
+
+query IIII
+SELECT MIN(a), MAX(a), MIN(b), MAX(b) FROM agg_dyn_late_bound;
+----
+1 8 5 7
+
+statement ok
+set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = true;
+
+query IIII
+SELECT MIN(a), MAX(a), MIN(b), MAX(b) FROM agg_dyn_late_bound;
+----
+1 8 5 7
+
+statement ok
+drop table agg_dyn_late_bound;
+
+statement ok
+reset datafusion.execution.batch_size;
+
# --- all-NULLs input: filter should stay `true` (no meaningful bound) ---
statement ok