Skip to content

Apache DataFusion SQL Query Engine

fix: do not push filters on volatile group keys below Aggregate (#25416)

Adrian Garcia Badaraccoc4f5a9e0f202
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs
index 8a1dcc12ef..fdbc3275c4 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -997,7 +997,16 @@ impl OptimizerRule for PushDownFilter {
}
LogicalPlan::Aggregate(mut agg) => {
// We can push down Predicate which in groupby_expr.
- let group_expr_columns = expr_columns(&agg.group_expr);
+ // Volatile group keys are excluded: below the aggregate, the
+ // predicate would evaluate the key again and see a different
+ // value than the one used for grouping.
+ let non_volatile_group_exprs: Vec<Expr> = agg
+ .group_expr
+ .iter()
+ .filter(|expr| !expr.is_volatile())
+ .cloned()
+ .collect();
+ let group_expr_columns = expr_columns(&non_volatile_group_exprs);
// As for plan Filter: Column(a+b) > 0 -- Agg: groupby:[Column(a)+Column(b)]
// After push, we need to replace `a+b` with Column(a)+Column(b)
@@ -4091,6 +4100,34 @@ mod tests {
)
}
+ #[test]
+ fn test_filter_on_volatile_group_key_not_pushed_below_aggregate() -> Result<()> {
+ // SELECT r, sum(b) FROM test1 GROUP BY a, TestScalarUDF() + 1 AS r HAVING a > 5 AND r > 0.5
+ let table_scan = test_table_scan_with_name("test1")?;
+ let fun = ScalarUDF::new_from_impl(TestScalarUDF {
+ signature: Signature::exact(vec![], Volatility::Volatile),
+ });
+ let expr = Expr::ScalarFunction(ScalarFunction::new_udf(Arc::new(fun), vec![]));
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(
+ vec![col("a"), add(expr, lit(1)).alias("r")],
+ vec![sum(col("b"))],
+ )?
+ .filter(col("a").gt(lit(5)).and(col("r").gt(lit(0.5))))?
+ .build()?;
+
+ // `a > 5` is pushed below the aggregate, `r > 0.5` must stay above it
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Filter: r > Float64(0.5)
+ Aggregate: groupBy=[[test1.a, TestScalarUDF() + Int32(1) AS r]], aggr=[[sum(test1.b)]]
+ TableScan: test1, full_filters=[test1.a > Int32(5)]
+ "
+ )
+ }
+
#[test]
fn test_push_down_volatile_function_in_join() -> Result<()> {
// SELECT t.a, t.r FROM (SELECT test1.a AS a, TestScalarUDF() AS r FROM test1 join test2 ON test1.a = test2.a) AS t WHERE t.r > 0.5;
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
index c327f8092f..c260efae95 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
@@ -640,3 +640,37 @@ drop table t1;
statement ok
drop table t2;
+
+# A filter on a volatile group key must stay above the aggregate. Below the
+# aggregate, the filter evaluates the key again and sees a different value.
+query TT
+EXPLAIN SELECT k, c FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY 1) WHERE k;
+----
+logical_plan
+01)Projection: random() < Float64(0.5) AS k, count(Int64(1)) AS c
+02)--Filter: random() < Float64(0.5)
+03)----Aggregate: groupBy=[[random() < Float64(0.5)]], aggr=[[count(Int64(1))]]
+04)------TableScan: generate_series() projection=[]
+physical_plan
+01)ProjectionExec: expr=[random() < Float64(0.5)@0 as k, count(Int64(1))@1 as c]
+02)--FilterExec: random() < Float64(0.5)@0
+03)----AggregateExec: mode=FinalPartitioned, gby=[random() < Float64(0.5)@0 as random() < Float64(0.5)], aggr=[count(Int64(1))]
+04)------RepartitionExec: partitioning=Hash([random() < Float64(0.5)@0], 4), input_partitions=4
+05)--------AggregateExec: mode=Partial, gby=[random() < 0.5 as random() < Float64(0.5)], aggr=[count(Int64(1))]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=10000, batch_size=8192]
+
+query B
+SELECT k FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY random() < 0.5) WHERE k;
+----
+true
+
+query B
+SELECT k FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY k) WHERE NOT k;
+----
+false
+
+query I
+SELECT sum(c) FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY random() < 0.5) WHERE k OR NOT k;
+----
+10000