Apache DataFusion SQL Query Engine
fix: do not push filters on volatile group keys below Aggregate (#25416)
Adrian Garcia Badaraccoc4f5a9e0f202
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex 8a1dcc12ef..fdbc3275c4 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -997,7 +997,16 @@ impl OptimizerRule for PushDownFilter {}LogicalPlan::Aggregate(mut agg) => {// We can push down Predicate which in groupby_expr.- let group_expr_columns = expr_columns(&agg.group_expr);+ // Volatile group keys are excluded: below the aggregate, the+ // predicate would evaluate the key again and see a different+ // value than the one used for grouping.+ let non_volatile_group_exprs: Vec<Expr> = agg+ .group_expr+ .iter()+ .filter(|expr| !expr.is_volatile())+ .cloned()+ .collect();+ let group_expr_columns = expr_columns(&non_volatile_group_exprs);// As for plan Filter: Column(a+b) > 0 -- Agg: groupby:[Column(a)+Column(b)]// After push, we need to replace `a+b` with Column(a)+Column(b)@@ -4091,6 +4100,34 @@ mod tests {)}+ #[test]+ fn test_filter_on_volatile_group_key_not_pushed_below_aggregate() -> Result<()> {+ // SELECT r, sum(b) FROM test1 GROUP BY a, TestScalarUDF() + 1 AS r HAVING a > 5 AND r > 0.5+ let table_scan = test_table_scan_with_name("test1")?;+ let fun = ScalarUDF::new_from_impl(TestScalarUDF {+ signature: Signature::exact(vec![], Volatility::Volatile),+ });+ let expr = Expr::ScalarFunction(ScalarFunction::new_udf(Arc::new(fun), vec![]));++ let plan = LogicalPlanBuilder::from(table_scan)+ .aggregate(+ vec![col("a"), add(expr, lit(1)).alias("r")],+ vec![sum(col("b"))],+ )?+ .filter(col("a").gt(lit(5)).and(col("r").gt(lit(0.5))))?+ .build()?;++ // `a > 5` is pushed below the aggregate, `r > 0.5` must stay above it+ assert_optimized_plan_equal!(+ plan,+ @r"+ Filter: r > Float64(0.5)+ Aggregate: groupBy=[[test1.a, TestScalarUDF() + Int32(1) AS r]], aggr=[[sum(test1.b)]]+ TableScan: test1, full_filters=[test1.a > Int32(5)]+ "+ )+ }+#[test]fn test_push_down_volatile_function_in_join() -> Result<()> {// SELECT t.a, t.r FROM (SELECT test1.a AS a, TestScalarUDF() AS r FROM test1 join test2 ON test1.a = test2.a) AS t WHERE t.r > 0.5;diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.sltindex c327f8092f..c260efae95 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt@@ -640,3 +640,37 @@ drop table t1;statement okdrop table t2;++# A filter on a volatile group key must stay above the aggregate. Below the+# aggregate, the filter evaluates the key again and sees a different value.+query TT+EXPLAIN SELECT k, c FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY 1) WHERE k;+----+logical_plan+01)Projection: random() < Float64(0.5) AS k, count(Int64(1)) AS c+02)--Filter: random() < Float64(0.5)+03)----Aggregate: groupBy=[[random() < Float64(0.5)]], aggr=[[count(Int64(1))]]+04)------TableScan: generate_series() projection=[]+physical_plan+01)ProjectionExec: expr=[random() < Float64(0.5)@0 as k, count(Int64(1))@1 as c]+02)--FilterExec: random() < Float64(0.5)@0+03)----AggregateExec: mode=FinalPartitioned, gby=[random() < Float64(0.5)@0 as random() < Float64(0.5)], aggr=[count(Int64(1))]+04)------RepartitionExec: partitioning=Hash([random() < Float64(0.5)@0], 4), input_partitions=4+05)--------AggregateExec: mode=Partial, gby=[random() < 0.5 as random() < Float64(0.5)], aggr=[count(Int64(1))]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=10000, batch_size=8192]++query B+SELECT k FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY random() < 0.5) WHERE k;+----+true++query B+SELECT k FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY k) WHERE NOT k;+----+false++query I+SELECT sum(c) FROM (SELECT random() < 0.5 AS k, count(*) AS c FROM generate_series(1, 10000) GROUP BY random() < 0.5) WHERE k OR NOT k;+----+10000