Apache DataFusion SQL Query Engine
fix: preserve grouping set member names during type coercion (#25891)
Peter L0566b55a4766
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 39310eec1f..47aba9e715 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -177,8 +177,7 @@ fn analyze_internal(TypeCoercionRewriter::new(&schema).with_session_time_zone(session_time_zone);let name_preserver = NamePreserver::new(&plan);- // apply coercion rewrite all expressions in the plan individually- plan.map_expressions(|expr| {+ let mut coerce_expr = |expr: Expr| {let original_name = name_preserver.save(&expr);// A lambda variable carries the field recorded when the plan was built, which@@ -192,6 +191,17 @@ fn analyze_internal(expr.rewrite(&mut expr_rewrite).map(|transformed| transformed.update_data(|e| original_name.restore(e)))+ };++ // apply coercion rewrite all expressions in the plan individually+ plan.map_expressions(|expr| {+ // Preserve the names of grouping set members, not of the grouping set itself.+ // An aliased grouping set is no longer recognized as a grouping set.+ if let Expr::GroupingSet(_) = &expr {+ expr.map_children(&mut coerce_expr)+ } else {+ coerce_expr(expr)+ }})?// some plans need extra coercion after their expressions are coerced.map_data(|plan| expr_rewrite.coerce_plan(plan))?@@ -1775,13 +1785,13 @@ mod test {use datafusion_expr::expr::{self, InSubquery, Like, ScalarFunction, SetComparison, SetQuantifier,};- use datafusion_expr::logical_plan::{EmptyRelation, Projection, Sort};+ use datafusion_expr::logical_plan::{EmptyRelation, Projection, Sort, table_scan};use datafusion_expr::test::function_stub::avg_udaf;use datafusion_expr::{AccumulatorFactoryFunction, AggregateUDF, BinaryExpr, Case, ColumnarValue, Expr,ExprSchemable, Filter, Limit, LogicalPlan, Operator, ScalarFunctionArgs,ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF, Subquery, Union,- Volatility, cast, col, create_udaf, is_true, lit, placeholder,+ Volatility, cast, col, create_udaf, grouping_set, is_true, lit, placeholder,};use datafusion_functions_aggregate::average::AvgAccumulator;@@ -2373,6 +2383,41 @@ mod test {)}+ #[test]+ fn scalar_udf_in_grouping_set() -> Result<()> {+ let udf = ScalarUDF::from(TestScalarUDF {+ signature: Signature::uniform(1, vec![DataType::Float32], Volatility::Stable),+ })+ .call(vec![lit(123_i32)]);+ let schema = Schema::new(vec![Field::new("a", Utf8, false)]);+ let plan = table_scan(Some("t"), &schema, None)?+ .aggregate(+ vec![grouping_set(vec![+ vec![col("t.a"), udf.clone()],+ vec![udf],+ vec![],+ ])],+ Vec::<Expr>::new(),+ )?+ .build()?;++ // Each renamed member is aliased to its original name, not the grouping set+ let analyzed = TypeCoercion::new().analyze(plan, &ConfigOptions::default())?;+ assert_snapshot!(+ analyzed,+ @r#"+ Aggregate: groupBy=[[GROUPING SETS ((t.a, TestScalarUDF(Float32(123)) AS TestScalarUDF(Int32(123))), (TestScalarUDF(Float32(123)) AS TestScalarUDF(Int32(123))), ())]], aggr=[[]]+ TableScan: t+ "#+ );++ // A second pass leaves the plan unchanged+ let reanalyzed =+ TypeCoercion::new().analyze(analyzed.clone(), &ConfigOptions::default())?;+ assert_eq!(analyzed, reanalyzed);+ Ok(())+ }+#[test]fn scalar_udf_invalid_input() -> Result<()> {let empty = empty();diff --git a/datafusion/sqllogictest/test_files/grouping.slt b/datafusion/sqllogictest/test_files/grouping.sltindex 738b7ae722..279ed32295 100644--- a/datafusion/sqllogictest/test_files/grouping.slt+++ b/datafusion/sqllogictest/test_files/grouping.slt@@ -445,3 +445,38 @@ query TTSELECT MIN(p), MAX(p) FROM (SELECT p FROM hive_partitioned_empty GROUP BY p)----NULL NULL++# Grouping-set keys keep their names when type coercion rewrites a member.+# `date_bin('5 minutes', ts)` has its Utf8 stride literal coerced to an interval,+# which renames the member expression.+statement ok+CREATE TABLE grouping_coerced (ts TIMESTAMP, k VARCHAR, calls BIGINT) AS VALUES+ ('1970-01-01T00:00:00'::timestamp, 'a', 1),+ ('1970-01-01T00:01:00'::timestamp, 'b', 2),+ ('1970-01-01T00:05:00'::timestamp, 'a', 4);++query PII+SELECT date_bin('5 minutes', ts) AS bucket, grouping(date_bin('5 minutes', ts)), SUM(calls)+FROM grouping_coerced GROUP BY ROLLUP(bucket) ORDER BY bucket NULLS FIRST+----+NULL 1 7+1970-01-01T00:00:00 0 3+1970-01-01T00:05:00 0 4++# A qualified column next to the coerced member, with the coerced member repeated+# across sets+query TPI+SELECT grouping_coerced.k, date_bin('5 minutes', grouping_coerced.ts) AS bucket, SUM(calls)+FROM grouping_coerced+GROUP BY GROUPING SETS ((grouping_coerced.k, bucket), (bucket), ())+ORDER BY grouping_coerced.k NULLS FIRST, bucket NULLS FIRST+----+NULL NULL 7+NULL 1970-01-01T00:00:00 3+NULL 1970-01-01T00:05:00 4+a 1970-01-01T00:00:00 1+a 1970-01-01T00:05:00 4+b 1970-01-01T00:00:00 2++statement ok+DROP TABLE grouping_coerced;