Skip to content

Apache DataFusion SQL Query Engine

fix: preserve grouping set member names during type coercion (#25891)

Peter L0566b55a4766
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index 39310eec1f..47aba9e715 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -177,8 +177,7 @@ fn analyze_internal(
TypeCoercionRewriter::new(&schema).with_session_time_zone(session_time_zone);
let name_preserver = NamePreserver::new(&plan);
- // apply coercion rewrite all expressions in the plan individually
- plan.map_expressions(|expr| {
+ let mut coerce_expr = |expr: Expr| {
let original_name = name_preserver.save(&expr);
// A lambda variable carries the field recorded when the plan was built, which
@@ -192,6 +191,17 @@ fn analyze_internal(
expr.rewrite(&mut expr_rewrite)
.map(|transformed| transformed.update_data(|e| original_name.restore(e)))
+ };
+
+ // apply coercion rewrite all expressions in the plan individually
+ plan.map_expressions(|expr| {
+ // Preserve the names of grouping set members, not of the grouping set itself.
+ // An aliased grouping set is no longer recognized as a grouping set.
+ if let Expr::GroupingSet(_) = &expr {
+ expr.map_children(&mut coerce_expr)
+ } else {
+ coerce_expr(expr)
+ }
})?
// some plans need extra coercion after their expressions are coerced
.map_data(|plan| expr_rewrite.coerce_plan(plan))?
@@ -1775,13 +1785,13 @@ mod test {
use datafusion_expr::expr::{
self, InSubquery, Like, ScalarFunction, SetComparison, SetQuantifier,
};
- use datafusion_expr::logical_plan::{EmptyRelation, Projection, Sort};
+ use datafusion_expr::logical_plan::{EmptyRelation, Projection, Sort, table_scan};
use datafusion_expr::test::function_stub::avg_udaf;
use datafusion_expr::{
AccumulatorFactoryFunction, AggregateUDF, BinaryExpr, Case, ColumnarValue, Expr,
ExprSchemable, Filter, Limit, LogicalPlan, Operator, ScalarFunctionArgs,
ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF, Subquery, Union,
- Volatility, cast, col, create_udaf, is_true, lit, placeholder,
+ Volatility, cast, col, create_udaf, grouping_set, is_true, lit, placeholder,
};
use datafusion_functions_aggregate::average::AvgAccumulator;
@@ -2373,6 +2383,41 @@ mod test {
)
}
+ #[test]
+ fn scalar_udf_in_grouping_set() -> Result<()> {
+ let udf = ScalarUDF::from(TestScalarUDF {
+ signature: Signature::uniform(1, vec![DataType::Float32], Volatility::Stable),
+ })
+ .call(vec![lit(123_i32)]);
+ let schema = Schema::new(vec![Field::new("a", Utf8, false)]);
+ let plan = table_scan(Some("t"), &schema, None)?
+ .aggregate(
+ vec![grouping_set(vec![
+ vec![col("t.a"), udf.clone()],
+ vec![udf],
+ vec![],
+ ])],
+ Vec::<Expr>::new(),
+ )?
+ .build()?;
+
+ // Each renamed member is aliased to its original name, not the grouping set
+ let analyzed = TypeCoercion::new().analyze(plan, &ConfigOptions::default())?;
+ assert_snapshot!(
+ analyzed,
+ @r#"
+ Aggregate: groupBy=[[GROUPING SETS ((t.a, TestScalarUDF(Float32(123)) AS TestScalarUDF(Int32(123))), (TestScalarUDF(Float32(123)) AS TestScalarUDF(Int32(123))), ())]], aggr=[[]]
+ TableScan: t
+ "#
+ );
+
+ // A second pass leaves the plan unchanged
+ let reanalyzed =
+ TypeCoercion::new().analyze(analyzed.clone(), &ConfigOptions::default())?;
+ assert_eq!(analyzed, reanalyzed);
+ Ok(())
+ }
+
#[test]
fn scalar_udf_invalid_input() -> Result<()> {
let empty = empty();
diff --git a/datafusion/sqllogictest/test_files/grouping.slt b/datafusion/sqllogictest/test_files/grouping.slt
index 738b7ae722..279ed32295 100644
--- a/datafusion/sqllogictest/test_files/grouping.slt
+++ b/datafusion/sqllogictest/test_files/grouping.slt
@@ -445,3 +445,38 @@ query TT
SELECT MIN(p), MAX(p) FROM (SELECT p FROM hive_partitioned_empty GROUP BY p)
----
NULL NULL
+
+# Grouping-set keys keep their names when type coercion rewrites a member.
+# `date_bin('5 minutes', ts)` has its Utf8 stride literal coerced to an interval,
+# which renames the member expression.
+statement ok
+CREATE TABLE grouping_coerced (ts TIMESTAMP, k VARCHAR, calls BIGINT) AS VALUES
+ ('1970-01-01T00:00:00'::timestamp, 'a', 1),
+ ('1970-01-01T00:01:00'::timestamp, 'b', 2),
+ ('1970-01-01T00:05:00'::timestamp, 'a', 4);
+
+query PII
+SELECT date_bin('5 minutes', ts) AS bucket, grouping(date_bin('5 minutes', ts)), SUM(calls)
+FROM grouping_coerced GROUP BY ROLLUP(bucket) ORDER BY bucket NULLS FIRST
+----
+NULL 1 7
+1970-01-01T00:00:00 0 3
+1970-01-01T00:05:00 0 4
+
+# A qualified column next to the coerced member, with the coerced member repeated
+# across sets
+query TPI
+SELECT grouping_coerced.k, date_bin('5 minutes', grouping_coerced.ts) AS bucket, SUM(calls)
+FROM grouping_coerced
+GROUP BY GROUPING SETS ((grouping_coerced.k, bucket), (bucket), ())
+ORDER BY grouping_coerced.k NULLS FIRST, bucket NULLS FIRST
+----
+NULL NULL 7
+NULL 1970-01-01T00:00:00 3
+NULL 1970-01-01T00:05:00 4
+a 1970-01-01T00:00:00 1
+a 1970-01-01T00:05:00 4
+b 1970-01-01T00:00:00 2
+
+statement ok
+DROP TABLE grouping_coerced;