Skip to content

Apache DataFusion SQL Query Engine

fix: infer placeholder types in GROUP BY, HAVING, QUALIFY and ORDER BY (fix for #24042) (#24043)

Braedon Wooding149bdb51b96f
diff --git a/datafusion/sql/src/expr/order_by.rs b/datafusion/sql/src/expr/order_by.rs
index faecfbcfec..0067a1ebd7 100644
--- a/datafusion/sql/src/expr/order_by.rs
+++ b/datafusion/sql/src/expr/order_by.rs
@@ -109,7 +109,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
))
}
e => {
- self.sql_expr_to_logical_expr(e, order_by_schema, planner_context)?
+ let expr = self.sql_expr_to_logical_expr(
+ e,
+ order_by_schema,
+ planner_context,
+ )?;
+ let (expr, _) = expr.infer_placeholder_types(order_by_schema)?;
+ expr
}
};
sort_expr_vec.push(make_sort_expr(expr, asc, nulls_first));
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs
index bdab013144..bbd9d203eb 100644
--- a/datafusion/sql/src/select.rs
+++ b/datafusion/sql/src/select.rs
@@ -191,7 +191,9 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// ON expression is a bare identifier. `b` resolves to the
// alias; `b + 0` keeps `b` as the input column.
let expr = substitute_top_level_alias(expr, &alias_map);
- normalize_col(expr, &projected_plan)
+ let expr = normalize_col(expr, &projected_plan)?;
+ let (expr, _) = expr.infer_placeholder_types(&on_expr_schema)?;
+ Ok(expr)
})
.collect::<Result<Vec<Expr>>>()?;
@@ -219,7 +221,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// SELECT c1, MAX(c2) AS m FROM t GROUP BY c1 HAVING MAX(c2) > 10;
//
let having_expr = resolve_aliases_to_exprs(having_expr, &alias_map)?;
- normalize_col(having_expr, &projected_plan)
+ let having_expr = normalize_col(having_expr, &projected_plan)?;
+ let (having_expr, _) =
+ having_expr.infer_placeholder_types(&combined_schema)?;
+ Ok(having_expr)
})
.transpose()?;
@@ -248,6 +253,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
base_plan.schema(),
std::slice::from_ref(&group_by_expr),
)?;
+ let (group_by_expr, _) =
+ group_by_expr.infer_placeholder_types(&combined_schema)?;
Ok(group_by_expr)
})
.collect::<Result<Vec<Expr>>>()?
@@ -286,7 +293,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// select row_number() over (PARTITION BY id) as rk from users qualify row_number() over (PARTITION BY id) > 1;
//
let qualify_expr = resolve_aliases_to_exprs(qualify_expr, &alias_map)?;
- normalize_col(qualify_expr, &projected_plan)
+ let qualify_expr = normalize_col(qualify_expr, &projected_plan)?;
+ let (qualify_expr, _) =
+ qualify_expr.infer_placeholder_types(&combined_schema)?;
+ Ok(qualify_expr)
})
.transpose()?;
diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs
index 4f282f5e06..08a95381b3 100644
--- a/datafusion/sql/tests/sql_integration.rs
+++ b/datafusion/sql/tests/sql_integration.rs
@@ -1504,6 +1504,113 @@ fn select_aggregate_with_group_by_with_having_using_count_star_not_in_select() {
);
}
+/// Asserts that placeholder `id` (e.g. `"$1"`) was inferred as `expected` type
+/// somewhere in `plan`.
+fn assert_placeholder_type(plan: &LogicalPlan, id: &str, expected: DataType) {
+ let param_types = plan.get_parameter_types().unwrap();
+ assert_eq!(param_types.get(id), Some(&Some(expected)));
+}
+
+/// An expression containing a placeholder, written in both the SELECT list and
+/// the GROUP BY, has to be recognised as one expression the way its literal
+/// equivalent is. Otherwise the columns inside it read as ungrouped, because the
+/// SELECT list has its placeholder types inferred and the grouping key does not.
+#[test]
+fn select_aggregate_with_group_by_placeholder_expression() {
+ let sql = "SELECT CASE WHEN age < $1 THEN 'young' ELSE 'old' END, count(*)
+ FROM person
+ GROUP BY CASE WHEN age < $1 THEN 'young' ELSE 'old' END";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ Projection: CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END, count(*)
+ Aggregate: groupBy=[[CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END]], aggr=[[count(*)]]
+ TableScan: person
+ "#
+ );
+ assert_placeholder_type(&plan, "$1", DataType::Int32);
+}
+
+/// The same, for a grouping expression repeated in HAVING.
+#[test]
+fn select_aggregate_with_having_placeholder_expression() {
+ let sql = "SELECT CASE WHEN age < $1 THEN 'young' ELSE 'old' END, count(*)
+ FROM person
+ GROUP BY CASE WHEN age < $1 THEN 'young' ELSE 'old' END
+ HAVING CASE WHEN age < $1 THEN 'young' ELSE 'old' END = 'young'";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ Projection: CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END, count(*)
+ Filter: CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END = Utf8("young")
+ Aggregate: groupBy=[[CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END]], aggr=[[count(*)]]
+ TableScan: person
+ "#
+ );
+ assert_placeholder_type(&plan, "$1", DataType::Int32);
+}
+
+/// The same, for a grouping expression repeated in ORDER BY.
+#[test]
+fn select_aggregate_with_order_by_placeholder_expression() {
+ let sql = "SELECT CASE WHEN age < $1 THEN 'young' ELSE 'old' END, count(*)
+ FROM person
+ GROUP BY CASE WHEN age < $1 THEN 'young' ELSE 'old' END
+ ORDER BY CASE WHEN age < $1 THEN 'young' ELSE 'old' END";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ Sort: CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END ASC NULLS LAST
+ Projection: CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END, count(*)
+ Aggregate: groupBy=[[CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END]], aggr=[[count(*)]]
+ TableScan: person
+ "#
+ );
+ assert_placeholder_type(&plan, "$1", DataType::Int32);
+}
+
+/// The same, for a window expression repeated in QUALIFY. Here the two spellings
+/// of the window expression collide by name instead, since they print alike but
+/// do not compare equal.
+#[test]
+fn select_window_with_qualify_placeholder_expression() {
+ let sql = "SELECT first_name,
+ row_number() OVER (PARTITION BY CASE WHEN age < $1 THEN 'young' ELSE 'old' END)
+ FROM person
+ QUALIFY row_number() OVER (PARTITION BY CASE WHEN age < $1 THEN 'young' ELSE 'old' END) = 1";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ Projection: person.first_name, row_number() PARTITION BY [CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+ Filter: row_number() PARTITION BY [CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING = Int64(1)
+ WindowAggr: windowExpr=[[row_number() PARTITION BY [CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+ TableScan: person
+ "#
+ );
+ assert_placeholder_type(&plan, "$1", DataType::Int32);
+}
+
+#[test]
+fn select_distinct_on_with_order_by_placeholder_expression() {
+ let sql =
+ "SELECT DISTINCT ON (CASE WHEN age < $1 THEN 'young' ELSE 'old' END) first_name
+ FROM person
+ ORDER BY CASE WHEN age < $1 THEN 'young' ELSE 'old' END";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ DistinctOn: on_expr=[[CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END]], select_expr=[[person.first_name]], sort_expr=[[CASE WHEN person.age < $1 THEN Utf8("young") ELSE Utf8("old") END ASC NULLS LAST]]
+ TableScan: person
+ "#
+ );
+ assert_placeholder_type(&plan, "$1", DataType::Int32);
+}
+
#[test]
fn select_binary_expr() {
let sql = "SELECT age + salary from person";