Skip to content

Apache DataFusion SQL Query Engine

fix: resolve pass-through columns against the input when merging an extraction projection (#25412)

Adrian Garcia Badaracco47135be91cf1
diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rs
index aa3cef881f..999c396745 100644
--- a/datafusion/optimizer/src/extract_leaf_expressions.rs
+++ b/datafusion/optimizer/src/extract_leaf_expressions.rs
@@ -672,6 +672,20 @@ impl<'a> LeafExpressionExtractor<'a> {
}
}
+/// Resolves `col` against `schema` and returns it with the qualifier that
+/// `schema` gives the field. Returns `None` if `schema` does not hold `col`,
+/// or if the name is ambiguous in `schema`.
+///
+/// Two spellings of the same input column (a bare `c` and a qualified `t.c`)
+/// resolve to the same `Column`, so they compare equal. A column pushed into
+/// a projection this way uses the name that the input gives it.
+fn resolve_against(schema: &DFSchema, col: &Column) -> Option<Column> {
+ schema
+ .qualified_field_from_column(col)
+ .ok()
+ .map(Column::from)
+}
+
/// Build an extraction projection above the target node (shared by both passes).
///
/// If the target is an existing projection, merges into it. This requires
@@ -737,29 +751,31 @@ fn build_extraction_projection_impl(
// than target_schema (the projection's output) because columns produced
// by alias expressions (e.g., CSE's __common_expr_N) exist in the output but
// not the input, and cannot be added as pass-through Column references.
+ //
+ // Resolve the pass-through columns of `existing` (including same-name
+ // aliases such as `t.c AS c`) and the columns in `columns_needed`
+ // against the input schema before comparing them. Otherwise a bare `c`
+ // and a qualified `t.c` do not match, and the merged projection holds
+ // both, which `Projection::try_new` rejects as ambiguous.
+ let input_schema = existing.input.schema();
let existing_cols: IndexSet<Column> = existing
.expr
.iter()
- .filter_map(|e| {
- if let Expr::Column(c) = e {
- Some(c.clone())
- } else {
- None
- }
- })
+ .filter_map(|e| resolve_against(input_schema, passthrough_column(e)?))
.collect();
- let input_schema = existing.input.schema();
for col in columns_needed {
let col_expr = Expr::Column(col.clone());
let resolved = replace_cols_by_name(col_expr, &replace_map)?;
if let Expr::Column(resolved_col) = &resolved
- && !existing_cols.contains(resolved_col)
- && input_schema.has_column(resolved_col)
+ && let Some(input_col) = resolve_against(input_schema, resolved_col)
+ && !existing_cols.contains(&input_col)
{
- proj_exprs.push(Expr::Column(resolved_col.clone()));
+ proj_exprs.push(Expr::Column(input_col));
}
- // If resolved to non-column expr, it's already computed by existing projection
+ // Skip a column that resolves to a non-column expr: the existing
+ // projection already computes it. Also skip a column that is not in
+ // the input or is ambiguous there: no single spelling is correct.
}
Projection::try_new(proj_exprs, Arc::clone(&existing.input))
@@ -2405,6 +2421,113 @@ mod tests {
"#)
}
+ /// A filter can name a column bare (`a`) while the projection below it
+ /// outputs the qualified `test.a`. The optimizer leaves this shape when it
+ /// removes the empty input of a `UNION ALL` (for example
+ /// `... UNION ALL SELECT ... WHERE 1 = 2`). The merge must match the two, and not add a bare `a`
+ /// beside `test.a`, which is an ambiguous schema.
+ #[test]
+ fn test_merge_bare_column_into_qualified_projection() -> Result<()> {
+ let table_scan = test_table_scan()?;
+ let projection = LogicalPlanBuilder::from(table_scan)
+ .project(vec![
+ col("test.a"),
+ col("test.b"),
+ (col("test.c") + lit(1)).alias("d"),
+ ])?
+ .build()?;
+ let predicate =
+ leaf_udf(Expr::Column(Column::new_unqualified("a")), "x").eq(lit(1));
+ let plan = LogicalPlan::Filter(datafusion_expr::Filter::try_new(
+ predicate,
+ Arc::new(projection),
+ )?);
+
+ assert_stages!(plan, @r#"
+ ## Original Plan
+ Filter: leaf_udf(a, Utf8("x")) = Int32(1)
+ Projection: test.a, test.b, test.c + Int32(1) AS d
+ TableScan: test projection=[a, b, c]
+
+ ## After Extraction
+ Projection: test.a, test.b, d
+ Filter: __datafusion_extracted_1 = Int32(1)
+ Projection: test.a, test.b, test.c + Int32(1) AS d, leaf_udf(a, Utf8("x")) AS __datafusion_extracted_1
+ TableScan: test projection=[a, b, c]
+
+ ## After Pushdown
+ (same as after extraction)
+
+ ## Optimized
+ (same as after pushdown)
+ "#)
+ }
+
+ /// When the merge appends a needed column that the projection does not
+ /// keep, it must append the input's qualified column (`test.b`), not the
+ /// bare `b` that the caller asked for.
+ #[test]
+ fn test_merge_appends_qualified_column_for_bare_request() -> Result<()> {
+ let projection = LogicalPlanBuilder::from(test_table_scan()?)
+ .project(vec![col("test.a")])?
+ .build()?;
+ let target_schema = Arc::clone(projection.schema());
+ let columns_needed: IndexSet<Column> =
+ IndexSet::from([Column::new_unqualified("b")]);
+
+ let merged = build_extraction_projection_impl(
+ &[],
+ &columns_needed,
+ &Arc::new(projection),
+ &target_schema,
+ )?;
+
+ assert_eq!(
+ merged.expr,
+ vec![
+ Expr::Column(Column::new(Some("test"), "a")),
+ Expr::Column(Column::new(Some("test"), "b")),
+ ]
+ );
+ Ok(())
+ }
+
+ /// A projection can spell a pass-through column as a same-name alias
+ /// (`test.a AS a`). Merging an extraction into it must treat that alias as
+ /// the pass-through it is, and not add `test.a` beside the `a` it outputs,
+ /// which is an ambiguous schema.
+ #[test]
+ fn test_merge_into_projection_with_same_name_alias() -> Result<()> {
+ let table_scan = test_table_scan()?;
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .project(vec![
+ col("test.a").alias("a"),
+ col("test.b").alias("b"),
+ col("test.c").alias("c"),
+ ])?
+ .filter(leaf_udf(col("a"), "x").eq(lit(1)))?
+ .build()?;
+
+ assert_stages!(plan, @r#"
+ ## Original Plan
+ Filter: leaf_udf(a, Utf8("x")) = Int32(1)
+ Projection: test.a AS a, test.b AS b, test.c AS c
+ TableScan: test projection=[a, b, c]
+
+ ## After Extraction
+ Projection: a, b, c
+ Filter: __datafusion_extracted_1 = Int32(1)
+ Projection: test.a AS a, test.b AS b, test.c AS c, leaf_udf(test.a, Utf8("x")) AS __datafusion_extracted_1
+ TableScan: test projection=[a, b, c]
+
+ ## After Pushdown
+ (same as after extraction)
+
+ ## Optimized
+ (same as after pushdown)
+ "#)
+ }
+
// =========================================================================
// Join extraction tests
// =========================================================================
diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt
index aa80ff0253..2eb91b712f 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -1906,3 +1906,54 @@ from (values (1),(2),(1)) group by 1;
----
{a: 1, z: NULL} 2
{a: 2, z: NULL} 1
+
+# Merging an extraction projection into a projection whose output lost its
+# qualifier. Eliminating the empty side of the union leaves a projection of
+# bare column names over a qualified input, and the merge used to add the
+# pass-through columns under those bare names beside the qualified ones the
+# projection already carried, which is an ambiguous schema.
+statement ok
+create table leaf_merge_source(v int, s struct<a int>, env varchar) as values (1, {a: 10}, 'prod'), (2, {a: 20}, 'dev');
+
+statement ok
+set datafusion.explain.logical_plan_only = true;
+
+query TT
+explain with samples as (
+ select v, s, env from leaf_merge_source
+),
+expanded as (
+ select v, s, env from samples
+ union all
+ select v, s, env from samples where 1 = 2
+)
+select env, sum(s['a']) from expanded group by env order by env;
+----
+logical_plan
+01)Sort: expanded.env ASC NULLS LAST
+02)--Projection: expanded.env, sum(__datafusion_extracted_1) AS sum(expanded.s[a])
+03)----Aggregate: groupBy=[[expanded.env]], aggr=[[sum(CAST(__datafusion_extracted_1 AS Int64))]]
+04)------SubqueryAlias: expanded
+05)--------SubqueryAlias: samples
+06)----------Projection: leaf_merge_source.env, get_field(s, Utf8("a")) AS __datafusion_extracted_1
+07)------------TableScan: leaf_merge_source projection=[s, env]
+
+statement ok
+set datafusion.explain.logical_plan_only = false;
+
+query TI
+with samples as (
+ select v, s, env from leaf_merge_source
+),
+expanded as (
+ select v, s, env from samples
+ union all
+ select v, s, env from samples where 1 = 2
+)
+select env, sum(s['a']) from expanded group by env order by env;
+----
+dev 20
+prod 10
+
+statement ok
+drop table leaf_merge_source;