Apache DataFusion SQL Query Engine
fix: align Union inputs after leaf expression pushdown (#26421)
Adrian Garcia Badaracco7c918f98546d
diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rsindex eff6e51988..ca12fb0bbb 100644--- a/datafusion/optimizer/src/extract_leaf_expressions.rs+++ b/datafusion/optimizer/src/extract_leaf_expressions.rs@@ -1443,6 +1443,7 @@ fn try_push_into_inputs(// so if the TopDown pass later pushes a child further (changing its output// schema), the parent node's schema becomes stale.let mut new_inputs: Vec<LogicalPlan> = Vec::with_capacity(num_inputs);+ let is_union = matches!(node, LogicalPlan::Union(_));for (idx, input) in inputs.into_iter().enumerate() {if per_input[idx].pairs.is_empty() {new_inputs.push(input.clone());@@ -1478,9 +1479,14 @@ fn try_push_into_inputs(// this input (e.g., through Filter → existing extraction projection).// This ensures the input's output schema is stable and won't change// when the TopDown pass later visits children.- match try_push_input(&proj_plan, alias_generator)? {- Some(pushed) => new_inputs.push(pushed),- None => new_inputs.push(proj_plan),+ let pushed = match try_push_input(&proj_plan, alias_generator)? {+ Some(pushed) => pushed,+ None => proj_plan,+ };+ if is_union {+ new_inputs.push(align_union_input(pushed, &per_input[idx])?);+ } else {+ new_inputs.push(pushed);}}}@@ -1501,6 +1507,40 @@ fn try_push_into_inputs(Ok(Some(new_node))}+/// Projects one input of a `Union` to the extracted aliases, in pair order,+/// followed by the needed columns, in column order.+///+/// `Union` matches its inputs by position, not by name. A merge into an+/// existing projection appends the aliases after the existing expressions,+/// while a new projection puts them first. Without this, two inputs that take+/// different paths disagree on the position of each column, and the union+/// reads the alias of one input from an unrelated column of another input.+fn align_union_input(+ input: LogicalPlan,+ target: &ExtractionTarget,+) -> Result<LogicalPlan> {+ let exprs: Vec<Expr> = target+ .pairs+ .iter()+ .map(|(_, alias)| Expr::Column(Column::new_unqualified(alias)))+ .chain(target.columns.iter().map(|col| Expr::Column(col.clone())))+ .collect();+ // Skip the extra projection when the input already has this exact layout.+ let fields = input.schema().fields();+ let already_aligned = fields.len() == exprs.len()+ && target+ .pairs+ .iter()+ .map(|(_, alias)| alias.as_str())+ .chain(target.columns.iter().map(|col| col.name.as_str()))+ .zip(fields.iter())+ .all(|(name, field)| field.name() == name);+ if already_aligned {+ return Ok(input);+ }+ Projection::try_new(exprs, Arc::new(input)).map(LogicalPlan::Projection)+}+#[cfg(test)]mod tests {diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex 2d5cb50af5..ed71ad4ea4 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -2560,3 +2560,78 @@ RESET datafusion.optimizer.max_passes;statement okSET datafusion.execution.target_partitions = 4;++#####################+# Union inputs that take different leaf pushdown paths+#####################++# A union matches its inputs by position. The first input is a projection over+# a limit, so the extraction merges into that projection. The second input is+# a bare limit, so the extraction gets a new projection. Both inputs must still+# place the extracted alias at the same position.++statement ok+CREATE TABLE union_leaf_a (s STRUCT<y VARCHAR>, k VARCHAR) AS VALUES (named_struct('y', 'struct-a'), 'column-a');++statement ok+CREATE TABLE union_leaf_b (s STRUCT<y VARCHAR>, k VARCHAR) AS VALUES (named_struct('y', 'struct-b'), 'column-b');++query T+SELECT get_field(s, 'y') AS y FROM (+ SELECT s, k2 AS k FROM (SELECT s, k AS k2 FROM union_leaf_a LIMIT 10)+ UNION ALL+ (SELECT s, k FROM union_leaf_b LIMIT 10)+) ORDER BY y;+----+struct-a+struct-b++statement ok+CREATE TABLE union_leaf_c (k VARCHAR, s STRUCT<x DOUBLE, y VARCHAR>) AS VALUES ('ka', named_struct('x', 1.5, 'y', 'ya'));++statement ok+CREATE TABLE union_leaf_d (k VARCHAR, s STRUCT<x DOUBLE, y VARCHAR>) AS VALUES ('kb', named_struct('x', 2.5, 'y', 'yb'));++query RT+SELECT get_field(s, 'x') AS x, get_field(s, 'y') AS y FROM (+ SELECT k2 AS k, s FROM (SELECT k AS k2, s FROM union_leaf_c LIMIT 10)+ UNION ALL+ (SELECT k, s FROM union_leaf_d LIMIT 10)+) ORDER BY x;+----+1.5 ya+2.5 yb++# Select an ordinary column alongside the extracted fields.+query RTT+SELECT get_field(s, 'x') AS x, get_field(s, 'y') AS y, k FROM (+ SELECT k2 AS k, s FROM (SELECT k AS k2, s FROM union_leaf_c LIMIT 10)+ UNION ALL+ (SELECT k, s FROM union_leaf_d LIMIT 10)+) ORDER BY x;+----+1.5 ya ka+2.5 yb kb++# Reverse the union inputs, so the other pushdown path supplies the output names.+query RT+SELECT get_field(s, 'x') AS x, get_field(s, 'y') AS y FROM (+ (SELECT k, s FROM union_leaf_d LIMIT 10)+ UNION ALL+ SELECT k2 AS k, s FROM (SELECT k AS k2, s FROM union_leaf_c LIMIT 10)+) ORDER BY x;+----+1.5 ya+2.5 yb++statement ok+DROP TABLE union_leaf_a;++statement ok+DROP TABLE union_leaf_b;++statement ok+DROP TABLE union_leaf_c;++statement ok+DROP TABLE union_leaf_d;