Skip to content

Apache DataFusion SQL Query Engine

fix: align Union inputs after leaf expression pushdown (#26421)

Adrian Garcia Badaracco7c918f98546d
diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rs
index eff6e51988..ca12fb0bbb 100644
--- a/datafusion/optimizer/src/extract_leaf_expressions.rs
+++ b/datafusion/optimizer/src/extract_leaf_expressions.rs
@@ -1443,6 +1443,7 @@ fn try_push_into_inputs(
// so if the TopDown pass later pushes a child further (changing its output
// schema), the parent node's schema becomes stale.
let mut new_inputs: Vec<LogicalPlan> = Vec::with_capacity(num_inputs);
+ let is_union = matches!(node, LogicalPlan::Union(_));
for (idx, input) in inputs.into_iter().enumerate() {
if per_input[idx].pairs.is_empty() {
new_inputs.push(input.clone());
@@ -1478,9 +1479,14 @@ fn try_push_into_inputs(
// this input (e.g., through Filter → existing extraction projection).
// This ensures the input's output schema is stable and won't change
// when the TopDown pass later visits children.
- match try_push_input(&proj_plan, alias_generator)? {
- Some(pushed) => new_inputs.push(pushed),
- None => new_inputs.push(proj_plan),
+ let pushed = match try_push_input(&proj_plan, alias_generator)? {
+ Some(pushed) => pushed,
+ None => proj_plan,
+ };
+ if is_union {
+ new_inputs.push(align_union_input(pushed, &per_input[idx])?);
+ } else {
+ new_inputs.push(pushed);
}
}
}
@@ -1501,6 +1507,40 @@ fn try_push_into_inputs(
Ok(Some(new_node))
}
+/// Projects one input of a `Union` to the extracted aliases, in pair order,
+/// followed by the needed columns, in column order.
+///
+/// `Union` matches its inputs by position, not by name. A merge into an
+/// existing projection appends the aliases after the existing expressions,
+/// while a new projection puts them first. Without this, two inputs that take
+/// different paths disagree on the position of each column, and the union
+/// reads the alias of one input from an unrelated column of another input.
+fn align_union_input(
+ input: LogicalPlan,
+ target: &ExtractionTarget,
+) -> Result<LogicalPlan> {
+ let exprs: Vec<Expr> = target
+ .pairs
+ .iter()
+ .map(|(_, alias)| Expr::Column(Column::new_unqualified(alias)))
+ .chain(target.columns.iter().map(|col| Expr::Column(col.clone())))
+ .collect();
+ // Skip the extra projection when the input already has this exact layout.
+ let fields = input.schema().fields();
+ let already_aligned = fields.len() == exprs.len()
+ && target
+ .pairs
+ .iter()
+ .map(|(_, alias)| alias.as_str())
+ .chain(target.columns.iter().map(|col| col.name.as_str()))
+ .zip(fields.iter())
+ .all(|(name, field)| field.name() == name);
+ if already_aligned {
+ return Ok(input);
+ }
+ Projection::try_new(exprs, Arc::new(input)).map(LogicalPlan::Projection)
+}
+
#[cfg(test)]
mod tests {
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index 2d5cb50af5..ed71ad4ea4 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -2560,3 +2560,78 @@ RESET datafusion.optimizer.max_passes;
statement ok
SET datafusion.execution.target_partitions = 4;
+
+#####################
+# Union inputs that take different leaf pushdown paths
+#####################
+
+# A union matches its inputs by position. The first input is a projection over
+# a limit, so the extraction merges into that projection. The second input is
+# a bare limit, so the extraction gets a new projection. Both inputs must still
+# place the extracted alias at the same position.
+
+statement ok
+CREATE TABLE union_leaf_a (s STRUCT<y VARCHAR>, k VARCHAR) AS VALUES (named_struct('y', 'struct-a'), 'column-a');
+
+statement ok
+CREATE TABLE union_leaf_b (s STRUCT<y VARCHAR>, k VARCHAR) AS VALUES (named_struct('y', 'struct-b'), 'column-b');
+
+query T
+SELECT get_field(s, 'y') AS y FROM (
+ SELECT s, k2 AS k FROM (SELECT s, k AS k2 FROM union_leaf_a LIMIT 10)
+ UNION ALL
+ (SELECT s, k FROM union_leaf_b LIMIT 10)
+) ORDER BY y;
+----
+struct-a
+struct-b
+
+statement ok
+CREATE TABLE union_leaf_c (k VARCHAR, s STRUCT<x DOUBLE, y VARCHAR>) AS VALUES ('ka', named_struct('x', 1.5, 'y', 'ya'));
+
+statement ok
+CREATE TABLE union_leaf_d (k VARCHAR, s STRUCT<x DOUBLE, y VARCHAR>) AS VALUES ('kb', named_struct('x', 2.5, 'y', 'yb'));
+
+query RT
+SELECT get_field(s, 'x') AS x, get_field(s, 'y') AS y FROM (
+ SELECT k2 AS k, s FROM (SELECT k AS k2, s FROM union_leaf_c LIMIT 10)
+ UNION ALL
+ (SELECT k, s FROM union_leaf_d LIMIT 10)
+) ORDER BY x;
+----
+1.5 ya
+2.5 yb
+
+# Select an ordinary column alongside the extracted fields.
+query RTT
+SELECT get_field(s, 'x') AS x, get_field(s, 'y') AS y, k FROM (
+ SELECT k2 AS k, s FROM (SELECT k AS k2, s FROM union_leaf_c LIMIT 10)
+ UNION ALL
+ (SELECT k, s FROM union_leaf_d LIMIT 10)
+) ORDER BY x;
+----
+1.5 ya ka
+2.5 yb kb
+
+# Reverse the union inputs, so the other pushdown path supplies the output names.
+query RT
+SELECT get_field(s, 'x') AS x, get_field(s, 'y') AS y FROM (
+ (SELECT k, s FROM union_leaf_d LIMIT 10)
+ UNION ALL
+ SELECT k2 AS k, s FROM (SELECT k AS k2, s FROM union_leaf_c LIMIT 10)
+) ORDER BY x;
+----
+1.5 ya
+2.5 yb
+
+statement ok
+DROP TABLE union_leaf_a;
+
+statement ok
+DROP TABLE union_leaf_b;
+
+statement ok
+DROP TABLE union_leaf_c;
+
+statement ok
+DROP TABLE union_leaf_d;