Apache DataFusion SQL Query Engine
fix: keep renamed columns intact in leaf projection pushdown (#25478)
Adrian Garcia Badaracco233aeab990a2
diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rsindex 999c396745..eff6e51988 100644--- a/datafusion/optimizer/src/extract_leaf_expressions.rs+++ b/datafusion/optimizer/src/extract_leaf_expressions.rs@@ -764,12 +764,32 @@ fn build_extraction_projection_impl(.filter_map(|e| resolve_against(input_schema, passthrough_column(e)?)).collect();+ // The names the merged projection already produces. The merge keeps+ // every expression of `existing`, so the parent can still read each of+ // these names, and a pass-through column that carries one of them makes+ // the output schema ambiguous.+ let output_names: std::collections::HashSet<&str> = existing+ .schema+ .fields()+ .iter()+ .map(|f| f.name().as_str())+ .collect();+for col in columns_needed {+ // The projection produces this name, so the parent reads it from the+ // projection's own output. Do not resolve the name through the rename+ // map: a rename such as `t.a AS b` is a computed output, and its+ // input column `t.a` beside the output field `a` of a second rename+ // gives an ambiguous schema (issue #25446).+ if output_names.contains(col.name.as_str()) {+ continue;+ }let col_expr = Expr::Column(col.clone());let resolved = replace_cols_by_name(col_expr, &replace_map)?;if let Expr::Column(resolved_col) = &resolved&& let Some(input_col) = resolve_against(input_schema, resolved_col)&& !existing_cols.contains(&input_col)+ && !output_names.contains(input_col.name.as_str()){proj_exprs.push(Expr::Column(input_col));}@@ -2955,14 +2975,11 @@ mod tests {## After PushdownProjection: __datafusion_extracted_1 AS leaf_udf(x,Utf8("a"))Filter: x IS NOT NULL- Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1, test.user+ Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1TableScan: test projection=[user]## Optimized- Projection: __datafusion_extracted_1 AS leaf_udf(x,Utf8("a"))- Filter: x IS NOT NULL- Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1- TableScan: test projection=[user]+ (same as after pushdown)"#)}@@ -2989,14 +3006,11 @@ mod tests {## After PushdownProjection: __datafusion_extracted_1 IS NOT NULL AS leaf_udf(x,Utf8("a")) IS NOT NULLFilter: x IS NOT NULL- Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1, test.user+ Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1TableScan: test projection=[user]## Optimized- Projection: __datafusion_extracted_1 IS NOT NULL AS leaf_udf(x,Utf8("a")) IS NOT NULL- Filter: x IS NOT NULL- Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1- TableScan: test projection=[user]+ (same as after pushdown)"#)}@@ -3018,17 +3032,92 @@ mod tests {## After ExtractionProjection: xFilter: __datafusion_extracted_1 = Utf8("active")- Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1, test.user+ Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1TableScan: test projection=[user]## After Pushdown(same as after extraction)## Optimized- Projection: x- Filter: __datafusion_extracted_1 = Utf8("active")- Projection: test.user AS x, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1- TableScan: test projection=[user]+ (same as after pushdown)+ "#)+ }++ /// A projection that swaps two column names, below a filter and a struct+ /// field read. The merge must not resolve the parent's column references+ /// through the rename map: the input column `test.user` beside the output+ /// field `user` of the other rename gives an ambiguous schema (#25446).+ #[test]+ fn test_extract_above_projection_that_swaps_column_names() -> Result<()> {+ let table_scan = test_table_scan_with_struct()?;+ let plan = LogicalPlanBuilder::from(table_scan)+ .project(vec![col("user").alias("id"), col("id").alias("user")])?+ .filter(col("user").gt(lit(0u32)))?+ .project(vec![col("id"), col("user"), leaf_udf(col("id"), "name")])?+ .build()?;++ assert_stages!(plan, @r#"+ ## Original Plan+ Projection: id, user, leaf_udf(id, Utf8("name"))+ Filter: user > UInt32(0)+ Projection: test.user AS id, test.id AS user+ TableScan: test projection=[id, user]++ ## After Extraction+ (same as original)++ ## After Pushdown+ Projection: id, user, __datafusion_extracted_1 AS leaf_udf(id,Utf8("name"))+ Filter: user > UInt32(0)+ Projection: test.user AS id, test.id AS user, leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1+ TableScan: test projection=[id, user]++ ## Optimized+ (same as after pushdown)+ "#)+ }++ /// A projection that gives a computed column the name of its own input+ /// column. The extraction projection goes below the filter, which puts a+ /// column named `id` under the rename, so the two plans have the same set+ /// of field names. The recovery projection must stay, or the rename is lost+ /// and the query gives wrong results.+ #[test]+ fn test_extract_above_projection_that_redefines_column_name() -> Result<()> {+ let table_scan = test_table_scan_with_struct()?;+ let plan = LogicalPlanBuilder::from(table_scan)+ .filter(col("id").gt(lit(0u32)))?+ .project(vec![(col("id") * lit(10u32)).alias("id"), col("user")])?+ .alias("sub")?+ .project(vec![col("sub.id"), leaf_udf(col("sub.user"), "name")])?+ .build()?;++ assert_stages!(plan, @r#"+ ## Original Plan+ Projection: sub.id, leaf_udf(sub.user, Utf8("name"))+ SubqueryAlias: sub+ Projection: test.id * UInt32(10) AS id, test.user+ Filter: test.id > UInt32(0)+ TableScan: test projection=[id, user]++ ## After Extraction+ (same as original)++ ## After Pushdown+ Projection: sub.id, __datafusion_extracted_1 AS leaf_udf(sub.user,Utf8("name"))+ SubqueryAlias: sub+ Projection: test.id * UInt32(10) AS id, test.user, __datafusion_extracted_1+ Filter: test.id > UInt32(0)+ Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.id, test.user+ TableScan: test projection=[id, user]++ ## Optimized+ Projection: sub.id, __datafusion_extracted_1 AS leaf_udf(sub.user,Utf8("name"))+ SubqueryAlias: sub+ Projection: test.id * UInt32(10) AS id, __datafusion_extracted_1+ Filter: test.id > UInt32(0)+ Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.id+ TableScan: test projection=[id, user]"#)}diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex 2eb91b712f..6e0e1b483b 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -1907,6 +1907,80 @@ from (values (1),(2),(1)) group by 1;{a: 1, z: NULL} 2{a: 2, z: NULL} 1+# A sub-query projection that renames a column to the name of a different+# column of the same input, with a struct field read above it. Leaf projection+# pushdown resolved the parent's column references through the rename map and+# added the renamed input column a second time, which made the output schema+# ambiguous (https://github.com/apache/datafusion/issues/25446).+statement ok+create table rename_swap_struct(a int, b int, s struct<x varchar>) as values (1, 10, {x: 'p'}), (2, 20, {x: 'q'});++# a rename beside a same-name alias, under a filter+query IIT+select b, a, s['x'] from (select rename_swap_struct.a as b, rename_swap_struct.b as a, s from rename_swap_struct) where a > 15 order by b;+----+2 20 q++# the same shape under a limit+query IIT rowsort+select b, a, s['x'] from (select rename_swap_struct.a as b, rename_swap_struct.b as a, s from rename_swap_struct) limit 10;+----+1 10 p+2 20 q++# a swap of two column names+query IIT+select a, c, s['x'] from (select rename_swap_struct.b as a, rename_swap_struct.a as c, s from rename_swap_struct) where a > 0 order by a;+----+10 1 p+20 2 q++# the same shape under an order by+query IIT+select b, a, s['x'] from (select rename_swap_struct.a as b, rename_swap_struct.b as a, s from rename_swap_struct) order by b;+----+1 10 p+2 20 q++# the same shape under a group by+query III+select b, a, count(s['x']) from (select rename_swap_struct.a as b, rename_swap_struct.b as a, s from rename_swap_struct) group by b, a order by b;+----+1 10 1+2 20 1++# the rename stays above the extraction projection, and the struct field is+# still read at the scan+query TT+explain select b, a, s['x'] from (select rename_swap_struct.a as b, rename_swap_struct.b as a, s from rename_swap_struct) where a > 15;+----+logical_plan+01)Projection: rename_swap_struct.a AS b, rename_swap_struct.b AS a, __datafusion_extracted_1 AS rename_swap_struct.s[x]+02)--Filter: rename_swap_struct.b > Int32(15)+03)----Projection: get_field(rename_swap_struct.s, Utf8("x")) AS __datafusion_extracted_1, rename_swap_struct.a, rename_swap_struct.b+04)------TableScan: rename_swap_struct projection=[a, b, s]+physical_plan+01)ProjectionExec: expr=[a@1 as b, b@2 as a, __datafusion_extracted_1@0 as rename_swap_struct.s[x]]+02)--FilterExec: b@2 > 15+03)----ProjectionExec: expr=[get_field(s@2, x) as __datafusion_extracted_1, a@0 as a, b@1 as b]+04)------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+drop table rename_swap_struct;++# The same swap where the struct field name is also a column name of the table.+statement ok+create table rename_swap_struct_field(a int, b int, s struct<b varchar>) as values (1, 10, {b: 'p'}), (2, 20, {b: 'q'});++query IIT+select a, b, s['b'] from (select b as a, a as b, s from rename_swap_struct_field limit 100) order by a;+----+10 1 p+20 2 q++statement ok+drop table rename_swap_struct_field;+# Merging an extraction projection into a projection whose output lost its# qualifier. Eliminating the empty side of the union leaves a projection of# bare column names over a qualified input, and the merge used to add the