Apache DataFusion SQL Query Engine
fix: preserve nested projection expressions (#24686)
discord97a1f468643f5
diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex b0543a871f..9604bf2f18 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -556,17 +556,6 @@ fn merge_consecutive_projections_one_level(return Projection::try_new_with_schema(expr, input, schema).map(Transformed::no);};- // A fast path: if the previous projection is same as the current projection- // we can directly remove the current projection and return child projection.- if prev_projection.expr == expr {- return Projection::try_new_with_schema(- expr,- Arc::clone(&prev_projection.input),- schema,- )- .map(Transformed::yes);- }-// Count usages (referrals) of each projection expression in its input fields:let mut column_referral_map = HashMap::<&Column, usize>::new();for expr in &expr {@@ -1210,6 +1199,69 @@ mod tests {)}+ #[test]+ fn merge_structurally_equal_non_idempotent_projections() -> Result<()> {+ let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);+ let projection = || col("i").add(lit(1)).alias("i");+ let plan = table_scan(TableReference::none(), &schema, None)?+ .project(vec![projection()])?+ .project(vec![projection()])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: ?table?.i + Int32(1) + Int32(1) AS i+ TableScan: ?table? projection=[i]+ "+ )+ }++ #[test]+ fn merge_deep_projection_chain_in_one_pass() -> Result<()> {+ let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);+ let mut plan = table_scan(TableReference::none(), &schema, None)?+ .project(vec![col("i").add(lit(1)).alias("i")])?+ .build()?;+ for _ in 1..12 {+ plan = LogicalPlanBuilder::from(plan)+ .project(vec![col("i").add(lit(1)).alias("i")])?+ .build()?;+ }++ let optimizer = Optimizer::with_rules(vec![Arc::new(OptimizeProjections::new())]);+ let optimized = optimizer.optimize(+ plan,+ &OptimizerContext::new().with_max_passes(1),+ observe,+ )?;+ let plan_string = format!("{optimized}");+ assert_eq!(12, plan_string.matches("Int32(1)").count());+ assert_eq!(1, plan_string.matches("Projection:").count());+ Ok(())+ }++ #[test]+ fn merge_columns_and_metadata_alias() -> Result<()> {+ let metadata =+ datafusion_common::metadata::FieldMetadata::from(HashMap::from([(+ "key".to_string(),+ "value".to_string(),+ )]));+ let plan = LogicalPlanBuilder::from(test_table_scan()?)+ .project(vec![col("a")])?+ .project(vec![col("a").alias_with_metadata("a", Some(metadata))])?+ .build()?;++ let optimized = optimize(plan)?;+ assert_eq!(+ "value",+ optimized.schema().field(0).metadata().get("key").unwrap()+ );+ assert_eq!(1, format!("{optimized}").matches("Projection:").count());+ Ok(())+ }+#[test]fn merge_alias() -> Result<()> {let table_scan = test_table_scan()?;diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.sltindex e18114bc51..bac94670f4 100644--- a/datafusion/sqllogictest/test_files/projection.slt+++ b/datafusion/sqllogictest/test_files/projection.slt@@ -217,6 +217,74 @@ SELECT column1 as a from (values (1), (2)) f where f.column1 = 2;----2+# Regression: preserve deep anonymous nested projections with one and default passes.+statement ok+CREATE TABLE nested_projection(i INT);++statement ok+INSERT INTO nested_projection VALUES (3), (4), (5);++statement ok+SET datafusion.optimizer.max_passes = 1;++query I rowsort+SELECT i+FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM nested_projection+ )+ )+ )+ )+ )+)+----+10+11+9++statement ok+RESET datafusion.optimizer.max_passes;++query I rowsort+SELECT i+FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM (+ SELECT i + 1 AS i+ FROM nested_projection+ )+ )+ )+ )+ )+)+----+10+11+9++statement ok+DROP TABLE nested_projection;+# clean datastatement okDROP TABLE aggregate_simple;