Skip to content

Apache DataFusion SQL Query Engine

fix: preserve nested projection expressions (#24686)

discord97a1f468643f5
diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs
index b0543a871f..9604bf2f18 100644
--- a/datafusion/optimizer/src/optimize_projections/mod.rs
+++ b/datafusion/optimizer/src/optimize_projections/mod.rs
@@ -556,17 +556,6 @@ fn merge_consecutive_projections_one_level(
return Projection::try_new_with_schema(expr, input, schema).map(Transformed::no);
};
- // A fast path: if the previous projection is same as the current projection
- // we can directly remove the current projection and return child projection.
- if prev_projection.expr == expr {
- return Projection::try_new_with_schema(
- expr,
- Arc::clone(&prev_projection.input),
- schema,
- )
- .map(Transformed::yes);
- }
-
// Count usages (referrals) of each projection expression in its input fields:
let mut column_referral_map = HashMap::<&Column, usize>::new();
for expr in &expr {
@@ -1210,6 +1199,69 @@ mod tests {
)
}
+ #[test]
+ fn merge_structurally_equal_non_idempotent_projections() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);
+ let projection = || col("i").add(lit(1)).alias("i");
+ let plan = table_scan(TableReference::none(), &schema, None)?
+ .project(vec![projection()])?
+ .project(vec![projection()])?
+ .build()?;
+
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: ?table?.i + Int32(1) + Int32(1) AS i
+ TableScan: ?table? projection=[i]
+ "
+ )
+ }
+
+ #[test]
+ fn merge_deep_projection_chain_in_one_pass() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]);
+ let mut plan = table_scan(TableReference::none(), &schema, None)?
+ .project(vec![col("i").add(lit(1)).alias("i")])?
+ .build()?;
+ for _ in 1..12 {
+ plan = LogicalPlanBuilder::from(plan)
+ .project(vec![col("i").add(lit(1)).alias("i")])?
+ .build()?;
+ }
+
+ let optimizer = Optimizer::with_rules(vec![Arc::new(OptimizeProjections::new())]);
+ let optimized = optimizer.optimize(
+ plan,
+ &OptimizerContext::new().with_max_passes(1),
+ observe,
+ )?;
+ let plan_string = format!("{optimized}");
+ assert_eq!(12, plan_string.matches("Int32(1)").count());
+ assert_eq!(1, plan_string.matches("Projection:").count());
+ Ok(())
+ }
+
+ #[test]
+ fn merge_columns_and_metadata_alias() -> Result<()> {
+ let metadata =
+ datafusion_common::metadata::FieldMetadata::from(HashMap::from([(
+ "key".to_string(),
+ "value".to_string(),
+ )]));
+ let plan = LogicalPlanBuilder::from(test_table_scan()?)
+ .project(vec![col("a")])?
+ .project(vec![col("a").alias_with_metadata("a", Some(metadata))])?
+ .build()?;
+
+ let optimized = optimize(plan)?;
+ assert_eq!(
+ "value",
+ optimized.schema().field(0).metadata().get("key").unwrap()
+ );
+ assert_eq!(1, format!("{optimized}").matches("Projection:").count());
+ Ok(())
+ }
+
#[test]
fn merge_alias() -> Result<()> {
let table_scan = test_table_scan()?;
diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.slt
index e18114bc51..bac94670f4 100644
--- a/datafusion/sqllogictest/test_files/projection.slt
+++ b/datafusion/sqllogictest/test_files/projection.slt
@@ -217,6 +217,74 @@ SELECT column1 as a from (values (1), (2)) f where f.column1 = 2;
----
2
+# Regression: preserve deep anonymous nested projections with one and default passes.
+statement ok
+CREATE TABLE nested_projection(i INT);
+
+statement ok
+INSERT INTO nested_projection VALUES (3), (4), (5);
+
+statement ok
+SET datafusion.optimizer.max_passes = 1;
+
+query I rowsort
+SELECT i
+FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM nested_projection
+ )
+ )
+ )
+ )
+ )
+)
+----
+10
+11
+9
+
+statement ok
+RESET datafusion.optimizer.max_passes;
+
+query I rowsort
+SELECT i
+FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM (
+ SELECT i + 1 AS i
+ FROM nested_projection
+ )
+ )
+ )
+ )
+ )
+)
+----
+10
+11
+9
+
+statement ok
+DROP TABLE nested_projection;
+
# clean data
statement ok
DROP TABLE aggregate_simple;