Apache DataFusion SQL Query Engine
Make PushDownFilter and CommonSubexprEliminate aware of Expr::placement (#20239)
Adrian Garcia Badaracco9333f74f2d70
diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rsindex 5d29892a23..2096c42770 100644--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs@@ -34,7 +34,9 @@ use datafusion_expr::expr::{Alias, ScalarFunction};use datafusion_expr::logical_plan::{Aggregate, Filter, LogicalPlan, Projection, Sort, Window,};-use datafusion_expr::{BinaryExpr, Case, Expr, Operator, SortExpr, col};+use datafusion_expr::{+ BinaryExpr, Case, Expr, ExpressionPlacement, Operator, SortExpr, col,+};const CSE_PREFIX: &str = "__common_expr";@@ -698,6 +700,18 @@ impl CSEController for ExprCSEController<'_> {}fn is_ignored(&self, node: &Expr) -> bool {+ // MoveTowardsLeafNodes expressions (e.g. get_field) are cheap struct+ // field accesses that the ExtractLeafExpressions / PushDownLeafProjections+ // rules deliberately duplicate when needed (one copy for a filter+ // predicate, another for an output column). CSE deduplicating them+ // creates intermediate projections that fight with those rules,+ // causing optimizer instability — ExtractLeafExpressions will undo+ // the dedup, creating an infinite loop that runs until the iteration+ // limit is hit. Skip them.+ if node.placement() == ExpressionPlacement::MoveTowardsLeafNodes {+ return true;+ }+// TODO: remove the next line after `Expr::Wildcard` is removed#[expect(deprecated)]let is_normal_minus_aggregates = matches!(@@ -830,6 +844,7 @@ mod test {use super::*;use crate::assert_optimized_plan_eq_snapshot;use crate::optimizer::OptimizerContext;+ use crate::test::udfs::leaf_udf_expr;use crate::test::*;use datafusion_expr::test::function_stub::{avg, sum};@@ -1831,4 +1846,56 @@ mod test {panic!("dummy - not implemented")}}++ /// Identical MoveTowardsLeafNodes expressions should NOT be deduplicated+ /// by CSE — they are cheap (e.g. struct field access) and the extraction+ /// rules deliberately duplicate them. Deduplicating causes optimizer+ /// instability where one optimizer rule will undo the work of another,+ /// resulting in an infinite optimization loop until the+ /// we hit the max iteration limit and then give up.+ #[test]+ fn test_leaf_expression_not_extracted() -> Result<()> {+ let table_scan = test_table_scan()?;++ let leaf = leaf_udf_expr(col("a"));+ let plan = LogicalPlanBuilder::from(table_scan)+ .project(vec![leaf.clone().alias("c1"), leaf.alias("c2")])?+ .build()?;++ // Plan should be unchanged — no __common_expr introduced+ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: leaf_udf(test.a) AS c1, leaf_udf(test.a) AS c2+ TableScan: test+ "+ )+ }++ /// When a MoveTowardsLeafNodes expression appears as a sub-expression of+ /// a larger expression that IS duplicated, only the outer expression gets+ /// deduplicated; the leaf sub-expression stays inline.+ #[test]+ fn test_leaf_subexpression_not_extracted() -> Result<()> {+ let table_scan = test_table_scan()?;++ // leaf_udf(a) + b appears twice — the outer `+` is a common+ // sub-expression, but leaf_udf(a) by itself is MoveTowardsLeafNodes+ // and should not be extracted separately.+ let common = leaf_udf_expr(col("a")) + col("b");+ let plan = LogicalPlanBuilder::from(table_scan)+ .project(vec![common.clone().alias("c1"), common.alias("c2")])?+ .build()?;++ // The whole `leaf_udf(a) + b` gets deduplicated as __common_expr_1,+ // but leaf_udf(a) alone is NOT pulled out.+ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: __common_expr_1 AS c1, __common_expr_1 AS c2+ Projection: leaf_udf(test.a) + test.b AS __common_expr_1, test.a, test.b, test.c+ TableScan: test+ "+ )+ }}diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex ecd6a89f2a..15bb5db07d 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -45,6 +45,7 @@ use crate::optimizer::ApplyOrder;use crate::simplify_expressions::simplify_predicates;use crate::utils::{has_all_column_refs, is_restrict_null_predicate};use crate::{OptimizerConfig, OptimizerRule};+use datafusion_expr::ExpressionPlacement;/// Optimizer rule for pushing (moving) filter expressions down in a plan so/// they are applied as early as possible.@@ -1295,10 +1296,13 @@ fn rewrite_projection(predicates: Vec<Expr>,mut projection: Projection,) -> Result<(Transformed<LogicalPlan>, Option<Expr>)> {- // A projection is filter-commutable if it do not contain volatile predicates or contain volatile- // predicates that are not used in the filter. However, we should re-writes all predicate expressions.- // collect projection.- let (volatile_map, non_volatile_map): (HashMap<_, _>, HashMap<_, _>) = projection+ // Partition projection expressions into non-pushable vs pushable.+ // Non-pushable expressions are volatile (must not be duplicated) or+ // MoveTowardsLeafNodes (cheap expressions like get_field where re-inlining+ // into a filter causes optimizer instability — ExtractLeafExpressions will+ // undo the push-down, creating an infinite loop that runs until the+ // iteration limit is hit).+ let (non_pushable_map, pushable_map): (HashMap<_, _>, HashMap<_, _>) = projection.schema.iter().zip(projection.expr.iter())@@ -1308,12 +1312,15 @@ fn rewrite_projection((qualified_name(qualifier, field.name()), expr)})- .partition(|(_, value)| value.is_volatile());+ .partition(|(_, value)| {+ value.is_volatile()+ || value.placement() == ExpressionPlacement::MoveTowardsLeafNodes+ });let mut push_predicates = vec![];let mut keep_predicates = vec![];for expr in predicates {- if contain(&expr, &volatile_map) {+ if contain(&expr, &non_pushable_map) {keep_predicates.push(expr);} else {push_predicates.push(expr);@@ -1325,7 +1332,7 @@ fn rewrite_projection(// re-write all filters based on this projection// E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"let new_filter = LogicalPlan::Filter(Filter::try_new(- replace_cols_by_name(expr, &non_volatile_map)?,+ replace_cols_by_name(expr, &pushable_map)?,std::mem::take(&mut projection.input),)?);@@ -1336,7 +1343,10 @@ fn rewrite_projection(conjunction(keep_predicates),))}- None => Ok((Transformed::no(LogicalPlan::Projection(projection)), None)),+ None => Ok((+ Transformed::no(LogicalPlan::Projection(projection)),+ conjunction(keep_predicates),+ )),}}@@ -1446,6 +1456,7 @@ mod tests {use crate::assert_optimized_plan_eq_snapshot;use crate::optimizer::Optimizer;use crate::simplify_expressions::SimplifyExpressions;+ use crate::test::udfs::leaf_udf_expr;use crate::test::*;use datafusion_expr::test::function_stub::sum;use insta::assert_snapshot;@@ -4221,4 +4232,68 @@ mod tests {")}++ /// Test that filters are NOT pushed through MoveTowardsLeafNodes projections.+ /// These are cheap expressions (like get_field) where re-inlining into a filter+ /// has no benefit and causes optimizer instability — ExtractLeafExpressions will+ /// undo the push-down, creating an infinite loop that runs until the iteration+ /// limit is hit.+ #[test]+ fn filter_not_pushed_through_move_towards_leaves_projection() -> Result<()> {+ let table_scan = test_table_scan()?;++ // Create a projection with a MoveTowardsLeafNodes expression+ let proj = LogicalPlanBuilder::from(table_scan)+ .project(vec![+ leaf_udf_expr(col("a")).alias("val"),+ col("b"),+ col("c"),+ ])?+ .build()?;++ // Put a filter on the MoveTowardsLeafNodes column+ let plan = LogicalPlanBuilder::from(proj)+ .filter(col("val").gt(lit(150i64)))?+ .build()?;++ // Filter should NOT be pushed through — val maps to a MoveTowardsLeafNodes expr+ assert_optimized_plan_equal!(+ plan,+ @r"+ Filter: val > Int64(150)+ Projection: leaf_udf(test.a) AS val, test.b, test.c+ TableScan: test+ "+ )+ }++ /// Test mixed predicates: Column predicate pushed, MoveTowardsLeafNodes kept.+ #[test]+ fn filter_mixed_predicates_partial_push() -> Result<()> {+ let table_scan = test_table_scan()?;++ // Create a projection with both MoveTowardsLeafNodes and Column expressions+ let proj = LogicalPlanBuilder::from(table_scan)+ .project(vec![+ leaf_udf_expr(col("a")).alias("val"),+ col("b"),+ col("c"),+ ])?+ .build()?;++ // Filter with both: val > 150 (MoveTowardsLeafNodes) AND b > 5 (Column)+ let plan = LogicalPlanBuilder::from(proj)+ .filter(col("val").gt(lit(150i64)).and(col("b").gt(lit(5i64))))?+ .build()?;++ // val > 150 should be kept above, b > 5 should be pushed through+ assert_optimized_plan_equal!(+ plan,+ @r"+ Filter: val > Int64(150)+ Projection: leaf_udf(test.a) AS val, test.b, test.c+ TableScan: test, full_filters=[test.b > Int64(5)]+ "+ )+ }}diff --git a/datafusion/optimizer/src/test/mod.rs b/datafusion/optimizer/src/test/mod.rsindex a459839504..75dea7c1a3 100644--- a/datafusion/optimizer/src/test/mod.rs+++ b/datafusion/optimizer/src/test/mod.rs@@ -24,6 +24,7 @@ use datafusion_common::{Result, assert_contains};use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, logical_plan::table_scan};use std::sync::Arc;+pub mod udfs;pub mod user_defined;pub fn test_table_scan_fields() -> Vec<Field> {diff --git a/datafusion/optimizer/src/test/udfs.rs b/datafusion/optimizer/src/test/udfs.rsnew file mode 100644index 0000000000..0e68568dec--- /dev/null+++ b/datafusion/optimizer/src/test/udfs.rs@@ -0,0 +1,87 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::any::Any;++use arrow::datatypes::DataType;+use datafusion_common::Result;+use datafusion_expr::{+ ColumnarValue, Expr, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF,+ ScalarUDFImpl, Signature, Volatility,+};++/// A configurable test UDF for optimizer tests.+/// Defaults to `MoveTowardsLeafNodes` placement. Use `with_placement()` to override.+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct PlacementTestUDF {+ signature: Signature,+ placement: ExpressionPlacement,+}++impl Default for PlacementTestUDF {+ fn default() -> Self {+ Self::new()+ }+}++impl PlacementTestUDF {+ pub fn new() -> Self {+ Self {+ signature: Signature::exact(vec![DataType::UInt32], Volatility::Immutable),+ placement: ExpressionPlacement::MoveTowardsLeafNodes,+ }+ }++ pub fn with_placement(mut self, placement: ExpressionPlacement) -> Self {+ self.placement = placement;+ self+ }+}++impl ScalarUDFImpl for PlacementTestUDF {+ fn as_any(&self) -> &dyn Any {+ self+ }+ fn name(&self) -> &str {+ match self.placement {+ ExpressionPlacement::MoveTowardsLeafNodes => "leaf_udf",+ ExpressionPlacement::KeepInPlace => "keep_in_place_udf",+ ExpressionPlacement::Column => "column_udf",+ ExpressionPlacement::Literal => "literal_udf",+ }+ }+ fn signature(&self) -> &Signature {+ &self.signature+ }+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::UInt32)+ }+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ panic!("PlacementTestUDF: not intended for execution")+ }+ fn placement(&self, _args: &[ExpressionPlacement]) -> ExpressionPlacement {+ self.placement+ }+}++/// Create a `leaf_udf(arg)` expression with `MoveTowardsLeafNodes` placement.+pub fn leaf_udf_expr(arg: Expr) -> Expr {+ let udf = ScalarUDF::new_from_impl(+ PlacementTestUDF::new().with_placement(ExpressionPlacement::MoveTowardsLeafNodes),+ );+ udf.call(vec![arg])+}diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex adcd2bb04f..50e26b2fb0 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -856,7 +856,7 @@ EXPLAIN SELECT id, s['value'], s['value'] + 10, s['label'] FROM simple_struct OR----logical_plan01)Sort: simple_struct.id ASC NULLS LAST, fetch=3-02)--Projection: simple_struct.id, get_field(simple_struct.s, Utf8("value")) AS simple_struct.s[value], get_field(simple_struct.s, Utf8("value")) + Int64(10), get_field(simple_struct.s, Utf8("label"))+02)--Projection: simple_struct.id, get_field(simple_struct.s, Utf8("value")), get_field(simple_struct.s, Utf8("value")) + Int64(10), get_field(simple_struct.s, Utf8("label"))03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]