Skip to content

Apache DataFusion SQL Query Engine

Make PushDownFilter and CommonSubexprEliminate aware of Expr::placement (#20239)

Adrian Garcia Badaracco9333f74f2d70
diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs
index 5d29892a23..2096c42770 100644
--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs
+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs
@@ -34,7 +34,9 @@ use datafusion_expr::expr::{Alias, ScalarFunction};
use datafusion_expr::logical_plan::{
Aggregate, Filter, LogicalPlan, Projection, Sort, Window,
};
-use datafusion_expr::{BinaryExpr, Case, Expr, Operator, SortExpr, col};
+use datafusion_expr::{
+ BinaryExpr, Case, Expr, ExpressionPlacement, Operator, SortExpr, col,
+};
const CSE_PREFIX: &str = "__common_expr";
@@ -698,6 +700,18 @@ impl CSEController for ExprCSEController<'_> {
}
fn is_ignored(&self, node: &Expr) -> bool {
+ // MoveTowardsLeafNodes expressions (e.g. get_field) are cheap struct
+ // field accesses that the ExtractLeafExpressions / PushDownLeafProjections
+ // rules deliberately duplicate when needed (one copy for a filter
+ // predicate, another for an output column). CSE deduplicating them
+ // creates intermediate projections that fight with those rules,
+ // causing optimizer instability — ExtractLeafExpressions will undo
+ // the dedup, creating an infinite loop that runs until the iteration
+ // limit is hit. Skip them.
+ if node.placement() == ExpressionPlacement::MoveTowardsLeafNodes {
+ return true;
+ }
+
// TODO: remove the next line after `Expr::Wildcard` is removed
#[expect(deprecated)]
let is_normal_minus_aggregates = matches!(
@@ -830,6 +844,7 @@ mod test {
use super::*;
use crate::assert_optimized_plan_eq_snapshot;
use crate::optimizer::OptimizerContext;
+ use crate::test::udfs::leaf_udf_expr;
use crate::test::*;
use datafusion_expr::test::function_stub::{avg, sum};
@@ -1831,4 +1846,56 @@ mod test {
panic!("dummy - not implemented")
}
}
+
+ /// Identical MoveTowardsLeafNodes expressions should NOT be deduplicated
+ /// by CSE — they are cheap (e.g. struct field access) and the extraction
+ /// rules deliberately duplicate them. Deduplicating causes optimizer
+ /// instability where one optimizer rule will undo the work of another,
+ /// resulting in an infinite optimization loop until the
+ /// we hit the max iteration limit and then give up.
+ #[test]
+ fn test_leaf_expression_not_extracted() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ let leaf = leaf_udf_expr(col("a"));
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .project(vec![leaf.clone().alias("c1"), leaf.alias("c2")])?
+ .build()?;
+
+ // Plan should be unchanged — no __common_expr introduced
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: leaf_udf(test.a) AS c1, leaf_udf(test.a) AS c2
+ TableScan: test
+ "
+ )
+ }
+
+ /// When a MoveTowardsLeafNodes expression appears as a sub-expression of
+ /// a larger expression that IS duplicated, only the outer expression gets
+ /// deduplicated; the leaf sub-expression stays inline.
+ #[test]
+ fn test_leaf_subexpression_not_extracted() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ // leaf_udf(a) + b appears twice — the outer `+` is a common
+ // sub-expression, but leaf_udf(a) by itself is MoveTowardsLeafNodes
+ // and should not be extracted separately.
+ let common = leaf_udf_expr(col("a")) + col("b");
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .project(vec![common.clone().alias("c1"), common.alias("c2")])?
+ .build()?;
+
+ // The whole `leaf_udf(a) + b` gets deduplicated as __common_expr_1,
+ // but leaf_udf(a) alone is NOT pulled out.
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: __common_expr_1 AS c1, __common_expr_1 AS c2
+ Projection: leaf_udf(test.a) + test.b AS __common_expr_1, test.a, test.b, test.c
+ TableScan: test
+ "
+ )
+ }
}
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs
index ecd6a89f2a..15bb5db07d 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -45,6 +45,7 @@ use crate::optimizer::ApplyOrder;
use crate::simplify_expressions::simplify_predicates;
use crate::utils::{has_all_column_refs, is_restrict_null_predicate};
use crate::{OptimizerConfig, OptimizerRule};
+use datafusion_expr::ExpressionPlacement;
/// Optimizer rule for pushing (moving) filter expressions down in a plan so
/// they are applied as early as possible.
@@ -1295,10 +1296,13 @@ fn rewrite_projection(
predicates: Vec<Expr>,
mut projection: Projection,
) -> Result<(Transformed<LogicalPlan>, Option<Expr>)> {
- // A projection is filter-commutable if it do not contain volatile predicates or contain volatile
- // predicates that are not used in the filter. However, we should re-writes all predicate expressions.
- // collect projection.
- let (volatile_map, non_volatile_map): (HashMap<_, _>, HashMap<_, _>) = projection
+ // Partition projection expressions into non-pushable vs pushable.
+ // Non-pushable expressions are volatile (must not be duplicated) or
+ // MoveTowardsLeafNodes (cheap expressions like get_field where re-inlining
+ // into a filter causes optimizer instability — ExtractLeafExpressions will
+ // undo the push-down, creating an infinite loop that runs until the
+ // iteration limit is hit).
+ let (non_pushable_map, pushable_map): (HashMap<_, _>, HashMap<_, _>) = projection
.schema
.iter()
.zip(projection.expr.iter())
@@ -1308,12 +1312,15 @@ fn rewrite_projection(
(qualified_name(qualifier, field.name()), expr)
})
- .partition(|(_, value)| value.is_volatile());
+ .partition(|(_, value)| {
+ value.is_volatile()
+ || value.placement() == ExpressionPlacement::MoveTowardsLeafNodes
+ });
let mut push_predicates = vec![];
let mut keep_predicates = vec![];
for expr in predicates {
- if contain(&expr, &volatile_map) {
+ if contain(&expr, &non_pushable_map) {
keep_predicates.push(expr);
} else {
push_predicates.push(expr);
@@ -1325,7 +1332,7 @@ fn rewrite_projection(
// re-write all filters based on this projection
// E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"
let new_filter = LogicalPlan::Filter(Filter::try_new(
- replace_cols_by_name(expr, &non_volatile_map)?,
+ replace_cols_by_name(expr, &pushable_map)?,
std::mem::take(&mut projection.input),
)?);
@@ -1336,7 +1343,10 @@ fn rewrite_projection(
conjunction(keep_predicates),
))
}
- None => Ok((Transformed::no(LogicalPlan::Projection(projection)), None)),
+ None => Ok((
+ Transformed::no(LogicalPlan::Projection(projection)),
+ conjunction(keep_predicates),
+ )),
}
}
@@ -1446,6 +1456,7 @@ mod tests {
use crate::assert_optimized_plan_eq_snapshot;
use crate::optimizer::Optimizer;
use crate::simplify_expressions::SimplifyExpressions;
+ use crate::test::udfs::leaf_udf_expr;
use crate::test::*;
use datafusion_expr::test::function_stub::sum;
use insta::assert_snapshot;
@@ -4221,4 +4232,68 @@ mod tests {
"
)
}
+
+ /// Test that filters are NOT pushed through MoveTowardsLeafNodes projections.
+ /// These are cheap expressions (like get_field) where re-inlining into a filter
+ /// has no benefit and causes optimizer instability — ExtractLeafExpressions will
+ /// undo the push-down, creating an infinite loop that runs until the iteration
+ /// limit is hit.
+ #[test]
+ fn filter_not_pushed_through_move_towards_leaves_projection() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ // Create a projection with a MoveTowardsLeafNodes expression
+ let proj = LogicalPlanBuilder::from(table_scan)
+ .project(vec![
+ leaf_udf_expr(col("a")).alias("val"),
+ col("b"),
+ col("c"),
+ ])?
+ .build()?;
+
+ // Put a filter on the MoveTowardsLeafNodes column
+ let plan = LogicalPlanBuilder::from(proj)
+ .filter(col("val").gt(lit(150i64)))?
+ .build()?;
+
+ // Filter should NOT be pushed through — val maps to a MoveTowardsLeafNodes expr
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Filter: val > Int64(150)
+ Projection: leaf_udf(test.a) AS val, test.b, test.c
+ TableScan: test
+ "
+ )
+ }
+
+ /// Test mixed predicates: Column predicate pushed, MoveTowardsLeafNodes kept.
+ #[test]
+ fn filter_mixed_predicates_partial_push() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ // Create a projection with both MoveTowardsLeafNodes and Column expressions
+ let proj = LogicalPlanBuilder::from(table_scan)
+ .project(vec![
+ leaf_udf_expr(col("a")).alias("val"),
+ col("b"),
+ col("c"),
+ ])?
+ .build()?;
+
+ // Filter with both: val > 150 (MoveTowardsLeafNodes) AND b > 5 (Column)
+ let plan = LogicalPlanBuilder::from(proj)
+ .filter(col("val").gt(lit(150i64)).and(col("b").gt(lit(5i64))))?
+ .build()?;
+
+ // val > 150 should be kept above, b > 5 should be pushed through
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Filter: val > Int64(150)
+ Projection: leaf_udf(test.a) AS val, test.b, test.c
+ TableScan: test, full_filters=[test.b > Int64(5)]
+ "
+ )
+ }
}
diff --git a/datafusion/optimizer/src/test/mod.rs b/datafusion/optimizer/src/test/mod.rs
index a459839504..75dea7c1a3 100644
--- a/datafusion/optimizer/src/test/mod.rs
+++ b/datafusion/optimizer/src/test/mod.rs
@@ -24,6 +24,7 @@ use datafusion_common::{Result, assert_contains};
use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, logical_plan::table_scan};
use std::sync::Arc;
+pub mod udfs;
pub mod user_defined;
pub fn test_table_scan_fields() -> Vec<Field> {
diff --git a/datafusion/optimizer/src/test/udfs.rs b/datafusion/optimizer/src/test/udfs.rs
new file mode 100644
index 0000000000..0e68568dec
--- /dev/null
+++ b/datafusion/optimizer/src/test/udfs.rs
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::any::Any;
+
+use arrow::datatypes::DataType;
+use datafusion_common::Result;
+use datafusion_expr::{
+ ColumnarValue, Expr, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF,
+ ScalarUDFImpl, Signature, Volatility,
+};
+
+/// A configurable test UDF for optimizer tests.
+/// Defaults to `MoveTowardsLeafNodes` placement. Use `with_placement()` to override.
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct PlacementTestUDF {
+ signature: Signature,
+ placement: ExpressionPlacement,
+}
+
+impl Default for PlacementTestUDF {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl PlacementTestUDF {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::exact(vec![DataType::UInt32], Volatility::Immutable),
+ placement: ExpressionPlacement::MoveTowardsLeafNodes,
+ }
+ }
+
+ pub fn with_placement(mut self, placement: ExpressionPlacement) -> Self {
+ self.placement = placement;
+ self
+ }
+}
+
+impl ScalarUDFImpl for PlacementTestUDF {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+ fn name(&self) -> &str {
+ match self.placement {
+ ExpressionPlacement::MoveTowardsLeafNodes => "leaf_udf",
+ ExpressionPlacement::KeepInPlace => "keep_in_place_udf",
+ ExpressionPlacement::Column => "column_udf",
+ ExpressionPlacement::Literal => "literal_udf",
+ }
+ }
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::UInt32)
+ }
+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ panic!("PlacementTestUDF: not intended for execution")
+ }
+ fn placement(&self, _args: &[ExpressionPlacement]) -> ExpressionPlacement {
+ self.placement
+ }
+}
+
+/// Create a `leaf_udf(arg)` expression with `MoveTowardsLeafNodes` placement.
+pub fn leaf_udf_expr(arg: Expr) -> Expr {
+ let udf = ScalarUDF::new_from_impl(
+ PlacementTestUDF::new().with_placement(ExpressionPlacement::MoveTowardsLeafNodes),
+ );
+ udf.call(vec![arg])
+}
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index adcd2bb04f..50e26b2fb0 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -856,7 +856,7 @@ EXPLAIN SELECT id, s['value'], s['value'] + 10, s['label'] FROM simple_struct OR
----
logical_plan
01)Sort: simple_struct.id ASC NULLS LAST, fetch=3
-02)--Projection: simple_struct.id, get_field(simple_struct.s, Utf8("value")) AS simple_struct.s[value], get_field(simple_struct.s, Utf8("value")) + Int64(10), get_field(simple_struct.s, Utf8("label"))
+02)--Projection: simple_struct.id, get_field(simple_struct.s, Utf8("value")), get_field(simple_struct.s, Utf8("value")) + Int64(10), get_field(simple_struct.s, Utf8("label"))
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]