Apache DataFusion SQL Query Engine
PushdownFilter optimizations (#21668)
Georgi Krastevd31832448135
diff --git a/datafusion-testing b/datafusion-testingindex 7833a65d5b..13bbae3877 160000--- a/datafusion-testing+++ b/datafusion-testing@@ -1 +1 @@-Subproject commit 7833a65d5b08be2ca484ea938f471cf01df54e18+Subproject commit 13bbae38776c2bfbc1fab1be7e7220222d4284bfdiff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex c572b202f0..2f1061c438 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -2495,6 +2495,19 @@ pub struct Filter {}impl Filter {+ /// Create a new filter operator.+ ///+ /// Skips the type-checking and dealiasing done in [Self::try_new].+ /// For internal use in DataFusion only.+ ///+ /// **Preconditions:**+ /// - the `predicate` expression returns a boolean value+ /// - the `predicate` expression is not aliased+ #[doc(hidden)]+ pub fn new(predicate: Expr, input: Arc<LogicalPlan>) -> Self {+ Self { predicate, input }+ }+/// Create a new filter operator.////// Notes: as Aliases have no effect on the output of a filter operator,diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex 46e129ad4b..9c2ac07ff0 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -30,17 +30,18 @@ use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,};use datafusion_common::{- Column, DFSchema, Result, assert_eq_or_internal_err, assert_or_internal_err,- internal_err, plan_err, qualified_name,+ Column, DFSchema, Result, assert_eq_or_internal_err, internal_err, plan_err,+ qualified_name,};use datafusion_expr::expr::WindowFunction;use datafusion_expr::expr_rewriter::replace_col;-use datafusion_expr::logical_plan::{Join, JoinType, LogicalPlan, TableScan, Union};+use datafusion_expr::logical_plan::{Join, JoinType, LogicalPlan};use datafusion_expr::utils::{conjunction, expr_to_columns, split_conjunction, split_conjunction_owned,};use datafusion_expr::{- BinaryExpr, Expr, Filter, Operator, Projection, TableProviderFilterPushDown, and, or,+ BinaryExpr, Distinct, Expr, Filter, Operator, Projection,+ TableProviderFilterPushDown, and, or,};use crate::optimizer::ApplyOrder;@@ -447,16 +448,13 @@ fn push_down_all_join(let mut on_filter_join_conditions = vec![];let (on_left_preserved, on_right_preserved) = on_lr_is_preserved(join.join_type);-- if !on_filter.is_empty() {- for on in on_filter {- if on_left_preserved && checker.is_left_only(&on) {- left_push.push(on)- } else if on_right_preserved && checker.is_right_only(&on) {- right_push.push(on)- } else {- on_filter_join_conditions.push(on)- }+ for on in on_filter {+ if on_left_preserved && checker.is_left_only(&on) {+ left_push.push(on)+ } else if on_right_preserved && checker.is_right_only(&on) {+ right_push.push(on)+ } else {+ on_filter_join_conditions.push(on)}}@@ -498,41 +496,46 @@ fn push_down_all_join());}+ // Add any new join conditions as the non join predicates+ let join_conditions_empty = join_conditions.is_empty();+ join_conditions.extend(on_filter_join_conditions);+ join.filter = conjunction(join_conditions);++ if join_conditions_empty && left_push.is_empty() && right_push.is_empty() {+ // wrap the join on the filter whose predicates must be kept, if any+ return Ok(Transformed::no(with_filters(+ keep_predicates,+ LogicalPlan::Join(join),+ )));+ }+if let Some(predicate) = conjunction(left_push) {- join.left = Arc::new(LogicalPlan::Filter(Filter::try_new(predicate, join.left)?));+ join.left = Arc::new(LogicalPlan::Filter(Filter::new(predicate, join.left)));}+if let Some(predicate) = conjunction(right_push) {- join.right =- Arc::new(LogicalPlan::Filter(Filter::try_new(predicate, join.right)?));+ join.right = Arc::new(LogicalPlan::Filter(Filter::new(predicate, join.right)));}- // Add any new join conditions as the non join predicates- join_conditions.extend(on_filter_join_conditions);- join.filter = conjunction(join_conditions);-// wrap the join on the filter whose predicates must be kept, if any- let plan = LogicalPlan::Join(join);- let plan = if let Some(predicate) = conjunction(keep_predicates) {- LogicalPlan::Filter(Filter::try_new(predicate, Arc::new(plan))?)- } else {- plan- };- Ok(Transformed::yes(plan))+ Ok(Transformed::yes(with_filters(+ keep_predicates,+ LogicalPlan::Join(join),+ )))}fn push_down_join(- join: Join,- parent_predicate: Option<&Expr>,+ mut join: Join,+ parent_predicate: Option<Expr>,) -> Result<Transformed<LogicalPlan>> {// Split the parent predicate into individual conjunctive parts.- let predicates = parent_predicate- .map_or_else(Vec::new, |pred| split_conjunction_owned(pred.clone()));+ let predicates = parent_predicate.map_or_else(Vec::new, split_conjunction_owned);// Extract conjunctions from the JOIN's ON filter, if present.let on_filters = join.filter- .as_ref()- .map_or_else(Vec::new, |filter| split_conjunction_owned(filter.clone()));+ .take()+ .map_or_else(Vec::new, split_conjunction_owned);// Are there any new join predicates that can be inferred from the filter expressions?let inferred_join_predicates = with_debug_timing("infer_join_predicates", || {@@ -773,13 +776,11 @@ impl OptimizerRule for PushDownFilter {plan: LogicalPlan,config: &dyn OptimizerConfig,) -> Result<Transformed<LogicalPlan>> {- let _ = config.options();+ let _ = config;if let LogicalPlan::Join(join) = plan {return push_down_join(join, None);};- let plan_schema = Arc::clone(plan.schema());-let LogicalPlan::Filter(mut filter) = plan else {return Ok(Transformed::no(plan));};@@ -812,44 +813,48 @@ impl OptimizerRule for PushDownFilter {}match Arc::unwrap_or_clone(filter.input) {- LogicalPlan::Filter(child_filter) => {- // child filters first to preserve execution order- let new_predicates = split_conjunction_owned(child_filter.predicate)- .into_iter()- .chain(split_conjunction_owned(filter.predicate))- // use IndexSet to remove duplicates while preserving predicate order- .collect::<IndexSet<_>>();+ LogicalPlan::Filter(mut child_filter) => {+ // Child filters first to preserve execution order.+ // Use IndexSet to remove duplicates while preserving predicate order.+ let new_predicates: IndexSet<Expr> =+ split_conjunction_owned(child_filter.predicate)+ .into_iter()+ .chain(split_conjunction_owned(filter.predicate))+ .collect();let Some(new_predicate) = conjunction(new_predicates) else {return plan_err!("at least one expression exists");};- let new_filter = LogicalPlan::Filter(Filter::try_new(- new_predicate,- child_filter.input,- )?);-- self.rewrite(new_filter, config)+ child_filter.predicate = new_predicate;+ self.rewrite(LogicalPlan::Filter(child_filter), config)}- LogicalPlan::Repartition(repartition) => {- let new_filter =- Filter::try_new(filter.predicate, Arc::clone(&repartition.input))- .map(LogicalPlan::Filter)?;- insert_below(LogicalPlan::Repartition(repartition), new_filter)+ LogicalPlan::Repartition(mut repartition) => {+ filter.input = repartition.input;+ repartition.input = Arc::new(LogicalPlan::Filter(filter));+ Ok(Transformed::yes(LogicalPlan::Repartition(repartition)))}LogicalPlan::Distinct(distinct) => {- let new_filter =- Filter::try_new(filter.predicate, Arc::clone(distinct.input()))- .map(LogicalPlan::Filter)?;- insert_below(LogicalPlan::Distinct(distinct), new_filter)+ let distinct = match distinct {+ Distinct::All(input) => {+ filter.input = input;+ Distinct::All(Arc::new(LogicalPlan::Filter(filter)))+ }+ Distinct::On(mut distinct) => {+ filter.input = distinct.input;+ distinct.input = Arc::new(LogicalPlan::Filter(filter));+ Distinct::On(distinct)+ }+ };++ Ok(Transformed::yes(LogicalPlan::Distinct(distinct)))}- LogicalPlan::Sort(sort) => {- let new_filter =- Filter::try_new(filter.predicate, Arc::clone(&sort.input))- .map(LogicalPlan::Filter)?;- insert_below(LogicalPlan::Sort(sort), new_filter)+ LogicalPlan::Sort(mut sort) => {+ filter.input = sort.input;+ sort.input = Arc::new(LogicalPlan::Filter(filter));+ Ok(Transformed::yes(LogicalPlan::Sort(sort)))}- LogicalPlan::SubqueryAlias(subquery_alias) => {+ LogicalPlan::SubqueryAlias(mut subquery_alias) => {let mut replace_map = HashMap::new();for (i, (qualifier, field)) insubquery_alias.input.schema().iter().enumerate()@@ -861,30 +866,24 @@ impl OptimizerRule for PushDownFilter {Expr::Column(Column::new(qualifier.cloned(), field.name())),);}- let new_predicate = replace_cols_by_name(filter.predicate, &replace_map)?;- let new_filter = LogicalPlan::Filter(Filter::try_new(- new_predicate,- Arc::clone(&subquery_alias.input),- )?);- insert_below(LogicalPlan::SubqueryAlias(subquery_alias), new_filter)+ filter.predicate = replace_cols_by_name(filter.predicate, &replace_map)?;+ filter.input = subquery_alias.input;+ subquery_alias.input = Arc::new(LogicalPlan::Filter(filter));+ Ok(Transformed::yes(LogicalPlan::SubqueryAlias(subquery_alias)))}LogicalPlan::Projection(projection) => {let predicates = split_conjunction_owned(filter.predicate.clone());- let (new_projection, keep_predicate) =+ let (mut result, keep_predicates) =rewrite_projection(predicates, projection)?;- if new_projection.transformed {- match keep_predicate {- None => Ok(new_projection),- Some(keep_predicate) => new_projection.map_data(|child_plan| {- Filter::try_new(keep_predicate, Arc::new(child_plan))- .map(LogicalPlan::Filter)- }),- }+ if result.transformed {+ result.data = with_filters(keep_predicates, result.data)} else {- filter.input = Arc::new(new_projection.data);- Ok(Transformed::no(LogicalPlan::Filter(filter)))+ filter.input = Arc::new(result.data);+ result.data = LogicalPlan::Filter(filter)}++ Ok(result)}LogicalPlan::Unnest(mut unnest) => {let predicates = split_conjunction_owned(filter.predicate.clone());@@ -895,11 +894,10 @@ impl OptimizerRule for PushDownFilter {for idx in &unnest.struct_type_columns {let (sub_qualifier, field) =unnest.input.schema().qualified_field(*idx);- let field_name = field.name().clone();-if let DataType::Struct(children) = field.data_type() {+ let field_name = field.name();for child in children {- let child_name = child.name().clone();+ let child_name = child.name();unnest_struct_columns.push(Column::new(sub_qualifier.cloned(),format!("{field_name}.{child_name}"),@@ -942,29 +940,21 @@ impl OptimizerRule for PushDownFilter {// Filter// Unnest Input (Projection)- let unnest_input = std::mem::take(&mut unnest.input);-- let filter_with_unnest_input = LogicalPlan::Filter(Filter::try_new(- conjunction(non_unnest_predicates).unwrap(), // Safe to unwrap since non_unnest_predicates is not empty.- unnest_input,- )?);-+ // Safe to unwrap since non_unnest_predicates is not empty.+ filter.predicate = conjunction(non_unnest_predicates).unwrap();+ filter.input = unnest.input;// Directly assign new filter plan as the new unnest's input.// The new filter plan will go through another rewrite pass since the rule itself// is applied recursively to all the child from top to down- let unnest_plan =- insert_below(LogicalPlan::Unnest(unnest), filter_with_unnest_input)?;-- match conjunction(unnest_predicates) {- None => Ok(unnest_plan),- Some(predicate) => Ok(Transformed::yes(LogicalPlan::Filter(- Filter::try_new(predicate, Arc::new(unnest_plan.data))?,- ))),- }+ unnest.input = Arc::new(LogicalPlan::Filter(filter));+ Ok(Transformed::yes(with_filters(+ unnest_predicates,+ LogicalPlan::Unnest(unnest),+ )))}- LogicalPlan::Union(ref union) => {+ LogicalPlan::Union(mut union) => {let mut inputs = Vec::with_capacity(union.inputs.len());- for input in &union.inputs {+ for input in union.inputs {let mut replace_map = HashMap::new();for (i, (qualifier, field)) in input.schema().iter().enumerate() {let (union_qualifier, union_field) =@@ -977,72 +967,51 @@ impl OptimizerRule for PushDownFilter {let push_predicate =replace_cols_by_name(filter.predicate.clone(), &replace_map)?;- inputs.push(Arc::new(LogicalPlan::Filter(Filter::try_new(+ inputs.push(Arc::new(LogicalPlan::Filter(Filter::new(push_predicate,- Arc::clone(input),- )?)))+ input,+ ))))}- Ok(Transformed::yes(LogicalPlan::Union(Union {- inputs,- schema: Arc::clone(&plan_schema),- })))++ union.inputs = inputs;+ Ok(Transformed::yes(LogicalPlan::Union(union)))}- LogicalPlan::Aggregate(agg) => {+ LogicalPlan::Aggregate(mut agg) => {// We can push down Predicate which in groupby_expr.- let group_expr_columns = agg- .group_expr- .iter()- .map(|e| {- let (relation, name) = e.qualified_name();- Column::new(relation, name)- })- .collect::<HashSet<_>>();+ let group_expr_columns = expr_columns(&agg.group_expr);- let predicates = split_conjunction_owned(filter.predicate);+ // As for plan Filter: Column(a+b) > 0 -- Agg: groupby:[Column(a)+Column(b)]+ // After push, we need to replace `a+b` with Column(a)+Column(b)+ // So we need create a replace_map, add {`a+b` --> Expr(Column(a)+Column(b))}+ let mut replace_map = HashMap::new();+ for expr in &agg.group_expr {+ replace_map.insert(expr.schema_name().to_string(), unalias(expr));+ }+ let predicates = split_conjunction_owned(filter.predicate);let mut keep_predicates = vec![];let mut push_predicates = vec![];for expr in predicates {let cols = expr.column_refs();if cols.iter().all(|c| group_expr_columns.contains(c)) {- push_predicates.push(expr);+ push_predicates.push(replace_cols_by_name(expr, &replace_map)?);} else {keep_predicates.push(expr);}}- // As for plan Filter: Column(a+b) > 0 -- Agg: groupby:[Column(a)+Column(b)]- // After push, we need to replace `a+b` with Column(a)+Column(b)- // So we need create a replace_map, add {`a+b` --> Expr(Column(a)+Column(b))}- let mut replace_map = HashMap::new();- for expr in &agg.group_expr {- replace_map.insert(expr.schema_name().to_string(), expr.clone());- }- let replaced_push_predicates = push_predicates- .into_iter()- .map(|expr| replace_cols_by_name(expr, &replace_map))- .collect::<Result<Vec<_>>>()?;-- let agg_input = Arc::clone(&agg.input);- Transformed::yes(LogicalPlan::Aggregate(agg))- .transform_data(|new_plan| {- // If we have a filter to push, we push it down to the input of the aggregate- if let Some(predicate) = conjunction(replaced_push_predicates) {- let new_filter = make_filter(predicate, agg_input)?;- insert_below(new_plan, new_filter)- } else {- Ok(Transformed::no(new_plan))- }- })?- .map_data(|child_plan| {- // if there are any remaining predicates we can't push, add them- // back as a filter- if let Some(predicate) = conjunction(keep_predicates) {- make_filter(predicate, Arc::new(child_plan))- } else {- Ok(child_plan)- }- })+ // If we have a filter to push, we push it down to the input of the aggregate+ let result = if let Some(predicate) = conjunction(push_predicates) {+ filter.predicate = predicate;+ filter.input = agg.input;+ agg.input = Arc::new(LogicalPlan::Filter(filter));+ Transformed::yes(LogicalPlan::Aggregate(agg))+ } else {+ Transformed::no(LogicalPlan::Aggregate(agg))+ };++ // If there are any remaining predicates we can't push, add them back as a filter+ result.map_data(|plan| Ok(with_filters(keep_predicates, plan)))}// Tries to push filters based on the partition key(s) of the window function(s) used.// Example:@@ -1054,22 +1023,16 @@ impl OptimizerRule for PushDownFilter {// Filter: (b > 1) and (c > 1)// Window: func() PARTITION BY [a] ...// Filter: (a > 1)- LogicalPlan::Window(window) => {+ LogicalPlan::Window(mut window) => {// Retrieve the set of potential partition keys where we can push filters by.// Unlike aggregations, where there is only one statement per SELECT, there can be// multiple window functions, each with potentially different partition keys.// Therefore, we need to ensure that any potential partition key returned is used in// ALL window functions. Otherwise, filters cannot be pushed by through that column.- let extract_partition_keys = |func: &WindowFunction| {- func.params- .partition_by- .iter()- .map(|c| {- let (relation, name) = c.qualified_name();- Column::new(relation, name)- })- .collect::<HashSet<_>>()- };+ fn extract_partition_keys(func: &WindowFunction) -> HashSet<Column> {+ expr_columns(&func.params.partition_by)+ }+let potential_partition_keys = window.window_expr.iter()@@ -1119,31 +1082,22 @@ impl OptimizerRule for PushDownFilter {// place, so we can use `push_predicates` directly. This is consistent with other// optimizers, such as the one used by Postgres.- let window_input = Arc::clone(&window.input);- Transformed::yes(LogicalPlan::Window(window))- .transform_data(|new_plan| {- // If we have a filter to push, we push it down to the input of the window- if let Some(predicate) = conjunction(push_predicates) {- let new_filter = make_filter(predicate, window_input)?;- insert_below(new_plan, new_filter)- } else {- Ok(Transformed::no(new_plan))- }- })?- .map_data(|child_plan| {- // if there are any remaining predicates we can't push, add them- // back as a filter- if let Some(predicate) = conjunction(keep_predicates) {- make_filter(predicate, Arc::new(child_plan))- } else {- Ok(child_plan)- }- })+ // If we have a filter to push, we push it down to the input of the aggregate+ let result = if let Some(predicate) = conjunction(push_predicates) {+ filter.predicate = predicate;+ filter.input = window.input;+ window.input = Arc::new(LogicalPlan::Filter(filter));+ Transformed::yes(LogicalPlan::Window(window))+ } else {+ Transformed::no(LogicalPlan::Window(window))+ };++ // If there are any remaining predicates we can't push, add them back as a filter+ result.map_data(|plan| Ok(with_filters(keep_predicates, plan)))}- LogicalPlan::Join(join) => push_down_join(join, Some(&filter.predicate)),- LogicalPlan::TableScan(scan) => {+ LogicalPlan::Join(join) => push_down_join(join, Some(filter.predicate)),+ LogicalPlan::TableScan(mut scan) => {let filter_predicates = split_conjunction(&filter.predicate);-// Filters containing scalar subqueries cannot be pushed to// providers because the subquery result is not available// until execution time.@@ -1169,13 +1123,21 @@ impl OptimizerRule for PushDownFilter {non_volatile_filters.len());+ if supported_filters+ .iter()+ .all(|res| res == &TableProviderFilterPushDown::Unsupported)+ {+ filter.input = Arc::new(LogicalPlan::TableScan(scan));+ return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ }+// Compose scan filters from non-volatile filters of `Exact` or `Inexact` pushdown type- let zip = non_volatile_filters.into_iter().zip(supported_filters);+ let zip = non_volatile_filters.iter().zip(supported_filters.iter());let new_scan_filters = zip.clone()- .filter(|(_, res)| res != &TableProviderFilterPushDown::Unsupported)- .map(|(pred, _)| pred);+ .filter(|(_, res)| *res != &TableProviderFilterPushDown::Unsupported)+ .map(|(&pred, _)| pred);// Add new scan filterslet new_scan_filters: Vec<Expr> = scan@@ -1186,28 +1148,31 @@ impl OptimizerRule for PushDownFilter {.cloned().collect();+ if supported_filters+ .iter()+ .all(|res| res == &TableProviderFilterPushDown::Inexact)+ && scan.filters == new_scan_filters+ {+ filter.input = Arc::new(LogicalPlan::TableScan(scan));+ return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ } else {+ scan.filters = new_scan_filters;+ }+// Compose predicates to be of `Unsupported` or `Inexact` pushdown type,// and also include volatile and subquery-containing filterslet new_predicate: Vec<Expr> = zip- .filter(|(_, res)| res != &TableProviderFilterPushDown::Exact)- .map(|(pred, _)| pred)+ .filter(|(_, res)| *res != &TableProviderFilterPushDown::Exact)+ .map(|(&pred, _)| pred).chain(volatile_filters).chain(subquery_filters).cloned().collect();- let new_scan = LogicalPlan::TableScan(TableScan {- filters: new_scan_filters,- ..scan- });-- Transformed::yes(new_scan).transform_data(|new_scan| {- if let Some(predicate) = conjunction(new_predicate) {- make_filter(predicate, Arc::new(new_scan)).map(Transformed::yes)- } else {- Ok(Transformed::no(new_scan))- }- })+ Ok(Transformed::yes(with_filters(+ new_predicate,+ LogicalPlan::TableScan(scan),+ )))}LogicalPlan::Extension(extension_plan) => {// This check prevents the Filter from being removed when the extension node has no children,@@ -1222,17 +1187,16 @@ impl OptimizerRule for PushDownFilter {// determine if we can push any predicates down past the extension node// each element is true for push, false to keep- let predicate_push_or_keep = split_conjunction(&filter.predicate)- .iter()- .map(|expr| {- let cols = expr.column_refs();- if cols.iter().any(|c| prevent_cols.contains(&c.name)) {- Ok(false) // No push (keep)- } else {- Ok(true) // push- }- })- .collect::<Result<Vec<_>>>()?;+ let predicate_push_or_keep: Vec<bool> =+ split_conjunction(&filter.predicate)+ .iter()+ .map(|expr| {+ !expr+ .column_refs()+ .iter()+ .any(|c| prevent_cols.contains(&c.name))+ })+ .collect();// all predicates are kept, no changes neededif predicate_push_or_keep.iter().all(|&x| !x) {@@ -1254,33 +1218,25 @@ impl OptimizerRule for PushDownFilter {}}- let new_children = match conjunction(push_predicates) {- Some(predicate) => extension_plan- .node- .inputs()- .into_iter()- .map(|child| {- Ok(LogicalPlan::Filter(Filter::try_new(- predicate.clone(),- Arc::new(child.clone()),- )?))- })- .collect::<Result<Vec<_>>>()?,- None => extension_plan.node.inputs().into_iter().cloned().collect(),- };+ // Unwrap - push_predicates is not empty, predicate_push_or_keep checked.+ let predicate = conjunction(push_predicates).unwrap();+ let new_children = extension_plan+ .node+ .inputs()+ .into_iter()+ .map(|child| {+ LogicalPlan::Filter(Filter::new(+ predicate.clone(),+ Arc::new(child.clone()),+ ))+ })+ .collect();+// extension with new inputs.- let child_plan = LogicalPlan::Extension(extension_plan);- let new_extension =- child_plan.with_new_exprs(child_plan.expressions(), new_children)?;-- let new_plan = match conjunction(keep_predicates) {- Some(predicate) => LogicalPlan::Filter(Filter::try_new(- predicate,- Arc::new(new_extension),- )?),- None => new_extension,- };- Ok(Transformed::yes(new_plan))+ let extension = LogicalPlan::Extension(extension_plan);+ let new_plan =+ extension.with_new_exprs(extension.expressions(), new_children)?;+ Ok(Transformed::yes(with_filters(keep_predicates, new_plan)))}child => {filter.input = Arc::new(child);@@ -1320,22 +1276,19 @@ impl OptimizerRule for PushDownFilter {fn rewrite_projection(predicates: Vec<Expr>,mut projection: Projection,-) -> Result<(Transformed<LogicalPlan>, Option<Expr>)> {+) -> Result<(Transformed<LogicalPlan>, Vec<Expr>)> {// Partition projection expressions into non-pushable vs pushable.// Non-pushable expressions are volatile (must not be duplicated) or// MoveTowardsLeafNodes (cheap expressions like get_field where re-inlining// into a filter causes optimizer instability — ExtractLeafExpressions will// undo the push-down, creating an infinite loop that runs until the// iteration limit is hit).- let (non_pushable_map, pushable_map): (HashMap<_, _>, HashMap<_, _>) = projection+ let (non_pushable_map, pushable_map) = projection.schema.iter().zip(projection.expr.iter()).map(|((qualifier, field), expr)| {- // strip alias, as they should not be part of filters- let expr = expr.clone().unalias();-- (qualified_name(qualifier, field.name()), expr)+ (qualified_name(qualifier, field.name()), unalias(expr))}).partition(|(_, value)| {value.is_volatile()@@ -1352,67 +1305,30 @@ fn rewrite_projection(}}- match conjunction(push_predicates) {- Some(expr) => {- // re-write all filters based on this projection- // E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"- let new_filter = LogicalPlan::Filter(Filter::try_new(- replace_cols_by_name(expr, &pushable_map)?,- std::mem::take(&mut projection.input),- )?);-- projection.input = Arc::new(new_filter);-- Ok((- Transformed::yes(LogicalPlan::Projection(projection)),- conjunction(keep_predicates),- ))- }- None => Ok((- Transformed::no(LogicalPlan::Projection(projection)),- conjunction(keep_predicates),- )),- }+ let projection = if let Some(expr) = conjunction(push_predicates) {+ // re-write all filters based on this projection+ // E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"+ projection.input = Arc::new(LogicalPlan::Filter(Filter::new(+ replace_cols_by_name(expr, &pushable_map)?,+ projection.input,+ )));++ Transformed::yes(LogicalPlan::Projection(projection))+ } else {+ Transformed::no(LogicalPlan::Projection(projection))+ };++ Ok((projection, keep_predicates))}/// Creates a new LogicalPlan::Filter node.+///+/// Deprecated: use [`Filter::try_new`] directly.+#[deprecated]pub fn make_filter(predicate: Expr, input: Arc<LogicalPlan>) -> Result<LogicalPlan> {Filter::try_new(predicate, input).map(LogicalPlan::Filter)}-/// Replace the existing child of the single input node with `new_child`.-///-/// Starting:-/// ```text-/// plan-/// child-/// ```-///-/// Ending:-/// ```text-/// plan-/// new_child-/// ```-fn insert_below(- plan: LogicalPlan,- new_child: LogicalPlan,-) -> Result<Transformed<LogicalPlan>> {- let mut new_child = Some(new_child);- let transformed_plan = plan.map_children(|_child| {- if let Some(new_child) = new_child.take() {- Ok(Transformed::yes(new_child))- } else {- // already took the new child- internal_err!("node had more than one input")- }- })?;-- // make sure we did the actual replacement- assert_or_internal_err!(new_child.is_none(), "node had no inputs");-- Ok(transformed_plan)-}-impl PushDownFilter {#[expect(missing_docs)]pub fn new() -> Self {@@ -1439,41 +1355,64 @@ where/// replaces columns by its name on the projection.pub fn replace_cols_by_name(e: Expr,- replace_map: &HashMap<String, Expr>,+ replace_map: &HashMap<String, impl AsRef<Expr>>,) -> Result<Expr> {e.transform_up(|expr| {- Ok(if let Expr::Column(c) = &expr {- match replace_map.get(&c.flat_name()) {- Some(new_c) => Transformed::yes(new_c.clone()),- None => Transformed::no(expr),- }+ if let Expr::Column(c) = &expr+ && let Some(new_expr) = replace_map.get(&c.flat_name())+ {+ Ok(Transformed::yes(new_expr.as_ref().clone()))} else {- Transformed::no(expr)- })+ Ok(Transformed::no(expr))+ }}).data()}+/// Unalias expression reference.+fn unalias(expr: &Expr) -> &Expr {+ if let Expr::Alias(alias) = expr {+ unalias(&alias.expr)+ } else {+ expr+ }+}+/// check whether the expression uses the columns in `check_map`.-fn contain(e: &Expr, check_map: &HashMap<String, Expr>) -> bool {+fn contain<T>(e: &Expr, check_map: &HashMap<String, T>) -> bool {let mut is_contain = false;e.apply(|expr| {- Ok(if let Expr::Column(c) = &expr {- match check_map.get(&c.flat_name()) {- Some(_) => {- is_contain = true;- TreeNodeRecursion::Stop- }- None => TreeNodeRecursion::Continue,- }+ if let Expr::Column(c) = &expr+ && check_map.contains_key(&c.flat_name())+ {+ is_contain = true;+ Ok(TreeNodeRecursion::Stop)} else {- TreeNodeRecursion::Continue- })+ Ok(TreeNodeRecursion::Continue)+ }}).unwrap();is_contain}+fn with_filters(predicates: Vec<Expr>, plan: LogicalPlan) -> LogicalPlan {+ if let Some(predicate) = conjunction(predicates) {+ LogicalPlan::Filter(Filter::new(predicate, Arc::new(plan)))+ } else {+ plan+ }+}++fn expr_columns(exprs: &[Expr]) -> HashSet<Column> {+ exprs+ .iter()+ .map(|expr| {+ let (relation, name) = expr.qualified_name();+ Column::new(relation, name)+ })+ .collect()+}+#[cfg(test)]mod tests {use std::cmp::Ordering;@@ -1487,9 +1426,9 @@ mod tests {use datafusion_expr::logical_plan::table_scan;use datafusion_expr::{ColumnarValue, ExprFunctionExt, Extension, LogicalPlanBuilder,- ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TableSource, TableType,- UserDefinedLogicalNodeCore, Volatility, WindowFunctionDefinition, col, in_list,- in_subquery, lit,+ ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TableScan, TableSource,+ TableType, UserDefinedLogicalNodeCore, Volatility, WindowFunctionDefinition, col,+ in_list, in_subquery, lit,};use crate::OptimizerContext;@@ -1536,6 +1475,17 @@ mod tests {}};}+ /// For testing that we don't return [Transformed::yes] when not necessary,+ /// as it triggers rebuilding parent plan nodes.+ macro_rules! assert_plan_not_transformed {+ ($plan:expr) => {{+ let transformed = PushDownFilter::new()+ .rewrite($plan, &OptimizerContext::new())+ .expect("failed to optimize plan");+ assert!(!transformed.transformed);+ }};+ }+#[test]fn filter_before_projection() -> Result<()> {let table_scan = test_table_scan()?;@@ -1684,6 +1634,8 @@ mod tests {.aggregate(vec![col("a")], vec![sum(col("b")).alias("b")])?.filter(col("b").gt(lit(10i64)))?.build()?;+ assert_plan_not_transformed!(plan.clone());+// filter of aggregate is after aggregation since they are non-commutativeassert_optimized_plan_equal!(plan,@@ -1876,6 +1828,7 @@ mod tests {.window(vec![window])?.filter(col("c").gt(lit(10i64)))?.build()?;+ assert_plan_not_transformed!(plan.clone());assert_optimized_plan_equal!(plan,@@ -3101,6 +3054,7 @@ mod tests {Some(filter),)?.build()?;+ assert_plan_not_transformed!(plan.clone());// not part of the test, just good to know:assert_snapshot!(plan,@@ -3207,15 +3161,16 @@ mod tests {let plan =table_scan_with_pushdown_provider(TableProviderFilterPushDown::Inexact)?;- let optimized_plan = PushDownFilter::new()+ let optimized = PushDownFilter::new().rewrite(plan, &OptimizerContext::new())- .expect("failed to optimize plan")- .data;+ .expect("failed to optimize plan");+ assert!(optimized.transformed);+ assert_plan_not_transformed!(optimized.data.clone());// Optimizing the same plan multiple times should produce the same plan// each time.assert_optimized_plan_equal!(- optimized_plan,+ optimized.data,@r"Filter: a = Int64(1)TableScan: test, partial_filters=[a = Int64(1)]@@ -3227,6 +3182,7 @@ mod tests {fn filter_with_table_provider_unsupported() -> Result<()> {let plan =table_scan_with_pushdown_provider(TableProviderFilterPushDown::Unsupported)?;+ assert_plan_not_transformed!(plan.clone());assert_optimized_plan_equal!(plan,@@ -4217,7 +4173,7 @@ mod tests {plan,@r"Projection: a, b- Filter: t.a > Int32(5) AND t.b > Int32(10) AND TestScalarUDF() > Float64(0.1)+ Filter: TestScalarUDF() > Float64(0.1) AND t.a > Int32(5) AND t.b > Int32(10)TableScan: test")diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rsindex e811ce7313..356f2711b7 100644--- a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs+++ b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs@@ -63,12 +63,14 @@ pub fn simplify_predicates(predicates: Vec<Expr>) -> Result<Vec<Expr>> {| Operator::Eq,right,}) => {- let left_col = extract_column_from_expr(left);- let right_col = extract_column_from_expr(right);- if let (Some(col), Some(_)) = (&left_col, right.as_literal()) {- column_predicates.entry(col.clone()).or_default().push(pred);- } else if let (Some(_), Some(col)) = (left.as_literal(), &right_col) {- column_predicates.entry(col.clone()).or_default().push(pred);+ if let (Some(col), Some(_)) =+ (extract_column_from_expr(left), right.as_literal())+ {+ column_predicates.entry(col).or_default().push(pred);+ } else if let (Some(_), Some(col)) =+ (left.as_literal(), extract_column_from_expr(right))+ {+ column_predicates.entry(col).or_default().push(pred);} else {other_predicates.push(pred);}