Apache DataFusion SQL Query Engine
feat: add ExpressionPlacement enum for optimizer expression placement decisions (#20065)
Adrian Garcia Badaracco9962911ee5d5
diff --git a/datafusion/expr-common/src/lib.rs b/datafusion/expr-common/src/lib.rsindex 0018694d18..c9a95fd294 100644--- a/datafusion/expr-common/src/lib.rs+++ b/datafusion/expr-common/src/lib.rs@@ -40,7 +40,10 @@ pub mod dyn_eq;pub mod groups_accumulator;pub mod interval_arithmetic;pub mod operator;+pub mod placement;pub mod signature;pub mod sort_properties;pub mod statistics;pub mod type_coercion;++pub use placement::ExpressionPlacement;diff --git a/datafusion/expr-common/src/placement.rs b/datafusion/expr-common/src/placement.rsnew file mode 100644index 0000000000..8212ba618e--- /dev/null+++ b/datafusion/expr-common/src/placement.rs@@ -0,0 +1,62 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Expression placement information for optimization decisions.++/// Describes where an expression should be placed in the query plan for+/// optimal execution. This is used by optimizers to make decisions about+/// expression placement, such as whether to push expressions down through+/// projections.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum ExpressionPlacement {+ /// A constant literal value.+ Literal,+ /// A simple column reference.+ Column,+ /// A cheap expression that can be pushed to leaf nodes in the plan.+ /// Examples include `get_field` for struct field access.+ /// Pushing these expressions down in the plan can reduce data early+ /// at low compute cost.+ /// See [`ExpressionPlacement::should_push_to_leaves`] for details.+ MoveTowardsLeafNodes,+ /// An expensive expression that should stay where it is in the plan.+ /// Examples include complex scalar functions or UDFs.+ KeepInPlace,+}++impl ExpressionPlacement {+ /// Returns true if the expression can be pushed down to leaf nodes+ /// in the query plan.+ ///+ /// This returns true for:+ /// - [`ExpressionPlacement::Column`]: Simple column references can be pushed down. They do no compute and do not increase or+ /// decrease the amount of data being processed.+ /// A projection that reduces the number of columns can eliminate unnecessary data early,+ /// but this method only considers one expression at a time, not a projection as a whole.+ /// - [`ExpressionPlacement::MoveTowardsLeafNodes`]: Cheap expressions can be pushed down to leaves to take advantage of+ /// early computation and potential optimizations at the data source level.+ /// For example `struct_col['field']` is cheap to compute (just an Arc clone of the nested array for `'field'`)+ /// and thus can reduce data early in the plan at very low compute cost.+ /// It may even be possible to eliminate the expression entirely if the data source can project only the needed field+ /// (as e.g. Parquet can).+ pub fn should_push_to_leaves(&self) -> bool {+ matches!(+ self,+ ExpressionPlacement::Column | ExpressionPlacement::MoveTowardsLeafNodes+ )+ }+}diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rsindex 8eae81bc5b..9bf0ac58cd 100644--- a/datafusion/expr/src/expr.rs+++ b/datafusion/expr/src/expr.rs@@ -38,6 +38,7 @@ use datafusion_common::tree_node::{use datafusion_common::{Column, DFSchema, HashMap, Result, ScalarValue, Spans, TableReference,};+use datafusion_expr_common::placement::ExpressionPlacement;use datafusion_functions_window_common::field::WindowUDFFieldArgs;#[cfg(feature = "sql")]use sqlparser::ast::{@@ -1536,6 +1537,23 @@ impl Expr {}}+ /// Returns placement information for this expression.+ ///+ /// This is used by optimizers to make decisions about expression placement,+ /// such as whether to push expressions down through projections.+ pub fn placement(&self) -> ExpressionPlacement {+ match self {+ Expr::Column(_) => ExpressionPlacement::Column,+ Expr::Literal(_, _) => ExpressionPlacement::Literal,+ Expr::ScalarFunction(func) => {+ let arg_placements: Vec<_> =+ func.args.iter().map(|arg| arg.placement()).collect();+ func.func.placement(&arg_placements)+ }+ _ => ExpressionPlacement::KeepInPlace,+ }+ }+/// Return String representation of the variant represented by `self`/// Useful for non-rust based bindingspub fn variant_name(&self) -> &str {diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rsindex 201f7a0251..cb136229bf 100644--- a/datafusion/expr/src/lib.rs+++ b/datafusion/expr/src/lib.rs@@ -95,6 +95,7 @@ pub use datafusion_expr_common::accumulator::Accumulator;pub use datafusion_expr_common::columnar_value::ColumnarValue;pub use datafusion_expr_common::groups_accumulator::{EmitTo, GroupsAccumulator};pub use datafusion_expr_common::operator::Operator;+pub use datafusion_expr_common::placement::ExpressionPlacement;pub use datafusion_expr_common::signature::{ArrayFunctionArgument, ArrayFunctionSignature, Coercion, Signature,TIMEZONE_WILDCARD, TypeSignature, TypeSignatureClass, Volatility,diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rsindex 2183bdbea4..405fb25680 100644--- a/datafusion/expr/src/udf.rs+++ b/datafusion/expr/src/udf.rs@@ -31,6 +31,7 @@ use datafusion_common::config::ConfigOptions;use datafusion_common::{ExprSchema, Result, ScalarValue, not_impl_err};use datafusion_expr_common::dyn_eq::{DynEq, DynHash};use datafusion_expr_common::interval_arithmetic::Interval;+use datafusion_expr_common::placement::ExpressionPlacement;use std::any::Any;use std::cmp::Ordering;use std::fmt::Debug;@@ -361,6 +362,13 @@ impl ScalarUDF {pub fn as_async(&self) -> Option<&AsyncScalarUDF> {self.inner().as_any().downcast_ref::<AsyncScalarUDF>()}++ /// Returns placement information for this function.+ ///+ /// See [`ScalarUDFImpl::placement`] for more details.+ pub fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {+ self.inner.placement(args)+ }}impl<F> From<F> for ScalarUDF@@ -964,6 +972,20 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync {fn documentation(&self) -> Option<&Documentation> {None}++ /// Returns placement information for this function.+ ///+ /// This is used by optimizers to make decisions about expression placement,+ /// such as whether to push expressions down through projections.+ ///+ /// The default implementation returns [`ExpressionPlacement::KeepInPlace`],+ /// meaning the expression should be kept where it is in the plan.+ ///+ /// Override this method to indicate that the function can be pushed down+ /// closer to the data source.+ fn placement(&self, _args: &[ExpressionPlacement]) -> ExpressionPlacement {+ ExpressionPlacement::KeepInPlace+ }}/// ScalarUDF that adds an alias to the underlying function. It is better to@@ -1091,6 +1113,10 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {fn documentation(&self) -> Option<&Documentation> {self.inner.documentation()}++ fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {+ self.inner.placement(args)+ }}#[cfg(test)]diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rsindex 47a903639d..8d1ffb7c4c 100644--- a/datafusion/functions/src/core/getfield.rs+++ b/datafusion/functions/src/core/getfield.rs@@ -33,8 +33,8 @@ use datafusion_common::{use datafusion_expr::expr::ScalarFunction;use datafusion_expr::simplify::ExprSimplifyResult;use datafusion_expr::{- ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,- ScalarUDFImpl, Signature, Volatility,+ ColumnarValue, Documentation, Expr, ExpressionPlacement, ReturnFieldArgs,+ ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,};use datafusion_macros::user_doc;@@ -499,6 +499,32 @@ impl ScalarUDFImpl for GetFieldFunc {fn documentation(&self) -> Option<&Documentation> {self.doc()}++ fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {+ // get_field can be pushed to leaves if:+ // 1. The base (first arg) is a column or already placeable at leaves+ // 2. All field keys (remaining args) are literals+ if args.is_empty() {+ return ExpressionPlacement::KeepInPlace;+ }++ let base_placement = args[0];+ let base_is_pushable = matches!(+ base_placement,+ ExpressionPlacement::Column | ExpressionPlacement::MoveTowardsLeafNodes+ );++ let all_keys_are_literals = args+ .iter()+ .skip(1)+ .all(|p| matches!(p, ExpressionPlacement::Literal));++ if base_is_pushable && all_keys_are_literals {+ ExpressionPlacement::MoveTowardsLeafNodes+ } else {+ ExpressionPlacement::KeepInPlace+ }+ }}#[cfg(test)]@@ -542,4 +568,92 @@ mod tests {Ok(())}++ #[test]+ fn test_placement_literal_key() {+ let func = GetFieldFunc::new();++ // get_field(col, 'literal') -> leaf-pushable (static field access)+ let args = vec![ExpressionPlacement::Column, ExpressionPlacement::Literal];+ assert_eq!(+ func.placement(&args),+ ExpressionPlacement::MoveTowardsLeafNodes+ );++ // get_field(col, 'a', 'b') -> leaf-pushable (nested static field access)+ let args = vec![+ ExpressionPlacement::Column,+ ExpressionPlacement::Literal,+ ExpressionPlacement::Literal,+ ];+ assert_eq!(+ func.placement(&args),+ ExpressionPlacement::MoveTowardsLeafNodes+ );++ // get_field(get_field(col, 'a'), 'b') represented as MoveTowardsLeafNodes for base+ let args = vec![+ ExpressionPlacement::MoveTowardsLeafNodes,+ ExpressionPlacement::Literal,+ ];+ assert_eq!(+ func.placement(&args),+ ExpressionPlacement::MoveTowardsLeafNodes+ );+ }++ #[test]+ fn test_placement_column_key() {+ let func = GetFieldFunc::new();++ // get_field(col, other_col) -> NOT leaf-pushable (dynamic per-row lookup)+ let args = vec![ExpressionPlacement::Column, ExpressionPlacement::Column];+ assert_eq!(func.placement(&args), ExpressionPlacement::KeepInPlace);++ // get_field(col, 'a', other_col) -> NOT leaf-pushable (dynamic nested lookup)+ let args = vec![+ ExpressionPlacement::Column,+ ExpressionPlacement::Literal,+ ExpressionPlacement::Column,+ ];+ assert_eq!(func.placement(&args), ExpressionPlacement::KeepInPlace);+ }++ #[test]+ fn test_placement_root() {+ let func = GetFieldFunc::new();++ // get_field(root_expr, 'literal') -> NOT leaf-pushable+ let args = vec![+ ExpressionPlacement::KeepInPlace,+ ExpressionPlacement::Literal,+ ];+ assert_eq!(func.placement(&args), ExpressionPlacement::KeepInPlace);++ // get_field(col, root_expr) -> NOT leaf-pushable+ let args = vec![+ ExpressionPlacement::Column,+ ExpressionPlacement::KeepInPlace,+ ];+ assert_eq!(func.placement(&args), ExpressionPlacement::KeepInPlace);+ }++ #[test]+ fn test_placement_edge_cases() {+ let func = GetFieldFunc::new();++ // Empty args -> NOT leaf-pushable+ assert_eq!(func.placement(&[]), ExpressionPlacement::KeepInPlace);++ // Just base, no key -> MoveTowardsLeafNodes (not a valid call but should handle gracefully)+ let args = vec![ExpressionPlacement::Column];+ assert_eq!(+ func.placement(&args),+ ExpressionPlacement::MoveTowardsLeafNodes+ );++ // Literal base with literal key -> NOT leaf-pushable (would be constant-folded)+ let args = vec![ExpressionPlacement::Literal, ExpressionPlacement::Literal];+ assert_eq!(func.placement(&args), ExpressionPlacement::KeepInPlace);+ }}diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rsindex d9273a8f60..5d29892a23 100644--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs@@ -702,6 +702,11 @@ impl CSEController for ExprCSEController<'_> {#[expect(deprecated)]let is_normal_minus_aggregates = matches!(node,+ // TODO: there's an argument for removing `Literal` from here,+ // maybe using `Expr::placemement().should_push_to_leaves()` instead+ // so that we extract common literals and don't broadcast them to num_batch_rows multiple times.+ // However that currently breaks things like `percentile_cont()` which expect literal arguments+ // (and would instead be getting `col(__common_expr_n)`).Expr::Literal(..)| Expr::Column(..)| Expr::ScalarVariable(..)diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex f97b05ea68..9cccb20bcc 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -525,15 +525,14 @@ fn merge_consecutive_projections(proj: Projection) -> Result<Transformed<Projectexpr.iter().for_each(|expr| expr.add_column_ref_counts(&mut column_referral_map));- // If an expression is non-trivial and appears more than once, do not merge+ // If an expression is non-trivial (KeepInPlace) and appears more than once, do not merge// them as consecutive projections will benefit from a compute-once approach.// For details, see: https://github.com/apache/datafusion/issues/8296if column_referral_map.into_iter().any(|(col, usage)| {usage > 1- && !is_expr_trivial(- &prev_projection.expr- [prev_projection.schema.index_of_column(col).unwrap()],- )+ && !prev_projection.expr[prev_projection.schema.index_of_column(col).unwrap()]+ .placement()+ .should_push_to_leaves()}) {// no changereturn Projection::try_new_with_schema(expr, input, schema).map(Transformed::no);@@ -586,11 +585,6 @@ fn merge_consecutive_projections(proj: Projection) -> Result<Transformed<Project}}-// Check whether `expr` is trivial; i.e. it doesn't imply any computation.-fn is_expr_trivial(expr: &Expr) -> bool {- matches!(expr, Expr::Column(_) | Expr::Literal(_, _))-}-/// Rewrites a projection expression using the projection before it (i.e. its input)/// This is a subroutine to the `merge_consecutive_projections` function.///diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rsindex 2358a21940..7107b0a900 100644--- a/datafusion/physical-expr-common/src/physical_expr.rs+++ b/datafusion/physical-expr-common/src/physical_expr.rs@@ -35,6 +35,7 @@ use datafusion_common::{};use datafusion_expr_common::columnar_value::ColumnarValue;use datafusion_expr_common::interval_arithmetic::Interval;+use datafusion_expr_common::placement::ExpressionPlacement;use datafusion_expr_common::sort_properties::ExprProperties;use datafusion_expr_common::statistics::Distribution;@@ -430,6 +431,16 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {fn is_volatile_node(&self) -> bool {false}++ /// Returns placement information for this expression.+ ///+ /// This is used by optimizers to make decisions about expression placement,+ /// such as whether to push expressions down through projections.+ ///+ /// The default implementation returns [`ExpressionPlacement::KeepInPlace`].+ fn placement(&self) -> ExpressionPlacement {+ ExpressionPlacement::KeepInPlace+ }}#[deprecated(diff --git a/datafusion/physical-expr/src/expressions/column.rs b/datafusion/physical-expr/src/expressions/column.rsindex 8c7e8c319f..cf844790a0 100644--- a/datafusion/physical-expr/src/expressions/column.rs+++ b/datafusion/physical-expr/src/expressions/column.rs@@ -30,6 +30,7 @@ use arrow::{use datafusion_common::tree_node::{Transformed, TreeNode};use datafusion_common::{Result, internal_err, plan_err};use datafusion_expr::ColumnarValue;+use datafusion_expr_common::placement::ExpressionPlacement;/// Represents the column at a given index in a RecordBatch///@@ -146,6 +147,10 @@ impl PhysicalExpr for Column {fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {write!(f, "{}", self.name)}++ fn placement(&self) -> ExpressionPlacement {+ ExpressionPlacement::Column+ }}impl Column {diff --git a/datafusion/physical-expr/src/expressions/literal.rs b/datafusion/physical-expr/src/expressions/literal.rsindex 1f3fefc60b..9105297c96 100644--- a/datafusion/physical-expr/src/expressions/literal.rs+++ b/datafusion/physical-expr/src/expressions/literal.rs@@ -33,6 +33,7 @@ use datafusion_common::{Result, ScalarValue};use datafusion_expr::Expr;use datafusion_expr_common::columnar_value::ColumnarValue;use datafusion_expr_common::interval_arithmetic::Interval;+use datafusion_expr_common::placement::ExpressionPlacement;use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties};/// Represents a literal value@@ -134,6 +135,10 @@ impl PhysicalExpr for Literal {fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {std::fmt::Display::fmt(self, f)}++ fn placement(&self) -> ExpressionPlacement {+ ExpressionPlacement::Literal+ }}/// Create a literal expressiondiff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rsindex aa090743ad..dab4153fa6 100644--- a/datafusion/physical-expr/src/scalar_function.rs+++ b/datafusion/physical-expr/src/scalar_function.rs@@ -45,8 +45,8 @@ use datafusion_expr::interval_arithmetic::Interval;use datafusion_expr::sort_properties::ExprProperties;use datafusion_expr::type_coercion::functions::fields_with_udf;use datafusion_expr::{- ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, Volatility,- expr_vec_fmt,+ ColumnarValue, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,+ Volatility, expr_vec_fmt,};/// Physical expression of a scalar function@@ -362,6 +362,12 @@ impl PhysicalExpr for ScalarFunctionExpr {fn is_volatile_node(&self) -> bool {self.fun.signature().volatility == Volatility::Volatile}++ fn placement(&self) -> ExpressionPlacement {+ let arg_placements: Vec<_> =+ self.args.iter().map(|arg| arg.placement()).collect();+ self.fun.placement(&arg_placements)+ }}#[cfg(test)]diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 8d4c775f87..f003602922 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -20,7 +20,7 @@//! of a projection on table `t1` where the expressions `a`, `b`, and `a+b` are the//! projection expressions. `SELECT` without `FROM` will only evaluate expressions.-use super::expressions::{Column, Literal};+use super::expressions::Column;use super::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream,@@ -48,6 +48,7 @@ use datafusion_common::tree_node::{};use datafusion_common::{DataFusionError, JoinSide, Result, internal_err};use datafusion_execution::TaskContext;+use datafusion_expr::ExpressionPlacement;use datafusion_physical_expr::equivalence::ProjectionMapping;use datafusion_physical_expr::projection::Projector;use datafusion_physical_expr::utils::{collect_columns, reassign_expr_columns};@@ -285,10 +286,13 @@ impl ExecutionPlan for ProjectionExec {.as_ref().iter().all(|proj_expr| {- proj_expr.expr.as_any().is::<Column>()- || proj_expr.expr.as_any().is::<Literal>()+ !matches!(+ proj_expr.expr.placement(),+ ExpressionPlacement::KeepInPlace+ )});- // If expressions are all either column_expr or Literal, then all computations in this projection are reorder or rename,+ // If expressions are all either column_expr or Literal (or other cheap expressions),+ // then all computations in this projection are reorder or rename,// and projection would not benefit from the repartition, benefits_from_input_partitioning will return false.vec![!all_simple_exprs]}@@ -1003,11 +1007,15 @@ fn try_unifying_projections(.unwrap();});// Merging these projections is not beneficial, e.g- // If an expression is not trivial and it is referred more than 1, unifies projections will be+ // If an expression is not trivial (KeepInPlace) and it is referred more than 1, unifies projections will be// beneficial as caching mechanism for non-trivial computations.// See discussion in: https://github.com/apache/datafusion/issues/8296if column_ref_map.iter().any(|(column, count)| {- *count > 1 && !is_expr_trivial(&Arc::clone(&child.expr()[column.index()].expr))+ *count > 1+ && !child.expr()[column.index()]+ .expr+ .placement()+ .should_push_to_leaves()}) {return Ok(None);}@@ -1117,13 +1125,6 @@ fn new_columns_for_join_on((new_columns.len() == hash_join_on.len()).then_some(new_columns)}-/// Checks if the given expression is trivial.-/// An expression is considered trivial if it is either a `Column` or a `Literal`.-fn is_expr_trivial(expr: &Arc<dyn PhysicalExpr>) -> bool {- expr.as_any().downcast_ref::<Column>().is_some()- || expr.as_any().downcast_ref::<Literal>().is_some()-}-#[cfg(test)]mod tests {use super::*;diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 4c49fae4dc..ab217b192b 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -7952,8 +7952,9 @@ logical_plan02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]03)----TableScan: t projection=[]physical_plan-01)ProjectionExec: expr=[2 as count(Int64(1)), 2 as count()]-02)--PlaceholderRowExec+01)ProjectionExec: expr=[count(Int64(1))@0 as count(Int64(1)), count(Int64(1))@0 as count()]+02)--ProjectionExec: expr=[2 as count(Int64(1))]+03)----PlaceholderRowExecquery IIselect count(1), count(*) from t;@@ -7968,8 +7969,9 @@ logical_plan02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]03)----TableScan: t projection=[]physical_plan-01)ProjectionExec: expr=[2 as count(Int64(1)), 2 as count(*)]-02)--PlaceholderRowExec+01)ProjectionExec: expr=[count(Int64(1))@0 as count(Int64(1)), count(Int64(1))@0 as count(*)]+02)--ProjectionExec: expr=[2 as count(Int64(1))]+03)----PlaceholderRowExecquery IIselect count(), count(*) from t;@@ -7984,8 +7986,9 @@ logical_plan02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]03)----TableScan: t projection=[]physical_plan-01)ProjectionExec: expr=[2 as count(), 2 as count(*)]-02)--PlaceholderRowExec+01)ProjectionExec: expr=[count(Int64(1))@0 as count(), count(Int64(1))@0 as count(*)]+02)--ProjectionExec: expr=[2 as count(Int64(1))]+03)----PlaceholderRowExecquery TTexplain select count(1) * count(2) from t;diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex 4be8358949..3c148561d9 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -932,6 +932,21 @@ SELECT id, id + 100 as computed FROM simple_struct ORDER BY id LIMIT 3;# plan extracts the shared get_field for efficient computation###+query TT+EXPLAIN SELECT (id + s['value']) * (id + s['value']) as id_and_value FROM simple_struct WHERE id > 2;+----+logical_plan+01)Projection: __common_expr_1 * __common_expr_1 AS id_and_value+02)--Projection: simple_struct.id + get_field(simple_struct.s, Utf8("value")) AS __common_expr_1+03)----Filter: simple_struct.id > Int64(2)+04)------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(2)]+physical_plan+01)ProjectionExec: expr=[__common_expr_1@0 * __common_expr_1@0 as id_and_value]+02)--ProjectionExec: expr=[id@0 + get_field(s@1, value) as __common_expr_1]+03)----FilterExec: id@0 > 2+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]++query TTEXPLAIN SELECT s['value'] + s['value'] as doubled FROM simple_struct WHERE id > 2;----@@ -941,10 +956,9 @@ logical_plan03)----Filter: simple_struct.id > Int64(2)04)------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(2)]physical_plan-01)ProjectionExec: expr=[__common_expr_1@0 + __common_expr_1@0 as doubled]-02)--ProjectionExec: expr=[get_field(s@0, value) as __common_expr_1]-03)----FilterExec: id@0 > 2, projection=[s@1]-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]+01)ProjectionExec: expr=[get_field(s@0, value) + get_field(s@0, value) as doubled]+02)--FilterExec: id@0 > 2, projection=[s@1]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]# Verify correctnessquery Idiff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.sltindex f939cd0154..1a6b82020c 100644--- a/datafusion/sqllogictest/test_files/unnest.slt+++ b/datafusion/sqllogictest/test_files/unnest.slt@@ -673,8 +673,8 @@ logical_planphysical_plan01)ProjectionExec: expr=[__unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1],depth=2)@0 as UNNEST(UNNEST(UNNEST(recursive_unnest_table.column3)[c1])), column3@1 as column3]02)--UnnestExec-03)----ProjectionExec: expr=[get_field(__unnest_placeholder(recursive_unnest_table.column3,depth=1)@0, c1) as __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1]), column3@1 as column3]-04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+04)------ProjectionExec: expr=[get_field(__unnest_placeholder(recursive_unnest_table.column3,depth=1)@0, c1) as __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1]), column3@1 as column3]05)--------UnnestExec06)----------ProjectionExec: expr=[column3@0 as __unnest_placeholder(recursive_unnest_table.column3), column3@0 as column3]07)------------DataSourceExec: partitions=1, partition_sizes=[1]