Apache DataFusion SQL Query Engine
refactor: pass SubqueryContext explicitly through planner traits (#23649)
Tim Saucer4184b0778e0c
diff --git a/datafusion-examples/examples/dataframe/cache_factory.rs b/datafusion-examples/examples/dataframe/cache_factory.rsindex a92c3dc4ce..dd145c715f 100644--- a/datafusion-examples/examples/dataframe/cache_factory.rs+++ b/datafusion-examples/examples/dataframe/cache_factory.rs@@ -29,6 +29,7 @@ use datafusion::error::Result;use datafusion::execution::context::QueryPlanner;use datafusion::execution::session_state::CacheFactory;use datafusion::execution::{SessionState, SessionStateBuilder};+use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;use datafusion::logical_expr::{Extension, LogicalPlan, UserDefinedLogicalNode, UserDefinedLogicalNodeCore,};@@ -146,6 +147,7 @@ impl ExtensionPlanner for CacheNodePlanner {logical_inputs: &[&LogicalPlan],physical_inputs: &[Arc<dyn ExecutionPlan>],session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {if let Some(cache_node) = node.as_any().downcast_ref::<CacheNode>() {assert_eq!(logical_inputs.len(), 1, "Inconsistent number of inputs");diff --git a/datafusion-examples/examples/query_planning/expr_api.rs b/datafusion-examples/examples/query_planning/expr_api.rsindex c087019c68..dd5145def3 100644--- a/datafusion-examples/examples/query_planning/expr_api.rs+++ b/datafusion-examples/examples/query_planning/expr_api.rs@@ -33,6 +33,7 @@ use datafusion::functions_aggregate::first_last::first_value_udaf;use datafusion::logical_expr::execution_props::ExecutionProps;use datafusion::logical_expr::expr::BinaryExpr;use datafusion::logical_expr::interval_arithmetic::Interval;+use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;use datafusion::logical_expr::simplify::SimplifyContext;use datafusion::logical_expr::{ColumnarValue, ExprFunctionExt, ExprSchemable, Operator};use datafusion::optimizer::analyzer::type_coercion::TypeCoercionRewriter;@@ -541,8 +542,12 @@ fn type_coercion_demo() -> Result<()> {// Evaluation with an expression that has not been type coerced cannot succeed.let props = ExecutionProps::default();- let physical_expr =- datafusion::physical_expr::create_physical_expr(&expr, &df_schema, &props)?;+ let physical_expr = datafusion::physical_expr::create_physical_expr(+ &expr,+ &df_schema,+ &props,+ &PhysicalPlanningContext::default(),+ )?;let e = physical_expr.evaluate(&batch).unwrap_err();assert!(e.find_root()@@ -566,6 +571,7 @@ fn type_coercion_demo() -> Result<()> {&coerced_expr,&df_schema,&props,+ &PhysicalPlanningContext::default(),)?;assert!(physical_expr.evaluate(&batch).is_ok());@@ -578,6 +584,7 @@ fn type_coercion_demo() -> Result<()> {&coerced_expr,&df_schema,&props,+ &PhysicalPlanningContext::default(),)?;assert!(physical_expr.evaluate(&batch).is_ok());@@ -606,6 +613,7 @@ fn type_coercion_demo() -> Result<()> {&coerced_expr,&df_schema,&props,+ &PhysicalPlanningContext::default(),)?;assert!(physical_expr.evaluate(&batch).is_ok());diff --git a/datafusion-examples/examples/query_planning/pruning.rs b/datafusion-examples/examples/query_planning/pruning.rsindex 7fdc4a7952..df26aa57b6 100644--- a/datafusion-examples/examples/query_planning/pruning.rs+++ b/datafusion-examples/examples/query_planning/pruning.rs@@ -26,6 +26,7 @@ use datafusion::common::pruning::PruningStatistics;use datafusion::common::{DFSchema, ScalarValue};use datafusion::error::Result;use datafusion::execution::context::ExecutionProps;+use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;use datafusion::physical_expr::create_physical_expr;use datafusion::physical_optimizer::pruning::PruningPredicate;use datafusion::prelude::*;@@ -194,7 +195,13 @@ impl PruningStatistics for MyCatalog {fn create_pruning_predicate(expr: Expr, schema: &SchemaRef) -> PruningPredicate {let df_schema = DFSchema::try_from(Arc::clone(schema)).unwrap();let props = ExecutionProps::new();- let physical_expr = create_physical_expr(&expr, &df_schema, &props).unwrap();+ let physical_expr = create_physical_expr(+ &expr,+ &df_schema,+ &props,+ &PhysicalPlanningContext::default(),+ )+ .unwrap();PruningPredicate::try_new(physical_expr, Arc::clone(schema)).unwrap()}diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rsindex 2c696d92d7..b0ccff8d10 100644--- a/datafusion-examples/examples/relation_planner/table_sample.rs+++ b/datafusion-examples/examples/relation_planner/table_sample.rs@@ -119,6 +119,7 @@ use datafusion_common::{DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,plan_datafusion_err, plan_err,};+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{UserDefinedLogicalNode, UserDefinedLogicalNodeCore,logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},@@ -587,6 +588,7 @@ impl ExtensionPlanner for TableSampleExtensionPlanner {_logical_inputs: &[&LogicalPlan],physical_inputs: &[Arc<dyn ExecutionPlan>],_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {let Some(sample_node) = node.as_any().downcast_ref::<TableSamplePlanNode>()else {diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rsindex 31f00b62ef..098f3d51ef 100644--- a/datafusion/catalog-listing/src/helpers.rs+++ b/datafusion/catalog-listing/src/helpers.rs@@ -34,6 +34,7 @@ use arrow::{record_batch::RecordBatch,};use datafusion_expr::execution_props::ExecutionProps;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use futures::stream::FuturesUnordered;use futures::{StreamExt, TryStreamExt, stream::BoxStream};use log::{debug, trace};@@ -328,7 +329,12 @@ pub fn filter_partitioned_file(let filter = utils::conjunction(filters.iter().cloned()).unwrap_or_else(|| lit(true));let props = ExecutionProps::new();- let expr = create_physical_expr(&filter, df_schema, &props)?;+ let expr = create_physical_expr(+ &filter,+ df_schema,+ &props,+ &PhysicalPlanningContext::default(),+ )?;// Since we're only operating on a single file, our batch and resulting "array" holds only one// value indicating if the input file matches the provided filtersdiff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex fabc7ca2a0..b3328cc063 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -44,6 +44,7 @@ use datafusion_execution::cache::cache_manager::{};use datafusion_expr::dml::InsertOp;use datafusion_expr::execution_props::ExecutionProps;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{Expr, Partitioning as LogicalPartitioning, TableProviderFilterPushDown, TableType,};@@ -614,6 +615,7 @@ impl TableProvider for ListingTable {output_partitioning,&df_schema,state.execution_props(),+ &PhysicalPlanningContext::default(),)?}};diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 075e462f4f..5d07133799 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -36,6 +36,7 @@ use datafusion_datasource::memory::{MemSink, MemorySourceConfig};use datafusion_datasource::sink::DataSinkExec;use datafusion_datasource::source::DataSourceExec;use datafusion_expr::dml::InsertOp;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{Expr, SortExpr, TableType};use datafusion_physical_expr::{LexOrdering, PhysicalExpr, create_physical_expr, create_physical_sort_exprs,@@ -209,8 +210,12 @@ impl TableProvider for MemTable {let eqp = state.execution_props();let mut file_sort_order = vec![];for sort_exprs in sort_order.iter() {- let physical_exprs =- create_physical_sort_exprs(sort_exprs, &df_schema, eqp)?;+ let physical_exprs = create_physical_sort_exprs(+ sort_exprs,+ &df_schema,+ eqp,+ &PhysicalPlanningContext::default(),+ )?;file_sort_order.extend(LexOrdering::new(physical_exprs));}source = source.try_with_sort_information(file_sort_order)?;@@ -356,8 +361,12 @@ impl TableProvider for MemTable {let physical_assignments: HashMap<String, Arc<dyn PhysicalExpr>> = assignments.iter().map(|(name, expr)| {- let physical_expr =- create_physical_expr(expr, &df_schema, state.execution_props())?;+ let physical_expr = create_physical_expr(+ expr,+ &df_schema,+ state.execution_props(),+ &PhysicalPlanningContext::default(),+ )?;Ok((name.clone(), physical_expr))}).collect::<Result<_>>()?;@@ -470,8 +479,12 @@ fn evaluate_filters_to_mask(let mut combined_mask: Option<BooleanArray> = None;for filter_expr in filters {- let physical_expr =- create_physical_expr(filter_expr, df_schema, execution_props)?;+ let physical_expr = create_physical_expr(+ filter_expr,+ df_schema,+ execution_props,+ &PhysicalPlanningContext::default(),+ )?;let result = physical_expr.evaluate(batch)?;let array = result.into_array(batch.num_rows())?;diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rsindex 5bfecef1fb..50f05355aa 100644--- a/datafusion/catalog/src/streaming.rs+++ b/datafusion/catalog/src/streaming.rs@@ -22,6 +22,7 @@ use std::sync::Arc;use arrow::datatypes::SchemaRef;use async_trait::async_trait;use datafusion_common::{DFSchema, Result, plan_err};+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{Expr, SortExpr, TableType};use datafusion_physical_expr::equivalence::project_ordering;use datafusion_physical_expr::projection::ProjectionMapping;@@ -131,8 +132,12 @@ impl TableProvider for StreamingTable {let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;let eqp = state.execution_props();- let original_sort_exprs =- create_physical_sort_exprs(&self.sort_order, &df_schema, eqp)?;+ let original_sort_exprs = create_physical_sort_exprs(+ &self.sort_order,+ &df_schema,+ eqp,+ &PhysicalPlanningContext::default(),+ )?;if let Some(p) = projection {// When performing a projection, the output columns will not matchdiff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 08c7463e21..281cb4dd79 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -2383,6 +2383,7 @@ mod tests {use arrow_schema::FieldRef;use datafusion_common::DataFusionError;use datafusion_common::datatype::DataTypeExt;+ use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use std::error::Error;use std::path::PathBuf;@@ -2851,6 +2852,7 @@ mod tests {_expr: &Expr,_input_dfschema: &DFSchema,_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Arc<dyn PhysicalExpr>> {unimplemented!()}diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex f7117c89ef..ff4ec20cfc 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -55,6 +55,7 @@ use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_expr::TableSource;use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::expr_rewriter::FunctionRewrite;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::planner::ExprPlanner;#[cfg(feature = "sql")]use datafusion_expr::planner::{RelationPlanner, TypePlanner};@@ -799,7 +800,12 @@ impl SessionState {.transform_up(|expr| rewrite.rewrite(expr, df_schema, config_options))?.data;}- create_physical_expr(&expr, df_schema, self.execution_props())+ create_physical_expr(+ &expr,+ df_schema,+ self.execution_props(),+ &PhysicalPlanningContext::default(),+ )}/// Return the session IDdiff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex b6d28e7b21..aef8036c74 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -80,7 +80,6 @@ use datafusion_common::{use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::memory::MemorySourceConfig;use datafusion_expr::dml::{CopyTo, InsertOp};-use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};use datafusion_expr::expr::{Alias, GroupingSet, NullTreatment, WindowFunction, WindowFunctionParams,physical_name,@@ -88,6 +87,9 @@ use datafusion_expr::expr::{use datafusion_expr::expr_rewriter::unnormalize_cols;use datafusion_expr::logical_plan::Subquery;use datafusion_expr::logical_plan::builder::wrap_projection_for_join_if_necessary;+use datafusion_expr::physical_planning_context::{+ PhysicalPlanningContext, ScalarSubqueryResults, SubqueryIndex,+};use datafusion_expr::utils::{expr_to_columns, split_conjunction};use datafusion_expr::{Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension,@@ -135,11 +137,19 @@ pub trait PhysicalPlanner: Send + Sync {/// `expr`: the expression to convert////// `input_dfschema`: the logical plan schema for evaluating `expr`+ ///+ /// `planning_ctx`: the [`PhysicalPlanningContext`] used to resolve+ /// `Expr::ScalarSubquery` nodes. During physical planning the planner+ /// threads the context of the plan currently being converted to a physical+ /// plan (for example into [`ExtensionPlanner::plan_extension`], which+ /// should forward it here). Callers creating physical expressions outside+ /// of a plan should pass `&PhysicalPlanningContext::default()`.fn create_physical_expr(&self,expr: &Expr,input_dfschema: &DFSchema,session_state: &SessionState,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Arc<dyn PhysicalExpr>>;}@@ -156,6 +166,12 @@ pub trait ExtensionPlanner {/// Returns `None` when the planner does not know how to plan the/// `node` and wants to delegate the planning to another/// [`ExtensionPlanner`].+ ///+ /// `planning_ctx` is the [`PhysicalPlanningContext`] of the plan subtree+ /// currently being converted to a physical plan. Forward it to+ /// [`PhysicalPlanner::create_physical_expr`] when creating this node's+ /// physical expressions so that scalar subqueries resolve against the same+ /// subquery state as the rest of the plan.async fn plan_extension(&self,planner: &dyn PhysicalPlanner,@@ -163,6 +179,7 @@ pub trait ExtensionPlanner {logical_inputs: &[&LogicalPlan],physical_inputs: &[Arc<dyn ExecutionPlan>],session_state: &SessionState,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>>;/// Create a physical plan for a [`LogicalPlan::TableScan`].@@ -202,6 +219,7 @@ pub trait ExtensionPlanner {/// _logical_inputs: &[&LogicalPlan],/// _physical_inputs: &[Arc<dyn ExecutionPlan>],/// _session_state: &SessionState,+ /// _planning_ctx: &PhysicalPlanningContext,/// ) -> Result<Option<Arc<dyn ExecutionPlan>>> {/// Ok(None)/// }@@ -211,6 +229,7 @@ pub trait ExtensionPlanner {/// _planner: &dyn PhysicalPlanner,/// scan: &TableScan,/// _session_state: &SessionState,+ /// _planning_ctx: &PhysicalPlanningContext,/// ) -> Result<Option<Arc<dyn ExecutionPlan>>> {/// // Check if this is your custom table source/// if scan.source.is::<MyCustomTableSource>() {@@ -235,6 +254,7 @@ pub trait ExtensionPlanner {_planner: &dyn PhysicalPlanner,_scan: &TableScan,_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {Ok(None)}@@ -295,8 +315,14 @@ impl PhysicalPlanner for DefaultPhysicalPlanner {expr: &Expr,input_dfschema: &DFSchema,session_state: &SessionState,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Arc<dyn PhysicalExpr>> {- create_physical_expr(expr, input_dfschema, session_state.execution_props())+ create_physical_expr(+ expr,+ input_dfschema,+ session_state.execution_props(),+ planning_ctx,+ )}}@@ -417,9 +443,9 @@ impl DefaultPhysicalPlanner {/// collected, planned as separate physical plans, and each assigned an/// index in a shared [`ScalarSubqueryResults`] container that will hold its/// result at execution time. The index map and shared results container are- /// registered in [`ExecutionProps`] so that [`create_physical_expr`] can- /// convert `Expr::ScalarSubquery` into [`ScalarSubqueryExpr`] nodes that- /// read from that container.+ /// stored in a [`PhysicalPlanningContext`] and passed explicitly to+ /// [`create_physical_expr`] so it can convert `Expr::ScalarSubquery` into+ /// [`ScalarSubqueryExpr`] nodes that read from that container.////// The resulting physical plan is wrapped in a [`ScalarSubqueryExec`] node/// that executes those subquery plans before any data flows through the@@ -458,27 +484,25 @@ impl DefaultPhysicalPlanner {if links.is_empty() {return self- .create_initial_plan_inner(logical_plan, session_state)+ .create_initial_plan_inner(+ logical_plan,+ session_state,+ &PhysicalPlanningContext::default(),+ ).await;}- // Create the shared `ScalarSubqueryResults` container and register- // it in `ExecutionProps` so that `create_physical_expr` can resolve- // `Expr::ScalarSubquery` into `ScalarSubqueryExpr` nodes. We clone- // the `SessionState` so these are available throughout physical- // planning without mutating the caller's state.- //- // Ideally, the subquery state would live in a dedicated planning- // context rather than in `ExecutionProps`. It's here because- // `create_physical_expr` only receives `&ExecutionProps`.+ // Build a `PhysicalPlanningContext` that carries the index map and+ // shared results container into calls that create physical expressions.+ // The context is threaded explicitly through physical planning rather+ // than being stashed in `ExecutionProps`, so the planner does not need+ // a mutable `SessionState` and each recursively planned subtree receives+ // the correct context.let results = ScalarSubqueryResults::new(links.len());- let mut owned = session_state.clone();- owned.execution_props_mut().subquery_indexes = index_map;- owned.execution_props_mut().subquery_results = results.clone();- let session_state = Cow::Owned(owned);+ let planning_ctx = PhysicalPlanningContext::new(index_map, results.clone());let plan = self- .create_initial_plan_inner(logical_plan, &session_state)+ .create_initial_plan_inner(logical_plan, session_state, &planning_ctx).await?;Ok(Arc::new(ScalarSubqueryExec::new(plan, links, results)))})@@ -490,6 +514,7 @@ impl DefaultPhysicalPlanner {&self,logical_plan: &LogicalPlan,session_state: &SessionState,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Arc<dyn ExecutionPlan>> {// DFS the tree to flatten it into a Vec.// This will allow us to build the Physical Plan from the leaves up@@ -540,9 +565,9 @@ impl DefaultPhysicalPlanner {let max_concurrency = planning_concurrency.min(flat_tree_leaf_indices.len());// Spawning tasks which will traverse leaf up to the root.- let tasks = flat_tree_leaf_indices- .into_iter()- .map(|index| self.task_helper(index, Arc::clone(&flat_tree), session_state));+ let tasks = flat_tree_leaf_indices.into_iter().map(|index| {+ self.task_helper(index, Arc::clone(&flat_tree), session_state, planning_ctx)+ });let mut outputs = futures::stream::iter(tasks).buffer_unordered(max_concurrency).try_collect::<Vec<_>>()@@ -570,6 +595,7 @@ impl DefaultPhysicalPlanner {leaf_starter_index: usize,flat_tree: Arc<Vec<LogicalNode<'a>>>,session_state: &'a SessionState,+ planning_ctx: &'a PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {// We always start with a leaf, so can ignore status and pass empty childrenlet mut node = flat_tree.get(leaf_starter_index).ok_or_else(|| {@@ -581,6 +607,7 @@ impl DefaultPhysicalPlanner {.map_logical_node_to_physical(node.node,session_state,+ planning_ctx,ChildrenContainer::None,).await?;@@ -598,6 +625,7 @@ impl DefaultPhysicalPlanner {.map_logical_node_to_physical(node.node,session_state,+ planning_ctx,ChildrenContainer::One(plan),).await?;@@ -634,7 +662,12 @@ impl DefaultPhysicalPlanner {let children = children.into_iter().map(|epc| epc.plan).collect();let children = ChildrenContainer::Multiple(children);plan = self- .map_logical_node_to_physical(node.node, session_state, children)+ .map_logical_node_to_physical(+ node.node,+ session_state,+ planning_ctx,+ children,+ ).await?;}}@@ -649,6 +682,7 @@ impl DefaultPhysicalPlanner {&self,node: &LogicalPlan,session_state: &SessionState,+ planning_ctx: &PhysicalPlanningContext,children: ChildrenContainer,) -> Result<Arc<dyn ExecutionPlan>> {let execution_props = session_state.execution_props();@@ -687,8 +721,9 @@ impl DefaultPhysicalPlanner {break;}- maybe_plan =- planner.plan_table_scan(self, scan, session_state).await?;+ maybe_plan = planner+ .plan_table_scan(self, scan, session_state, planning_ctx)+ .await?;}let plan = match maybe_plan {@@ -714,7 +749,12 @@ impl DefaultPhysicalPlanner {.map(|row| {row.iter().map(|expr| {- create_physical_expr(expr, schema, execution_props)+ create_physical_expr(+ expr,+ schema,+ execution_props,+ planning_ctx,+ )}).collect::<Result<Vec<Arc<dyn PhysicalExpr>>>>()})@@ -973,7 +1013,14 @@ impl DefaultPhysicalPlanner {let logical_schema = node.schema();let window_expr = window_expr.iter()- .map(|e| create_window_expr(e, logical_schema, execution_props))+ .map(|e| {+ create_window_expr(+ e,+ logical_schema,+ execution_props,+ planning_ctx,+ )+ }).collect::<Result<Vec<_>>>()?;let can_repartition = session_state.config().target_partitions() > 1@@ -1079,6 +1126,7 @@ impl DefaultPhysicalPlanner {logical_input_schema,&physical_input_schema,execution_props,+ planning_ctx,)?;let agg_filter = aggr_expr@@ -1089,6 +1137,7 @@ impl DefaultPhysicalPlanner {logical_input_schema,&physical_input_schema,execution_props,+ planning_ctx,).build().map(lowered_aggregate_to_tuple)@@ -1186,6 +1235,7 @@ impl DefaultPhysicalPlanner {LogicalPlan::Projection(Projection { input, expr, .. }) => self.create_project_physical_exec_with_props(execution_props,+ planning_ctx,children.one()?,input,expr,@@ -1195,8 +1245,12 @@ impl DefaultPhysicalPlanner {}) => {let physical_input = children.one()?;let input_dfschema = input.schema();- let runtime_expr =- create_physical_expr(predicate, input_dfschema, execution_props)?;+ let runtime_expr = create_physical_expr(+ predicate,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;let input_schema = input.schema();let filter = match self.try_plan_async_exprs(@@ -1258,6 +1312,7 @@ impl DefaultPhysicalPlanner {partitioning_scheme,input_dfschema,execution_props,+ planning_ctx,)?;Arc::new(RepartitionExec::try_new(physical_input,@@ -1269,8 +1324,12 @@ impl DefaultPhysicalPlanner {}) => {let physical_input = children.one()?;let input_dfschema = input.as_ref().schema();- let sort_exprs =- create_physical_sort_exprs(expr, input_dfschema, execution_props)?;+ let sort_exprs = create_physical_sort_exprs(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;let Some(ordering) = LexOrdering::new(sort_exprs) else {return internal_err!("SortExec requires at least one sort expression"@@ -1399,6 +1458,7 @@ impl DefaultPhysicalPlanner {LogicalPlan::Projection(Projection { input, expr, .. }),) => self.create_project_physical_exec_with_props(execution_props,+ planning_ctx,physical_left,input,expr,@@ -1412,6 +1472,7 @@ impl DefaultPhysicalPlanner {LogicalPlan::Projection(Projection { input, expr, .. }),) => self.create_project_physical_exec_with_props(execution_props,+ planning_ctx,physical_right,input,expr,@@ -1479,9 +1540,18 @@ impl DefaultPhysicalPlanner {let join_on = keys.iter().map(|(l, r)| {- let l = create_physical_expr(l, left_df_schema, execution_props)?;- let r =- create_physical_expr(r, right_df_schema, execution_props)?;+ let l = create_physical_expr(+ l,+ left_df_schema,+ execution_props,+ planning_ctx,+ )?;+ let r = create_physical_expr(+ r,+ right_df_schema,+ execution_props,+ planning_ctx,+ )?;Ok((l, r))}).collect::<Result<join_utils::JoinOn>>()?;@@ -1582,6 +1652,7 @@ impl DefaultPhysicalPlanner {expr,&filter_df_schema,execution_props,+ planning_ctx,)?;let column_indices = join_utils::JoinFilter::build_column_indices(left_field_indices,@@ -1702,11 +1773,13 @@ impl DefaultPhysicalPlanner {lhs_logical,left_df_schema,execution_props,+ planning_ctx,)?;let on_right = create_physical_expr(rhs_logical,right_df_schema,execution_props,+ planning_ctx,)?;Arc::new(PiecewiseMergeJoinExec::try_new(@@ -1778,6 +1851,7 @@ impl DefaultPhysicalPlanner {if let Some((input, expr)) = new_project {self.create_project_physical_exec_with_props(execution_props,+ planning_ctx,join,input,expr,@@ -1820,6 +1894,7 @@ impl DefaultPhysicalPlanner {&logical_input,&children,session_state,+ planning_ctx,).await?;}@@ -1882,6 +1957,7 @@ impl DefaultPhysicalPlanner {input_dfschema: &DFSchema,input_schema: &Schema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<PhysicalGroupBy> {if group_expr.len() == 1 {match &group_expr[0] {@@ -1891,6 +1967,7 @@ impl DefaultPhysicalPlanner {input_dfschema,input_schema,execution_props,+ planning_ctx,)}Expr::GroupingSet(GroupingSet::Cube(exprs)) => create_cube_physical_expr(@@ -1898,6 +1975,7 @@ impl DefaultPhysicalPlanner {input_dfschema,input_schema,execution_props,+ planning_ctx,),Expr::GroupingSet(GroupingSet::Rollup(exprs)) => {create_rollup_physical_expr(@@ -1905,10 +1983,16 @@ impl DefaultPhysicalPlanner {input_dfschema,input_schema,execution_props,+ planning_ctx,)}expr => Ok(PhysicalGroupBy::new_single(vec![tuple_err((- create_physical_expr(expr, input_dfschema, execution_props),+ create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ ),physical_name(expr),))?])),}@@ -1922,7 +2006,12 @@ impl DefaultPhysicalPlanner {.iter().map(|e| {tuple_err((- create_physical_expr(e, input_dfschema, execution_props),+ create_physical_expr(+ e,+ input_dfschema,+ execution_props,+ planning_ctx,+ ),physical_name(e),))})@@ -1947,6 +2036,7 @@ fn merge_grouping_set_physical_expr(input_dfschema: &DFSchema,input_schema: &Schema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<PhysicalGroupBy> {let num_groups = grouping_sets.len();let mut all_exprs: Vec<Expr> = vec![];@@ -1961,6 +2051,7 @@ fn merge_grouping_set_physical_expr(expr,input_dfschema,execution_props,+ planning_ctx,)?);null_exprs.push(get_null_physical_expr_pair(@@ -1968,6 +2059,7 @@ fn merge_grouping_set_physical_expr(input_dfschema,input_schema,execution_props,+ planning_ctx,)?);}}@@ -1998,6 +2090,7 @@ fn create_cube_physical_expr(input_dfschema: &DFSchema,input_schema: &Schema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<PhysicalGroupBy> {let num_of_exprs = exprs.len();let num_groups = num_of_exprs * num_of_exprs;@@ -2013,12 +2106,14 @@ fn create_cube_physical_expr(input_dfschema,input_schema,execution_props,+ planning_ctx,)?);all_exprs.push(get_physical_expr_pair(expr,input_dfschema,execution_props,+ planning_ctx,)?)}@@ -2044,6 +2139,7 @@ fn create_rollup_physical_expr(input_dfschema: &DFSchema,input_schema: &Schema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<PhysicalGroupBy> {let num_of_exprs = exprs.len();@@ -2060,12 +2156,14 @@ fn create_rollup_physical_expr(input_dfschema,input_schema,execution_props,+ planning_ctx,)?);all_exprs.push(get_physical_expr_pair(expr,input_dfschema,execution_props,+ planning_ctx,)?)}@@ -2092,8 +2190,10 @@ fn get_null_physical_expr_pair(input_dfschema: &DFSchema,input_schema: &Schema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<(Arc<dyn PhysicalExpr>, String)> {- let physical_expr = create_physical_expr(expr, input_dfschema, execution_props)?;+ let physical_expr =+ create_physical_expr(expr, input_dfschema, execution_props, planning_ctx)?;let physical_name = physical_name(&expr.clone())?;let data_type = physical_expr.data_type(input_schema)?;@@ -2163,8 +2263,10 @@ fn get_physical_expr_pair(expr: &Expr,input_dfschema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<(Arc<dyn PhysicalExpr>, String)> {- let physical_expr = create_physical_expr(expr, input_dfschema, execution_props)?;+ let physical_expr =+ create_physical_expr(expr, input_dfschema, execution_props, planning_ctx)?;let physical_name = physical_name(expr)?;Ok((physical_expr, physical_name))}@@ -2417,11 +2519,14 @@ pub fn is_window_frame_bound_valid(window_frame: &WindowFrame) -> bool {}/// Create a window expression with a name from a logical expression+///+/// See [`create_physical_expr`] for details on the `planning_ctx` argument.pub fn create_window_expr_with_name(e: &Expr,name: impl Into<String>,logical_schema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Arc<dyn WindowExpr>> {let name = name.into();let physical_schema = Arc::clone(logical_schema.inner());@@ -2440,12 +2545,24 @@ pub fn create_window_expr_with_name(filter,},} = window_fun.as_ref();- let physical_args =- create_physical_exprs(args, logical_schema, execution_props)?;- let partition_by =- create_physical_exprs(partition_by, logical_schema, execution_props)?;- let order_by =- create_physical_sort_exprs(order_by, logical_schema, execution_props)?;+ let physical_args = create_physical_exprs(+ args,+ logical_schema,+ execution_props,+ planning_ctx,+ )?;+ let partition_by = create_physical_exprs(+ partition_by,+ logical_schema,+ execution_props,+ planning_ctx,+ )?;+ let order_by = create_physical_sort_exprs(+ order_by,+ logical_schema,+ execution_props,+ planning_ctx,+ )?;if !is_window_frame_bound_valid(window_frame) {return plan_err!(@@ -2460,7 +2577,9 @@ pub fn create_window_expr_with_name(== NullTreatment::IgnoreNulls;let physical_filter = filter.as_ref()- .map(|f| create_physical_expr(f, logical_schema, execution_props))+ .map(|f| {+ create_physical_expr(f, logical_schema, execution_props, planning_ctx)+ }).transpose()?;windows::create_window_expr(@@ -2481,10 +2600,13 @@ pub fn create_window_expr_with_name(}/// Create a window expression from a logical expression or an alias+///+/// See [`create_physical_expr`] for details on the `planning_ctx` argument.pub fn create_window_expr(e: &Expr,logical_schema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Arc<dyn WindowExpr>> {// unpack aliased logical expressions, e.g. "sum(col) over () as total"let (name, e) = match e {@@ -2494,7 +2616,7 @@ pub fn create_window_expr(),_ => (e.schema_name().to_string(), e.clone()),};- create_window_expr_with_name(&e, name, logical_schema, execution_props)+ create_window_expr_with_name(&e, name, logical_schema, execution_props, planning_ctx)}type AggregateExprWithOptionalArgs = (@@ -2515,11 +2637,13 @@ pub fn create_aggregate_expr_with_name_and_maybe_filter(physical_input_schema: &Schema,execution_props: &ExecutionProps,) -> Result<AggregateExprWithOptionalArgs> {+ let planning_ctx = PhysicalPlanningContext::default();let mut builder = LoweredAggregateBuilder::new(e,logical_input_schema,physical_input_schema,execution_props,+ &planning_ctx,).with_human_display(human_display);@@ -2554,11 +2678,13 @@ pub fn create_aggregate_expr_and_maybe_filter(_ => (None, String::default(), e.clone()),};+ let planning_ctx = PhysicalPlanningContext::default();let mut builder = LoweredAggregateBuilder::new(&e,logical_input_schema,physical_input_schema,execution_props,+ &planning_ctx,).with_human_display(human_display);@@ -2953,6 +3079,7 @@ impl DefaultPhysicalPlanner {fn create_project_physical_exec_with_props(&self,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,input_exec: Arc<dyn ExecutionPlan>,input: &Arc<LogicalPlan>,expr: &[Expr],@@ -2990,8 +3117,12 @@ impl DefaultPhysicalPlanner {physical_name(e)};- let physical_expr =- create_physical_expr(e, input_logical_schema, execution_props);+ let physical_expr = create_physical_expr(+ e,+ input_logical_schema,+ execution_props,+ planning_ctx,+ );tuple_err((physical_expr, physical_name))})@@ -3248,6 +3379,7 @@ mod tests {Accumulator, AggregateUDF, AggregateUDFImpl, ExprFunctionExt, LogicalPlanBuilder,Partitioning as LogicalPartitioning, RangePartitioning, Signature, TableSource,UserDefinedLogicalNodeCore, Volatility, WindowFunctionDefinition, col, lit,+ scalar_subquery,};use datafusion_functions_aggregate::count::{count_all, count_udaf};use datafusion_functions_aggregate::expr_fn::sum;@@ -3357,8 +3489,12 @@ mod tests {)).alias_with_metadata("window_alias", Some(metadata));- let window_expr =- create_window_expr(&expr, &logical_schema, &ExecutionProps::new())?;+ let window_expr = create_window_expr(+ &expr,+ &logical_schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )?;assert_eq!(window_expr.name(), "window_alias");Ok(())@@ -3613,6 +3749,7 @@ mod tests {logical_input_schema,physical_input_schema,session_state.execution_props(),+ &PhysicalPlanningContext::default(),);insta::assert_debug_snapshot!(cube, @r#"@@ -3744,6 +3881,7 @@ mod tests {logical_input_schema,physical_input_schema,session_state.execution_props(),+ &PhysicalPlanningContext::default(),);insta::assert_debug_snapshot!(rollup, @r#"@@ -3848,6 +3986,7 @@ mod tests {&col("a").not(),&dfschema,&make_session_state(),+ &PhysicalPlanningContext::default(),)?;let expected = expressions::not(expressions::col("a", &schema)?)?;@@ -3875,6 +4014,29 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn scalar_subquery_in_extension_expr_plans() -> Result<()> {+ let subquery = LogicalPlanBuilder::empty(true)+ .project(vec![lit(42_i32)])?+ .build()?;+ let logical_plan = LogicalPlan::Extension(Extension {+ node: Arc::new(NoOpExtensionNode {+ expressions: vec![scalar_subquery(Arc::new(subquery))],+ ..Default::default()+ }),+ });+ let planner = DefaultPhysicalPlanner::with_extension_planners(vec![Arc::new(+ ExpressionExtensionPlanner,+ )]);++ let plan = planner+ .create_physical_plan(&logical_plan, &make_session_state())+ .await?;++ assert_contains!(format!("{plan:?}"), "ScalarSubqueryExec");+ Ok(())+ }+#[tokio::test]async fn error_during_extension_planning() {let session_state = make_session_state();@@ -4396,6 +4558,7 @@ mod tests {_logical_inputs: &[&LogicalPlan],_physical_inputs: &[Arc<dyn ExecutionPlan>],_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {internal_err!("BOOM")}@@ -4404,6 +4567,7 @@ mod tests {#[derive(PartialEq, Eq, Hash)]struct NoOpExtensionNode {schema: DFSchemaRef,+ expressions: Vec<Expr>,}impl Default for NoOpExtensionNode {@@ -4416,6 +4580,7 @@ mod tests {).unwrap(),),+ expressions: vec![],}}}@@ -4448,7 +4613,7 @@ mod tests {}fn expressions(&self) -> Vec<Expr> {- vec![]+ self.expressions.clone()}fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {@@ -4457,10 +4622,13 @@ mod tests {fn with_exprs_and_inputs(&self,- _exprs: Vec<Expr>,+ exprs: Vec<Expr>,_inputs: Vec<LogicalPlan>,) -> Result<Self> {- unimplemented!("NoOp");+ Ok(Self {+ schema: Arc::clone(&self.schema),+ expressions: exprs,+ })}fn supports_limit_pushdown(&self) -> bool {@@ -4522,9 +4690,13 @@ mod tests {fn with_new_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- unimplemented!("NoOpExecutionPlan::with_new_children");+ if children.is_empty() {+ Ok(self)+ } else {+ exec_err!("NoOpExecutionPlan does not support children")+ }}fn execute(@@ -4536,6 +4708,33 @@ mod tests {}}+ struct ExpressionExtensionPlanner;++ #[async_trait]+ impl ExtensionPlanner for ExpressionExtensionPlanner {+ async fn plan_extension(+ &self,+ planner: &dyn PhysicalPlanner,+ node: &dyn UserDefinedLogicalNode,+ _logical_inputs: &[&LogicalPlan],+ _physical_inputs: &[Arc<dyn ExecutionPlan>],+ session_state: &SessionState,+ planning_ctx: &PhysicalPlanningContext,+ ) -> Result<Option<Arc<dyn ExecutionPlan>>> {+ for expr in node.expressions() {+ planner.create_physical_expr(+ &expr,+ node.schema(),+ session_state,+ planning_ctx,+ )?;+ }+ Ok(Some(Arc::new(NoOpExecutionPlan::new(Arc::clone(+ node.schema().inner(),+ )))))+ }+ }+// Produces an execution plan where the schema is mismatched from// the logical plan node.struct BadExtensionPlanner {}@@ -4550,6 +4749,7 @@ mod tests {_logical_inputs: &[&LogicalPlan],_physical_inputs: &[Arc<dyn ExecutionPlan>],_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {Ok(Some(Arc::new(NoOpExecutionPlan::new(SchemaRef::new(Schema::new(vec![Field::new("b", DataType::Int32, false)]),@@ -4995,9 +5195,8 @@ digraph {}#[tokio::test]- // When schemas match, planning proceeds past the schema_satisfied_by check.- // It then panics on unimplemented error in NoOpExecutionPlan.- #[should_panic(expected = "NoOpExecutionPlan")]+ // When schemas match, planning proceeds past the schema_satisfied_by check+ // and succeeds.async fn test_aggregate_schema_check_passes() {let schema =Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));@@ -5184,6 +5383,7 @@ digraph {_logical_inputs: &[&LogicalPlan],_physical_inputs: &[Arc<dyn ExecutionPlan>],_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {Ok(None)}@@ -5193,6 +5393,7 @@ digraph {_planner: &dyn PhysicalPlanner,scan: &TableScan,_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {if scan.source.is::<MockTableSource>() {Ok(Some(Arc::new(EmptyExec::new(Arc::clone(diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rsindex c534954213..d1018f3fb0 100644--- a/datafusion/core/src/test_util/parquet.rs+++ b/datafusion/core/src/test_util/parquet.rs@@ -29,6 +29,7 @@ use crate::datasource::object_store::ObjectStoreUrl;use crate::datasource::physical_plan::ParquetSource;use crate::error::Result;use crate::logical_expr::execution_props::ExecutionProps;+use crate::logical_expr::physical_planning_context::PhysicalPlanningContext;use crate::logical_expr::simplify::SimplifyContext;use crate::optimizer::simplify_expressions::ExprSimplifier;use crate::physical_expr::create_physical_expr;@@ -172,8 +173,12 @@ impl TestParquetFile {if let Some(filter) = maybe_filter {let simplifier = ExprSimplifier::new(context);let filter = simplifier.coerce(filter, &df_schema).unwrap();- let physical_filter_expr =- create_physical_expr(&filter, &df_schema, &ExecutionProps::default())?;+ let physical_filter_expr = create_physical_expr(+ &filter,+ &df_schema,+ &ExecutionProps::default(),+ &PhysicalPlanningContext::default(),+ )?;let source = Arc::new(ParquetSource::new(Arc::clone(&self.schema))diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rsindex a41803191a..372a7a601d 100644--- a/datafusion/core/tests/parquet/page_pruning.rs+++ b/datafusion/core/tests/parquet/page_pruning.rs@@ -38,6 +38,7 @@ use datafusion_expr::{Expr, col, lit};use datafusion_physical_expr::create_physical_expr;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use futures::StreamExt;use object_store::ObjectMeta;use object_store::path::Path;@@ -74,7 +75,13 @@ async fn get_parquet_exec(let df_schema = schema.clone().to_dfschema().unwrap();let execution_props = ExecutionProps::new();- let predicate = create_physical_expr(&filter, &df_schema, &execution_props).unwrap();+ let predicate = create_physical_expr(+ &filter,+ &df_schema,+ &execution_props,+ &PhysicalPlanningContext::default(),+ )+ .unwrap();let source = Arc::new(ParquetSource::new(schema.clone())diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rsindex 75738bcfe1..354a1b3110 100644--- a/datafusion/core/tests/user_defined/user_defined_plan.rs+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs@@ -99,6 +99,7 @@ use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use async_trait::async_trait;use datafusion_common::cast::as_string_view_array;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use futures::{Stream, StreamExt};/// Execute the specified sql and return the resulting record batches@@ -630,6 +631,7 @@ impl ExtensionPlanner for TopKPlanner {logical_inputs: &[&LogicalPlan],physical_inputs: &[Arc<dyn ExecutionPlan>],_session_state: &SessionState,+ _planning_ctx: &PhysicalPlanningContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {Ok(if let Some(topk_node) = node.as_any().downcast_ref::<TopKPlanNode>() {diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex c0b47a9a52..962df06302 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -1664,6 +1664,7 @@ mod tests {use chrono::TimeZone;use datafusion_common::DFSchema;use datafusion_expr::execution_props::ExecutionProps;+ use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use object_store::{ObjectMeta, path::Path};struct File {@@ -1877,6 +1878,7 @@ mod tests {&expr,&DFSchema::try_from(Arc::clone(&table_schema))?,&ExecutionProps::default(),+ &PhysicalPlanningContext::default(),)}).collect::<Result<Vec<_>>>()?,@@ -2243,7 +2245,10 @@ mod tests {#[test]fn test_split_groups_by_statistics_with_target_partitions() -> Result<()> {use datafusion_common::DFSchema;- use datafusion_expr::{col, execution_props::ExecutionProps};+ use datafusion_expr::{+ col, execution_props::ExecutionProps,+ physical_planning_context::PhysicalPlanningContext,+ };let schema = Arc::new(Schema::new(vec![Field::new("value",@@ -2257,7 +2262,13 @@ mod tests {let sort_expr = [col("value").sort(true, false)];let sort_ordering = sort_expr.map(|expr| {- create_physical_sort_expr(&expr, &df_schema, &exec_props).unwrap()+ create_physical_sort_expr(+ &expr,+ &df_schema,+ &exec_props,+ &PhysicalPlanningContext::default(),+ )+ .unwrap()}).into();diff --git a/datafusion/datasource/src/projection.rs b/datafusion/datasource/src/projection.rsindex de822ae602..3cf4f29a77 100644--- a/datafusion/datasource/src/projection.rs+++ b/datafusion/datasource/src/projection.rs@@ -298,7 +298,10 @@ mod test {use arrow::datatypes as arrow_schema;use arrow::datatypes::{DataType, Field, SchemaRef};use datafusion_common::{DFSchema, ScalarValue, config::ConfigOptions};- use datafusion_expr::{Expr, ScalarUDF, col, execution_props::ExecutionProps};+ use datafusion_expr::{+ Expr, ScalarUDF, col, execution_props::ExecutionProps,+ physical_planning_context::PhysicalPlanningContext,+ };use datafusion_functions::core::input_file_name::InputFileNameFunc;use datafusion_physical_expr::{ScalarFunctionExpr, create_physical_exprs, projection::ProjectionExpr,@@ -325,8 +328,13 @@ mod test {schema: &SchemaRef,) -> ProjectionExprs {let df_schema = DFSchema::try_from(Arc::clone(schema)).unwrap();- let physical_exprs =- create_physical_exprs(exprs, &df_schema, &ExecutionProps::default()).unwrap();+ let physical_exprs = create_physical_exprs(+ exprs,+ &df_schema,+ &ExecutionProps::default(),+ &PhysicalPlanningContext::default(),+ )+ .unwrap();let projection_exprs = physical_exprs.into_iter().enumerate()diff --git a/datafusion/expr/src/execution_props.rs b/datafusion/expr/src/execution_props.rsindex 649f74ed39..9910918c6e 100644--- a/datafusion/expr/src/execution_props.rs+++ b/datafusion/expr/src/execution_props.rs@@ -18,14 +18,10 @@use crate::var_provider::{VarProvider, VarType};use chrono::{DateTime, Utc};use datafusion_common::HashMap;-use datafusion_common::ScalarValue;use datafusion_common::TableReference;use datafusion_common::alias::AliasGenerator;use datafusion_common::config::ConfigOptions;-use datafusion_common::{Result, internal_err};-use std::fmt;-use std::hash::{Hash, Hasher};-use std::sync::{Arc, Mutex};+use std::sync::Arc;/// Holds properties and scratch state used while optimizing a [`LogicalPlan`]/// and translating it into an executable physical plan, such as the statement@@ -64,12 +60,6 @@ pub struct ExecutionProps {pub config_options: Option<Arc<ConfigOptions>>,/// Providers for scalar variablespub var_providers: Option<HashMap<VarType, Arc<dyn VarProvider + Send + Sync>>>,- /// Maps each logical `Subquery` to its index in `subquery_results`.- /// Populated by the physical planner before calling `create_physical_expr`.- pub subquery_indexes: HashMap<crate::logical_plan::Subquery, SubqueryIndex>,- /// Shared results container for uncorrelated scalar subquery values.- /// Populated at execution time by `ScalarSubqueryExec`.- pub subquery_results: ScalarSubqueryResults,/// Maps each lambda variable name to its lambda qualifier generated/// during physical planning. Populated by the physical planner for/// each lambda before calling `create_physical_expr`.@@ -90,8 +80,6 @@ impl ExecutionProps {alias_generator: Arc::new(AliasGenerator::new()),config_options: None,var_providers: None,- subquery_indexes: HashMap::new(),- subquery_results: ScalarSubqueryResults::default(),lambda_variable_qualifier: HashMap::new(),}}@@ -169,103 +157,6 @@ impl ExecutionProps {}}-/// Index of a scalar subquery within a [`ScalarSubqueryResults`] container.-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]-pub struct SubqueryIndex(usize);--impl SubqueryIndex {- /// Creates a new subquery index.- pub const fn new(index: usize) -> Self {- Self(index)- }-- /// Returns the underlying slot index.- pub const fn as_usize(self) -> usize {- self.0- }-}--/// Shared results container for uncorrelated scalar subqueries.-///-/// Each entry corresponds to one scalar subquery, identified by its index.-/// Each slot is populated at execution time by `ScalarSubqueryExec`, read by-/// `ScalarSubqueryExpr` instances that share this container, and cleared when-/// the plan is reset for re-execution.-#[derive(Clone, Default)]-pub struct ScalarSubqueryResults {- slots: Arc<Vec<Mutex<Option<ScalarValue>>>>,-}--impl ScalarSubqueryResults {- /// Creates a new shared results container with `n` empty slots.- pub fn new(n: usize) -> Self {- Self {- slots: Arc::new((0..n).map(|_| Mutex::new(None)).collect()),- }- }-- /// Returns the scalar value stored at `index`, if it has been populated.- pub fn get(&self, index: SubqueryIndex) -> Option<ScalarValue> {- let slot = self.slots.get(index.as_usize())?;- slot.lock().unwrap().clone()- }-- /// Stores `value` in the slot at `index`.- pub fn set(&self, index: SubqueryIndex, value: ScalarValue) -> Result<()> {- let Some(slot) = self.slots.get(index.as_usize()) else {- return internal_err!(- "ScalarSubqueryResults: result index {} is out of bounds",- index.as_usize()- );- };-- let mut slot = slot.lock().unwrap();- if slot.is_some() {- return internal_err!(- "ScalarSubqueryResults: result for index {} was already populated",- index.as_usize()- );- }- *slot = Some(value);-- Ok(())- }-- /// Clears all populated results so the container can be reused.- pub fn clear(&self) {- for slot in self.slots.iter() {- *slot.lock().unwrap() = None;- }- }-- /// Returns true if `this` and `other` point to the same shared container.- pub fn ptr_eq(this: &Self, other: &Self) -> bool {- Arc::ptr_eq(&this.slots, &other.slots)- }-}--impl fmt::Debug for ScalarSubqueryResults {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.debug_list()- .entries(self.slots.iter().map(|slot| slot.lock().unwrap().clone()))- .finish()- }-}--impl PartialEq for ScalarSubqueryResults {- fn eq(&self, other: &Self) -> bool {- Self::ptr_eq(self, other)- }-}--impl Eq for ScalarSubqueryResults {}--impl Hash for ScalarSubqueryResults {- fn hash<H: Hasher>(&self, state: &mut H) {- Arc::as_ptr(&self.slots).hash(state);- }-}-#[cfg(test)]mod test {use super::*;@@ -274,44 +165,8 @@ mod test {fn debug() {let props = ExecutionProps::new();assert_eq!(- "ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None, subquery_indexes: {}, subquery_results: [], lambda_variable_qualifier: {} }",+ "ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None, lambda_variable_qualifier: {} }",format!("{props:?}"));}-- #[test]- fn scalar_subquery_results_set_and_get() -> Result<()> {- let results = ScalarSubqueryResults::new(1);- assert_eq!(results.get(SubqueryIndex::new(0)), None);-- results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(42)))?;- assert_eq!(- results.get(SubqueryIndex::new(0)),- Some(ScalarValue::Int32(Some(42)))- );- assert!(- results- .set(SubqueryIndex::new(0), ScalarValue::Int32(Some(7)))- .is_err()- );-- Ok(())- }-- #[test]- fn scalar_subquery_results_clear() -> Result<()> {- let results = ScalarSubqueryResults::new(1);- results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(42)))?;-- results.clear();-- assert_eq!(results.get(SubqueryIndex::new(0)), None);- results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(7)))?;- assert_eq!(- results.get(SubqueryIndex::new(0)),- Some(ScalarValue::Int32(Some(7)))- );-- Ok(())- }}diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rsindex 43cb3fdc20..1033952642 100644--- a/datafusion/expr/src/lib.rs+++ b/datafusion/expr/src/lib.rs@@ -55,6 +55,7 @@ pub mod expr_rewriter;pub mod expr_schema;pub mod extension_types;pub mod function;+pub mod physical_planning_context;pub mod select_expr;pub mod groups_accumulator {pub use datafusion_expr_common::groups_accumulator::*;diff --git a/datafusion/expr/src/physical_planning_context.rs b/datafusion/expr/src/physical_planning_context.rsnew file mode 100644index 0000000000..b1ba63e071--- /dev/null+++ b/datafusion/expr/src/physical_planning_context.rs@@ -0,0 +1,211 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::fmt;+use std::hash::{Hash, Hasher};+use std::sync::{Arc, Mutex};++use datafusion_common::{HashMap, Result, ScalarValue, internal_err};++/// Context used while converting a logical plan subtree into a physical plan.+///+/// Unlike [`ExecutionProps`](crate::execution_props::ExecutionProps), which+/// applies to the overall planning and execution of a query, this context can+/// differ between recursively planned subtrees. It currently carries the state+/// needed to create physical expressions for [`Expr::ScalarSubquery`] nodes+/// that read from a shared+/// [`ScalarSubqueryResults`] container.+///+/// The physical planner builds this context from the set of uncorrelated scalar+/// subqueries it has scheduled for a subtree. It is then passed explicitly+/// through `create_physical_expr` so that function can find the slot index for+/// each [`Subquery`].+///+/// An empty [`PhysicalPlanningContext`] (the [`Default`]) is what every+/// non-physical-planner caller passes; if such a caller encounters a scalar+/// subquery, `create_physical_expr` returns a `not_impl_err`.+///+/// [`Expr::ScalarSubquery`]: crate::Expr::ScalarSubquery+/// [`Subquery`]: crate::logical_plan::Subquery+#[derive(Clone, Debug, Default)]+pub struct PhysicalPlanningContext {+ indexes: HashMap<crate::logical_plan::Subquery, SubqueryIndex>,+ results: ScalarSubqueryResults,+}++impl PhysicalPlanningContext {+ /// Create a [`PhysicalPlanningContext`] from an index map and a shared+ /// results container. The index map must use the same indices as slots in+ /// `results`.+ pub fn new(+ indexes: HashMap<crate::logical_plan::Subquery, SubqueryIndex>,+ results: ScalarSubqueryResults,+ ) -> Self {+ Self { indexes, results }+ }++ /// Returns the slot index assigned to `subquery`, if any.+ pub fn index_of(+ &self,+ subquery: &crate::logical_plan::Subquery,+ ) -> Option<SubqueryIndex> {+ self.indexes.get(subquery).copied()+ }++ /// Returns the shared results container.+ pub fn results(&self) -> &ScalarSubqueryResults {+ &self.results+ }+}++/// Index of a scalar subquery within a [`ScalarSubqueryResults`] container.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+pub struct SubqueryIndex(usize);++impl SubqueryIndex {+ /// Creates a new subquery index.+ pub const fn new(index: usize) -> Self {+ Self(index)+ }++ /// Returns the underlying slot index.+ pub const fn as_usize(self) -> usize {+ self.0+ }+}++/// Shared results container for uncorrelated scalar subqueries.+///+/// Each entry corresponds to one scalar subquery, identified by its index.+/// Each slot is populated at execution time by `ScalarSubqueryExec`, read by+/// `ScalarSubqueryExpr` instances that share this container, and cleared when+/// the plan is reset for re-execution.+#[derive(Clone, Default)]+pub struct ScalarSubqueryResults {+ slots: Arc<Vec<Mutex<Option<ScalarValue>>>>,+}++impl ScalarSubqueryResults {+ /// Creates a new shared results container with `n` empty slots.+ pub fn new(n: usize) -> Self {+ Self {+ slots: Arc::new((0..n).map(|_| Mutex::new(None)).collect()),+ }+ }++ /// Returns the scalar value stored at `index`, if it has been populated.+ pub fn get(&self, index: SubqueryIndex) -> Option<ScalarValue> {+ let slot = self.slots.get(index.as_usize())?;+ slot.lock().unwrap().clone()+ }++ /// Stores `value` in the slot at `index`.+ pub fn set(&self, index: SubqueryIndex, value: ScalarValue) -> Result<()> {+ let Some(slot) = self.slots.get(index.as_usize()) else {+ return internal_err!(+ "ScalarSubqueryResults: result index {} is out of bounds",+ index.as_usize()+ );+ };++ let mut slot = slot.lock().unwrap();+ if slot.is_some() {+ return internal_err!(+ "ScalarSubqueryResults: result for index {} was already populated",+ index.as_usize()+ );+ }+ *slot = Some(value);++ Ok(())+ }++ /// Clears all populated results so the container can be reused.+ pub fn clear(&self) {+ for slot in self.slots.iter() {+ *slot.lock().unwrap() = None;+ }+ }++ /// Returns true if `this` and `other` point to the same shared container.+ pub fn ptr_eq(this: &Self, other: &Self) -> bool {+ Arc::ptr_eq(&this.slots, &other.slots)+ }+}++impl fmt::Debug for ScalarSubqueryResults {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_list()+ .entries(self.slots.iter().map(|slot| slot.lock().unwrap().clone()))+ .finish()+ }+}++impl PartialEq for ScalarSubqueryResults {+ fn eq(&self, other: &Self) -> bool {+ Self::ptr_eq(self, other)+ }+}++impl Eq for ScalarSubqueryResults {}++impl Hash for ScalarSubqueryResults {+ fn hash<H: Hasher>(&self, state: &mut H) {+ Arc::as_ptr(&self.slots).hash(state);+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn scalar_subquery_results_set_and_get() -> Result<()> {+ let results = ScalarSubqueryResults::new(1);+ assert_eq!(results.get(SubqueryIndex::new(0)), None);++ results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(42)))?;+ assert_eq!(+ results.get(SubqueryIndex::new(0)),+ Some(ScalarValue::Int32(Some(42)))+ );+ assert!(+ results+ .set(SubqueryIndex::new(0), ScalarValue::Int32(Some(7)))+ .is_err()+ );++ Ok(())+ }++ #[test]+ fn scalar_subquery_results_clear() -> Result<()> {+ let results = ScalarSubqueryResults::new(1);+ results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(42)))?;++ results.clear();++ assert_eq!(results.get(SubqueryIndex::new(0)), None);+ results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(7)))?;+ assert_eq!(+ results.get(SubqueryIndex::new(0)),+ Some(ScalarValue::Int32(Some(7)))+ );++ Ok(())+ }+}diff --git a/datafusion/functions-nested/src/array_any_match.rs b/datafusion/functions-nested/src/array_any_match.rsindex 8e6e67ed2a..b83c56e9e2 100644--- a/datafusion/functions-nested/src/array_any_match.rs+++ b/datafusion/functions-nested/src/array_any_match.rs@@ -255,6 +255,7 @@ mod tests {execution_props::ExecutionProps,expr::{HigherOrderFunction, LambdaVariable},lambda, lit,+ physical_planning_context::PhysicalPlanningContext,};use datafusion_physical_expr::create_physical_expr;@@ -290,6 +291,7 @@ mod tests {)),&schema,&ExecutionProps::new(),+ &PhysicalPlanningContext::default(),)?.evaluate(&RecordBatch::try_new(Arc::clone(schema.inner()),@@ -323,6 +325,7 @@ mod tests {)),&schema,&ExecutionProps::new(),+ &PhysicalPlanningContext::default(),)?.evaluate(&RecordBatch::try_new(Arc::clone(schema.inner()),diff --git a/datafusion/functions-nested/src/lambda_utils.rs b/datafusion/functions-nested/src/lambda_utils.rsindex ce596d0f1b..927ca5a514 100644--- a/datafusion/functions-nested/src/lambda_utils.rs+++ b/datafusion/functions-nested/src/lambda_utils.rs@@ -141,6 +141,7 @@ pub(crate) mod test_utils {execution_props::ExecutionProps,expr::{HigherOrderFunction, LambdaVariable},lambda,+ physical_planning_context::PhysicalPlanningContext,};use datafusion_physical_expr::create_physical_expr;@@ -175,6 +176,7 @@ pub(crate) mod test_utils {)),&schema,&ExecutionProps::new(),+ &PhysicalPlanningContext::default(),)?.evaluate(&RecordBatch::try_new(Arc::clone(schema.inner()),diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 39c8541b51..e4a22a3419 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -40,6 +40,7 @@ use datafusion_common::{tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter},};use datafusion_expr::expr::HigherOrderFunction;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{BinaryExpr, Case, ColumnarValue, Expr, ExprSchemable, Like, Operator, Volatility,and, binary::BinaryTypeCoercer, lit, or, preimage::PreimageResult,@@ -707,11 +708,15 @@ impl ConstEvaluator {return ConstSimplifyResult::NotSimplified(s, m);}- let phys_expr =- match create_physical_expr(&expr, &DUMMY_DF_SCHEMA, &self.execution_props) {- Ok(e) => e,- Err(err) => return ConstSimplifyResult::SimplifyRuntimeError(err, expr),- };+ let phys_expr = match create_physical_expr(+ &expr,+ &DUMMY_DF_SCHEMA,+ &self.execution_props,+ &PhysicalPlanningContext::default(),+ ) {+ Ok(e) => e,+ Err(err) => return ConstSimplifyResult::SimplifyRuntimeError(err, expr),+ };let metadata = phys_expr.return_field(DUMMY_BATCH.schema_ref()).ok()diff --git a/datafusion/optimizer/src/utils.rs b/datafusion/optimizer/src/utils.rsindex b29649e9ea..4ea1589cfa 100644--- a/datafusion/optimizer/src/utils.rs+++ b/datafusion/optimizer/src/utils.rs@@ -29,6 +29,7 @@ use datafusion_common::{Column, DFSchema, Result, ScalarValue};use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::expr::{Exists, InSubquery, SetComparison};use datafusion_expr::expr_rewriter::replace_col;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{ColumnarValue, Expr, logical_plan::LogicalPlan};use datafusion_physical_expr::create_physical_expr;use log::{debug, trace};@@ -233,8 +234,13 @@ fn evaluate_expr_with_null_column<'a>(let replaced_predicate = replace_col(predicate, &join_cols_to_replace)?;let coerced_predicate = coerce(replaced_predicate, &input_schema)?;- create_physical_expr(&coerced_predicate, &input_schema, &execution_props)?- .evaluate(&input_batch)+ create_physical_expr(+ &coerced_predicate,+ &input_schema,+ &execution_props,+ &PhysicalPlanningContext::default(),+ )?+ .evaluate(&input_batch)}fn coerce(expr: Expr, schema: &DFSchema) -> Result<Expr> {diff --git a/datafusion/physical-expr/src/aggregate.rs b/datafusion/physical-expr/src/aggregate.rsindex b774658679..013779cf8c 100644--- a/datafusion/physical-expr/src/aggregate.rs+++ b/datafusion/physical-expr/src/aggregate.rs@@ -51,6 +51,7 @@ use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams, NullTreatment, physical_name,};+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{AggregateUDF, Expr, ReversedUDAF, SetMonotonicity};use datafusion_expr_common::accumulator::Accumulator;use datafusion_expr_common::groups_accumulator::GroupsAccumulator;@@ -423,6 +424,7 @@ pub struct LoweredAggregateBuilder<'a> {logical_input_schema: &'a DFSchema,physical_input_schema: &'a Schema,execution_props: &'a ExecutionProps,+ planning_ctx: &'a PhysicalPlanningContext,}impl<'a> LoweredAggregateBuilder<'a> {@@ -430,12 +432,17 @@ impl<'a> LoweredAggregateBuilder<'a> {////// `logical_input_schema` is used to resolve logical expressions such as/// columns, while `physical_input_schema` is the input schema used by the- /// physical aggregate expression.+ /// physical aggregate expression. `planning_ctx` is used when creating+ /// physical expressions that reference uncorrelated scalar subqueries.+ /// Callers creating physical aggregates outside of physical planning should+ /// pass `&PhysicalPlanningContext::default()`, in which case converting a+ /// scalar-subquery expression returns a planning error.pub fn new(expr: &'a Expr,logical_input_schema: &'a DFSchema,physical_input_schema: &'a Schema,execution_props: &'a ExecutionProps,+ planning_ctx: &'a PhysicalPlanningContext,) -> Self {Self {expr,@@ -446,6 +453,7 @@ impl<'a> LoweredAggregateBuilder<'a> {logical_input_schema,physical_input_schema,execution_props,+ planning_ctx,}}@@ -484,6 +492,7 @@ impl<'a> LoweredAggregateBuilder<'a> {logical_input_schema,physical_input_schema,execution_props,+ planning_ctx,} = self;let (name, human_display, output_metadata, expr) = lower_aggregate_display(@@ -515,16 +524,29 @@ impl<'a> LoweredAggregateBuilder<'a> {physical_name(&expr)?};- let physical_args =- create_physical_exprs(args, logical_input_schema, execution_props)?;+ let physical_args = create_physical_exprs(+ args,+ logical_input_schema,+ execution_props,+ planning_ctx,+ )?;let filter = filter.as_ref().map(|filter| {- create_physical_expr(filter, logical_input_schema, execution_props)+ create_physical_expr(+ filter,+ logical_input_schema,+ execution_props,+ planning_ctx,+ )}).transpose()?;- let order_bys =- create_physical_sort_exprs(order_by, logical_input_schema, execution_props)?;+ let order_bys = create_physical_sort_exprs(+ order_by,+ logical_input_schema,+ execution_props,+ planning_ctx,+ )?;let ignore_nulls = null_treatment.unwrap_or(NullTreatment::RespectNulls)== NullTreatment::IgnoreNulls;@@ -1162,6 +1184,7 @@ mod tests {&logical_schema,&schema,&ExecutionProps::new(),+ &PhysicalPlanningContext::default(),).build()?;@@ -1185,6 +1208,7 @@ mod tests {&logical_schema,&schema,&ExecutionProps::new(),+ &PhysicalPlanningContext::default(),).with_human_display(expr.human_display().to_string()).build()?;diff --git a/datafusion/physical-expr/src/analysis.rs b/datafusion/physical-expr/src/analysis.rsindex 1dca36b75f..a00fc19ae9 100644--- a/datafusion/physical-expr/src/analysis.rs+++ b/datafusion/physical-expr/src/analysis.rs@@ -350,6 +350,7 @@ mod tests {use datafusion_common::{DFSchema, ScalarValue, assert_contains, stats::Precision};use datafusion_expr::{Expr, col, execution_props::ExecutionProps, interval_arithmetic::Interval, lit,+ physical_planning_context::PhysicalPlanningContext,};use crate::{AnalysisContext, create_physical_expr, expressions::Column};@@ -412,8 +413,13 @@ mod tests {for (expr, lower, upper) in test_cases {let boundaries = ExprBoundaries::try_new_unbounded(&schema).unwrap();let df_schema = DFSchema::try_from(Arc::clone(&schema)).unwrap();- let physical_expr =- create_physical_expr(&expr, &df_schema, &ExecutionProps::new()).unwrap();+ let physical_expr = create_physical_expr(+ &expr,+ &df_schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )+ .unwrap();let analysis_result = analyze(&physical_expr,AnalysisContext::new(boundaries),@@ -453,8 +459,13 @@ mod tests {for expr in test_cases {let boundaries = ExprBoundaries::try_new_unbounded(&schema).unwrap();let df_schema = DFSchema::try_from(Arc::clone(&schema)).unwrap();- let physical_expr =- create_physical_expr(&expr, &df_schema, &ExecutionProps::new()).unwrap();+ let physical_expr = create_physical_expr(+ &expr,+ &df_schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )+ .unwrap();let analysis_result = analyze(&physical_expr,AnalysisContext::new(boundaries),@@ -475,8 +486,13 @@ mod tests {let expected_error = "OR operator cannot yet propagate true intervals";let boundaries = ExprBoundaries::try_new_unbounded(&schema).unwrap();let df_schema = DFSchema::try_from(Arc::clone(&schema)).unwrap();- let physical_expr =- create_physical_expr(&expr, &df_schema, &ExecutionProps::new()).unwrap();+ let physical_expr = create_physical_expr(+ &expr,+ &df_schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )+ .unwrap();let analysis_error = analyze(&physical_expr,AnalysisContext::new(boundaries),diff --git a/datafusion/physical-expr/src/physical_expr.rs b/datafusion/physical-expr/src/physical_expr.rsindex cfc9866fc8..d45d0fe149 100644--- a/datafusion/physical-expr/src/physical_expr.rs+++ b/datafusion/physical-expr/src/physical_expr.rs@@ -26,6 +26,7 @@ use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{DFSchema, HashMap, ScalarValue, SplitPoint};use datafusion_common::{Result, plan_err};use datafusion_expr::execution_props::ExecutionProps;+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{Expr, Partitioning as LogicalPartitioning, SortExpr};use datafusion_expr_common::casts::try_cast_literal_to_type;@@ -190,47 +191,68 @@ pub fn create_lex_ordering(exprs,&df_schema,execution_props,+ &PhysicalPlanningContext::default(),)?));}Ok(all_sort_orders)}/// Create a physical sort expression from a logical expression+///+/// See [`create_physical_expr`] for details on the `planning_ctx` argument.pub fn create_physical_sort_expr(e: &SortExpr,input_dfschema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<PhysicalSortExpr> {- create_physical_expr(&e.expr, input_dfschema, execution_props).map(|expr| {- let options = SortOptions::new(!e.asc, e.nulls_first);- PhysicalSortExpr::new(expr, options)- })+ create_physical_expr(&e.expr, input_dfschema, execution_props, planning_ctx).map(+ |expr| {+ let options = SortOptions::new(!e.asc, e.nulls_first);+ PhysicalSortExpr::new(expr, options)+ },+ )}/// Create vector of physical sort expression from a vector of logical expression+///+/// See [`create_physical_expr`] for details on the `planning_ctx` argument.pub fn create_physical_sort_exprs(exprs: &[SortExpr],input_dfschema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Vec<PhysicalSortExpr>> {exprs.iter()- .map(|e| create_physical_sort_expr(e, input_dfschema, execution_props))+ .map(|e| {+ create_physical_sort_expr(e, input_dfschema, execution_props, planning_ctx)+ }).collect()}/// Create physical partitioning from logical partitioning.+///+/// See [`create_physical_expr`] for details on the `planning_ctx` argument.pub fn create_physical_partitioning(partitioning: &LogicalPartitioning,input_dfschema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Partitioning> {match partitioning {LogicalPartitioning::RoundRobinBatch(n) => Ok(Partitioning::RoundRobinBatch(*n)),LogicalPartitioning::Hash(exprs, partition_count) => {let exprs = exprs.iter()- .map(|expr| create_physical_expr(expr, input_dfschema, execution_props))+ .map(|expr| {+ create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )+ }).collect::<Result<Vec<_>>>()?;Ok(Partitioning::Hash(exprs, *partition_count))}@@ -239,6 +261,7 @@ pub fn create_physical_partitioning(range.ordering(),input_dfschema,execution_props,+ planning_ctx,)?;let Some(ordering) = LexOrdering::new(ordering) else {return plan_err!("Range partitioning requires non-empty ordering");diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rsindex d0d0508a10..3cdd64f7a7 100644--- a/datafusion/physical-expr/src/planner.rs+++ b/datafusion/physical-expr/src/planner.rs@@ -37,6 +37,7 @@ use datafusion_expr::expr::{Alias, Cast, HigherOrderFunction, InList, Lambda, LambdaVariable, Placeholder,ScalarFunction,};+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::var_provider::VarType;use datafusion_expr::var_provider::is_system_variables;use datafusion_expr::{@@ -63,6 +64,7 @@ use datafusion_expr::{/// # use datafusion_expr::{Expr, col, lit};/// # use datafusion_physical_expr::create_physical_expr;/// # use datafusion_expr::execution_props::ExecutionProps;+/// # use datafusion_expr::physical_planning_context::PhysicalPlanningContext;/// // For a logical expression `a = 1`, we can create a physical expression/// let expr = col("a").eq(lit(1));/// // To create a PhysicalExpr we need 1. a schema@@ -70,8 +72,11 @@ use datafusion_expr::{/// let df_schema = DFSchema::try_from(schema).unwrap();/// // 2. ExecutionProps/// let props = ExecutionProps::new();-/// // We can now create a PhysicalExpr:-/// let physical_expr = create_physical_expr(&expr, &df_schema, &props).unwrap();+/// // We can now create a PhysicalExpr. Expressions with no scalar+/// // subqueries use an empty `PhysicalPlanningContext`:+/// let physical_expr =+/// create_physical_expr(&expr, &df_schema, &props, &PhysicalPlanningContext::default())+/// .unwrap();/// ```////// # Example: Executing a PhysicalExpr to obtain [ColumnarValue]@@ -83,12 +88,15 @@ use datafusion_expr::{/// # use datafusion_expr::{Expr, col, lit, ColumnarValue};/// # use datafusion_physical_expr::create_physical_expr;/// # use datafusion_expr::execution_props::ExecutionProps;+/// # use datafusion_expr::physical_planning_context::PhysicalPlanningContext;/// # let expr = col("a").eq(lit(1));/// # let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);/// # let df_schema = DFSchema::try_from(schema.clone()).unwrap();/// # let props = ExecutionProps::new();/// // Given a PhysicalExpr, for `a = 1` we can evaluate it against a RecordBatch like this:-/// let physical_expr = create_physical_expr(&expr, &df_schema, &props).unwrap();+/// let physical_expr =+/// create_physical_expr(&expr, &df_schema, &props, &PhysicalPlanningContext::default())+/// .unwrap();/// // Input of [1,2,3]/// let input_batch = RecordBatch::try_from_iter(vec![/// ("a", Arc::new(Int32Array::from(vec![1, 2, 3])) as _)@@ -111,11 +119,20 @@ use datafusion_expr::{/// * `e` - The logical expression/// * `input_dfschema` - The DataFusion schema for the input, used to resolve `Column` references/// to qualified or unqualified fields by name.+/// * `execution_props` - Per-execution properties such as the query start time.+/// * `planning_ctx` - The [`PhysicalPlanningContext`] used to resolve+/// `Expr::ScalarSubquery` nodes. The physical planner threads the subquery+/// index map and shared results container from its `ScalarSubqueryExec`+/// construction into calls to `create_physical_expr`. Callers creating+/// physical expressions outside of physical planning should pass+/// `&PhysicalPlanningContext::default()`; converting a scalar subquery then returns a+/// planning error.#[cfg_attr(feature = "recursive_protection", recursive::recursive)]pub fn create_physical_expr(e: &Expr,input_dfschema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Arc<dyn PhysicalExpr>> {let input_schema = input_dfschema.as_arrow();@@ -131,7 +148,12 @@ pub fn create_physical_expr(new_metadata,)))} else {- Ok(create_physical_expr(expr, input_dfschema, execution_props)?)+ Ok(create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?)}}Expr::Column(c) => {@@ -167,12 +189,22 @@ pub fn create_physical_expr(Operator::IsNotDistinctFrom,lit(true),);- create_physical_expr(&binary_op, input_dfschema, execution_props)+ create_physical_expr(+ &binary_op,+ input_dfschema,+ execution_props,+ planning_ctx,+ )}Expr::IsNotTrue(expr) => {let binary_op =binary_expr(expr.as_ref().clone(), Operator::IsDistinctFrom, lit(true));- create_physical_expr(&binary_op, input_dfschema, execution_props)+ create_physical_expr(+ &binary_op,+ input_dfschema,+ execution_props,+ planning_ctx,+ )}Expr::IsFalse(expr) => {let binary_op = binary_expr(@@ -180,12 +212,22 @@ pub fn create_physical_expr(Operator::IsNotDistinctFrom,lit(false),);- create_physical_expr(&binary_op, input_dfschema, execution_props)+ create_physical_expr(+ &binary_op,+ input_dfschema,+ execution_props,+ planning_ctx,+ )}Expr::IsNotFalse(expr) => {let binary_op =binary_expr(expr.as_ref().clone(), Operator::IsDistinctFrom, lit(false));- create_physical_expr(&binary_op, input_dfschema, execution_props)+ create_physical_expr(+ &binary_op,+ input_dfschema,+ execution_props,+ planning_ctx,+ )}Expr::IsUnknown(expr) => {let binary_op = binary_expr(@@ -193,7 +235,12 @@ pub fn create_physical_expr(Operator::IsNotDistinctFrom,Expr::Literal(ScalarValue::Boolean(None), None),);- create_physical_expr(&binary_op, input_dfschema, execution_props)+ create_physical_expr(+ &binary_op,+ input_dfschema,+ execution_props,+ planning_ctx,+ )}Expr::IsNotUnknown(expr) => {let binary_op = binary_expr(@@ -201,12 +248,27 @@ pub fn create_physical_expr(Operator::IsDistinctFrom,Expr::Literal(ScalarValue::Boolean(None), None),);- create_physical_expr(&binary_op, input_dfschema, execution_props)+ create_physical_expr(+ &binary_op,+ input_dfschema,+ execution_props,+ planning_ctx,+ )}Expr::BinaryExpr(BinaryExpr { left, op, right }) => {// Create physical expressions for left and right operands- let lhs = create_physical_expr(left, input_dfschema, execution_props)?;- let rhs = create_physical_expr(right, input_dfschema, execution_props)?;+ let lhs = create_physical_expr(+ left,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;+ let rhs = create_physical_expr(+ right,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;// Note that the logical planner is responsible// for type coercion on the arguments (e.g. if one// argument was originally Int32 and one was@@ -229,10 +291,18 @@ pub fn create_physical_expr("LIKE does not support escape_char other than the backslash (\\)");}- let physical_expr =- create_physical_expr(expr, input_dfschema, execution_props)?;- let physical_pattern =- create_physical_expr(pattern, input_dfschema, execution_props)?;+ let physical_expr = create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;+ let physical_pattern = create_physical_expr(+ pattern,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;like(*negated,*case_insensitive,@@ -251,10 +321,18 @@ pub fn create_physical_expr(if escape_char.is_some() {return exec_err!("SIMILAR TO does not support escape_char yet");}- let physical_expr =- create_physical_expr(expr, input_dfschema, execution_props)?;- let physical_pattern =- create_physical_expr(pattern, input_dfschema, execution_props)?;+ let physical_expr = create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;+ let physical_pattern = create_physical_expr(+ pattern,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;similar_to(*negated, *case_insensitive, physical_expr, physical_pattern)}Expr::Case(case) => {@@ -263,6 +341,7 @@ pub fn create_physical_expr(e.as_ref(),input_dfschema,execution_props,+ planning_ctx,)?)} else {None@@ -272,10 +351,18 @@ pub fn create_physical_expr(.iter().map(|(w, t)| (w.as_ref(), t.as_ref())).unzip();- let when_expr =- create_physical_exprs(when_expr, input_dfschema, execution_props)?;- let then_expr =- create_physical_exprs(then_expr, input_dfschema, execution_props)?;+ let when_expr = create_physical_exprs(+ when_expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;+ let then_expr = create_physical_exprs(+ then_expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;let when_then_expr: Vec<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> =when_expr.iter()@@ -288,6 +375,7 @@ pub fn create_physical_expr(e.as_ref(),input_dfschema,execution_props,+ planning_ctx,)?)} else {None@@ -295,7 +383,7 @@ pub fn create_physical_expr(Ok(expressions::case(expr, when_then_expr, else_expr)?)}Expr::Cast(Cast { expr, field }) => expressions::cast_with_target_field(- create_physical_expr(expr, input_dfschema, execution_props)?,+ create_physical_expr(expr, input_dfschema, execution_props, planning_ctx)?,input_schema,Arc::clone(field),None,@@ -314,31 +402,45 @@ pub fn create_physical_expr(}expressions::try_cast(- create_physical_expr(expr, input_dfschema, execution_props)?,+ create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?,input_schema,field.data_type().clone(),)}- Expr::Not(expr) => {- expressions::not(create_physical_expr(expr, input_dfschema, execution_props)?)- }+ Expr::Not(expr) => expressions::not(create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?),Expr::Negative(expr) => expressions::negative(- create_physical_expr(expr, input_dfschema, execution_props)?,+ create_physical_expr(expr, input_dfschema, execution_props, planning_ctx)?,input_schema,),Expr::IsNull(expr) => expressions::is_null(create_physical_expr(expr,input_dfschema,execution_props,+ planning_ctx,)?),Expr::IsNotNull(expr) => expressions::is_not_null(create_physical_expr(expr,input_dfschema,execution_props,+ planning_ctx,)?),Expr::ScalarFunction(ScalarFunction { func, args }) => {- let physical_args =- create_physical_exprs(args, input_dfschema, execution_props)?;+ let physical_args = create_physical_exprs(+ args,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;let config_options = match execution_props.config_options.as_ref() {Some(config_options) => Arc::clone(config_options),None => Arc::new(ConfigOptions::default()),@@ -357,9 +459,20 @@ pub fn create_physical_expr(low,high,}) => {- let value_expr = create_physical_expr(expr, input_dfschema, execution_props)?;- let low_expr = create_physical_expr(low, input_dfschema, execution_props)?;- let high_expr = create_physical_expr(high, input_dfschema, execution_props)?;+ let value_expr = create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;+ let low_expr =+ create_physical_expr(low, input_dfschema, execution_props, planning_ctx)?;+ let high_expr = create_physical_expr(+ high,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;// rewrite the between into the two binary operatorslet binary_expr = binary(@@ -394,17 +507,25 @@ pub fn create_physical_expr(Ok(expressions::lit(ScalarValue::Boolean(None)))}_ => {- let value_expr =- create_physical_expr(expr, input_dfschema, execution_props)?;+ let value_expr = create_physical_expr(+ expr,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;- let list_exprs =- create_physical_exprs(list, input_dfschema, execution_props)?;+ let list_exprs = create_physical_exprs(+ list,+ input_dfschema,+ execution_props,+ planning_ctx,+ )?;expressions::in_list(value_expr, list_exprs, negated, input_schema)}},Expr::ScalarSubquery(sq) => {- match execution_props.subquery_indexes.get(sq) {- Some(&index) => {+ match planning_ctx.index_of(sq) {+ Some(index) => {let schema = sq.subquery.schema();if schema.fields().len() != 1 {return plan_err!(@@ -418,7 +539,7 @@ pub fn create_physical_expr(dt,nullable,index,- execution_props.subquery_results.clone(),+ planning_ctx.results().clone(),)))}None => {@@ -495,9 +616,19 @@ pub fn create_physical_expr(.clone().with_qualified_lambda_variables(&qualifier, &lambda.params);- create_physical_expr(arg, &lambda_schema, &execution_props)+ create_physical_expr(+ arg,+ &lambda_schema,+ &execution_props,+ planning_ctx,+ )}- _ => create_physical_expr(arg, input_dfschema, execution_props),+ _ => create_physical_expr(+ arg,+ input_dfschema,+ execution_props,+ planning_ctx,+ ),}).collect::<Result<_>>()?;@@ -515,7 +646,7 @@ pub fn create_physical_expr(}Expr::Lambda(Lambda { params, body }) => expressions::lambda(params,- create_physical_expr(body, input_dfschema, execution_props)?,+ create_physical_expr(body, input_dfschema, execution_props, planning_ctx)?,),Expr::LambdaVariable(LambdaVariable {name,@@ -572,17 +703,22 @@ pub fn create_physical_expr(}/// Create vector of Physical Expression from a vector of logical expression+///+/// See [`create_physical_expr`] for details on the `planning_ctx` argument.pub fn create_physical_exprs<'a, I>(exprs: I,input_dfschema: &DFSchema,execution_props: &ExecutionProps,+ planning_ctx: &PhysicalPlanningContext,) -> Result<Vec<Arc<dyn PhysicalExpr>>>whereI: IntoIterator<Item = &'a Expr>,{exprs.into_iter()- .map(|expr| create_physical_expr(expr, input_dfschema, execution_props))+ .map(|expr| {+ create_physical_expr(expr, input_dfschema, execution_props, planning_ctx)+ }).collect()}@@ -591,7 +727,13 @@ pub fn logical2physical(expr: &Expr, schema: &Schema) -> Arc<dyn PhysicalExpr> {// TODO this makes a deep copy of the Schema. Should take SchemaRef instead and avoid deep copylet df_schema = schema.clone().to_dfschema().unwrap();let execution_props = ExecutionProps::new();- create_physical_expr(expr, &df_schema, &execution_props).unwrap()+ create_physical_expr(+ expr,+ &df_schema,+ &execution_props,+ &PhysicalPlanningContext::default(),+ )+ .unwrap()}#[cfg(test)]@@ -608,7 +750,12 @@ mod tests {fn lower_cast_expr(expr: &Expr, schema: &Schema) -> Result<Arc<dyn PhysicalExpr>> {let df_schema = DFSchema::try_from(schema.clone())?;- create_physical_expr(expr, &df_schema, &ExecutionProps::new())+ create_physical_expr(+ expr,+ &df_schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )}fn as_planner_cast(physical: &Arc<dyn PhysicalExpr>) -> &expressions::CastExpr {@@ -623,7 +770,12 @@ mod tests {let schema = Schema::new(vec![Field::new("letter", DataType::Utf8, false)]);let df_schema = DFSchema::try_from_qualified_schema("data", &schema)?;- let p = create_physical_expr(&expr, &df_schema, &ExecutionProps::new())?;+ let p = create_physical_expr(+ &expr,+ &df_schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )?;let batch = RecordBatch::try_new(Arc::new(schema),@@ -728,8 +880,12 @@ mod tests {let df_schema = DFSchema::try_from(schema)?;// This should not stack overflow- let _physical_expr =- create_physical_expr(&expr, &df_schema, &ExecutionProps::new())?;+ let _physical_expr = create_physical_expr(+ &expr,+ &df_schema,+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )?;Ok(())}diff --git a/datafusion/physical-expr/src/scalar_subquery.rs b/datafusion/physical-expr/src/scalar_subquery.rsindex f7270f8054..473b52a5cb 100644--- a/datafusion/physical-expr/src/scalar_subquery.rs+++ b/datafusion/physical-expr/src/scalar_subquery.rs@@ -24,7 +24,7 @@ use std::sync::Arc;use arrow::datatypes::{DataType, Field, FieldRef, Schema};use arrow::record_batch::RecordBatch;use datafusion_common::{Result, internal_datafusion_err};-use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};+use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};use datafusion_expr_common::columnar_value::ColumnarValue;use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rsindex 74de1f11bf..2e04b5456b 100644--- a/datafusion/physical-plan/src/scalar_subquery.rs+++ b/datafusion/physical-plan/src/scalar_subquery.rs@@ -29,7 +29,7 @@ use std::sync::Arc;use datafusion_common::{Result, ScalarValue, Statistics, exec_err, internal_err};use datafusion_execution::TaskContext;-use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};+use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};use crate::execution_plan::{CardinalityEffect, ExecutionPlan, PlanProperties};use crate::joins::utils::{OnceAsync, OnceFut};@@ -202,9 +202,9 @@ impl ExecutionPlan for ScalarSubqueryExec {) -> Result<SendableRecordBatchStream> {let subqueries = self.subqueries.clone();let results = self.results.clone();- let subquery_ctx = Arc::clone(&context);+ let planning_ctx = Arc::clone(&context);let mut subquery_future = self.subquery_future.try_once(move || {- Ok(async move { execute_subqueries(subqueries, results, subquery_ctx).await })+ Ok(async move { execute_subqueries(subqueries, results, planning_ctx).await })})?;let input = Arc::clone(&self.input);let schema = self.schema();diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex e5f8aa072d..4f72668813 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -53,7 +53,7 @@ use datafusion_datasource_parquet::source::ParquetSource;#[cfg(feature = "parquet")]use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{FunctionRegistry, TaskContext};-use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};+use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};use datafusion_functions_table::generate_series::{Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex cbd6fd912a..d87efcf986 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -111,7 +111,7 @@ use datafusion_expr::{Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, HigherOrderUDF,ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,WindowFrame, WindowFrameBound, WindowUDF,- execution_props::{ScalarSubqueryResults, SubqueryIndex},+ physical_planning_context::{ScalarSubqueryResults, SubqueryIndex},};use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;use datafusion_functions_aggregate::array_agg::array_agg_udaf;diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex 28ceb27d63..67f90c649e 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -681,3 +681,98 @@ months are ignored, as in PostgreSQL. The result keeps the input time's unitunit is truncated -- so `time(s) + interval '1 nanosecond'` is a no-op.See [PR #23279](https://github.com/apache/datafusion/pull/23279) for details.++### Scalar-subquery state moved to an explicit `PhysicalPlanningContext`++The `subquery_indexes` and `subquery_results` public fields on+`datafusion_expr::execution_props::ExecutionProps` have been removed. They were+added in `54.0.0` as the channel through which the physical planner passed+uncorrelated scalar-subquery state to functions that create physical+`Arc<dyn PhysicalExpr>` values from logical `Expr` values.++That state is now carried by a dedicated+`datafusion_expr::physical_planning_context::PhysicalPlanningContext` passed explicitly+through functions and planner traits. Unlike `ExecutionProps`, which applies+throughout the planning of an entire query, this context is scoped to the+logical plan subtree currently being converted. This removes the need for the+physical planner to clone and mutate a `SessionState`, is a prerequisite for+letting the planner take `&dyn Session`, and lets `ExtensionPlanner`+implementations create physical+expressions containing scalar subqueries against the same subquery state as the+rest of the plan.++The following functions take a new trailing+`planning_ctx: &PhysicalPlanningContext` parameter:++- `datafusion_physical_expr::create_physical_expr` / `create_physical_exprs`+- `datafusion_physical_expr::create_physical_sort_expr` /+ `create_physical_sort_exprs` / `create_physical_partitioning`+- `datafusion::physical_planner::create_window_expr` /+ `create_window_expr_with_name`+- `datafusion_physical_expr::aggregate::LoweredAggregateBuilder::new`++The planner traits changed accordingly:++- `PhysicalPlanner::create_physical_expr` takes+ `planning_ctx: &PhysicalPlanningContext`+- `ExtensionPlanner::plan_extension` and `plan_table_scan` receive+ `planning_ctx: &PhysicalPlanningContext` and should forward it to+ `PhysicalPlanner::create_physical_expr` when creating physical expressions++Convenience methods such as `SessionContext::create_physical_expr` and+`SessionState::create_physical_expr` are unchanged.++**Who is affected:**++- Code calling the functions above: pass+ `&PhysicalPlanningContext::default()` unless you are creating physical+ expressions as part of a physical plan that contains uncorrelated scalar+ subqueries.+- Custom `PhysicalPlanner` or `ExtensionPlanner` implementations: add the new+ parameter and forward it.+- Code that read or wrote `execution_props.subquery_indexes` /+ `execution_props.subquery_results`: build a `PhysicalPlanningContext` instead.++**Migration guide:**++When creating a physical expression outside of physical planning, pass an empty+context:++```rust,ignore+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;+use datafusion_physical_expr::create_physical_expr;++// Before+let phys = create_physical_expr(&expr, &schema, &props)?;++// After+let phys = create_physical_expr(+ &expr,+ &schema,+ &props,+ &PhysicalPlanningContext::default(),+)?;+```++For `ExtensionPlanner` implementations, accept and forward the context:++```rust,ignore+async fn plan_extension(+ &self,+ planner: &dyn PhysicalPlanner,+ node: &dyn UserDefinedLogicalNode,+ logical_inputs: &[&LogicalPlan],+ physical_inputs: &[Arc<dyn ExecutionPlan>],+ session_state: &SessionState,+ planning_ctx: &PhysicalPlanningContext, // new parameter+) -> Result<Option<Arc<dyn ExecutionPlan>>> {+ for expr in node.expressions() {+ // Forward the context so scalar subqueries in this node's+ // expressions resolve against the plan's subquery state+ planner.create_physical_expr(&expr, node.schema(), session_state, planning_ctx)?;+ }+ // ...+}+```++See [PR #23649](https://github.com/apache/datafusion/pull/23649) for details.