Apache DataFusion SQL Query Engine
perf: Implement physical execution of uncorrelated scalar subqueries (#21240)
Neil Conway1bb588ed0039
diff --git a/.github/workflows/large_files.yml b/.github/workflows/large_files.ymlindex 12a5599189..d5fb586144 100644--- a/.github/workflows/large_files.yml+++ b/.github/workflows/large_files.yml@@ -34,9 +34,9 @@ jobs:fetch-depth: 0- name: Check size of new Git objectsenv:- # 1 MB ought to be enough for anybody.+ # 1.5 MB ought to be enough for anybody.# TODO in case we may want to consciously commit a bigger file to the repo without using Git LFS we may disable the check e.g. with a label- MAX_FILE_SIZE_BYTES: 1048576+ MAX_FILE_SIZE_BYTES: 1572864shell: bashrun: |if [ "${{ github.event_name }}" = "merge_group" ]; thendiff --git a/Cargo.lock b/Cargo.lockindex 9d60540d9c..b567d0b93c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1802,6 +1802,7 @@ dependencies = ["flate2","futures","glob",+ "indexmap 2.14.0","insta","itertools 0.14.0","liblzma",diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rsindex 22f8589d6f..d82bd2097c 100644--- a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs@@ -17,9 +17,9 @@//! See `main.rs` for how to run it.//!-//! This example demonstrates how to use the `PhysicalExtensionCodec` trait's-//! interception methods (`serialize_physical_plan` and `deserialize_physical_plan`)-//! to implement custom serialization logic.+//! This example demonstrates how to use the `PhysicalProtoConverterExtension`+//! trait's interception methods (`execution_plan_to_proto` and+//! `proto_to_execution_plan`) to implement custom serialization logic.//!//! The key insight is that `FileScanConfig::expr_adapter_factory` is NOT serialized by//! default. This example shows how to:@@ -28,9 +28,10 @@//! 3. Store the inner DataSourceExec (without adapter) as a child in the extension's inputs field//! 4. Unwrap and restore the adapter during deserialization//!-//! This demonstrates nested serialization (protobuf outer, JSON inner) and the power-//! of the `PhysicalExtensionCodec` interception pattern. Both plan and expression-//! serialization route through the codec, enabling interception at every node in the tree.+//! This demonstrates nested serialization (protobuf outer, JSON inner) and the+//! power of `PhysicalProtoConverterExtension`. Both plan and expression+//! serialization route through converter hooks, enabling interception at every+//! node in the tree.use std::fmt::Debug;use std::sync::Arc;@@ -61,7 +62,7 @@ use datafusion_proto::bytes::{use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;use datafusion_proto::physical_plan::{- PhysicalExtensionCodec, PhysicalProtoConverterExtension,+ PhysicalExtensionCodec, PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,};use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;use datafusion_proto::protobuf::{@@ -177,7 +178,7 @@ pub async fn adapter_serialization() -> Result<()> {println!("\n=== Example Complete! ===");println!("Key takeaways:");println!(- " 1. PhysicalExtensionCodec provides serialize_physical_plan/deserialize_physical_plan hooks"+ " 1. PhysicalProtoConverterExtension provides execution_plan_to_proto/proto_to_execution_plan hooks");println!(" 2. Custom metadata can be wrapped as PhysicalExtensionNode");println!(" 3. Nested serialization (protobuf + JSON) works seamlessly");@@ -303,9 +304,10 @@ impl PhysicalExtensionCodec for AdapterPreservingCodec {_node: Arc<dyn ExecutionPlan>,_buf: &mut Vec<u8>,) -> Result<()> {- // We don't need this for the example - we use serialize_physical_plan instead+ // We don't need this for the example - adapter wrapping happens in+ // `execution_plan_to_proto` instead.not_impl_err!(- "try_encode not used - adapter wrapping happens in serialize_physical_plan"+ "try_encode not used - adapter wrapping happens in execution_plan_to_proto")}}@@ -371,9 +373,8 @@ impl PhysicalProtoConverterExtension for AdapterPreservingCodec {// Interception point: override deserialization to unwrap adaptersfn proto_to_execution_plan(&self,- ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,proto: &PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {// Check if this is our custom extension wrapperif let Some(PhysicalPlanType::Extension(extension)) = &proto.physical_plan_type@@ -395,11 +396,7 @@ impl PhysicalProtoConverterExtension for AdapterPreservingCodec {let inner_proto = &extension.inputs[0];// Deserialize the inner plan- let inner_plan = inner_proto.try_into_physical_plan_with_converter(- ctx,- extension_codec,- self,- )?;+ let inner_plan = self.default_proto_to_execution_plan(inner_proto, ctx)?;// Recreate the adapter factorylet adapter_factory = create_adapter_factory(&payload.adapter_metadata.tag);@@ -409,17 +406,16 @@ impl PhysicalProtoConverterExtension for AdapterPreservingCodec {}// Not our extension - use default deserialization- proto.try_into_physical_plan_with_converter(ctx, extension_codec, self)+ self.default_proto_to_execution_plan(proto, ctx)}fn proto_to_physical_expr(&self,proto: &PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>> {- parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)+ parse_physical_expr_with_converter(proto, input_schema, ctx, self)}fn physical_expr_to_proto(diff --git a/datafusion-examples/examples/proto/expression_deduplication.rs b/datafusion-examples/examples/proto/expression_deduplication.rsindex ec076b5249..26d246b2ef 100644--- a/datafusion-examples/examples/proto/expression_deduplication.rs+++ b/datafusion-examples/examples/proto/expression_deduplication.rs@@ -17,8 +17,9 @@//! See `main.rs` for how to run it.//!-//! This example demonstrates how to use the `PhysicalExtensionCodec` trait's-//! interception methods to implement expression deduplication during deserialization.+//! This example demonstrates how to use the+//! `PhysicalProtoConverterExtension` trait's interception methods to+//! implement expression deduplication during deserialization.//!//! This pattern is inspired by PR #18192, which introduces expression caching//! to reduce memory usage when deserializing plans with duplicate expressions.@@ -29,8 +30,9 @@//! 2. Reduce memory allocation during deserialization//! 3. Enable downstream optimizations that rely on Arc pointer equality//!-//! This demonstrates the decorator pattern enabled by the `PhysicalExtensionCodec` trait,-//! where all expression serialization/deserialization routes through the codec methods.+//! This demonstrates the decorator pattern enabled by+//! `PhysicalProtoConverterExtension`, where physical-expression+//! serialization and deserialization route through converter hooks.use std::collections::HashMap;use std::fmt::Debug;@@ -49,7 +51,7 @@ use datafusion::prelude::SessionContext;use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;use datafusion_proto::physical_plan::{- DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,+ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalPlanDecodeContext,PhysicalProtoConverterExtension,};use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};@@ -202,11 +204,10 @@ impl PhysicalExtensionCodec for CachingCodec {impl PhysicalProtoConverterExtension for CachingCodec {fn proto_to_execution_plan(&self,- ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,proto: &PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {- proto.try_into_physical_plan_with_converter(ctx, extension_codec, self)+ self.default_proto_to_execution_plan(proto, ctx)}fn execution_plan_to_proto(@@ -225,9 +226,8 @@ impl PhysicalProtoConverterExtension for CachingCodec {fn proto_to_physical_expr(&self,proto: &PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>> {// Create cache key from protobuf byteslet mut key = Vec::new();@@ -249,8 +249,7 @@ impl PhysicalProtoConverterExtension for CachingCodec {}// Cache miss - deserialize and store- let expr =- parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)?;+ let expr = parse_physical_expr_with_converter(proto, input_schema, ctx, self)?;// Store in cache{diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex a2a07d4598..0a3783de9d 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -144,6 +144,7 @@ datafusion-session = { workspace = true }datafusion-sql = { workspace = true, optional = true }flate2 = { workspace = true, optional = true }futures = { workspace = true }+indexmap = { workspace = true }itertools = { workspace = true }liblzma = { workspace = true, optional = true }log = { workspace = true }diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex a025446aa3..3b2c7a78e8 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -63,6 +63,7 @@ use arrow::datatypes::Schema;use arrow_schema::Field;use datafusion_catalog::ScanArgs;use datafusion_common::Column;+use datafusion_common::HashMap as DFHashMap;use datafusion_common::display::ToStringifiedPlan;use datafusion_common::format::ExplainAnalyzeCategories;use datafusion_common::tree_node::{@@ -78,11 +79,13 @@ use datafusion_common::{use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::memory::MemorySourceConfig;use datafusion_expr::dml::{CopyTo, InsertOp};+use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams, Alias, GroupingSet, NullTreatment,WindowFunction, WindowFunctionParams, physical_name,};use datafusion_expr::expr_rewriter::unnormalize_cols;+use datafusion_expr::logical_plan::Subquery;use datafusion_expr::logical_plan::builder::wrap_projection_for_join_if_necessary;use datafusion_expr::utils::{expr_to_columns, split_conjunction};use datafusion_expr::{@@ -101,11 +104,13 @@ use datafusion_physical_plan::execution_plan::InvariantLevel;use datafusion_physical_plan::joins::PiecewiseMergeJoinExec;use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;use datafusion_physical_plan::recursive_query::RecursiveQueryExec;+use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink};use datafusion_physical_plan::unnest::ListUnnest;use async_trait::async_trait;use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper};use futures::{StreamExt, TryStreamExt};+use indexmap::IndexSet;use itertools::{Itertools, multiunzip};use log::debug;use tokio::sync::Mutex;@@ -379,8 +384,93 @@ impl DefaultPhysicalPlanner {Ok(())}- /// Create a physical plan from a logical plan- async fn create_initial_plan(+ /// Collect uncorrelated scalar subqueries. We don't descend into nested+ /// subqueries here: each call to `create_initial_plan` handles subqueries+ /// at its level and then recurses in order to handle nested subqueries.+ #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // Subquery contains Arc with interior mutability but is intentionally used as hash key+ fn collect_scalar_subqueries(plan: &LogicalPlan) -> Vec<Subquery> {+ let mut subqueries = IndexSet::new();+ plan.apply(|node| {+ for expr in node.expressions() {+ expr.apply(|e| {+ if let Expr::ScalarSubquery(sq) = e+ && sq.outer_ref_columns.is_empty()+ {+ subqueries.insert(sq.clone());+ }+ Ok(TreeNodeRecursion::Continue)+ })+ .expect("infallible");+ }+ Ok(TreeNodeRecursion::Continue)+ })+ .expect("infallible");+ subqueries.into_iter().collect()+ }++ /// Create a physical plan from a logical plan.+ ///+ /// Uncorrelated scalar subqueries in the plan's own expressions are+ /// collected, planned as separate physical plans, and each assigned an+ /// index in a shared [`ScalarSubqueryResults`] container that will hold its+ /// result at execution time. The index map and shared results container are+ /// registered in [`ExecutionProps`] so that [`create_physical_expr`] can+ /// convert `Expr::ScalarSubquery` into [`ScalarSubqueryExpr`] nodes that+ /// read from that container.+ ///+ /// The resulting physical plan is wrapped in a [`ScalarSubqueryExec`] node+ /// that executes those subquery plans before any data flows through the+ /// main plan. If a subquery itself contains nested uncorrelated subqueries,+ /// the recursive call produces its own [`ScalarSubqueryExec`] inside the+ /// subquery plan — each level manages only its own subqueries.+ ///+ /// Returns a [`BoxFuture`] rather than using `async fn` because of+ /// this recursion.+ ///+ /// [`ScalarSubqueryExpr`]: datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr+ /// [`BoxFuture`]: futures::future::BoxFuture+ fn create_initial_plan<'a>(+ &'a self,+ logical_plan: &'a LogicalPlan,+ session_state: &'a SessionState,+ ) -> futures::future::BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(async move {+ let all_subqueries = Self::collect_scalar_subqueries(logical_plan);+ let (links, index_map) = self+ .plan_scalar_subqueries(all_subqueries, session_state)+ .await?;++ if links.is_empty() {+ return self+ .create_initial_plan_inner(logical_plan, session_state)+ .await;+ }++ // Create the shared `ScalarSubqueryResults` container and register+ // it in `ExecutionProps` so that `create_physical_expr` can resolve+ // `Expr::ScalarSubquery` into `ScalarSubqueryExpr` nodes. We clone+ // the `SessionState` so these are available throughout physical+ // planning without mutating the caller's state.+ //+ // Ideally, the subquery state would live in a dedicated planning+ // context rather than in `ExecutionProps`. It's here because+ // `create_physical_expr` only receives `&ExecutionProps`.+ let results = ScalarSubqueryResults::new(links.len());+ let mut owned = session_state.clone();+ owned.execution_props_mut().subquery_indexes = index_map;+ owned.execution_props_mut().subquery_results = results.clone();+ let session_state = Cow::Owned(owned);++ let plan = self+ .create_initial_plan_inner(logical_plan, &session_state)+ .await?;+ Ok(Arc::new(ScalarSubqueryExec::new(plan, links, results)))+ })+ }++ /// Inner physical planning that converts a logical plan tree into an+ /// execution plan tree without collecting scalar subqueries.+ async fn create_initial_plan_inner(&self,logical_plan: &LogicalPlan,session_state: &SessionState,@@ -545,6 +635,7 @@ impl DefaultPhysicalPlanner {session_state: &SessionState,children: ChildrenContainer,) -> Result<Arc<dyn ExecutionPlan>> {+ let execution_props = session_state.execution_props();let exec_node: Arc<dyn ExecutionPlan> = match node {// Leaves (no children)LogicalPlan::TableScan(scan) => {@@ -601,7 +692,7 @@ impl DefaultPhysicalPlanner {.map(|row| {row.iter().map(|expr| {- self.create_physical_expr(expr, schema, session_state)+ create_physical_expr(expr, schema, execution_props)}).collect::<Result<Vec<Arc<dyn PhysicalExpr>>>>()})@@ -860,13 +951,7 @@ impl DefaultPhysicalPlanner {let logical_schema = node.schema();let window_expr = window_expr.iter()- .map(|e| {- create_window_expr(- e,- logical_schema,- session_state.execution_props(),- )- })+ .map(|e| create_window_expr(e, logical_schema, execution_props)).collect::<Result<Vec<_>>>()?;let can_repartition = session_state.config().target_partitions() > 1@@ -971,7 +1056,7 @@ impl DefaultPhysicalPlanner {group_expr,logical_input_schema,&physical_input_schema,- session_state,+ execution_props,)?;let agg_filter = aggr_expr@@ -981,7 +1066,7 @@ impl DefaultPhysicalPlanner {e,logical_input_schema,&physical_input_schema,- session_state.execution_props(),+ execution_props,)}).collect::<Result<Vec<_>>>()?;@@ -1075,8 +1160,8 @@ impl DefaultPhysicalPlanner {)?)}LogicalPlan::Projection(Projection { input, expr, .. }) => self- .create_project_physical_exec(- session_state,+ .create_project_physical_exec_with_props(+ execution_props,children.one()?,input,expr,@@ -1086,9 +1171,8 @@ impl DefaultPhysicalPlanner {}) => {let physical_input = children.one()?;let input_dfschema = input.schema();-let runtime_expr =- self.create_physical_expr(predicate, input_dfschema, session_state)?;+ create_physical_expr(predicate, input_dfschema, execution_props)?;let input_schema = input.schema();let filter = match self.try_plan_async_exprs(@@ -1136,7 +1220,9 @@ impl DefaultPhysicalPlanner {.options().optimizer.default_filter_selectivity;- Arc::new(filter.with_default_selectivity(selectivity)?)+ let filter_exec: Arc<dyn ExecutionPlan> =+ Arc::new(filter.with_default_selectivity(selectivity)?);+ filter_exec}LogicalPlan::Repartition(Repartition {input,@@ -1152,11 +1238,7 @@ impl DefaultPhysicalPlanner {let runtime_expr = expr.iter().map(|e| {- self.create_physical_expr(- e,- input_dfschema,- session_state,- )+ create_physical_expr(e, input_dfschema, execution_props)}).collect::<Result<Vec<_>>>()?;Partitioning::Hash(runtime_expr, *n)@@ -1177,11 +1259,8 @@ impl DefaultPhysicalPlanner {}) => {let physical_input = children.one()?;let input_dfschema = input.as_ref().schema();- let sort_exprs = create_physical_sort_exprs(- expr,- input_dfschema,- session_state.execution_props(),- )?;+ let sort_exprs =+ create_physical_sort_exprs(expr, input_dfschema, execution_props)?;let Some(ordering) = LexOrdering::new(sort_exprs) else {return internal_err!("SortExec requires at least one sort expression"@@ -1308,8 +1387,8 @@ impl DefaultPhysicalPlanner {(true,LogicalPlan::Projection(Projection { input, expr, .. }),- ) => self.create_project_physical_exec(- session_state,+ ) => self.create_project_physical_exec_with_props(+ execution_props,physical_left,input,expr,@@ -1321,8 +1400,8 @@ impl DefaultPhysicalPlanner {(true,LogicalPlan::Projection(Projection { input, expr, .. }),- ) => self.create_project_physical_exec(- session_state,+ ) => self.create_project_physical_exec_with_props(+ execution_props,physical_right,input,expr,@@ -1387,7 +1466,6 @@ impl DefaultPhysicalPlanner {// All equi-join keys are columns now, create physical join planlet left_df_schema = left.schema();let right_df_schema = right.schema();- let execution_props = session_state.execution_props();let join_on = keys.iter().map(|(l, r)| {@@ -1493,7 +1571,7 @@ impl DefaultPhysicalPlanner {let filter_expr = create_physical_expr(expr,&filter_df_schema,- session_state.execution_props(),+ execution_props,)?;let column_indices = join_utils::JoinFilter::build_column_indices(left_field_indices,@@ -1613,12 +1691,12 @@ impl DefaultPhysicalPlanner {let on_left = create_physical_expr(lhs_logical,left_df_schema,- session_state.execution_props(),+ execution_props,)?;let on_right = create_physical_expr(rhs_logical,right_df_schema,- session_state.execution_props(),+ execution_props,)?;Arc::new(PiecewiseMergeJoinExec::try_new(@@ -1688,7 +1766,12 @@ impl DefaultPhysicalPlanner {// If plan was mutated previously then need to create the ExecutionPlan// for the new Projection that was applied on top.if let Some((input, expr)) = new_project {- self.create_project_physical_exec(session_state, join, input, expr)?+ self.create_project_physical_exec_with_props(+ execution_props,+ join,+ input,+ expr,+ )?} else {join}@@ -1782,7 +1865,7 @@ impl DefaultPhysicalPlanner {group_expr: &[Expr],input_dfschema: &DFSchema,input_schema: &Schema,- session_state: &SessionState,+ execution_props: &ExecutionProps,) -> Result<PhysicalGroupBy> {if group_expr.len() == 1 {match &group_expr[0] {@@ -1791,25 +1874,25 @@ impl DefaultPhysicalPlanner {grouping_sets,input_dfschema,input_schema,- session_state,+ execution_props,)}Expr::GroupingSet(GroupingSet::Cube(exprs)) => create_cube_physical_expr(exprs,input_dfschema,input_schema,- session_state,+ execution_props,),Expr::GroupingSet(GroupingSet::Rollup(exprs)) => {create_rollup_physical_expr(exprs,input_dfschema,input_schema,- session_state,+ execution_props,)}expr => Ok(PhysicalGroupBy::new_single(vec![tuple_err((- self.create_physical_expr(expr, input_dfschema, session_state),+ create_physical_expr(expr, input_dfschema, execution_props),physical_name(expr),))?])),}@@ -1823,7 +1906,7 @@ impl DefaultPhysicalPlanner {.iter().map(|e| {tuple_err((- self.create_physical_expr(e, input_dfschema, session_state),+ create_physical_expr(e, input_dfschema, execution_props),physical_name(e),))})@@ -1847,7 +1930,7 @@ fn merge_grouping_set_physical_expr(grouping_sets: &[Vec<Expr>],input_dfschema: &DFSchema,input_schema: &Schema,- session_state: &SessionState,+ execution_props: &ExecutionProps,) -> Result<PhysicalGroupBy> {let num_groups = grouping_sets.len();let mut all_exprs: Vec<Expr> = vec![];@@ -1861,14 +1944,14 @@ fn merge_grouping_set_physical_expr(grouping_set_expr.push(get_physical_expr_pair(expr,input_dfschema,- session_state,+ execution_props,)?);null_exprs.push(get_null_physical_expr_pair(expr,input_dfschema,input_schema,- session_state,+ execution_props,)?);}}@@ -1898,7 +1981,7 @@ fn create_cube_physical_expr(exprs: &[Expr],input_dfschema: &DFSchema,input_schema: &Schema,- session_state: &SessionState,+ execution_props: &ExecutionProps,) -> Result<PhysicalGroupBy> {let num_of_exprs = exprs.len();let num_groups = num_of_exprs * num_of_exprs;@@ -1913,10 +1996,14 @@ fn create_cube_physical_expr(expr,input_dfschema,input_schema,- session_state,+ execution_props,)?);- all_exprs.push(get_physical_expr_pair(expr, input_dfschema, session_state)?)+ all_exprs.push(get_physical_expr_pair(+ expr,+ input_dfschema,+ execution_props,+ )?)}let mut groups: Vec<Vec<bool>> = Vec::with_capacity(num_groups);@@ -1940,7 +2027,7 @@ fn create_rollup_physical_expr(exprs: &[Expr],input_dfschema: &DFSchema,input_schema: &Schema,- session_state: &SessionState,+ execution_props: &ExecutionProps,) -> Result<PhysicalGroupBy> {let num_of_exprs = exprs.len();@@ -1956,10 +2043,14 @@ fn create_rollup_physical_expr(expr,input_dfschema,input_schema,- session_state,+ execution_props,)?);- all_exprs.push(get_physical_expr_pair(expr, input_dfschema, session_state)?)+ all_exprs.push(get_physical_expr_pair(+ expr,+ input_dfschema,+ execution_props,+ )?)}for total in 0..=num_of_exprs {@@ -1984,10 +2075,9 @@ fn get_null_physical_expr_pair(expr: &Expr,input_dfschema: &DFSchema,input_schema: &Schema,- session_state: &SessionState,+ execution_props: &ExecutionProps,) -> Result<(Arc<dyn PhysicalExpr>, String)> {- let physical_expr =- create_physical_expr(expr, input_dfschema, session_state.execution_props())?;+ let physical_expr = create_physical_expr(expr, input_dfschema, execution_props)?;let physical_name = physical_name(&expr.clone())?;let data_type = physical_expr.data_type(input_schema)?;@@ -2056,10 +2146,9 @@ fn qualify_join_schema_sides(fn get_physical_expr_pair(expr: &Expr,input_dfschema: &DFSchema,- session_state: &SessionState,+ execution_props: &ExecutionProps,) -> Result<(Arc<dyn PhysicalExpr>, String)> {- let physical_expr =- create_physical_expr(expr, input_dfschema, session_state.execution_props())?;+ let physical_expr = create_physical_expr(expr, input_dfschema, execution_props)?;let physical_name = physical_name(expr)?;Ok((physical_expr, physical_name))}@@ -2839,9 +2928,37 @@ impl DefaultPhysicalPlanner {Ok(mem_exec)}- fn create_project_physical_exec(+ /// Build physical plans for scalar subqueries and assign each an ordinal+ /// `SubqueryIndex`. Returns the links (plan + index) and a map from logical+ /// `Subquery` to its index.+ async fn plan_scalar_subqueries(&self,+ subqueries: Vec<Subquery>,session_state: &SessionState,+ ) -> Result<(Vec<ScalarSubqueryLink>, DFHashMap<Subquery, SubqueryIndex>)> {+ let mut links = Vec::with_capacity(subqueries.len());+ let mut index_map = DFHashMap::with_capacity(subqueries.len());+ for sq in subqueries {+ // Callers deduplicate, but guard against accidental double-planning.+ if index_map.contains_key(&sq) {+ continue;+ }+ let physical_plan = self+ .create_initial_plan(&sq.subquery, session_state)+ .await?;+ let index = SubqueryIndex::new(links.len());+ links.push(ScalarSubqueryLink {+ plan: physical_plan,+ index,+ });+ index_map.insert(sq, index);+ }+ Ok((links, index_map))+ }++ fn create_project_physical_exec_with_props(+ &self,+ execution_props: &ExecutionProps,input_exec: Arc<dyn ExecutionPlan>,input: &Arc<LogicalPlan>,expr: &[Expr],@@ -2880,7 +2997,7 @@ impl DefaultPhysicalPlanner {};let physical_expr =- self.create_physical_expr(e, input_logical_schema, session_state);+ create_physical_expr(e, input_logical_schema, execution_props);tuple_err((physical_expr, physical_name))})@@ -3124,7 +3241,7 @@ mod tests {use arrow_schema::SchemaRef;use datafusion_common::config::ConfigOptions;use datafusion_common::{- DFSchemaRef, TableReference, ToDFSchema as _, assert_contains,+ DFSchemaRef, TableReference, ToDFSchema as _, assert_batches_eq, assert_contains,};use datafusion_execution::TaskContext;use datafusion_execution::runtime_env::RuntimeEnv;@@ -3158,6 +3275,16 @@ mod tests {.await}+ async fn plan_sql(query: &str) -> Result<Arc<dyn ExecutionPlan>> {+ let ctx = SessionContext::new();+ ctx.sql(query).await?.create_physical_plan().await+ }++ async fn collect_sql(query: &str) -> Result<Vec<RecordBatch>> {+ let ctx = SessionContext::new();+ ctx.sql(query).await?.collect().await+ }+#[tokio::test]async fn test_all_operators() -> Result<()> {let logical_plan = test_csv_scan()@@ -3181,6 +3308,132 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn scalar_subquery_in_sort_expr_plans() -> Result<()> {+ let plan = plan_sql(+ "SELECT x \+ FROM (VALUES (2), (1)) AS t(x) \+ ORDER BY x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y))",+ )+ .await?;++ assert_contains!(format!("{plan:?}"), "ScalarSubqueryExec");+ Ok(())+ }++ #[tokio::test]+ async fn scalar_subquery_in_sort_expr_executes() -> Result<()> {+ let batches = collect_sql(+ "SELECT x \+ FROM (VALUES (2), (1), (3)) AS t(x) \+ ORDER BY x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) DESC",+ )+ .await?;++ assert_batches_eq!(+ &[+ "+---+", "| x |", "+---+", "| 3 |", "| 2 |", "| 1 |", "+---+",+ ],+ &batches+ );+ Ok(())+ }++ #[tokio::test]+ async fn scalar_subquery_in_aggregate_arg_plans() -> Result<()> {+ let plan = plan_sql(+ "SELECT sum(x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y))) \+ FROM (VALUES (2), (1)) AS t(x)",+ )+ .await?;++ assert_contains!(format!("{plan:?}"), "ScalarSubqueryExec");+ Ok(())+ }++ #[tokio::test]+ async fn scalar_subquery_in_aggregate_arg_executes() -> Result<()> {+ let batches = collect_sql(+ "SELECT sum(x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y))) AS s \+ FROM (VALUES (2), (1)) AS t(x)",+ )+ .await?;++ assert_batches_eq!(+ &["+----+", "| s |", "+----+", "| 43 |", "+----+",],+ &batches+ );+ Ok(())+ }++ #[tokio::test]+ async fn scalar_subquery_in_join_on_plans() -> Result<()> {+ let plan = plan_sql(+ "SELECT l.x, r.y \+ FROM (VALUES (1), (2)) AS l(x) \+ JOIN (VALUES (11), (12)) AS r(y) \+ ON l.x + (SELECT 10) = r.y",+ )+ .await?;++ let formatted = format!("{plan:?}");+ assert_contains!(&formatted, "ScalarSubqueryExec");+ assert!(+ formatted.contains("HashJoinExec")+ || formatted.contains("SortMergeJoinExec")+ || formatted.contains("NestedLoopJoinExec")+ );+ Ok(())+ }++ #[tokio::test]+ async fn scalar_subquery_mixed_correlated_and_uncorrelated_executes() -> Result<()> {+ let query = "SELECT t.x, \+ (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) + \+ (SELECT count(*) FROM (VALUES (1), (1), (2)) AS v(z) WHERE v.z = t.x) AS total \+ FROM (VALUES (1), (2), (3)) AS t(x) \+ ORDER BY x";+ let plan = plan_sql(query).await?;++ let formatted = format!("{plan:?}");+ assert_eq!(formatted.matches("ScalarSubqueryExec").count(), 1);+ assert!(+ formatted.contains("HashJoinExec")+ || formatted.contains("SortMergeJoinExec")+ || formatted.contains("NestedLoopJoinExec")+ );++ let batches = collect_sql(query).await?;+ assert_batches_eq!(+ &[+ "+---+-------+",+ "| x | total |",+ "+---+-------+",+ "| 1 | 22 |",+ "| 2 | 21 |",+ "| 3 | 20 |",+ "+---+-------+",+ ],+ &batches+ );+ Ok(())+ }++ #[tokio::test]+ async fn scalar_subquery_in_projection_and_filter_plans() -> Result<()> {+ let plan = plan_sql(+ "SELECT x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) \+ FROM (VALUES (2), (1)) AS t(x) \+ WHERE x > (SELECT min(y) FROM (VALUES (0), (1)) AS v(y))",+ )+ .await?;++ let formatted = format!("{plan:?}");+ // All uncorrelated scalar subqueries are hoisted to a single root node.+ assert_eq!(formatted.matches("ScalarSubqueryExec").count(), 1);+ Ok(())+ }+#[tokio::test]async fn test_create_cube_expr() -> Result<()> {let logical_plan = test_csv_scan().await?.build()?;@@ -3198,7 +3451,7 @@ mod tests {&exprs,logical_input_schema,physical_input_schema,- &session_state,+ session_state.execution_props(),);insta::assert_debug_snapshot!(cube, @r#"@@ -3329,7 +3582,7 @@ mod tests {&exprs,logical_input_schema,physical_input_schema,- &session_state,+ session_state.execution_props(),);insta::assert_debug_snapshot!(rollup, @r#"diff --git a/datafusion/expr/src/execution_props.rs b/datafusion/expr/src/execution_props.rsindex ff53dc495f..24d0f333a6 100644--- a/datafusion/expr/src/execution_props.rs+++ b/datafusion/expr/src/execution_props.rs@@ -18,9 +18,13 @@use crate::var_provider::{VarProvider, VarType};use chrono::{DateTime, Utc};use datafusion_common::HashMap;+use datafusion_common::ScalarValue;use datafusion_common::alias::AliasGenerator;use datafusion_common::config::ConfigOptions;-use std::sync::Arc;+use datafusion_common::{Result, internal_err};+use std::fmt;+use std::hash::{Hash, Hasher};+use std::sync::{Arc, Mutex};/// Holds properties and scratch state used while optimizing a [`LogicalPlan`]/// and translating it into an executable physical plan, such as the statement@@ -59,6 +63,12 @@ pub struct ExecutionProps {pub config_options: Option<Arc<ConfigOptions>>,/// Providers for scalar variablespub var_providers: Option<HashMap<VarType, Arc<dyn VarProvider + Send + Sync>>>,+ /// Maps each logical `Subquery` to its index in `subquery_results`.+ /// Populated by the physical planner before calling `create_physical_expr`.+ pub subquery_indexes: HashMap<crate::logical_plan::Subquery, SubqueryIndex>,+ /// Shared results container for uncorrelated scalar subquery values.+ /// Populated at execution time by `ScalarSubqueryExec`.+ pub subquery_results: ScalarSubqueryResults,}impl Default for ExecutionProps {@@ -75,6 +85,8 @@ impl ExecutionProps {alias_generator: Arc::new(AliasGenerator::new()),config_options: None,var_providers: None,+ subquery_indexes: HashMap::new(),+ subquery_results: ScalarSubqueryResults::default(),}}@@ -102,8 +114,7 @@ impl ExecutionProps {&*self}- /// Registers a variable provider, returning the existing- /// provider, if any+ /// Registers a variable provider, returning the existing provider, if anypub fn add_var_provider(&mut self,var_type: VarType,@@ -136,15 +147,149 @@ impl ExecutionProps {}}+/// Index of a scalar subquery within a [`ScalarSubqueryResults`] container.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+pub struct SubqueryIndex(usize);++impl SubqueryIndex {+ /// Creates a new subquery index.+ pub const fn new(index: usize) -> Self {+ Self(index)+ }++ /// Returns the underlying slot index.+ pub const fn as_usize(self) -> usize {+ self.0+ }+}++/// Shared results container for uncorrelated scalar subqueries.+///+/// Each entry corresponds to one scalar subquery, identified by its index.+/// Each slot is populated at execution time by `ScalarSubqueryExec`, read by+/// `ScalarSubqueryExpr` instances that share this container, and cleared when+/// the plan is reset for re-execution.+#[derive(Clone, Default)]+pub struct ScalarSubqueryResults {+ slots: Arc<Vec<Mutex<Option<ScalarValue>>>>,+}++impl ScalarSubqueryResults {+ /// Creates a new shared results container with `n` empty slots.+ pub fn new(n: usize) -> Self {+ Self {+ slots: Arc::new((0..n).map(|_| Mutex::new(None)).collect()),+ }+ }++ /// Returns the scalar value stored at `index`, if it has been populated.+ pub fn get(&self, index: SubqueryIndex) -> Option<ScalarValue> {+ let slot = self.slots.get(index.as_usize())?;+ slot.lock().unwrap().clone()+ }++ /// Stores `value` in the slot at `index`.+ pub fn set(&self, index: SubqueryIndex, value: ScalarValue) -> Result<()> {+ let Some(slot) = self.slots.get(index.as_usize()) else {+ return internal_err!(+ "ScalarSubqueryResults: result index {} is out of bounds",+ index.as_usize()+ );+ };++ let mut slot = slot.lock().unwrap();+ if slot.is_some() {+ return internal_err!(+ "ScalarSubqueryResults: result for index {} was already populated",+ index.as_usize()+ );+ }+ *slot = Some(value);++ Ok(())+ }++ /// Clears all populated results so the container can be reused.+ pub fn clear(&self) {+ for slot in self.slots.iter() {+ *slot.lock().unwrap() = None;+ }+ }++ /// Returns true if `this` and `other` point to the same shared container.+ pub fn ptr_eq(this: &Self, other: &Self) -> bool {+ Arc::ptr_eq(&this.slots, &other.slots)+ }+}++impl fmt::Debug for ScalarSubqueryResults {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_list()+ .entries(self.slots.iter().map(|slot| slot.lock().unwrap().clone()))+ .finish()+ }+}++impl PartialEq for ScalarSubqueryResults {+ fn eq(&self, other: &Self) -> bool {+ Self::ptr_eq(self, other)+ }+}++impl Eq for ScalarSubqueryResults {}++impl Hash for ScalarSubqueryResults {+ fn hash<H: Hasher>(&self, state: &mut H) {+ Arc::as_ptr(&self.slots).hash(state);+ }+}+#[cfg(test)]mod test {use super::*;+#[test]fn debug() {let props = ExecutionProps::new();assert_eq!(- "ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None }",+ "ExecutionProps { query_execution_start_time: None, alias_generator: AliasGenerator { next_id: 1 }, config_options: None, var_providers: None, subquery_indexes: {}, subquery_results: [] }",format!("{props:?}"));}++ #[test]+ fn scalar_subquery_results_set_and_get() -> Result<()> {+ let results = ScalarSubqueryResults::new(1);+ assert_eq!(results.get(SubqueryIndex::new(0)), None);++ results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(42)))?;+ assert_eq!(+ results.get(SubqueryIndex::new(0)),+ Some(ScalarValue::Int32(Some(42)))+ );+ assert!(+ results+ .set(SubqueryIndex::new(0), ScalarValue::Int32(Some(7)))+ .is_err()+ );++ Ok(())+ }++ #[test]+ fn scalar_subquery_results_clear() -> Result<()> {+ let results = ScalarSubqueryResults::new(1);+ results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(42)))?;++ results.clear();++ assert_eq!(results.get(SubqueryIndex::new(0)), None);+ results.set(SubqueryIndex::new(0), ScalarValue::Int32(Some(7)))?;+ assert_eq!(+ results.get(SubqueryIndex::new(0)),+ Some(ScalarValue::Int32(Some(7)))+ );++ Ok(())+ }}diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rsindex 7dba4f0579..c3dffe553d 100644--- a/datafusion/expr/src/expr.rs+++ b/datafusion/expr/src/expr.rs@@ -2059,6 +2059,12 @@ impl Expr {.expect("exists closure is infallible")}+ /// Returns true if the expression contains a scalar subquery.+ pub fn contains_scalar_subquery(&self) -> bool {+ self.exists(|expr| Ok(matches!(expr, Expr::ScalarSubquery(_))))+ .expect("exists closure is infallible")+ }+/// Returns true if the expression node is volatile, i.e. whether it can return/// different results when evaluated multiple times with the same input./// Note: unlike [`Self::is_volatile`], this function does not consider inputs:diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex 9cc9843f35..db8b82fe87 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -3770,6 +3770,7 @@ impl PartialOrd for Aggregate {/// index among identical entries. For example, if the same set appears three/// times, the ordinals are 0, 1, 2 and this function returns 2./// Returns 0 when no grouping set is duplicated.+#[allow(clippy::allow_attributes, clippy::mutable_key_type)] // Expr contains Arc with interior mutability but is intentionally used as hash keyfn max_grouping_set_duplicate_ordinal(group_expr: &[Expr]) -> usize {if let Some(Expr::GroupingSet(GroupingSet::GroupingSets(sets))) = group_expr.first() {let mut counts: HashMap<&[Expr], usize> = HashMap::new();diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex 77991db1bf..ef9382a572 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -820,7 +820,7 @@ impl LogicalPlan {transform_down_up_with_subqueries_impl(self, &mut f_down, &mut f_up)}- /// Similarly to [`Self::apply`], calls `f` on this node and its inputs+ /// Similarly to [`Self::apply`], calls `f` on this node and its inputs,/// including subqueries that may appear in expressions such as `IN (SELECT/// ...)`.pub fn apply_subqueries<F: FnMut(&Self) -> Result<TreeNodeRecursion>>(@@ -833,9 +833,7 @@ impl LogicalPlan {| Expr::InSubquery(InSubquery { subquery, .. })| Expr::SetComparison(SetComparison { subquery, .. })| Expr::ScalarSubquery(subquery) => {- // use a synthetic plan so the collector sees a- // LogicalPlan::Subquery (even though it is- // actually a Subquery alias)+ // Wrap in LogicalPlan::Subquery to match f's signaturef(&LogicalPlan::Subquery(subquery.clone()))}_ => Ok(TreeNodeRecursion::Continue),@@ -900,4 +898,18 @@ impl LogicalPlan {})})}++ /// Similar to [`Self::map_subqueries`], but only applies `f` to+ /// uncorrelated subqueries (those with no outer column references).+ pub fn map_uncorrelated_subqueries<F: FnMut(Self) -> Result<Transformed<Self>>>(+ self,+ mut f: F,+ ) -> Result<Transformed<Self>> {+ self.map_subqueries(|subquery_plan| match &subquery_plan {+ LogicalPlan::Subquery(sq) if sq.outer_ref_columns.is_empty() => {+ f(subquery_plan)+ }+ _ => Ok(Transformed::no(subquery_plan)),+ })+ }}diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rsindex 4213c23ccc..c02ba60247 100644--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs@@ -586,8 +586,12 @@ impl OptimizerRule for CommonSubexprEliminate {| LogicalPlan::Unnest(_)| LogicalPlan::RecursiveQuery(_) => {// This rule handles recursion itself in a `ApplyOrder::TopDown` like- // manner.- plan.map_children(|c| self.rewrite(c, config))?+ // manner. Process uncorrelated subqueries in expressions+ // (e.g., Expr::ScalarSubquery), then direct children.+ plan.map_uncorrelated_subqueries(|c| self.rewrite(c, config))?+ .transform_sibling(|plan| {+ plan.map_children(|c| self.rewrite(c, config))+ })?}};diff --git a/datafusion/optimizer/src/eliminate_cross_join.rs b/datafusion/optimizer/src/eliminate_cross_join.rsindex 3cb0516a6d..8306d4b54c 100644--- a/datafusion/optimizer/src/eliminate_cross_join.rs+++ b/datafusion/optimizer/src/eliminate_cross_join.rs@@ -212,7 +212,12 @@ fn rewrite_children(plan: LogicalPlan,config: &dyn OptimizerConfig,) -> Result<Transformed<LogicalPlan>> {- let transformed_plan = plan.map_children(|input| optimizer.rewrite(input, config))?;+ // Process uncorrelated subqueries in expressions, then direct children.+ let transformed_plan = plan+ .map_uncorrelated_subqueries(|input| optimizer.rewrite(input, config))?+ .transform_sibling(|plan| {+ plan.map_children(|input| optimizer.rewrite(input, config))+ })?;// recompute schema if the plan was transformedif transformed_plan.transformed {diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex 14badcf143..af944abc6f 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -135,9 +135,11 @@ fn optimize_projections(// their parents' required indices.match plan {LogicalPlan::Projection(proj) => {- return merge_consecutive_projections(proj)?.transform_data(|proj| {- rewrite_projection_given_requirements(proj, config, &indices)- });+ return merge_consecutive_projections(proj)?+ .transform_data(|proj| {+ rewrite_projection_given_requirements(proj, config, &indices)+ })?+ .transform_data(|plan| optimize_subqueries(plan, config));}LogicalPlan::Aggregate(aggregate) => {// Split parent requirements to GROUP BY and aggregate sections:@@ -222,7 +224,8 @@ fn optimize_projections(new_aggr_expr,).map(LogicalPlan::Aggregate)- });+ })?+ .transform_data(|plan| optimize_subqueries(plan, config));}LogicalPlan::Window(window) => {let input_schema = Arc::clone(window.input.schema());@@ -262,7 +265,8 @@ fn optimize_projections(.map(LogicalPlan::Window).map(Transformed::yes)}- });+ })?+ .transform_data(|plan| optimize_subqueries(plan, config));}LogicalPlan::TableScan(table_scan) => {let TableScan {@@ -283,7 +287,8 @@ fn optimize_projections(let new_scan =TableScan::try_new(table_name, source, Some(projection), filters, fetch)?;- return Ok(Transformed::yes(LogicalPlan::TableScan(new_scan)));+ return Transformed::yes(LogicalPlan::TableScan(new_scan))+ .transform_data(|plan| optimize_subqueries(plan, config));}// Other node types are handled below_ => {}@@ -476,6 +481,9 @@ fn optimize_projections()})?;+ let transformed_plan =+ transformed_plan.transform_data(|plan| optimize_subqueries(plan, config))?;+// If any of the children are transformed, we need to potentially update the plan's schemaif transformed_plan.transformed {transformed_plan.map_data(|plan| plan.recompute_schema())@@ -484,8 +492,19 @@ fn optimize_projections(}}-/// Merges consecutive projections.-///+/// Optimizes uncorrelated subquery plans embedded in expressions of the given+/// plan node (e.g., `Expr::ScalarSubquery`). `map_children` only visits direct+/// plan inputs, so subqueries must be handled separately.+fn optimize_subqueries(+ plan: LogicalPlan,+ config: &dyn OptimizerConfig,+) -> Result<Transformed<LogicalPlan>> {+ plan.map_uncorrelated_subqueries(|subquery_plan| {+ let indices = RequiredIndices::new_for_all_exprs(&subquery_plan);+ optimize_projections(subquery_plan, config, indices)+ })+}+/// Given a projection `proj`, this function attempts to merge it with a previous/// projection if it exists and if merging is beneficial. Merging is considered/// beneficial when expressions in the current projection are non-trivial anddiff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex e645509907..35b2e3e3b6 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -1144,8 +1144,16 @@ impl OptimizerRule for PushDownFilter {LogicalPlan::TableScan(scan) => {let filter_predicates = split_conjunction(&filter.predicate);- let (volatile_filters, non_volatile_filters): (Vec<&Expr>, Vec<&Expr>) =+ // Filters containing scalar subqueries cannot be pushed to+ // providers because the subquery result is not available+ // until execution time.+ let (subquery_filters, pushdown_candidates): (Vec<&Expr>, Vec<&Expr>) =filter_predicates+ .into_iter()+ .partition(|pred| pred.contains_scalar_subquery());++ let (volatile_filters, non_volatile_filters): (Vec<&Expr>, Vec<&Expr>) =+ pushdown_candidates.into_iter().partition(|pred| pred.is_volatile());@@ -1178,11 +1186,13 @@ impl OptimizerRule for PushDownFilter {.cloned().collect();- // Compose predicates to be of `Unsupported` or `Inexact` pushdown type, and also include volatile filters+ // Compose predicates to be of `Unsupported` or `Inexact` pushdown type,+ // and also include volatile and subquery-containing filterslet new_predicate: Vec<Expr> = zip.filter(|(_, res)| res != &TableProviderFilterPushDown::Exact).map(|(pred, _)| pred).chain(volatile_filters)+ .chain(subquery_filters).cloned().collect();diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rsindex c54cd287db..941fcffb79 100644--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-//! [`ScalarSubqueryToJoin`] rewriting scalar subquery filters to `JOIN`s+//! [`ScalarSubqueryToJoin`] rewriting correlated scalar subquery filters to `JOIN`suse std::collections::{BTreeSet, HashMap};use std::sync::Arc;@@ -36,8 +36,8 @@ use datafusion_expr::logical_plan::{JoinType, Subquery};use datafusion_expr::utils::conjunction;use datafusion_expr::{EmptyRelation, Expr, LogicalPlan, LogicalPlanBuilder, expr};-/// Optimizer rule for rewriting subquery filters to joins-/// and places additional projection on top of the filter, to preserve+/// Optimizer rule that rewrites correlated scalar subquery filters to joins and+/// places an additional projection on top of the filter, to preserve the/// original schema.#[derive(Default, Debug)]pub struct ScalarSubqueryToJoin {}@@ -48,10 +48,15 @@ impl ScalarSubqueryToJoin {Self::default()}- /// Finds expressions that have a scalar subquery in them (and recurses when found)+ /// Finds expressions that contain correlated scalar subqueries (and+ /// recurses when found).////// # Arguments- /// * `predicate` - A conjunction to split and search+ /// * `predicate` - A conjunction to split and search.+ /// * `alias_gen` - Generator used to produce unique aliases for each+ /// extracted scalar subquery (e.g. `__scalar_sq_1`, `__scalar_sq_2`).+ /// Each subquery is replaced by a column reference using the generated+ /// alias, and the same alias is later used to construct the join.////// Returns a tuple (subqueries, alias)fn extract_subquery_exprs(@@ -85,7 +90,7 @@ impl OptimizerRule for ScalarSubqueryToJoin {LogicalPlan::Filter(filter) => {// Optimization: skip the rest of the rule and its copies if// there are no scalar subqueries- if !contains_scalar_subquery(&filter.predicate) {+ if !contains_correlated_scalar_subquery(&filter.predicate) {return Ok(Transformed::no(LogicalPlan::Filter(filter)));}@@ -137,9 +142,13 @@ impl OptimizerRule for ScalarSubqueryToJoin {Ok(Transformed::yes(new_plan))}LogicalPlan::Projection(projection) => {- // Optimization: skip the rest of the rule and its copies if- // there are no scalar subqueries- if !projection.expr.iter().any(contains_scalar_subquery) {+ // Optimization: skip the rest of the rule and its copies if there+ // are no correlated scalar subqueries+ if !projection+ .expr+ .iter()+ .any(contains_correlated_scalar_subquery)+ {return Ok(Transformed::no(LogicalPlan::Projection(projection)));}@@ -226,11 +235,14 @@ impl OptimizerRule for ScalarSubqueryToJoin {}}-/// Returns true if the expression has a scalar subquery somewhere in it-/// false otherwise-fn contains_scalar_subquery(expr: &Expr) -> bool {- expr.exists(|expr| Ok(matches!(expr, Expr::ScalarSubquery(_))))- .expect("Inner is always Ok")+/// Returns true if the expression contains a correlated scalar subquery, false+/// otherwise. Uncorrelated scalar subqueries are handled by the physical+/// planner via `ScalarSubqueryExec` and do not need to be converted to joins.+fn contains_correlated_scalar_subquery(expr: &Expr) -> bool {+ expr.exists(|expr| {+ Ok(matches!(expr, Expr::ScalarSubquery(sq) if !sq.outer_ref_columns.is_empty()))+ })+ .expect("Inner is always Ok")}struct ExtractScalarSubQuery<'a> {@@ -243,19 +255,21 @@ impl TreeNodeRewriter for ExtractScalarSubQuery<'_> {fn f_down(&mut self, expr: Expr) -> Result<Transformed<Expr>> {match expr {- Expr::ScalarSubquery(subquery) => {- let subqry_alias = self.alias_gen.next("__scalar_sq");- self.sub_query_info- .push((subquery.clone(), subqry_alias.clone()));+ // Skip uncorrelated scalar subqueries+ Expr::ScalarSubquery(ref subquery)+ if !subquery.outer_ref_columns.is_empty() =>+ {+ let subquery = subquery.clone();let scalar_expr = subquery.subquery.head_output_expr()?.map_or(plan_err!("single expression required."), Ok)?;+ let subqry_alias = self.alias_gen.next("__scalar_sq");+ let col =+ create_col_from_scalar_expr(&scalar_expr, subqry_alias.clone())?;+ self.sub_query_info.push((subquery, subqry_alias));Ok(Transformed::new(- Expr::Column(create_col_from_scalar_expr(- &scalar_expr,- subqry_alias,- )?),+ Expr::Column(col),true,TreeNodeRecursion::Jump,))@@ -627,15 +641,13 @@ mod tests {plan,@r"Projection: customer.c_custkey [c_custkey:Int64]- Projection: customer.c_custkey, customer.c_name [c_custkey:Int64, c_name:Utf8]- Filter: customer.c_custkey = __scalar_sq_1.max(orders.o_custkey) [c_custkey:Int64, c_name:Utf8, max(orders.o_custkey):Int64;N]- Left Join: Filter: Boolean(true) [c_custkey:Int64, c_name:Utf8, max(orders.o_custkey):Int64;N]- TableScan: customer [c_custkey:Int64, c_name:Utf8]- SubqueryAlias: __scalar_sq_1 [max(orders.o_custkey):Int64;N]- Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]- Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]- Filter: orders.o_custkey = orders.o_custkey [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]- TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ Filter: customer.c_custkey = (<subquery>) [c_custkey:Int64, c_name:Utf8]+ Subquery: [max(orders.o_custkey):Int64;N]+ Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]+ Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]+ Filter: orders.o_custkey = orders.o_custkey [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ TableScan: customer [c_custkey:Int64, c_name:Utf8]")}@@ -1032,14 +1044,12 @@ mod tests {plan,@r"Projection: customer.c_custkey [c_custkey:Int64]- Projection: customer.c_custkey, customer.c_name [c_custkey:Int64, c_name:Utf8]- Filter: customer.c_custkey < __scalar_sq_1.max(orders.o_custkey) [c_custkey:Int64, c_name:Utf8, max(orders.o_custkey):Int64;N]- Left Join: Filter: Boolean(true) [c_custkey:Int64, c_name:Utf8, max(orders.o_custkey):Int64;N]- TableScan: customer [c_custkey:Int64, c_name:Utf8]- SubqueryAlias: __scalar_sq_1 [max(orders.o_custkey):Int64;N]- Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]- Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]- TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ Filter: customer.c_custkey < (<subquery>) [c_custkey:Int64, c_name:Utf8]+ Subquery: [max(orders.o_custkey):Int64;N]+ Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]+ Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ TableScan: customer [c_custkey:Int64, c_name:Utf8]")}@@ -1062,14 +1072,12 @@ mod tests {plan,@r"Projection: customer.c_custkey [c_custkey:Int64]- Projection: customer.c_custkey, customer.c_name [c_custkey:Int64, c_name:Utf8]- Filter: customer.c_custkey = __scalar_sq_1.max(orders.o_custkey) [c_custkey:Int64, c_name:Utf8, max(orders.o_custkey):Int64;N]- Left Join: Filter: Boolean(true) [c_custkey:Int64, c_name:Utf8, max(orders.o_custkey):Int64;N]- TableScan: customer [c_custkey:Int64, c_name:Utf8]- SubqueryAlias: __scalar_sq_1 [max(orders.o_custkey):Int64;N]- Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]- Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]- TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ Filter: customer.c_custkey = (<subquery>) [c_custkey:Int64, c_name:Utf8]+ Subquery: [max(orders.o_custkey):Int64;N]+ Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]+ Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ TableScan: customer [c_custkey:Int64, c_name:Utf8]")}@@ -1161,19 +1169,16 @@ mod tests {plan,@r"Projection: customer.c_custkey [c_custkey:Int64]- Projection: customer.c_custkey, customer.c_name [c_custkey:Int64, c_name:Utf8]- Filter: customer.c_custkey BETWEEN __scalar_sq_1.min(orders.o_custkey) AND __scalar_sq_2.max(orders.o_custkey) [c_custkey:Int64, c_name:Utf8, min(orders.o_custkey):Int64;N, max(orders.o_custkey):Int64;N]- Left Join: Filter: Boolean(true) [c_custkey:Int64, c_name:Utf8, min(orders.o_custkey):Int64;N, max(orders.o_custkey):Int64;N]- Left Join: Filter: Boolean(true) [c_custkey:Int64, c_name:Utf8, min(orders.o_custkey):Int64;N]- TableScan: customer [c_custkey:Int64, c_name:Utf8]- SubqueryAlias: __scalar_sq_1 [min(orders.o_custkey):Int64;N]- Projection: min(orders.o_custkey) [min(orders.o_custkey):Int64;N]- Aggregate: groupBy=[[]], aggr=[[min(orders.o_custkey)]] [min(orders.o_custkey):Int64;N]- TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]- SubqueryAlias: __scalar_sq_2 [max(orders.o_custkey):Int64;N]- Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]- Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]- TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ Filter: customer.c_custkey BETWEEN (<subquery>) AND (<subquery>) [c_custkey:Int64, c_name:Utf8]+ Subquery: [min(orders.o_custkey):Int64;N]+ Projection: min(orders.o_custkey) [min(orders.o_custkey):Int64;N]+ Aggregate: groupBy=[[]], aggr=[[min(orders.o_custkey)]] [min(orders.o_custkey):Int64;N]+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ Subquery: [max(orders.o_custkey):Int64;N]+ Projection: max(orders.o_custkey) [max(orders.o_custkey):Int64;N]+ Aggregate: groupBy=[[]], aggr=[[max(orders.o_custkey)]] [max(orders.o_custkey):Int64;N]+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ TableScan: customer [c_custkey:Int64, c_name:Utf8]")}diff --git a/datafusion/optimizer/tests/optimizer_integration.rs b/datafusion/optimizer/tests/optimizer_integration.rsindex ad7427acf9..b1ba313e1d 100644--- a/datafusion/optimizer/tests/optimizer_integration.rs+++ b/datafusion/optimizer/tests/optimizer_integration.rs@@ -136,15 +136,13 @@ fn subquery_filter_with_cast() -> Result<()> {assert_snapshot!(format!("{plan}"),@r#"- Projection: test.col_int32- Inner Join: Filter: CAST(test.col_int32 AS Float64) > __scalar_sq_1.avg(test.col_int32)- TableScan: test projection=[col_int32]- SubqueryAlias: __scalar_sq_1- Aggregate: groupBy=[[]], aggr=[[avg(CAST(test.col_int32 AS Float64))]]- Projection: test.col_int32- Filter: __common_expr_4 >= Date32("2002-05-08") AND __common_expr_4 <= Date32("2002-05-13")- Projection: CAST(test.col_utf8 AS Date32) AS __common_expr_4, test.col_int32- TableScan: test projection=[col_int32, col_utf8]+ Filter: CAST(test.col_int32 AS Float64) > (<subquery>)+ Subquery:+ Aggregate: groupBy=[[]], aggr=[[avg(CAST(test.col_int32 AS Float64))]]+ Projection: test.col_int32+ Filter: CAST(test.col_utf8 AS Date32) >= Date32("2002-05-08") AND CAST(test.col_utf8 AS Date32) <= Date32("2002-05-13")+ TableScan: test projection=[col_int32, col_utf8]+ TableScan: test projection=[col_int32]"#);Ok(())diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rsindex bedd348dab..9c567ce862 100644--- a/datafusion/physical-expr/src/lib.rs+++ b/datafusion/physical-expr/src/lib.rs@@ -40,6 +40,7 @@ mod physical_expr;pub mod planner;pub mod projection;mod scalar_function;+pub mod scalar_subquery;pub mod simplifier;pub mod statistics;pub mod utils;diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rsindex 0880f1a139..5d4c5d4ce1 100644--- a/datafusion/physical-expr/src/planner.rs+++ b/datafusion/physical-expr/src/planner.rs@@ -18,6 +18,7 @@use std::sync::Arc;use crate::ScalarFunctionExpr;+use crate::scalar_subquery::ScalarSubqueryExpr;use crate::{PhysicalExpr,expressions::{self, Column, Literal, binary, like, similar_to},@@ -396,6 +397,35 @@ pub fn create_physical_expr(expressions::in_list(value_expr, list_exprs, negated, input_schema)}},+ Expr::ScalarSubquery(sq) => {+ match execution_props.subquery_indexes.get(sq) {+ Some(&index) => {+ let schema = sq.subquery.schema();+ if schema.fields().len() != 1 {+ return plan_err!(+ "Scalar subquery must return exactly one column, got {}",+ schema.fields().len()+ );+ }+ let dt = schema.field(0).data_type().clone();+ let nullable = schema.field(0).is_nullable();+ Ok(Arc::new(ScalarSubqueryExpr::new(+ dt,+ nullable,+ index,+ execution_props.subquery_results.clone(),+ )))+ }+ None => {+ // Not found: either a correlated subquery that wasn't+ // rewritten to a join, or an uncorrelated one that wasn't+ // registered by the physical planner.+ not_impl_err!(+ "Physical plan does not support logical expression {e:?}"+ )+ }+ }+ }Expr::Placeholder(Placeholder { id, .. }) => {exec_err!("Placeholder '{id}' was not provided a value for execution.")}diff --git a/datafusion/physical-expr/src/scalar_subquery.rs b/datafusion/physical-expr/src/scalar_subquery.rsnew file mode 100644index 0000000000..ea00847151--- /dev/null+++ b/datafusion/physical-expr/src/scalar_subquery.rs@@ -0,0 +1,240 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Physical expression for uncorrelated scalar subqueries.++use std::fmt;+use std::hash::Hash;+use std::sync::Arc;++use arrow::datatypes::{DataType, Field, FieldRef, Schema};+use arrow::record_batch::RecordBatch;+use datafusion_common::{Result, internal_datafusion_err};+use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};+use datafusion_expr_common::columnar_value::ColumnarValue;+use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties};+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;++/// A physical expression whose value is provided by a scalar subquery.+///+/// Subquery execution is handled by `ScalarSubqueryExec`, which stores the+/// result in a shared [`ScalarSubqueryResults`] container. This expression+/// simply reads from that container at the appropriate index.+#[derive(Debug)]+pub struct ScalarSubqueryExpr {+ data_type: DataType,+ nullable: bool,+ /// Index of this subquery in the shared results container.+ index: SubqueryIndex,+ /// Shared results container populated by `ScalarSubqueryExec`.+ results: ScalarSubqueryResults,+}++impl ScalarSubqueryExpr {+ pub fn new(+ data_type: DataType,+ nullable: bool,+ index: SubqueryIndex,+ results: ScalarSubqueryResults,+ ) -> Self {+ Self {+ data_type,+ nullable,+ index,+ results,+ }+ }++ pub fn data_type(&self) -> &DataType {+ &self.data_type+ }++ pub fn nullable(&self) -> bool {+ self.nullable+ }++ /// Returns the index of this subquery in the shared results container.+ pub fn index(&self) -> SubqueryIndex {+ self.index+ }++ pub fn results(&self) -> &ScalarSubqueryResults {+ &self.results+ }+}++impl fmt::Display for ScalarSubqueryExpr {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match self.results.get(self.index) {+ Some(v) => write!(f, "scalar_subquery({v})"),+ None => write!(f, "scalar_subquery(<pending>)"),+ }+ }+}++// Two ScalarSubqueryExprs are considered the "same" if they refer to the+// same underlying shared results container and the same index within it.+impl Hash for ScalarSubqueryExpr {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.results.hash(state);+ self.index.hash(state);+ }+}++impl PartialEq for ScalarSubqueryExpr {+ fn eq(&self, other: &Self) -> bool {+ self.results == other.results && self.index == other.index+ }+}++impl Eq for ScalarSubqueryExpr {}++impl PhysicalExpr for ScalarSubqueryExpr {+ fn return_field(&self, _input_schema: &Schema) -> Result<FieldRef> {+ Ok(Arc::new(Field::new(+ "scalar_subquery",+ self.data_type.clone(),+ self.nullable,+ )))+ }++ fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {+ let value = self.results.get(self.index).ok_or_else(|| {+ internal_datafusion_err!(+ "ScalarSubqueryExpr evaluated before the subquery was executed"+ )+ })?;+ Ok(ColumnarValue::Scalar(value))+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ _children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(self)+ }++ fn get_properties(&self, _children: &[ExprProperties]) -> Result<ExprProperties> {+ Ok(ExprProperties::new_unknown().with_order(SortProperties::Singleton))+ }++ fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ write!(f, "(scalar subquery)")+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ use arrow::array::Int32Array;+ use arrow::datatypes::Field;+ use datafusion_common::ScalarValue;++ fn make_results(values: Vec<Option<ScalarValue>>) -> ScalarSubqueryResults {+ let results = ScalarSubqueryResults::new(values.len());+ for (index, value) in values.into_iter().enumerate() {+ if let Some(value) = value {+ results.set(SubqueryIndex::new(index), value).unwrap();+ }+ }+ results+ }++ #[test]+ fn test_evaluate_with_value() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ let a = Int32Array::from(vec![1, 2, 3]);+ let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)])?;++ let results = make_results(vec![Some(ScalarValue::Int32(Some(42)))]);+ let expr = ScalarSubqueryExpr::new(+ DataType::Int32,+ false,+ SubqueryIndex::new(0),+ results,+ );++ let result = expr.evaluate(&batch)?;+ match result {+ ColumnarValue::Scalar(ScalarValue::Int32(Some(42))) => {}+ other => panic!("Expected Scalar(Int32(42)), got {other:?}"),+ }+ Ok(())+ }++ #[test]+ fn test_evaluate_before_populated() {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ let a = Int32Array::from(vec![1]);+ let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();++ let results = ScalarSubqueryResults::new(1);+ let expr = ScalarSubqueryExpr::new(+ DataType::Int32,+ false,+ SubqueryIndex::new(0),+ results,+ );++ let result = expr.evaluate(&batch);+ assert!(result.is_err());+ }++ #[test]+ fn test_identity_equality() {+ let results = make_results(vec![None, None]);++ let e1a = ScalarSubqueryExpr::new(+ DataType::Int32,+ false,+ SubqueryIndex::new(0),+ results.clone(),+ );+ let e1b = ScalarSubqueryExpr::new(+ DataType::Int32,+ false,+ SubqueryIndex::new(0),+ results.clone(),+ );+ let e2 = ScalarSubqueryExpr::new(+ DataType::Int32,+ false,+ SubqueryIndex::new(1),+ results.clone(),+ );++ // Same container + same index → equal+ assert_eq!(e1a, e1b);+ // Same container, different index → not equal+ assert_ne!(e1a, e2);++ // Different container, same index → not equal+ let other_results = make_results(vec![None]);+ let e3 = ScalarSubqueryExpr::new(+ DataType::Int32,+ false,+ SubqueryIndex::new(0),+ other_results,+ );+ assert_ne!(e1a, e3);+ }+}diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex 54fc97c154..3005e97542 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -85,6 +85,7 @@ pub mod placeholder_row;pub mod projection;pub mod recursive_query;pub mod repartition;+pub mod scalar_subquery;pub mod sort_pushdown;pub mod sorts;pub mod spill;diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rsnew file mode 100644index 0000000000..b6ad7f91f0--- /dev/null+++ b/datafusion/physical-plan/src/scalar_subquery.rs@@ -0,0 +1,574 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Execution plan for uncorrelated scalar subqueries.+//!+//! [`ScalarSubqueryExec`] wraps a main input plan and a set of subquery plans.+//! At execution time, it runs each subquery exactly once, extracts the scalar+//! result, and populates a shared [`ScalarSubqueryResults`] container that+//! [`ScalarSubqueryExpr`] instances hold directly and read from by index.+//!+//! [`ScalarSubqueryExpr`]: datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr++use std::fmt;+use std::sync::Arc;++use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::{Result, ScalarValue, Statistics, exec_err, internal_err};+use datafusion_execution::TaskContext;+use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};+use datafusion_physical_expr::PhysicalExpr;++use crate::execution_plan::{CardinalityEffect, ExecutionPlan, PlanProperties};+use crate::joins::utils::{OnceAsync, OnceFut};+use crate::stream::RecordBatchStreamAdapter;+use crate::{DisplayAs, DisplayFormatType, SendableRecordBatchStream};++use futures::StreamExt;+use futures::TryStreamExt;++/// Links a scalar subquery's execution plan to its index in the shared results+/// container. The [`ScalarSubqueryExec`] that owns these links populates+/// `results[index]` at execution time, and [`ScalarSubqueryExpr`] instances+/// with the same index read from it.+///+/// [`ScalarSubqueryExpr`]: datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr+#[derive(Debug, Clone)]+pub struct ScalarSubqueryLink {+ /// The physical plan for the subquery.+ pub plan: Arc<dyn ExecutionPlan>,+ /// Index into the shared results container.+ pub index: SubqueryIndex,+}++/// Manages execution of uncorrelated scalar subqueries for a single plan+/// level.+///+/// From a query-results perspective, this node is a pass-through: it yields+/// the same batches as its main input and exists only to populate scalar+/// subquery results as a side effect before those batches are produced.+///+/// The first child node is the **main input plan**, whose batches are passed+/// through unchanged. The remaining children are **subquery plans**, each of+/// which must produce exactly zero or one row. Before any batches from the main+/// input are yielded, all subquery plans are executed and their scalar results+/// are stored in a shared [`ScalarSubqueryResults`] container owned by this+/// node. [`ScalarSubqueryExpr`] nodes embedded in the main input's expressions+/// hold the same container and read from it by index.+///+/// All subqueries are evaluated eagerly when the first output partition is+/// requested, before any rows from the main input are produced.+///+/// TODO: Consider overlapping computation of the subqueries with evaluating the+/// main query.+///+/// [`ScalarSubqueryExpr`]: datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr+#[derive(Debug)]+pub struct ScalarSubqueryExec {+ /// The main input plan whose output is passed through.+ input: Arc<dyn ExecutionPlan>,+ /// Subquery plans and their result indexes.+ subqueries: Vec<ScalarSubqueryLink>,+ /// Shared one-time async computation of subquery results.+ subquery_future: Arc<OnceAsync<()>>,+ /// Shared results container; the corresponding `ScalarSubqueryExpr`+ /// nodes in the input plan hold the same underlying container.+ results: ScalarSubqueryResults,+ /// Cached plan properties (copied from input).+ cache: Arc<PlanProperties>,+}++impl ScalarSubqueryExec {+ pub fn new(+ input: Arc<dyn ExecutionPlan>,+ subqueries: Vec<ScalarSubqueryLink>,+ results: ScalarSubqueryResults,+ ) -> Self {+ let cache = Arc::clone(input.properties());+ Self {+ input,+ subqueries,+ subquery_future: Arc::default(),+ results,+ cache,+ }+ }++ pub fn input(&self) -> &Arc<dyn ExecutionPlan> {+ &self.input+ }++ pub fn subqueries(&self) -> &[ScalarSubqueryLink] {+ &self.subqueries+ }++ pub fn results(&self) -> &ScalarSubqueryResults {+ &self.results+ }++ /// Returns a per-child bool vec that is `true` for the main input+ /// (child 0) and `false` for every subquery child.+ fn true_for_input_only(&self) -> Vec<bool> {+ std::iter::once(true)+ .chain(std::iter::repeat_n(false, self.subqueries.len()))+ .collect()+ }+}++impl DisplayAs for ScalarSubqueryExec {+ fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {+ match t {+ DisplayFormatType::Default | DisplayFormatType::Verbose => {+ write!(+ f,+ "ScalarSubqueryExec: subqueries={}",+ self.subqueries.len()+ )+ }+ DisplayFormatType::TreeRender => {+ write!(f, "")+ }+ }+ }+}++impl ExecutionPlan for ScalarSubqueryExec {+ fn name(&self) -> &'static str {+ "ScalarSubqueryExec"+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ &self.cache+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ let mut children = vec![&self.input];+ for sq in &self.subqueries {+ children.push(&sq.plan);+ }+ children+ }++ fn with_new_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // First child is the main input, the rest are subquery plans.+ let input = children.remove(0);+ let subqueries = self+ .subqueries+ .iter()+ .zip(children)+ .map(|(sq, new_plan)| ScalarSubqueryLink {+ plan: new_plan,+ index: sq.index,+ })+ .collect();+ Ok(Arc::new(ScalarSubqueryExec::new(+ input,+ subqueries,+ self.results.clone(),+ )))+ }++ fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {+ self.results.clear();+ Ok(Arc::new(ScalarSubqueryExec {+ input: Arc::clone(&self.input),+ subqueries: self.subqueries.clone(),+ subquery_future: Arc::default(),+ results: self.results.clone(),+ cache: Arc::clone(&self.cache),+ }))+ }++ fn execute(+ &self,+ partition: usize,+ context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ let subqueries = self.subqueries.clone();+ let results = self.results.clone();+ let subquery_ctx = Arc::clone(&context);+ let mut subquery_future = self.subquery_future.try_once(move || {+ Ok(async move { execute_subqueries(subqueries, results, subquery_ctx).await })+ })?;+ let input = Arc::clone(&self.input);+ let schema = self.schema();++ Ok(Box::pin(RecordBatchStreamAdapter::new(+ schema,+ futures::stream::once(async move {+ // Execute all subqueries exactly once, even when multiple+ // partitions call execute() concurrently.+ wait_for_subqueries(&mut subquery_future).await?;++ // Now that the subqueries have finished execution, we can+ // safely execute the main input+ input.execute(partition, context)+ })+ .try_flatten(),+ )))+ }++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }++ fn maintains_input_order(&self) -> Vec<bool> {+ // Only the main input (first child); subquery children don't contribute+ // to ordering.+ self.true_for_input_only()+ }++ fn benefits_from_input_partitioning(&self) -> Vec<bool> {+ // Only the main input; subquery children produce at most one row, so+ // repartitioning them has no benefit.+ self.true_for_input_only()+ }++ fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {+ self.input.partition_statistics(partition)+ }++ fn cardinality_effect(&self) -> CardinalityEffect {+ CardinalityEffect::Equal+ }+}++/// Wait for the subquery execution future to complete.+async fn wait_for_subqueries(fut: &mut OnceFut<()>) -> Result<()> {+ std::future::poll_fn(|cx| fut.get_shared(cx)).await?;+ Ok(())+}++async fn execute_subqueries(+ subqueries: Vec<ScalarSubqueryLink>,+ results: ScalarSubqueryResults,+ context: Arc<TaskContext>,+) -> Result<()> {+ // Evaluate subqueries in parallel; wait for them all to finish evaluation+ // before returning.+ let futures = subqueries.iter().map(|sq| {+ let plan = Arc::clone(&sq.plan);+ let ctx = Arc::clone(&context);+ let results = results.clone();+ let index = sq.index;+ async move {+ let value = execute_scalar_subquery(plan, ctx).await?;+ results.set(index, value)?;+ Ok(()) as Result<()>+ }+ });+ futures::future::try_join_all(futures).await?;+ Ok(())+}++/// Execute a single subquery plan and extract the scalar value.+/// Returns NULL for 0 rows, the scalar value for exactly 1 row,+/// or an error for >1 rows.+async fn execute_scalar_subquery(+ plan: Arc<dyn ExecutionPlan>,+ context: Arc<TaskContext>,+) -> Result<ScalarValue> {+ let schema = plan.schema();+ if schema.fields().len() != 1 {+ // Should be enforced by the physical planner.+ return internal_err!(+ "Scalar subquery must return exactly one column, got {}",+ schema.fields().len()+ );+ }++ let mut stream = crate::execute_stream(plan, context)?;+ let mut result: Option<ScalarValue> = None;++ while let Some(batch) = stream.next().await.transpose()? {+ if batch.num_rows() == 0 {+ continue;+ }+ if result.is_some() || batch.num_rows() > 1 {+ return exec_err!("Scalar subquery returned more than one row");+ }+ result = Some(ScalarValue::try_from_array(batch.column(0), 0)?);+ }++ // 0 rows → typed NULL per SQL semantics+ match result {+ Some(v) => Ok(v),+ None => ScalarValue::try_from(schema.field(0).data_type()),+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::test::{self, TestMemoryExec};+ use crate::{+ execution_plan::reset_plan_states,+ projection::{ProjectionExec, ProjectionExpr},+ };++ use std::sync::atomic::{AtomicUsize, Ordering};++ use crate::test::exec::ErrorExec;+ use arrow::array::{Int32Array, Int64Array};+ use arrow::datatypes::{DataType, Field, Schema};+ use arrow::record_batch::RecordBatch;+ use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;++ enum ExpectedSubqueryResult {+ Value(ScalarValue),+ Error(&'static str),+ }++ #[derive(Debug)]+ struct CountingExec {+ inner: Arc<dyn ExecutionPlan>,+ execute_calls: Arc<AtomicUsize>,+ }++ impl CountingExec {+ fn new(inner: Arc<dyn ExecutionPlan>, execute_calls: Arc<AtomicUsize>) -> Self {+ Self {+ inner,+ execute_calls,+ }+ }+ }++ impl DisplayAs for CountingExec {+ fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {+ match t {+ DisplayFormatType::Default | DisplayFormatType::Verbose => {+ write!(f, "CountingExec")+ }+ DisplayFormatType::TreeRender => write!(f, ""),+ }+ }+ }++ impl ExecutionPlan for CountingExec {+ fn name(&self) -> &'static str {+ "CountingExec"+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ self.inner.properties()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![&self.inner]+ }++ fn with_new_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(Self::new(+ children.remove(0),+ Arc::clone(&self.execute_calls),+ )))+ }++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }++ fn execute(+ &self,+ partition: usize,+ context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ self.execute_calls.fetch_add(1, Ordering::SeqCst);+ self.inner.execute(partition, context)+ }+ }++ fn make_subquery_plan(batches: Vec<RecordBatch>) -> Arc<dyn ExecutionPlan> {+ let schema = batches[0].schema();+ TestMemoryExec::try_new_exec(&[batches], schema, None).unwrap()+ }++ fn int32_batch(values: Vec<i32>) -> RecordBatch {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(values))]).unwrap()+ }++ fn empty_int64_batch() -> RecordBatch {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, true)]));+ RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![] as Vec<i64>))])+ .unwrap()+ }++ fn placeholder_input() -> Arc<dyn ExecutionPlan> {+ Arc::new(crate::placeholder_row::PlaceholderRowExec::new(+ test::aggr_test_schema(),+ ))+ }++ fn single_subquery_exec(+ input: Arc<dyn ExecutionPlan>,+ subquery_plan: Arc<dyn ExecutionPlan>,+ results: ScalarSubqueryResults,+ ) -> ScalarSubqueryExec {+ ScalarSubqueryExec::new(+ input,+ vec![ScalarSubqueryLink {+ plan: subquery_plan,+ index: SubqueryIndex::new(0),+ }],+ results,+ )+ }++ fn scalar_subquery_projection_input(+ results: ScalarSubqueryResults,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(ScalarSubqueryExpr::new(+ DataType::Int32,+ false,+ SubqueryIndex::new(0),+ results,+ )),+ alias: "sq".to_string(),+ }],+ placeholder_input(),+ )?))+ }++ fn extract_single_int32_value(batches: &[RecordBatch]) -> i32 {+ assert_eq!(batches.len(), 1);+ let values = batches[0]+ .column(0)+ .as_any()+ .downcast_ref::<Int32Array>()+ .unwrap();+ assert_eq!(values.len(), 1);+ values.value(0)+ }++ #[tokio::test]+ async fn test_execute_scalar_subquery_row_count_semantics() -> Result<()> {+ for (name, plan, expected) in [+ (+ "single_row",+ make_subquery_plan(vec![int32_batch(vec![42])]),+ ExpectedSubqueryResult::Value(ScalarValue::Int32(Some(42))),+ ),+ (+ "zero_rows",+ make_subquery_plan(vec![empty_int64_batch()]),+ ExpectedSubqueryResult::Value(ScalarValue::Int64(None)),+ ),+ (+ "multiple_rows",+ make_subquery_plan(vec![int32_batch(vec![1, 2, 3])]),+ ExpectedSubqueryResult::Error("more than one row"),+ ),+ ] {+ let actual =+ execute_scalar_subquery(plan, Arc::new(TaskContext::default())).await;+ match expected {+ ExpectedSubqueryResult::Value(expected) => {+ assert_eq!(actual?, expected, "{name}");+ }+ ExpectedSubqueryResult::Error(expected) => {+ let err = actual.expect_err(name);+ assert!(+ err.to_string().contains(expected),+ "{name}: expected error containing '{expected}', got {err}"+ );+ }+ }+ }++ Ok(())+ }++ #[tokio::test]+ async fn test_failed_subquery_is_not_retried() -> Result<()> {+ let execute_calls = Arc::new(AtomicUsize::new(0));+ let subquery_plan = Arc::new(CountingExec::new(+ Arc::new(ErrorExec::new()),+ Arc::clone(&execute_calls),+ ));+ let exec = single_subquery_exec(+ placeholder_input(),+ subquery_plan,+ ScalarSubqueryResults::new(1),+ );++ let ctx = Arc::new(TaskContext::default());+ let stream = exec.execute(0, Arc::clone(&ctx))?;+ assert!(crate::common::collect(stream).await.is_err());++ let stream = exec.execute(0, ctx)?;+ assert!(crate::common::collect(stream).await.is_err());++ assert_eq!(execute_calls.load(Ordering::SeqCst), 1);+ Ok(())+ }++ #[tokio::test]+ async fn test_reset_state_clears_results_and_reexecutes_subqueries() -> Result<()> {+ let execute_calls = Arc::new(AtomicUsize::new(0));+ let results = ScalarSubqueryResults::new(1);+ let subquery_plan = Arc::new(CountingExec::new(+ make_subquery_plan(vec![int32_batch(vec![42])]),+ Arc::clone(&execute_calls),+ ));+ let exec: Arc<dyn ExecutionPlan> = Arc::new(single_subquery_exec(+ scalar_subquery_projection_input(results.clone())?,+ subquery_plan,+ results.clone(),+ ));++ let batches =+ crate::common::collect(exec.execute(0, Arc::new(TaskContext::default()))?)+ .await?;+ assert_eq!(extract_single_int32_value(&batches), 42);+ assert_eq!(+ results.get(SubqueryIndex::new(0)),+ Some(ScalarValue::Int32(Some(42)))+ );++ let reset_exec = reset_plan_states(Arc::clone(&exec))?;+ assert_eq!(results.get(SubqueryIndex::new(0)), None);++ let reset_batches = crate::common::collect(+ reset_exec.execute(0, Arc::new(TaskContext::default()))?,+ )+ .await?;+ assert_eq!(extract_single_int32_value(&reset_batches), 42);+ assert_eq!(+ results.get(SubqueryIndex::new(0)),+ Some(ScalarValue::Int32(Some(42)))+ );+ assert_eq!(execute_calls.load(Ordering::SeqCst), 2);++ Ok(())+ }+}diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex c61226fb52..29b71929f1 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -426,6 +426,8 @@ message LogicalExprNode {Unnest unnest = 35;+ // Subquery expressions+ ScalarSubqueryExprNode scalar_subquery_expr = 36;}}@@ -433,6 +435,15 @@ message Wildcard {TableReference qualifier = 1;}+message SubqueryNode {+ LogicalPlanNode subquery = 1;+ repeated LogicalExprNode outer_ref_columns = 2;+}++message ScalarSubqueryExprNode {+ SubqueryNode subquery = 1;+}+message PlaceholderNode {string id = 1;// We serialize the data type, metadata, and nullability separately to maintain@@ -775,6 +786,7 @@ message PhysicalPlanNode {AsyncFuncExecNode async_func = 36;BufferExecNode buffer = 37;ArrowScanExecNode arrow_scan = 38;+ ScalarSubqueryExecNode scalar_subquery = 39;}}@@ -920,6 +932,8 @@ message PhysicalExprNode {UnknownColumn unknown_column = 20;PhysicalHashExprNode hash_expr = 21;++ PhysicalScalarSubqueryExprNode scalar_subquery = 22;}}@@ -1477,4 +1491,15 @@ message AsyncFuncExecNode {message BufferExecNode {PhysicalPlanNode input = 1;uint64 capacity = 2;+}++message ScalarSubqueryExecNode {+ PhysicalPlanNode input = 1;+ repeated PhysicalPlanNode subqueries = 2;+}++message PhysicalScalarSubqueryExprNode {+ datafusion_common.ArrowType data_type = 1;+ bool nullable = 2;+ uint32 index = 3;}\ No newline at end of filediff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rsindex 84b15ea9a8..2b7d7ed8e8 100644--- a/datafusion/proto/src/bytes/mod.rs+++ b/datafusion/proto/src/bytes/mod.rs@@ -22,28 +22,20 @@ use crate::logical_plan::{};use crate::physical_plan::{DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,- PhysicalProtoConverterExtension,+ PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,};use crate::protobuf;use datafusion_common::{Result, plan_datafusion_err};use datafusion_execution::TaskContext;-use datafusion_expr::{- AggregateUDF, Expr, LogicalPlan, Volatility, WindowUDF, create_udaf, create_udf,- create_udwf,-};+use datafusion_expr::{Expr, LogicalPlan};use prost::{Message,bytes::{Bytes, BytesMut},};use std::sync::Arc;-// Reexport Bytes which appears in the API-use datafusion_execution::registry::FunctionRegistry;-use datafusion_expr::planner::ExprPlanner;use datafusion_physical_plan::ExecutionPlan;-mod registry;-/// Encodes something (such as [`Expr`]) to/from a stream of/// bytes.///@@ -65,26 +57,21 @@ pub trait Serializeable: Sized {/// Convert `self` to an opaque byte streamfn to_bytes(&self) -> Result<Bytes>;- /// Convert `bytes` (the output of [`to_bytes`]) back into an- /// object. This will error if the serialized bytes contain any- /// user defined functions, in which case use- /// [`from_bytes_with_registry`]+ /// Convert `bytes` (the output of [`to_bytes`]) back into an object. This+ /// will error if the serialized bytes contain any user defined functions,+ /// in which case use [`from_bytes_with_ctx`]////// [`to_bytes`]: Self::to_bytes- /// [`from_bytes_with_registry`]: Self::from_bytes_with_registry+ /// [`from_bytes_with_ctx`]: Self::from_bytes_with_ctxfn from_bytes(bytes: &[u8]) -> Result<Self> {- Self::from_bytes_with_registry(bytes, ®istry::NoRegistry {})+ Self::from_bytes_with_ctx(bytes, &TaskContext::default())}- /// Convert `bytes` (the output of [`to_bytes`]) back into an- /// object resolving user defined functions with the specified- /// `registry`+ /// Convert `bytes` (the output of [`to_bytes`]) back into an object,+ /// resolving user defined functions with the specified `ctx`////// [`to_bytes`]: Self::to_bytes- fn from_bytes_with_registry(- bytes: &[u8],- registry: &dyn FunctionRegistry,- ) -> Result<Self>;+ fn from_bytes_with_ctx(bytes: &[u8], ctx: &TaskContext) -> Result<Self>;}impl Serializeable for Expr {@@ -100,100 +87,22 @@ impl Serializeable for Expr {let bytes: Bytes = buffer.into();- // the produced byte stream may lead to "recursion limit" errors, see+ // The produced byte stream may lead to "recursion limit" errors, see// https://github.com/apache/datafusion/issues/3968- // Until the underlying prost issue ( https://github.com/tokio-rs/prost/issues/736 ) is fixed, we try to- // deserialize the data here and check for errors.- //- // Need to provide some placeholder registry because the stream may contain UDFs- struct PlaceHolderRegistry;-- impl FunctionRegistry for PlaceHolderRegistry {- fn udfs(&self) -> std::collections::HashSet<String> {- std::collections::HashSet::default()- }-- fn udf(&self, name: &str) -> Result<Arc<datafusion_expr::ScalarUDF>> {- Ok(Arc::new(create_udf(- name,- vec![],- arrow::datatypes::DataType::Null,- Volatility::Immutable,- Arc::new(|_| unimplemented!()),- )))- }-- fn udaf(&self, name: &str) -> Result<Arc<AggregateUDF>> {- Ok(Arc::new(create_udaf(- name,- vec![arrow::datatypes::DataType::Null],- Arc::new(arrow::datatypes::DataType::Null),- Volatility::Immutable,- Arc::new(|_| unimplemented!()),- Arc::new(vec![]),- )))- }-- fn udwf(&self, name: &str) -> Result<Arc<WindowUDF>> {- Ok(Arc::new(create_udwf(- name,- arrow::datatypes::DataType::Null,- Arc::new(arrow::datatypes::DataType::Null),- Volatility::Immutable,- Arc::new(|| unimplemented!()),- )))- }- fn register_udaf(- &mut self,- _udaf: Arc<AggregateUDF>,- ) -> Result<Option<Arc<AggregateUDF>>> {- datafusion_common::internal_err!(- "register_udaf called in Placeholder Registry!"- )- }- fn register_udf(- &mut self,- _udf: Arc<datafusion_expr::ScalarUDF>,- ) -> Result<Option<Arc<datafusion_expr::ScalarUDF>>> {- datafusion_common::internal_err!(- "register_udf called in Placeholder Registry!"- )- }- fn register_udwf(- &mut self,- _udaf: Arc<WindowUDF>,- ) -> Result<Option<Arc<WindowUDF>>> {- datafusion_common::internal_err!(- "register_udwf called in Placeholder Registry!"- )- }-- fn expr_planners(&self) -> Vec<Arc<dyn ExprPlanner>> {- vec![]- }-- fn udafs(&self) -> std::collections::HashSet<String> {- std::collections::HashSet::default()- }-- fn udwfs(&self) -> std::collections::HashSet<String> {- std::collections::HashSet::default()- }- }- Expr::from_bytes_with_registry(&bytes, &PlaceHolderRegistry)?;+ // Until the underlying prost issue ( https://github.com/tokio-rs/prost/issues/736 )+ // is fixed, verify the bytes can be decoded without hitting that limit.+ protobuf::LogicalExprNode::decode(bytes.as_ref())+ .map_err(|e| plan_datafusion_err!("Error decoding expr as protobuf: {e}"))?;Ok(bytes)}- fn from_bytes_with_registry(- bytes: &[u8],- registry: &dyn FunctionRegistry,- ) -> Result<Self> {+ fn from_bytes_with_ctx(bytes: &[u8], ctx: &TaskContext) -> Result<Self> {let protobuf = protobuf::LogicalExprNode::decode(bytes).map_err(|e| plan_datafusion_err!("Error decoding expr as protobuf: {e}"))?;let extension_codec = DefaultLogicalExtensionCodec {};- logical_plan::from_proto::parse_expr(&protobuf, registry, &extension_codec)+ logical_plan::from_proto::parse_expr(&protobuf, ctx, &extension_codec).map_err(|e| plan_datafusion_err!("Error parsing protobuf into Expr: {e}"))}}@@ -327,7 +236,8 @@ pub fn physical_plan_from_json(.map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;let extension_codec = DefaultPhysicalExtensionCodec {};let proto_converter = DefaultPhysicalProtoConverter {};- proto_converter.proto_to_execution_plan(ctx, &extension_codec, &back)+ let decode_ctx = PhysicalPlanDecodeContext::new(ctx, &extension_codec);+ proto_converter.proto_to_execution_plan(&back, &decode_ctx)}/// Deserialize a PhysicalPlan from bytes@@ -369,5 +279,6 @@ pub fn physical_plan_from_bytes_with_proto_converter() -> Result<Arc<dyn ExecutionPlan>> {let protobuf = protobuf::PhysicalPlanNode::decode(bytes).map_err(|e| plan_datafusion_err!("Error decoding expr as protobuf: {e}"))?;- proto_converter.proto_to_execution_plan(ctx, extension_codec, &protobuf)+ let decode_ctx = PhysicalPlanDecodeContext::new(ctx, extension_codec);+ proto_converter.proto_to_execution_plan(&protobuf, &decode_ctx)}diff --git a/datafusion/proto/src/bytes/registry.rs b/datafusion/proto/src/bytes/registry.rsdeleted file mode 100644index a3f74787e2..0000000000--- a/datafusion/proto/src/bytes/registry.rs+++ /dev/null@@ -1,85 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--use std::{collections::HashSet, sync::Arc};--use datafusion_common::Result;-use datafusion_common::plan_err;-use datafusion_execution::registry::FunctionRegistry;-use datafusion_expr::planner::ExprPlanner;-use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};--/// A default [`FunctionRegistry`] registry that does not resolve any-/// user defined functions-pub(crate) struct NoRegistry {}--impl FunctionRegistry for NoRegistry {- fn udfs(&self) -> HashSet<String> {- HashSet::new()- }-- fn udf(&self, name: &str) -> Result<Arc<ScalarUDF>> {- plan_err!(- "No function registry provided to deserialize, so can not deserialize User Defined Function '{name}'"- )- }-- fn udaf(&self, name: &str) -> Result<Arc<AggregateUDF>> {- plan_err!(- "No function registry provided to deserialize, so can not deserialize User Defined Aggregate Function '{name}'"- )- }-- fn udwf(&self, name: &str) -> Result<Arc<WindowUDF>> {- plan_err!(- "No function registry provided to deserialize, so can not deserialize User Defined Window Function '{name}'"- )- }- fn register_udaf(- &mut self,- udaf: Arc<AggregateUDF>,- ) -> Result<Option<Arc<AggregateUDF>>> {- plan_err!(- "No function registry provided to deserialize, so can not register User Defined Aggregate Function '{}'",- udaf.inner().name()- )- }- fn register_udf(&mut self, udf: Arc<ScalarUDF>) -> Result<Option<Arc<ScalarUDF>>> {- plan_err!(- "No function registry provided to deserialize, so can not deserialize User Defined Function '{}'",- udf.inner().name()- )- }- fn register_udwf(&mut self, udwf: Arc<WindowUDF>) -> Result<Option<Arc<WindowUDF>>> {- plan_err!(- "No function registry provided to deserialize, so can not deserialize User Defined Window Function '{}'",- udwf.inner().name()- )- }-- fn expr_planners(&self) -> Vec<Arc<dyn ExprPlanner>> {- vec![]- }-- fn udafs(&self) -> HashSet<String> {- HashSet::new()- }-- fn udwfs(&self) -> HashSet<String> {- HashSet::new()- }-}diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 82bcdac898..d84fdae345 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -12193,6 +12193,9 @@ impl serde::Serialize for LogicalExprNode {logical_expr_node::ExprType::Unnest(v) => {struct_ser.serialize_field("unnest", v)?;}+ logical_expr_node::ExprType::ScalarSubqueryExpr(v) => {+ struct_ser.serialize_field("scalarSubqueryExpr", v)?;+ }}}struct_ser.end()@@ -12254,6 +12257,8 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {"similarTo","placeholder","unnest",+ "scalar_subquery_expr",+ "scalarSubqueryExpr",];#[allow(clippy::enum_variant_names)]@@ -12289,6 +12294,7 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {SimilarTo,Placeholder,Unnest,+ ScalarSubqueryExpr,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -12341,6 +12347,7 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {"similarTo" | "similar_to" => Ok(GeneratedField::SimilarTo),"placeholder" => Ok(GeneratedField::Placeholder),"unnest" => Ok(GeneratedField::Unnest),+ "scalarSubqueryExpr" | "scalar_subquery_expr" => Ok(GeneratedField::ScalarSubqueryExpr),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -12578,6 +12585,13 @@ impl<'de> serde::Deserialize<'de> for LogicalExprNode {return Err(serde::de::Error::duplicate_field("unnest"));}expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_expr_node::ExprType::Unnest)+;+ }+ GeneratedField::ScalarSubqueryExpr => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("scalarSubqueryExpr"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(logical_expr_node::ExprType::ScalarSubqueryExpr);}}@@ -16610,6 +16624,9 @@ impl serde::Serialize for PhysicalExprNode {physical_expr_node::ExprType::HashExpr(v) => {struct_ser.serialize_field("hashExpr", v)?;}+ physical_expr_node::ExprType::ScalarSubquery(v) => {+ struct_ser.serialize_field("scalarSubquery", v)?;+ }}}struct_ser.end()@@ -16656,6 +16673,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"unknownColumn","hash_expr","hashExpr",+ "scalar_subquery",+ "scalarSubquery",];#[allow(clippy::enum_variant_names)]@@ -16680,6 +16699,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {Extension,UnknownColumn,HashExpr,+ ScalarSubquery,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -16721,6 +16741,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"extension" => Ok(GeneratedField::Extension),"unknownColumn" | "unknown_column" => Ok(GeneratedField::UnknownColumn),"hashExpr" | "hash_expr" => Ok(GeneratedField::HashExpr),+ "scalarSubquery" | "scalar_subquery" => Ok(GeneratedField::ScalarSubquery),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -16883,6 +16904,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {return Err(serde::de::Error::duplicate_field("hashExpr"));}expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::HashExpr)+;+ }+ GeneratedField::ScalarSubquery => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("scalarSubquery"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::ScalarSubquery);}}@@ -18121,6 +18149,9 @@ impl serde::Serialize for PhysicalPlanNode {physical_plan_node::PhysicalPlanType::ArrowScan(v) => {struct_ser.serialize_field("arrowScan", v)?;}+ physical_plan_node::PhysicalPlanType::ScalarSubquery(v) => {+ struct_ser.serialize_field("scalarSubquery", v)?;+ }}}struct_ser.end()@@ -18191,6 +18222,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {"buffer","arrow_scan","arrowScan",+ "scalar_subquery",+ "scalarSubquery",];#[allow(clippy::enum_variant_names)]@@ -18232,6 +18265,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {AsyncFunc,Buffer,ArrowScan,+ ScalarSubquery,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -18290,6 +18324,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {"asyncFunc" | "async_func" => Ok(GeneratedField::AsyncFunc),"buffer" => Ok(GeneratedField::Buffer),"arrowScan" | "arrow_scan" => Ok(GeneratedField::ArrowScan),+ "scalarSubquery" | "scalar_subquery" => Ok(GeneratedField::ScalarSubquery),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -18569,6 +18604,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {return Err(serde::de::Error::duplicate_field("arrowScan"));}physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::ArrowScan)+;+ }+ GeneratedField::ScalarSubquery => {+ if physical_plan_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("scalarSubquery"));+ }+ physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::ScalarSubquery);}}@@ -18581,6 +18623,134 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {deserializer.deserialize_struct("datafusion.PhysicalPlanNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for PhysicalScalarSubqueryExprNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.data_type.is_some() {+ len += 1;+ }+ if self.nullable {+ len += 1;+ }+ if self.index != 0 {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalScalarSubqueryExprNode", len)?;+ if let Some(v) = self.data_type.as_ref() {+ struct_ser.serialize_field("dataType", v)?;+ }+ if self.nullable {+ struct_ser.serialize_field("nullable", &self.nullable)?;+ }+ if self.index != 0 {+ struct_ser.serialize_field("index", &self.index)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalScalarSubqueryExprNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "data_type",+ "dataType",+ "nullable",+ "index",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ DataType,+ Nullable,+ Index,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "dataType" | "data_type" => Ok(GeneratedField::DataType),+ "nullable" => Ok(GeneratedField::Nullable),+ "index" => Ok(GeneratedField::Index),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalScalarSubqueryExprNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalScalarSubqueryExprNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalScalarSubqueryExprNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut data_type__ = None;+ let mut nullable__ = None;+ let mut index__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::DataType => {+ if data_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("dataType"));+ }+ data_type__ = map_.next_value()?;+ }+ GeneratedField::Nullable => {+ if nullable__.is_some() {+ return Err(serde::de::Error::duplicate_field("nullable"));+ }+ nullable__ = Some(map_.next_value()?);+ }+ GeneratedField::Index => {+ if index__.is_some() {+ return Err(serde::de::Error::duplicate_field("index"));+ }+ index__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ }+ }+ Ok(PhysicalScalarSubqueryExprNode {+ data_type: data_type__,+ nullable: nullable__.unwrap_or_default(),+ index: index__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalScalarSubqueryExprNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for PhysicalScalarUdfNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -21250,6 +21420,205 @@ impl<'de> serde::Deserialize<'de> for RollupNode {deserializer.deserialize_struct("datafusion.RollupNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for ScalarSubqueryExecNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.input.is_some() {+ len += 1;+ }+ if !self.subqueries.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.ScalarSubqueryExecNode", len)?;+ if let Some(v) = self.input.as_ref() {+ struct_ser.serialize_field("input", v)?;+ }+ if !self.subqueries.is_empty() {+ struct_ser.serialize_field("subqueries", &self.subqueries)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for ScalarSubqueryExecNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "input",+ "subqueries",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Input,+ Subqueries,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "input" => Ok(GeneratedField::Input),+ "subqueries" => Ok(GeneratedField::Subqueries),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = ScalarSubqueryExecNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.ScalarSubqueryExecNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<ScalarSubqueryExecNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut input__ = None;+ let mut subqueries__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Input => {+ if input__.is_some() {+ return Err(serde::de::Error::duplicate_field("input"));+ }+ input__ = map_.next_value()?;+ }+ GeneratedField::Subqueries => {+ if subqueries__.is_some() {+ return Err(serde::de::Error::duplicate_field("subqueries"));+ }+ subqueries__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(ScalarSubqueryExecNode {+ input: input__,+ subqueries: subqueries__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.ScalarSubqueryExecNode", FIELDS, GeneratedVisitor)+ }+}+impl serde::Serialize for ScalarSubqueryExprNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.subquery.is_some() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.ScalarSubqueryExprNode", len)?;+ if let Some(v) = self.subquery.as_ref() {+ struct_ser.serialize_field("subquery", v)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for ScalarSubqueryExprNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "subquery",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Subquery,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "subquery" => Ok(GeneratedField::Subquery),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = ScalarSubqueryExprNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.ScalarSubqueryExprNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<ScalarSubqueryExprNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut subquery__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Subquery => {+ if subquery__.is_some() {+ return Err(serde::de::Error::duplicate_field("subquery"));+ }+ subquery__ = map_.next_value()?;+ }+ }+ }+ Ok(ScalarSubqueryExprNode {+ subquery: subquery__,+ })+ }+ }+ deserializer.deserialize_struct("datafusion.ScalarSubqueryExprNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for ScalarUdfExprNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -22927,6 +23296,115 @@ impl<'de> serde::Deserialize<'de> for SubqueryAliasNode {deserializer.deserialize_struct("datafusion.SubqueryAliasNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for SubqueryNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.subquery.is_some() {+ len += 1;+ }+ if !self.outer_ref_columns.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.SubqueryNode", len)?;+ if let Some(v) = self.subquery.as_ref() {+ struct_ser.serialize_field("subquery", v)?;+ }+ if !self.outer_ref_columns.is_empty() {+ struct_ser.serialize_field("outerRefColumns", &self.outer_ref_columns)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for SubqueryNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "subquery",+ "outer_ref_columns",+ "outerRefColumns",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Subquery,+ OuterRefColumns,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "subquery" => Ok(GeneratedField::Subquery),+ "outerRefColumns" | "outer_ref_columns" => Ok(GeneratedField::OuterRefColumns),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = SubqueryNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.SubqueryNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<SubqueryNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut subquery__ = None;+ let mut outer_ref_columns__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Subquery => {+ if subquery__.is_some() {+ return Err(serde::de::Error::duplicate_field("subquery"));+ }+ subquery__ = map_.next_value()?;+ }+ GeneratedField::OuterRefColumns => {+ if outer_ref_columns__.is_some() {+ return Err(serde::de::Error::duplicate_field("outerRefColumns"));+ }+ outer_ref_columns__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(SubqueryNode {+ subquery: subquery__,+ outer_ref_columns: outer_ref_columns__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.SubqueryNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for SymmetricHashJoinExecNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex ab60c3058d..1dc259926a 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -195,8 +195,8 @@ pub mod projection_node {pub struct SelectionNode {#[prost(message, optional, boxed, tag = "1")]pub input: ::core::option::Option<::prost::alloc::boxed::Box<LogicalPlanNode>>,- #[prost(message, optional, tag = "2")]- pub expr: ::core::option::Option<LogicalExprNode>,+ #[prost(message, optional, boxed, tag = "2")]+ pub expr: ::core::option::Option<::prost::alloc::boxed::Box<LogicalExprNode>>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct SortNode {@@ -382,8 +382,8 @@ pub struct JoinNode {pub right_join_key: ::prost::alloc::vec::Vec<LogicalExprNode>,#[prost(enumeration = "super::datafusion_common::NullEquality", tag = "7")]pub null_equality: i32,- #[prost(message, optional, tag = "8")]- pub filter: ::core::option::Option<LogicalExprNode>,+ #[prost(message, optional, boxed, tag = "8")]+ pub filter: ::core::option::Option<::prost::alloc::boxed::Box<LogicalExprNode>>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct DistinctNode {@@ -578,7 +578,7 @@ pub struct SubqueryAliasNode {pub struct LogicalExprNode {#[prost(oneof = "logical_expr_node::ExprType",- tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35"+ tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36")]pub expr_type: ::core::option::Option<logical_expr_node::ExprType>,}@@ -656,6 +656,9 @@ pub mod logical_expr_node {Placeholder(super::PlaceholderNode),#[prost(message, tag = "35")]Unnest(super::Unnest),+ /// Subquery expressions+ #[prost(message, tag = "36")]+ ScalarSubqueryExpr(::prost::alloc::boxed::Box<super::ScalarSubqueryExprNode>),}}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]@@ -664,6 +667,18 @@ pub struct Wildcard {pub qualifier: ::core::option::Option<TableReference>,}#[derive(Clone, PartialEq, ::prost::Message)]+pub struct SubqueryNode {+ #[prost(message, optional, boxed, tag = "1")]+ pub subquery: ::core::option::Option<::prost::alloc::boxed::Box<LogicalPlanNode>>,+ #[prost(message, repeated, tag = "2")]+ pub outer_ref_columns: ::prost::alloc::vec::Vec<LogicalExprNode>,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct ScalarSubqueryExprNode {+ #[prost(message, optional, boxed, tag = "1")]+ pub subquery: ::core::option::Option<::prost::alloc::boxed::Box<SubqueryNode>>,+}+#[derive(Clone, PartialEq, ::prost::Message)]pub struct PlaceholderNode {#[prost(string, tag = "1")]pub id: ::prost::alloc::string::String,@@ -1102,7 +1117,7 @@ pub mod table_reference {pub struct PhysicalPlanNode {#[prost(oneof = "physical_plan_node::PhysicalPlanType",- tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38"+ tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39")]pub physical_plan_type: ::core::option::Option<physical_plan_node::PhysicalPlanType>,}@@ -1186,6 +1201,8 @@ pub mod physical_plan_node {Buffer(::prost::alloc::boxed::Box<super::BufferExecNode>),#[prost(message, tag = "38")]ArrowScan(super::ArrowScanExecNode),+ #[prost(message, tag = "39")]+ ScalarSubquery(::prost::alloc::boxed::Box<super::ScalarSubqueryExecNode>),}}#[derive(Clone, PartialEq, ::prost::Message)]@@ -1317,7 +1334,7 @@ pub struct PhysicalExprNode {pub expr_id: ::core::option::Option<u64>,#[prost(oneof = "physical_expr_node::ExprType",- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21"+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22")]pub expr_type: ::core::option::Option<physical_expr_node::ExprType>,}@@ -1370,6 +1387,8 @@ pub mod physical_expr_node {UnknownColumn(super::UnknownColumn),#[prost(message, tag = "21")]HashExpr(super::PhysicalHashExprNode),+ #[prost(message, tag = "22")]+ ScalarSubquery(super::PhysicalScalarSubqueryExprNode),}}#[derive(Clone, PartialEq, ::prost::Message)]@@ -2202,6 +2221,22 @@ pub struct BufferExecNode {#[prost(uint64, tag = "2")]pub capacity: u64,}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct ScalarSubqueryExecNode {+ #[prost(message, optional, boxed, tag = "1")]+ pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,+ #[prost(message, repeated, tag = "2")]+ pub subqueries: ::prost::alloc::vec::Vec<PhysicalPlanNode>,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalScalarSubqueryExprNode {+ #[prost(message, optional, tag = "1")]+ pub data_type: ::core::option::Option<super::datafusion_common::ArrowType>,+ #[prost(bool, tag = "2")]+ pub nullable: bool,+ #[prost(uint32, tag = "3")]+ pub index: u32,+}/// Identifies a built-in file format supported by DataFusion./// Used by DefaultLogicalExtensionCodec to serialize/deserialize/// FileFormatFactory instances (e.g. in CopyTo plans).diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rsindex ed33d9fab1..78ffd362c8 100644--- a/datafusion/proto/src/logical_plan/from_proto.rs+++ b/datafusion/proto/src/logical_plan/from_proto.rs@@ -23,10 +23,12 @@ use datafusion_common::{NullEquality, RecursionUnnestOption, Result, ScalarValue, TableReference,UnnestOptions, exec_datafusion_err, internal_err, plan_datafusion_err,};+use datafusion_execution::TaskContext;use datafusion_execution::registry::FunctionRegistry;use datafusion_expr::dml::InsertOp;use datafusion_expr::expr::{Alias, NullTreatment, Placeholder, Sort};use datafusion_expr::expr::{Unnest, WildcardOptions};+use datafusion_expr::logical_plan::Subquery;use datafusion_expr::{Between, BinaryExpr, Case, Cast, Expr, GroupingSet,GroupingSet::GroupingSets,@@ -52,7 +54,7 @@ use crate::protobuf::{},};-use super::LogicalExtensionCodec;+use super::{AsLogicalPlan, LogicalExtensionCodec};impl From<&protobuf::UnnestOptions> for UnnestOptions {fn from(opts: &protobuf::UnnestOptions) -> Self {@@ -256,7 +258,7 @@ impl From<protobuf::NullTreatment> for NullTreatment {pub fn parse_expr(proto: &protobuf::LogicalExprNode,- registry: &dyn FunctionRegistry,+ ctx: &TaskContext,codec: &dyn LogicalExtensionCodec,) -> Result<Expr, Error> {use protobuf::{logical_expr_node::ExprType, window_expr_node};@@ -269,7 +271,7 @@ pub fn parse_expr(match expr_type {ExprType::BinaryExpr(binary_expr) => {let op = from_proto_binary_op(&binary_expr.op)?;- let operands = parse_exprs(&binary_expr.operands, registry, codec)?;+ let operands = parse_exprs(&binary_expr.operands, ctx, codec)?;if operands.len() < 2 {return Err(proto_error(@@ -296,8 +298,8 @@ pub fn parse_expr(.window_function.as_ref().ok_or_else(|| Error::required("window_function"))?;- let partition_by = parse_exprs(&expr.partition_by, registry, codec)?;- let mut order_by = parse_sorts(&expr.order_by, registry, codec)?;+ let partition_by = parse_exprs(&expr.partition_by, ctx, codec)?;+ let mut order_by = parse_sorts(&expr.order_by, ctx, codec)?;let window_frame = expr.window_frame.as_ref()@@ -329,7 +331,7 @@ pub fn parse_expr(window_expr_node::WindowFunction::Udaf(udaf_name) => {let udaf_function = match &expr.fun_definition {Some(buf) => codec.try_decode_udaf(udaf_name, buf)?,- None => registry+ None => ctx.udaf(udaf_name).or_else(|_| codec.try_decode_udaf(udaf_name, &[]))?,};@@ -338,7 +340,7 @@ pub fn parse_expr(window_expr_node::WindowFunction::Udwf(udwf_name) => {let udwf_function = match &expr.fun_definition {Some(buf) => codec.try_decode_udwf(udwf_name, buf)?,- None => registry+ None => ctx.udwf(udwf_name).or_else(|_| codec.try_decode_udwf(udwf_name, &[]))?,};@@ -346,7 +348,7 @@ pub fn parse_expr(}};- let args = parse_exprs(&expr.exprs, registry, codec)?;+ let args = parse_exprs(&expr.exprs, ctx, codec)?;let mut builder = Expr::from(WindowFunction::new(agg_fn, args)).partition_by(partition_by).order_by(order_by)@@ -357,8 +359,7 @@ pub fn parse_expr(builder = builder.distinct();};- if let Some(filter) =- parse_optional_expr(expr.filter.as_deref(), registry, codec)?+ if let Some(filter) = parse_optional_expr(expr.filter.as_deref(), ctx, codec)?{builder = builder.filter(filter);}@@ -366,7 +367,7 @@ pub fn parse_expr(builder.build().map_err(Error::DataFusionError)}ExprType::Alias(alias) => Ok(Expr::Alias(Alias::new(- parse_required_expr(alias.expr.as_deref(), registry, "expr", codec)?,+ parse_required_expr(alias.expr.as_deref(), ctx, "expr", codec)?,alias.relation.first()@@ -376,69 +377,69 @@ pub fn parse_expr())),ExprType::IsNullExpr(is_null) => Ok(Expr::IsNull(Box::new(parse_required_expr(is_null.expr.as_deref(),- registry,+ ctx,"expr",codec,)?))),ExprType::IsNotNullExpr(is_not_null) => Ok(Expr::IsNotNull(Box::new(- parse_required_expr(is_not_null.expr.as_deref(), registry, "expr", codec)?,+ parse_required_expr(is_not_null.expr.as_deref(), ctx, "expr", codec)?,))),ExprType::NotExpr(not) => Ok(Expr::Not(Box::new(parse_required_expr(not.expr.as_deref(),- registry,+ ctx,"expr",codec,)?))),ExprType::IsTrue(msg) => Ok(Expr::IsTrue(Box::new(parse_required_expr(msg.expr.as_deref(),- registry,+ ctx,"expr",codec,)?))),ExprType::IsFalse(msg) => Ok(Expr::IsFalse(Box::new(parse_required_expr(msg.expr.as_deref(),- registry,+ ctx,"expr",codec,)?))),ExprType::IsUnknown(msg) => Ok(Expr::IsUnknown(Box::new(parse_required_expr(msg.expr.as_deref(),- registry,+ ctx,"expr",codec,)?))),ExprType::IsNotTrue(msg) => Ok(Expr::IsNotTrue(Box::new(parse_required_expr(msg.expr.as_deref(),- registry,+ ctx,"expr",codec,)?))),ExprType::IsNotFalse(msg) => Ok(Expr::IsNotFalse(Box::new(parse_required_expr(msg.expr.as_deref(),- registry,+ ctx,"expr",codec,)?))),ExprType::IsNotUnknown(msg) => Ok(Expr::IsNotUnknown(Box::new(- parse_required_expr(msg.expr.as_deref(), registry, "expr", codec)?,+ parse_required_expr(msg.expr.as_deref(), ctx, "expr", codec)?,))),ExprType::Between(between) => Ok(Expr::Between(Between::new(Box::new(parse_required_expr(between.expr.as_deref(),- registry,+ ctx,"expr",codec,)?),between.negated,Box::new(parse_required_expr(between.low.as_deref(),- registry,+ ctx,"expr",codec,)?),Box::new(parse_required_expr(between.high.as_deref(),- registry,+ ctx,"expr",codec,)?),@@ -447,13 +448,13 @@ pub fn parse_expr(like.negated,Box::new(parse_required_expr(like.expr.as_deref(),- registry,+ ctx,"expr",codec,)?),Box::new(parse_required_expr(like.pattern.as_deref(),- registry,+ ctx,"pattern",codec,)?),@@ -464,13 +465,13 @@ pub fn parse_expr(like.negated,Box::new(parse_required_expr(like.expr.as_deref(),- registry,+ ctx,"expr",codec,)?),Box::new(parse_required_expr(like.pattern.as_deref(),- registry,+ ctx,"pattern",codec,)?),@@ -481,13 +482,13 @@ pub fn parse_expr(like.negated,Box::new(parse_required_expr(like.expr.as_deref(),- registry,+ ctx,"expr",codec,)?),Box::new(parse_required_expr(like.pattern.as_deref(),- registry,+ ctx,"pattern",codec,)?),@@ -501,13 +502,13 @@ pub fn parse_expr(.map(|e| {let when_expr = parse_required_expr(e.when_expr.as_ref(),- registry,+ ctx,"when_expr",codec,)?;let then_expr = parse_required_expr(e.then_expr.as_ref(),- registry,+ ctx,"then_expr",codec,)?;@@ -515,16 +516,15 @@ pub fn parse_expr(}).collect::<Result<Vec<(Box<Expr>, Box<Expr>)>, Error>>()?;Ok(Expr::Case(Case::new(- parse_optional_expr(case.expr.as_deref(), registry, codec)?.map(Box::new),+ parse_optional_expr(case.expr.as_deref(), ctx, codec)?.map(Box::new),when_then_expr,- parse_optional_expr(case.else_expr.as_deref(), registry, codec)?- .map(Box::new),+ parse_optional_expr(case.else_expr.as_deref(), ctx, codec)?.map(Box::new),)))}ExprType::Cast(cast) => {let expr = Box::new(parse_required_expr(cast.expr.as_deref(),- registry,+ ctx,"expr",codec,)?);@@ -537,7 +537,7 @@ pub fn parse_expr(ExprType::TryCast(cast) => {let expr = Box::new(parse_required_expr(cast.expr.as_deref(),- registry,+ ctx,"expr",codec,)?);@@ -551,10 +551,10 @@ pub fn parse_expr()))}ExprType::Negative(negative) => Ok(Expr::Negative(Box::new(- parse_required_expr(negative.expr.as_deref(), registry, "expr", codec)?,+ parse_required_expr(negative.expr.as_deref(), ctx, "expr", codec)?,))),ExprType::Unnest(unnest) => {- let mut exprs = parse_exprs(&unnest.exprs, registry, codec)?;+ let mut exprs = parse_exprs(&unnest.exprs, ctx, codec)?;if exprs.len() != 1 {return Err(proto_error("Unnest must have exactly one expression"));}@@ -563,11 +563,11 @@ pub fn parse_expr(ExprType::InList(in_list) => Ok(Expr::InList(InList::new(Box::new(parse_required_expr(in_list.expr.as_deref(),- registry,+ ctx,"expr",codec,)?),- parse_exprs(&in_list.list, registry, codec)?,+ parse_exprs(&in_list.list, ctx, codec)?,in_list.negated,))),ExprType::Wildcard(protobuf::Wildcard { qualifier }) => {@@ -585,19 +585,19 @@ pub fn parse_expr(}) => {let scalar_fn = match fun_definition {Some(buf) => codec.try_decode_udf(fun_name, buf)?,- None => registry+ None => ctx.udf(fun_name.as_str()).or_else(|_| codec.try_decode_udf(fun_name, &[]))?,};Ok(Expr::ScalarFunction(expr::ScalarFunction::new_udf(scalar_fn,- parse_exprs(args, registry, codec)?,+ parse_exprs(args, ctx, codec)?,)))}ExprType::AggregateUdfExpr(pb) => {let agg_fn = match &pb.fun_definition {Some(buf) => codec.try_decode_udaf(&pb.fun_name, buf)?,- None => registry+ None => ctx.udaf(&pb.fun_name).or_else(|_| codec.try_decode_udaf(&pb.fun_name, &[]))?,};@@ -616,10 +616,10 @@ pub fn parse_expr(Ok(Expr::AggregateFunction(expr::AggregateFunction::new_udf(agg_fn,- parse_exprs(&pb.args, registry, codec)?,+ parse_exprs(&pb.args, ctx, codec)?,pb.distinct,- parse_optional_expr(pb.filter.as_deref(), registry, codec)?.map(Box::new),- parse_sorts(&pb.order_by, registry, codec)?,+ parse_optional_expr(pb.filter.as_deref(), ctx, codec)?.map(Box::new),+ parse_sorts(&pb.order_by, ctx, codec)?,null_treatment,)))}@@ -627,15 +627,15 @@ pub fn parse_expr(ExprType::GroupingSet(GroupingSetNode { expr }) => {Ok(Expr::GroupingSet(GroupingSets(expr.iter()- .map(|expr_list| parse_exprs(&expr_list.expr, registry, codec))+ .map(|expr_list| parse_exprs(&expr_list.expr, ctx, codec)).collect::<Result<Vec<_>, Error>>()?,)))}ExprType::Cube(CubeNode { expr }) => Ok(Expr::GroupingSet(GroupingSet::Cube(- parse_exprs(expr, registry, codec)?,+ parse_exprs(expr, ctx, codec)?,))),ExprType::Rollup(RollupNode { expr }) => Ok(Expr::GroupingSet(- GroupingSet::Rollup(parse_exprs(expr, registry, codec)?),+ GroupingSet::Rollup(parse_exprs(expr, ctx, codec)?),)),ExprType::Placeholder(PlaceholderNode {id,@@ -657,13 +657,41 @@ pub fn parse_expr()))}},+ ExprType::ScalarSubqueryExpr(sq) => {+ let subquery = parse_subquery(+ sq.subquery+ .as_deref()+ .ok_or_else(|| Error::required("ScalarSubqueryExprNode.subquery"))?,+ ctx,+ codec,+ )?;+ Ok(Expr::ScalarSubquery(subquery))+ }}}+fn parse_subquery(+ proto: &protobuf::SubqueryNode,+ ctx: &TaskContext,+ codec: &dyn LogicalExtensionCodec,+) -> Result<Subquery, Error> {+ let plan_node = proto+ .subquery+ .as_ref()+ .ok_or_else(|| Error::required("SubqueryNode.subquery"))?;+ let plan = plan_node.try_into_logical_plan(ctx, codec)?;+ let outer_ref_columns = parse_exprs(&proto.outer_ref_columns, ctx, codec)?;+ Ok(Subquery {+ subquery: Arc::new(plan),+ outer_ref_columns,+ spans: Default::default(),+ })+}+/// Parse a vector of `protobuf::LogicalExprNode`s.pub fn parse_exprs<'a, I>(protos: I,- registry: &dyn FunctionRegistry,+ ctx: &TaskContext,codec: &dyn LogicalExtensionCodec,) -> Result<Vec<Expr>, Error>where@@ -672,7 +700,7 @@ wherelet res = protos.into_iter().map(|elem| {- parse_expr(elem, registry, codec).map_err(|e| plan_datafusion_err!("{}", e))+ parse_expr(elem, ctx, codec).map_err(|e| plan_datafusion_err!("{}", e))}).collect::<Result<Vec<_>>>()?;Ok(res)@@ -680,7 +708,7 @@ wherepub fn parse_sorts<'a, I>(protos: I,- registry: &dyn FunctionRegistry,+ ctx: &TaskContext,codec: &dyn LogicalExtensionCodec,) -> Result<Vec<Sort>, Error>where@@ -688,17 +716,17 @@ where{protos.into_iter()- .map(|sort| parse_sort(sort, registry, codec))+ .map(|sort| parse_sort(sort, ctx, codec)).collect::<Result<Vec<Sort>, Error>>()}pub fn parse_sort(sort: &protobuf::SortExprNode,- registry: &dyn FunctionRegistry,+ ctx: &TaskContext,codec: &dyn LogicalExtensionCodec,) -> Result<Sort, Error> {Ok(Sort::new(- parse_required_expr(sort.expr.as_ref(), registry, "expr", codec)?,+ parse_required_expr(sort.expr.as_ref(), ctx, "expr", codec)?,sort.asc,sort.nulls_first,))@@ -754,23 +782,23 @@ pub fn from_proto_binary_op(op: &str) -> Result<Operator, Error> {fn parse_optional_expr(p: Option<&protobuf::LogicalExprNode>,- registry: &dyn FunctionRegistry,+ ctx: &TaskContext,codec: &dyn LogicalExtensionCodec,) -> Result<Option<Expr>, Error> {match p {- Some(expr) => parse_expr(expr, registry, codec).map(Some),+ Some(expr) => parse_expr(expr, ctx, codec).map(Some),None => Ok(None),}}fn parse_required_expr(p: Option<&protobuf::LogicalExprNode>,- registry: &dyn FunctionRegistry,+ ctx: &TaskContext,field: impl Into<String>,codec: &dyn LogicalExtensionCodec,) -> Result<Expr, Error> {match p {- Some(expr) => parse_expr(expr, registry, codec),+ Some(expr) => parse_expr(expr, ctx, codec),None => Err(Error::required(field)),}}diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 9715ecf8d9..eea2eb3364 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -1321,10 +1321,10 @@ impl AsLogicalPlan for LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::Selection(Box::new(protobuf::SelectionNode {input: Some(Box::new(input)),- expr: Some(serialize_expr(+ expr: Some(Box::new(serialize_expr(&filter.predicate,extension_codec,- )?),+ )?)),},))),})@@ -1440,7 +1440,7 @@ impl AsLogicalPlan for LogicalPlanNode {null_equality.to_owned().into();let filter = filter.as_ref()- .map(|e| serialize_expr(e, extension_codec))+ .map(|e| serialize_expr(e, extension_codec).map(Box::new)).map_or(Ok(None), |v| v.map(Some))?;Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::Join(Box::new(@@ -1457,8 +1457,14 @@ impl AsLogicalPlan for LogicalPlanNode {))),})}- LogicalPlan::Subquery(_) => {- not_impl_err!("LogicalPlan serde is not yet implemented for subqueries")+ LogicalPlan::Subquery(subquery) => {+ // Serialize the inner subquery plan directly — the+ // LogicalPlan::Subquery wrapper is reconstructed during+ // expression deserialization.+ LogicalPlanNode::try_from_logical_plan(+ &subquery.subquery,+ extension_codec,+ )}LogicalPlan::SubqueryAlias(SubqueryAlias { input, alias, .. }) => {let input: LogicalPlanNode = LogicalPlanNode::try_from_logical_plan(diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rsindex 6fcb738992..bd5c4b585c 100644--- a/datafusion/proto/src/logical_plan/to_proto.rs+++ b/datafusion/proto/src/logical_plan/to_proto.rs@@ -28,6 +28,7 @@ use datafusion_expr::expr::{self, AggregateFunctionParams, Alias, Between, BinaryExpr, Cast, GroupingSet, InList,Like, NullTreatment, Placeholder, ScalarFunction, Unnest,};+use datafusion_expr::logical_plan::Subquery;use datafusion_expr::{Expr, JoinConstraint, JoinType, SortExpr, TryCast, WindowFrame, WindowFrameBound,WindowFrameUnits, WindowFunctionDefinition, logical_plan::PlanType,@@ -48,7 +49,8 @@ use crate::protobuf::{},};-use super::LogicalExtensionCodec;+use super::{AsLogicalPlan, LogicalExtensionCodec};+use crate::protobuf::LogicalPlanNode;impl From<&UnnestOptions> for protobuf::UnnestOptions {fn from(opts: &UnnestOptions) -> Self {@@ -579,14 +581,20 @@ pub fn serialize_expr(qualifier: qualifier.to_owned().map(|x| x.into()),})),},- Expr::ScalarSubquery(_)- | Expr::InSubquery(_)- | Expr::Exists { .. }- | Expr::SetComparison(_)- | Expr::OuterReferenceColumn { .. } => {- // we would need to add logical plan operators to datafusion.proto to support this- // see discussion in https://github.com/apache/datafusion/issues/2565- return Err(Error::General("Proto serialization error: Expr::ScalarSubquery(_) | Expr::InSubquery(_) | Expr::Exists { .. } | Exp:OuterReferenceColumn not supported".to_string()));+ Expr::ScalarSubquery(subquery) => protobuf::LogicalExprNode {+ expr_type: Some(ExprType::ScalarSubqueryExpr(Box::new(+ protobuf::ScalarSubqueryExprNode {+ subquery: Some(Box::new(serialize_subquery(subquery, codec)?)),+ },+ ))),+ },+ Expr::InSubquery(_)+ | Expr::Exists(_)+ | Expr::OuterReferenceColumn(_, _)+ | Expr::SetComparison(_) => {+ return Err(Error::General(format!(+ "Proto serialization error: {expr} is not yet supported"+ )));}Expr::GroupingSet(GroupingSet::Cube(exprs)) => protobuf::LogicalExprNode {expr_type: Some(ExprType::Cube(CubeNode {@@ -631,6 +639,19 @@ pub fn serialize_expr(Ok(expr_node)}+fn serialize_subquery(+ subquery: &Subquery,+ codec: &dyn LogicalExtensionCodec,+) -> Result<protobuf::SubqueryNode, Error> {+ let plan = LogicalPlanNode::try_from_logical_plan(&subquery.subquery, codec)+ .map_err(|e| Error::General(e.to_string()))?;+ let outer_ref_columns = serialize_exprs(&subquery.outer_ref_columns, codec)?;+ Ok(protobuf::SubqueryNode {+ subquery: Some(Box::new(plan)),+ outer_ref_columns,+ })+}+pub fn serialize_sorts<'a, I>(sorts: I,codec: &dyn LogicalExtensionCodec,diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 61665db607..732676941b 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -38,7 +38,9 @@ use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{FunctionRegistry, TaskContext};use datafusion_expr::WindowFunctionDefinition;use datafusion_expr::dml::InsertOp;+use datafusion_expr::execution_props::SubqueryIndex;use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};+use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};use datafusion_physical_plan::expressions::{BinaryExpr, CaseExpr, CastExpr, Column, IsNotNullExpr, IsNullExpr, LikeExpr, Literal,@@ -52,7 +54,7 @@ use object_store::ObjectMeta;use object_store::path::Path;use super::{- DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,PhysicalProtoConverterExtension,};use crate::logical_plan::{self};@@ -70,24 +72,21 @@ impl From<&protobuf::PhysicalColumn> for Column {/// # Arguments////// * `proto` - Input proto with physical sort expression node-/// * `registry` - A registry knows how to build logical expressions out of user-defined function names/// * `input_schema` - The Arrow schema for the input, used for determining expression data types/// when performing type coercion.-/// * `codec` - An extension codec used to decode custom UDFs.+/// * `ctx` - Decode context carrying the task context, extension codec, and+/// any scoped state needed during recursive deserialization.+/// * `proto_converter` - Converter hooks used for recursive physical plan and+/// expression deserialization.pub fn parse_physical_sort_expr(proto: &protobuf::PhysicalSortExprNode,- ctx: &TaskContext,+ ctx: &PhysicalPlanDecodeContext<'_>,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<PhysicalSortExpr> {if let Some(expr) = &proto.expr {- let expr = proto_converter.proto_to_physical_expr(- expr.as_ref(),- ctx,- input_schema,- codec,- )?;+ let expr =+ proto_converter.proto_to_physical_expr(expr.as_ref(), input_schema, ctx)?;let options = SortOptions {descending: !proto.asc,nulls_first: proto.nulls_first,@@ -103,21 +102,22 @@ pub fn parse_physical_sort_expr(/// # Arguments////// * `proto` - Input proto with vector of physical sort expression node-/// * `registry` - A registry knows how to build logical expressions out of user-defined function names/// * `input_schema` - The Arrow schema for the input, used for determining expression data types/// when performing type coercion.-/// * `codec` - An extension codec used to decode custom UDFs.+/// * `ctx` - Decode context carrying the task context, extension codec, and+/// any scoped state needed during recursive deserialization.+/// * `proto_converter` - Converter hooks used for recursive physical plan and+/// expression deserialization.pub fn parse_physical_sort_exprs(proto: &[protobuf::PhysicalSortExprNode],- ctx: &TaskContext,+ ctx: &PhysicalPlanDecodeContext<'_>,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Vec<PhysicalSortExpr>> {proto.iter().map(|sort_expr| {- parse_physical_sort_expr(sort_expr, ctx, input_schema, codec, proto_converter)+ parse_physical_sort_expr(sort_expr, ctx, input_schema, proto_converter)}).collect()}@@ -128,34 +128,25 @@ pub fn parse_physical_sort_exprs(////// * `proto` - Input proto with physical window expression node./// * `name` - Name of the window expression.-/// * `registry` - A registry knows how to build logical expressions out of user-defined function names-/// * `input_schema` - The Arrow schema for the input, used for determining expression data types-/// when performing type coercion.-/// * `codec` - An extension codec used to decode custom UDFs.+/// * `input_schema` - The Arrow schema for the input, used for determining+/// expression data types when performing type coercion.+/// * `ctx` - Decode context carrying the task context, extension codec, and+/// any scoped state needed during recursive deserialization.+/// * `proto_converter` - Converter hooks used for recursive physical plan and+/// expression deserialization.pub fn parse_physical_window_expr(proto: &protobuf::PhysicalWindowExprNode,- ctx: &TaskContext,+ ctx: &PhysicalPlanDecodeContext<'_>,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn WindowExpr>> {let window_node_expr =- parse_physical_exprs(&proto.args, ctx, input_schema, codec, proto_converter)?;- let partition_by = parse_physical_exprs(- &proto.partition_by,- ctx,- input_schema,- codec,- proto_converter,- )?;+ parse_physical_exprs(&proto.args, ctx, input_schema, proto_converter)?;+ let partition_by =+ parse_physical_exprs(&proto.partition_by, ctx, input_schema, proto_converter)?;- let order_by = parse_physical_sort_exprs(- &proto.order_by,- ctx,- input_schema,- codec,- proto_converter,- )?;+ let order_by =+ parse_physical_sort_exprs(&proto.order_by, ctx, input_schema, proto_converter)?;let window_frame = proto.window_frame@@ -171,14 +162,20 @@ pub fn parse_physical_window_expr(match window_func {protobuf::physical_window_expr_node::WindowFunction::UserDefinedAggrFunction(udaf_name) => {WindowFunctionDefinition::AggregateUDF(match &proto.fun_definition {- Some(buf) => codec.try_decode_udaf(udaf_name, buf)?,- None => ctx.udaf(udaf_name).or_else(|_| codec.try_decode_udaf(udaf_name, &[]))?,+ Some(buf) => ctx.codec().try_decode_udaf(udaf_name, buf)?,+ None => ctx+ .task_ctx()+ .udaf(udaf_name)+ .or_else(|_| ctx.codec().try_decode_udaf(udaf_name, &[]))?,})}protobuf::physical_window_expr_node::WindowFunction::UserDefinedWindowFunction(udwf_name) => {WindowFunctionDefinition::WindowUDF(match &proto.fun_definition {- Some(buf) => codec.try_decode_udwf(udwf_name, buf)?,- None => ctx.udwf(udwf_name).or_else(|_| codec.try_decode_udwf(udwf_name, &[]))?+ Some(buf) => ctx.codec().try_decode_udwf(udwf_name, buf)?,+ None => ctx+ .task_ctx()+ .udwf(udwf_name)+ .or_else(|_| ctx.codec().try_decode_udwf(udwf_name, &[]))?})}}@@ -206,9 +203,8 @@ pub fn parse_physical_window_expr(pub fn parse_physical_exprs<'a, I>(protos: I,- ctx: &TaskContext,+ ctx: &PhysicalPlanDecodeContext<'_>,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Vec<Arc<dyn PhysicalExpr>>>where@@ -216,7 +212,7 @@ where{protos.into_iter()- .map(|p| proto_converter.proto_to_physical_expr(p, ctx, input_schema, codec))+ .map(|p| proto_converter.proto_to_physical_expr(p, input_schema, ctx)).collect::<Result<Vec<_>>>()}@@ -225,21 +221,22 @@ where/// # Arguments////// * `proto` - Input proto with physical expression node-/// * `registry` - A registry knows how to build logical expressions out of user-defined function names-/// * `input_schema` - The Arrow schema for the input, used for determining expression data types-/// when performing type coercion.-/// * `codec` - An extension codec used to decode custom UDFs.+/// * `ctx` - Task context used to resolve registered functions.+/// * `input_schema` - The Arrow schema for the input, used for determining+/// expression data types when performing type coercion.+/// * `codec` - Physical extension codec used to construct the root decode+/// context for deserialization.pub fn parse_physical_expr(proto: &protobuf::PhysicalExprNode,ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,) -> Result<Arc<dyn PhysicalExpr>> {+ let decode_ctx = PhysicalPlanDecodeContext::new(ctx, codec);parse_physical_expr_with_converter(proto,- ctx,input_schema,- codec,+ &decode_ctx,&DefaultPhysicalProtoConverter {},)}@@ -249,16 +246,16 @@ pub fn parse_physical_expr(/// # Arguments////// * `proto` - Input proto with physical expression node-/// * `registry` - A registry knows how to build logical expressions out of user-defined function names-/// * `input_schema` - The Arrow schema for the input, used for determining expression data types-/// when performing type coercion.-/// * `codec` - An extension codec used to decode custom UDFs.-/// * `proto_converter` - Conversion functions for physical plans and expressions+/// * `input_schema` - The Arrow schema for the input, used for determining+/// expression data types when performing type coercion.+/// * `ctx` - Decode context carrying the task context, extension codec, and+/// any scoped state needed during recursive deserialization.+/// * `proto_converter` - Converter hooks used for recursive physical plan and+/// expression deserialization.pub fn parse_physical_expr_with_converter(proto: &protobuf::PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn PhysicalExpr>> {let expr_type = proto@@ -281,14 +278,7 @@ pub fn parse_physical_expr_with_converter(let operands: Vec<Arc<dyn PhysicalExpr>> = binary_expr.operands.iter()- .map(|e| {- proto_converter.proto_to_physical_expr(- e,- ctx,- input_schema,- codec,- )- })+ .map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx)).collect::<Result<Vec<_>>>()?;if operands.len() < 2 {@@ -311,7 +301,6 @@ pub fn parse_physical_expr_with_converter(ctx,"left",input_schema,- codec,proto_converter,)?,op,@@ -320,7 +309,6 @@ pub fn parse_physical_expr_with_converter(ctx,"right",input_schema,- codec,proto_converter,)?,))@@ -345,7 +333,6 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?))}@@ -355,7 +342,6 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?))}@@ -364,7 +350,6 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?)),ExprType::Negative(e) => {@@ -373,7 +358,6 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?))}@@ -383,10 +367,9 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?,- parse_physical_exprs(&e.list, ctx, input_schema, codec, proto_converter)?,+ parse_physical_exprs(&e.list, ctx, input_schema, proto_converter)?,&e.negated,input_schema,)?,@@ -394,12 +377,7 @@ pub fn parse_physical_expr_with_converter(e.expr.as_ref().map(|e| {- proto_converter.proto_to_physical_expr(- e.as_ref(),- ctx,- input_schema,- codec,- )+ proto_converter.proto_to_physical_expr(e.as_ref(), input_schema, ctx)}).transpose()?,e.when_then_expr@@ -411,7 +389,6 @@ pub fn parse_physical_expr_with_converter(ctx,"when_expr",input_schema,- codec,proto_converter,)?,parse_required_physical_expr(@@ -419,7 +396,6 @@ pub fn parse_physical_expr_with_converter(ctx,"then_expr",input_schema,- codec,proto_converter,)?,))@@ -428,12 +404,7 @@ pub fn parse_physical_expr_with_converter(e.else_expr.as_ref().map(|e| {- proto_converter.proto_to_physical_expr(- e.as_ref(),- ctx,- input_schema,- codec,- )+ proto_converter.proto_to_physical_expr(e.as_ref(), input_schema, ctx)}).transpose()?,)?),@@ -443,7 +414,6 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?,convert_required!(e.arrow_type)?,@@ -455,24 +425,23 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?,convert_required!(e.arrow_type)?,)),ExprType::ScalarUdf(e) => {let udf = match &e.fun_definition {- Some(buf) => codec.try_decode_udf(&e.name, buf)?,+ Some(buf) => ctx.codec().try_decode_udf(&e.name, buf)?,None => ctx+ .task_ctx().udf(e.name.as_str())- .or_else(|_| codec.try_decode_udf(&e.name, &[]))?,+ .or_else(|_| ctx.codec().try_decode_udf(&e.name, &[]))?,};let scalar_fun_def = Arc::clone(&udf);- let args =- parse_physical_exprs(&e.args, ctx, input_schema, codec, proto_converter)?;+ let args = parse_physical_exprs(&e.args, ctx, input_schema, proto_converter)?;- let config_options = Arc::clone(ctx.session_config().options());+ let config_options = Arc::clone(ctx.task_ctx().session_config().options());Arc::new(ScalarFunctionExpr::new(@@ -498,7 +467,6 @@ pub fn parse_physical_expr_with_converter(ctx,"expr",input_schema,- codec,proto_converter,)?,parse_required_physical_expr(@@ -506,7 +474,6 @@ pub fn parse_physical_expr_with_converter(ctx,"pattern",input_schema,- codec,proto_converter,)?,)),@@ -515,7 +482,6 @@ pub fn parse_physical_expr_with_converter(&hash_expr.on_columns,ctx,input_schema,- codec,proto_converter,)?;Arc::new(HashExpr::new(@@ -524,15 +490,35 @@ pub fn parse_physical_expr_with_converter(hash_expr.description.clone(),))}+ ExprType::ScalarSubquery(sq) => {+ let data_type: arrow::datatypes::DataType = sq+ .data_type+ .as_ref()+ .ok_or_else(|| {+ proto_error("Missing data_type in PhysicalScalarSubqueryExprNode")+ })?+ .try_into()?;+ let results = ctx.scalar_subquery_results().ok_or_else(|| {+ proto_error(+ "ScalarSubqueryExpr can only be deserialized as part \+ of a surrounding ScalarSubqueryExec",+ )+ })?;+ Arc::new(ScalarSubqueryExpr::new(+ data_type,+ sq.nullable,+ SubqueryIndex::new(sq.index as usize),+ results.clone(),+ ))+ }ExprType::Extension(extension) => {let inputs: Vec<Arc<dyn PhysicalExpr>> = extension.inputs.iter()- .map(|e| {- proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec)- })+ .map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx)).collect::<Result<_>>()?;- codec.try_decode_expr(extension.expr.as_slice(), &inputs)? as _+ ctx.codec()+ .try_decode_expr(extension.expr.as_slice(), &inputs)? as _}};@@ -541,22 +527,20 @@ pub fn parse_physical_expr_with_converter(fn parse_required_physical_expr(expr: Option<&protobuf::PhysicalExprNode>,- ctx: &TaskContext,+ ctx: &PhysicalPlanDecodeContext<'_>,field: &str,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn PhysicalExpr>> {- expr.map(|e| proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec))+ expr.map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx)).transpose()?.ok_or_else(|| internal_datafusion_err!("Missing required field {field:?}"))}pub fn parse_protobuf_hash_partitioning(partitioning: Option<&protobuf::PhysicalHashRepartition>,- ctx: &TaskContext,+ ctx: &PhysicalPlanDecodeContext<'_>,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Partitioning>> {match partitioning {@@ -565,7 +549,6 @@ pub fn parse_protobuf_hash_partitioning(&hash_part.hash_expr,ctx,input_schema,- codec,proto_converter,)?;@@ -580,9 +563,8 @@ pub fn parse_protobuf_hash_partitioning(pub fn parse_protobuf_partitioning(partitioning: Option<&protobuf::Partitioning>,- ctx: &TaskContext,+ ctx: &PhysicalPlanDecodeContext<'_>,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Partitioning>> {match partitioning {@@ -597,7 +579,6 @@ pub fn parse_protobuf_partitioning(Some(hash_repartition),ctx,input_schema,- codec,proto_converter,)}@@ -651,8 +632,7 @@ pub fn parse_table_schema_from_proto(pub fn parse_protobuf_file_scan_config(proto: &protobuf::FileScanExecConf,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,file_source: Arc<dyn FileSource>,) -> Result<FileScanConfig> {@@ -678,7 +658,6 @@ pub fn parse_protobuf_file_scan_config(&node_collection.physical_sort_expr_nodes,ctx,&schema,- codec,proto_converter,)?;output_ordering.extend(LexOrdering::new(sort_exprs));@@ -694,9 +673,8 @@ pub fn parse_protobuf_file_scan_config(proto_expr.expr.as_ref().ok_or_else(|| {internal_datafusion_err!("ProjectionExpr missing expr field")})?,- ctx,&schema,- codec,+ ctx,)?;Ok(ProjectionExpr::new(expr, proto_expr.alias.clone()))})diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 206f4378d3..6145df6503 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -51,6 +51,7 @@ use datafusion_datasource_parquet::source::ParquetSource;#[cfg(feature = "parquet")]use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{FunctionRegistry, TaskContext};+use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};use datafusion_functions_table::generate_series::{Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,@@ -83,6 +84,7 @@ use datafusion_physical_plan::metrics::{MetricCategory, MetricType};use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::repartition::RepartitionExec;+use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink};use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::union::{InterleaveExec, UnionExec};@@ -117,6 +119,58 @@ use crate::{convert_required, into_required};pub mod from_proto;pub mod to_proto;+/// Context threaded through physical-plan deserialization.+///+/// This bundles the stable per-call inputs for deserialization and the+/// per-scope `ScalarSubqueryResults` handle needed while reconstructing+/// `ScalarSubqueryExpr` nodes inside a `ScalarSubqueryExec` input plan.+#[derive(Clone)]+pub struct PhysicalPlanDecodeContext<'a> {+ task_ctx: &'a TaskContext,+ codec: &'a dyn PhysicalExtensionCodec,+ scalar_subquery_results: Option<ScalarSubqueryResults>,+}++impl<'a> PhysicalPlanDecodeContext<'a> {+ /// Creates a new root decode context.+ pub fn new(task_ctx: &'a TaskContext, codec: &'a dyn PhysicalExtensionCodec) -> Self {+ Self {+ task_ctx,+ codec,+ scalar_subquery_results: None,+ }+ }++ /// Returns the task context used for deserialization.+ pub fn task_ctx(&self) -> &'a TaskContext {+ self.task_ctx+ }++ /// Returns the physical extension codec used for deserialization.+ pub fn codec(&self) -> &'a dyn PhysicalExtensionCodec {+ self.codec+ }++ /// Returns the scalar subquery results container for the current scope, if+ /// one is active.+ pub fn scalar_subquery_results(&self) -> Option<&ScalarSubqueryResults> {+ self.scalar_subquery_results.as_ref()+ }++ /// Returns a child context with a different scalar subquery results+ /// container.+ pub fn with_scalar_subquery_results(+ &self,+ scalar_subquery_results: ScalarSubqueryResults,+ ) -> Self {+ Self {+ task_ctx: self.task_ctx,+ codec: self.codec,+ scalar_subquery_results: Some(scalar_subquery_results),+ }+ }+}+impl AsExecutionPlan for protobuf::PhysicalPlanNode {fn try_decode(buf: &[u8]) -> Result<Self>where@@ -168,9 +222,17 @@ impl protobuf::PhysicalPlanNode {pub fn try_into_physical_plan_with_converter(&self,ctx: &TaskContext,-codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let decode_ctx = PhysicalPlanDecodeContext::new(ctx, codec);+ self.try_into_physical_plan_with_context(&decode_ctx, proto_converter)+ }++ pub(crate) fn try_into_physical_plan_with_context(+ &self,+ ctx: &PhysicalPlanDecodeContext<'_>,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let plan = self.physical_plan_type.as_ref().ok_or_else(|| {proto_error(format!(@@ -179,147 +241,124 @@ impl protobuf::PhysicalPlanNode {})?;match plan {PhysicalPlanType::Explain(explain) => {- self.try_into_explain_physical_plan(explain, ctx, codec, proto_converter)+ self.try_into_explain_physical_plan(explain, ctx, proto_converter)+ }+ PhysicalPlanType::Projection(projection) => {+ self.try_into_projection_physical_plan(projection, ctx, proto_converter)}- PhysicalPlanType::Projection(projection) => self- .try_into_projection_physical_plan(- projection,- ctx,- codec,- proto_converter,- ),PhysicalPlanType::Filter(filter) => {- self.try_into_filter_physical_plan(filter, ctx, codec, proto_converter)+ self.try_into_filter_physical_plan(filter, ctx, proto_converter)}PhysicalPlanType::CsvScan(scan) => {- self.try_into_csv_scan_physical_plan(scan, ctx, codec, proto_converter)+ self.try_into_csv_scan_physical_plan(scan, ctx, proto_converter)}PhysicalPlanType::JsonScan(scan) => {- self.try_into_json_scan_physical_plan(scan, ctx, codec, proto_converter)+ self.try_into_json_scan_physical_plan(scan, ctx, proto_converter)+ }+ PhysicalPlanType::ParquetScan(scan) => {+ self.try_into_parquet_scan_physical_plan(scan, ctx, proto_converter)}- PhysicalPlanType::ParquetScan(scan) => self- .try_into_parquet_scan_physical_plan(scan, ctx, codec, proto_converter),PhysicalPlanType::AvroScan(scan) => {- self.try_into_avro_scan_physical_plan(scan, ctx, codec, proto_converter)+ self.try_into_avro_scan_physical_plan(scan, ctx, proto_converter)}PhysicalPlanType::MemoryScan(scan) => {- self.try_into_memory_scan_physical_plan(scan, ctx, codec, proto_converter)+ self.try_into_memory_scan_physical_plan(scan, ctx, proto_converter)}PhysicalPlanType::ArrowScan(scan) => {- self.try_into_arrow_scan_physical_plan(scan, ctx, codec, proto_converter)+ self.try_into_arrow_scan_physical_plan(scan, ctx, proto_converter)}PhysicalPlanType::CoalesceBatches(coalesce_batches) => self.try_into_coalesce_batches_physical_plan(coalesce_batches,ctx,- codec,proto_converter,),PhysicalPlanType::Merge(merge) => {- self.try_into_merge_physical_plan(merge, ctx, codec, proto_converter)+ self.try_into_merge_physical_plan(merge, ctx, proto_converter)+ }+ PhysicalPlanType::Repartition(repart) => {+ self.try_into_repartition_physical_plan(repart, ctx, proto_converter)+ }+ PhysicalPlanType::GlobalLimit(limit) => {+ self.try_into_global_limit_physical_plan(limit, ctx, proto_converter)+ }+ PhysicalPlanType::LocalLimit(limit) => {+ self.try_into_local_limit_physical_plan(limit, ctx, proto_converter)+ }+ PhysicalPlanType::Window(window_agg) => {+ self.try_into_window_physical_plan(window_agg, ctx, proto_converter)+ }+ PhysicalPlanType::Aggregate(hash_agg) => {+ self.try_into_aggregate_physical_plan(hash_agg, ctx, proto_converter)+ }+ PhysicalPlanType::HashJoin(hashjoin) => {+ self.try_into_hash_join_physical_plan(hashjoin, ctx, proto_converter)}- PhysicalPlanType::Repartition(repart) => self- .try_into_repartition_physical_plan(repart, ctx, codec, proto_converter),- PhysicalPlanType::GlobalLimit(limit) => self- .try_into_global_limit_physical_plan(limit, ctx, codec, proto_converter),- PhysicalPlanType::LocalLimit(limit) => self- .try_into_local_limit_physical_plan(limit, ctx, codec, proto_converter),- PhysicalPlanType::Window(window_agg) => self.try_into_window_physical_plan(- window_agg,- ctx,- codec,- proto_converter,- ),- PhysicalPlanType::Aggregate(hash_agg) => self- .try_into_aggregate_physical_plan(hash_agg, ctx, codec, proto_converter),- PhysicalPlanType::HashJoin(hashjoin) => self- .try_into_hash_join_physical_plan(hashjoin, ctx, codec, proto_converter),PhysicalPlanType::SymmetricHashJoin(sym_join) => self.try_into_symmetric_hash_join_physical_plan(sym_join,ctx,- codec,proto_converter,),PhysicalPlanType::Union(union) => {- self.try_into_union_physical_plan(union, ctx, codec, proto_converter)+ self.try_into_union_physical_plan(union, ctx, proto_converter)+ }+ PhysicalPlanType::Interleave(interleave) => {+ self.try_into_interleave_physical_plan(interleave, ctx, proto_converter)+ }+ PhysicalPlanType::CrossJoin(crossjoin) => {+ self.try_into_cross_join_physical_plan(crossjoin, ctx, proto_converter)}- PhysicalPlanType::Interleave(interleave) => self- .try_into_interleave_physical_plan(- interleave,- ctx,- codec,- proto_converter,- ),- PhysicalPlanType::CrossJoin(crossjoin) => self- .try_into_cross_join_physical_plan(- crossjoin,- ctx,- codec,- proto_converter,- ),PhysicalPlanType::Empty(empty) => {- self.try_into_empty_physical_plan(empty, ctx, codec, proto_converter)+ self.try_into_empty_physical_plan(empty, ctx, proto_converter)}PhysicalPlanType::PlaceholderRow(placeholder) => {- self.try_into_placeholder_row_physical_plan(placeholder, ctx, codec)+ self.try_into_placeholder_row_physical_plan(placeholder, ctx)}PhysicalPlanType::Sort(sort) => {- self.try_into_sort_physical_plan(sort, ctx, codec, proto_converter)+ self.try_into_sort_physical_plan(sort, ctx, proto_converter)}PhysicalPlanType::SortPreservingMerge(sort) => self- .try_into_sort_preserving_merge_physical_plan(- sort,- ctx,- codec,- proto_converter,- ),- PhysicalPlanType::Extension(extension) => self- .try_into_extension_physical_plan(extension, ctx, codec, proto_converter),- PhysicalPlanType::NestedLoopJoin(join) => self- .try_into_nested_loop_join_physical_plan(- join,- ctx,- codec,- proto_converter,- ),+ .try_into_sort_preserving_merge_physical_plan(sort, ctx, proto_converter),+ PhysicalPlanType::Extension(extension) => {+ self.try_into_extension_physical_plan(extension, ctx, proto_converter)+ }+ PhysicalPlanType::NestedLoopJoin(join) => {+ self.try_into_nested_loop_join_physical_plan(join, ctx, proto_converter)+ }PhysicalPlanType::Analyze(analyze) => {- self.try_into_analyze_physical_plan(analyze, ctx, codec, proto_converter)+ self.try_into_analyze_physical_plan(analyze, ctx, proto_converter)}PhysicalPlanType::JsonSink(sink) => {- self.try_into_json_sink_physical_plan(sink, ctx, codec, proto_converter)+ self.try_into_json_sink_physical_plan(sink, ctx, proto_converter)}PhysicalPlanType::CsvSink(sink) => {- self.try_into_csv_sink_physical_plan(sink, ctx, codec, proto_converter)+ self.try_into_csv_sink_physical_plan(sink, ctx, proto_converter)}#[cfg_attr(not(feature = "parquet"), allow(unused_variables))]- PhysicalPlanType::ParquetSink(sink) => self- .try_into_parquet_sink_physical_plan(sink, ctx, codec, proto_converter),+ PhysicalPlanType::ParquetSink(sink) => {+ self.try_into_parquet_sink_physical_plan(sink, ctx, proto_converter)+ }PhysicalPlanType::Unnest(unnest) => {- self.try_into_unnest_physical_plan(unnest, ctx, codec, proto_converter)+ self.try_into_unnest_physical_plan(unnest, ctx, proto_converter)+ }+ PhysicalPlanType::Cooperative(cooperative) => {+ self.try_into_cooperative_physical_plan(cooperative, ctx, proto_converter)}- PhysicalPlanType::Cooperative(cooperative) => self- .try_into_cooperative_physical_plan(- cooperative,- ctx,- codec,- proto_converter,- ),PhysicalPlanType::GenerateSeries(generate_series) => {self.try_into_generate_series_physical_plan(generate_series)}PhysicalPlanType::SortMergeJoin(sort_join) => {- self.try_into_sort_join(sort_join, ctx, codec, proto_converter)+ self.try_into_sort_join(sort_join, ctx, proto_converter)+ }+ PhysicalPlanType::AsyncFunc(async_func) => {+ self.try_into_async_func_physical_plan(async_func, ctx, proto_converter)}- PhysicalPlanType::AsyncFunc(async_func) => self- .try_into_async_func_physical_plan(- async_func,- ctx,- codec,- proto_converter,- ),PhysicalPlanType::Buffer(buffer) => {- self.try_into_buffer_physical_plan(buffer, ctx, codec, proto_converter)+ self.try_into_buffer_physical_plan(buffer, ctx, proto_converter)+ }+ PhysicalPlanType::ScalarSubquery(sq) => {+ self.try_into_scalar_subquery_physical_plan(sq, ctx, proto_converter)}}}@@ -569,6 +608,14 @@ impl protobuf::PhysicalPlanNode {);}+ if let Some(exec) = plan.downcast_ref::<ScalarSubqueryExec>() {+ return protobuf::PhysicalPlanNode::try_from_scalar_subquery_exec(+ exec,+ codec,+ proto_converter,+ );+ }+let mut buf: Vec<u8> = vec![];match codec.try_encode(Arc::clone(&plan_clone), &mut buf) {Ok(_) => {@@ -602,9 +649,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_explain_physical_plan(&self,explain: &protobuf::ExplainExecNode,- _ctx: &TaskContext,-- _codec: &dyn PhysicalExtensionCodec,+ _ctx: &PhysicalPlanDecodeContext<'_>,_proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(ExplainExec::new(@@ -621,13 +666,11 @@ impl protobuf::PhysicalPlanNode {fn try_into_projection_physical_plan(&self,projection: &protobuf::ProjectionExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&projection.input, ctx, codec, proto_converter)?;+ into_physical_plan(&projection.input, ctx, proto_converter)?;let exprs = projection.expr.iter()@@ -636,9 +679,8 @@ impl protobuf::PhysicalPlanNode {Ok((proto_converter.proto_to_physical_expr(expr,- ctx,input.schema().as_ref(),- codec,+ ctx,)?,name.to_string(),))@@ -654,24 +696,17 @@ impl protobuf::PhysicalPlanNode {fn try_into_filter_physical_plan(&self,filter: &protobuf::FilterExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&filter.input, ctx, codec, proto_converter)?;+ into_physical_plan(&filter.input, ctx, proto_converter)?;let predicate = filter.expr.as_ref().map(|expr| {- proto_converter.proto_to_physical_expr(- expr,- ctx,- input.schema().as_ref(),- codec,- )+ proto_converter.proto_to_physical_expr(expr, input.schema().as_ref(), ctx)}).transpose()?.ok_or_else(|| {@@ -711,9 +746,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_csv_scan_physical_plan(&self,scan: &protobuf::CsvScanExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let escape =@@ -755,7 +788,6 @@ impl protobuf::PhysicalPlanNode {let conf = FileScanConfigBuilder::from(parse_protobuf_file_scan_config(scan.base_conf.as_ref().unwrap(),ctx,- codec,proto_converter,source,)?)@@ -767,9 +799,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_json_scan_physical_plan(&self,scan: &protobuf::JsonScanExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let base_conf = scan.base_conf.as_ref().unwrap();@@ -777,7 +807,6 @@ impl protobuf::PhysicalPlanNode {let scan_conf = parse_protobuf_file_scan_config(base_conf,ctx,- codec,proto_converter,Arc::new(JsonSource::new(table_schema)),)?;@@ -787,8 +816,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_arrow_scan_physical_plan(&self,scan: &protobuf::ArrowScanExecNode,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let base_conf = scan.base_conf.as_ref().ok_or_else(|| {@@ -798,7 +826,6 @@ impl protobuf::PhysicalPlanNode {let scan_conf = parse_protobuf_file_scan_config(base_conf,ctx,- codec,proto_converter,Arc::new(ArrowSource::new_file_source(table_schema)),)?;@@ -809,8 +836,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_parquet_scan_physical_plan(&self,scan: &protobuf::ParquetScanExecNode,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "parquet")]@@ -839,9 +865,8 @@ impl protobuf::PhysicalPlanNode {.map(|expr| {proto_converter.proto_to_physical_expr(expr,- ctx,predicate_schema.as_ref(),- codec,+ ctx,)}).transpose()?;@@ -857,9 +882,15 @@ impl protobuf::PhysicalPlanNode {false => ObjectStoreUrl::parse(&base_conf.object_store_url)?,true => ObjectStoreUrl::local_filesystem(),};- let store = ctx.runtime_env().object_store(object_store_url)?;- let metadata_cache =- ctx.runtime_env().cache_manager.get_file_metadata_cache();+ let store = ctx+ .task_ctx()+ .runtime_env()+ .object_store(object_store_url)?;+ let metadata_cache = ctx+ .task_ctx()+ .runtime_env()+ .cache_manager+ .get_file_metadata_cache();let reader_factory =Arc::new(CachedParquetFileReaderFactory::new(store, metadata_cache));@@ -873,7 +904,6 @@ impl protobuf::PhysicalPlanNode {let base_config = parse_protobuf_file_scan_config(base_conf,ctx,- codec,proto_converter,Arc::new(source),)?;@@ -889,8 +919,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_avro_scan_physical_plan(&self,scan: &protobuf::AvroScanExecNode,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "avro")]@@ -900,7 +929,6 @@ impl protobuf::PhysicalPlanNode {let conf = parse_protobuf_file_scan_config(scan.base_conf.as_ref().unwrap(),ctx,- codec,proto_converter,Arc::new(AvroSource::new(table_schema)),)?;@@ -914,9 +942,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_memory_scan_physical_plan(&self,scan: &protobuf::MemoryScanExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let partitions = scan@@ -947,7 +973,6 @@ impl protobuf::PhysicalPlanNode {&ordering.physical_sort_expr_nodes,ctx,&schema,- codec,proto_converter,)?;sort_information.extend(LexOrdering::new(sort_exprs));@@ -965,13 +990,11 @@ impl protobuf::PhysicalPlanNode {fn try_into_coalesce_batches_physical_plan(&self,coalesce_batches: &protobuf::CoalesceBatchesExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&coalesce_batches.input, ctx, codec, proto_converter)?;+ into_physical_plan(&coalesce_batches.input, ctx, proto_converter)?;Ok(Arc::new(#[expect(deprecated)]CoalesceBatchesExec::new(input, coalesce_batches.target_batch_size as usize)@@ -982,13 +1005,11 @@ impl protobuf::PhysicalPlanNode {fn try_into_merge_physical_plan(&self,merge: &protobuf::CoalescePartitionsExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&merge.input, ctx, codec, proto_converter)?;+ into_physical_plan(&merge.input, ctx, proto_converter)?;Ok(Arc::new(CoalescePartitionsExec::new(input).with_fetch(merge.fetch.map(|f| f as usize)),@@ -998,18 +1019,15 @@ impl protobuf::PhysicalPlanNode {fn try_into_repartition_physical_plan(&self,repart: &protobuf::RepartitionExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&repart.input, ctx, codec, proto_converter)?;+ into_physical_plan(&repart.input, ctx, proto_converter)?;let partitioning = parse_protobuf_partitioning(repart.partitioning.as_ref(),ctx,input.schema().as_ref(),- codec,proto_converter,)?;let mut repart_exec = RepartitionExec::try_new(input, partitioning.unwrap())?;@@ -1022,13 +1040,11 @@ impl protobuf::PhysicalPlanNode {fn try_into_global_limit_physical_plan(&self,limit: &protobuf::GlobalLimitExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&limit.input, ctx, codec, proto_converter)?;+ into_physical_plan(&limit.input, ctx, proto_converter)?;let fetch = if limit.fetch >= 0 {Some(limit.fetch as usize)} else {@@ -1044,26 +1060,22 @@ impl protobuf::PhysicalPlanNode {fn try_into_local_limit_physical_plan(&self,limit: &protobuf::LocalLimitExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&limit.input, ctx, codec, proto_converter)?;+ into_physical_plan(&limit.input, ctx, proto_converter)?;Ok(Arc::new(LocalLimitExec::new(input, limit.fetch as usize)))}fn try_into_window_physical_plan(&self,window_agg: &protobuf::WindowAggExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&window_agg.input, ctx, codec, proto_converter)?;+ into_physical_plan(&window_agg.input, ctx, proto_converter)?;let input_schema = input.schema();let physical_window_expr: Vec<Arc<dyn WindowExpr>> = window_agg@@ -1074,7 +1086,6 @@ impl protobuf::PhysicalPlanNode {window_expr,ctx,input_schema.as_ref(),- codec,proto_converter,)})@@ -1084,12 +1095,7 @@ impl protobuf::PhysicalPlanNode {.partition_keys.iter().map(|expr| {- proto_converter.proto_to_physical_expr(- expr,- ctx,- input.schema().as_ref(),- codec,- )+ proto_converter.proto_to_physical_expr(expr, input.schema().as_ref(), ctx)}).collect::<Result<Vec<Arc<dyn PhysicalExpr>>>>()?;@@ -1122,13 +1128,11 @@ impl protobuf::PhysicalPlanNode {fn try_into_aggregate_physical_plan(&self,hash_agg: &protobuf::AggregateExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&hash_agg.input, ctx, codec, proto_converter)?;+ into_physical_plan(&hash_agg.input, ctx, proto_converter)?;let mode = protobuf::AggregateMode::try_from(hash_agg.mode).map_err(|_| {proto_error(format!("Received a AggregateNode message with unknown AggregateMode {}",@@ -1154,7 +1158,7 @@ impl protobuf::PhysicalPlanNode {.zip(hash_agg.group_expr_name.iter()).map(|(expr, name)| {proto_converter- .proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec)+ .proto_to_physical_expr(expr, input.schema().as_ref(), ctx).map(|expr| (expr, name.to_string()))}).collect::<Result<Vec<_>, _>>()?;@@ -1165,7 +1169,7 @@ impl protobuf::PhysicalPlanNode {.zip(hash_agg.group_expr_name.iter()).map(|(expr, name)| {proto_converter- .proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec)+ .proto_to_physical_expr(expr, input.schema().as_ref(), ctx).map(|expr| (expr, name.to_string()))}).collect::<Result<Vec<_>, _>>()?;@@ -1194,12 +1198,7 @@ impl protobuf::PhysicalPlanNode {expr.expr.as_ref().map(|e| {- proto_converter.proto_to_physical_expr(- e,- ctx,- &physical_schema,- codec,- )+ proto_converter.proto_to_physical_expr(e, &physical_schema, ctx)}).transpose()})@@ -1222,9 +1221,8 @@ impl protobuf::PhysicalPlanNode {.map(|e| {proto_converter.proto_to_physical_expr(e,- ctx,&physical_schema,- codec,+ ctx,)}).collect::<Result<Vec<_>>>()?;@@ -1236,7 +1234,6 @@ impl protobuf::PhysicalPlanNode {e,ctx,&physical_schema,- codec,proto_converter,)})@@ -1248,11 +1245,14 @@ impl protobuf::PhysicalPlanNode {AggregateFunction::UserDefinedAggrFunction(udaf_name) => {let agg_udf = match &agg_node.fun_definition {Some(buf) => {- codec.try_decode_udaf(udaf_name, buf)?+ ctx.codec().try_decode_udaf(udaf_name, buf)?}- None => ctx.udaf(udaf_name).or_else(|_| {- codec.try_decode_udaf(udaf_name, &[])- })?,+ None => ctx.task_ctx().udaf(udaf_name).or_else(+ |_| {+ ctx.codec()+ .try_decode_udaf(udaf_name, &[])+ },+ )?,};AggregateExprBuilder::new(agg_udf, input_phy_expr)@@ -1304,15 +1304,13 @@ impl protobuf::PhysicalPlanNode {fn try_into_hash_join_physical_plan(&self,hashjoin: &protobuf::HashJoinExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let left: Arc<dyn ExecutionPlan> =- into_physical_plan(&hashjoin.left, ctx, codec, proto_converter)?;+ into_physical_plan(&hashjoin.left, ctx, proto_converter)?;let right: Arc<dyn ExecutionPlan> =- into_physical_plan(&hashjoin.right, ctx, codec, proto_converter)?;+ into_physical_plan(&hashjoin.right, ctx, proto_converter)?;let left_schema = left.schema();let right_schema = right.schema();let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = hashjoin@@ -1321,15 +1319,13 @@ impl protobuf::PhysicalPlanNode {.map(|col| {let left = proto_converter.proto_to_physical_expr(&col.left.clone().unwrap(),- ctx,left_schema.as_ref(),- codec,+ ctx,)?;let right = proto_converter.proto_to_physical_expr(&col.right.clone().unwrap(),- ctx,right_schema.as_ref(),- codec,+ ctx,)?;Ok((left, right))})@@ -1362,8 +1358,8 @@ impl protobuf::PhysicalPlanNode {f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,- ctx, &schema,- codec,+ &schema,+ ctx,)?;let column_indices = f.column_indices.iter()@@ -1424,13 +1420,11 @@ impl protobuf::PhysicalPlanNode {fn try_into_symmetric_hash_join_physical_plan(&self,sym_join: &protobuf::SymmetricHashJoinExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let left = into_physical_plan(&sym_join.left, ctx, codec, proto_converter)?;- let right = into_physical_plan(&sym_join.right, ctx, codec, proto_converter)?;+ let left = into_physical_plan(&sym_join.left, ctx, proto_converter)?;+ let right = into_physical_plan(&sym_join.right, ctx, proto_converter)?;let left_schema = left.schema();let right_schema = right.schema();let on = sym_join@@ -1439,15 +1433,13 @@ impl protobuf::PhysicalPlanNode {.map(|col| {let left = proto_converter.proto_to_physical_expr(&col.left.clone().unwrap(),- ctx,left_schema.as_ref(),- codec,+ ctx,)?;let right = proto_converter.proto_to_physical_expr(&col.right.clone().unwrap(),- ctx,right_schema.as_ref(),- codec,+ ctx,)?;Ok((left, right))})@@ -1480,8 +1472,8 @@ impl protobuf::PhysicalPlanNode {f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,- ctx, &schema,- codec,+ &schema,+ ctx,)?;let column_indices = f.column_indices.iter()@@ -1507,7 +1499,6 @@ impl protobuf::PhysicalPlanNode {&sym_join.left_sort_exprs,ctx,&left_schema,- codec,proto_converter,)?;let left_sort_exprs = LexOrdering::new(left_sort_exprs);@@ -1516,7 +1507,6 @@ impl protobuf::PhysicalPlanNode {&sym_join.right_sort_exprs,ctx,&right_schema,- codec,proto_converter,)?;let right_sort_exprs = LexOrdering::new(right_sort_exprs);@@ -1555,14 +1545,12 @@ impl protobuf::PhysicalPlanNode {fn try_into_union_physical_plan(&self,union: &protobuf::UnionExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];for input in &union.inputs {- inputs.push(proto_converter.proto_to_execution_plan(ctx, codec, input)?);+ inputs.push(proto_converter.proto_to_execution_plan(input, ctx)?);}UnionExec::try_new(inputs)}@@ -1570,14 +1558,12 @@ impl protobuf::PhysicalPlanNode {fn try_into_interleave_physical_plan(&self,interleave: &protobuf::InterleaveExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];for input in &interleave.inputs {- inputs.push(proto_converter.proto_to_execution_plan(ctx, codec, input)?);+ inputs.push(proto_converter.proto_to_execution_plan(input, ctx)?);}Ok(Arc::new(InterleaveExec::try_new(inputs)?))}@@ -1585,24 +1571,20 @@ impl protobuf::PhysicalPlanNode {fn try_into_cross_join_physical_plan(&self,crossjoin: &protobuf::CrossJoinExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let left: Arc<dyn ExecutionPlan> =- into_physical_plan(&crossjoin.left, ctx, codec, proto_converter)?;+ into_physical_plan(&crossjoin.left, ctx, proto_converter)?;let right: Arc<dyn ExecutionPlan> =- into_physical_plan(&crossjoin.right, ctx, codec, proto_converter)?;+ into_physical_plan(&crossjoin.right, ctx, proto_converter)?;Ok(Arc::new(CrossJoinExec::new(left, right)))}fn try_into_empty_physical_plan(&self,empty: &protobuf::EmptyExecNode,- _ctx: &TaskContext,-- _codec: &dyn PhysicalExtensionCodec,+ _ctx: &PhysicalPlanDecodeContext<'_>,_proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let schema = Arc::new(convert_required!(empty.schema)?);@@ -1612,9 +1594,7 @@ impl protobuf::PhysicalPlanNode {fn try_into_placeholder_row_physical_plan(&self,placeholder: &protobuf::PlaceholderRowExecNode,- _ctx: &TaskContext,-- _codec: &dyn PhysicalExtensionCodec,+ _ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {let schema = Arc::new(convert_required!(placeholder.schema)?);Ok(Arc::new(PlaceholderRowExec::new(schema)))@@ -1623,12 +1603,10 @@ impl protobuf::PhysicalPlanNode {fn try_into_sort_physical_plan(&self,sort: &protobuf::SortExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sort.input, ctx, codec, proto_converter)?;+ let input = into_physical_plan(&sort.input, ctx, proto_converter)?;let exprs = sort.expr.iter()@@ -1649,7 +1627,11 @@ impl protobuf::PhysicalPlanNode {})?.as_ref();Ok(PhysicalSortExpr {- expr: proto_converter.proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec)?,+ expr: proto_converter.proto_to_physical_expr(+ expr,+ input.schema().as_ref(),+ ctx,+ )?,options: SortOptions {descending: !sort_expr.asc,nulls_first: sort_expr.nulls_first,@@ -1676,12 +1658,10 @@ impl protobuf::PhysicalPlanNode {fn try_into_sort_preserving_merge_physical_plan(&self,sort: &protobuf::SortPreservingMergeExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sort.input, ctx, codec, proto_converter)?;+ let input = into_physical_plan(&sort.input, ctx, proto_converter)?;let exprs = sort.expr.iter()@@ -1704,9 +1684,8 @@ impl protobuf::PhysicalPlanNode {Ok(PhysicalSortExpr {expr: proto_converter.proto_to_physical_expr(expr,- ctx,input.schema().as_ref(),- codec,+ ctx,)?,options: SortOptions {descending: !sort_expr.asc,@@ -1730,18 +1709,18 @@ impl protobuf::PhysicalPlanNode {fn try_into_extension_physical_plan(&self,extension: &protobuf::PhysicalExtensionNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let inputs: Vec<Arc<dyn ExecutionPlan>> = extension.inputs.iter()- .map(|i| proto_converter.proto_to_execution_plan(ctx, codec, i))+ .map(|i| proto_converter.proto_to_execution_plan(i, ctx)).collect::<Result<_>>()?;- let extension_node = codec.try_decode(extension.node.as_slice(), &inputs, ctx)?;+ let extension_node =+ ctx.codec()+ .try_decode(extension.node.as_slice(), &inputs, ctx.task_ctx())?;Ok(extension_node)}@@ -1749,15 +1728,13 @@ impl protobuf::PhysicalPlanNode {fn try_into_nested_loop_join_physical_plan(&self,join: &protobuf::NestedLoopJoinExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let left: Arc<dyn ExecutionPlan> =- into_physical_plan(&join.left, ctx, codec, proto_converter)?;+ into_physical_plan(&join.left, ctx, proto_converter)?;let right: Arc<dyn ExecutionPlan> =- into_physical_plan(&join.right, ctx, codec, proto_converter)?;+ into_physical_plan(&join.right, ctx, proto_converter)?;let join_type = protobuf::JoinType::try_from(join.join_type).map_err(|_| {proto_error(format!("Received a NestedLoopJoinExecNode message with unknown JoinType {}",@@ -1774,12 +1751,13 @@ impl protobuf::PhysicalPlanNode {.ok_or_else(|| proto_error("Missing JoinFilter schema"))?.try_into()?;- let expression = proto_converter.proto_to_physical_expr(+ let expression = proto_converter+ .proto_to_physical_expr(f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,- ctx, &schema,- codec,+ &schema,+ ctx,)?;let column_indices = f.column_indices.iter()@@ -1824,13 +1802,11 @@ impl protobuf::PhysicalPlanNode {fn try_into_analyze_physical_plan(&self,analyze: &protobuf::AnalyzeExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&analyze.input, ctx, codec, proto_converter)?;+ into_physical_plan(&analyze.input, ctx, proto_converter)?;let metric_categories = if analyze.has_metric_categories {let cats: Result<Vec<MetricCategory>> = analyze.metric_categories@@ -1854,12 +1830,10 @@ impl protobuf::PhysicalPlanNode {fn try_into_json_sink_physical_plan(&self,sink: &protobuf::JsonSinkExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;+ let input = into_physical_plan(&sink.input, ctx, proto_converter)?;let data_sink: JsonSink = sink.sink@@ -1875,7 +1849,6 @@ impl protobuf::PhysicalPlanNode {&collection.physical_sort_expr_nodes,ctx,&sink_schema,- codec,proto_converter,).map(|sort_exprs| {@@ -1894,12 +1867,10 @@ impl protobuf::PhysicalPlanNode {fn try_into_csv_sink_physical_plan(&self,sink: &protobuf::CsvSinkExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;+ let input = into_physical_plan(&sink.input, ctx, proto_converter)?;let data_sink: CsvSink = sink.sink@@ -1915,7 +1886,6 @@ impl protobuf::PhysicalPlanNode {&collection.physical_sort_expr_nodes,ctx,&sink_schema,- codec,proto_converter,).map(|sort_exprs| {@@ -1935,14 +1905,12 @@ impl protobuf::PhysicalPlanNode {fn try_into_parquet_sink_physical_plan(&self,sink: &protobuf::ParquetSinkExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "parquet")]{- let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;+ let input = into_physical_plan(&sink.input, ctx, proto_converter)?;let data_sink: ParquetSink = sink.sink@@ -1958,7 +1926,6 @@ impl protobuf::PhysicalPlanNode {&collection.physical_sort_expr_nodes,ctx,&sink_schema,- codec,proto_converter,).map(|sort_exprs| {@@ -1980,12 +1947,10 @@ impl protobuf::PhysicalPlanNode {fn try_into_unnest_physical_plan(&self,unnest: &protobuf::UnnestExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&unnest.input, ctx, codec, proto_converter)?;+ let input = into_physical_plan(&unnest.input, ctx, proto_converter)?;Ok(Arc::new(UnnestExec::new(input,@@ -2012,14 +1977,12 @@ impl protobuf::PhysicalPlanNode {fn try_into_sort_join(&self,sort_join: &SortMergeJoinExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let left = into_physical_plan(&sort_join.left, ctx, codec, proto_converter)?;+ let left = into_physical_plan(&sort_join.left, ctx, proto_converter)?;let left_schema = left.schema();- let right = into_physical_plan(&sort_join.right, ctx, codec, proto_converter)?;+ let right = into_physical_plan(&sort_join.right, ctx, proto_converter)?;let right_schema = right.schema();let filter = sort_join@@ -2036,9 +1999,8 @@ impl protobuf::PhysicalPlanNode {f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,- ctx,&schema,- codec,+ ctx,)?;let column_indices = f.column_indices@@ -2097,15 +2059,13 @@ impl protobuf::PhysicalPlanNode {.map(|col| {let left = proto_converter.proto_to_physical_expr(&col.left.clone().unwrap(),- ctx,left_schema.as_ref(),- codec,+ ctx,)?;let right = proto_converter.proto_to_physical_expr(&col.right.clone().unwrap(),- ctx,right_schema.as_ref(),- codec,+ ctx,)?;Ok((left, right))})@@ -2190,24 +2150,21 @@ impl protobuf::PhysicalPlanNode {fn try_into_cooperative_physical_plan(&self,field_stream: &protobuf::CooperativeExecNode,- ctx: &TaskContext,-- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&field_stream.input, ctx, codec, proto_converter)?;+ let input = into_physical_plan(&field_stream.input, ctx, proto_converter)?;Ok(Arc::new(CooperativeExec::new(input)))}fn try_into_async_func_physical_plan(&self,async_func: &protobuf::AsyncFuncExecNode,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&async_func.input, ctx, codec, proto_converter)?;+ into_physical_plan(&async_func.input, ctx, proto_converter)?;if async_func.async_exprs.len() != async_func.async_expr_names.len() {return internal_err!(@@ -2222,9 +2179,8 @@ impl protobuf::PhysicalPlanNode {.map(|(expr, name)| {let physical_expr = proto_converter.proto_to_physical_expr(expr,- ctx,input.schema().as_ref(),- codec,+ ctx,)?;Ok(Arc::new(AsyncFuncExpr::try_new(@@ -2241,16 +2197,49 @@ impl protobuf::PhysicalPlanNode {fn try_into_buffer_physical_plan(&self,buffer: &protobuf::BufferExecNode,- ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&buffer.input, ctx, extension_codec, proto_converter)?;+ into_physical_plan(&buffer.input, ctx, proto_converter)?;Ok(Arc::new(BufferExec::new(input, buffer.capacity as usize)))}+ fn try_into_scalar_subquery_physical_plan(+ &self,+ sq: &protobuf::ScalarSubqueryExecNode,+ ctx: &PhysicalPlanDecodeContext<'_>,+ proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // First, deserialize the main input plan. We set up the subquery results+ // container first, so that ScalarSubqueryExpr nodes can reference it.+ let subquery_results = ScalarSubqueryResults::new(sq.subqueries.len());+ let input_ctx = ctx.with_scalar_subquery_results(subquery_results.clone());+ let input = into_physical_plan(&sq.input, &input_ctx, proto_converter)?;++ // Now deserialize the subquery children.+ let subqueries: Vec<ScalarSubqueryLink> = sq+ .subqueries+ .iter()+ .enumerate()+ .map(|(index, sq_plan)| {+ let plan =+ sq_plan.try_into_physical_plan_with_context(ctx, proto_converter)?;+ Ok(ScalarSubqueryLink {+ plan,+ index: SubqueryIndex::new(index),+ })+ })+ .collect::<Result<Vec<_>>>()?;++ Ok(Arc::new(ScalarSubqueryExec::new(+ input,+ subqueries,+ subquery_results,+ )))+ }+fn try_from_explain_exec(exec: &ExplainExec,_codec: &dyn PhysicalExtensionCodec,@@ -3643,6 +3632,38 @@ impl protobuf::PhysicalPlanNode {))),})}++ fn try_from_scalar_subquery_exec(+ exec: &ScalarSubqueryExec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Self> {+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::clone(exec.input()),+ codec,+ proto_converter,+ )?;+ let subqueries = exec+ .subqueries()+ .iter()+ .map(|sq| {+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::clone(&sq.plan),+ codec,+ proto_converter,+ )+ })+ .collect::<Result<Vec<_>>>()?;++ Ok(protobuf::PhysicalPlanNode {+ physical_plan_type: Some(PhysicalPlanType::ScalarSubquery(Box::new(+ protobuf::ScalarSubqueryExecNode {+ input: Some(Box::new(input)),+ subqueries,+ },+ ))),+ })+ }}pub trait AsExecutionPlan: Debug + Send + Sync + Clone {@@ -3751,11 +3772,21 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {pub trait PhysicalProtoConverterExtension {fn proto_to_execution_plan(&self,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,proto: &protobuf::PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>>;+ fn default_proto_to_execution_plan(+ &self,+ proto: &protobuf::PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,+ ) -> Result<Arc<dyn ExecutionPlan>>+ where+ Self: Sized,+ {+ proto.try_into_physical_plan_with_context(ctx, self)+ }+fn execution_plan_to_proto(&self,plan: &Arc<dyn ExecutionPlan>,@@ -3765,11 +3796,22 @@ pub trait PhysicalProtoConverterExtension {fn proto_to_physical_expr(&self,proto: &protobuf::PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>>;+ fn default_proto_to_physical_expr(+ &self,+ proto: &protobuf::PhysicalExprNode,+ input_schema: &Schema,+ ctx: &PhysicalPlanDecodeContext<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>>+ where+ Self: Sized,+ {+ parse_physical_expr_with_converter(proto, input_schema, ctx, self)+ }+fn physical_expr_to_proto(&self,expr: &Arc<dyn PhysicalExpr>,@@ -3790,15 +3832,15 @@ struct DataEncoderTuple {pub blob: Vec<u8>,}-pub struct DefaultPhysicalProtoConverter;+pub struct DefaultPhysicalProtoConverter {}+impl PhysicalProtoConverterExtension for DefaultPhysicalProtoConverter {fn proto_to_execution_plan(&self,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,proto: &protobuf::PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {- proto.try_into_physical_plan_with_converter(ctx, codec, self)+ proto.try_into_physical_plan_with_context(ctx, self)}fn execution_plan_to_proto(@@ -3819,15 +3861,14 @@ impl PhysicalProtoConverterExtension for DefaultPhysicalProtoConverter {fn proto_to_physical_expr(&self,proto: &protobuf::PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>>whereSelf: Sized,{// Default implementation calls the free function- parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)+ parse_physical_expr_with_converter(proto, input_schema, ctx, self)}fn physical_expr_to_proto(@@ -3857,9 +3898,8 @@ impl DeduplicatingSerializer {impl PhysicalProtoConverterExtension for DeduplicatingSerializer {fn proto_to_execution_plan(&self,- _ctx: &TaskContext,- _codec: &dyn PhysicalExtensionCodec,_proto: &protobuf::PhysicalPlanNode,+ _ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {internal_err!("DeduplicatingSerializer cannot deserialize execution plans")}@@ -3882,9 +3922,8 @@ impl PhysicalProtoConverterExtension for DeduplicatingSerializer {fn proto_to_physical_expr(&self,_proto: &protobuf::PhysicalExprNode,- _ctx: &TaskContext,_input_schema: &Schema,- _codec: &dyn PhysicalExtensionCodec,+ _ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>>whereSelf: Sized,@@ -3924,11 +3963,10 @@ struct DeduplicatingDeserializer {impl PhysicalProtoConverterExtension for DeduplicatingDeserializer {fn proto_to_execution_plan(&self,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,proto: &protobuf::PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {- proto.try_into_physical_plan_with_converter(ctx, codec, self)+ proto.try_into_physical_plan_with_context(ctx, self)}fn execution_plan_to_proto(@@ -3945,9 +3983,8 @@ impl PhysicalProtoConverterExtension for DeduplicatingDeserializer {fn proto_to_physical_expr(&self,proto: &protobuf::PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>>whereSelf: Sized,@@ -3958,17 +3995,12 @@ impl PhysicalProtoConverterExtension for DeduplicatingDeserializer {return Ok(Arc::clone(cached));}// Deserialize and cache- let expr = parse_physical_expr_with_converter(- proto,- ctx,- input_schema,- codec,- self,- )?;+ let expr =+ parse_physical_expr_with_converter(proto, input_schema, ctx, self)?;self.cache.borrow_mut().insert(expr_id, Arc::clone(&expr));Ok(expr)} else {- parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)+ parse_physical_expr_with_converter(proto, input_schema, ctx, self)}}@@ -4002,12 +4034,11 @@ pub struct DeduplicatingProtoConverter {}impl PhysicalProtoConverterExtension for DeduplicatingProtoConverter {fn proto_to_execution_plan(&self,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,proto: &protobuf::PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {let deserializer = DeduplicatingDeserializer::default();- proto.try_into_physical_plan_with_converter(ctx, codec, &deserializer)+ proto.try_into_physical_plan_with_context(ctx, &deserializer)}fn execution_plan_to_proto(@@ -4029,15 +4060,14 @@ impl PhysicalProtoConverterExtension for DeduplicatingProtoConverter {fn proto_to_physical_expr(&self,proto: &protobuf::PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>>whereSelf: Sized,{let deserializer = DeduplicatingDeserializer::default();- deserializer.proto_to_physical_expr(proto, ctx, input_schema, codec)+ deserializer.proto_to_physical_expr(proto, input_schema, ctx)}fn physical_expr_to_proto(@@ -4151,12 +4181,11 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {fn into_physical_plan(node: &Option<Box<protobuf::PhysicalPlanNode>>,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {if let Some(field) = node {- proto_converter.proto_to_execution_plan(ctx, codec, field)+ proto_converter.proto_to_execution_plan(field, ctx)} else {Err(proto_error("Missing required field in protobuf"))}diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex f0eb6d27aa..cb69b913b5 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -32,6 +32,7 @@ use datafusion_datasource_json::file_format::JsonSink;use datafusion_datasource_parquet::file_format::ParquetSink;use datafusion_expr::WindowFrame;use datafusion_physical_expr::ScalarFunctionExpr;+use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;@@ -526,6 +527,17 @@ pub fn serialize_physical_expr_with_converter(},)),})+ } else if let Some(expr) = expr.downcast_ref::<ScalarSubqueryExpr>() {+ Ok(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarSubquery(+ protobuf::PhysicalScalarSubqueryExprNode {+ data_type: Some(expr.data_type().try_into()?),+ nullable: expr.nullable(),+ index: expr.index().as_usize() as u32,+ },+ )),+ })} else {let mut buf: Vec<u8> = vec![];match codec.try_encode_expr(value, &mut buf) {diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex 6de9dd4caa..c0570881e3 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -134,7 +134,8 @@ fn roundtrip_expr_test_with_codec() {let proto: protobuf::LogicalExprNode = serialize_expr(&initial_struct, codec).unwrap_or_else(|e| panic!("Error serializing expression: {e:?}"));- let round_trip: Expr = from_proto::parse_expr(&proto, &ctx, codec).unwrap();+ let round_trip: Expr =+ from_proto::parse_expr(&proto, ctx.task_ctx().as_ref(), codec).unwrap();assert_eq!(format!("{:?}", &initial_struct), format!("{round_trip:?}"));@@ -2571,7 +2572,8 @@ fn roundtrip_scalar_udf_extension_codec() {let ctx = SessionContext::new();let proto = serialize_expr(&test_expr, &UDFExtensionCodec).expect("serialize expr");let round_trip =- from_proto::parse_expr(&proto, &ctx, &UDFExtensionCodec).expect("parse expr");+ from_proto::parse_expr(&proto, ctx.task_ctx().as_ref(), &UDFExtensionCodec)+ .expect("parse expr");assert_eq!(format!("{:?}", &test_expr), format!("{round_trip:?}"));roundtrip_json_test(&proto);@@ -2584,7 +2586,8 @@ fn roundtrip_aggregate_udf_extension_codec() {let ctx = SessionContext::new();let proto = serialize_expr(&test_expr, &UDFExtensionCodec).expect("serialize expr");let round_trip =- from_proto::parse_expr(&proto, &ctx, &UDFExtensionCodec).expect("parse expr");+ from_proto::parse_expr(&proto, ctx.task_ctx().as_ref(), &UDFExtensionCodec)+ .expect("parse expr");assert_eq!(format!("{:?}", &test_expr), format!("{round_trip:?}"));roundtrip_json_test(&proto);diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 3f1c306603..e7d38b57a1 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -74,6 +74,9 @@ use datafusion::physical_plan::metrics::MetricType;use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion::physical_plan::repartition::RepartitionExec;+use datafusion::physical_plan::scalar_subquery::{+ ScalarSubqueryExec, ScalarSubqueryLink,+};use datafusion::physical_plan::sorts::sort::SortExec;use datafusion::physical_plan::union::{InterleaveExec, UnionExec};use datafusion::physical_plan::unnest::{ListUnnest, UnnestExec};@@ -102,6 +105,7 @@ use datafusion_expr::{Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue,ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,WindowFrame, WindowFrameBound, WindowUDF,+ execution_props::{ScalarSubqueryResults, SubqueryIndex},};use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;use datafusion_functions_aggregate::array_agg::array_agg_udaf;@@ -109,15 +113,15 @@ use datafusion_functions_aggregate::average::avg_udaf;use datafusion_functions_aggregate::min_max::max_udaf;use datafusion_functions_aggregate::nth_value::nth_value_udaf;use datafusion_functions_aggregate::string_agg::string_agg_udaf;+use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_proto::bytes::{physical_plan_from_bytes_with_proto_converter,physical_plan_to_bytes_with_proto_converter,};-use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;use datafusion_proto::physical_plan::{AsExecutionPlan, DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,- DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,PhysicalProtoConverterExtension,};use datafusion_proto::protobuf;@@ -2556,9 +2560,8 @@ fn custom_proto_converter_intercepts() -> Result<()> {impl PhysicalProtoConverterExtension for CustomConverterInterceptor {fn proto_to_execution_plan(&self,- ctx: &TaskContext,- codec: &dyn PhysicalExtensionCodec,proto: &protobuf::PhysicalPlanNode,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {{let mut counter = self@@ -2567,7 +2570,7 @@ fn custom_proto_converter_intercepts() -> Result<()> {.map_err(|err| exec_datafusion_err!("{err}"))?;*counter += 1;}- proto.try_into_physical_plan_with_converter(ctx, codec, self)+ self.default_proto_to_execution_plan(proto, ctx)}fn execution_plan_to_proto(@@ -2595,9 +2598,8 @@ fn custom_proto_converter_intercepts() -> Result<()> {fn proto_to_physical_expr(&self,proto: &PhysicalExprNode,- ctx: &TaskContext,input_schema: &Schema,- codec: &dyn PhysicalExtensionCodec,+ ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn PhysicalExpr>>whereSelf: Sized,@@ -2609,7 +2611,7 @@ fn custom_proto_converter_intercepts() -> Result<()> {.map_err(|err| exec_datafusion_err!("{err}"))?;*counter += 1;}- parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)+ self.default_proto_to_physical_expr(proto, input_schema, ctx)}fn physical_expr_to_proto(@@ -2837,14 +2839,11 @@ fn test_backward_compatibility_no_expr_id() -> Result<()> {let ctx = SessionContext::new();let codec = DefaultPhysicalExtensionCodec {};let proto_converter = DefaultPhysicalProtoConverter {};+ let task_ctx = ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);// Should deserialize without error- let result = proto_converter.proto_to_physical_expr(- &proto,- ctx.task_ctx().as_ref(),- &schema,- &codec,- )?;+ let result = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;// Verify the result is correctlet col = result.downcast_ref::<Column>().expect("Expected Column");@@ -2964,17 +2963,14 @@ fn test_deduplication_within_expr_deserialization() -> Result<()> {let ctx = SessionContext::new();let codec = DefaultPhysicalExtensionCodec {};let proto_converter = DeduplicatingProtoConverter {};+ let task_ctx = ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);// Serialize the expressionlet proto = proto_converter.physical_expr_to_proto(&binary_expr, &codec)?;// First expression deserialization- let expr1 = proto_converter.proto_to_physical_expr(- &proto,- ctx.task_ctx().as_ref(),- &schema,- &codec,- )?;+ let expr1 = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;// Check that deduplication worked within the deserializationlet binary1 = expr1@@ -2986,12 +2982,7 @@ fn test_deduplication_within_expr_deserialization() -> Result<()> {);// Second expression deserialization- let expr2 = proto_converter.proto_to_physical_expr(- &proto,- ctx.task_ctx().as_ref(),- &schema,- &codec,- )?;+ let expr2 = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;// Check that the second expression was also deserialized correctlylet binary2 = expr2@@ -3167,6 +3158,243 @@ fn roundtrip_lead_with_default_value() -> Result<()> {)?))}+/// Verify that ScalarSubqueryExpr nodes in the input plan are connected to the+/// same shared results container as ScalarSubqueryExec after a proto round-trip.+#[test]+fn roundtrip_scalar_subquery_exec() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let results = ScalarSubqueryResults::new(1);++ // Build the input plan: a filter whose predicate references the+ // scalar subquery result via ScalarSubqueryExpr.+ let sq_expr = Arc::new(ScalarSubqueryExpr::new(+ DataType::Int64,+ true,+ SubqueryIndex::new(0),+ results.clone(),+ ));+ let predicate = binary(col("a", &schema)?, Operator::Eq, sq_expr, &schema)?;+ let filter =+ FilterExec::try_new(predicate, Arc::new(EmptyExec::new(schema.clone())))?;++ // Build a trivial subquery plan.+ let subquery_plan =+ Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new(+ "x",+ DataType::Int64,+ true,+ )]))));++ let exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(+ Arc::new(filter),+ vec![ScalarSubqueryLink {+ plan: subquery_plan,+ index: SubqueryIndex::new(0),+ }],+ results,+ ));++ // Perform the round-trip using DeduplicatingProtoConverter, which+ // creates a DeduplicatingDeserializer that threads scalar subquery+ // results through expression deserialization.+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&exec),+ &codec,+ &converter,+ )?;+ let ctx = SessionContext::new();+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ // Verify the deserialized ScalarSubqueryExec's results container is+ // shared with the ScalarSubqueryExpr in the input plan.+ let sq_exec = deserialized+ .downcast_ref::<ScalarSubqueryExec>()+ .expect("expected ScalarSubqueryExec");+ let exec_results = sq_exec.results();++ // Walk the input plan to find the ScalarSubqueryExpr and verify it+ // points to the same results container.+ let filter_exec = sq_exec+ .input()+ .downcast_ref::<FilterExec>()+ .expect("expected FilterExec");+ let binary_expr = filter_exec+ .predicate()+ .downcast_ref::<BinaryExpr>()+ .expect("expected BinaryExpr");+ let deserialized_sq_expr = binary_expr+ .right()+ .downcast_ref::<ScalarSubqueryExpr>()+ .expect("expected ScalarSubqueryExpr");++ assert!(+ ScalarSubqueryResults::ptr_eq(exec_results, deserialized_sq_expr.results()),+ "ScalarSubqueryExpr should share the same results container as ScalarSubqueryExec"+ );+ Ok(())+}++/// Verify that nested ScalarSubqueryExec nodes deserialize with distinct+/// scoped results containers, and that each ScalarSubqueryExpr is wired to the+/// container for its own surrounding ScalarSubqueryExec.+#[test]+fn roundtrip_nested_scalar_subquery_exec_scopes_results() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let subquery_schema =+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));++ let inner_results = ScalarSubqueryResults::new(1);+ let inner_sq_expr = Arc::new(ScalarSubqueryExpr::new(+ DataType::Int64,+ true,+ SubqueryIndex::new(0),+ inner_results.clone(),+ ));+ let inner_predicate =+ binary(col("a", &schema)?, Operator::Eq, inner_sq_expr, &schema)?;+ let inner_filter = Arc::new(FilterExec::try_new(+ inner_predicate,+ Arc::new(EmptyExec::new(schema.clone())),+ )?);+ let inner_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(+ inner_filter,+ vec![ScalarSubqueryLink {+ plan: Arc::new(EmptyExec::new(subquery_schema.clone())),+ index: SubqueryIndex::new(0),+ }],+ inner_results,+ ));++ let outer_results = ScalarSubqueryResults::new(1);+ let outer_sq_expr = Arc::new(ScalarSubqueryExpr::new(+ DataType::Int64,+ true,+ SubqueryIndex::new(0),+ outer_results.clone(),+ ));+ let outer_predicate =+ binary(col("a", &schema)?, Operator::Eq, outer_sq_expr, &schema)?;+ let outer_filter = Arc::new(FilterExec::try_new(outer_predicate, inner_exec)?);+ let outer_exec: Arc<dyn ExecutionPlan> = Arc::new(ScalarSubqueryExec::new(+ outer_filter,+ vec![ScalarSubqueryLink {+ plan: Arc::new(EmptyExec::new(subquery_schema)),+ index: SubqueryIndex::new(0),+ }],+ outer_results,+ ));++ let bytes = datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&outer_exec))?;+ let ctx = SessionContext::new();+ let deserialized = datafusion_proto::bytes::physical_plan_from_bytes(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ )?;++ let outer_exec = deserialized+ .downcast_ref::<ScalarSubqueryExec>()+ .expect("expected outer ScalarSubqueryExec");+ let outer_results = outer_exec.results();+ let outer_filter = outer_exec+ .input()+ .downcast_ref::<FilterExec>()+ .expect("expected outer FilterExec");+ let outer_binary = outer_filter+ .predicate()+ .downcast_ref::<BinaryExpr>()+ .expect("expected outer BinaryExpr");+ let outer_sq_expr = outer_binary+ .right()+ .downcast_ref::<ScalarSubqueryExpr>()+ .expect("expected outer ScalarSubqueryExpr");++ let inner_exec = outer_filter+ .input()+ .downcast_ref::<ScalarSubqueryExec>()+ .expect("expected inner ScalarSubqueryExec");+ let inner_results = inner_exec.results();+ let inner_filter = inner_exec+ .input()+ .downcast_ref::<FilterExec>()+ .expect("expected inner FilterExec");+ let inner_binary = inner_filter+ .predicate()+ .downcast_ref::<BinaryExpr>()+ .expect("expected inner BinaryExpr");+ let inner_sq_expr = inner_binary+ .right()+ .downcast_ref::<ScalarSubqueryExpr>()+ .expect("expected inner ScalarSubqueryExpr");++ assert!(+ ScalarSubqueryResults::ptr_eq(outer_results, outer_sq_expr.results()),+ "outer ScalarSubqueryExpr should use outer ScalarSubqueryExec results"+ );+ assert!(+ ScalarSubqueryResults::ptr_eq(inner_results, inner_sq_expr.results()),+ "inner ScalarSubqueryExpr should use inner ScalarSubqueryExec results"+ );+ assert!(+ !ScalarSubqueryResults::ptr_eq(outer_results, inner_results),+ "nested ScalarSubqueryExec nodes should not share results containers"+ );+ assert!(+ !ScalarSubqueryResults::ptr_eq(outer_results, inner_sq_expr.results()),+ "inner ScalarSubqueryExpr must not read from outer results"+ );+ assert!(+ !ScalarSubqueryResults::ptr_eq(inner_results, outer_sq_expr.results()),+ "outer ScalarSubqueryExpr must not read from inner results"+ );++ Ok(())+}++/// Verify that the default physical plan bytes round-trip preserves executable+/// scalar subquery plans.+#[tokio::test]+async fn roundtrip_scalar_subquery_exec_with_default_converter_executes() -> Result<()> {+ let ctx = SessionContext::new();+ let sql = "SELECT x + (SELECT max(y) FROM (VALUES (10), (20)) AS u(y)) AS s \+ FROM (VALUES (2), (1)) AS t(x) \+ ORDER BY s";++ let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;+ assert!(+ format!("{initial_plan:?}").contains("ScalarSubqueryExec"),+ "expected ScalarSubqueryExec in plan:\n{initial_plan:?}"+ );++ let bytes =+ datafusion_proto::bytes::physical_plan_to_bytes(Arc::clone(&initial_plan))?;+ let roundtripped = datafusion_proto::bytes::physical_plan_from_bytes(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ )?;+ assert!(+ format!("{roundtripped:?}").contains("ScalarSubqueryExec"),+ "expected ScalarSubqueryExec after roundtrip:\n{roundtripped:?}"+ );++ let batches = datafusion::physical_plan::common::collect(+ roundtripped.execute(0, ctx.task_ctx())?,+ )+ .await?;+ datafusion::assert_batches_eq!(+ &["+----+", "| s |", "+----+", "| 21 |", "| 22 |", "+----+",],+ &batches+ );++ Ok(())+}+/// Test that a chain of the same operator (a AND b AND c) is linearized/// and roundtrips correctly.#[test]diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rsindex bb955a426c..850fd42ce1 100644--- a/datafusion/proto/tests/cases/serialize.rs+++ b/datafusion/proto/tests/cases/serialize.rs@@ -77,7 +77,8 @@ fn udf_roundtrip_with_registry() {.call(vec![lit("")]);let bytes = expr.to_bytes().unwrap();- let deserialized_expr = Expr::from_bytes_with_registry(&bytes, &ctx).unwrap();+ let deserialized_expr =+ Expr::from_bytes_with_ctx(&bytes, ctx.task_ctx().as_ref()).unwrap();assert_eq!(expr, deserialized_expr);}@@ -281,7 +282,8 @@ fn test_expression_serialization_roundtrip() {let extension_codec = DefaultLogicalExtensionCodec {};let proto = serialize_expr(&expr, &extension_codec).unwrap();- let deserialize = parse_expr(&proto, &ctx, &extension_codec).unwrap();+ let deserialize =+ parse_expr(&proto, ctx.task_ctx().as_ref(), &extension_codec).unwrap();let serialize_name = extract_function_name(&expr);let deserialize_name = extract_function_name(&deserialize);diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex f584e3c342..3fea8df260 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -48,11 +48,9 @@ NULL bar3 bazquery I rowsort-SELECT (- SELECT id FROM table_with_metadata- ) UNION (- SELECT id FROM table_with_metadata- );+SELECT id FROM table_with_metadata+UNION+SELECT id FROM table_with_metadata;----13diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 368d252e25..e70f24303e 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -688,8 +688,10 @@ query TTexplain SELECT t1_id, (SELECT t2_id FROM t2 limit 0) FROM t1----logical_plan-01)Projection: t1.t1_id, Int32(NULL) AS t2_id-02)--TableScan: t1 projection=[t1_id]+01)Projection: t1.t1_id, (<subquery>)+02)--Subquery:+03)----EmptyRelation: rows=0+04)--TableScan: t1 projection=[t1_id]query II rowsortSELECT t1_id, (SELECT t2_id FROM t2 limit 0) FROM t1@@ -720,26 +722,79 @@ query TTexplain select (select count(*) from t1) as b----logical_plan-01)Projection: __scalar_sq_1.count(*) AS b-02)--SubqueryAlias: __scalar_sq_1+01)Projection: (<subquery>) AS b+02)--Subquery:03)----Projection: count(Int64(1)) AS count(*)04)------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]05)--------TableScan: t1 projection=[]+06)--EmptyRelation: rows=1#simple_uncorrelated_scalar_subquery2query TTexplain select (select count(*) from t1) as b, (select count(1) from t2)----logical_plan-01)Projection: __scalar_sq_1.count(*) AS b, __scalar_sq_2.count(Int64(1)) AS count(Int64(1))-02)--Left Join:-03)----SubqueryAlias: __scalar_sq_1-04)------Projection: count(Int64(1)) AS count(*)-05)--------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]-06)----------TableScan: t1 projection=[]-07)----SubqueryAlias: __scalar_sq_2-08)------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]-09)--------TableScan: t2 projection=[]+01)Projection: (<subquery>) AS b, (<subquery>)+02)--Subquery:+03)----Projection: count(Int64(1)) AS count(*)+04)------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]+05)--------TableScan: t1 projection=[]+06)--Subquery:+07)----Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]+08)------TableScan: t2 projection=[]+09)--EmptyRelation: rows=1++# Verify projection pushdown works inside uncorrelated scalar subqueries.+# Each test targets a different early-return path in OptimizeProjections+# to ensure subquery plans are optimized regardless of where the subquery+# expression appears.++# Subquery in a Filter predicate: the TableScan inside the subquery should+# only read t2_id, not all columns.+query TT+explain select t1_id from t1 where t1_id > (select max(t2_id) from t2)+----+logical_plan+01)Filter: t1.t1_id > (<subquery>)+02)--Subquery:+03)----Aggregate: groupBy=[[]], aggr=[[max(t2.t2_id)]]+04)------TableScan: t2 projection=[t2_id]+05)--TableScan: t1 projection=[t1_id]++# Subquery in a Projection expression+query TT+explain select t1_id, (select max(t2_id) from t2) as max_t2 from t1+----+logical_plan+01)Projection: t1.t1_id, (<subquery>) AS max_t2+02)--Subquery:+03)----Aggregate: groupBy=[[]], aggr=[[max(t2.t2_id)]]+04)------TableScan: t2 projection=[t2_id]+05)--TableScan: t1 projection=[t1_id]++# Subquery in an Aggregate expression+query TT+explain select sum(t1_int + (select min(t2_int) from t2)) as s from t1+----+logical_plan+01)Projection: sum(t1.t1_int + min(t2.t2_int)) AS s+02)--Aggregate: groupBy=[[]], aggr=[[sum(CAST(t1.t1_int + (<subquery>) AS Int64))]]+03)----Subquery:+04)------Aggregate: groupBy=[[]], aggr=[[min(t2.t2_int)]]+05)--------TableScan: t2 projection=[t2_int]+06)----TableScan: t1 projection=[t1_int]++# Subquery in a Window expression+query TT+explain select t1_id, sum(t1_int + (select min(t2_int) from t2)) over () as win from t1+----+logical_plan+01)Projection: t1.t1_id, sum(t1.t1_int + min(t2.t2_int)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS win+02)--WindowAggr: windowExpr=[[sum(CAST(t1.t1_int + (<subquery>) AS Int64)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]+03)----Subquery:+04)------Aggregate: groupBy=[[]], aggr=[[min(t2.t2_int)]]+05)--------TableScan: t2 projection=[t2_int]+06)----TableScan: t1 projection=[t1_id, t1_int]statement okset datafusion.explain.logical_plan_only = false;@@ -748,22 +803,23 @@ query TTexplain select (select count(*) from t1) as b, (select count(1) from t2)----logical_plan-01)Projection: __scalar_sq_1.count(*) AS b, __scalar_sq_2.count(Int64(1)) AS count(Int64(1))-02)--Left Join:-03)----SubqueryAlias: __scalar_sq_1-04)------Projection: count(Int64(1)) AS count(*)-05)--------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]-06)----------TableScan: t1 projection=[]-07)----SubqueryAlias: __scalar_sq_2-08)------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]-09)--------TableScan: t2 projection=[]+01)Projection: (<subquery>) AS b, (<subquery>)+02)--Subquery:+03)----Projection: count(Int64(1)) AS count(*)+04)------Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]+05)--------TableScan: t1 projection=[]+06)--Subquery:+07)----Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]]+08)------TableScan: t2 projection=[]+09)--EmptyRelation: rows=1physical_plan-01)ProjectionExec: expr=[count(*)@0 as b, count(Int64(1))@1 as count(Int64(1))]-02)--NestedLoopJoinExec: join_type=Left-03)----ProjectionExec: expr=[4 as count(*)]-04)------PlaceholderRowExec-05)----ProjectionExec: expr=[4 as count(Int64(1))]-06)------PlaceholderRowExec+01)ScalarSubqueryExec: subqueries=2+02)--ProjectionExec: expr=[scalar_subquery(<pending>) as b, scalar_subquery(<pending>) as count(Int64(1))]+03)----PlaceholderRowExec+04)--ProjectionExec: expr=[4 as count(*)]+05)----PlaceholderRowExec+06)--ProjectionExec: expr=[4 as count(Int64(1))]+07)----PlaceholderRowExecstatement okset datafusion.explain.logical_plan_only = true;@@ -1669,6 +1725,316 @@ drop table employees;statement count 0drop table project_assignments;+#############+## Uncorrelated scalar subquery row-count semantics+## A scalar subquery must return at most one row; returning more is an error.+#############++statement ok+CREATE TABLE sq_values(v INT) AS VALUES (1), (2), (3);++statement ok+CREATE TABLE sq_main(x INT) AS VALUES (10), (20);++statement ok+CREATE TABLE sq_empty(v INT) AS VALUES (1) LIMIT 0;++# Scalar subquery returning multiple rows in SELECT position → error+query error DataFusion error: Execution error: Scalar subquery returned more than one row+SELECT (SELECT v FROM sq_values);++# Scalar subquery returning multiple rows in WHERE position → error+query error DataFusion error: Execution error: Scalar subquery returned more than one row+SELECT x FROM sq_main WHERE x > (SELECT v FROM sq_values);++# Scalar subquery returning multiple rows as a function argument → error+query error DataFusion error: Execution error: Scalar subquery returned more than one row+SELECT x + (SELECT v FROM sq_values) FROM sq_main;++# Scalar subquery returning exactly one row → success+query I+SELECT (SELECT v FROM sq_values LIMIT 1);+----+1++# Scalar subquery returning exactly one row in WHERE → success+query I rowsort+SELECT x FROM sq_main WHERE x > (SELECT v FROM sq_values LIMIT 1);+----+10+20++# Scalar subquery returning zero rows → NULL+query I+SELECT (SELECT v FROM sq_empty);+----+NULL++# Scalar subquery returning zero rows in arithmetic → NULL propagation+query I+SELECT x + (SELECT v FROM sq_empty) FROM sq_main;+----+NULL+NULL++# Scalar subquery returning zero rows in WHERE comparison → no matching rows+query I+SELECT x FROM sq_main WHERE x > (SELECT v FROM sq_empty);+----++# Aggregated subquery always returns one row, even on empty input → success+query I+SELECT (SELECT count(*) FROM sq_empty);+----+0++# Aggregated subquery on multi-row table → success (aggregation reduces to 1 row)+query I+SELECT (SELECT max(v) FROM sq_values);+----+3++# Multiple scalar subqueries, one returns multiple rows → error+query error DataFusion error: Execution error: Scalar subquery returned more than one row+SELECT (SELECT count(*) FROM sq_empty), (SELECT v FROM sq_values);++#############+## Uncorrelated scalar subqueries in various expression contexts+#############++# HAVING clause with uncorrelated scalar subquery+query II rowsort+SELECT x, count(*) AS cnt FROM sq_main GROUP BY x+HAVING count(*) > (SELECT min(v) FROM sq_values);+----++# CASE WHEN with uncorrelated scalar subquery as condition+query T rowsort+SELECT CASE WHEN x > (SELECT min(v) FROM sq_values)+ THEN 'big' ELSE 'small' END AS label+FROM sq_main;+----+big+big++# ORDER BY with uncorrelated scalar subquery+query I+SELECT x FROM sq_main ORDER BY x + (SELECT max(v) FROM sq_values);+----+10+20++# Aggregate function argument containing uncorrelated scalar subquery+query I+SELECT sum(x + (SELECT max(v) FROM sq_values)) AS s FROM sq_main;+----+36++# JOIN ON condition with uncorrelated scalar subquery+query II rowsort+SELECT l.x, r.x AS rx+FROM sq_main AS l JOIN sq_main AS r+ON l.x = r.x + (SELECT min(v) FROM sq_values);+----++# Nested uncorrelated-in-uncorrelated scalar subquery.+query I+SELECT (SELECT max(v) + (SELECT min(v) FROM sq_values) FROM sq_values);+----+4++# Verify nested subqueries are not hoisted: the root ScalarSubqueryExec+# should manage only the outer subquery (subqueries=1), not both.+query TT+EXPLAIN SELECT (SELECT max(v) + (SELECT min(v) FROM sq_values) FROM sq_values);+----+logical_plan+01)Projection: (<subquery>)+02)--Subquery:+03)----Projection: max(sq_values.v) + (<subquery>)+04)------Subquery:+05)--------Aggregate: groupBy=[[]], aggr=[[min(sq_values.v)]]+06)----------TableScan: sq_values projection=[v]+07)------Aggregate: groupBy=[[]], aggr=[[max(sq_values.v)]]+08)--------TableScan: sq_values projection=[v]+09)--EmptyRelation: rows=1+physical_plan+01)ScalarSubqueryExec: subqueries=1+02)--ProjectionExec: expr=[scalar_subquery(<pending>) as max(sq_values.v) + min(sq_values.v)]+03)----PlaceholderRowExec+04)--ScalarSubqueryExec: subqueries=1+05)----ProjectionExec: expr=[max(sq_values.v)@0 + scalar_subquery(<pending>) as max(sq_values.v) + min(sq_values.v)]+06)------AggregateExec: mode=Single, gby=[], aggr=[max(sq_values.v)]+07)--------DataSourceExec: partitions=1, partition_sizes=[1]+08)----AggregateExec: mode=Single, gby=[], aggr=[min(sq_values.v)]+09)------DataSourceExec: partitions=1, partition_sizes=[1]++# CTE as source inside uncorrelated scalar subquery+query I+SELECT (SELECT s FROM (WITH cte AS (SELECT max(v) AS s FROM sq_values) SELECT s FROM cte));+----+3++# Window function with uncorrelated scalar subquery+query II rowsort+SELECT x, sum(x + (SELECT max(v) FROM sq_values)) OVER () AS win_sum FROM sq_main;+----+10 36+20 36++# Duplicate uncorrelated scalar subqueries only appear in the query plan once+statement ok+set datafusion.explain.logical_plan_only = false;++query TT+explain SELECT (SELECT max(v) FROM sq_values) + (SELECT max(v) FROM sq_values) AS doubled;+----+logical_plan+01)Projection: __common_expr_1 + __common_expr_1 AS doubled+02)--Projection: (<subquery>) AS __common_expr_1+03)----Subquery:+04)------Aggregate: groupBy=[[]], aggr=[[max(sq_values.v)]]+05)--------TableScan: sq_values projection=[v]+06)----EmptyRelation: rows=1+physical_plan+01)ScalarSubqueryExec: subqueries=1+02)--ProjectionExec: expr=[__common_expr_1@0 + __common_expr_1@0 as doubled]+03)----ProjectionExec: expr=[scalar_subquery(<pending>) as __common_expr_1]+04)------PlaceholderRowExec+05)--AggregateExec: mode=Single, gby=[], aggr=[max(sq_values.v)]+06)----DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+RESET datafusion.explain.logical_plan_only;++#############+## Additional edge cases inspired by DuckDB scalar subquery bugs and tests.+## References:+## DuckDB issue #13639: volatile functions in uncorrelated subqueries+## DuckDB issue #13469 / PR #13514: multi-row error semantics+## DuckDB issue #4113: uncorrelated scalar subquery wrong results+## DuckDB test: test/sql/subquery/scalar/test_uncorrelated_scalar_subquery.test+## DuckDB test: test/sql/subquery/scalar/test_scalar_subquery.test+## DuckDB test: test/sql/subquery/scalar/test_scalar_subquery_cte.test+## DuckDB test: test/sql/order/test_limit.test+## DuckDB PR #8519: TopN optimization with scalar subquery in LIMIT/OFFSET+#############++# Volatile function in uncorrelated subquery: random() is evaluated once and+# the same value is used for every outer row. Both Postgres and DuckDB (#13639)+# exhibit this behavior; it is correct per the SQL standard.+# We cannot assert the exact value, but we can verify all rows are identical.+query B+SELECT count(DISTINCT r) = 1 FROM (+ SELECT (SELECT random()) AS r+ FROM (VALUES (1), (2), (3)) AS t(x)+);+----+true++# Subquery as GROUP BY key.+# Ref: DuckDB test_uncorrelated_scalar_subquery.test, test #6+query II+SELECT (SELECT 42) AS k, max(x) FROM (VALUES (1), (2), (3)) AS t(x) GROUP BY k;+----+42 3++# Subquery inside an aggregate function argument.+# Ref: DuckDB test_uncorrelated_scalar_subquery.test, test #7+query II+SELECT x, max((SELECT 42)) FROM (VALUES (1), (2), (3)) AS t(x) GROUP BY x ORDER BY x;+----+1 42+2 42+3 42++# Doubly-nested constant subquery.+# Ref: DuckDB test_scalar_subquery.test+query I+SELECT (SELECT (SELECT 42));+----+42++# Triple-nested constant subquery.+query I+SELECT (SELECT (SELECT (SELECT 99)));+----+99++# Star expansion: single column is OK.+# Ref: DuckDB test_uncorrelated_scalar_subquery.test, tests #16-17+query I+SELECT (SELECT * FROM (VALUES (1)) AS t(x));+----+1++# Star expansion: two columns must error.+query error Too many columns+SELECT (SELECT * FROM (VALUES (1, 2)) AS t(x, y));++# Subquery in BETWEEN bounds.+query I+SELECT x FROM (VALUES (1), (2), (3), (4), (5)) AS t(x)+WHERE x BETWEEN (SELECT 2) AND (SELECT 4)+ORDER BY x;+----+2+3+4++# DISTINCT subquery returning exactly one distinct value (multi-row input).+query I+SELECT (SELECT DISTINCT 42 FROM (VALUES (1), (2), (3)) AS t(x));+----+42++# DISTINCT subquery returning multiple distinct values must error.+query error DataFusion error: Execution error: Scalar subquery returned more than one row+SELECT (SELECT DISTINCT x FROM (VALUES (1), (2), (3)) AS t(x));++# NULL comparison semantics through subquery boundary.+# Ref: DuckDB test_scalar_subquery.test, NULL edge cases+query B+SELECT 1 = (SELECT CAST(NULL AS INT));+----+NULL++query B+SELECT CAST(NULL AS INT) = (SELECT 1);+----+NULL++# Nested CTE inside scalar subquery.+# Ref: DuckDB test_scalar_subquery_cte.test, nested CTE test+query I+SELECT (WITH cte1 AS (WITH cte2 AS (SELECT 42) SELECT * FROM cte2) SELECT * FROM cte1);+----+42++# Subquery in LIMIT: not yet supported, verify clear error message.+# Ref: DuckDB PR #8519, DuckDB test/sql/order/test_limit.test+query error This feature is not implemented: Unsupported LIMIT expression+SELECT * FROM (VALUES (1), (2), (3)) AS t(x) ORDER BY x LIMIT (SELECT 2);++# Subquery in OFFSET: not yet supported, verify clear error message.+query error This feature is not implemented: Unsupported OFFSET expression+SELECT * FROM (VALUES (1), (2), (3)) AS t(x) ORDER BY x OFFSET (SELECT 1);++# UNION ALL subquery with ORDER BY + LIMIT 1 to avoid multi-row error.+query I+SELECT (SELECT v FROM (SELECT 1 AS v UNION ALL SELECT 2) AS t ORDER BY v LIMIT 1);+----+1++statement count 0+DROP TABLE sq_values;++statement count 0+DROP TABLE sq_main;++statement count 0+DROP TABLE sq_empty;+# https://github.com/apache/datafusion/issues/21205statement okCREATE TABLE dup_filter_t1(id INTEGER) AS VALUES (1), (2), (3);@@ -1736,3 +2102,38 @@ DROP TABLE sq_name_t1;statement okDROP TABLE sq_name_t2;++# Test: scalar subquery in filter on a partition column of a partitioned table.+# This exercises the code path where filters are pushed down to the table+# provider for partition pruning. Scalar subqueries must not be pushed to the+# provider because the subquery result is not available at partition listing+# time.++query I+COPY (VALUES(1, 'a'), (2, 'b'), (3, 'c'))+TO 'test_files/scratch/subquery/partition_pruning/part=1/file1.parquet';+----+3++query I+COPY (VALUES(4, 'd'), (5, 'e'))+TO 'test_files/scratch/subquery/partition_pruning/part=2/file1.parquet';+----+2++statement ok+CREATE EXTERNAL TABLE subquery_partitioned+STORED AS PARQUET+LOCATION 'test_files/scratch/subquery/partition_pruning/';++query IT+SELECT column1, column2 FROM subquery_partitioned+WHERE part = (SELECT 1)+ORDER BY column1;+----+1 a+2 b+3 c++statement ok+DROP TABLE subquery_partitioned;diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.partindex a31579eb1e..0c5b6d76dc 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part@@ -49,61 +49,62 @@ limit 10;logical_plan01)Sort: value DESC NULLS FIRST, fetch=1002)--Projection: partsupp.ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty) AS value-03)----Inner Join: Filter: CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty) AS Decimal128(38, 15)) > __scalar_sq_1.sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)-04)------Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]-05)--------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost-06)----------Inner Join: supplier.s_nationkey = nation.n_nationkey-07)------------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey-08)--------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey-09)----------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], partial_filters=[Boolean(true)]-10)----------------TableScan: supplier projection=[s_suppkey, s_nationkey]-11)------------Projection: nation.n_nationkey-12)--------------Filter: nation.n_name = Utf8View("GERMANY")-13)----------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY")]-14)------SubqueryAlias: __scalar_sq_1-15)--------Projection: CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty) AS Float64) * Float64(0.0001) AS Decimal128(38, 15))-16)----------Aggregate: groupBy=[[]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]-17)------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost-18)--------------Inner Join: supplier.s_nationkey = nation.n_nationkey-19)----------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey-20)------------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey-21)--------------------TableScan: partsupp projection=[ps_suppkey, ps_availqty, ps_supplycost]-22)--------------------TableScan: supplier projection=[s_suppkey, s_nationkey]-23)----------------Projection: nation.n_nationkey-24)------------------Filter: nation.n_name = Utf8View("GERMANY")-25)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY")]+03)----Filter: CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty) AS Decimal128(38, 15)) > (<subquery>)+04)------Subquery:+05)--------Projection: CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty) AS Float64) * Float64(0.0001) AS Decimal128(38, 15))+06)----------Aggregate: groupBy=[[]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]+07)------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost+08)--------------Inner Join: supplier.s_nationkey = nation.n_nationkey+09)----------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey+10)------------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey+11)--------------------TableScan: partsupp projection=[ps_suppkey, ps_availqty, ps_supplycost]+12)--------------------TableScan: supplier projection=[s_suppkey, s_nationkey]+13)----------------Projection: nation.n_nationkey+14)------------------Filter: nation.n_name = Utf8View("GERMANY")+15)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY")]+16)------Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]+17)--------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost+18)----------Inner Join: supplier.s_nationkey = nation.n_nationkey+19)------------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey+20)--------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey+21)----------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost]+22)----------------TableScan: supplier projection=[s_suppkey, s_nationkey]+23)------------Projection: nation.n_nationkey+24)--------------Filter: nation.n_name = Utf8View("GERMANY")+25)----------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY")]physical_plan-01)SortExec: TopK(fetch=10), expr=[value@1 DESC], preserve_partitioning=[false]-02)--ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value]-03)----NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, projection=[ps_partkey@0, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@3]-04)------ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1]-05)--------CoalescePartitionsExec-06)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-07)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4-08)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-09)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]-10)------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=4-11)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]-12)----------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4-13)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false-14)----------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-15)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false-16)------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4-17)--------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]-18)----------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-19)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false-20)------ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)]-21)--------AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-22)----------CoalescePartitionsExec-23)------------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-24)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]-25)----------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4-26)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]-27)--------------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4-28)----------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false-29)--------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-30)----------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false-31)----------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4-32)------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]-33)--------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-34)----------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false+01)ScalarSubqueryExec: subqueries=1+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true+03)----SortPreservingMergeExec: [value@1 DESC], fetch=10+04)------SortExec: TopK(fetch=10), expr=[value@1 DESC], preserve_partitioning=[true]+05)--------ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value]+06)----------FilterExec: CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) > scalar_subquery(<pending>)+07)------------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+08)--------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4+09)----------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+10)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]+11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=4+12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]+13)------------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4+14)--------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false+15)------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+16)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false+17)--------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4+18)----------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]+19)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+20)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false+21)--ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)]+22)----AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+23)------CoalescePartitionsExec+24)--------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+25)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]+26)------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4+27)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]+28)----------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4+29)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false+30)----------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+31)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false+32)------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4+33)--------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]+34)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+35)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.partindex ae0c0a93a3..3e1aca318b 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.part@@ -52,43 +52,44 @@ order bylogical_plan01)Sort: supplier.s_suppkey ASC NULLS LAST02)--Projection: supplier.s_suppkey, supplier.s_name, supplier.s_address, supplier.s_phone, revenue0.total_revenue-03)----Inner Join: revenue0.total_revenue = __scalar_sq_1.max(revenue0.total_revenue)-04)------Projection: supplier.s_suppkey, supplier.s_name, supplier.s_address, supplier.s_phone, revenue0.total_revenue-05)--------Inner Join: supplier.s_suppkey = revenue0.supplier_no-06)----------TableScan: supplier projection=[s_suppkey, s_name, s_address, s_phone], partial_filters=[Boolean(true)]-07)----------SubqueryAlias: revenue0-08)------------Projection: lineitem.l_suppkey AS supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS total_revenue-09)--------------Aggregate: groupBy=[[lineitem.l_suppkey]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(Some(1),20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]-10)----------------Projection: lineitem.l_suppkey, lineitem.l_extendedprice, lineitem.l_discount-11)------------------Filter: lineitem.l_shipdate >= Date32("1996-01-01") AND lineitem.l_shipdate < Date32("1996-04-01")-12)--------------------TableScan: lineitem projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], partial_filters=[lineitem.l_shipdate >= Date32("1996-01-01"), lineitem.l_shipdate < Date32("1996-04-01")]-13)------SubqueryAlias: __scalar_sq_1-14)--------Aggregate: groupBy=[[]], aggr=[[max(revenue0.total_revenue)]]-15)----------SubqueryAlias: revenue0-16)------------Projection: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS total_revenue-17)--------------Aggregate: groupBy=[[lineitem.l_suppkey]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(Some(1),20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]-18)----------------Projection: lineitem.l_suppkey, lineitem.l_extendedprice, lineitem.l_discount-19)------------------Filter: lineitem.l_shipdate >= Date32("1996-01-01") AND lineitem.l_shipdate < Date32("1996-04-01")-20)--------------------TableScan: lineitem projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], partial_filters=[lineitem.l_shipdate >= Date32("1996-01-01"), lineitem.l_shipdate < Date32("1996-04-01")]+03)----Inner Join: supplier.s_suppkey = revenue0.supplier_no+04)------TableScan: supplier projection=[s_suppkey, s_name, s_address, s_phone]+05)------SubqueryAlias: revenue0+06)--------Projection: lineitem.l_suppkey AS supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS total_revenue+07)----------Filter: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) = (<subquery>)+08)------------Subquery:+09)--------------Aggregate: groupBy=[[]], aggr=[[max(revenue0.total_revenue)]]+10)----------------SubqueryAlias: revenue0+11)------------------Projection: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS total_revenue+12)--------------------Aggregate: groupBy=[[lineitem.l_suppkey]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(Some(1),20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]+13)----------------------Projection: lineitem.l_suppkey, lineitem.l_extendedprice, lineitem.l_discount+14)------------------------Filter: lineitem.l_shipdate >= Date32("1996-01-01") AND lineitem.l_shipdate < Date32("1996-04-01")+15)--------------------------TableScan: lineitem projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], partial_filters=[lineitem.l_shipdate >= Date32("1996-01-01"), lineitem.l_shipdate < Date32("1996-04-01")]+16)------------Aggregate: groupBy=[[lineitem.l_suppkey]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(Some(1),20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]+17)--------------Projection: lineitem.l_suppkey, lineitem.l_extendedprice, lineitem.l_discount+18)----------------Filter: lineitem.l_shipdate >= Date32("1996-01-01") AND lineitem.l_shipdate < Date32("1996-04-01")+19)------------------TableScan: lineitem projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], partial_filters=[lineitem.l_shipdate >= Date32("1996-01-01"), lineitem.l_shipdate < Date32("1996-04-01")]physical_plan-01)SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST]-02)--SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)], projection=[s_suppkey@1, s_name@2, s_address@3, s_phone@4, total_revenue@5]-04)------AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)]-05)--------CoalescePartitionsExec-06)----------AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)]-07)------------ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]-08)--------------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-09)----------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4-10)------------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-11)--------------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]-12)----------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=false-13)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5]-14)--------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-15)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=csv, has_header=false-16)--------ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]-17)----------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-18)------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4-19)--------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-20)----------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]-21)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=false+01)ScalarSubqueryExec: subqueries=1+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true+03)----SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST]+04)------SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true]+05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5]+06)----------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=csv, has_header=false+08)----------ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]+09)------------FilterExec: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 = scalar_subquery(<pending>)+10)--------------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+11)----------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4+12)------------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+13)--------------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]+14)----------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=false+15)--AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)]+16)----CoalescePartitionsExec+17)------AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)]+18)--------ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]+19)----------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+20)------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4+21)--------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+22)----------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]+23)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.partindex add578c3b0..3240cbfb69 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part@@ -61,40 +61,36 @@ logical_plan03)----Aggregate: groupBy=[[custsale.cntrycode]], aggr=[[count(Int64(1)), sum(custsale.c_acctbal)]]04)------SubqueryAlias: custsale05)--------Projection: substr(customer.c_phone, Int64(1), Int64(2)) AS cntrycode, customer.c_acctbal-06)----------Inner Join: Filter: CAST(customer.c_acctbal AS Decimal128(19, 6)) > __scalar_sq_2.avg(customer.c_acctbal)-07)------------Projection: customer.c_phone, customer.c_acctbal-08)--------------LeftAnti Join: customer.c_custkey = __correlated_sq_1.o_custkey-09)----------------Filter: substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")])-10)------------------TableScan: customer projection=[c_custkey, c_phone, c_acctbal], partial_filters=[substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")]), Boolean(true)]-11)----------------SubqueryAlias: __correlated_sq_1-12)------------------TableScan: orders projection=[o_custkey]-13)------------SubqueryAlias: __scalar_sq_2-14)--------------Aggregate: groupBy=[[]], aggr=[[avg(customer.c_acctbal)]]-15)----------------Projection: customer.c_acctbal-16)------------------Filter: customer.c_acctbal > Decimal128(Some(0),15,2) AND substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")])-17)--------------------TableScan: customer projection=[c_phone, c_acctbal], partial_filters=[customer.c_acctbal > Decimal128(Some(0),15,2), substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")])]+06)----------LeftAnti Join: customer.c_custkey = __correlated_sq_1.o_custkey+07)------------Filter: substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")]) AND CAST(customer.c_acctbal AS Decimal128(19, 6)) > (<subquery>)+08)--------------Subquery:+09)----------------Aggregate: groupBy=[[]], aggr=[[avg(customer.c_acctbal)]]+10)------------------Projection: customer.c_acctbal+11)--------------------Filter: customer.c_acctbal > Decimal128(Some(0),15,2) AND substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")])+12)----------------------TableScan: customer projection=[c_phone, c_acctbal], partial_filters=[customer.c_acctbal > Decimal128(Some(0),15,2), substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")])]+13)--------------TableScan: customer projection=[c_custkey, c_phone, c_acctbal], partial_filters=[substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8View("13"), Utf8View("31"), Utf8View("23"), Utf8View("29"), Utf8View("30"), Utf8View("18"), Utf8View("17")])]+14)------------SubqueryAlias: __correlated_sq_1+15)--------------TableScan: orders projection=[o_custkey]physical_plan-01)SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST]-02)--SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal]-04)------AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]-05)--------RepartitionExec: partitioning=Hash([cntrycode@0], 4), input_partitions=4-06)----------AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]-07)------------ProjectionExec: expr=[substr(c_phone@0, 1, 2) as cntrycode, c_acctbal@1 as c_acctbal]-08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-09)----------------NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[c_phone@0, c_acctbal@1, avg(customer.c_acctbal)@3]-10)------------------ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1]-11)--------------------CoalescePartitionsExec-12)----------------------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2]-13)------------------------RepartitionExec: partitioning=Hash([c_custkey@0], 4), input_partitions=4-14)--------------------------FilterExec: substr(c_phone@1, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17])-15)----------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-16)------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_custkey, c_phone, c_acctbal], file_type=csv, has_header=false-17)------------------------RepartitionExec: partitioning=Hash([o_custkey@0], 4), input_partitions=4-18)--------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:0..4223281], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:4223281..8446562], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:8446562..12669843], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:12669843..16893122]]}, projection=[o_custkey], file_type=csv, has_header=false-19)------------------AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)]-20)--------------------CoalescePartitionsExec-21)----------------------AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)]-22)------------------------FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1]-23)--------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-24)----------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_phone, c_acctbal], file_type=csv, has_header=false+01)ScalarSubqueryExec: subqueries=1+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true+03)----SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST]+04)------SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true]+05)--------ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal]+06)----------AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]+07)------------RepartitionExec: partitioning=Hash([cntrycode@0], 4), input_partitions=4+08)--------------AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]+09)----------------ProjectionExec: expr=[substr(c_phone@0, 1, 2) as cntrycode, c_acctbal@1 as c_acctbal]+10)------------------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2]+11)--------------------RepartitionExec: partitioning=Hash([c_custkey@0], 4), input_partitions=4+12)----------------------FilterExec: substr(c_phone@1, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]) AND CAST(c_acctbal@2 AS Decimal128(19, 6)) > scalar_subquery(<pending>)+13)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+14)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_custkey, c_phone, c_acctbal], file_type=csv, has_header=false+15)--------------------RepartitionExec: partitioning=Hash([o_custkey@0], 4), input_partitions=4+16)----------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:0..4223281], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:4223281..8446562], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:8446562..12669843], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:12669843..16893122]]}, projection=[o_custkey], file_type=csv, has_header=false+17)--AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)]+18)----CoalescePartitionsExec+19)------AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)]+20)--------FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1]+21)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+22)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_phone, c_acctbal], file_type=csv, has_header=falsediff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.mdindex cadd365e1f..854dee0724 100644--- a/docs/source/library-user-guide/upgrading/54.0.0.md+++ b/docs/source/library-user-guide/upgrading/54.0.0.md@@ -320,6 +320,86 @@ The difference is only observable for strings containing combining charactersclusters (e.g., ZWJ emoji sequences). For ASCII and most common Unicode text,behavior is unchanged.+### Scalar subquery execution changes++Uncorrelated scalar subqueries (e.g. `SELECT ... WHERE x > (SELECT max(v) FROM t)`)+are now executed by a dedicated physical operator rather than being rewritten to+a join. Correlated scalar subqueries are unchanged.++This produces two user-visible changes:++- **Subqueries that return multiple rows now fail at runtime.** An uncorrelated+ scalar subquery that returns more than one row fails with `Execution error: Scalar subquery returned more than one row`. This matches the SQL standard and+ the behavior of most other SQL implementations. The previous join-based+ rewrite could silently produce multi-row output. Add a `LIMIT 1` or an+ aggregate to the subquery to fix such queries.+- **Plan shape changes.** Uncorrelated `Expr::ScalarSubquery` nodes now survive+ into the final logical plan instead of being replaced by a join; the+ corresponding physical plan contains a new `ScalarSubqueryExec` node and a+ `ScalarSubqueryExpr` expression. Code that walks or transforms `LogicalPlan` /+ `ExecutionPlan` trees, as well as `EXPLAIN` output, may need updating.++### `datafusion-proto`: expression deserialization now takes a `TaskContext`++`Serializeable::from_bytes_with_registry` is renamed to `from_bytes_with_ctx`+and takes a `&TaskContext` instead of a `&dyn FunctionRegistry`. `parse_expr`,+`parse_exprs`, and `parse_sorts` take the same change. `Expr::from_bytes`+(without a registry argument) is unchanged.++```diff+-let expr = Expr::from_bytes_with_registry(&bytes, ®istry)?;++let expr = Expr::from_bytes_with_ctx(&bytes, ctx.task_ctx().as_ref())?;+```++```diff+-let expr = parse_expr(&proto, ®istry, &codec)?;++let expr = parse_expr(&proto, ctx.task_ctx().as_ref(), &codec)?;+```++### `datafusion-proto`: `PhysicalProtoConverterExtension` reshaped++`PhysicalProtoConverterExtension` and the `parse_physical_*_with_converter`+helpers now take a single `&PhysicalPlanDecodeContext<'_>` that bundles the+`TaskContext` and the `PhysicalExtensionCodec`. Implementations update like+this:++```diff+ impl PhysicalProtoConverterExtension for MyConverter {+ fn proto_to_execution_plan(+ &self,+- ctx: &TaskContext,+- codec: &dyn PhysicalExtensionCodec,+ proto: &protobuf::PhysicalPlanNode,++ ctx: &PhysicalPlanDecodeContext<'_>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+- proto.try_into_physical_plan_with_converter(ctx, codec, self)++ self.default_proto_to_execution_plan(proto, ctx)+ }++ fn proto_to_physical_expr(+ &self,+ proto: &PhysicalExprNode,+- ctx: &TaskContext,+ input_schema: &Schema,+- codec: &dyn PhysicalExtensionCodec,++ ctx: &PhysicalPlanDecodeContext<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+- parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)++ self.default_proto_to_physical_expr(proto, input_schema, ctx)+ }+ }+```++Pull out the `TaskContext` or codec inside these methods with+`ctx.task_ctx()` and `ctx.codec()`. Construct a fresh context at an API+boundary with `PhysicalPlanDecodeContext::new(task_ctx, codec)`.++### `ExecutionProps` has new fields++`ExecutionProps` gained new public fields. Code that constructs it via a+struct literal, or pattern-matches it without `..`, no longer compiles. Use+`ExecutionProps::new()` and include `..` in exhaustive patterns.+### Items in `datafusion_functions::strings` are no longer public`StringArrayBuilder`, `LargeStringArrayBuilder`, `StringViewArrayBuilder`,