Apache DataFusion SQL Query Engine
refactor(proto): destructure plan and proto structs in core plan serde hooks (#24167)
Adrian Garcia Badaraccoa4e071d17a4d
diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rsindex 5879b98c34..72df0f5345 100644--- a/datafusion/physical-plan/src/buffer.rs+++ b/datafusion/physical-plan/src/buffer.rs@@ -332,13 +332,24 @@ impl ExecutionPlan for BufferExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `BufferExec` is a compile error here until it is either serialized or+ // explicitly documented as not needing to be.+ let Self {+ input,+ // Derived from the input's properties at construction time.+ properties: _,+ capacity,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ } = self;+ let input = ctx.encode_child(input)?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Buffer(Box::new(protobuf::BufferExecNode {input: Some(Box::new(input)),- capacity: self.capacity() as u64,+ capacity: *capacity as u64,},)),),@@ -363,9 +374,11 @@ impl BufferExec {protobuf::physical_plan_node::PhysicalPlanType::Buffer,"BufferExec",);- let input =- ctx.decode_required_child(buffer.input.as_deref(), "BufferExec", "input")?;- Ok(Arc::new(BufferExec::new(input, buffer.capacity as usize)))+ // Destructure exhaustively so that a new field on `BufferExecNode` is a+ // compile error here rather than a silently dropped field.+ let protobuf::BufferExecNode { input, capacity } = &**buffer;+ let input = ctx.decode_required_child(input.as_deref(), "BufferExec", "input")?;+ Ok(Arc::new(BufferExec::new(input, *capacity as usize)))}}diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rsindex cb0f9b2ce4..87957ced7b 100644--- a/datafusion/physical-plan/src/coalesce_batches.rs+++ b/datafusion/physical-plan/src/coalesce_batches.rs@@ -323,14 +323,26 @@ impl ExecutionPlan for CoalesceBatchesExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `CoalesceBatchesExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ target_batch_size,+ fetch,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ } = self;+ let input = ctx.encode_child(input)?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::CoalesceBatches(Box::new(protobuf::CoalesceBatchesExecNode {input: Some(Box::new(input)),- target_batch_size: self.target_batch_size() as u32,- fetch: self.fetch().map(|n| n as u32),+ target_batch_size: *target_batch_size as u32,+ fetch: fetch.map(|n| n as u32),}),),),@@ -361,14 +373,19 @@ impl CoalesceBatchesExec {protobuf::physical_plan_node::PhysicalPlanType::CoalesceBatches,"CoalesceBatchesExec",);- let input = ctx.decode_required_child(- coalesce_batches.input.as_deref(),- "CoalesceBatchesExec",- "input",- )?;+ // Destructure exhaustively so that a new field on+ // `CoalesceBatchesExecNode` is a compile error here rather than a+ // silently dropped field.+ let protobuf::CoalesceBatchesExecNode {+ input,+ target_batch_size,+ fetch,+ } = &**coalesce_batches;+ let input =+ ctx.decode_required_child(input.as_deref(), "CoalesceBatchesExec", "input")?;Ok(Arc::new(- CoalesceBatchesExec::new(input, coalesce_batches.target_batch_size as usize)- .with_fetch(coalesce_batches.fetch.map(|f| f as usize)),+ CoalesceBatchesExec::new(input, *target_batch_size as usize)+ .with_fetch(fetch.map(|f| f as usize)),))}}diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex 6f58eb2f1e..7c8bca772e 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -384,13 +384,24 @@ impl ExecutionPlan for CoalescePartitionsExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `CoalescePartitionsExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ fetch,+ } = self;+ let input = ctx.encode_child(input)?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Merge(Box::new(protobuf::CoalescePartitionsExecNode {input: Some(Box::new(input)),- fetch: self.fetch().map(|f| f as u32),+ fetch: fetch.map(|f| f as u32),},)),),@@ -417,14 +428,17 @@ impl CoalescePartitionsExec {protobuf::physical_plan_node::PhysicalPlanType::Merge,"CoalescePartitionsExec",);+ // Destructure exhaustively so that a new field on+ // `CoalescePartitionsExecNode` is a compile error here rather than a+ // silently dropped field.+ let protobuf::CoalescePartitionsExecNode { input, fetch } = &**merge;let input = ctx.decode_required_child(- merge.input.as_deref(),+ input.as_deref(),"CoalescePartitionsExec","input",)?;Ok(Arc::new(- CoalescePartitionsExec::new(input)- .with_fetch(merge.fetch.map(|f| f as usize)),+ CoalescePartitionsExec::new(input).with_fetch(fetch.map(|f| f as usize)),))}}diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rsindex 9e27b26d6e..17166e287e 100644--- a/datafusion/physical-plan/src/coop.rs+++ b/datafusion/physical-plan/src/coop.rs@@ -402,7 +402,15 @@ impl ExecutionPlan for CooperativeExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `CooperativeExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ // Derived from the input's properties at construction time.+ properties: _,+ } = self;+ let input = ctx.encode_child(input)?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Cooperative(Box::new(@@ -432,11 +440,12 @@ impl CooperativeExec {protobuf::physical_plan_node::PhysicalPlanType::Cooperative,"CooperativeExec",);- let input = ctx.decode_required_child(- cooperative.input.as_deref(),- "CooperativeExec",- "input",- )?;+ // Destructure exhaustively so that a new field on+ // `CooperativeExecNode` is a compile error here rather than a silently+ // dropped field.+ let protobuf::CooperativeExecNode { input } = &**cooperative;+ let input =+ ctx.decode_required_child(input.as_deref(), "CooperativeExec", "input")?;Ok(Arc::new(CooperativeExec::new(input)))}}diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rsindex dd08ff36a9..a1d79890b0 100644--- a/datafusion/physical-plan/src/empty.rs+++ b/datafusion/physical-plan/src/empty.rs@@ -214,16 +214,22 @@ impl ExecutionPlan for EmptyExec {_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let schema = self.schema().as_ref().try_into()?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `EmptyExec` is a compile error here until it is either serialized or+ // explicitly documented as not needing to be.+ let Self {+ schema,+ partitions,+ // Derived from `schema` and `partitions`, recomputed on decode.+ cache: _,+ } = self;+ let schema = schema.as_ref().try_into()?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(protobuf::EmptyExecNode {schema: Some(schema),- partitions: self- .properties()- .output_partitioning()- .partition_count() as u32,+ partitions: *partitions as u32,},),),@@ -244,7 +250,10 @@ impl EmptyExec {protobuf::physical_plan_node::PhysicalPlanType::Empty,"EmptyExec",);- let schema = empty.schema.as_ref().ok_or_else(|| {+ // Destructure exhaustively so that a new field on `EmptyExecNode` is a+ // compile error here rather than a silently dropped field.+ let protobuf::EmptyExecNode { schema, partitions } = empty;+ let schema = schema.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("EmptyExec is missing required field 'schema'")@@ -252,7 +261,7 @@ impl EmptyExec {let schema = Arc::new(arrow::datatypes::Schema::try_from(schema)?);// A zero (absent) partition count comes from a plan encoded before the// field existed, which always meant a single partition.- let partitions = empty.partitions.max(1) as usize;+ let partitions = (*partitions).max(1) as usize;Ok(Arc::new(EmptyExec::new(schema).with_partitions(partitions)))}}diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rsindex 3b31ee748b..7ff44c36d8 100644--- a/datafusion/physical-plan/src/explain.rs+++ b/datafusion/physical-plan/src/explain.rs@@ -215,17 +215,26 @@ impl ExecutionPlan for ExplainExec {) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `ExplainExec` is a compile error here until it is either serialized+ // or explicitly documented as not needing to be.+ let Self {+ schema,+ stringified_plans,+ verbose,+ // Derived from `schema`, recomputed on decode.+ cache: _,+ } = self;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Explain(protobuf::ExplainExecNode {- schema: Some(self.schema().as_ref().try_into()?),- stringified_plans: self- .stringified_plans()+ schema: Some(schema.as_ref().try_into()?),+ stringified_plans: stringified_plans.iter().map(stringified_plan_to_proto).collect(),- verbose: self.verbose(),+ verbose: *verbose,},),),@@ -247,19 +256,25 @@ impl ExplainExec {protobuf::physical_plan_node::PhysicalPlanType::Explain,"ExplainExec",);- let schema = explain.schema.as_ref().ok_or_else(|| {+ // Destructure exhaustively so that a new field on `ExplainExecNode` is+ // a compile error here rather than a silently dropped field.+ let protobuf::ExplainExecNode {+ schema,+ stringified_plans,+ verbose,+ } = explain;+ let schema = schema.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("ExplainExec is missing required field 'schema'")})?;Ok(Arc::new(ExplainExec::new(Arc::new(arrow::datatypes::Schema::try_from(schema)?),- explain- .stringified_plans+ stringified_plans.iter().map(stringified_plan_from_proto).collect(),- explain.verbose,+ *verbose,)))}}diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex a1be69849b..186c49562b 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -850,15 +850,30 @@ impl ExecutionPlan for FilterExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;- let expr = ctx.encode_expr(self.predicate())?;- // Preserve the exact wire format: `None` (full projection) is serialized- // as the identity projection `[0, 1, ..., num_fields - 1]` so that it is- // distinguishable from an explicit projection on decode.- let projection = if let Some(v) = self.projection() {+ // Destructure exhaustively (no `..`) so that adding a field to+ // `FilterExec` is a compile error here until it is either serialized or+ // explicitly documented as not needing to be.+ let Self {+ predicate,+ input,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ default_selectivity,+ // Derived plan properties, recomputed on decode.+ cache: _,+ projection,+ batch_size,+ fetch,+ } = self;+ let input_node = ctx.encode_child(input)?;+ let expr = ctx.encode_expr(predicate)?;+ // The identity projection `[0, 1, ..., num_fields - 1]` is the+ // canonical wire representation of a full projection, so `None` is+ // encoded that way (and decodes back to `None`).+ let projection = if let Some(v) = projection {v.iter().map(|x| *x as u32).collect()} else {- (0..self.input().schema().fields().len())+ (0..input.schema().fields().len()).map(|i| i as u32).collect()};@@ -866,12 +881,12 @@ impl ExecutionPlan for FilterExec {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Filter(Box::new(protobuf::FilterExecNode {- input: Some(Box::new(input)),+ input: Some(Box::new(input_node)),expr: Some(expr),- default_filter_selectivity: self.default_selectivity() as u32,+ default_filter_selectivity: *default_selectivity as u32,projection,- batch_size: self.batch_size() as u32,- fetch: self.fetch().map(|f| f as u32),+ batch_size: *batch_size as u32,+ fetch: fetch.map(|f| f as u32),},)),),@@ -893,28 +908,37 @@ impl FilterExec {ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {use datafusion_proto_models::protobuf;- let filter = crate::expect_plan_variant!(+ let filter_node = crate::expect_plan_variant!(node,protobuf::physical_plan_node::PhysicalPlanType::Filter,"FilterExec",);- let input =- ctx.decode_required_child(filter.input.as_deref(), "FilterExec", "input")?;+ // Destructure exhaustively so that a new field on `FilterExecNode` is a+ // compile error here rather than a silently dropped field.+ let protobuf::FilterExecNode {+ input,+ expr,+ default_filter_selectivity,+ projection,+ batch_size,+ fetch,+ } = &**filter_node;+ let input = ctx.decode_required_child(input.as_deref(), "FilterExec", "input")?;let predicate = ctx.decode_required_expr(- filter.expr.as_ref(),+ expr.as_ref(),input.schema().as_ref(),"FilterExec","expr",)?;- let filter_selectivity = filter.default_filter_selectivity.try_into();+ let filter_selectivity = (*default_filter_selectivity).try_into();// `None` is encoded as the full identity projection. Reconstruct it only// when all input columns are present in order, leaving an empty list as// `Some(vec![])`.let num_fields = input.schema().fields().len();- let mut is_full_projection = filter.projection.len() == num_fields;- let mut projection_vec: Vec<usize> = Vec::with_capacity(filter.projection.len());- for (i, idx) in filter.projection.iter().enumerate() {+ let mut is_full_projection = projection.len() == num_fields;+ let mut projection_vec: Vec<usize> = Vec::with_capacity(projection.len());+ for (i, idx) in projection.iter().enumerate() {let idx = *idx as usize;is_full_projection &= idx == i;projection_vec.push(idx);@@ -926,8 +950,8 @@ impl FilterExec {};let filter = FilterExecBuilder::new(predicate, input).apply_projection(projection)?- .with_batch_size(filter.batch_size as usize)- .with_fetch(filter.fetch.map(|f| f as usize))+ .with_batch_size(*batch_size as usize)+ .with_fetch(fetch.map(|f| f as usize)).build()?;match filter_selectivity {Ok(filter_selectivity) => Ok(Arc::new(diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rsindex dd62c93d1c..4e5d9200d9 100644--- a/datafusion/physical-plan/src/limit.rs+++ b/datafusion/physical-plan/src/limit.rs@@ -278,19 +278,30 @@ impl ExecutionPlan for GlobalLimitExec {) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_physical_expr_common::sort_expr::optional_ordering_try_to_proto;use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;- let required_ordering = optional_ordering_try_to_proto(- self.required_ordering.as_ref(),- &ctx.expr_ctx(),- )?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `GlobalLimitExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ skip,+ fetch,+ required_ordering,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ } = self;+ let input = ctx.encode_child(input)?;+ let required_ordering =+ optional_ordering_try_to_proto(required_ordering.as_ref(), &ctx.expr_ctx())?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit(Box::new(protobuf::GlobalLimitExecNode {input: Some(Box::new(input)),- skip: self.skip() as u32,- fetch: match self.fetch() {- Some(n) => n as i64,+ skip: *skip as u32,+ fetch: match fetch {+ Some(n) => *n as i64,_ => -1, // no limit},required_ordering,@@ -314,21 +325,27 @@ impl GlobalLimitExec {protobuf::physical_plan_node::PhysicalPlanType::GlobalLimit,"GlobalLimitExec",);- let input = ctx.decode_required_child(- limit.input.as_deref(),- "GlobalLimitExec",- "input",- )?;- let fetch = if limit.fetch >= 0 {- Some(limit.fetch as usize)+ // Destructure exhaustively so that a new field on+ // `GlobalLimitExecNode` is a compile error here rather than a silently+ // dropped field.+ let protobuf::GlobalLimitExecNode {+ input,+ skip,+ fetch,+ required_ordering,+ } = &**limit;+ let input =+ ctx.decode_required_child(input.as_deref(), "GlobalLimitExec", "input")?;+ let fetch = if *fetch >= 0 {+ Some(*fetch as usize)} else {None};let required_ordering = optional_ordering_try_from_proto(- &limit.required_ordering,+ required_ordering,&ctx.expr_ctx(input.schema().as_ref()),)?;- let mut exec = GlobalLimitExec::new(input, limit.skip as usize, fetch);+ let mut exec = GlobalLimitExec::new(input, *skip as usize, fetch);exec.set_required_ordering(required_ordering);Ok(Arc::new(exec))}@@ -538,17 +555,27 @@ impl ExecutionPlan for LocalLimitExec {) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_physical_expr_common::sort_expr::optional_ordering_try_to_proto;use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;- let required_ordering = optional_ordering_try_to_proto(- self.required_ordering.as_ref(),- &ctx.expr_ctx(),- )?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `LocalLimitExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ fetch,+ required_ordering,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ } = self;+ let input = ctx.encode_child(input)?;+ let required_ordering =+ optional_ordering_try_to_proto(required_ordering.as_ref(), &ctx.expr_ctx())?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::LocalLimit(Box::new(protobuf::LocalLimitExecNode {input: Some(Box::new(input)),- fetch: self.fetch() as u32,+ fetch: *fetch as u32,required_ordering,},)),@@ -570,13 +597,20 @@ impl LocalLimitExec {protobuf::physical_plan_node::PhysicalPlanType::LocalLimit,"LocalLimitExec",);+ // Destructure exhaustively so that a new field on `LocalLimitExecNode`+ // is a compile error here rather than a silently dropped field.+ let protobuf::LocalLimitExecNode {+ input,+ fetch,+ required_ordering,+ } = &**limit;let input =- ctx.decode_required_child(limit.input.as_deref(), "LocalLimitExec", "input")?;+ ctx.decode_required_child(input.as_deref(), "LocalLimitExec", "input")?;let required_ordering = optional_ordering_try_from_proto(- &limit.required_ordering,+ required_ordering,&ctx.expr_ctx(input.schema().as_ref()),)?;- let mut exec = LocalLimitExec::new(input, limit.fetch as usize);+ let mut exec = LocalLimitExec::new(input, *fetch as usize);exec.set_required_ordering(required_ordering);Ok(Arc::new(exec))}diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rsindex 67c063b65c..33833cd1e7 100644--- a/datafusion/physical-plan/src/placeholder_row.rs+++ b/datafusion/physical-plan/src/placeholder_row.rs@@ -214,16 +214,22 @@ impl ExecutionPlan for PlaceholderRowExec {_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let schema = self.schema().as_ref().try_into()?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `PlaceholderRowExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ schema,+ partitions,+ // Derived from `schema` and `partitions`, recomputed on decode.+ cache: _,+ } = self;+ let schema = schema.as_ref().try_into()?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(protobuf::PlaceholderRowExecNode {schema: Some(schema),- partitions: self- .properties()- .output_partitioning()- .partition_count() as u32,+ partitions: *partitions as u32,},),),@@ -244,7 +250,11 @@ impl PlaceholderRowExec {protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow,"PlaceholderRowExec",);- let schema = placeholder.schema.as_ref().ok_or_else(|| {+ // Destructure exhaustively so that a new field on+ // `PlaceholderRowExecNode` is a compile error here rather than a+ // silently dropped field.+ let protobuf::PlaceholderRowExecNode { schema, partitions } = placeholder;+ let schema = schema.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("PlaceholderRowExec is missing required field 'schema'")@@ -252,7 +262,7 @@ impl PlaceholderRowExec {let schema = Arc::new(Schema::try_from(schema)?);// A zero (absent) partition count comes from a plan encoded before the// field existed, which always meant a single partition.- let partitions = placeholder.partitions.max(1) as usize;+ let partitions = (*partitions).max(1) as usize;Ok(Arc::new(PlaceholderRowExec::new(schema).with_partitions(partitions),))diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 1ec278eb9c..2a24eb60e6 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -619,9 +619,23 @@ impl ExecutionPlan for ProjectionExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;- let expr = ctx.encode_expressions(self.expr().iter().map(|p| &p.expr))?;- let expr_name = self.expr().iter().map(|p| p.alias.clone()).collect();+ // Destructure exhaustively (no `..`) so that adding a field to+ // `ProjectionExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ // The projector is rebuilt from the projection expressions and the+ // input schema on decode; the expressions themselves are serialized.+ projector,+ input,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ } = self;+ let projection_exprs = projector.projection().as_ref();+ let input = ctx.encode_child(input)?;+ let expr = ctx.encode_expressions(projection_exprs.iter().map(|p| &p.expr))?;+ let expr_name = projection_exprs.iter().map(|p| p.alias.clone()).collect();Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(Box::new(@@ -658,16 +672,19 @@ impl ProjectionExec {protobuf::physical_plan_node::PhysicalPlanType::Projection,"ProjectionExec",);- let input = ctx.decode_required_child(- projection.input.as_deref(),- "ProjectionExec",- "input",- )?;+ // Destructure exhaustively so that a new field on `ProjectionExecNode`+ // is a compile error here rather than a silently dropped field.+ let protobuf::ProjectionExecNode {+ input,+ expr,+ expr_name,+ } = &**projection;+ let input =+ ctx.decode_required_child(input.as_deref(), "ProjectionExec", "input")?;let input_schema = input.schema();- let exprs = projection- .expr+ let exprs = expr.iter()- .zip(projection.expr_name.iter())+ .zip(expr_name.iter()).map(|(expr, name)| {Ok(ProjectionExpr {expr: ctx.decode_expr(expr, input_schema.as_ref())?,diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 11ddcec24d..8f4b8558a5 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -1960,9 +1960,25 @@ impl ExecutionPlan for RepartitionExec {&self,ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<protobuf::PhysicalPlanNode>> {- let input = ctx.encode_child(self.input())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `RepartitionExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ // Execution-time channel state, created on `execute()`.+ state: _,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ preserve_order,+ // Derived plan properties. The output partitioning lives here (it is+ // the plan's own `partitioning`) and *is* serialized below; the rest+ // is recomputed on decode.+ cache,+ } = self;- let partitioning = self.partitioning().try_to_proto(&ctx.expr_ctx())?;+ let input = ctx.encode_child(input)?;++ let partitioning = cache.partitioning.try_to_proto(&ctx.expr_ctx())?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(@@ -1970,7 +1986,7 @@ impl ExecutionPlan for RepartitionExec {protobuf::RepartitionExecNode {input: Some(Box::new(input)),partitioning: Some(partitioning),- preserve_order: self.preserve_order(),+ preserve_order: *preserve_order,},)),),@@ -1990,15 +2006,19 @@ impl RepartitionExec {protobuf::physical_plan_node::PhysicalPlanType::Repartition,"RepartitionExec",);- let input = ctx.decode_required_child(- repart.input.as_deref(),- "RepartitionExec",- "input",- )?;+ // Destructure exhaustively so that a new field on+ // `RepartitionExecNode` is a compile error here rather than a silently+ // dropped field.+ let protobuf::RepartitionExecNode {+ input,+ partitioning,+ preserve_order,+ } = &**repart;+ let input =+ ctx.decode_required_child(input.as_deref(), "RepartitionExec", "input")?;let input_schema = input.schema();- let partitioning = repart- .partitioning+ let partitioning = partitioning.as_ref().map(|partitioning| {Partitioning::try_from_proto(@@ -2015,7 +2035,7 @@ impl RepartitionExec {})?;let mut repart_exec = RepartitionExec::try_new(input, partitioning)?;- if repart.preserve_order {+ if *preserve_order {repart_exec = repart_exec.with_preserve_order();}Ok(Arc::new(repart_exec))diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rsindex f2b7c5e0b5..cac8f92503 100644--- a/datafusion/physical-plan/src/scalar_subquery.rs+++ b/datafusion/physical-plan/src/scalar_subquery.rs@@ -285,10 +285,29 @@ impl ExecutionPlan for ScalarSubqueryExec {) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `ScalarSubqueryExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ subqueries,+ // Execution-time one-shot future, created on `execute()`.+ subquery_future: _,+ // Runtime results container, rebuilt (empty) on decode and shared+ // with the input's `ScalarSubqueryExpr` nodes.+ results: _,+ // Copied from the input's properties, recomputed on decode.+ cache: _,+ } = self;+ let input = ctx.encode_child(input)?;// Subquery indices are positional and recovered during decoding.- let subqueries =- ctx.encode_children(self.subqueries().iter().map(|subquery| &subquery.plan))?;+ let subqueries = ctx.encode_children(subqueries.iter().map(+ |ScalarSubqueryLink {+ plan,+ // Positional: recovered from the element's position on decode.+ index: _,+ }| plan,+ ))?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery(Box::new(@@ -316,8 +335,12 @@ impl ScalarSubqueryExec {protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery,"ScalarSubqueryExec",);- let results = ScalarSubqueryResults::new(scalar_subquery.subqueries.len());- let input_node = scalar_subquery.input.as_deref().ok_or_else(|| {+ // Destructure exhaustively so that a new field on+ // `ScalarSubqueryExecNode` is a compile error here rather than a+ // silently dropped field.+ let protobuf::ScalarSubqueryExecNode { input, subqueries } = &**scalar_subquery;+ let results = ScalarSubqueryResults::new(subqueries.len());+ let input_node = input.as_deref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("ScalarSubqueryExec is missing required field 'input'")@@ -325,8 +348,7 @@ impl ScalarSubqueryExec {// The input's ScalarSubqueryExpr nodes must share this results container.let input =ctx.decode_child_with_scalar_subquery_results(input_node, results.clone())?;- let subqueries = scalar_subquery- .subqueries+ let subqueries = subqueries.iter().enumerate().map(|(index, plan)| {diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex bd47668cf3..c91df46351 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -1603,9 +1603,24 @@ impl ExecutionPlan for SortExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;- let expr = self- .expr()+ // Destructure exhaustively (no `..`) so that adding a field to+ // `SortExec` is a compile error here until it is either serialized or+ // explicitly documented as not needing to be.+ let Self {+ input,+ expr,+ // Runtime metrics, not part of the plan shape.+ metrics_set: _,+ preserve_partitioning,+ fetch,+ // Derived from `input` and `expr` at construction time.+ common_sort_prefix: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ filter,+ } = self;+ let input = ctx.encode_child(input)?;+ let expr = expr.iter().map(|sort_expr| {let sort_node = Box::new(protobuf::PhysicalSortExprNode {@@ -1621,11 +1636,12 @@ impl ExecutionPlan for SortExec {})}).collect::<Result<Vec<_>>>()?;- let dynamic_filter = self- .dynamic_expressions_produced()- .into_iter()- .next()- .map(|expr| ctx.encode_expr(&expr))+ let dynamic_filter = filter+ .as_ref()+ .map(|filter| {+ let df_expr: Arc<dyn PhysicalExpr> = filter.read().expr();+ ctx.encode_expr(&df_expr)+ }).transpose()?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(@@ -1633,11 +1649,11 @@ impl ExecutionPlan for SortExec {protobuf::SortExecNode {input: Some(Box::new(input)),expr,- fetch: match self.fetch() {- Some(n) => n as i64,+ fetch: match fetch {+ Some(n) => *n as i64,None => -1,},- preserve_partitioning: self.preserve_partitioning(),+ preserve_partitioning: *preserve_partitioning,dynamic_filter,},)),@@ -1659,11 +1675,18 @@ impl SortExec {protobuf::physical_plan_node::PhysicalPlanType::Sort,"SortExec",);- let input =- ctx.decode_required_child(sort.input.as_deref(), "SortExec", "input")?;+ // Destructure exhaustively so that a new field on `SortExecNode` is a+ // compile error here rather than a silently dropped field.+ let protobuf::SortExecNode {+ input,+ expr,+ fetch,+ preserve_partitioning,+ dynamic_filter,+ } = &**sort;+ let input = ctx.decode_required_child(input.as_deref(), "SortExec", "input")?;let input_schema = input.schema();- let exprs = sort- .expr+ let exprs = expr.iter().map(|expr| {let Some(ExprType::Sort(sort_expr)) = expr.expr_type.as_ref() else {@@ -1688,12 +1711,12 @@ impl SortExec {let Some(ordering) = LexOrdering::new(exprs) else {return datafusion_common::internal_err!("SortExec requires an ordering");};- let fetch = (sort.fetch >= 0).then_some(sort.fetch as usize);+ let fetch = (*fetch >= 0).then_some(*fetch as usize);let new_sort = SortExec::new(ordering, input).with_fetch(fetch)- .with_preserve_partitioning(sort.preserve_partitioning);+ .with_preserve_partitioning(*preserve_partitioning);- let new_sort = if let Some(df_proto) = &sort.dynamic_filter {+ let new_sort = if let Some(df_proto) = dynamic_filter {let df_expr =ctx.decode_expr(df_proto, new_sort.input().schema().as_ref())?;let df = (df_expr as Arc<dyn std::any::Any + Send + Sync>)diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex e088d92d65..9fadeb972d 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -473,9 +473,24 @@ impl ExecutionPlan for SortPreservingMergeExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let input = ctx.encode_child(self.input())?;- let expr = self- .expr()+ // Destructure exhaustively (no `..`) so that adding a field to+ // `SortPreservingMergeExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ input,+ expr,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ fetch,+ // Derived plan properties, recomputed on decode.+ cache: _,+ // Not serialized: `SortPreservingMergeExecNode` has no field for it,+ // so decoding always yields the `true` default from+ // `SortPreservingMergeExec::new`.+ enable_round_robin_repartition: _,+ } = self;+ let input = ctx.encode_child(input)?;+ let expr = expr.iter().map(|e| {Ok(protobuf::PhysicalExprNode {@@ -496,7 +511,7 @@ impl ExecutionPlan for SortPreservingMergeExec {Box::new(protobuf::SortPreservingMergeExecNode {input: Some(Box::new(input)),expr,- fetch: self.fetch().map(|f| f as i64).unwrap_or(-1),+ fetch: fetch.map(|f| f as i64).unwrap_or(-1),}),),),@@ -518,14 +533,17 @@ impl SortPreservingMergeExec {protobuf::physical_plan_node::PhysicalPlanType::SortPreservingMerge,"SortPreservingMergeExec",);+ // Destructure exhaustively so that a new field on+ // `SortPreservingMergeExecNode` is a compile error here rather than a+ // silently dropped field.+ let protobuf::SortPreservingMergeExecNode { input, expr, fetch } = &**spm;let input = ctx.decode_required_child(- spm.input.as_deref(),+ input.as_deref(),"SortPreservingMergeExec","input",)?;let input_schema = input.schema();- let exprs = spm- .expr+ let exprs = expr.iter().map(|e| {let Some(protobuf::physical_expr_node::ExprType::Sort(sort)) =@@ -553,7 +571,7 @@ impl SortPreservingMergeExec {let Some(ordering) = LexOrdering::new(exprs) else {return internal_err!("SortPreservingMergeExec requires an ordering");};- let fetch = (spm.fetch >= 0).then_some(spm.fetch as usize);+ let fetch = (*fetch >= 0).then_some(*fetch as usize);Ok(Arc::new(SortPreservingMergeExec::new(ordering, input).with_fetch(fetch),))diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 160772dc22..557d8f21d1 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -574,7 +574,17 @@ impl ExecutionPlan for UnionExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let inputs = ctx.encode_children(self.inputs())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `UnionExec` is a compile error here until it is either serialized or+ // explicitly documented as not needing to be.+ let Self {+ inputs,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ } = self;+ let inputs = ctx.encode_children(inputs)?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Union(@@ -597,8 +607,10 @@ impl UnionExec {protobuf::physical_plan_node::PhysicalPlanType::Union,"UnionExec",);- let inputs = union- .inputs+ // Destructure exhaustively so that a new field on `UnionExecNode` is a+ // compile error here rather than a silently dropped field.+ let protobuf::UnionExecNode { inputs } = union;+ let inputs = inputs.iter().map(|input| ctx.decode_child(input)).collect::<Result<Vec<_>>>()?;@@ -854,7 +866,17 @@ impl ExecutionPlan for InterleaveExec {ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {use datafusion_proto_models::protobuf;- let inputs = ctx.encode_children(self.inputs())?;+ // Destructure exhaustively (no `..`) so that adding a field to+ // `InterleaveExec` is a compile error here until it is either+ // serialized or explicitly documented as not needing to be.+ let Self {+ inputs,+ // Runtime metrics, not part of the plan shape.+ metrics: _,+ // Derived plan properties, recomputed on decode.+ cache: _,+ } = self;+ let inputs = ctx.encode_children(inputs)?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Interleave(@@ -877,8 +899,10 @@ impl InterleaveExec {protobuf::physical_plan_node::PhysicalPlanType::Interleave,"InterleaveExec",);- let inputs = interleave- .inputs+ // Destructure exhaustively so that a new field on `InterleaveExecNode`+ // is a compile error here rather than a silently dropped field.+ let protobuf::InterleaveExecNode { inputs } = interleave;+ let inputs = inputs.iter().map(|input| ctx.decode_child(input)).collect::<Result<Vec<_>>>()?;