Apache DataFusion SQL Query Engine
fix: preserve EmptyExec and PlaceholderRowExec partition count across proto round-trip (#23643)
Andy Grove0840e5c4e941
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex b22fad9c0e..fdb33c4fd2 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1369,10 +1369,16 @@ message JoinOn {message EmptyExecNode {datafusion_common.Schema schema = 1;+ // Number of output partitions. Absent (0) means a single partition, so that+ // plans encoded before this field existed decode to the previous default.+ uint32 partitions = 2;}message PlaceholderRowExecNode {datafusion_common.Schema schema = 1;+ // Number of output partitions. Absent (0) means a single partition, so that+ // plans encoded before this field existed decode to the previous default.+ uint32 partitions = 2;}message ProjectionExecNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex c334eac2f5..050d780d08 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -5857,10 +5857,16 @@ impl serde::Serialize for EmptyExecNode {if self.schema.is_some() {len += 1;}+ if self.partitions != 0 {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.EmptyExecNode", len)?;if let Some(v) = self.schema.as_ref() {struct_ser.serialize_field("schema", v)?;}+ if self.partitions != 0 {+ struct_ser.serialize_field("partitions", &self.partitions)?;+ }struct_ser.end()}}@@ -5872,11 +5878,13 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode {{const FIELDS: &[&str] = &["schema",+ "partitions",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Schema,+ Partitions,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -5899,6 +5907,7 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode {{match value {"schema" => Ok(GeneratedField::Schema),+ "partitions" => Ok(GeneratedField::Partitions),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -5919,6 +5928,7 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode {V: serde::de::MapAccess<'de>,{let mut schema__ = None;+ let mut partitions__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Schema => {@@ -5927,10 +5937,19 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode {}schema__ = map_.next_value()?;}+ GeneratedField::Partitions => {+ if partitions__.is_some() {+ return Err(serde::de::Error::duplicate_field("partitions"));+ }+ partitions__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }}}Ok(EmptyExecNode {schema: schema__,+ partitions: partitions__.unwrap_or_default(),})}}@@ -22125,10 +22144,16 @@ impl serde::Serialize for PlaceholderRowExecNode {if self.schema.is_some() {len += 1;}+ if self.partitions != 0 {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.PlaceholderRowExecNode", len)?;if let Some(v) = self.schema.as_ref() {struct_ser.serialize_field("schema", v)?;}+ if self.partitions != 0 {+ struct_ser.serialize_field("partitions", &self.partitions)?;+ }struct_ser.end()}}@@ -22140,11 +22165,13 @@ impl<'de> serde::Deserialize<'de> for PlaceholderRowExecNode {{const FIELDS: &[&str] = &["schema",+ "partitions",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Schema,+ Partitions,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -22167,6 +22194,7 @@ impl<'de> serde::Deserialize<'de> for PlaceholderRowExecNode {{match value {"schema" => Ok(GeneratedField::Schema),+ "partitions" => Ok(GeneratedField::Partitions),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -22187,6 +22215,7 @@ impl<'de> serde::Deserialize<'de> for PlaceholderRowExecNode {V: serde::de::MapAccess<'de>,{let mut schema__ = None;+ let mut partitions__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Schema => {@@ -22195,10 +22224,19 @@ impl<'de> serde::Deserialize<'de> for PlaceholderRowExecNode {}schema__ = map_.next_value()?;}+ GeneratedField::Partitions => {+ if partitions__.is_some() {+ return Err(serde::de::Error::duplicate_field("partitions"));+ }+ partitions__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }}}Ok(PlaceholderRowExecNode {schema: schema__,+ partitions: partitions__.unwrap_or_default(),})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex db51edfd5d..7fb3f15752 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -2068,11 +2068,19 @@ pub struct JoinOn {pub struct EmptyExecNode {#[prost(message, optional, tag = "1")]pub schema: ::core::option::Option<super::datafusion_common::Schema>,+ /// Number of output partitions. Absent (0) means a single partition, so that+ /// plans encoded before this field existed decode to the previous default.+ #[prost(uint32, tag = "2")]+ pub partitions: u32,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct PlaceholderRowExecNode {#[prost(message, optional, tag = "1")]pub schema: ::core::option::Option<super::datafusion_common::Schema>,+ /// Number of output partitions. Absent (0) means a single partition, so that+ /// plans encoded before this field existed decode to the previous default.+ #[prost(uint32, tag = "2")]+ pub partitions: u32,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct ProjectionExecNode {diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 0744a94dce..f3d14ec53f 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -1710,7 +1710,10 @@ pub trait PhysicalPlanNodeExt: Sized {_proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let schema = Arc::new(convert_required!(empty.schema)?);- Ok(Arc::new(EmptyExec::new(schema)))+ // A zero (absent) partition count comes from a plan encoded before the+ // field existed, which always meant a single partition.+ let partitions = empty.partitions.max(1) as usize;+ Ok(Arc::new(EmptyExec::new(schema).with_partitions(partitions)))}fn try_into_placeholder_row_physical_plan(@@ -1719,7 +1722,12 @@ pub trait PhysicalPlanNodeExt: Sized {_ctx: &PhysicalPlanDecodeContext<'_>,) -> Result<Arc<dyn ExecutionPlan>> {let schema = Arc::new(convert_required!(placeholder.schema)?);- Ok(Arc::new(PlaceholderRowExec::new(schema)))+ // A zero (absent) partition count comes from a plan encoded before the+ // field existed, which always meant a single partition.+ let partitions = placeholder.partitions.max(1) as usize;+ Ok(Arc::new(+ PlaceholderRowExec::new(schema).with_partitions(partitions),+ ))}fn try_into_sort_physical_plan(@@ -3015,6 +3023,8 @@ pub trait PhysicalPlanNodeExt: Sized {Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Empty(protobuf::EmptyExecNode {schema: Some(schema),+ partitions: empty.properties().output_partitioning().partition_count()+ as u32,})),})}@@ -3028,6 +3038,8 @@ pub trait PhysicalPlanNodeExt: Sized {physical_plan_type: Some(PhysicalPlanType::PlaceholderRow(protobuf::PlaceholderRowExecNode {schema: Some(schema),+ partitions: empty.properties().output_partitioning().partition_count()+ as u32,},)),})diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 6ede6fc0e9..1dc4580302 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -88,9 +88,9 @@ use datafusion::physical_plan::windows::{create_udwf_window_expr,};use datafusion::physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode, Partitioning,- PhysicalExpr, PlanProperties, RangePartitioning, SendableRecordBatchStream,- SplitPoint, Statistics, displayable,+ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, InputOrderMode,+ Partitioning, PhysicalExpr, PlanProperties, RangePartitioning,+ SendableRecordBatchStream, SplitPoint, Statistics, displayable,};use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion::scalar::ScalarValue;@@ -235,6 +235,58 @@ fn roundtrip_empty() -> Result<()> {roundtrip_test(Arc::new(EmptyExec::new(Arc::new(Schema::empty()))))}+#[test]+fn roundtrip_empty_with_partitions() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let plan = Arc::new(EmptyExec::new(Arc::new(Schema::empty())).with_partitions(4));+ let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;+ assert_eq!(plan.output_partitioning().partition_count(), 4);+ Ok(())+}++#[test]+fn roundtrip_placeholder_row_with_partitions() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let plan =+ Arc::new(PlaceholderRowExec::new(Arc::new(Schema::empty())).with_partitions(4));+ let plan = roundtrip_test_and_return(plan, &ctx, &codec, &proto_converter)?;+ assert_eq!(plan.output_partitioning().partition_count(), 4);+ Ok(())+}++/// Plans encoded before `partitions` was added carry no value for it, which+/// decodes as zero and must be treated as the previous default of one.+#[test]+fn decode_empty_and_placeholder_row_without_partitions() -> Result<()> {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let schema: protobuf::Schema = (&Schema::empty()).try_into()?;++ for physical_plan_type in [+ protobuf::physical_plan_node::PhysicalPlanType::Empty(protobuf::EmptyExecNode {+ schema: Some(schema.clone()),+ partitions: 0,+ }),+ protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(+ protobuf::PlaceholderRowExecNode {+ schema: Some(schema.clone()),+ partitions: 0,+ },+ ),+ ] {+ let node = protobuf::PhysicalPlanNode {+ physical_plan_type: Some(physical_plan_type),+ };+ let plan = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?;+ assert_eq!(plan.output_partitioning().partition_count(), 1);+ }+ Ok(())+}+#[derive(Debug)]struct DowncastDelegatingExec {inner: Arc<dyn ExecutionPlan>,diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex c221104117..807c0bd1b2 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -598,3 +598,36 @@ metadata and schema fingerprint match.- Pass the current schema fingerprint to `CachedFileMetadata::is_valid_for`.See [PR #23201](https://github.com/apache/datafusion/pull/23201) for details.++### `EmptyExecNode` and `PlaceholderRowExecNode` gained a `partitions` field++The generated protobuf structs `EmptyExecNode` and `PlaceholderRowExecNode`+encoded only a schema, so the partition count set by `EmptyExec::with_partitions`+was silently dropped when a physical plan was serialized and deserialized: a plan+that reported `n` partitions before encoding reported `1` after. Both messages now+carry a `partitions` field that round-trips the count.++**Who is affected:**++- Users constructing `EmptyExecNode` or `PlaceholderRowExecNode` with an+ exhaustive struct literal.++**Migration guide:**++Set the new field, or fill it from `Default`:++```rust,ignore+// Before+EmptyExecNode { schema: Some(schema) }++// After+EmptyExecNode { schema: Some(schema), partitions: 4 }+// or+EmptyExecNode { schema: Some(schema), ..Default::default() }+```++The wire format stays compatible in both directions. Plans encoded before this+field existed decode as a single partition, the previous default, and plans+encoded after it add a field that older readers ignore.++See [PR #23643](https://github.com/apache/datafusion/pull/23643) for details.