Apache DataFusion SQL Query Engine
fix: preserve ProjectionExec metadata during serialization (#25009)
Goutam Adwant5c0e73f87034
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 89fe640f88..061afc6e6e 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -652,13 +652,25 @@ impl ExecutionPlan for ProjectionExec {metrics: _,// Derived plan properties, recomputed on decode.cache: _,- // Derived metadata comparison, recomputed with the projector.- overrides_metadata: _,+ overrides_metadata,} = self;let projection_exprs = projector.projection().as_ref();let input = ctx.encode_child(input)?;let expr = ctx.encode_expressions(projection_exprs.iter().map(|p| &p.expr))?;let expr_name = projection_exprs.iter().map(|p| p.alias.clone()).collect();+ let output_schema = projector.output_schema();+ // Preserve explicit overrides and field metadata that cannot be+ // reconstructed from the child. Inherited schema metadata can be.+ let schema = if *overrides_metadata+ || output_schema+ .fields()+ .iter()+ .any(|field| !field.metadata().is_empty())+ {+ Some(output_schema.as_ref().try_into()?)+ } else {+ None+ };Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(Box::new(@@ -666,6 +678,7 @@ impl ExecutionPlan for ProjectionExec {input: Some(Box::new(input)),expr,expr_name,+ schema,},)),),@@ -701,6 +714,7 @@ impl ProjectionExec {input,expr,expr_name,+ schema,} = &**projection;let input =ctx.decode_required_child(input.as_deref(), "ProjectionExec", "input")?;@@ -715,7 +729,15 @@ impl ProjectionExec {})}).collect::<Result<Vec<ProjectionExpr>>>()?;- Ok(Arc::new(ProjectionExec::try_new(exprs, input)?))+ let projection = match schema {+ Some(schema) => ProjectionExec::try_new_with_schema_metadata(+ exprs,+ input,+ &Schema::try_from(schema)?,+ )?,+ None => ProjectionExec::try_new(exprs, input)?,+ };+ Ok(Arc::new(projection))}}diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 2e6eaea107..b82efd3b2b 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1526,6 +1526,11 @@ message ProjectionExecNode {PhysicalPlanNode input = 1;repeated PhysicalExprNode expr = 2;repeated string expr_name = 3;+ // Only field and schema metadata are used; output types are derived from expr.+ // Present when the projection output has metadata or explicitly overrides+ // inherited metadata, including an explicit clear. Absent for older plans+ // and metadata-free projections.+ datafusion_common.Schema schema = 4;}enum AggregateMode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 665d01eb96..01b7f02563 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -24658,6 +24658,9 @@ impl serde::Serialize for ProjectionExecNode {if !self.expr_name.is_empty() {len += 1;}+ if self.schema.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.ProjectionExecNode", len)?;if let Some(v) = self.input.as_ref() {struct_ser.serialize_field("input", v)?;@@ -24668,6 +24671,9 @@ impl serde::Serialize for ProjectionExecNode {if !self.expr_name.is_empty() {struct_ser.serialize_field("exprName", &self.expr_name)?;}+ if let Some(v) = self.schema.as_ref() {+ struct_ser.serialize_field("schema", v)?;+ }struct_ser.end()}}@@ -24682,6 +24688,7 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {"expr","expr_name","exprName",+ "schema",];#[allow(clippy::enum_variant_names)]@@ -24689,6 +24696,7 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {Input,Expr,ExprName,+ Schema,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -24713,6 +24721,7 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {"input" => Ok(GeneratedField::Input),"expr" => Ok(GeneratedField::Expr),"exprName" | "expr_name" => Ok(GeneratedField::ExprName),+ "schema" => Ok(GeneratedField::Schema),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -24735,6 +24744,7 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {let mut input__ = None;let mut expr__ = None;let mut expr_name__ = None;+ let mut schema__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Input => {@@ -24755,12 +24765,19 @@ impl<'de> serde::Deserialize<'de> for ProjectionExecNode {}expr_name__ = Some(map_.next_value()?);}+ GeneratedField::Schema => {+ if schema__.is_some() {+ return Err(serde::de::Error::duplicate_field("schema"));+ }+ schema__ = map_.next_value()?;+ }}}Ok(ProjectionExecNode {input: input__,expr: expr__.unwrap_or_default(),expr_name: expr_name__.unwrap_or_default(),+ schema: schema__,})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 5275c34268..61ba256704 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -2302,6 +2302,12 @@ pub struct ProjectionExecNode {pub expr: ::prost::alloc::vec::Vec<PhysicalExprNode>,#[prost(string, repeated, tag = "3")]pub expr_name: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,+ /// Only field and schema metadata are used; output types are derived from expr.+ /// Present when the projection output has metadata or explicitly overrides+ /// inherited metadata, including an explicit clear. Absent for older plans+ /// and metadata-free projections.+ #[prost(message, optional, tag = "4")]+ pub schema: ::core::option::Option<super::datafusion_common::Schema>,}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]pub struct PartiallySortedInputOrderMode {diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rsindex f0828b5a66..aea02f258d 100644--- a/datafusion/proto/tests/cases/plans/exprs.rs+++ b/datafusion/proto/tests/cases/plans/exprs.rs@@ -43,16 +43,312 @@ use datafusion_proto::physical_plan::{};use datafusion_proto::protobuf;use datafusion_proto::protobuf::PhysicalPlanNode;+use prost::Message;+use std::collections::HashMap;use std::sync::Arc;use std::vec;+#[test]+fn roundtrip_projection_metadata() -> Result<()> {+ let input_schema = Arc::new(Schema::new(vec![Field::new(+ "value",+ DataType::Int32,+ false,+ )]));+ let projected_schema =+ Schema::new_with_metadata(+ vec![Field::new("value", DataType::Int32, false).with_metadata(+ HashMap::from([("field-key".to_string(), "field-value".to_string())]),+ )],+ HashMap::from([("schema-key".to_string(), "schema-value".to_string())]),+ );+ let plan = Arc::new(ProjectionExec::try_new_with_schema_metadata(+ vec![(col("value", &input_schema)?, "value".to_string())],+ Arc::new(EmptyExec::new(input_schema)),+ &projected_schema,+ )?);+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DefaultPhysicalProtoConverter {};+ let decoded = roundtrip_test_and_return(plan, &ctx, &codec, &converter)?;+ assert_eq!(decoded.schema().as_ref(), &projected_schema);+ Ok(())+}++#[test]+fn roundtrip_projection_metadata_overrides() -> Result<()> {+ let field_metadata =+ HashMap::from([("field-key".to_string(), "field-value".to_string())]);+ let extension_metadata = HashMap::from([+ ("ARROW:extension:name".to_string(), "arrow.uuid".to_string()),+ ("ARROW:extension:metadata".to_string(), String::new()),+ ]);+ for (input_field, output_field, input_metadata) in [+ (+ Field::new("value", DataType::Int32, false).with_metadata(field_metadata),+ Field::new("value", DataType::Int32, false),+ HashMap::from([("input-schema".to_string(), "input-value".to_string())]),+ ),+ (+ Field::new("value", DataType::FixedSizeBinary(16), true),+ Field::new("value", DataType::FixedSizeBinary(16), true)+ .with_metadata(extension_metadata),+ HashMap::new(),+ ),+ ] {+ let input_schema =+ Arc::new(Schema::new_with_metadata(vec![input_field], input_metadata));+ let projected_schema = Schema::new(vec![output_field]);+ let plan = Arc::new(ProjectionExec::try_new_with_schema_metadata(+ vec![(col("value", &input_schema)?, "value".to_string())],+ Arc::new(EmptyExec::new(input_schema)),+ &projected_schema,+ )?);+ let codec = DefaultPhysicalExtensionCodec {};+ let ctx = SessionContext::new();+ let node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(projection)) =+ node.physical_plan_type.as_ref()+ else {+ unreachable!("expected ProjectionExecNode")+ };+ assert!(projection.schema.is_some());+ for node in projection_roundtrip_nodes(&node) {+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ assert_eq!(decoded.schema().as_ref(), &projected_schema);+ }+ }+ Ok(())+}++#[test]+fn roundtrip_projection_without_schema() -> Result<()> {+ let input_schema =+ Arc::new(Schema::new_with_metadata(+ vec![Field::new("value", DataType::Int32, false).with_metadata(+ HashMap::from([("field-key".to_string(), "field-value".to_string())]),+ )],+ HashMap::from([("schema-key".to_string(), "schema-value".to_string())]),+ ));+ let plan = Arc::new(ProjectionExec::try_new(+ vec![(col("value", &input_schema)?, "value".to_string())],+ Arc::new(EmptyExec::new(Arc::clone(&input_schema))),+ )?);+ let codec = DefaultPhysicalExtensionCodec {};+ let ctx = SessionContext::new();+ let mut node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(projection)) =+ node.physical_plan_type.as_mut()+ else {+ unreachable!("expected ProjectionExecNode")+ };+ // Simulate a plan encoded before the schema field existed.+ projection.schema = None;+ for node in projection_roundtrip_nodes(&node) {+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ assert_eq!(decoded.schema(), input_schema);+ }+ Ok(())+}++fn projection_roundtrip_nodes(node: &PhysicalPlanNode) -> Vec<PhysicalPlanNode> {+ vec![+ PhysicalPlanNode::decode(node.encode_to_vec().as_slice()).unwrap(),+ #[cfg(feature = "json")]+ serde_json::from_str(&serde_json::to_string(node).unwrap()).unwrap(),+ ]+}++#[test]+fn roundtrip_projection_metadata_without_child_metadata() -> Result<()> {+ let field_metadata =+ HashMap::from([("field-key".to_string(), "field-value".to_string())]);+ let schema_metadata =+ HashMap::from([("schema-key".to_string(), "schema-value".to_string())]);+ let extension_metadata = HashMap::from([+ ("ARROW:extension:name".to_string(), "arrow.uuid".to_string()),+ ("ARROW:extension:metadata".to_string(), String::new()),+ ]);+ for (data_type, field_metadata, schema_metadata) in [+ (+ DataType::Int32,+ field_metadata.clone(),+ schema_metadata.clone(),+ ),+ (DataType::Int32, field_metadata, HashMap::new()),+ (+ DataType::FixedSizeBinary(16),+ extension_metadata,+ HashMap::new(),+ ),+ ] {+ let input_field = Field::new("value", data_type, false);+ let input_without_metadata = Arc::new(Schema::new(vec![input_field.clone()]));+ let input_schema = Arc::new(Schema::new_with_metadata(+ vec![input_field.with_metadata(field_metadata)],+ schema_metadata,+ ));+ // This constructor derives metadata from the input, without an override.+ let plan = Arc::new(ProjectionExec::try_new(+ vec![(col("value", &input_schema)?, "output".to_string())],+ Arc::new(EmptyExec::new(input_schema)),+ )?);+ let expected_schema = plan.schema();+ let codec = DefaultPhysicalExtensionCodec {};+ let ctx = SessionContext::new();+ let mut node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(projection)) =+ node.physical_plan_type.as_mut()+ else {+ unreachable!("expected ProjectionExecNode")+ };+ // The child cannot supply metadata when the projection is reconstructed.+ projection.input = Some(Box::new(PhysicalPlanNode::try_from_physical_plan(+ Arc::new(EmptyExec::new(input_without_metadata)),+ &codec,+ )?));+ for node in projection_roundtrip_nodes(&node) {+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ assert_eq!(decoded.schema(), expected_schema);+ assert!(decoded.children()[0].schema().metadata().is_empty());+ assert!(+ decoded.children()[0]+ .schema()+ .field(0)+ .metadata()+ .is_empty()+ );+ }+ }+ Ok(())+}++#[test]+fn roundtrip_projection_inherited_schema_metadata_from_child() -> Result<()> {+ let schema_metadata =+ HashMap::from([("schema-key".to_string(), "schema-value".to_string())]);+ let input_schema = Arc::new(Schema::new_with_metadata(+ vec![Field::new("value", DataType::Int32, false)],+ schema_metadata,+ ));+ let plan = Arc::new(ProjectionExec::try_new(+ vec![(col("value", &input_schema)?, "output".to_string())],+ Arc::new(EmptyExec::new(input_schema)),+ )?);+ let expected_schema = plan.schema();+ let codec = DefaultPhysicalExtensionCodec {};+ let ctx = SessionContext::new();+ let node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(projection)) =+ node.physical_plan_type.as_ref()+ else {+ unreachable!("expected ProjectionExecNode")+ };+ assert!(projection.schema.is_none());+ for node in projection_roundtrip_nodes(&node) {+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ assert_eq!(decoded.schema(), expected_schema);+ }+ Ok(())+}++#[test]+fn roundtrip_projection_without_metadata() -> Result<()> {+ let input_schema = Arc::new(Schema::new(vec![Field::new(+ "value",+ DataType::Int32,+ false,+ )]));+ let plan = Arc::new(ProjectionExec::try_new(+ vec![(col("value", &input_schema)?, "value".to_string())],+ Arc::new(EmptyExec::new(Arc::clone(&input_schema))),+ )?);+ let codec = DefaultPhysicalExtensionCodec {};+ let ctx = SessionContext::new();+ let node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(projection)) =+ node.physical_plan_type.as_ref()+ else {+ unreachable!("expected ProjectionExecNode")+ };+ assert!(projection.schema.is_none());+ for node in projection_roundtrip_nodes(&node) {+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ assert_eq!(decoded.schema(), input_schema);+ }+ Ok(())+}++#[test]+fn decode_projection_schema_only_replaces_metadata() -> Result<()> {+ let input_schema = Arc::new(Schema::new(vec![Field::new(+ "input",+ DataType::Int32,+ false,+ )]));+ let plan = Arc::new(ProjectionExec::try_new(+ vec![(col("input", &input_schema)?, "output".to_string())],+ Arc::new(EmptyExec::new(input_schema)),+ )?);+ let codec = DefaultPhysicalExtensionCodec {};+ let ctx = SessionContext::new();+ let mut node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(projection)) =+ node.physical_plan_type.as_mut()+ else {+ unreachable!("expected ProjectionExecNode")+ };+ let field_metadata =+ HashMap::from([("field-key".to_string(), "field-value".to_string())]);+ let schema_metadata =+ HashMap::from([("schema-key".to_string(), "schema-value".to_string())]);+ let metadata_schema = Schema::new_with_metadata(+ vec![+ Field::new("ignored", DataType::Utf8, true)+ .with_metadata(field_metadata.clone()),+ ],+ schema_metadata.clone(),+ );+ projection.schema = Some((&metadata_schema).try_into()?);+ let expected_schema = Schema::new_with_metadata(+ vec![Field::new("output", DataType::Int32, false).with_metadata(field_metadata)],+ schema_metadata,+ );+ for node in projection_roundtrip_nodes(&node) {+ let decoded = node.try_into_physical_plan(&ctx.task_ctx(), &codec)?;+ assert_eq!(decoded.schema().as_ref(), &expected_schema);+ }+ for field_count in [0, 2] {+ let Some(protobuf::physical_plan_node::PhysicalPlanType::Projection(projection)) =+ node.physical_plan_type.as_mut()+ else {+ unreachable!("expected ProjectionExecNode")+ };+ let metadata_schema =+ Schema::new(vec![+ Field::new("ignored", DataType::Utf8, true);+ field_count+ ]);+ projection.schema = Some((&metadata_schema).try_into()?);+ for node in projection_roundtrip_nodes(&node) {+ let error = node+ .try_into_physical_plan(&ctx.task_ctx(), &codec)+ .unwrap_err();+ assert!(error.strip_backtrace().contains(&format!(+ "Projection has 1 output fields but metadata schema has {field_count} fields"+ )));+ }+ }+ Ok(())+}+#[test]fn roundtrip_literal_metadata() -> Result<()> {use datafusion_common::metadata::FieldMetadata;use datafusion_proto::bytes::{physical_plan_from_bytes, physical_plan_to_bytes};let schema = Arc::new(Schema::empty());- let metadata = FieldMetadata::from(std::collections::HashMap::from([+ let metadata = FieldMetadata::from(HashMap::from([("ARROW:extension:name".to_string(), "example.id".to_string()),("ARROW:extension:metadata".to_string(), "{}".to_string()),("description".to_string(), "identifier".to_string()),