Apache DataFusion SQL Query Engine
fix(proto): preserve EmptyRelation schema across logical plan round trip (#25944)
Luca Bigon7d4893cac33e
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex ba5dda6e8a..d437145618 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -181,6 +181,7 @@ message HashRepartition {message EmptyRelationNode {bool produce_one_row = 1;+ datafusion_common.DfSchema schema = 2;}message CreateExternalTableNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 05e066e1e2..f578c8a812 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -6695,10 +6695,16 @@ impl serde::Serialize for EmptyRelationNode {if self.produce_one_row {len += 1;}+ if self.schema.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.EmptyRelationNode", len)?;if self.produce_one_row {struct_ser.serialize_field("produceOneRow", &self.produce_one_row)?;}+ if let Some(v) = self.schema.as_ref() {+ struct_ser.serialize_field("schema", v)?;+ }struct_ser.end()}}@@ -6711,11 +6717,13 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {const FIELDS: &[&str] = &["produce_one_row","produceOneRow",+ "schema",];#[allow(clippy::enum_variant_names)]enum GeneratedField {ProduceOneRow,+ Schema,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -6738,6 +6746,7 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {{match value {"produceOneRow" | "produce_one_row" => Ok(GeneratedField::ProduceOneRow),+ "schema" => Ok(GeneratedField::Schema),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -6758,6 +6767,7 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {V: serde::de::MapAccess<'de>,{let mut produce_one_row__ = None;+ let mut schema__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::ProduceOneRow => {@@ -6766,10 +6776,17 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {}produce_one_row__ = Some(map_.next_value()?);}+ GeneratedField::Schema => {+ if schema__.is_some() {+ return Err(serde::de::Error::duplicate_field("schema"));+ }+ schema__ = map_.next_value()?;+ }}}Ok(EmptyRelationNode {produce_one_row: produce_one_row__.unwrap_or_default(),+ schema: schema__,})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 4bfa7c6fe8..3522207cdf 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -264,10 +264,12 @@ pub struct HashRepartition {#[prost(uint64, tag = "2")]pub partition_count: u64,}-#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]+#[derive(Clone, PartialEq, ::prost::Message)]pub struct EmptyRelationNode {#[prost(bool, tag = "1")]pub produce_one_row: bool,+ #[prost(message, optional, tag = "2")]+ pub schema: ::core::option::Option<super::datafusion_common::DfSchema>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct CreateExternalTableNode {diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 9ebc21c876..5302b477f7 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -805,7 +805,14 @@ impl AsLogicalPlan for LogicalPlanNode {.build()}LogicalPlanType::EmptyRelation(empty_relation) => {- LogicalPlanBuilder::empty(empty_relation.produce_one_row).build()+ if let Some(schema) = &empty_relation.schema {+ Ok(LogicalPlan::EmptyRelation(EmptyRelation {+ produce_one_row: empty_relation.produce_one_row,+ schema: Arc::new(schema.try_into()?),+ }))+ } else {+ LogicalPlanBuilder::empty(empty_relation.produce_one_row).build()+ }}LogicalPlanType::CreateExternalTable(create_extern_table) => {let pb_schema = (create_extern_table.schema.clone()).ok_or_else(|| {@@ -2012,11 +2019,13 @@ impl AsLogicalPlan for LogicalPlanNode {})}LogicalPlan::EmptyRelation(EmptyRelation {- produce_one_row, ..+ produce_one_row,+ schema,}) => Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::EmptyRelation(protobuf::EmptyRelationNode {produce_one_row: *produce_one_row,+ schema: Some(schema.try_into()?),},)),}),diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex 4a3e666932..d27aaf27a0 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -3853,6 +3853,34 @@ async fn roundtrip_empty_table_scan_with_projection() -> Result<()> {Ok(())}+// Regression test for https://github.com/apache/datafusion/issues/25937:+// the optimizer reduces `LIMIT 0` and `WHERE false` to an `EmptyRelation`, which+// must keep its schema across a to_proto -> from_proto round trip: on its own,+// and under a node that still references its columns.+#[tokio::test]+async fn roundtrip_empty_relation_schema() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("name", DataType::Utf8, true),+ ]));+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));++ let ctx = SessionContext::new();+ ctx.register_table("empty", table)?;++ for query in [+ "SELECT * FROM empty LIMIT 0",+ "SELECT max(id), count(*) FROM empty WHERE false",+ ] {+ let plan = ctx.sql(query).await?.into_optimized_plan()?;+ let bytes = logical_plan_to_bytes(&plan)?;+ let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;++ assert_eq!(plan, restored, "{query}");+ }+ Ok(())+}+// Regression test for https://github.com/apache/datafusion/issues/22065:// the decoder must preserve `null_aware = true` (NOT IN semantics), and the// `NOT IN` value key count of a multi-column `(a, b) NOT IN` (which also holdsdiff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rsindex c71ca27b3e..72d96c0900 100644--- a/datafusion/proto/tests/cases/serialize.rs+++ b/datafusion/proto/tests/cases/serialize.rs@@ -109,7 +109,7 @@ fn plan_to_json() {schema: Arc::new(DFSchema::empty()),});let actual = logical_plan_to_json(&plan).unwrap();- let expected = r#"{"emptyRelation":{}}"#.to_string();+ let expected = r#"{"emptyRelation":{"schema":{}}}"#.to_string();assert_eq!(actual, expected);}