Skip to content

Apache DataFusion SQL Query Engine

fix(proto): preserve EmptyRelation schema across logical plan round trip (#25944)

Luca Bigon7d4893cac33e
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index ba5dda6e8a..d437145618 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -181,6 +181,7 @@ message HashRepartition {
message EmptyRelationNode {
bool produce_one_row = 1;
+ datafusion_common.DfSchema schema = 2;
}
message CreateExternalTableNode {
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 05e066e1e2..f578c8a812 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -6695,10 +6695,16 @@ impl serde::Serialize for EmptyRelationNode {
if self.produce_one_row {
len += 1;
}
+ if self.schema.is_some() {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.EmptyRelationNode", len)?;
if self.produce_one_row {
struct_ser.serialize_field("produceOneRow", &self.produce_one_row)?;
}
+ if let Some(v) = self.schema.as_ref() {
+ struct_ser.serialize_field("schema", v)?;
+ }
struct_ser.end()
}
}
@@ -6711,11 +6717,13 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
const FIELDS: &[&str] = &[
"produce_one_row",
"produceOneRow",
+ "schema",
];
#[allow(clippy::enum_variant_names)]
enum GeneratedField {
ProduceOneRow,
+ Schema,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -6738,6 +6746,7 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
{
match value {
"produceOneRow" | "produce_one_row" => Ok(GeneratedField::ProduceOneRow),
+ "schema" => Ok(GeneratedField::Schema),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -6758,6 +6767,7 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
V: serde::de::MapAccess<'de>,
{
let mut produce_one_row__ = None;
+ let mut schema__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::ProduceOneRow => {
@@ -6766,10 +6776,17 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
}
produce_one_row__ = Some(map_.next_value()?);
}
+ GeneratedField::Schema => {
+ if schema__.is_some() {
+ return Err(serde::de::Error::duplicate_field("schema"));
+ }
+ schema__ = map_.next_value()?;
+ }
}
}
Ok(EmptyRelationNode {
produce_one_row: produce_one_row__.unwrap_or_default(),
+ schema: schema__,
})
}
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index 4bfa7c6fe8..3522207cdf 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -264,10 +264,12 @@ pub struct HashRepartition {
#[prost(uint64, tag = "2")]
pub partition_count: u64,
}
-#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+#[derive(Clone, PartialEq, ::prost::Message)]
pub struct EmptyRelationNode {
#[prost(bool, tag = "1")]
pub produce_one_row: bool,
+ #[prost(message, optional, tag = "2")]
+ pub schema: ::core::option::Option<super::datafusion_common::DfSchema>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CreateExternalTableNode {
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index 9ebc21c876..5302b477f7 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -805,7 +805,14 @@ impl AsLogicalPlan for LogicalPlanNode {
.build()
}
LogicalPlanType::EmptyRelation(empty_relation) => {
- LogicalPlanBuilder::empty(empty_relation.produce_one_row).build()
+ if let Some(schema) = &empty_relation.schema {
+ Ok(LogicalPlan::EmptyRelation(EmptyRelation {
+ produce_one_row: empty_relation.produce_one_row,
+ schema: Arc::new(schema.try_into()?),
+ }))
+ } else {
+ LogicalPlanBuilder::empty(empty_relation.produce_one_row).build()
+ }
}
LogicalPlanType::CreateExternalTable(create_extern_table) => {
let pb_schema = (create_extern_table.schema.clone()).ok_or_else(|| {
@@ -2012,11 +2019,13 @@ impl AsLogicalPlan for LogicalPlanNode {
})
}
LogicalPlan::EmptyRelation(EmptyRelation {
- produce_one_row, ..
+ produce_one_row,
+ schema,
}) => Ok(LogicalPlanNode {
logical_plan_type: Some(LogicalPlanType::EmptyRelation(
protobuf::EmptyRelationNode {
produce_one_row: *produce_one_row,
+ schema: Some(schema.try_into()?),
},
)),
}),
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index 4a3e666932..d27aaf27a0 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -3853,6 +3853,34 @@ async fn roundtrip_empty_table_scan_with_projection() -> Result<()> {
Ok(())
}
+// Regression test for https://github.com/apache/datafusion/issues/25937:
+// the optimizer reduces `LIMIT 0` and `WHERE false` to an `EmptyRelation`, which
+// must keep its schema across a to_proto -> from_proto round trip: on its own,
+// and under a node that still references its columns.
+#[tokio::test]
+async fn roundtrip_empty_relation_schema() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("name", DataType::Utf8, true),
+ ]));
+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));
+
+ let ctx = SessionContext::new();
+ ctx.register_table("empty", table)?;
+
+ for query in [
+ "SELECT * FROM empty LIMIT 0",
+ "SELECT max(id), count(*) FROM empty WHERE false",
+ ] {
+ let plan = ctx.sql(query).await?.into_optimized_plan()?;
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+
+ assert_eq!(plan, restored, "{query}");
+ }
+ Ok(())
+}
+
// Regression test for https://github.com/apache/datafusion/issues/22065:
// the decoder must preserve `null_aware = true` (NOT IN semantics), and the
// `NOT IN` value key count of a multi-column `(a, b) NOT IN` (which also holds
diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rs
index c71ca27b3e..72d96c0900 100644
--- a/datafusion/proto/tests/cases/serialize.rs
+++ b/datafusion/proto/tests/cases/serialize.rs
@@ -109,7 +109,7 @@ fn plan_to_json() {
schema: Arc::new(DFSchema::empty()),
});
let actual = logical_plan_to_json(&plan).unwrap();
- let expected = r#"{"emptyRelation":{}}"#.to_string();
+ let expected = r#"{"emptyRelation":{"schema":{}}}"#.to_string();
assert_eq!(actual, expected);
}