Skip to content

Apache DataFusion SQL Query Engine

refactor(proto): destructure Arrow and Avro source serde hooks (#24696)

Burak Şen777626616200
diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs
index b909263748..7e8189a78d 100644
--- a/datafusion/datasource-arrow/src/source.rs
+++ b/datafusion/datasource-arrow/src/source.rs
@@ -414,7 +414,20 @@ impl FileSource for ArrowSource {
use datafusion_proto_models::protobuf;
use protobuf::physical_plan_node::PhysicalPlanType;
- let format = match self.format {
+ // Exhaustive destructure: adding a field to `ArrowSource` without
+ // deciding how it is serialized is a compile error, not a silent
+ // round-trip gap.
+ let Self {
+ format,
+ // Runtime metrics, not part of the plan.
+ metrics: _,
+ // Serialized in `base` and reapplied on decode.
+ projection: _,
+ // Serialized in `base` and used to rebuild the source on decode.
+ table_schema: _,
+ } = self;
+
+ let format = match format {
ArrowFormat::File => protobuf::ArrowIpcFormat::File,
ArrowFormat::Stream => protobuf::ArrowIpcFormat::Stream,
};
@@ -452,16 +465,18 @@ impl ArrowSource {
);
};
- let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
+ let protobuf::ArrowScanExecNode { base_conf, format } = scan;
+
+ let base_conf = base_conf.as_ref().ok_or_else(|| {
datafusion_common::internal_datafusion_err!(
"ArrowScanExecNode is missing required field 'base_conf'"
)
})?;
- let format = protobuf::ArrowIpcFormat::try_from(scan.format).map_err(|_| {
+ let format = protobuf::ArrowIpcFormat::try_from(*format).map_err(|_| {
datafusion_common::internal_datafusion_err!(
"Unknown ArrowIpcFormat: {}",
- scan.format
+ format
)
})?;
diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rs
index 7c29adad2b..e42a97e80c 100644
--- a/datafusion/datasource-avro/src/source.rs
+++ b/datafusion/datasource-avro/src/source.rs
@@ -189,6 +189,20 @@ impl FileSource for AvroSource {
use datafusion_proto_models::protobuf;
use protobuf::physical_plan_node::PhysicalPlanType;
+ // Exhaustive destructure: adding a field to `AvroSource` without
+ // deciding how it is serialized is a compile error, not a silent
+ // round-trip gap.
+ let Self {
+ // Serialized in `base` and used to rebuild the source on decode.
+ table_schema: _,
+ // Set from `FileScanConfig` when the scan is opened.
+ batch_size: _,
+ // Serialized in `base` and reapplied on decode.
+ projection: _,
+ // Runtime metrics, not part of the plan.
+ metrics: _,
+ } = self;
+
let node = protobuf::AvroScanExecNode {
base_conf: Some(base.try_to_proto(ctx)?),
};
@@ -216,7 +230,9 @@ impl AvroSource {
);
};
- let base_conf = scan.base_conf.as_ref().ok_or_else(|| {
+ let protobuf::AvroScanExecNode { base_conf } = scan;
+
+ let base_conf = base_conf.as_ref().ok_or_else(|| {
datafusion_common::internal_datafusion_err!(
"AvroScanExecNode is missing required field 'base_conf'"
)