Apache DataFusion SQL Query Engine
fix(proto): destructure JSON source and sink serde hooks (#24945)
Burak Şen4cb8c2185f8a
diff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rsindex 2a9900bc23..aef89560a3 100644--- a/datafusion/datasource-json/src/file_format.rs+++ b/datafusion/datasource-json/src/file_format.rs@@ -500,6 +500,13 @@ impl DataSink for JsonSink {use datafusion_proto_models::protobuf;use protobuf::physical_plan_node::PhysicalPlanType;+ // Keep an exhaustive guard in the active hook while centralizing the+ // field mapping in the exhaustive `TryFrom<&JsonSink>` below.+ let Self {+ config: _,+ writer_options: _,+ } = self;+let input = ctx.encode_child(exec.input())?;let sort_order = exec.encode_sort_order(ctx)?;let sink = protobuf::JsonSink::try_from(self)?;@@ -520,9 +527,14 @@ impl TryFrom<&JsonSink> for datafusion_proto_models::protobuf::JsonSink {type Error = datafusion_common::DataFusionError;fn try_from(value: &JsonSink) -> Result<Self> {+ // Keep this public conversion exhaustive like the active serde hook.+ let JsonSink {+ config,+ writer_options,+ } = value;Ok(Self {- config: Some(value.config().try_into()?),- writer_options: Some(value.writer_options().try_into()?),+ config: Some(config.try_into()?),+ writer_options: Some(writer_options.try_into()?),})}}@@ -532,14 +544,17 @@ impl TryFrom<&datafusion_proto_models::protobuf::JsonSink> for JsonSink {type Error = datafusion_common::DataFusionError;fn try_from(value: &datafusion_proto_models::protobuf::JsonSink) -> Result<Self> {- let config =- FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {- datafusion_common::internal_datafusion_err!(- "JsonSink is missing required field 'config'"- )- })?)?;- let writer_options = value- .writer_options+ // Exhaustive destructure: new wire fields must be explicitly restored.+ let datafusion_proto_models::protobuf::JsonSink {+ config,+ writer_options,+ } = value;+ let config = FileSinkConfig::try_from(config.as_ref().ok_or_else(|| {+ datafusion_common::internal_datafusion_err!(+ "JsonSink is missing required field 'config'"+ )+ })?)?;+ let writer_options = writer_options.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!(@@ -566,19 +581,26 @@ impl JsonSink {protobuf::physical_plan_node::PhysicalPlanType::JsonSink,"JsonSink",);- let input = ctx.decode_required_child(- sink_node.input.as_deref(),- "JsonSinkExecNode",- "input",- )?;- let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {+ // Exhaustive destructure: a new field on `JsonSinkExecNode` is a+ // compile error here rather than a silently ignored wire field.+ let protobuf::JsonSinkExecNode {+ input,+ sink,+ // The output schema is recomputed by `DataSinkExec::new`.+ sink_schema: _,+ sort_order,+ } = sink_node.as_ref();++ let input =+ ctx.decode_required_child(input.as_deref(), "JsonSinkExecNode", "input")?;+ let proto_sink = sink.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("JsonSinkExecNode is missing required field 'sink'")})?;let data_sink = JsonSink::try_from(proto_sink)?;let sort_order = DataSinkExec::decode_sort_order(- sink_node.sort_order.as_ref(),+ sort_order.as_ref(),ctx,input.schema().as_ref(),)?;diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex c523dd47f3..f4c7021c86 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -257,9 +257,24 @@ impl FileSource for JsonSource {use datafusion_proto_models::protobuf;use protobuf::physical_plan_node::PhysicalPlanType;+ // Exhaustive destructure: adding a field to `JsonSource` without+ // deciding how it is serialized is a compile error, not a silent+ // round-trip gap.+ let Self {+ // Serialized in `base` and used to rebuild the source on decode.+ table_schema: _,+ // Set from `FileScanConfig` when the scan is opened.+ batch_size: _,+ // Runtime metrics, not part of the plan.+ metrics: _,+ // Serialized in `base` and reapplied on decode.+ projection: _,+ newline_delimited,+ } = self;+let node = protobuf::JsonScanExecNode {base_conf: Some(base.try_to_proto(ctx)?),- newline_delimited: if self.newline_delimited {+ newline_delimited: if *newline_delimited {None} else {Some(false)@@ -292,7 +307,14 @@ impl JsonSource {);};- let base_conf = scan.base_conf.as_ref().ok_or_else(|| {+ // Exhaustive destructure: a new field on `JsonScanExecNode` is a+ // compile error here rather than a silently ignored wire field.+ let protobuf::JsonScanExecNode {+ base_conf,+ newline_delimited,+ } = scan;++ let base_conf = base_conf.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("JsonScanExecNode is missing required field 'base_conf'")@@ -301,7 +323,7 @@ impl JsonSource {let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;let source = Arc::new(JsonSource::new(table_schema)- .with_newline_delimited(scan.newline_delimited.unwrap_or(true)),+ .with_newline_delimited(newline_delimited.unwrap_or(true)),);let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex 27d1101036..712212f6b6 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -431,6 +431,7 @@ enum CompressionTypeVariant {message JsonWriterOptions {CompressionTypeVariant compression = 1;+ optional uint32 compression_level = 2;}diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 7dab16406b..0f176ced0d 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -997,8 +997,19 @@ impl TryFrom<&protobuf::JsonWriterOptions> for JsonWriterOptions {fn try_from(opts: &protobuf::JsonWriterOptions,) -> datafusion_common::Result<Self, Self::Error> {- let compression: CompressionTypeVariant = opts.compression().into();- Ok(JsonWriterOptions::new(compression))+ // Exhaustive destructure: new wire fields must be explicitly restored.+ let protobuf::JsonWriterOptions {+ compression,+ compression_level,+ } = opts;+ let compression: CompressionTypeVariant =+ protobuf::CompressionTypeVariant::try_from(*compression)+ .unwrap_or_default()+ .into();+ Ok(JsonWriterOptions {+ compression,+ compression_level: *compression_level,+ })}}diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex feaeaf3c5c..2e94368bfd 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -5164,12 +5164,18 @@ impl serde::Serialize for JsonWriterOptions {if self.compression != 0 {len += 1;}+ if self.compression_level.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion_common.JsonWriterOptions", len)?;if self.compression != 0 {let v = CompressionTypeVariant::try_from(self.compression).map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.compression)))?;struct_ser.serialize_field("compression", &v)?;}+ if let Some(v) = self.compression_level.as_ref() {+ struct_ser.serialize_field("compressionLevel", v)?;+ }struct_ser.end()}}@@ -5181,11 +5187,14 @@ impl<'de> serde::Deserialize<'de> for JsonWriterOptions {{const FIELDS: &[&str] = &["compression",+ "compression_level",+ "compressionLevel",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Compression,+ CompressionLevel,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -5208,6 +5217,7 @@ impl<'de> serde::Deserialize<'de> for JsonWriterOptions {{match value {"compression" => Ok(GeneratedField::Compression),+ "compressionLevel" | "compression_level" => Ok(GeneratedField::CompressionLevel),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -5228,6 +5238,7 @@ impl<'de> serde::Deserialize<'de> for JsonWriterOptions {V: serde::de::MapAccess<'de>,{let mut compression__ = None;+ let mut compression_level__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Compression => {@@ -5236,10 +5247,19 @@ impl<'de> serde::Deserialize<'de> for JsonWriterOptions {}compression__ = Some(map_.next_value::<CompressionTypeVariant>()? as i32);}+ GeneratedField::CompressionLevel => {+ if compression_level__.is_some() {+ return Err(serde::de::Error::duplicate_field("compressionLevel"));+ }+ compression_level__ =+ map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0)+ ;+ }}}Ok(JsonWriterOptions {compression: compression__.unwrap_or_default(),+ compression_level: compression_level__,})}}diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex bdbe38538e..c0e79aec6d 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -583,6 +583,8 @@ pub struct EmptyMessage {}pub struct JsonWriterOptions {#[prost(enumeration = "CompressionTypeVariant", tag = "1")]pub compression: i32,+ #[prost(uint32, optional, tag = "2")]+ pub compression_level: ::core::option::Option<u32>,}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]pub struct CsvWriterOptions {diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex 3609817465..ae220a53d8 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -885,9 +885,16 @@ impl TryFrom<&JsonWriterOptions> for protobuf::JsonWriterOptions {fn try_from(opts: &JsonWriterOptions,) -> datafusion_common::Result<Self, Self::Error> {- let compression: protobuf::CompressionTypeVariant = opts.compression.into();+ // Exhaustive destructure: new writer options must be explicitly+ // included in the wire representation.+ let JsonWriterOptions {+ compression,+ compression_level,+ } = opts;+ let compression: protobuf::CompressionTypeVariant = (*compression).into();Ok(protobuf::JsonWriterOptions {compression: compression.into(),+ compression_level: *compression_level,})}}diff --git a/datafusion/proto-models/src/generated/datafusion_proto_common.rs b/datafusion/proto-models/src/generated/datafusion_proto_common.rsindex bdbe38538e..c0e79aec6d 100644--- a/datafusion/proto-models/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto-models/src/generated/datafusion_proto_common.rs@@ -583,6 +583,8 @@ pub struct EmptyMessage {}pub struct JsonWriterOptions {#[prost(enumeration = "CompressionTypeVariant", tag = "1")]pub compression: i32,+ #[prost(uint32, optional, tag = "2")]+ pub compression_level: ::core::option::Option<u32>,}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]pub struct CsvWriterOptions {diff --git a/datafusion/proto/tests/cases/plans/sinks.rs b/datafusion/proto/tests/cases/plans/sinks.rsindex 517659e9e9..63bf11dd30 100644--- a/datafusion/proto/tests/cases/plans/sinks.rs+++ b/datafusion/proto/tests/cases/plans/sinks.rs@@ -27,10 +27,12 @@ use datafusion::datasource::file_format::json::JsonSink;use datafusion::datasource::file_format::parquet::ParquetSink;use datafusion::datasource::listing::{ListingTableUrl, PartitionedFile};use datafusion::datasource::object_store::ObjectStoreUrl;-use datafusion::datasource::physical_plan::{FileGroup, FileOutputMode, FileSinkConfig};+use datafusion::datasource::physical_plan::{+ FileGroup, FileOutputMode, FileSink, FileSinkConfig,+};use datafusion::datasource::sink::{DataSink, DataSinkExec};use datafusion::execution::TaskContext;-use datafusion::physical_expr::PhysicalSortRequirement;+use datafusion::physical_expr::{LexRequirement, PhysicalSortRequirement};use datafusion::physical_plan::expressions::Column;use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;use datafusion::physical_plan::proto::ExecutionPlanEncodeCtx;@@ -218,9 +220,9 @@ fn roundtrip_json_sink() -> Result<()> {};let data_sink = Arc::new(JsonSink::new(file_sink_config,- JsonWriterOptions::new(CompressionTypeVariant::UNCOMPRESSED),+ JsonWriterOptions::new_with_level(CompressionTypeVariant::ZSTD, 7),));- let sort_order = [PhysicalSortRequirement::new(+ let sort_order: LexRequirement = [PhysicalSortRequirement::new(Arc::new(Column::new("plan_type", 0)),Some(SortOptions {descending: true,@@ -229,11 +231,45 @@ fn roundtrip_json_sink() -> Result<()> {)].into();- roundtrip_test(Arc::new(DataSinkExec::new(- input,- data_sink,- Some(sort_order),- )))+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let roundtrip_plan = roundtrip_test_and_return(+ Arc::new(DataSinkExec::new(+ input,+ data_sink,+ Some(sort_order.clone()),+ )),+ &ctx,+ &codec,+ &proto_converter,+ )?;++ let roundtrip_plan =+ roundtrip_plan+ .downcast_ref::<DataSinkExec>()+ .ok_or_else(|| {+ datafusion_common::internal_datafusion_err!("Expected DataSinkExec")+ })?;+ let json_sink = roundtrip_plan+ .sink()+ .downcast_ref::<JsonSink>()+ .ok_or_else(|| {+ datafusion_common::internal_datafusion_err!("Expected JsonSink")+ })?;+ assert_eq!(json_sink.config().insert_op, InsertOp::Overwrite);+ assert!(json_sink.config().keep_partition_by_columns);+ assert_eq!(+ json_sink.config().file_output_mode,+ FileOutputMode::SingleFile+ );+ assert_eq!(+ json_sink.writer_options().compression,+ CompressionTypeVariant::ZSTD+ );+ assert_eq!(json_sink.writer_options().compression_level, Some(7));+ assert_eq!(roundtrip_plan.sort_order(), &Some(sort_order));+ Ok(())}#[test]diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 90bc85cc73..2be2df94ef 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -138,6 +138,41 @@ previous default (`use_statistics_registry = false`).`StatisticsRegistry::default_with_builtin_providers()`.- To opt out, register nothing (the default).+### Generated protobuf `JsonWriterOptions` gained a `compression_level` field++The generated public protobuf `JsonWriterOptions` struct now carries an optional+`compression_level` so JSON sink plans preserve explicitly configured+compression levels across serialization.++**Who is affected:**++- Users constructing generated `JsonWriterOptions` values with an exhaustive+ struct literal.++**Migration guide:**++Set `compression_level` explicitly, or fill it from `Default`:++```rust,ignore+// Before+JsonWriterOptions { compression }++// After+JsonWriterOptions {+ compression,+ compression_level: None,+}+// or+JsonWriterOptions {+ compression,+ ..Default::default()+}+```++The protobuf wire format remains backward compatible.++See [PR #24945](https://github.com/apache/datafusion/pull/24945) for details.+### `DefaultStatisticsProvider` is deprecated`datafusion_physical_plan::operator_statistics::DefaultStatisticsProvider` is