Apache DataFusion SQL Query Engine
fix(proto): preserve Parquet source and sink state (#25057)
Burak Şen768fc8a108c0
diff --git a/datafusion/datasource-parquet/src/sink.rs b/datafusion/datasource-parquet/src/sink.rsindex 3c66d4dcd7..92286c37a6 100644--- a/datafusion/datasource-parquet/src/sink.rs+++ b/datafusion/datasource-parquet/src/sink.rs@@ -423,6 +423,18 @@ impl DataSink for ParquetSink {use datafusion_proto_models::protobuf;use protobuf::physical_plan_node::PhysicalPlanType;+ // Keep the active hook exhaustive while centralizing field mapping in+ // the exhaustive `TryFrom<&ParquetSink>` below.+ let Self {+ config: _,+ parquet_options: _,+ // Runtime output state, not part of the plan.+ written: _,+ sorting_columns: _,+ // Runtime metrics are recreated on decode.+ metrics: _,+ } = self;+let input = ctx.encode_child(exec.input())?;let sort_order = exec.encode_sort_order(ctx)?;let sink = protobuf::ParquetSink::try_from(self)?;@@ -443,9 +455,43 @@ impl TryFrom<&ParquetSink> for datafusion_proto_models::protobuf::ParquetSink {type Error = DataFusionError;fn try_from(value: &ParquetSink) -> Result<Self> {+ use datafusion_proto_models::protobuf;++ let ParquetSink {+ config,+ parquet_options,+ // Runtime output state, not part of the plan.+ written: _,+ sorting_columns,+ // Runtime metrics are recreated on decode.+ metrics: _,+ } = value;+ let sorting_columns =+ sorting_columns+ .as_ref()+ .map(|columns| protobuf::ParquetSortingColumns {+ columns: columns+ .iter()+ .map(+ |&SortingColumn {+ column_idx,+ descending,+ nulls_first,+ }| {+ protobuf::ParquetSortingColumn {+ column_idx,+ descending,+ nulls_first,+ }+ },+ )+ .collect(),+ });+Ok(Self {- config: Some(value.config().try_into()?),- parquet_options: Some(value.parquet_options().try_into()?),+ config: Some(config.try_into()?),+ parquet_options: Some(parquet_options.try_into()?),+ sorting_columns,})}}@@ -455,14 +501,19 @@ impl TryFrom<&datafusion_proto_models::protobuf::ParquetSink> for ParquetSink {type Error = DataFusionError;fn try_from(value: &datafusion_proto_models::protobuf::ParquetSink) -> Result<Self> {- let config =- FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {- datafusion_common::internal_datafusion_err!(- "ParquetSink is missing required field 'config'"- )- })?)?;- let parquet_options = value- .parquet_options+ use datafusion_proto_models::protobuf;++ let protobuf::ParquetSink {+ config,+ parquet_options,+ sorting_columns,+ } = value;+ let config = FileSinkConfig::try_from(config.as_ref().ok_or_else(|| {+ datafusion_common::internal_datafusion_err!(+ "ParquetSink is missing required field 'config'"+ )+ })?)?;+ let parquet_options = parquet_options.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!(@@ -470,8 +521,26 @@ impl TryFrom<&datafusion_proto_models::protobuf::ParquetSink> for ParquetSink {)})?.try_into()?;+ let sorting_columns = sorting_columns.as_ref().map(+ |protobuf::ParquetSortingColumns { columns }| {+ columns+ .iter()+ .map(+ |&protobuf::ParquetSortingColumn {+ column_idx,+ descending,+ nulls_first,+ }| SortingColumn {+ column_idx,+ descending,+ nulls_first,+ },+ )+ .collect()+ },+ );- Ok(Self::new(config, parquet_options))+ Ok(Self::new(config, parquet_options).with_sorting_columns(sorting_columns))}}@@ -489,19 +558,23 @@ impl ParquetSink {protobuf::physical_plan_node::PhysicalPlanType::ParquetSink,"ParquetSink",);- let input = ctx.decode_required_child(- sink_node.input.as_deref(),- "ParquetSinkExecNode",- "input",- )?;- let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {+ let protobuf::ParquetSinkExecNode {+ input,+ sink,+ // Recomputed by `DataSinkExec::new`.+ sink_schema: _,+ sort_order,+ } = sink_node.as_ref();+ let input =+ ctx.decode_required_child(input.as_deref(), "ParquetSinkExecNode", "input")?;+ let proto_sink = sink.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("ParquetSinkExecNode is missing required field 'sink'")})?;let data_sink = ParquetSink::try_from(proto_sink)?;let sort_order = DataSinkExec::decode_sort_order(- sink_node.sort_order.as_ref(),+ sort_order.as_ref(),ctx,input.schema().as_ref(),)?;diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 1015bdc600..e1434824e4 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -42,6 +42,8 @@ use arrow::datatypes::TimeUnit;use datafusion_common::DataFusionError;use datafusion_common::config::TableParquetOptions;use datafusion_common::tree_node::TreeNodeRecursion;+#[cfg(feature = "proto")]+use datafusion_common::utils::{usize_from_wire, usize_to_wire};use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -1109,16 +1111,38 @@ impl FileSource for ParquetSource {use datafusion_proto_models::protobuf;use protobuf::physical_plan_node::PhysicalPlanType;- if self.schema_provider.is_some() {+ let Self {+ table_parquet_options,+ // Runtime metrics are recreated when the source is decoded.+ metrics: _,+ // Carried by `base`.+ table_schema: _,+ predicate,+ // Rebuilt from the decode context.+ parquet_file_reader_factory: _,+ // Requires a custom codec for serialization.+ schema_provider,+ // Applied by `FileScanConfig` before execution.+ batch_size: _,+ metadata_size_hint,+ // Carried by `base` as projection expressions.+ projection: _,+ // Not serialized or restored by the default decoder.+ #[cfg(feature = "parquet_encryption")]+ encryption_factory: _,+ reverse_row_groups,+ sort_order_for_reorder,+ } = self;++ if schema_provider.is_some() {return Ok(None);}- let predicate = self- .filter()- .map(|pred| ctx.encode_expr(&pred))+ let predicate = predicate+ .as_ref()+ .map(|pred| ctx.encode_expr(pred)).transpose()?;- let sort_order_for_reorder = self- .sort_order_for_reorder+ let sort_order_for_reorder = sort_order_for_reorder.as_ref().map(|ordering| -> datafusion_common::Result<_> {Ok(protobuf::PhysicalSortExprNodeCollection {@@ -1129,13 +1153,17 @@ impl FileSource for ParquetSource {})}).transpose()?;+ let metadata_size_hint = metadata_size_hint+ .map(|hint| usize_to_wire(hint, "ParquetSource", "metadata_size_hint"))+ .transpose()?;let node = protobuf::ParquetScanExecNode {base_conf: Some(base.try_to_proto(ctx)?),predicate,- parquet_options: Some(self.table_parquet_options().try_into()?),+ parquet_options: Some(table_parquet_options.try_into()?),sort_order_for_reorder,- reverse_row_groups: self.reverse_row_groups,+ reverse_row_groups: *reverse_row_groups,+ metadata_size_hint,};Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::ParquetScan(node)),@@ -1148,6 +1176,8 @@ impl ParquetSource {/// Reconstructs a `DataSourceExec` from a protobuf `ParquetScan`.////// Rebuilds the reader factory from the decode context because it is not serialized.+ /// Encryption factories and crypto options are not serialized or restored;+ /// plans that rely on them require custom handling.pub fn try_from_proto(node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,@@ -1168,7 +1198,16 @@ impl ParquetSource {);};- let base_conf = scan.base_conf.as_ref().ok_or_else(|| {+ let protobuf::ParquetScanExecNode {+ base_conf,+ predicate,+ parquet_options,+ sort_order_for_reorder,+ reverse_row_groups,+ metadata_size_hint,+ } = scan;++ let base_conf = base_conf.as_ref().ok_or_else(|| {datafusion_common::internal_datafusion_err!("ParquetScanExecNode is missing required field 'base_conf'")@@ -1200,13 +1239,11 @@ impl ParquetSource {schema};- let predicate = scan- .predicate+ let predicate = predicate.as_ref().map(|expr| ctx.decode_expr(expr, predicate_schema.as_ref())).transpose()?;- let sort_order_for_reorder = scan- .sort_order_for_reorder+ let sort_order_for_reorder = sort_order_for_reorder.as_ref().map(|ordering| {optional_ordering_try_from_proto(@@ -1216,9 +1253,12 @@ impl ParquetSource {}).transpose()?.flatten();+ let metadata_size_hint = metadata_size_hint+ .map(|hint| usize_from_wire(hint, "ParquetSource", "metadata_size_hint"))+ .transpose()?;let mut options = TableParquetOptions::default();- if let Some(table_options) = scan.parquet_options.as_ref() {+ if let Some(table_options) = parquet_options.as_ref() {options = table_options.try_into()?;}@@ -1243,7 +1283,8 @@ impl ParquetSource {.with_parquet_file_reader_factory(reader_factory).with_table_parquet_options(options);source.sort_order_for_reorder = sort_order_for_reorder;- source.reverse_row_groups = scan.reverse_row_groups;+ source.reverse_row_groups = *reverse_row_groups;+ source.metadata_size_hint = metadata_size_hint;if let Some(predicate) = predicate {source = source.with_predicate(predicate);diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex fac5ff2719..0f9386969a 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -986,9 +986,25 @@ message CsvSinkExecNode {PhysicalSortExprNodeCollection sort_order = 4;}+message ParquetSortingColumn {+ // Zero-based ordinal of the leaf column in the Parquet schema.+ int32 column_idx = 1;+ // Whether the column is sorted in descending order.+ bool descending = 2;+ // Whether nulls sort before non-null values.+ bool nulls_first = 3;+}++message ParquetSortingColumns {+ // The wrapper preserves the distinction between no sorting metadata and an empty list.+ repeated ParquetSortingColumn columns = 1;+}+message ParquetSink {FileSinkConfig config = 1;datafusion_common.TableParquetOptions parquet_options = 2;+ // Sorting-column metadata to write to each Parquet row group.+ ParquetSortingColumns sorting_columns = 3;}message ParquetSinkExecNode {@@ -1308,6 +1324,9 @@ message ParquetScanExecNode {PhysicalSortExprNodeCollection sort_order_for_reorder = 5;bool reverse_row_groups = 6;++ // Source-specific footer prefetch size. Absent means no hint.+ optional uint64 metadata_size_hint = 7;}message CsvScanExecNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex e06f5b7011..d97912d265 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -16764,6 +16764,9 @@ impl serde::Serialize for ParquetScanExecNode {if self.reverse_row_groups {len += 1;}+ if self.metadata_size_hint.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.ParquetScanExecNode", len)?;if let Some(v) = self.base_conf.as_ref() {struct_ser.serialize_field("baseConf", v)?;@@ -16780,6 +16783,11 @@ impl serde::Serialize for ParquetScanExecNode {if self.reverse_row_groups {struct_ser.serialize_field("reverseRowGroups", &self.reverse_row_groups)?;}+ if let Some(v) = self.metadata_size_hint.as_ref() {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("metadataSizeHint", ToString::to_string(&v).as_str())?;+ }struct_ser.end()}}@@ -16799,6 +16807,8 @@ impl<'de> serde::Deserialize<'de> for ParquetScanExecNode {"sortOrderForReorder","reverse_row_groups","reverseRowGroups",+ "metadata_size_hint",+ "metadataSizeHint",];#[allow(clippy::enum_variant_names)]@@ -16808,6 +16818,7 @@ impl<'de> serde::Deserialize<'de> for ParquetScanExecNode {ParquetOptions,SortOrderForReorder,ReverseRowGroups,+ MetadataSizeHint,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -16834,6 +16845,7 @@ impl<'de> serde::Deserialize<'de> for ParquetScanExecNode {"parquetOptions" | "parquet_options" => Ok(GeneratedField::ParquetOptions),"sortOrderForReorder" | "sort_order_for_reorder" => Ok(GeneratedField::SortOrderForReorder),"reverseRowGroups" | "reverse_row_groups" => Ok(GeneratedField::ReverseRowGroups),+ "metadataSizeHint" | "metadata_size_hint" => Ok(GeneratedField::MetadataSizeHint),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -16858,6 +16870,7 @@ impl<'de> serde::Deserialize<'de> for ParquetScanExecNode {let mut parquet_options__ = None;let mut sort_order_for_reorder__ = None;let mut reverse_row_groups__ = None;+ let mut metadata_size_hint__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::BaseConf => {@@ -16890,6 +16903,14 @@ impl<'de> serde::Deserialize<'de> for ParquetScanExecNode {}reverse_row_groups__ = Some(map_.next_value()?);}+ GeneratedField::MetadataSizeHint => {+ if metadata_size_hint__.is_some() {+ return Err(serde::de::Error::duplicate_field("metadataSizeHint"));+ }+ metadata_size_hint__ =+ map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0)+ ;+ }}}Ok(ParquetScanExecNode {@@ -16898,6 +16919,7 @@ impl<'de> serde::Deserialize<'de> for ParquetScanExecNode {parquet_options: parquet_options__,sort_order_for_reorder: sort_order_for_reorder__,reverse_row_groups: reverse_row_groups__.unwrap_or_default(),+ metadata_size_hint: metadata_size_hint__,})}}@@ -16918,6 +16940,9 @@ impl serde::Serialize for ParquetSink {if self.parquet_options.is_some() {len += 1;}+ if self.sorting_columns.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.ParquetSink", len)?;if let Some(v) = self.config.as_ref() {struct_ser.serialize_field("config", v)?;@@ -16925,6 +16950,9 @@ impl serde::Serialize for ParquetSink {if let Some(v) = self.parquet_options.as_ref() {struct_ser.serialize_field("parquetOptions", v)?;}+ if let Some(v) = self.sorting_columns.as_ref() {+ struct_ser.serialize_field("sortingColumns", v)?;+ }struct_ser.end()}}@@ -16938,12 +16966,15 @@ impl<'de> serde::Deserialize<'de> for ParquetSink {"config","parquet_options","parquetOptions",+ "sorting_columns",+ "sortingColumns",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Config,ParquetOptions,+ SortingColumns,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -16967,6 +16998,7 @@ impl<'de> serde::Deserialize<'de> for ParquetSink {match value {"config" => Ok(GeneratedField::Config),"parquetOptions" | "parquet_options" => Ok(GeneratedField::ParquetOptions),+ "sortingColumns" | "sorting_columns" => Ok(GeneratedField::SortingColumns),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -16988,6 +17020,7 @@ impl<'de> serde::Deserialize<'de> for ParquetSink {{let mut config__ = None;let mut parquet_options__ = None;+ let mut sorting_columns__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Config => {@@ -17002,11 +17035,18 @@ impl<'de> serde::Deserialize<'de> for ParquetSink {}parquet_options__ = map_.next_value()?;}+ GeneratedField::SortingColumns => {+ if sorting_columns__.is_some() {+ return Err(serde::de::Error::duplicate_field("sortingColumns"));+ }+ sorting_columns__ = map_.next_value()?;+ }}}Ok(ParquetSink {config: config__,parquet_options: parquet_options__,+ sorting_columns: sorting_columns__,})}}@@ -17157,6 +17197,226 @@ impl<'de> serde::Deserialize<'de> for ParquetSinkExecNode {deserializer.deserialize_struct("datafusion.ParquetSinkExecNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for ParquetSortingColumn {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.column_idx != 0 {+ len += 1;+ }+ if self.descending {+ len += 1;+ }+ if self.nulls_first {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.ParquetSortingColumn", len)?;+ if self.column_idx != 0 {+ struct_ser.serialize_field("columnIdx", &self.column_idx)?;+ }+ if self.descending {+ struct_ser.serialize_field("descending", &self.descending)?;+ }+ if self.nulls_first {+ struct_ser.serialize_field("nullsFirst", &self.nulls_first)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for ParquetSortingColumn {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "column_idx",+ "columnIdx",+ "descending",+ "nulls_first",+ "nullsFirst",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ ColumnIdx,+ Descending,+ NullsFirst,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "columnIdx" | "column_idx" => Ok(GeneratedField::ColumnIdx),+ "descending" => Ok(GeneratedField::Descending),+ "nullsFirst" | "nulls_first" => Ok(GeneratedField::NullsFirst),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = ParquetSortingColumn;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.ParquetSortingColumn")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<ParquetSortingColumn, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut column_idx__ = None;+ let mut descending__ = None;+ let mut nulls_first__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::ColumnIdx => {+ if column_idx__.is_some() {+ return Err(serde::de::Error::duplicate_field("columnIdx"));+ }+ column_idx__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::Descending => {+ if descending__.is_some() {+ return Err(serde::de::Error::duplicate_field("descending"));+ }+ descending__ = Some(map_.next_value()?);+ }+ GeneratedField::NullsFirst => {+ if nulls_first__.is_some() {+ return Err(serde::de::Error::duplicate_field("nullsFirst"));+ }+ nulls_first__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(ParquetSortingColumn {+ column_idx: column_idx__.unwrap_or_default(),+ descending: descending__.unwrap_or_default(),+ nulls_first: nulls_first__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.ParquetSortingColumn", FIELDS, GeneratedVisitor)+ }+}+impl serde::Serialize for ParquetSortingColumns {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.columns.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.ParquetSortingColumns", len)?;+ if !self.columns.is_empty() {+ struct_ser.serialize_field("columns", &self.columns)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for ParquetSortingColumns {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "columns",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Columns,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "columns" => Ok(GeneratedField::Columns),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = ParquetSortingColumns;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.ParquetSortingColumns")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<ParquetSortingColumns, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut columns__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Columns => {+ if columns__.is_some() {+ return Err(serde::de::Error::duplicate_field("columns"));+ }+ columns__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(ParquetSortingColumns {+ columns: columns__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.ParquetSortingColumns", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for PartialTableReference {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 149839dacc..f8cf7f82f4 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -1530,6 +1530,24 @@ pub struct CsvSinkExecNode {#[prost(message, optional, tag = "4")]pub sort_order: ::core::option::Option<PhysicalSortExprNodeCollection>,}+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]+pub struct ParquetSortingColumn {+ /// Zero-based ordinal of the leaf column in the Parquet schema.+ #[prost(int32, tag = "1")]+ pub column_idx: i32,+ /// Whether the column is sorted in descending order.+ #[prost(bool, tag = "2")]+ pub descending: bool,+ /// Whether nulls sort before non-null values.+ #[prost(bool, tag = "3")]+ pub nulls_first: bool,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct ParquetSortingColumns {+ /// The wrapper preserves the distinction between no sorting metadata and an empty list.+ #[prost(message, repeated, tag = "1")]+ pub columns: ::prost::alloc::vec::Vec<ParquetSortingColumn>,+}#[derive(Clone, PartialEq, ::prost::Message)]pub struct ParquetSink {#[prost(message, optional, tag = "1")]@@ -1538,6 +1556,9 @@ pub struct ParquetSink {pub parquet_options: ::core::option::Option<super::datafusion_common::TableParquetOptions,>,+ /// Sorting-column metadata to write to each Parquet row group.+ #[prost(message, optional, tag = "3")]+ pub sorting_columns: ::core::option::Option<ParquetSortingColumns>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct ParquetSinkExecNode {@@ -2005,6 +2026,9 @@ pub struct ParquetScanExecNode {pub sort_order_for_reorder: ::core::option::Option<PhysicalSortExprNodeCollection>,#[prost(bool, tag = "6")]pub reverse_row_groups: bool,+ /// Source-specific footer prefetch size. Absent means no hint.+ #[prost(uint64, optional, tag = "7")]+ pub metadata_size_hint: ::core::option::Option<u64>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct CsvScanExecNode {diff --git a/datafusion/proto/tests/cases/plans/mod.rs b/datafusion/proto/tests/cases/plans/mod.rsindex 8dad1eff67..40839d4b3a 100644--- a/datafusion/proto/tests/cases/plans/mod.rs+++ b/datafusion/proto/tests/cases/plans/mod.rs@@ -92,6 +92,23 @@ fn roundtrip_test_and_return(Ok(result_exec_plan)}+/// Round-trip through the public JSON APIs and return the decoded plan for field assertions.+#[cfg(feature = "json")]+fn roundtrip_test_json_and_return(+ exec_plan: Arc<dyn ExecutionPlan>,+ ctx: &SessionContext,+) -> Result<Arc<dyn ExecutionPlan>> {+ use datafusion_proto::bytes::{physical_plan_from_json, physical_plan_to_json};++ let json = physical_plan_to_json(Arc::clone(&exec_plan))?;+ let result_exec_plan = physical_plan_from_json(&json, ctx.task_ctx().as_ref())?;+ pretty_assertions::assert_eq!(+ format!("{exec_plan:?}"),+ format!("{result_exec_plan:?}")+ );+ Ok(result_exec_plan)+}+/// Perform a serde roundtrip and assert that the string representation of the before and after plans/// are identical. Note that this often isn't sufficient to guarantee that no information is/// lost during serde because the string representation of a plan often only shows a subset of state.diff --git a/datafusion/proto/tests/cases/plans/sinks.rs b/datafusion/proto/tests/cases/plans/sinks.rsindex 8d8b736000..1bc8c9d41e 100644--- a/datafusion/proto/tests/cases/plans/sinks.rs+++ b/datafusion/proto/tests/cases/plans/sinks.rs@@ -17,7 +17,7 @@//! Data sinks and their file sink configurations.-use super::{roundtrip_test, roundtrip_test_and_return};+use super::roundtrip_test_and_return;use arrow::csv::writer::Terminator;use arrow::csv::{QuoteStyle, WriterBuilder};use async_trait::async_trait;@@ -33,6 +33,7 @@ use datafusion::datasource::physical_plan::{};use datafusion::datasource::sink::{DataSink, DataSinkExec};use datafusion::execution::TaskContext;+use datafusion::parquet::file::metadata::SortingColumn;use datafusion::physical_expr::{LexRequirement, PhysicalSortRequirement};use datafusion::physical_plan::expressions::Column;use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;@@ -425,7 +426,7 @@ fn roundtrip_parquet_sink() -> Result<()> {let field_a = Field::new("plan_type", DataType::Utf8, false);let field_b = Field::new("plan", DataType::Utf8, false);let schema = Arc::new(Schema::new(vec![field_a, field_b]));- let input = Arc::new(PlaceholderRowExec::new(schema.clone()));+ let input: Arc<dyn ExecutionPlan> = Arc::new(PlaceholderRowExec::new(schema.clone()));let file_sink_config = FileSinkConfig {original_url: String::default(),@@ -439,11 +440,7 @@ fn roundtrip_parquet_sink() -> Result<()> {file_extension: "parquet".into(),file_output_mode: FileOutputMode::Automatic,};- let data_sink = Arc::new(ParquetSink::new(- file_sink_config,- TableParquetOptions::default(),- ));- let sort_order = [PhysicalSortRequirement::new(+ let sort_order: LexRequirement = [PhysicalSortRequirement::new(Arc::new(Column::new("plan_type", 0)),Some(SortOptions {descending: true,@@ -452,9 +449,65 @@ fn roundtrip_parquet_sink() -> Result<()> {)].into();- roundtrip_test(Arc::new(DataSinkExec::new(- input,- data_sink,- Some(sort_order),- )))+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let sorting_columns = vec![+ SortingColumn {+ column_idx: 0,+ descending: true,+ nulls_first: false,+ },+ SortingColumn {+ column_idx: 1,+ descending: false,+ nulls_first: true,+ },+ ];+ for sorting_columns in [+ None,+ Some(vec![]),+ Some(sorting_columns[..1].to_vec()),+ Some(sorting_columns),+ ] {+ let data_sink = Arc::new(+ ParquetSink::new(file_sink_config.clone(), TableParquetOptions::default())+ .with_sorting_columns(sorting_columns.clone()),+ );+ let roundtripped = roundtrip_test_and_return(+ Arc::new(DataSinkExec::new(+ Arc::clone(&input),+ data_sink,+ Some(sort_order.clone()),+ )),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ #[cfg(feature = "json")]+ let roundtripped = super::roundtrip_test_json_and_return(roundtripped, &ctx)?;+ let node = PhysicalPlanNode::try_from_physical_plan(roundtripped, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetSink(node)) =+ node.physical_plan_type+ else {+ panic!("expected ParquetSink node");+ };+ let actual = node+ .sink+ .expect("ParquetSinkExecNode should contain a sink")+ .sorting_columns+ .map(|columns| {+ columns+ .columns+ .into_iter()+ .map(|column| SortingColumn {+ column_idx: column.column_idx,+ descending: column.descending,+ nulls_first: column.nulls_first,+ })+ .collect::<Vec<_>>()+ });+ assert_eq!(actual, sorting_columns);+ }+ Ok(())}diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rsindex 7f04a794f2..c4897357ee 100644--- a/datafusion/proto/tests/cases/plans/sources.rs+++ b/datafusion/proto/tests/cases/plans/sources.rs@@ -111,7 +111,7 @@ fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));let predicate = Arc::new(BinaryExpr::new(- Arc::new(Column::new("col", 1)),+ Arc::new(Column::new("col", 0)),Operator::Eq,lit("1"),));@@ -119,28 +119,51 @@ fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {let mut options = TableParquetOptions::new();options.global.pushdown_filters = true;- let file_source = Arc::new(- ParquetSource::new(Arc::clone(&file_schema))- .with_table_parquet_options(options)- .with_predicate(predicate),- );-- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(Statistics {- num_rows: Precision::Inexact(100),- total_byte_size: Precision::Inexact(1024),- column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(- vec![Field::new("col", DataType::Utf8, false)],- ))),- })- .build();+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ for metadata_size_hint in [None, Some(0), Some(8192), Some(usize::MAX)] {+ let mut file_source = ParquetSource::new(Arc::clone(&file_schema))+ .with_table_parquet_options(options.clone())+ .with_predicate(predicate.clone());+ if let Some(hint) = metadata_size_hint {+ file_source = file_source.with_metadata_size_hint(hint);+ }+ let scan_config = FileScanConfigBuilder::new(+ ObjectStoreUrl::local_filesystem(),+ Arc::new(file_source),+ )+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(Statistics {+ num_rows: Precision::Inexact(100),+ total_byte_size: Precision::Inexact(1024),+ column_statistics: Statistics::unknown_column(&file_schema),+ })+ .build();- roundtrip_test(DataSourceExec::from_data_source(scan_config))+ let roundtripped = roundtrip_test_and_return(+ DataSourceExec::from_data_source(scan_config),+ &ctx,+ &codec,+ &proto_converter,+ )?;+ #[cfg(feature = "json")]+ let roundtripped = super::roundtrip_test_json_and_return(roundtripped, &ctx)?;+ let node = PhysicalPlanNode::try_from_physical_plan(roundtripped, &codec)?;+ let Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetScan(scan)) =+ node.physical_plan_type+ else {+ return internal_err!("Expected ParquetScan node");+ };+ assert_eq!(+ scan.metadata_size_hint,+ metadata_size_hint.map(|hint| hint as u64)+ );+ }+ Ok(())}#[tokio::test]diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 0a9828ff7f..f83c8b3d5c 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -227,6 +227,27 @@ The protobuf wire format remains backward compatible.See [PR #24945](https://github.com/apache/datafusion/pull/24945) for details.+### Generated protobuf Parquet structs gained state fields++The generated public protobuf `ParquetScanExecNode` struct now carries an+optional `metadata_size_hint`, and `ParquetSink` now carries optional+`sorting_columns`. These fields preserve the corresponding Parquet source and+sink settings across physical-plan serialization.++**Who is affected:**++- Users constructing either generated struct with an exhaustive struct literal.++**Migration guide:**++Add `metadata_size_hint: None` to `ParquetScanExecNode` literals and+`sorting_columns: None` to `ParquetSink` literals to retain the previous+behavior, or use `..Default::default()` for unspecified fields.++The protobuf wire format remains backward compatible.++See [PR #25057](https://github.com/apache/datafusion/pull/25057) for details.+### Generated protobuf `CsvWriterOptions` changedThe generated `CsvWriterOptions` now includes `compression_level`,