Skip to content

Apache DataFusion SQL Query Engine

Proto: migrate file sink serialization (#23781)

Phoenix491de04ec18e
diff --git a/Cargo.lock b/Cargo.lock
index ab79bfa4a3..7ace154139 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2018,6 +2018,7 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"futures",
"object_store",
@@ -2039,6 +2040,7 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"futures",
"object_store",
@@ -2069,6 +2071,7 @@ dependencies = [
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-pruning",
"datafusion-session",
"futures",
diff --git a/datafusion/datasource-csv/Cargo.toml b/datafusion/datasource-csv/Cargo.toml
index 2950925127..4026e6e808 100644
--- a/datafusion/datasource-csv/Cargo.toml
+++ b/datafusion/datasource-csv/Cargo.toml
@@ -30,6 +30,13 @@ version.workspace = true
[package.metadata.docs.rs]
all-features = true
+[features]
+proto = [
+ "dep:datafusion-proto-models",
+ "datafusion-datasource/proto",
+ "datafusion-physical-plan/proto",
+]
+
[dependencies]
arrow = { workspace = true }
async-trait = { workspace = true }
@@ -41,6 +48,7 @@ datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-physical-expr-common = { workspace = true }
datafusion-physical-plan = { workspace = true }
+datafusion-proto-models = { workspace = true, optional = true }
datafusion-session = { workspace = true }
futures = { workspace = true }
object_store = { workspace = true }
diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs
index a7f01f6ffe..7161519001 100644
--- a/datafusion/datasource-csv/src/file_format.rs
+++ b/datafusion/datasource-csv/src/file_format.rs
@@ -825,6 +825,105 @@ impl DataSink for CsvSink {
) -> Result<u64> {
FileSink::write_all(self, data, context).await
}
+
+ #[cfg(feature = "proto")]
+ fn try_to_proto(
+ &self,
+ exec: &DataSinkExec,
+ ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
+ use datafusion_proto_models::protobuf;
+ use protobuf::physical_plan_node::PhysicalPlanType;
+
+ let input = ctx.encode_child(exec.input())?;
+ let sort_order = exec.encode_sort_order(ctx)?;
+ let sink = protobuf::CsvSink::try_from(self)?;
+ let node = protobuf::CsvSinkExecNode {
+ input: Some(Box::new(input)),
+ sink: Some(sink),
+ sink_schema: Some(exec.schema().as_ref().try_into()?),
+ sort_order,
+ };
+ Ok(Some(protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(node))),
+ }))
+ }
+}
+
+#[cfg(feature = "proto")]
+impl TryFrom<&CsvSink> for datafusion_proto_models::protobuf::CsvSink {
+ type Error = DataFusionError;
+
+ fn try_from(value: &CsvSink) -> Result<Self> {
+ Ok(Self {
+ config: Some(value.config().try_into()?),
+ writer_options: Some(value.writer_options().try_into()?),
+ })
+ }
+}
+
+#[cfg(feature = "proto")]
+impl TryFrom<&datafusion_proto_models::protobuf::CsvSink> for CsvSink {
+ type Error = DataFusionError;
+
+ fn try_from(value: &datafusion_proto_models::protobuf::CsvSink) -> Result<Self> {
+ let config =
+ FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "CsvSink is missing required field 'config'"
+ )
+ })?)?;
+ let writer_options = value
+ .writer_options
+ .as_ref()
+ .ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "CsvSink is missing required field 'writer_options'"
+ )
+ })?
+ .try_into()?;
+
+ Ok(Self::new(config, writer_options))
+ }
+}
+
+#[cfg(feature = "proto")]
+impl CsvSink {
+ /// Reconstructs a [`DataSinkExec`] containing a `CsvSink` from protobuf.
+ pub fn try_from_proto(
+ node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
+ ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ use datafusion_proto_models::protobuf;
+
+ let sink_node = datafusion_physical_plan::expect_plan_variant!(
+ node,
+ protobuf::physical_plan_node::PhysicalPlanType::CsvSink,
+ "CsvSink",
+ );
+ let input = ctx.decode_required_child(
+ sink_node.input.as_deref(),
+ "CsvSinkExecNode",
+ "input",
+ )?;
+ let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "CsvSinkExecNode is missing required field 'sink'"
+ )
+ })?;
+ let data_sink = CsvSink::try_from(proto_sink)?;
+ let sort_order = DataSinkExec::decode_sort_order(
+ sink_node.sort_order.as_ref(),
+ ctx,
+ input.schema().as_ref(),
+ )?;
+
+ Ok(Arc::new(DataSinkExec::new(
+ input,
+ Arc::new(data_sink),
+ sort_order,
+ )))
+ }
}
#[cfg(test)]
diff --git a/datafusion/datasource-json/Cargo.toml b/datafusion/datasource-json/Cargo.toml
index b5947ea5c4..7aefbb42c1 100644
--- a/datafusion/datasource-json/Cargo.toml
+++ b/datafusion/datasource-json/Cargo.toml
@@ -30,6 +30,13 @@ version.workspace = true
[package.metadata.docs.rs]
all-features = true
+[features]
+proto = [
+ "dep:datafusion-proto-models",
+ "datafusion-datasource/proto",
+ "datafusion-physical-plan/proto",
+]
+
[dependencies]
arrow = { workspace = true }
async-trait = { workspace = true }
@@ -41,6 +48,7 @@ datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-physical-expr-common = { workspace = true }
datafusion-physical-plan = { workspace = true }
+datafusion-proto-models = { workspace = true, optional = true }
datafusion-session = { workspace = true }
futures = { workspace = true }
object_store = { workspace = true }
diff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rs
index 43bde2a039..1ef8ba7e4a 100644
--- a/datafusion/datasource-json/src/file_format.rs
+++ b/datafusion/datasource-json/src/file_format.rs
@@ -490,6 +490,105 @@ impl DataSink for JsonSink {
) -> Result<u64> {
FileSink::write_all(self, data, context).await
}
+
+ #[cfg(feature = "proto")]
+ fn try_to_proto(
+ &self,
+ exec: &DataSinkExec,
+ ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
+ use datafusion_proto_models::protobuf;
+ use protobuf::physical_plan_node::PhysicalPlanType;
+
+ let input = ctx.encode_child(exec.input())?;
+ let sort_order = exec.encode_sort_order(ctx)?;
+ let sink = protobuf::JsonSink::try_from(self)?;
+ let node = protobuf::JsonSinkExecNode {
+ input: Some(Box::new(input)),
+ sink: Some(sink),
+ sink_schema: Some(exec.schema().as_ref().try_into()?),
+ sort_order,
+ };
+ Ok(Some(protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(node))),
+ }))
+ }
+}
+
+#[cfg(feature = "proto")]
+impl TryFrom<&JsonSink> for datafusion_proto_models::protobuf::JsonSink {
+ type Error = datafusion_common::DataFusionError;
+
+ fn try_from(value: &JsonSink) -> Result<Self> {
+ Ok(Self {
+ config: Some(value.config().try_into()?),
+ writer_options: Some(value.writer_options().try_into()?),
+ })
+ }
+}
+
+#[cfg(feature = "proto")]
+impl TryFrom<&datafusion_proto_models::protobuf::JsonSink> for JsonSink {
+ type Error = datafusion_common::DataFusionError;
+
+ fn try_from(value: &datafusion_proto_models::protobuf::JsonSink) -> Result<Self> {
+ let config =
+ FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "JsonSink is missing required field 'config'"
+ )
+ })?)?;
+ let writer_options = value
+ .writer_options
+ .as_ref()
+ .ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "JsonSink is missing required field 'writer_options'"
+ )
+ })?
+ .try_into()?;
+
+ Ok(Self::new(config, writer_options))
+ }
+}
+
+#[cfg(feature = "proto")]
+impl JsonSink {
+ /// Reconstructs a [`DataSinkExec`] containing a `JsonSink` from protobuf.
+ pub fn try_from_proto(
+ node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
+ ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ use datafusion_proto_models::protobuf;
+
+ let sink_node = datafusion_physical_plan::expect_plan_variant!(
+ node,
+ protobuf::physical_plan_node::PhysicalPlanType::JsonSink,
+ "JsonSink",
+ );
+ let input = ctx.decode_required_child(
+ sink_node.input.as_deref(),
+ "JsonSinkExecNode",
+ "input",
+ )?;
+ let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "JsonSinkExecNode is missing required field 'sink'"
+ )
+ })?;
+ let data_sink = JsonSink::try_from(proto_sink)?;
+ let sort_order = DataSinkExec::decode_sort_order(
+ sink_node.sort_order.as_ref(),
+ ctx,
+ input.schema().as_ref(),
+ )?;
+
+ Ok(Arc::new(DataSinkExec::new(
+ input,
+ Arc::new(data_sink),
+ sort_order,
+ )))
+ }
}
#[derive(Debug)]
diff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.toml
index 32424069c1..a2589af19a 100644
--- a/datafusion/datasource-parquet/Cargo.toml
+++ b/datafusion/datasource-parquet/Cargo.toml
@@ -46,6 +46,7 @@ datafusion-physical-expr = { workspace = true }
datafusion-physical-expr-adapter = { workspace = true }
datafusion-physical-expr-common = { workspace = true }
datafusion-physical-plan = { workspace = true }
+datafusion-proto-models = { workspace = true, optional = true }
datafusion-pruning = { workspace = true }
datafusion-session = { workspace = true }
futures = { workspace = true }
@@ -74,6 +75,11 @@ name = "datafusion_datasource_parquet"
path = "src/mod.rs"
[features]
+proto = [
+ "dep:datafusion-proto-models",
+ "datafusion-datasource/proto",
+ "datafusion-physical-plan/proto",
+]
parquet_encryption = [
"parquet/encryption",
"datafusion-common/parquet_encryption",
diff --git a/datafusion/datasource-parquet/src/sink.rs b/datafusion/datasource-parquet/src/sink.rs
index df2f17c6be..e11f1d29d7 100644
--- a/datafusion/datasource-parquet/src/sink.rs
+++ b/datafusion/datasource-parquet/src/sink.rs
@@ -33,6 +33,8 @@ use datafusion_datasource::display::FileGroupDisplay;
use datafusion_datasource::file_compression_type::FileCompressionType;
use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};
use datafusion_datasource::sink::DataSink;
+#[cfg(feature = "proto")]
+use datafusion_datasource::sink::DataSinkExec;
use datafusion_datasource::write::demux::DemuxedStreamReceiver;
use datafusion_datasource::write::{
ObjectWriterBuilder, SharedBuffer, get_writer_schema,
@@ -40,6 +42,8 @@ use datafusion_datasource::write::{
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation};
use datafusion_execution::runtime_env::RuntimeEnv;
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
+#[cfg(feature = "proto")]
+use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::metrics::{
ElapsedComputeFutureExt, ExecutionPlanMetricsSet, MetricBuilder, MetricCategory,
MetricsSet, Time,
@@ -409,6 +413,105 @@ impl DataSink for ParquetSink {
) -> Result<u64> {
FileSink::write_all(self, data, context).await
}
+
+ #[cfg(feature = "proto")]
+ fn try_to_proto(
+ &self,
+ exec: &DataSinkExec,
+ ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
+ use datafusion_proto_models::protobuf;
+ use protobuf::physical_plan_node::PhysicalPlanType;
+
+ let input = ctx.encode_child(exec.input())?;
+ let sort_order = exec.encode_sort_order(ctx)?;
+ let sink = protobuf::ParquetSink::try_from(self)?;
+ let node = protobuf::ParquetSinkExecNode {
+ input: Some(Box::new(input)),
+ sink: Some(sink),
+ sink_schema: Some(exec.schema().as_ref().try_into()?),
+ sort_order,
+ };
+ Ok(Some(protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::ParquetSink(Box::new(node))),
+ }))
+ }
+}
+
+#[cfg(feature = "proto")]
+impl TryFrom<&ParquetSink> for datafusion_proto_models::protobuf::ParquetSink {
+ type Error = DataFusionError;
+
+ fn try_from(value: &ParquetSink) -> Result<Self> {
+ Ok(Self {
+ config: Some(value.config().try_into()?),
+ parquet_options: Some(value.parquet_options().try_into()?),
+ })
+ }
+}
+
+#[cfg(feature = "proto")]
+impl TryFrom<&datafusion_proto_models::protobuf::ParquetSink> for ParquetSink {
+ type Error = DataFusionError;
+
+ fn try_from(value: &datafusion_proto_models::protobuf::ParquetSink) -> Result<Self> {
+ let config =
+ FileSinkConfig::try_from(value.config.as_ref().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "ParquetSink is missing required field 'config'"
+ )
+ })?)?;
+ let parquet_options = value
+ .parquet_options
+ .as_ref()
+ .ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "ParquetSink is missing required field 'parquet_options'"
+ )
+ })?
+ .try_into()?;
+
+ Ok(Self::new(config, parquet_options))
+ }
+}
+
+#[cfg(feature = "proto")]
+impl ParquetSink {
+ /// Reconstructs a [`DataSinkExec`] containing a `ParquetSink` from protobuf.
+ pub fn try_from_proto(
+ node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
+ ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ use datafusion_proto_models::protobuf;
+
+ let sink_node = datafusion_physical_plan::expect_plan_variant!(
+ node,
+ protobuf::physical_plan_node::PhysicalPlanType::ParquetSink,
+ "ParquetSink",
+ );
+ let input = ctx.decode_required_child(
+ sink_node.input.as_deref(),
+ "ParquetSinkExecNode",
+ "input",
+ )?;
+ let proto_sink = sink_node.sink.as_ref().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "ParquetSinkExecNode is missing required field 'sink'"
+ )
+ })?;
+ let data_sink = ParquetSink::try_from(proto_sink)?;
+ let sort_order = DataSinkExec::decode_sort_order(
+ sink_node.sort_order.as_ref(),
+ ctx,
+ input.schema().as_ref(),
+ )?;
+
+ Ok(Arc::new(DataSinkExec::new(
+ input,
+ Arc::new(data_sink),
+ sort_order,
+ )))
+ }
}
/// Consumes a stream of [ArrowLeafColumn] via a channel and serializes them using an [ArrowColumnWriter]
diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rs
index 89a39c2ed4..25b559f780 100644
--- a/datafusion/datasource/src/sink.rs
+++ b/datafusion/datasource/src/sink.rs
@@ -77,8 +77,7 @@ pub trait DataSink: Any + DisplayAs + Debug + Send + Sync {
/// Implementations can use `ctx` to encode the input plan, sink-specific
/// expressions, and [`DataSinkExec::encode_sort_order`].
///
- /// Returning `Ok(None)` preserves the legacy central serialization fallback
- /// without eagerly encoding any child plans or expressions.
+ /// Returning `Ok(None)` lets the caller try its extension codec instead.
#[cfg(feature = "proto")]
fn try_to_proto(
&self,
@@ -194,6 +193,42 @@ impl DataSinkExec {
.transpose()
}
+ /// Decode the optional sink ordering from a protobuf plan node.
+ #[cfg(feature = "proto")]
+ pub fn decode_sort_order(
+ collection: Option<
+ &datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection,
+ >,
+ ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
+ schema: &Schema,
+ ) -> Result<Option<LexRequirement>> {
+ use arrow::compute::SortOptions;
+ use datafusion_physical_expr::PhysicalSortExpr;
+
+ let Some(collection) = collection else {
+ return Ok(None);
+ };
+ let sort_exprs = collection
+ .physical_sort_expr_nodes
+ .iter()
+ .map(|node| {
+ let expr = node.expr.as_ref().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!(
+ "Unexpected empty physical expression"
+ )
+ })?;
+ Ok(PhysicalSortExpr {
+ expr: ctx.decode_expr(expr, schema)?,
+ options: SortOptions {
+ descending: !node.asc,
+ nulls_first: node.nulls_first,
+ },
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ Ok(LexRequirement::new(sort_exprs.into_iter().map(Into::into)))
+ }
+
fn create_schema(
input: &Arc<dyn ExecutionPlan>,
schema: SchemaRef,
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index dd2cf8e219..3144809379 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -63,9 +63,9 @@ datafusion-common = { workspace = true }
datafusion-datasource = { workspace = true, features = ["proto"] }
datafusion-datasource-arrow = { workspace = true }
datafusion-datasource-avro = { workspace = true, optional = true }
-datafusion-datasource-csv = { workspace = true }
-datafusion-datasource-json = { workspace = true }
-datafusion-datasource-parquet = { workspace = true, optional = true }
+datafusion-datasource-csv = { workspace = true, features = ["proto"] }
+datafusion-datasource-json = { workspace = true, features = ["proto"] }
+datafusion-datasource-parquet = { workspace = true, optional = true, features = ["proto"] }
datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-functions-table = { workspace = true }
diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs
index 7c9b372348..ade6ea183b 100644
--- a/datafusion/proto/src/physical_plan/from_proto.rs
+++ b/datafusion/proto/src/physical_plan/from_proto.rs
@@ -57,7 +57,7 @@ use super::{
};
use crate::convert::TryFromProto;
use crate::protobuf::physical_expr_node::ExprType;
-use crate::{convert_required, convert_required_proto, protobuf};
+use crate::{convert_required, protobuf};
use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
/// Parses a physical sort expression from a protobuf.
@@ -586,10 +586,7 @@ impl TryFromProto<&protobuf::JsonSink> for JsonSink {
type Error = DataFusionError;
fn try_from_proto(value: &protobuf::JsonSink) -> Result<Self, Self::Error> {
- Ok(Self::new(
- convert_required_proto!(FileSinkConfig, value.config)?,
- convert_required!(value.writer_options)?,
- ))
+ Self::try_from(value)
}
}
@@ -598,10 +595,7 @@ impl TryFromProto<&protobuf::ParquetSink> for ParquetSink {
type Error = DataFusionError;
fn try_from_proto(value: &protobuf::ParquetSink) -> Result<Self, Self::Error> {
- Ok(Self::new(
- convert_required_proto!(FileSinkConfig, value.config)?,
- convert_required!(value.parquet_options)?,
- ))
+ Self::try_from(value)
}
}
@@ -609,10 +603,7 @@ impl TryFromProto<&protobuf::CsvSink> for CsvSink {
type Error = DataFusionError;
fn try_from_proto(value: &protobuf::CsvSink) -> Result<Self, Self::Error> {
- Ok(Self::new(
- convert_required_proto!(FileSinkConfig, value.config)?,
- convert_required!(value.writer_options)?,
- ))
+ Self::try_from(value)
}
}
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 3524106ee1..bb17f9dbca 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -54,7 +54,7 @@ use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};
use datafusion_functions_table::generate_series::{
Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,
};
-use datafusion_physical_expr::{LexOrdering, LexRequirement};
+use datafusion_physical_expr::LexOrdering;
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
use datafusion_physical_plan::aggregates::AggregateExec;
@@ -95,7 +95,6 @@ use prost::Message;
use prost::bytes::BufMut;
use crate::common::{byte_to_string, str_to_byte};
-use crate::convert::TryFromProto;
use crate::convert_required;
use crate::physical_plan::from_proto::{
parse_physical_expr_with_converter, parse_physical_sort_exprs,
@@ -782,15 +781,19 @@ pub trait PhysicalPlanNodeExt: Sized {
PhysicalPlanType::Analyze(_) => {
AnalyzeExec::try_from_proto(self.node(), &decode_ctx)
}
- PhysicalPlanType::JsonSink(sink) => {
- self.try_into_json_sink_physical_plan(sink, ctx, proto_converter)
+ PhysicalPlanType::JsonSink(_) => {
+ JsonSink::try_from_proto(self.node(), &decode_ctx)
}
- PhysicalPlanType::CsvSink(sink) => {
- self.try_into_csv_sink_physical_plan(sink, ctx, proto_converter)
+ PhysicalPlanType::CsvSink(_) => {
+ CsvSink::try_from_proto(self.node(), &decode_ctx)
}
- #[cfg_attr(not(feature = "parquet"), allow(unused_variables))]
- PhysicalPlanType::ParquetSink(sink) => {
- self.try_into_parquet_sink_physical_plan(sink, ctx, proto_converter)
+ PhysicalPlanType::ParquetSink(_) => {
+ #[cfg(feature = "parquet")]
+ {
+ ParquetSink::try_from_proto(self.node(), &decode_ctx)
+ }
+ #[cfg(not(feature = "parquet"))]
+ not_impl_err!("ParquetSink requires the `parquet` feature")
}
PhysicalPlanType::Unnest(_) => {
UnnestExec::try_from_proto(self.node(), &decode_ctx)
@@ -854,16 +857,6 @@ pub trait PhysicalPlanNodeExt: Sized {
return Ok(node);
}
- if let Some(exec) = plan.downcast_ref::<DataSinkExec>()
- && let Some(node) = protobuf::PhysicalPlanNode::try_from_data_sink_exec(
- exec,
- codec,
- proto_converter,
- )?
- {
- return Ok(node);
- }
-
if let Some(exec) = plan.downcast_ref::<LazyMemoryExec>()
&& let Some(node) =
protobuf::PhysicalPlanNode::try_from_lazy_memory_exec(exec)?
@@ -1630,81 +1623,53 @@ pub trait PhysicalPlanNodeExt: Sized {
AnalyzeExec::try_from_proto(self.node(), &decode_ctx)
}
+ #[deprecated(
+ since = "55.0.0",
+ note = "unused by DataFusion; `JsonSink` deserializes itself via `JsonSink::try_from_proto`"
+ )]
fn try_into_json_sink_physical_plan(
&self,
sink: &protobuf::JsonSinkExecNode,
ctx: &PhysicalPlanDecodeContext<'_>,
proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&sink.input, ctx, proto_converter)?;
-
- let data_sink = JsonSink::try_from_proto(
- sink.sink
- .as_ref()
- .ok_or_else(|| proto_error("Missing required field in protobuf"))?,
- )?;
- let sink_schema = input.schema();
- let sort_order = sink
- .sort_order
- .as_ref()
- .map(|collection| {
- parse_physical_sort_exprs(
- &collection.physical_sort_expr_nodes,
- ctx,
- &sink_schema,
- proto_converter,
- )
- .map(|sort_exprs| {
- LexRequirement::new(sort_exprs.into_iter().map(Into::into))
- })
- })
- .transpose()?
- .flatten();
- Ok(Arc::new(DataSinkExec::new(
- input,
- Arc::new(data_sink),
- sort_order,
- )))
+ let node = protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(sink.clone()))),
+ };
+ let decoder = ConverterPlanDecoder {
+ ctx,
+ proto_converter,
+ };
+ let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
+ JsonSink::try_from_proto(&node, &decode_ctx)
}
+ #[deprecated(
+ since = "55.0.0",
+ note = "unused by DataFusion; `CsvSink` deserializes itself via `CsvSink::try_from_proto`"
+ )]
fn try_into_csv_sink_physical_plan(
&self,
sink: &protobuf::CsvSinkExecNode,
ctx: &PhysicalPlanDecodeContext<'_>,
proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&sink.input, ctx, proto_converter)?;
-
- let data_sink = CsvSink::try_from_proto(
- sink.sink
- .as_ref()
- .ok_or_else(|| proto_error("Missing required field in protobuf"))?,
- )?;
- let sink_schema = input.schema();
- let sort_order = sink
- .sort_order
- .as_ref()
- .map(|collection| {
- parse_physical_sort_exprs(
- &collection.physical_sort_expr_nodes,
- ctx,
- &sink_schema,
- proto_converter,
- )
- .map(|sort_exprs| {
- LexRequirement::new(sort_exprs.into_iter().map(Into::into))
- })
- })
- .transpose()?
- .flatten();
- Ok(Arc::new(DataSinkExec::new(
- input,
- Arc::new(data_sink),
- sort_order,
- )))
+ let node = protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(sink.clone()))),
+ };
+ let decoder = ConverterPlanDecoder {
+ ctx,
+ proto_converter,
+ };
+ let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
+ CsvSink::try_from_proto(&node, &decode_ctx)
}
#[cfg_attr(not(feature = "parquet"), expect(unused_variables))]
+ #[deprecated(
+ since = "55.0.0",
+ note = "unused by DataFusion; `ParquetSink` deserializes itself via `ParquetSink::try_from_proto`"
+ )]
fn try_into_parquet_sink_physical_plan(
&self,
sink: &protobuf::ParquetSinkExecNode,
@@ -1713,38 +1678,20 @@ pub trait PhysicalPlanNodeExt: Sized {
) -> Result<Arc<dyn ExecutionPlan>> {
#[cfg(feature = "parquet")]
{
- let input = into_physical_plan(&sink.input, ctx, proto_converter)?;
-
- let data_sink = ParquetSink::try_from_proto(
- sink.sink
- .as_ref()
- .ok_or_else(|| proto_error("Missing required field in protobuf"))?,
- )?;
- let sink_schema = input.schema();
- let sort_order = sink
- .sort_order
- .as_ref()
- .map(|collection| {
- parse_physical_sort_exprs(
- &collection.physical_sort_expr_nodes,
- ctx,
- &sink_schema,
- proto_converter,
- )
- .map(|sort_exprs| {
- LexRequirement::new(sort_exprs.into_iter().map(Into::into))
- })
- })
- .transpose()?
- .flatten();
- Ok(Arc::new(DataSinkExec::new(
- input,
- Arc::new(data_sink),
- sort_order,
- )))
+ let node = protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::ParquetSink(Box::new(
+ sink.clone(),
+ ))),
+ };
+ let decoder = ConverterPlanDecoder {
+ ctx,
+ proto_converter,
+ };
+ let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
+ ParquetSink::try_from_proto(&node, &decode_ctx)
}
#[cfg(not(feature = "parquet"))]
- panic!("Trying to use ParquetSink without `parquet` feature enabled");
+ not_impl_err!("ParquetSink requires the `parquet` feature")
}
#[deprecated(
@@ -2568,66 +2515,21 @@ pub trait PhysicalPlanNodeExt: Sized {
})
}
+ #[deprecated(
+ since = "55.0.0",
+ note = "unused by DataFusion; `DataSinkExec` serializes itself via `ExecutionPlan::try_to_proto`"
+ )]
fn try_from_data_sink_exec(
exec: &DataSinkExec,
codec: &dyn PhysicalExtensionCodec,
proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Option<protobuf::PhysicalPlanNode>> {
- let input: protobuf::PhysicalPlanNode =
- protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
- exec.input().to_owned(),
- codec,
- proto_converter,
- )?;
let encoder = ConverterPlanEncoder {
codec,
proto_converter,
};
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
- let sort_order = exec.encode_sort_order(&encode_ctx)?;
-
- if let Some(sink) = exec.sink().downcast_ref::<JsonSink>() {
- return Ok(Some(protobuf::PhysicalPlanNode {
- physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(
- protobuf::JsonSinkExecNode {
- input: Some(Box::new(input)),
- sink: Some(protobuf::JsonSink::try_from_proto(sink)?),
- sink_schema: Some(exec.schema().as_ref().try_into()?),
- sort_order,
- },
- ))),
- }));
- }
-
- if let Some(sink) = exec.sink().downcast_ref::<CsvSink>() {
- return Ok(Some(protobuf::PhysicalPlanNode {
- physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(
- protobuf::CsvSinkExecNode {
- input: Some(Box::new(input)),
- sink: Some(protobuf::CsvSink::try_from_proto(sink)?),
- sink_schema: Some(exec.schema().as_ref().try_into()?),
- sort_order,
- },
- ))),
- }));
- }
-
- #[cfg(feature = "parquet")]
- if let Some(sink) = exec.sink().downcast_ref::<ParquetSink>() {
- return Ok(Some(protobuf::PhysicalPlanNode {
- physical_plan_type: Some(PhysicalPlanType::ParquetSink(Box::new(
- protobuf::ParquetSinkExecNode {
- input: Some(Box::new(input)),
- sink: Some(protobuf::ParquetSink::try_from_proto(sink)?),
- sink_schema: Some(exec.schema().as_ref().try_into()?),
- sort_order,
- },
- ))),
- }));
- }
-
- // If unknown DataSink then let extension handle it
- Ok(None)
+ exec.try_to_proto(&encode_ctx)
}
#[deprecated(
@@ -3357,18 +3259,6 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {
}
}
-fn into_physical_plan(
- node: &Option<Box<protobuf::PhysicalPlanNode>>,
- ctx: &PhysicalPlanDecodeContext<'_>,
- proto_converter: &dyn PhysicalProtoConverterExtension,
-) -> Result<Arc<dyn ExecutionPlan>> {
- if let Some(field) = node {
- proto_converter.proto_to_execution_plan(field, ctx)
- } else {
- Err(proto_error("Missing required field in protobuf"))
- }
-}
-
/// Adapter backing [`ExecutionPlanEncodeCtx`] for plans migrated to the
/// `try_to_proto` hook (#22419). Routes child-plan and child-expr encoding back
/// through the central converter so nested plans honor their own hooks.
diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs
index 236ae654c3..2d4aa72ff0 100644
--- a/datafusion/proto/src/physical_plan/to_proto.rs
+++ b/datafusion/proto/src/physical_plan/to_proto.rs
@@ -24,7 +24,7 @@ use datafusion_common::{
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
};
use datafusion_datasource::file_scan_config::FileScanConfig;
-use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};
+use datafusion_datasource::file_sink_config::FileSinkConfig;
use datafusion_datasource::{FileRange, PartitionedFile};
use datafusion_datasource_csv::file_format::CsvSink;
use datafusion_datasource_json::file_format::JsonSink;
@@ -518,10 +518,7 @@ impl TryFromProto<&JsonSink> for protobuf::JsonSink {
type Error = DataFusionError;
fn try_from_proto(value: &JsonSink) -> Result<Self, Self::Error> {
- Ok(Self {
- config: Some(protobuf::FileSinkConfig::try_from_proto(value.config())?),
- writer_options: Some(value.writer_options().try_into()?),
- })
+ Self::try_from(value)
}
}
@@ -529,10 +526,7 @@ impl TryFromProto<&CsvSink> for protobuf::CsvSink {
type Error = DataFusionError;
fn try_from_proto(value: &CsvSink) -> Result<Self, Self::Error> {
- Ok(Self {
- config: Some(protobuf::FileSinkConfig::try_from_proto(value.config())?),
- writer_options: Some(value.writer_options().try_into()?),
- })
+ Self::try_from(value)
}
}
@@ -541,10 +535,7 @@ impl TryFromProto<&ParquetSink> for protobuf::ParquetSink {
type Error = DataFusionError;
fn try_from_proto(value: &ParquetSink) -> Result<Self, Self::Error> {
- Ok(Self {
- config: Some(protobuf::FileSinkConfig::try_from_proto(value.config())?),
- parquet_options: Some(value.parquet_options().try_into()?),
- })
+ Self::try_from(value)
}
}