Skip to content

Apache DataFusion SQL Query Engine

feat: implement protobuf converter trait to allow control over serialization and deserialization processes (#19437)

Tim Saucer41d48b3f6c8a
diff --git a/Cargo.lock b/Cargo.lock
index 9b87670734..b219b83cd7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2100,6 +2100,7 @@ dependencies = [
"object_store",
"prost",
"rand 0.9.2",
+ "serde",
"serde_json",
"strum",
"strum_macros",
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 368cc67641..4cbec17272 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -62,6 +62,7 @@ mimalloc = { version = "0.1", default-features = false }
object_store = { workspace = true, features = ["aws", "http"] }
prost = { workspace = true }
rand = { workspace = true }
+serde = { version = "1", features = ["derive"] }
serde_json = { workspace = true }
strum = { workspace = true }
strum_macros = { workspace = true }
diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
new file mode 100644
index 0000000000..f72c9d2b61
--- /dev/null
+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
@@ -0,0 +1,544 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! See `main.rs` for how to run it.
+//!
+//! This example demonstrates how to use the `PhysicalExtensionCodec` trait's
+//! interception methods (`serialize_physical_plan` and `deserialize_physical_plan`)
+//! to implement custom serialization logic.
+//!
+//! The key insight is that `FileScanConfig::expr_adapter_factory` is NOT serialized by
+//! default. This example shows how to:
+//! 1. Detect plans with custom adapters during serialization
+//! 2. Wrap them as Extension nodes with JSON-serialized adapter metadata
+//! 3. Unwrap and restore the adapter during deserialization
+//!
+//! This demonstrates nested serialization (protobuf outer, JSON inner) and the power
+//! of the `PhysicalExtensionCodec` interception pattern. Both plan and expression
+//! serialization route through the codec, enabling interception at every node in the tree.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use arrow::array::record_batch;
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion::assert_batches_eq;
+use datafusion::common::{Result, not_impl_err};
+use datafusion::datasource::listing::{
+ ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,
+};
+use datafusion::datasource::physical_plan::{FileScanConfig, FileScanConfigBuilder};
+use datafusion::datasource::source::DataSourceExec;
+use datafusion::execution::TaskContext;
+use datafusion::execution::context::SessionContext;
+use datafusion::execution::object_store::ObjectStoreUrl;
+use datafusion::parquet::arrow::ArrowWriter;
+use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::SessionConfig;
+use datafusion_physical_expr_adapter::{
+ DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
+};
+use datafusion_proto::bytes::{
+ physical_plan_from_bytes_with_proto_converter,
+ physical_plan_to_bytes_with_proto_converter,
+};
+use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;
+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
+use datafusion_proto::physical_plan::{
+ PhysicalExtensionCodec, PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;
+use datafusion_proto::protobuf::{
+ PhysicalExprNode, PhysicalExtensionNode, PhysicalPlanNode,
+};
+use object_store::memory::InMemory;
+use object_store::path::Path;
+use object_store::{ObjectStore, PutPayload};
+use prost::Message;
+use serde::{Deserialize, Serialize};
+
+/// Example showing how to preserve custom adapter information during plan serialization.
+///
+/// This demonstrates:
+/// 1. Creating a custom PhysicalExprAdapter with metadata
+/// 2. Using PhysicalExtensionCodec to intercept serialization
+/// 3. Wrapping adapter info as Extension nodes
+/// 4. Restoring adapters during deserialization
+pub async fn adapter_serialization() -> Result<()> {
+ println!("=== PhysicalExprAdapter Serialization Example ===\n");
+
+ // Step 1: Create sample Parquet data in memory
+ println!("Step 1: Creating sample Parquet data...");
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let batch = record_batch!(("id", Int32, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))?;
+ let path = Path::from("data.parquet");
+ write_parquet(&store, &path, &batch).await?;
+
+ // Step 2: Set up session with custom adapter
+ println!("Step 2: Setting up session with custom adapter...");
+ let logical_schema =
+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+
+ let mut cfg = SessionConfig::new();
+ cfg.options_mut().execution.parquet.pushdown_filters = true;
+ let ctx = SessionContext::new_with_config(cfg);
+ ctx.runtime_env().register_object_store(
+ ObjectStoreUrl::parse("memory://")?.as_ref(),
+ Arc::clone(&store),
+ );
+
+ // Create a table with our custom MetadataAdapterFactory
+ let adapter_factory = Arc::new(MetadataAdapterFactory::new("v1"));
+ let listing_config =
+ ListingTableConfig::new(ListingTableUrl::parse("memory:///data.parquet")?)
+ .infer_options(&ctx.state())
+ .await?
+ .with_schema(logical_schema)
+ .with_expr_adapter_factory(
+ Arc::clone(&adapter_factory) as Arc<dyn PhysicalExprAdapterFactory>
+ );
+ let table = ListingTable::try_new(listing_config)?;
+ ctx.register_table("my_table", Arc::new(table))?;
+
+ // Step 3: Create physical plan with filter
+ println!("Step 3: Creating physical plan with filter...");
+ let df = ctx.sql("SELECT * FROM my_table WHERE id > 5").await?;
+ let original_plan = df.create_physical_plan().await?;
+
+ // Verify adapter is present in original plan
+ let has_adapter_before = verify_adapter_in_plan(&original_plan, "original");
+ println!(" Original plan has adapter: {has_adapter_before}");
+
+ // Step 4: Serialize with our custom codec
+ println!("\nStep 4: Serializing plan with AdapterPreservingCodec...");
+ let codec = AdapterPreservingCodec;
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&original_plan),
+ &codec,
+ &codec,
+ )?;
+ println!(" Serialized {} bytes", bytes.len());
+ println!(" (DataSourceExec with adapter was wrapped as PhysicalExtensionNode)");
+
+ // Step 5: Deserialize with our custom codec
+ println!("\nStep 5: Deserializing plan with AdapterPreservingCodec...");
+ let task_ctx = ctx.task_ctx();
+ let restored_plan =
+ physical_plan_from_bytes_with_proto_converter(&bytes, &task_ctx, &codec, &codec)?;
+
+ // Verify adapter is restored
+ let has_adapter_after = verify_adapter_in_plan(&restored_plan, "restored");
+ println!(" Restored plan has adapter: {has_adapter_after}");
+
+ // Step 6: Execute and compare results
+ println!("\nStep 6: Executing plans and comparing results...");
+ let original_results =
+ datafusion::physical_plan::collect(Arc::clone(&original_plan), task_ctx.clone())
+ .await?;
+ let restored_results =
+ datafusion::physical_plan::collect(restored_plan, task_ctx).await?;
+
+ #[rustfmt::skip]
+ let expected = [
+ "+----+",
+ "| id |",
+ "+----+",
+ "| 6 |",
+ "| 7 |",
+ "| 8 |",
+ "| 9 |",
+ "| 10 |",
+ "+----+",
+ ];
+
+ println!("\n Original plan results:");
+ arrow::util::pretty::print_batches(&original_results)?;
+ assert_batches_eq!(expected, &original_results);
+
+ println!("\n Restored plan results:");
+ arrow::util::pretty::print_batches(&restored_results)?;
+ assert_batches_eq!(expected, &restored_results);
+
+ println!("\n=== Example Complete! ===");
+ println!("Key takeaways:");
+ println!(
+ " 1. PhysicalExtensionCodec provides serialize_physical_plan/deserialize_physical_plan hooks"
+ );
+ println!(" 2. Custom metadata can be wrapped as PhysicalExtensionNode");
+ println!(" 3. Nested serialization (protobuf + JSON) works seamlessly");
+ println!(
+ " 4. Both plans produce identical results despite serialization round-trip"
+ );
+ println!(" 5. Adapters are fully preserved through the serialization round-trip");
+
+ Ok(())
+}
+
+// ============================================================================
+// MetadataAdapter - A simple custom adapter with a tag
+// ============================================================================
+
+/// A custom PhysicalExprAdapter that wraps another adapter.
+/// The tag metadata is stored in the factory, not the adapter itself.
+#[derive(Debug)]
+struct MetadataAdapter {
+ inner: Arc<dyn PhysicalExprAdapter>,
+}
+
+impl PhysicalExprAdapter for MetadataAdapter {
+ fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {
+ // Simply delegate to inner adapter
+ self.inner.rewrite(expr)
+ }
+}
+
+// ============================================================================
+// MetadataAdapterFactory - Factory for creating MetadataAdapter instances
+// ============================================================================
+
+/// Factory for creating MetadataAdapter instances.
+/// The tag is stored in the factory and extracted via Debug formatting in `extract_adapter_tag`.
+#[derive(Debug)]
+struct MetadataAdapterFactory {
+ // Note: This field is read via Debug formatting in `extract_adapter_tag`.
+ // Rust's dead code analysis doesn't recognize Debug-based field access.
+ // In PR #19234, this field is used by `with_partition_values`, but that method
+ // doesn't exist in upstream DataFusion's PhysicalExprAdapter trait.
+ #[allow(dead_code)]
+ tag: String,
+}
+
+impl MetadataAdapterFactory {
+ fn new(tag: impl Into<String>) -> Self {
+ Self { tag: tag.into() }
+ }
+}
+
+impl PhysicalExprAdapterFactory for MetadataAdapterFactory {
+ fn create(
+ &self,
+ logical_file_schema: SchemaRef,
+ physical_file_schema: SchemaRef,
+ ) -> Arc<dyn PhysicalExprAdapter> {
+ let inner = DefaultPhysicalExprAdapterFactory
+ .create(logical_file_schema, physical_file_schema);
+ Arc::new(MetadataAdapter { inner })
+ }
+}
+
+// ============================================================================
+// AdapterPreservingCodec - Custom codec that preserves adapters
+// ============================================================================
+
+/// Extension payload structure for serializing adapter info
+#[derive(Serialize, Deserialize)]
+struct ExtensionPayload {
+ /// Marker to identify this is our custom extension
+ marker: String,
+ /// JSON-serialized adapter metadata
+ adapter_metadata: AdapterMetadata,
+ /// Protobuf-serialized inner DataSourceExec (without adapter)
+ inner_plan_bytes: Vec<u8>,
+}
+
+/// Metadata about the adapter to recreate it during deserialization
+#[derive(Serialize, Deserialize)]
+struct AdapterMetadata {
+ /// The adapter tag (e.g., "v1")
+ tag: String,
+}
+
+const EXTENSION_MARKER: &str = "adapter_preserving_extension_v1";
+
+/// A codec that intercepts serialization to preserve adapter information.
+#[derive(Debug)]
+struct AdapterPreservingCodec;
+
+impl PhysicalExtensionCodec for AdapterPreservingCodec {
+ // Required method: decode custom extension nodes
+ fn try_decode(
+ &self,
+ buf: &[u8],
+ _inputs: &[Arc<dyn ExecutionPlan>],
+ ctx: &TaskContext,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ // Try to parse as our extension payload
+ if let Ok(payload) = serde_json::from_slice::<ExtensionPayload>(buf)
+ && payload.marker == EXTENSION_MARKER
+ {
+ // Decode the inner plan
+ let inner_proto = PhysicalPlanNode::decode(&payload.inner_plan_bytes[..])
+ .map_err(|e| {
+ datafusion::error::DataFusionError::Plan(format!(
+ "Failed to decode inner plan: {e}"
+ ))
+ })?;
+
+ // Deserialize the inner plan using default implementation
+ let inner_plan =
+ inner_proto.try_into_physical_plan_with_converter(ctx, self, self)?;
+
+ // Recreate the adapter factory
+ let adapter_factory = create_adapter_factory(&payload.adapter_metadata.tag);
+
+ // Inject adapter into the plan
+ return inject_adapter_into_plan(inner_plan, adapter_factory);
+ }
+
+ not_impl_err!("Unknown extension type")
+ }
+
+ // Required method: encode custom execution plans
+ fn try_encode(
+ &self,
+ _node: Arc<dyn ExecutionPlan>,
+ _buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ // We don't need this for the example - we use serialize_physical_plan instead
+ not_impl_err!(
+ "try_encode not used - adapter wrapping happens in serialize_physical_plan"
+ )
+ }
+}
+
+impl PhysicalProtoConverterExtension for AdapterPreservingCodec {
+ fn execution_plan_to_proto(
+ &self,
+ plan: &Arc<dyn ExecutionPlan>,
+ extension_codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<PhysicalPlanNode> {
+ // Check if this is a DataSourceExec with adapter
+ if let Some(exec) = plan.as_any().downcast_ref::<DataSourceExec>()
+ && let Some(config) =
+ exec.data_source().as_any().downcast_ref::<FileScanConfig>()
+ && let Some(adapter_factory) = &config.expr_adapter_factory
+ && let Some(tag) = extract_adapter_tag(adapter_factory.as_ref())
+ {
+ // Try to extract our MetadataAdapterFactory's tag
+ println!(" [Serialize] Found DataSourceExec with adapter tag: {tag}");
+
+ // 1. Create adapter metadata
+ let adapter_metadata = AdapterMetadata { tag };
+
+ // 2. Create a copy of the config without the adapter
+ let config_without_adapter = rebuild_config_without_adapter(config);
+
+ // 3. Create a new DataSourceExec without adapter
+ let plan_without_adapter: Arc<dyn ExecutionPlan> =
+ DataSourceExec::from_data_source(config_without_adapter);
+
+ // 4. Serialize the inner plan to protobuf bytes
+ let inner_proto = PhysicalPlanNode::try_from_physical_plan_with_converter(
+ plan_without_adapter,
+ extension_codec,
+ self,
+ )?;
+
+ let mut inner_bytes = Vec::new();
+ inner_proto.encode(&mut inner_bytes).map_err(|e| {
+ datafusion::error::DataFusionError::Plan(format!(
+ "Failed to encode inner plan: {e}"
+ ))
+ })?;
+
+ // 5. Create extension payload
+ let payload = ExtensionPayload {
+ marker: EXTENSION_MARKER.to_string(),
+ adapter_metadata,
+ inner_plan_bytes: inner_bytes,
+ };
+ let payload_bytes = serde_json::to_vec(&payload).map_err(|e| {
+ datafusion::error::DataFusionError::Plan(format!(
+ "Failed to serialize payload: {e}"
+ ))
+ })?;
+
+ // 6. Return as PhysicalExtensionNode
+ return Ok(PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::Extension(
+ PhysicalExtensionNode {
+ node: payload_bytes,
+ inputs: vec![], // Leaf node
+ },
+ )),
+ });
+ }
+
+ // No adapter found - use default serialization
+ PhysicalPlanNode::try_from_physical_plan_with_converter(
+ Arc::clone(plan),
+ extension_codec,
+ self,
+ )
+ }
+
+ // Interception point: override deserialization to unwrap adapters
+ fn proto_to_execution_plan(
+ &self,
+ ctx: &TaskContext,
+ extension_codec: &dyn PhysicalExtensionCodec,
+ proto: &PhysicalPlanNode,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ // Check if this is our custom extension wrapper
+ if let Some(PhysicalPlanType::Extension(extension)) = &proto.physical_plan_type
+ && let Ok(payload) =
+ serde_json::from_slice::<ExtensionPayload>(&extension.node)
+ && payload.marker == EXTENSION_MARKER
+ {
+ println!(
+ " [Deserialize] Found adapter extension with tag: {}",
+ payload.adapter_metadata.tag
+ );
+
+ // Decode the inner plan
+ let inner_proto = PhysicalPlanNode::decode(&payload.inner_plan_bytes[..])
+ .map_err(|e| {
+ datafusion::error::DataFusionError::Plan(format!(
+ "Failed to decode inner plan: {e}"
+ ))
+ })?;
+
+ // Deserialize the inner plan using default implementation
+ let inner_plan = inner_proto.try_into_physical_plan_with_converter(
+ ctx,
+ extension_codec,
+ self,
+ )?;
+
+ // Recreate the adapter factory
+ let adapter_factory = create_adapter_factory(&payload.adapter_metadata.tag);
+
+ // Inject adapter into the plan
+ return inject_adapter_into_plan(inner_plan, adapter_factory);
+ }
+
+ // Not our extension - use default deserialization
+ proto.try_into_physical_plan_with_converter(ctx, extension_codec, self)
+ }
+
+ fn proto_to_physical_expr(
+ &self,
+ proto: &PhysicalExprNode,
+ ctx: &TaskContext,
+ input_schema: &Schema,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)
+ }
+
+ fn physical_expr_to_proto(
+ &self,
+ expr: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<PhysicalExprNode> {
+ serialize_physical_expr_with_converter(expr, codec, self)
+ }
+}
+
+// ============================================================================
+// Helper functions
+// ============================================================================
+
+/// Write a RecordBatch to Parquet in the object store
+async fn write_parquet(
+ store: &dyn ObjectStore,
+ path: &Path,
+ batch: &arrow::record_batch::RecordBatch,
+) -> Result<()> {
+ let mut buf = vec![];
+ let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), None)?;
+ writer.write(batch)?;
+ writer.close()?;
+
+ let payload = PutPayload::from_bytes(buf.into());
+ store.put(path, payload).await?;
+ Ok(())
+}
+
+/// Extract the tag from a MetadataAdapterFactory.
+///
+/// Note: Since `PhysicalExprAdapterFactory` doesn't provide `as_any()` for downcasting,
+/// we parse the Debug output. In a production system, you might add a dedicated trait
+/// method for metadata extraction.
+fn extract_adapter_tag(factory: &dyn PhysicalExprAdapterFactory) -> Option<String> {
+ let debug_str = format!("{factory:?}");
+ if debug_str.contains("MetadataAdapterFactory") {
+ // Extract tag from debug output: MetadataAdapterFactory { tag: "v1" }
+ if let Some(start) = debug_str.find("tag: \"") {
+ let after_tag = &debug_str[start + 6..];
+ if let Some(end) = after_tag.find('"') {
+ return Some(after_tag[..end].to_string());
+ }
+ }
+ }
+ None
+}
+
+/// Create an adapter factory from a tag
+fn create_adapter_factory(tag: &str) -> Arc<dyn PhysicalExprAdapterFactory> {
+ Arc::new(MetadataAdapterFactory::new(tag))
+}
+
+/// Rebuild a FileScanConfig without the adapter
+fn rebuild_config_without_adapter(config: &FileScanConfig) -> FileScanConfig {
+ FileScanConfigBuilder::from(config.clone())
+ .with_expr_adapter(None)
+ .build()
+}
+
+/// Inject an adapter into a plan (assumes plan is a DataSourceExec with FileScanConfig)
+fn inject_adapter_into_plan(
+ plan: Arc<dyn ExecutionPlan>,
+ adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
+) -> Result<Arc<dyn ExecutionPlan>> {
+ if let Some(exec) = plan.as_any().downcast_ref::<DataSourceExec>()
+ && let Some(config) = exec.data_source().as_any().downcast_ref::<FileScanConfig>()
+ {
+ let new_config = FileScanConfigBuilder::from(config.clone())
+ .with_expr_adapter(Some(adapter_factory))
+ .build();
+ return Ok(DataSourceExec::from_data_source(new_config));
+ }
+ // If not a DataSourceExec with FileScanConfig, return as-is
+ Ok(plan)
+}
+
+/// Helper to verify if a plan has an adapter (for testing/validation)
+fn verify_adapter_in_plan(plan: &Arc<dyn ExecutionPlan>, label: &str) -> bool {
+ // Walk the plan tree to find DataSourceExec with adapter
+ fn check_plan(plan: &dyn ExecutionPlan) -> bool {
+ if let Some(exec) = plan.as_any().downcast_ref::<DataSourceExec>()
+ && let Some(config) =
+ exec.data_source().as_any().downcast_ref::<FileScanConfig>()
+ && config.expr_adapter_factory.is_some()
+ {
+ return true;
+ }
+ // Check children
+ for child in plan.children() {
+ if check_plan(child.as_ref()) {
+ return true;
+ }
+ }
+ false
+ }
+
+ let has_adapter = check_plan(plan.as_ref());
+ println!(" [Verify] {label} plan adapter check: {has_adapter}");
+ has_adapter
+}
diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rs
index 5846626d81..73e65182a0 100644
--- a/datafusion-examples/examples/custom_data_source/main.rs
+++ b/datafusion-examples/examples/custom_data_source/main.rs
@@ -26,6 +26,7 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
+//! - `adapter_serialization` — preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception
//! - `csv_json_opener` — use low level FileOpener APIs to read CSV/JSON into Arrow RecordBatches
//! - `csv_sql_streaming` — build and run a streaming query plan from a SQL statement against a local CSV file
//! - `custom_datasource` — run queries against a custom datasource (TableProvider)
@@ -34,6 +35,7 @@
//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter
//! - `file_stream_provider` — run a query on FileStreamProvider which implements StreamProvider for reading and writing to arbitrary stream sources/sinks
+mod adapter_serialization;
mod csv_json_opener;
mod csv_sql_streaming;
mod custom_datasource;
@@ -50,6 +52,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};
#[strum(serialize_all = "snake_case")]
enum ExampleKind {
All,
+ AdapterSerialization,
CsvJsonOpener,
CsvSqlStreaming,
CustomDatasource,
@@ -74,6 +77,9 @@ impl ExampleKind {
Box::pin(example.run()).await?;
}
}
+ ExampleKind::AdapterSerialization => {
+ adapter_serialization::adapter_serialization().await?
+ }
ExampleKind::CsvJsonOpener => csv_json_opener::csv_json_opener().await?,
ExampleKind::CsvSqlStreaming => {
csv_sql_streaming::csv_sql_streaming().await?
diff --git a/datafusion-examples/examples/proto/expression_deduplication.rs b/datafusion-examples/examples/proto/expression_deduplication.rs
new file mode 100644
index 0000000000..0dec807f80
--- /dev/null
+++ b/datafusion-examples/examples/proto/expression_deduplication.rs
@@ -0,0 +1,275 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! See `main.rs` for how to run it.
+//!
+//! This example demonstrates how to use the `PhysicalExtensionCodec` trait's
+//! interception methods to implement expression deduplication during deserialization.
+//!
+//! This pattern is inspired by PR #18192, which introduces expression caching
+//! to reduce memory usage when deserializing plans with duplicate expressions.
+//!
+//! The key insight is that identical expressions serialize to identical protobuf bytes.
+//! By caching deserialized expressions keyed by their protobuf bytes, we can:
+//! 1. Return the same Arc for duplicate expressions
+//! 2. Reduce memory allocation during deserialization
+//! 3. Enable downstream optimizations that rely on Arc pointer equality
+//!
+//! This demonstrates the decorator pattern enabled by the `PhysicalExtensionCodec` trait,
+//! where all expression serialization/deserialization routes through the codec methods.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::sync::{Arc, RwLock};
+
+use arrow::datatypes::{DataType, Field, Schema};
+use datafusion::common::Result;
+use datafusion::execution::TaskContext;
+use datafusion::logical_expr::Operator;
+use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::expressions::{BinaryExpr, col};
+use datafusion::physical_plan::filter::FilterExec;
+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
+use datafusion::prelude::SessionContext;
+use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;
+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};
+use prost::Message;
+
+/// Example showing how to implement expression deduplication using the codec decorator pattern.
+///
+/// This demonstrates:
+/// 1. Creating a CachingCodec that caches expressions by their protobuf bytes
+/// 2. Intercepting deserialization to return cached Arcs for duplicate expressions
+/// 3. Verifying that duplicate expressions share the same Arc after deserialization
+///
+/// Deduplication is keyed by the protobuf bytes representing the expression,
+/// in reality deduplication could be done based on e.g. the pointer address of the
+/// serialized expression in memory, but this is simpler to demonstrate.
+///
+/// In this case our expression is trivial and just for demonstration purposes.
+/// In real scenarios, expressions can be much more complex, e.g. a large InList
+/// expression could be megabytes in size, so deduplication can save significant memory
+/// in addition to more correctly representing the original plan structure.
+pub async fn expression_deduplication() -> Result<()> {
+ println!("=== Expression Deduplication Example ===\n");
+
+ // Create a schema for our test expressions
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));
+
+ // Step 1: Create expressions with duplicates
+ println!("Step 1: Creating expressions with duplicates...");
+
+ // Create expression: col("a")
+ let a = col("a", &schema)?;
+
+ // Create a clone to show duplicates
+ let a_clone = Arc::clone(&a);
+
+ // Combine: a OR a_clone
+ let combined_expr =
+ Arc::new(BinaryExpr::new(a, Operator::Or, a_clone)) as Arc<dyn PhysicalExpr>;
+ println!(" Created expression: a OR a with duplicates");
+ println!(" Note: a appears twice in the expression tree\n");
+ // Step 2: Create a filter plan with this expression
+ println!("Step 2: Creating physical plan with the expression...");
+
+ let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema)));
+ let filter_plan: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(combined_expr, input)?);
+
+ println!(" Created FilterExec with duplicate sub-expressions\n");
+
+ // Step 3: Serialize with the caching codec
+ println!("Step 3: Serializing plan...");
+
+ let extension_codec = DefaultPhysicalExtensionCodec {};
+ let caching_converter = CachingCodec::new();
+ let proto =
+ caching_converter.execution_plan_to_proto(&filter_plan, &extension_codec)?;
+
+ // Serialize to bytes
+ let mut bytes = Vec::new();
+ proto.encode(&mut bytes).unwrap();
+ println!(" Serialized plan to {} bytes\n", bytes.len());
+
+ // Step 4: Deserialize with the caching codec
+ println!("Step 4: Deserializing plan with CachingCodec...");
+
+ let ctx = SessionContext::new();
+ let deserialized_plan = proto.try_into_physical_plan_with_converter(
+ &ctx.task_ctx(),
+ &extension_codec,
+ &caching_converter,
+ )?;
+
+ // Step 5: check that we deduplicated expressions
+ println!("Step 5: Checking for deduplicated expressions...");
+ let Some(filter_exec) = deserialized_plan.as_any().downcast_ref::<FilterExec>()
+ else {
+ panic!("Deserialized plan is not a FilterExec");
+ };
+ let predicate = Arc::clone(filter_exec.predicate());
+ let binary_expr = predicate
+ .as_any()
+ .downcast_ref::<BinaryExpr>()
+ .expect("Predicate is not a BinaryExpr");
+ let left = &binary_expr.left();
+ let right = &binary_expr.right();
+ // Check if left and right point to the same Arc
+ let deduplicated = Arc::ptr_eq(left, right);
+ if deduplicated {
+ println!(" Success: Duplicate expressions were deduplicated!");
+ println!(
+ " Cache Stats: hits={}, misses={}",
+ caching_converter.stats.read().unwrap().cache_hits,
+ caching_converter.stats.read().unwrap().cache_misses,
+ );
+ } else {
+ println!(" Failure: Duplicate expressions were NOT deduplicated.");
+ }
+
+ Ok(())
+}
+
+// ============================================================================
+// CachingCodec - Implements expression deduplication
+// ============================================================================
+
+/// Statistics for cache performance monitoring
+#[derive(Debug, Default)]
+struct CacheStats {
+ cache_hits: usize,
+ cache_misses: usize,
+}
+
+/// A codec that caches deserialized expressions to enable deduplication.
+///
+/// When deserializing, if we've already seen the same protobuf bytes,
+/// we return the cached Arc instead of creating a new allocation.
+#[derive(Debug, Default)]
+struct CachingCodec {
+ /// Cache mapping protobuf bytes -> deserialized expression
+ expr_cache: RwLock<HashMap<Vec<u8>, Arc<dyn PhysicalExpr>>>,
+ /// Statistics for demonstration
+ stats: RwLock<CacheStats>,
+}
+
+impl CachingCodec {
+ fn new() -> Self {
+ Self::default()
+ }
+}
+
+impl PhysicalExtensionCodec for CachingCodec {
+ // Required: decode custom extension nodes
+ fn try_decode(
+ &self,
+ _buf: &[u8],
+ _inputs: &[Arc<dyn ExecutionPlan>],
+ _ctx: &TaskContext,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ datafusion::common::not_impl_err!("No custom extension nodes")
+ }
+
+ // Required: encode custom execution plans
+ fn try_encode(
+ &self,
+ _node: Arc<dyn ExecutionPlan>,
+ _buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ datafusion::common::not_impl_err!("No custom extension nodes")
+ }
+}
+
+impl PhysicalProtoConverterExtension for CachingCodec {
+ fn proto_to_execution_plan(
+ &self,
+ ctx: &TaskContext,
+ extension_codec: &dyn PhysicalExtensionCodec,
+ proto: &PhysicalPlanNode,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ proto.try_into_physical_plan_with_converter(ctx, extension_codec, self)
+ }
+
+ fn execution_plan_to_proto(
+ &self,
+ plan: &Arc<dyn ExecutionPlan>,
+ extension_codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<PhysicalPlanNode> {
+ PhysicalPlanNode::try_from_physical_plan_with_converter(
+ Arc::clone(plan),
+ extension_codec,
+ self,
+ )
+ }
+
+ // CACHING IMPLEMENTATION: Intercept expression deserialization
+ fn proto_to_physical_expr(
+ &self,
+ proto: &PhysicalExprNode,
+ ctx: &TaskContext,
+ input_schema: &Schema,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ // Create cache key from protobuf bytes
+ let mut key = Vec::new();
+ proto.encode(&mut key).map_err(|e| {
+ datafusion::error::DataFusionError::Internal(format!(
+ "Failed to encode proto for cache key: {e}"
+ ))
+ })?;
+
+ // Check cache first
+ {
+ let cache = self.expr_cache.read().unwrap();
+ if let Some(cached) = cache.get(&key) {
+ // Cache hit! Update stats and return cached Arc
+ let mut stats = self.stats.write().unwrap();
+ stats.cache_hits += 1;
+ return Ok(Arc::clone(cached));
+ }
+ }
+
+ // Cache miss - deserialize and store
+ let expr =
+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)?;
+
+ // Store in cache
+ {
+ let mut cache = self.expr_cache.write().unwrap();
+ cache.insert(key, Arc::clone(&expr));
+ let mut stats = self.stats.write().unwrap();
+ stats.cache_misses += 1;
+ }
+
+ Ok(expr)
+ }
+
+ fn physical_expr_to_proto(
+ &self,
+ expr: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<PhysicalExprNode> {
+ serialize_physical_expr_with_converter(expr, codec, self)
+ }
+}
diff --git a/datafusion-examples/examples/proto/main.rs b/datafusion-examples/examples/proto/main.rs
index f56078b319..16fcd70f34 100644
--- a/datafusion-examples/examples/proto/main.rs
+++ b/datafusion-examples/examples/proto/main.rs
@@ -27,8 +27,10 @@
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
//! - `composed_extension_codec` — example of using multiple extension codecs for serialization / deserialization
+//! - `expression_deduplication` — example of expression caching/deduplication using the codec decorator pattern
mod composed_extension_codec;
+mod expression_deduplication;
use datafusion::error::{DataFusionError, Result};
use strum::{IntoEnumIterator, VariantNames};
@@ -39,6 +41,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};
enum ExampleKind {
All,
ComposedExtensionCodec,
+ ExpressionDeduplication,
}
impl ExampleKind {
@@ -59,6 +62,9 @@ impl ExampleKind {
ExampleKind::ComposedExtensionCodec => {
composed_extension_codec::composed_extension_codec().await?
}
+ ExampleKind::ExpressionDeduplication => {
+ expression_deduplication::expression_deduplication().await?
+ }
}
Ok(())
}
diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs
index d95bdd3886..84b15ea9a8 100644
--- a/datafusion/proto/src/bytes/mod.rs
+++ b/datafusion/proto/src/bytes/mod.rs
@@ -21,7 +21,8 @@ use crate::logical_plan::{
self, AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec,
};
use crate::physical_plan::{
- AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,
+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
};
use crate::protobuf;
use datafusion_common::{Result, plan_datafusion_err};
@@ -276,16 +277,18 @@ pub fn logical_plan_from_json_with_extension_codec(
/// Serialize a PhysicalPlan as bytes
pub fn physical_plan_to_bytes(plan: Arc<dyn ExecutionPlan>) -> Result<Bytes> {
let extension_codec = DefaultPhysicalExtensionCodec {};
- physical_plan_to_bytes_with_extension_codec(plan, &extension_codec)
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ physical_plan_to_bytes_with_proto_converter(plan, &extension_codec, &proto_converter)
}
/// Serialize a PhysicalPlan as JSON
#[cfg(feature = "json")]
pub fn physical_plan_to_json(plan: Arc<dyn ExecutionPlan>) -> Result<String> {
let extension_codec = DefaultPhysicalExtensionCodec {};
- let protobuf =
- protobuf::PhysicalPlanNode::try_from_physical_plan(plan, &extension_codec)
- .map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ let protobuf = proto_converter
+ .execution_plan_to_proto(&plan, &extension_codec)
+ .map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;
serde_json::to_string(&protobuf)
.map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))
}
@@ -295,8 +298,18 @@ pub fn physical_plan_to_bytes_with_extension_codec(
plan: Arc<dyn ExecutionPlan>,
extension_codec: &dyn PhysicalExtensionCodec,
) -> Result<Bytes> {
- let protobuf =
- protobuf::PhysicalPlanNode::try_from_physical_plan(plan, extension_codec)?;
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ physical_plan_to_bytes_with_proto_converter(plan, extension_codec, &proto_converter)
+}
+
+/// Serialize a PhysicalPlan as bytes, using the provided extension codec
+/// and protobuf converter.
+pub fn physical_plan_to_bytes_with_proto_converter(
+ plan: Arc<dyn ExecutionPlan>,
+ extension_codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+) -> Result<Bytes> {
+ let protobuf = proto_converter.execution_plan_to_proto(&plan, extension_codec)?;
let mut buffer = BytesMut::new();
protobuf
.encode(&mut buffer)
@@ -313,7 +326,8 @@ pub fn physical_plan_from_json(
let back: protobuf::PhysicalPlanNode = serde_json::from_str(json)
.map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;
let extension_codec = DefaultPhysicalExtensionCodec {};
- back.try_into_physical_plan(ctx, &extension_codec)
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ proto_converter.proto_to_execution_plan(ctx, &extension_codec, &back)
}
/// Deserialize a PhysicalPlan from bytes
@@ -322,7 +336,13 @@ pub fn physical_plan_from_bytes(
ctx: &TaskContext,
) -> Result<Arc<dyn ExecutionPlan>> {
let extension_codec = DefaultPhysicalExtensionCodec {};
- physical_plan_from_bytes_with_extension_codec(bytes, ctx, &extension_codec)
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ physical_plan_from_bytes_with_proto_converter(
+ bytes,
+ ctx,
+ &extension_codec,
+ &proto_converter,
+ )
}
/// Deserialize a PhysicalPlan from bytes
@@ -330,8 +350,24 @@ pub fn physical_plan_from_bytes_with_extension_codec(
bytes: &[u8],
ctx: &TaskContext,
extension_codec: &dyn PhysicalExtensionCodec,
+) -> Result<Arc<dyn ExecutionPlan>> {
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ physical_plan_from_bytes_with_proto_converter(
+ bytes,
+ ctx,
+ extension_codec,
+ &proto_converter,
+ )
+}
+
+/// Deserialize a PhysicalPlan from bytes
+pub fn physical_plan_from_bytes_with_proto_converter(
+ bytes: &[u8],
+ ctx: &TaskContext,
+ extension_codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let protobuf = protobuf::PhysicalPlanNode::decode(bytes)
.map_err(|e| plan_datafusion_err!("Error decoding expr as protobuf: {e}"))?;
- protobuf.try_into_physical_plan(ctx, extension_codec)
+ proto_converter.proto_to_execution_plan(ctx, extension_codec, &protobuf)
}
diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs
index 3cfc796700..fc8eba12c5 100644
--- a/datafusion/proto/src/physical_plan/from_proto.rs
+++ b/datafusion/proto/src/physical_plan/from_proto.rs
@@ -21,14 +21,9 @@ use std::sync::Arc;
use arrow::array::RecordBatch;
use arrow::compute::SortOptions;
-use arrow::datatypes::Field;
+use arrow::datatypes::{Field, Schema};
use arrow::ipc::reader::StreamReader;
use chrono::{TimeZone, Utc};
-use datafusion_expr::dml::InsertOp;
-use object_store::ObjectMeta;
-use object_store::path::Path;
-
-use arrow::datatypes::Schema;
use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_groups::FileGroup;
@@ -42,6 +37,7 @@ use datafusion_datasource_parquet::file_format::ParquetSink;
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_execution::{FunctionRegistry, TaskContext};
use datafusion_expr::WindowFunctionDefinition;
+use datafusion_expr::dml::InsertOp;
use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};
use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};
use datafusion_physical_plan::expressions::{
@@ -52,13 +48,16 @@ use datafusion_physical_plan::joins::{HashExpr, SeededRandomState};
use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};
use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};
use datafusion_proto_common::common::proto_error;
+use object_store::ObjectMeta;
+use object_store::path::Path;
-use crate::convert_required;
+use super::{
+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
+};
use crate::logical_plan::{self};
-use crate::protobuf;
use crate::protobuf::physical_expr_node::ExprType;
-
-use super::PhysicalExtensionCodec;
+use crate::{convert_required, protobuf};
impl From<&protobuf::PhysicalColumn> for Column {
fn from(c: &protobuf::PhysicalColumn) -> Column {
@@ -80,9 +79,15 @@ pub fn parse_physical_sort_expr(
ctx: &TaskContext,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<PhysicalSortExpr> {
if let Some(expr) = &proto.expr {
- let expr = parse_physical_expr(expr.as_ref(), ctx, input_schema, codec)?;
+ let expr = proto_converter.proto_to_physical_expr(
+ expr.as_ref(),
+ ctx,
+ input_schema,
+ codec,
+ )?;
let options = SortOptions {
descending: !proto.asc,
nulls_first: proto.nulls_first,
@@ -107,10 +112,13 @@ pub fn parse_physical_sort_exprs(
ctx: &TaskContext,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Vec<PhysicalSortExpr>> {
proto
.iter()
- .map(|sort_expr| parse_physical_sort_expr(sort_expr, ctx, input_schema, codec))
+ .map(|sort_expr| {
+ parse_physical_sort_expr(sort_expr, ctx, input_schema, codec, proto_converter)
+ })
.collect()
}
@@ -129,12 +137,25 @@ pub fn parse_physical_window_expr(
ctx: &TaskContext,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn WindowExpr>> {
- let window_node_expr = parse_physical_exprs(&proto.args, ctx, input_schema, codec)?;
- let partition_by =
- parse_physical_exprs(&proto.partition_by, ctx, input_schema, codec)?;
-
- let order_by = parse_physical_sort_exprs(&proto.order_by, ctx, input_schema, codec)?;
+ let window_node_expr =
+ parse_physical_exprs(&proto.args, ctx, input_schema, codec, proto_converter)?;
+ let partition_by = parse_physical_exprs(
+ &proto.partition_by,
+ ctx,
+ input_schema,
+ codec,
+ proto_converter,
+ )?;
+
+ let order_by = parse_physical_sort_exprs(
+ &proto.order_by,
+ ctx,
+ input_schema,
+ codec,
+ proto_converter,
+ )?;
let window_frame = proto
.window_frame
@@ -188,13 +209,14 @@ pub fn parse_physical_exprs<'a, I>(
ctx: &TaskContext,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Vec<Arc<dyn PhysicalExpr>>>
where
I: IntoIterator<Item = &'a protobuf::PhysicalExprNode>,
{
protos
.into_iter()
- .map(|p| parse_physical_expr(p, ctx, input_schema, codec))
+ .map(|p| proto_converter.proto_to_physical_expr(p, ctx, input_schema, codec))
.collect::<Result<Vec<_>>>()
}
@@ -212,6 +234,32 @@ pub fn parse_physical_expr(
ctx: &TaskContext,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ parse_physical_expr_with_converter(
+ proto,
+ ctx,
+ input_schema,
+ codec,
+ &DefaultPhysicalProtoConverter {},
+ )
+}
+
+/// Parses a physical expression from a protobuf.
+///
+/// # Arguments
+///
+/// * `proto` - Input proto with physical expression node
+/// * `registry` - A registry knows how to build logical expressions out of user-defined function names
+/// * `input_schema` - The Arrow schema for the input, used for determining expression data types
+/// when performing type coercion.
+/// * `codec` - An extension codec used to decode custom UDFs.
+/// * `proto_converter` - Conversion functions for physical plans and expressions
+pub fn parse_physical_expr_with_converter(
+ proto: &protobuf::PhysicalExprNode,
+ ctx: &TaskContext,
+ input_schema: &Schema,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn PhysicalExpr>> {
let expr_type = proto
.expr_type
@@ -232,6 +280,7 @@ pub fn parse_physical_expr(
"left",
input_schema,
codec,
+ proto_converter,
)?,
logical_plan::from_proto::from_proto_binary_op(&binary_expr.op)?,
parse_required_physical_expr(
@@ -240,6 +289,7 @@ pub fn parse_physical_expr(
"right",
input_schema,
codec,
+ proto_converter,
)?,
)),
ExprType::AggregateExpr(_) => {
@@ -262,6 +312,7 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?))
}
ExprType::IsNotNullExpr(e) => {
@@ -271,6 +322,7 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?))
}
ExprType::NotExpr(e) => Arc::new(NotExpr::new(parse_required_physical_expr(
@@ -279,6 +331,7 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?)),
ExprType::Negative(e) => {
Arc::new(NegativeExpr::new(parse_required_physical_expr(
@@ -287,6 +340,7 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?))
}
ExprType::InList(e) => in_list(
@@ -296,15 +350,23 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?,
- parse_physical_exprs(&e.list, ctx, input_schema, codec)?,
+ parse_physical_exprs(&e.list, ctx, input_schema, codec, proto_converter)?,
&e.negated,
input_schema,
)?,
ExprType::Case(e) => Arc::new(CaseExpr::try_new(
e.expr
.as_ref()
- .map(|e| parse_physical_expr(e.as_ref(), ctx, input_schema, codec))
+ .map(|e| {
+ proto_converter.proto_to_physical_expr(
+ e.as_ref(),
+ ctx,
+ input_schema,
+ codec,
+ )
+ })
.transpose()?,
e.when_then_expr
.iter()
@@ -316,6 +378,7 @@ pub fn parse_physical_expr(
"when_expr",
input_schema,
codec,
+ proto_converter,
)?,
parse_required_physical_expr(
e.then_expr.as_ref(),
@@ -323,13 +386,21 @@ pub fn parse_physical_expr(
"then_expr",
input_schema,
codec,
+ proto_converter,
)?,
))
})
.collect::<Result<Vec<_>>>()?,
e.else_expr
.as_ref()
- .map(|e| parse_physical_expr(e.as_ref(), ctx, input_schema, codec))
+ .map(|e| {
+ proto_converter.proto_to_physical_expr(
+ e.as_ref(),
+ ctx,
+ input_schema,
+ codec,
+ )
+ })
.transpose()?,
)?),
ExprType::Cast(e) => Arc::new(CastExpr::new(
@@ -339,6 +410,7 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?,
convert_required!(e.arrow_type)?,
None,
@@ -350,6 +422,7 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?,
convert_required!(e.arrow_type)?,
)),
@@ -362,7 +435,8 @@ pub fn parse_physical_expr(
};
let scalar_fun_def = Arc::clone(&udf);
- let args = parse_physical_exprs(&e.args, ctx, input_schema, codec)?;
+ let args =
+ parse_physical_exprs(&e.args, ctx, input_schema, codec, proto_converter)?;
let config_options = Arc::clone(ctx.session_config().options());
@@ -391,6 +465,7 @@ pub fn parse_physical_expr(
"expr",
input_schema,
codec,
+ proto_converter,
)?,
parse_required_physical_expr(
like_expr.pattern.as_deref(),
@@ -398,11 +473,17 @@ pub fn parse_physical_expr(
"pattern",
input_schema,
codec,
+ proto_converter,
)?,
)),
ExprType::HashExpr(hash_expr) => {
- let on_columns =
- parse_physical_exprs(&hash_expr.on_columns, ctx, input_schema, codec)?;
+ let on_columns = parse_physical_exprs(
+ &hash_expr.on_columns,
+ ctx,
+ input_schema,
+ codec,
+ proto_converter,
+ )?;
Arc::new(HashExpr::new(
on_columns,
SeededRandomState::with_seeds(
@@ -418,9 +499,11 @@ pub fn parse_physical_expr(
let inputs: Vec<Arc<dyn PhysicalExpr>> = extension
.inputs
.iter()
- .map(|e| parse_physical_expr(e, ctx, input_schema, codec))
+ .map(|e| {
+ proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec)
+ })
.collect::<Result<_>>()?;
- (codec.try_decode_expr(extension.expr.as_slice(), &inputs)?) as _
+ codec.try_decode_expr(extension.expr.as_slice(), &inputs)? as _
}
};
@@ -433,8 +516,9 @@ fn parse_required_physical_expr(
field: &str,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn PhysicalExpr>> {
- expr.map(|e| parse_physical_expr(e, ctx, input_schema, codec))
+ expr.map(|e| proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec))
.transpose()?
.ok_or_else(|| internal_datafusion_err!("Missing required field {field:?}"))
}
@@ -444,11 +528,17 @@ pub fn parse_protobuf_hash_partitioning(
ctx: &TaskContext,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Option<Partitioning>> {
match partitioning {
Some(hash_part) => {
- let expr =
- parse_physical_exprs(&hash_part.hash_expr, ctx, input_schema, codec)?;
+ let expr = parse_physical_exprs(
+ &hash_part.hash_expr,
+ ctx,
+ input_schema,
+ codec,
+ proto_converter,
+ )?;
Ok(Some(Partitioning::Hash(
expr,
@@ -464,6 +554,7 @@ pub fn parse_protobuf_partitioning(
ctx: &TaskContext,
input_schema: &Schema,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Option<Partitioning>> {
match partitioning {
Some(protobuf::Partitioning { partition_method }) => match partition_method {
@@ -478,6 +569,7 @@ pub fn parse_protobuf_partitioning(
ctx,
input_schema,
codec,
+ proto_converter,
)
}
Some(protobuf::partitioning::PartitionMethod::Unknown(partition_count)) => {
@@ -532,6 +624,7 @@ pub fn parse_protobuf_file_scan_config(
proto: &protobuf::FileScanExecConf,
ctx: &TaskContext,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
file_source: Arc<dyn FileSource>,
) -> Result<FileScanConfig> {
let schema: Arc<Schema> = parse_protobuf_file_scan_schema(proto)?;
@@ -557,6 +650,7 @@ pub fn parse_protobuf_file_scan_config(
ctx,
&schema,
codec,
+ proto_converter,
)?;
output_ordering.extend(LexOrdering::new(sort_exprs));
}
@@ -567,7 +661,7 @@ pub fn parse_protobuf_file_scan_config(
.projections
.iter()
.map(|proto_expr| {
- let expr = parse_physical_expr(
+ let expr = proto_converter.proto_to_physical_expr(
proto_expr.expr.as_ref().ok_or_else(|| {
internal_datafusion_err!("ProjectionExpr missing expr field")
})?,
@@ -743,12 +837,13 @@ impl TryFrom<&protobuf::FileSinkConfig> for FileSinkConfig {
#[cfg(test)]
mod tests {
- use super::*;
use chrono::{TimeZone, Utc};
use datafusion_datasource::PartitionedFile;
use object_store::ObjectMeta;
use object_store::path::Path;
+ use super::*;
+
#[test]
fn partitioned_file_path_roundtrip_percent_encoded() {
let path_str = "foo/foo%2Fbar/baz%252Fqux";
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 54892597b5..ca213bc722 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -18,30 +18,8 @@
use std::fmt::Debug;
use std::sync::Arc;
-use self::from_proto::parse_protobuf_partitioning;
-use self::to_proto::{serialize_partitioning, serialize_physical_expr};
-use crate::common::{byte_to_string, str_to_byte};
-use crate::physical_plan::from_proto::{
- parse_physical_expr, parse_physical_sort_expr, parse_physical_sort_exprs,
- parse_physical_window_expr, parse_protobuf_file_scan_config, parse_record_batches,
- parse_table_schema_from_proto,
-};
-use crate::physical_plan::to_proto::{
- serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr,
- serialize_physical_sort_exprs, serialize_physical_window_expr,
- serialize_record_batches,
-};
-use crate::protobuf::physical_aggregate_expr_node::AggregateFunction;
-use crate::protobuf::physical_expr_node::ExprType;
-use crate::protobuf::physical_plan_node::PhysicalPlanType;
-use crate::protobuf::{
- self, ListUnnest as ProtoListUnnest, SortExprNode, SortMergeJoinExecNode,
- proto_error, window_agg_exec_node,
-};
-use crate::{convert_required, into_required};
-
use arrow::compute::SortOptions;
-use arrow::datatypes::{IntervalMonthDayNanoType, SchemaRef};
+use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
use datafusion_catalog::memory::MemorySourceConfig;
use datafusion_common::config::CsvOptions;
use datafusion_common::{
@@ -68,12 +46,14 @@ use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
use datafusion_functions_table::generate_series::{
Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,
};
-use datafusion_physical_expr::aggregate::AggregateExprBuilder;
-use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
+use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;
use datafusion_physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};
-use datafusion_physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};
-use datafusion_physical_plan::aggregates::{AggregateMode, LimitOptions};
+use datafusion_physical_plan::aggregates::{
+ AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,
+};
use datafusion_physical_plan::analyze::AnalyzeExec;
+use datafusion_physical_plan::async_func::AsyncFuncExec;
#[expect(deprecated)]
use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
@@ -84,10 +64,9 @@ use datafusion_physical_plan::expressions::PhysicalSortExpr;
use datafusion_physical_plan::filter::{FilterExec, FilterExecBuilder};
use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};
use datafusion_physical_plan::joins::{
- CrossJoinExec, NestedLoopJoinExec, SortMergeJoinExec, StreamJoinPartitionMode,
- SymmetricHashJoinExec,
+ CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec,
+ StreamJoinPartitionMode, SymmetricHashJoinExec,
};
-use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::memory::LazyMemoryExec;
use datafusion_physical_plan::metrics::MetricType;
@@ -100,12 +79,31 @@ use datafusion_physical_plan::union::{InterleaveExec, UnionExec};
use datafusion_physical_plan::unnest::{ListUnnest, UnnestExec};
use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};
use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr};
-
-use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;
-use datafusion_physical_plan::async_func::AsyncFuncExec;
use prost::Message;
use prost::bytes::BufMut;
+use self::from_proto::parse_protobuf_partitioning;
+use self::to_proto::serialize_partitioning;
+use crate::common::{byte_to_string, str_to_byte};
+use crate::physical_plan::from_proto::{
+ parse_physical_expr_with_converter, parse_physical_sort_expr,
+ parse_physical_sort_exprs, parse_physical_window_expr,
+ parse_protobuf_file_scan_config, parse_record_batches, parse_table_schema_from_proto,
+};
+use crate::physical_plan::to_proto::{
+ serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr,
+ serialize_physical_expr_with_converter, serialize_physical_sort_exprs,
+ serialize_physical_window_expr, serialize_record_batches,
+};
+use crate::protobuf::physical_aggregate_expr_node::AggregateFunction;
+use crate::protobuf::physical_expr_node::ExprType;
+use crate::protobuf::physical_plan_node::PhysicalPlanType;
+use crate::protobuf::{
+ self, ListUnnest as ProtoListUnnest, SortExprNode, SortMergeJoinExecNode,
+ proto_error, window_agg_exec_node,
+};
+use crate::{convert_required, into_required};
+
pub mod from_proto;
pub mod to_proto;
@@ -132,8 +130,37 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
fn try_into_physical_plan(
&self,
ctx: &TaskContext,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ self.try_into_physical_plan_with_converter(
+ ctx,
+ codec,
+ &DefaultPhysicalProtoConverter {},
+ )
+ }
- extension_codec: &dyn PhysicalExtensionCodec,
+ fn try_from_physical_plan(
+ plan: Arc<dyn ExecutionPlan>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<Self>
+ where
+ Self: Sized,
+ {
+ Self::try_from_physical_plan_with_converter(
+ plan,
+ codec,
+ &DefaultPhysicalProtoConverter {},
+ )
+ }
+}
+
+impl protobuf::PhysicalPlanNode {
+ pub fn try_into_physical_plan_with_converter(
+ &self,
+ ctx: &TaskContext,
+
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let plan = self.physical_plan_type.as_ref().ok_or_else(|| {
proto_error(format!(
@@ -142,125 +169,149 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
})?;
match plan {
PhysicalPlanType::Explain(explain) => {
- self.try_into_explain_physical_plan(explain, ctx, extension_codec)
- }
- PhysicalPlanType::Projection(projection) => {
- self.try_into_projection_physical_plan(projection, ctx, extension_codec)
+ self.try_into_explain_physical_plan(explain, ctx, codec, proto_converter)
}
+ PhysicalPlanType::Projection(projection) => self
+ .try_into_projection_physical_plan(
+ projection,
+ ctx,
+ codec,
+ proto_converter,
+ ),
PhysicalPlanType::Filter(filter) => {
- self.try_into_filter_physical_plan(filter, ctx, extension_codec)
+ self.try_into_filter_physical_plan(filter, ctx, codec, proto_converter)
}
PhysicalPlanType::CsvScan(scan) => {
- self.try_into_csv_scan_physical_plan(scan, ctx, extension_codec)
+ self.try_into_csv_scan_physical_plan(scan, ctx, codec, proto_converter)
}
PhysicalPlanType::JsonScan(scan) => {
- self.try_into_json_scan_physical_plan(scan, ctx, extension_codec)
- }
- PhysicalPlanType::ParquetScan(scan) => {
- self.try_into_parquet_scan_physical_plan(scan, ctx, extension_codec)
+ self.try_into_json_scan_physical_plan(scan, ctx, codec, proto_converter)
}
+ PhysicalPlanType::ParquetScan(scan) => self
+ .try_into_parquet_scan_physical_plan(scan, ctx, codec, proto_converter),
PhysicalPlanType::AvroScan(scan) => {
- self.try_into_avro_scan_physical_plan(scan, ctx, extension_codec)
+ self.try_into_avro_scan_physical_plan(scan, ctx, codec, proto_converter)
}
PhysicalPlanType::MemoryScan(scan) => {
- self.try_into_memory_scan_physical_plan(scan, ctx, extension_codec)
+ self.try_into_memory_scan_physical_plan(scan, ctx, codec, proto_converter)
}
PhysicalPlanType::CoalesceBatches(coalesce_batches) => self
.try_into_coalesce_batches_physical_plan(
coalesce_batches,
ctx,
- extension_codec,
+ codec,
+ proto_converter,
),
PhysicalPlanType::Merge(merge) => {
- self.try_into_merge_physical_plan(merge, ctx, extension_codec)
- }
- PhysicalPlanType::Repartition(repart) => {
- self.try_into_repartition_physical_plan(repart, ctx, extension_codec)
- }
- PhysicalPlanType::GlobalLimit(limit) => {
- self.try_into_global_limit_physical_plan(limit, ctx, extension_codec)
- }
- PhysicalPlanType::LocalLimit(limit) => {
- self.try_into_local_limit_physical_plan(limit, ctx, extension_codec)
- }
- PhysicalPlanType::Window(window_agg) => {
- self.try_into_window_physical_plan(window_agg, ctx, extension_codec)
- }
- PhysicalPlanType::Aggregate(hash_agg) => {
- self.try_into_aggregate_physical_plan(hash_agg, ctx, extension_codec)
- }
- PhysicalPlanType::HashJoin(hashjoin) => {
- self.try_into_hash_join_physical_plan(hashjoin, ctx, extension_codec)
+ self.try_into_merge_physical_plan(merge, ctx, codec, proto_converter)
}
+ PhysicalPlanType::Repartition(repart) => self
+ .try_into_repartition_physical_plan(repart, ctx, codec, proto_converter),
+ PhysicalPlanType::GlobalLimit(limit) => self
+ .try_into_global_limit_physical_plan(limit, ctx, codec, proto_converter),
+ PhysicalPlanType::LocalLimit(limit) => self
+ .try_into_local_limit_physical_plan(limit, ctx, codec, proto_converter),
+ PhysicalPlanType::Window(window_agg) => self.try_into_window_physical_plan(
+ window_agg,
+ ctx,
+ codec,
+ proto_converter,
+ ),
+ PhysicalPlanType::Aggregate(hash_agg) => self
+ .try_into_aggregate_physical_plan(hash_agg, ctx, codec, proto_converter),
+ PhysicalPlanType::HashJoin(hashjoin) => self
+ .try_into_hash_join_physical_plan(hashjoin, ctx, codec, proto_converter),
PhysicalPlanType::SymmetricHashJoin(sym_join) => self
.try_into_symmetric_hash_join_physical_plan(
sym_join,
ctx,
- extension_codec,
+ codec,
+ proto_converter,
),
PhysicalPlanType::Union(union) => {
- self.try_into_union_physical_plan(union, ctx, extension_codec)
- }
- PhysicalPlanType::Interleave(interleave) => {
- self.try_into_interleave_physical_plan(interleave, ctx, extension_codec)
+ self.try_into_union_physical_plan(union, ctx, codec, proto_converter)
}
- PhysicalPlanType::CrossJoin(crossjoin) => {
- self.try_into_cross_join_physical_plan(crossjoin, ctx, extension_codec)
- }
- PhysicalPlanType::Empty(empty) => {
- self.try_into_empty_physical_plan(empty, ctx, extension_codec)
- }
- PhysicalPlanType::PlaceholderRow(placeholder) => self
- .try_into_placeholder_row_physical_plan(
- placeholder,
+ PhysicalPlanType::Interleave(interleave) => self
+ .try_into_interleave_physical_plan(
+ interleave,
ctx,
- extension_codec,
+ codec,
+ proto_converter,
),
- PhysicalPlanType::Sort(sort) => {
- self.try_into_sort_physical_plan(sort, ctx, extension_codec)
+ PhysicalPlanType::CrossJoin(crossjoin) => self
+ .try_into_cross_join_physical_plan(
+ crossjoin,
+ ctx,
+ codec,
+ proto_converter,
+ ),
+ PhysicalPlanType::Empty(empty) => {
+ self.try_into_empty_physical_plan(empty, ctx, codec, proto_converter)
}
- PhysicalPlanType::SortPreservingMerge(sort) => self
- .try_into_sort_preserving_merge_physical_plan(sort, ctx, extension_codec),
- PhysicalPlanType::Extension(extension) => {
- self.try_into_extension_physical_plan(extension, ctx, extension_codec)
+ PhysicalPlanType::PlaceholderRow(placeholder) => {
+ self.try_into_placeholder_row_physical_plan(placeholder, ctx, codec)
}
- PhysicalPlanType::NestedLoopJoin(join) => {
- self.try_into_nested_loop_join_physical_plan(join, ctx, extension_codec)
+ PhysicalPlanType::Sort(sort) => {
+ self.try_into_sort_physical_plan(sort, ctx, codec, proto_converter)
}
+ PhysicalPlanType::SortPreservingMerge(sort) => self
+ .try_into_sort_preserving_merge_physical_plan(
+ sort,
+ ctx,
+ codec,
+ proto_converter,
+ ),
+ PhysicalPlanType::Extension(extension) => self
+ .try_into_extension_physical_plan(extension, ctx, codec, proto_converter),
+ PhysicalPlanType::NestedLoopJoin(join) => self
+ .try_into_nested_loop_join_physical_plan(
+ join,
+ ctx,
+ codec,
+ proto_converter,
+ ),
PhysicalPlanType::Analyze(analyze) => {
- self.try_into_analyze_physical_plan(analyze, ctx, extension_codec)
+ self.try_into_analyze_physical_plan(analyze, ctx, codec, proto_converter)
}
PhysicalPlanType::JsonSink(sink) => {
- self.try_into_json_sink_physical_plan(sink, ctx, extension_codec)
+ self.try_into_json_sink_physical_plan(sink, ctx, codec, proto_converter)
}
PhysicalPlanType::CsvSink(sink) => {
- self.try_into_csv_sink_physical_plan(sink, ctx, extension_codec)
+ self.try_into_csv_sink_physical_plan(sink, ctx, codec, proto_converter)
}
#[cfg_attr(not(feature = "parquet"), allow(unused_variables))]
- PhysicalPlanType::ParquetSink(sink) => {
- self.try_into_parquet_sink_physical_plan(sink, ctx, extension_codec)
- }
+ PhysicalPlanType::ParquetSink(sink) => self
+ .try_into_parquet_sink_physical_plan(sink, ctx, codec, proto_converter),
PhysicalPlanType::Unnest(unnest) => {
- self.try_into_unnest_physical_plan(unnest, ctx, extension_codec)
- }
- PhysicalPlanType::Cooperative(cooperative) => {
- self.try_into_cooperative_physical_plan(cooperative, ctx, extension_codec)
+ self.try_into_unnest_physical_plan(unnest, ctx, codec, proto_converter)
}
+ PhysicalPlanType::Cooperative(cooperative) => self
+ .try_into_cooperative_physical_plan(
+ cooperative,
+ ctx,
+ codec,
+ proto_converter,
+ ),
PhysicalPlanType::GenerateSeries(generate_series) => {
self.try_into_generate_series_physical_plan(generate_series)
}
PhysicalPlanType::SortMergeJoin(sort_join) => {
- self.try_into_sort_join(sort_join, ctx, extension_codec)
- }
- PhysicalPlanType::AsyncFunc(async_func) => {
- self.try_into_async_func_physical_plan(async_func, ctx, extension_codec)
+ self.try_into_sort_join(sort_join, ctx, codec, proto_converter)
}
+ PhysicalPlanType::AsyncFunc(async_func) => self
+ .try_into_async_func_physical_plan(
+ async_func,
+ ctx,
+ codec,
+ proto_converter,
+ ),
}
}
- fn try_from_physical_plan(
+ pub fn try_from_physical_plan_with_converter(
plan: Arc<dyn ExecutionPlan>,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self>
where
Self: Sized,
@@ -269,93 +320,96 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
let plan = plan.as_any();
if let Some(exec) = plan.downcast_ref::<ExplainExec>() {
- return protobuf::PhysicalPlanNode::try_from_explain_exec(
- exec,
- extension_codec,
- );
+ return protobuf::PhysicalPlanNode::try_from_explain_exec(exec, codec);
}
if let Some(exec) = plan.downcast_ref::<ProjectionExec>() {
return protobuf::PhysicalPlanNode::try_from_projection_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {
return protobuf::PhysicalPlanNode::try_from_analyze_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<FilterExec>() {
return protobuf::PhysicalPlanNode::try_from_filter_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(limit) = plan.downcast_ref::<GlobalLimitExec>() {
return protobuf::PhysicalPlanNode::try_from_global_limit_exec(
limit,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(limit) = plan.downcast_ref::<LocalLimitExec>() {
return protobuf::PhysicalPlanNode::try_from_local_limit_exec(
limit,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<HashJoinExec>() {
return protobuf::PhysicalPlanNode::try_from_hash_join_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<SymmetricHashJoinExec>() {
return protobuf::PhysicalPlanNode::try_from_symmetric_hash_join_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<SortMergeJoinExec>() {
return protobuf::PhysicalPlanNode::try_from_sort_merge_join_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<CrossJoinExec>() {
return protobuf::PhysicalPlanNode::try_from_cross_join_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<AggregateExec>() {
return protobuf::PhysicalPlanNode::try_from_aggregate_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(empty) = plan.downcast_ref::<EmptyExec>() {
- return protobuf::PhysicalPlanNode::try_from_empty_exec(
- empty,
- extension_codec,
- );
+ return protobuf::PhysicalPlanNode::try_from_empty_exec(empty, codec);
}
if let Some(empty) = plan.downcast_ref::<PlaceholderRowExec>() {
return protobuf::PhysicalPlanNode::try_from_placeholder_row_exec(
- empty,
- extension_codec,
+ empty, codec,
);
}
@@ -363,14 +417,16 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
if let Some(coalesce_batches) = plan.downcast_ref::<CoalesceBatchesExec>() {
return protobuf::PhysicalPlanNode::try_from_coalesce_batches_exec(
coalesce_batches,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(data_source_exec) = plan.downcast_ref::<DataSourceExec>()
&& let Some(node) = protobuf::PhysicalPlanNode::try_from_data_source_exec(
data_source_exec,
- extension_codec,
+ codec,
+ proto_converter,
)?
{
return Ok(node);
@@ -379,67 +435,80 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
if let Some(exec) = plan.downcast_ref::<CoalescePartitionsExec>() {
return protobuf::PhysicalPlanNode::try_from_coalesce_partitions_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<RepartitionExec>() {
return protobuf::PhysicalPlanNode::try_from_repartition_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<SortExec>() {
- return protobuf::PhysicalPlanNode::try_from_sort_exec(exec, extension_codec);
+ return protobuf::PhysicalPlanNode::try_from_sort_exec(
+ exec,
+ codec,
+ proto_converter,
+ );
}
if let Some(union) = plan.downcast_ref::<UnionExec>() {
return protobuf::PhysicalPlanNode::try_from_union_exec(
union,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(interleave) = plan.downcast_ref::<InterleaveExec>() {
return protobuf::PhysicalPlanNode::try_from_interleave_exec(
interleave,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<SortPreservingMergeExec>() {
return protobuf::PhysicalPlanNode::try_from_sort_preserving_merge_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<NestedLoopJoinExec>() {
return protobuf::PhysicalPlanNode::try_from_nested_loop_join_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<WindowAggExec>() {
return protobuf::PhysicalPlanNode::try_from_window_agg_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<BoundedWindowAggExec>() {
return protobuf::PhysicalPlanNode::try_from_bounded_window_agg_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<DataSinkExec>()
&& let Some(node) = protobuf::PhysicalPlanNode::try_from_data_sink_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
)?
{
return Ok(node);
@@ -448,14 +517,16 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
if let Some(exec) = plan.downcast_ref::<UnnestExec>() {
return protobuf::PhysicalPlanNode::try_from_unnest_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
if let Some(exec) = plan.downcast_ref::<CooperativeExec>() {
return protobuf::PhysicalPlanNode::try_from_cooperative_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
@@ -469,21 +540,23 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
if let Some(exec) = plan.downcast_ref::<AsyncFuncExec>() {
return protobuf::PhysicalPlanNode::try_from_async_func_exec(
exec,
- extension_codec,
+ codec,
+ proto_converter,
);
}
let mut buf: Vec<u8> = vec![];
- match extension_codec.try_encode(Arc::clone(&plan_clone), &mut buf) {
+ match codec.try_encode(Arc::clone(&plan_clone), &mut buf) {
Ok(_) => {
let inputs: Vec<protobuf::PhysicalPlanNode> = plan_clone
.children()
.into_iter()
.cloned()
.map(|i| {
- protobuf::PhysicalPlanNode::try_from_physical_plan(
+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
i,
- extension_codec,
+ codec,
+ proto_converter,
)
})
.collect::<Result<_>>()?;
@@ -507,7 +580,8 @@ impl protobuf::PhysicalPlanNode {
explain: &protobuf::ExplainExecNode,
_ctx: &TaskContext,
- _extension_codec: &dyn PhysicalExtensionCodec,
+ _codec: &dyn PhysicalExtensionCodec,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
Ok(Arc::new(ExplainExec::new(
Arc::new(explain.schema.as_ref().unwrap().try_into()?),
@@ -525,21 +599,22 @@ impl protobuf::PhysicalPlanNode {
projection: &protobuf::ProjectionExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&projection.input, ctx, extension_codec)?;
+ into_physical_plan(&projection.input, ctx, codec, proto_converter)?;
let exprs = projection
.expr
.iter()
.zip(projection.expr_name.iter())
.map(|(expr, name)| {
Ok((
- parse_physical_expr(
+ proto_converter.proto_to_physical_expr(
expr,
ctx,
input.schema().as_ref(),
- extension_codec,
+ codec,
)?,
name.to_string(),
))
@@ -557,16 +632,22 @@ impl protobuf::PhysicalPlanNode {
filter: &protobuf::FilterExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&filter.input, ctx, extension_codec)?;
+ into_physical_plan(&filter.input, ctx, codec, proto_converter)?;
let predicate = filter
.expr
.as_ref()
.map(|expr| {
- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)
+ proto_converter.proto_to_physical_expr(
+ expr,
+ ctx,
+ input.schema().as_ref(),
+ codec,
+ )
})
.transpose()?
.ok_or_else(|| {
@@ -607,7 +688,8 @@ impl protobuf::PhysicalPlanNode {
scan: &protobuf::CsvScanExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let escape =
if let Some(protobuf::csv_scan_exec_node::OptionalEscape::Escape(escape)) =
@@ -648,7 +730,8 @@ impl protobuf::PhysicalPlanNode {
let conf = FileScanConfigBuilder::from(parse_protobuf_file_scan_config(
scan.base_conf.as_ref().unwrap(),
ctx,
- extension_codec,
+ codec,
+ proto_converter,
source,
)?)
.with_file_compression_type(FileCompressionType::UNCOMPRESSED)
@@ -661,14 +744,16 @@ impl protobuf::PhysicalPlanNode {
scan: &protobuf::JsonScanExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let base_conf = scan.base_conf.as_ref().unwrap();
let table_schema = parse_table_schema_from_proto(base_conf)?;
let scan_conf = parse_protobuf_file_scan_config(
base_conf,
ctx,
- extension_codec,
+ codec,
+ proto_converter,
Arc::new(JsonSource::new(table_schema)),
)?;
Ok(DataSourceExec::from_data_source(scan_conf))
@@ -679,7 +764,8 @@ impl protobuf::PhysicalPlanNode {
&self,
scan: &protobuf::ParquetScanExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
#[cfg(feature = "parquet")]
{
@@ -696,7 +782,7 @@ impl protobuf::PhysicalPlanNode {
.iter()
.map(|&i| schema.field(i as usize).clone())
.collect();
- Arc::new(arrow::datatypes::Schema::new(projected_fields))
+ Arc::new(Schema::new(projected_fields))
} else {
schema
};
@@ -705,11 +791,11 @@ impl protobuf::PhysicalPlanNode {
.predicate
.as_ref()
.map(|expr| {
- parse_physical_expr(
+ proto_converter.proto_to_physical_expr(
expr,
ctx,
predicate_schema.as_ref(),
- extension_codec,
+ codec,
)
})
.transpose()?;
@@ -731,7 +817,8 @@ impl protobuf::PhysicalPlanNode {
let base_config = parse_protobuf_file_scan_config(
base_conf,
ctx,
- extension_codec,
+ codec,
+ proto_converter,
Arc::new(source),
)?;
Ok(DataSourceExec::from_data_source(base_config))
@@ -747,7 +834,8 @@ impl protobuf::PhysicalPlanNode {
&self,
scan: &protobuf::AvroScanExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
#[cfg(feature = "avro")]
{
@@ -756,7 +844,8 @@ impl protobuf::PhysicalPlanNode {
let conf = parse_protobuf_file_scan_config(
scan.base_conf.as_ref().unwrap(),
ctx,
- extension_codec,
+ codec,
+ proto_converter,
Arc::new(AvroSource::new(table_schema)),
)?;
Ok(DataSourceExec::from_data_source(conf))
@@ -771,7 +860,8 @@ impl protobuf::PhysicalPlanNode {
scan: &protobuf::MemoryScanExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let partitions = scan
.partitions
@@ -801,7 +891,8 @@ impl protobuf::PhysicalPlanNode {
&ordering.physical_sort_expr_nodes,
ctx,
&schema,
- extension_codec,
+ codec,
+ proto_converter,
)?;
sort_information.extend(LexOrdering::new(sort_exprs));
}
@@ -820,10 +911,11 @@ impl protobuf::PhysicalPlanNode {
coalesce_batches: &protobuf::CoalesceBatchesExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&coalesce_batches.input, ctx, extension_codec)?;
+ into_physical_plan(&coalesce_batches.input, ctx, codec, proto_converter)?;
Ok(Arc::new(
#[expect(deprecated)]
CoalesceBatchesExec::new(input, coalesce_batches.target_batch_size as usize)
@@ -836,10 +928,11 @@ impl protobuf::PhysicalPlanNode {
merge: &protobuf::CoalescePartitionsExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&merge.input, ctx, extension_codec)?;
+ into_physical_plan(&merge.input, ctx, codec, proto_converter)?;
Ok(Arc::new(
CoalescePartitionsExec::new(input)
.with_fetch(merge.fetch.map(|f| f as usize)),
@@ -851,15 +944,17 @@ impl protobuf::PhysicalPlanNode {
repart: &protobuf::RepartitionExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&repart.input, ctx, extension_codec)?;
+ into_physical_plan(&repart.input, ctx, codec, proto_converter)?;
let partitioning = parse_protobuf_partitioning(
repart.partitioning.as_ref(),
ctx,
input.schema().as_ref(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(Arc::new(RepartitionExec::try_new(
input,
@@ -872,10 +967,11 @@ impl protobuf::PhysicalPlanNode {
limit: &protobuf::GlobalLimitExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&limit.input, ctx, extension_codec)?;
+ into_physical_plan(&limit.input, ctx, codec, proto_converter)?;
let fetch = if limit.fetch >= 0 {
Some(limit.fetch as usize)
} else {
@@ -893,10 +989,11 @@ impl protobuf::PhysicalPlanNode {
limit: &protobuf::LocalLimitExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&limit.input, ctx, extension_codec)?;
+ into_physical_plan(&limit.input, ctx, codec, proto_converter)?;
Ok(Arc::new(LocalLimitExec::new(input, limit.fetch as usize)))
}
@@ -905,10 +1002,11 @@ impl protobuf::PhysicalPlanNode {
window_agg: &protobuf::WindowAggExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&window_agg.input, ctx, extension_codec)?;
+ into_physical_plan(&window_agg.input, ctx, codec, proto_converter)?;
let input_schema = input.schema();
let physical_window_expr: Vec<Arc<dyn WindowExpr>> = window_agg
@@ -919,7 +1017,8 @@ impl protobuf::PhysicalPlanNode {
window_expr,
ctx,
input_schema.as_ref(),
- extension_codec,
+ codec,
+ proto_converter,
)
})
.collect::<Result<Vec<_>, _>>()?;
@@ -928,7 +1027,12 @@ impl protobuf::PhysicalPlanNode {
.partition_keys
.iter()
.map(|expr| {
- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)
+ proto_converter.proto_to_physical_expr(
+ expr,
+ ctx,
+ input.schema().as_ref(),
+ codec,
+ )
})
.collect::<Result<Vec<Arc<dyn PhysicalExpr>>>>()?;
@@ -963,10 +1067,11 @@ impl protobuf::PhysicalPlanNode {
hash_agg: &protobuf::AggregateExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&hash_agg.input, ctx, extension_codec)?;
+ into_physical_plan(&hash_agg.input, ctx, codec, proto_converter)?;
let mode = protobuf::AggregateMode::try_from(hash_agg.mode).map_err(|_| {
proto_error(format!(
"Received a AggregateNode message with unknown AggregateMode {}",
@@ -990,7 +1095,8 @@ impl protobuf::PhysicalPlanNode {
.iter()
.zip(hash_agg.group_expr_name.iter())
.map(|(expr, name)| {
- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)
+ proto_converter
+ .proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec)
.map(|expr| (expr, name.to_string()))
})
.collect::<Result<Vec<_>, _>>()?;
@@ -1000,7 +1106,8 @@ impl protobuf::PhysicalPlanNode {
.iter()
.zip(hash_agg.group_expr_name.iter())
.map(|(expr, name)| {
- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)
+ proto_converter
+ .proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec)
.map(|expr| (expr, name.to_string()))
})
.collect::<Result<Vec<_>, _>>()?;
@@ -1029,7 +1136,12 @@ impl protobuf::PhysicalPlanNode {
expr.expr
.as_ref()
.map(|e| {
- parse_physical_expr(e, ctx, &physical_schema, extension_codec)
+ proto_converter.proto_to_physical_expr(
+ e,
+ ctx,
+ &physical_schema,
+ codec,
+ )
})
.transpose()
})
@@ -1050,11 +1162,11 @@ impl protobuf::PhysicalPlanNode {
.expr
.iter()
.map(|e| {
- parse_physical_expr(
+ proto_converter.proto_to_physical_expr(
e,
ctx,
&physical_schema,
- extension_codec,
+ codec,
)
})
.collect::<Result<Vec<_>>>()?;
@@ -1066,7 +1178,8 @@ impl protobuf::PhysicalPlanNode {
e,
ctx,
&physical_schema,
- extension_codec,
+ codec,
+ proto_converter,
)
})
.collect::<Result<_>>()?;
@@ -1076,11 +1189,11 @@ impl protobuf::PhysicalPlanNode {
.map(|func| match func {
AggregateFunction::UserDefinedAggrFunction(udaf_name) => {
let agg_udf = match &agg_node.fun_definition {
- Some(buf) => extension_codec
- .try_decode_udaf(udaf_name, buf)?,
+ Some(buf) => {
+ codec.try_decode_udaf(udaf_name, buf)?
+ }
None => ctx.udaf(udaf_name).or_else(|_| {
- extension_codec
- .try_decode_udaf(udaf_name, &[])
+ codec.try_decode_udaf(udaf_name, &[])
})?,
};
@@ -1135,29 +1248,30 @@ impl protobuf::PhysicalPlanNode {
hashjoin: &protobuf::HashJoinExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let left: Arc<dyn ExecutionPlan> =
- into_physical_plan(&hashjoin.left, ctx, extension_codec)?;
+ into_physical_plan(&hashjoin.left, ctx, codec, proto_converter)?;
let right: Arc<dyn ExecutionPlan> =
- into_physical_plan(&hashjoin.right, ctx, extension_codec)?;
+ into_physical_plan(&hashjoin.right, ctx, codec, proto_converter)?;
let left_schema = left.schema();
let right_schema = right.schema();
let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = hashjoin
.on
.iter()
.map(|col| {
- let left = parse_physical_expr(
+ let left = proto_converter.proto_to_physical_expr(
&col.left.clone().unwrap(),
ctx,
left_schema.as_ref(),
- extension_codec,
+ codec,
)?;
- let right = parse_physical_expr(
+ let right = proto_converter.proto_to_physical_expr(
&col.right.clone().unwrap(),
ctx,
right_schema.as_ref(),
- extension_codec,
+ codec,
)?;
Ok((left, right))
})
@@ -1186,12 +1300,12 @@ impl protobuf::PhysicalPlanNode {
.ok_or_else(|| proto_error("Missing JoinFilter schema"))?
.try_into()?;
- let expression = parse_physical_expr(
+ let expression = proto_converter.proto_to_physical_expr(
f.expression.as_ref().ok_or_else(|| {
proto_error("Unexpected empty filter expression")
})?,
ctx, &schema,
- extension_codec,
+ codec,
)?;
let column_indices = f.column_indices
.iter()
@@ -1254,27 +1368,28 @@ impl protobuf::PhysicalPlanNode {
sym_join: &protobuf::SymmetricHashJoinExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let left = into_physical_plan(&sym_join.left, ctx, extension_codec)?;
- let right = into_physical_plan(&sym_join.right, ctx, extension_codec)?;
+ let left = into_physical_plan(&sym_join.left, ctx, codec, proto_converter)?;
+ let right = into_physical_plan(&sym_join.right, ctx, codec, proto_converter)?;
let left_schema = left.schema();
let right_schema = right.schema();
let on = sym_join
.on
.iter()
.map(|col| {
- let left = parse_physical_expr(
+ let left = proto_converter.proto_to_physical_expr(
&col.left.clone().unwrap(),
ctx,
left_schema.as_ref(),
- extension_codec,
+ codec,
)?;
- let right = parse_physical_expr(
+ let right = proto_converter.proto_to_physical_expr(
&col.right.clone().unwrap(),
ctx,
right_schema.as_ref(),
- extension_codec,
+ codec,
)?;
Ok((left, right))
})
@@ -1303,12 +1418,12 @@ impl protobuf::PhysicalPlanNode {
.ok_or_else(|| proto_error("Missing JoinFilter schema"))?
.try_into()?;
- let expression = parse_physical_expr(
+ let expression = proto_converter.proto_to_physical_expr(
f.expression.as_ref().ok_or_else(|| {
proto_error("Unexpected empty filter expression")
})?,
ctx, &schema,
- extension_codec,
+ codec,
)?;
let column_indices = f.column_indices
.iter()
@@ -1334,7 +1449,8 @@ impl protobuf::PhysicalPlanNode {
&sym_join.left_sort_exprs,
ctx,
&left_schema,
- extension_codec,
+ codec,
+ proto_converter,
)?;
let left_sort_exprs = LexOrdering::new(left_sort_exprs);
@@ -1342,7 +1458,8 @@ impl protobuf::PhysicalPlanNode {
&sym_join.right_sort_exprs,
ctx,
&right_schema,
- extension_codec,
+ codec,
+ proto_converter,
)?;
let right_sort_exprs = LexOrdering::new(right_sort_exprs);
@@ -1382,11 +1499,12 @@ impl protobuf::PhysicalPlanNode {
union: &protobuf::UnionExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];
for input in &union.inputs {
- inputs.push(input.try_into_physical_plan(ctx, extension_codec)?);
+ inputs.push(proto_converter.proto_to_execution_plan(ctx, codec, input)?);
}
UnionExec::try_new(inputs)
}
@@ -1396,11 +1514,12 @@ impl protobuf::PhysicalPlanNode {
interleave: &protobuf::InterleaveExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];
for input in &interleave.inputs {
- inputs.push(input.try_into_physical_plan(ctx, extension_codec)?);
+ inputs.push(proto_converter.proto_to_execution_plan(ctx, codec, input)?);
}
Ok(Arc::new(InterleaveExec::try_new(inputs)?))
}
@@ -1410,12 +1529,13 @@ impl protobuf::PhysicalPlanNode {
crossjoin: &protobuf::CrossJoinExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let left: Arc<dyn ExecutionPlan> =
- into_physical_plan(&crossjoin.left, ctx, extension_codec)?;
+ into_physical_plan(&crossjoin.left, ctx, codec, proto_converter)?;
let right: Arc<dyn ExecutionPlan> =
- into_physical_plan(&crossjoin.right, ctx, extension_codec)?;
+ into_physical_plan(&crossjoin.right, ctx, codec, proto_converter)?;
Ok(Arc::new(CrossJoinExec::new(left, right)))
}
@@ -1424,7 +1544,8 @@ impl protobuf::PhysicalPlanNode {
empty: &protobuf::EmptyExecNode,
_ctx: &TaskContext,
- _extension_codec: &dyn PhysicalExtensionCodec,
+ _codec: &dyn PhysicalExtensionCodec,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let schema = Arc::new(convert_required!(empty.schema)?);
Ok(Arc::new(EmptyExec::new(schema)))
@@ -1435,7 +1556,7 @@ impl protobuf::PhysicalPlanNode {
placeholder: &protobuf::PlaceholderRowExecNode,
_ctx: &TaskContext,
- _extension_codec: &dyn PhysicalExtensionCodec,
+ _codec: &dyn PhysicalExtensionCodec,
) -> Result<Arc<dyn ExecutionPlan>> {
let schema = Arc::new(convert_required!(placeholder.schema)?);
Ok(Arc::new(PlaceholderRowExec::new(schema)))
@@ -1446,9 +1567,10 @@ impl protobuf::PhysicalPlanNode {
sort: &protobuf::SortExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&sort.input, ctx, extension_codec)?;
+ let input = into_physical_plan(&sort.input, ctx, codec, proto_converter)?;
let exprs = sort
.expr
.iter()
@@ -1469,7 +1591,7 @@ impl protobuf::PhysicalPlanNode {
})?
.as_ref();
Ok(PhysicalSortExpr {
- expr: parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)?,
+ expr: proto_converter.proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec)?,
options: SortOptions {
descending: !sort_expr.asc,
nulls_first: sort_expr.nulls_first,
@@ -1498,9 +1620,10 @@ impl protobuf::PhysicalPlanNode {
sort: &protobuf::SortPreservingMergeExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&sort.input, ctx, extension_codec)?;
+ let input = into_physical_plan(&sort.input, ctx, codec, proto_converter)?;
let exprs = sort
.expr
.iter()
@@ -1521,11 +1644,11 @@ impl protobuf::PhysicalPlanNode {
})?
.as_ref();
Ok(PhysicalSortExpr {
- expr: parse_physical_expr(
+ expr: proto_converter.proto_to_physical_expr(
expr,
ctx,
input.schema().as_ref(),
- extension_codec,
+ codec,
)?,
options: SortOptions {
descending: !sort_expr.asc,
@@ -1551,16 +1674,16 @@ impl protobuf::PhysicalPlanNode {
extension: &protobuf::PhysicalExtensionNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let inputs: Vec<Arc<dyn ExecutionPlan>> = extension
.inputs
.iter()
- .map(|i| i.try_into_physical_plan(ctx, extension_codec))
+ .map(|i| proto_converter.proto_to_execution_plan(ctx, codec, i))
.collect::<Result<_>>()?;
- let extension_node =
- extension_codec.try_decode(extension.node.as_slice(), &inputs, ctx)?;
+ let extension_node = codec.try_decode(extension.node.as_slice(), &inputs, ctx)?;
Ok(extension_node)
}
@@ -1570,12 +1693,13 @@ impl protobuf::PhysicalPlanNode {
join: &protobuf::NestedLoopJoinExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let left: Arc<dyn ExecutionPlan> =
- into_physical_plan(&join.left, ctx, extension_codec)?;
+ into_physical_plan(&join.left, ctx, codec, proto_converter)?;
let right: Arc<dyn ExecutionPlan> =
- into_physical_plan(&join.right, ctx, extension_codec)?;
+ into_physical_plan(&join.right, ctx, codec, proto_converter)?;
let join_type = protobuf::JoinType::try_from(join.join_type).map_err(|_| {
proto_error(format!(
"Received a NestedLoopJoinExecNode message with unknown JoinType {}",
@@ -1592,12 +1716,12 @@ impl protobuf::PhysicalPlanNode {
.ok_or_else(|| proto_error("Missing JoinFilter schema"))?
.try_into()?;
- let expression = parse_physical_expr(
+ let expression = proto_converter.proto_to_physical_expr(
f.expression.as_ref().ok_or_else(|| {
proto_error("Unexpected empty filter expression")
})?,
ctx, &schema,
- extension_codec,
+ codec,
)?;
let column_indices = f.column_indices
.iter()
@@ -1644,10 +1768,11 @@ impl protobuf::PhysicalPlanNode {
analyze: &protobuf::AnalyzeExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&analyze.input, ctx, extension_codec)?;
+ into_physical_plan(&analyze.input, ctx, codec, proto_converter)?;
Ok(Arc::new(AnalyzeExec::new(
analyze.verbose,
analyze.show_statistics,
@@ -1662,9 +1787,10 @@ impl protobuf::PhysicalPlanNode {
sink: &protobuf::JsonSinkExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&sink.input, ctx, extension_codec)?;
+ let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;
let data_sink: JsonSink = sink
.sink
@@ -1680,7 +1806,8 @@ impl protobuf::PhysicalPlanNode {
&collection.physical_sort_expr_nodes,
ctx,
&sink_schema,
- extension_codec,
+ codec,
+ proto_converter,
)
.map(|sort_exprs| {
LexRequirement::new(sort_exprs.into_iter().map(Into::into))
@@ -1700,9 +1827,10 @@ impl protobuf::PhysicalPlanNode {
sink: &protobuf::CsvSinkExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&sink.input, ctx, extension_codec)?;
+ let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;
let data_sink: CsvSink = sink
.sink
@@ -1718,7 +1846,8 @@ impl protobuf::PhysicalPlanNode {
&collection.physical_sort_expr_nodes,
ctx,
&sink_schema,
- extension_codec,
+ codec,
+ proto_converter,
)
.map(|sort_exprs| {
LexRequirement::new(sort_exprs.into_iter().map(Into::into))
@@ -1739,11 +1868,12 @@ impl protobuf::PhysicalPlanNode {
sink: &protobuf::ParquetSinkExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
#[cfg(feature = "parquet")]
{
- let input = into_physical_plan(&sink.input, ctx, extension_codec)?;
+ let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;
let data_sink: ParquetSink = sink
.sink
@@ -1759,7 +1889,8 @@ impl protobuf::PhysicalPlanNode {
&collection.physical_sort_expr_nodes,
ctx,
&sink_schema,
- extension_codec,
+ codec,
+ proto_converter,
)
.map(|sort_exprs| {
LexRequirement::new(sort_exprs.into_iter().map(Into::into))
@@ -1782,9 +1913,10 @@ impl protobuf::PhysicalPlanNode {
unnest: &protobuf::UnnestExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&unnest.input, ctx, extension_codec)?;
+ let input = into_physical_plan(&unnest.input, ctx, codec, proto_converter)?;
Ok(Arc::new(UnnestExec::new(
input,
@@ -1813,11 +1945,12 @@ impl protobuf::PhysicalPlanNode {
sort_join: &SortMergeJoinExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let left = into_physical_plan(&sort_join.left, ctx, extension_codec)?;
+ let left = into_physical_plan(&sort_join.left, ctx, codec, proto_converter)?;
let left_schema = left.schema();
- let right = into_physical_plan(&sort_join.right, ctx, extension_codec)?;
+ let right = into_physical_plan(&sort_join.right, ctx, codec, proto_converter)?;
let right_schema = right.schema();
let filter = sort_join
@@ -1830,13 +1963,13 @@ impl protobuf::PhysicalPlanNode {
.ok_or_else(|| proto_error("Missing JoinFilter schema"))?
.try_into()?;
- let expression = parse_physical_expr(
+ let expression = proto_converter.proto_to_physical_expr(
f.expression.as_ref().ok_or_else(|| {
proto_error("Unexpected empty filter expression")
})?,
ctx,
&schema,
- extension_codec,
+ codec,
)?;
let column_indices = f
.column_indices
@@ -1893,17 +2026,17 @@ impl protobuf::PhysicalPlanNode {
.on
.iter()
.map(|col| {
- let left = parse_physical_expr(
+ let left = proto_converter.proto_to_physical_expr(
&col.left.clone().unwrap(),
ctx,
left_schema.as_ref(),
- extension_codec,
+ codec,
)?;
- let right = parse_physical_expr(
+ let right = proto_converter.proto_to_physical_expr(
&col.right.clone().unwrap(),
ctx,
right_schema.as_ref(),
- extension_codec,
+ codec,
)?;
Ok((left, right))
})
@@ -1990,9 +2123,10 @@ impl protobuf::PhysicalPlanNode {
field_stream: &protobuf::CooperativeExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let input = into_physical_plan(&field_stream.input, ctx, extension_codec)?;
+ let input = into_physical_plan(&field_stream.input, ctx, codec, proto_converter)?;
Ok(Arc::new(CooperativeExec::new(input)))
}
@@ -2000,10 +2134,11 @@ impl protobuf::PhysicalPlanNode {
&self,
async_func: &protobuf::AsyncFuncExecNode,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let input: Arc<dyn ExecutionPlan> =
- into_physical_plan(&async_func.input, ctx, extension_codec)?;
+ into_physical_plan(&async_func.input, ctx, codec, proto_converter)?;
if async_func.async_exprs.len() != async_func.async_expr_names.len() {
return internal_err!(
@@ -2016,11 +2151,11 @@ impl protobuf::PhysicalPlanNode {
.iter()
.zip(async_func.async_expr_names.iter())
.map(|(expr, name)| {
- let physical_expr = parse_physical_expr(
+ let physical_expr = proto_converter.proto_to_physical_expr(
expr,
ctx,
input.schema().as_ref(),
- extension_codec,
+ codec,
)?;
Ok(Arc::new(AsyncFuncExpr::try_new(
@@ -2036,7 +2171,7 @@ impl protobuf::PhysicalPlanNode {
fn try_from_explain_exec(
exec: &ExplainExec,
- _extension_codec: &dyn PhysicalExtensionCodec,
+ _codec: &dyn PhysicalExtensionCodec,
) -> Result<Self> {
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Explain(
@@ -2055,16 +2190,20 @@ impl protobuf::PhysicalPlanNode {
fn try_from_projection_exec(
exec: &ProjectionExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let expr = exec
.expr()
.iter()
- .map(|proj_expr| serialize_physical_expr(&proj_expr.expr, extension_codec))
+ .map(|proj_expr| {
+ proto_converter.physical_expr_to_proto(&proj_expr.expr, codec)
+ })
.collect::<Result<Vec<_>>>()?;
let expr_name = exec
.expr()
@@ -2084,11 +2223,13 @@ impl protobuf::PhysicalPlanNode {
fn try_from_analyze_exec(
exec: &AnalyzeExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Analyze(Box::new(
@@ -2104,20 +2245,22 @@ impl protobuf::PhysicalPlanNode {
fn try_from_filter_exec(
exec: &FilterExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Filter(Box::new(
protobuf::FilterExecNode {
input: Some(Box::new(input)),
- expr: Some(serialize_physical_expr(
- exec.predicate(),
- extension_codec,
- )?),
+ expr: Some(
+ proto_converter
+ .physical_expr_to_proto(exec.predicate(), codec)?,
+ ),
default_filter_selectivity: exec.default_selectivity() as u32,
projection: exec.projection().as_ref().map_or_else(Vec::new, |v| {
v.iter().map(|x| *x as u32).collect::<Vec<u32>>()
@@ -2130,11 +2273,13 @@ impl protobuf::PhysicalPlanNode {
fn try_from_global_limit_exec(
limit: &GlobalLimitExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
limit.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
@@ -2153,11 +2298,13 @@ impl protobuf::PhysicalPlanNode {
fn try_from_local_limit_exec(
limit: &LocalLimitExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
limit.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::LocalLimit(Box::new(
@@ -2171,22 +2318,25 @@ impl protobuf::PhysicalPlanNode {
fn try_from_hash_join_exec(
exec: &HashJoinExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.left().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.right().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let on: Vec<protobuf::JoinOn> = exec
.on()
.iter()
.map(|tuple| {
- let l = serialize_physical_expr(&tuple.0, extension_codec)?;
- let r = serialize_physical_expr(&tuple.1, extension_codec)?;
+ let l = proto_converter.physical_expr_to_proto(&tuple.0, codec)?;
+ let r = proto_converter.physical_expr_to_proto(&tuple.1, codec)?;
Ok::<_, DataFusionError>(protobuf::JoinOn {
left: Some(l),
right: Some(r),
@@ -2200,7 +2350,7 @@ impl protobuf::PhysicalPlanNode {
.as_ref()
.map(|f| {
let expression =
- serialize_physical_expr(f.expression(), extension_codec)?;
+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;
let column_indices = f
.column_indices()
.iter()
@@ -2248,22 +2398,25 @@ impl protobuf::PhysicalPlanNode {
fn try_from_symmetric_hash_join_exec(
exec: &SymmetricHashJoinExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.left().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.right().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let on = exec
.on()
.iter()
.map(|tuple| {
- let l = serialize_physical_expr(&tuple.0, extension_codec)?;
- let r = serialize_physical_expr(&tuple.1, extension_codec)?;
+ let l = proto_converter.physical_expr_to_proto(&tuple.0, codec)?;
+ let r = proto_converter.physical_expr_to_proto(&tuple.1, codec)?;
Ok::<_, DataFusionError>(protobuf::JoinOn {
left: Some(l),
right: Some(r),
@@ -2277,7 +2430,7 @@ impl protobuf::PhysicalPlanNode {
.as_ref()
.map(|f| {
let expression =
- serialize_physical_expr(f.expression(), extension_codec)?;
+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;
let column_indices = f
.column_indices()
.iter()
@@ -2314,10 +2467,10 @@ impl protobuf::PhysicalPlanNode {
.iter()
.map(|expr| {
Ok(protobuf::PhysicalSortExprNode {
- expr: Some(Box::new(serialize_physical_expr(
- &expr.expr,
- extension_codec,
- )?)),
+ expr: Some(Box::new(
+ proto_converter
+ .physical_expr_to_proto(&expr.expr, codec)?,
+ )),
asc: !expr.options.descending,
nulls_first: expr.options.nulls_first,
})
@@ -2334,10 +2487,10 @@ impl protobuf::PhysicalPlanNode {
.iter()
.map(|expr| {
Ok(protobuf::PhysicalSortExprNode {
- expr: Some(Box::new(serialize_physical_expr(
- &expr.expr,
- extension_codec,
- )?)),
+ expr: Some(Box::new(
+ proto_converter
+ .physical_expr_to_proto(&expr.expr, codec)?,
+ )),
asc: !expr.options.descending,
nulls_first: expr.options.nulls_first,
})
@@ -2366,22 +2519,25 @@ impl protobuf::PhysicalPlanNode {
fn try_from_sort_merge_join_exec(
exec: &SortMergeJoinExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.left().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.right().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let on = exec
.on()
.iter()
.map(|tuple| {
- let l = serialize_physical_expr(&tuple.0, extension_codec)?;
- let r = serialize_physical_expr(&tuple.1, extension_codec)?;
+ let l = proto_converter.physical_expr_to_proto(&tuple.0, codec)?;
+ let r = proto_converter.physical_expr_to_proto(&tuple.1, codec)?;
Ok::<_, DataFusionError>(protobuf::JoinOn {
left: Some(l),
right: Some(r),
@@ -2395,7 +2551,7 @@ impl protobuf::PhysicalPlanNode {
.as_ref()
.map(|f| {
let expression =
- serialize_physical_expr(f.expression(), extension_codec)?;
+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;
let column_indices = f
.column_indices()
.iter()
@@ -2435,7 +2591,7 @@ impl protobuf::PhysicalPlanNode {
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::SortMergeJoin(Box::new(
- protobuf::SortMergeJoinExecNode {
+ SortMergeJoinExecNode {
left: Some(Box::new(left)),
right: Some(Box::new(right)),
on,
@@ -2450,15 +2606,18 @@ impl protobuf::PhysicalPlanNode {
fn try_from_cross_join_exec(
exec: &CrossJoinExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.left().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.right().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::CrossJoin(Box::new(
@@ -2472,7 +2631,8 @@ impl protobuf::PhysicalPlanNode {
fn try_from_aggregate_exec(
exec: &AggregateExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
let groups: Vec<bool> = exec
.group_expr()
@@ -2492,13 +2652,15 @@ impl protobuf::PhysicalPlanNode {
let filter = exec
.filter_expr()
.iter()
- .map(|expr| serialize_maybe_filter(expr.to_owned(), extension_codec))
+ .map(|expr| serialize_maybe_filter(expr.to_owned(), codec, proto_converter))
.collect::<Result<Vec<_>>>()?;
let agg = exec
.aggr_expr()
.iter()
- .map(|expr| serialize_physical_aggr_expr(expr.to_owned(), extension_codec))
+ .map(|expr| {
+ serialize_physical_aggr_expr(expr.to_owned(), codec, proto_converter)
+ })
.collect::<Result<Vec<_>>>()?;
let agg_names = exec
@@ -2517,23 +2679,24 @@ impl protobuf::PhysicalPlanNode {
}
};
let input_schema = exec.input_schema();
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let null_expr = exec
.group_expr()
.null_expr()
.iter()
- .map(|expr| serialize_physical_expr(&expr.0, extension_codec))
+ .map(|expr| proto_converter.physical_expr_to_proto(&expr.0, codec))
.collect::<Result<Vec<_>>>()?;
let group_expr = exec
.group_expr()
.expr()
.iter()
- .map(|expr| serialize_physical_expr(&expr.0, extension_codec))
+ .map(|expr| proto_converter.physical_expr_to_proto(&expr.0, codec))
.collect::<Result<Vec<_>>>()?;
let limit = exec.limit_options().map(|config| protobuf::AggLimit {
@@ -2563,7 +2726,7 @@ impl protobuf::PhysicalPlanNode {
fn try_from_empty_exec(
empty: &EmptyExec,
- _extension_codec: &dyn PhysicalExtensionCodec,
+ _codec: &dyn PhysicalExtensionCodec,
) -> Result<Self> {
let schema = empty.schema().as_ref().try_into()?;
Ok(protobuf::PhysicalPlanNode {
@@ -2575,7 +2738,7 @@ impl protobuf::PhysicalPlanNode {
fn try_from_placeholder_row_exec(
empty: &PlaceholderRowExec,
- _extension_codec: &dyn PhysicalExtensionCodec,
+ _codec: &dyn PhysicalExtensionCodec,
) -> Result<Self> {
let schema = empty.schema().as_ref().try_into()?;
Ok(protobuf::PhysicalPlanNode {
@@ -2590,11 +2753,13 @@ impl protobuf::PhysicalPlanNode {
#[expect(deprecated)]
fn try_from_coalesce_batches_exec(
coalesce_batches: &CoalesceBatchesExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
coalesce_batches.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::CoalesceBatches(Box::new(
@@ -2609,7 +2774,8 @@ impl protobuf::PhysicalPlanNode {
fn try_from_data_source_exec(
data_source_exec: &DataSourceExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Option<Self>> {
let data_source = data_source_exec.data_source();
if let Some(maybe_csv) = data_source.as_any().downcast_ref::<FileScanConfig>() {
@@ -2620,7 +2786,8 @@ impl protobuf::PhysicalPlanNode {
protobuf::CsvScanExecNode {
base_conf: Some(serialize_file_scan_config(
maybe_csv,
- extension_codec,
+ codec,
+ proto_converter,
)?),
has_header: csv_config.has_header(),
delimiter: byte_to_string(
@@ -2661,7 +2828,8 @@ impl protobuf::PhysicalPlanNode {
protobuf::JsonScanExecNode {
base_conf: Some(serialize_file_scan_config(
scan_conf,
- extension_codec,
+ codec,
+ proto_converter,
)?),
},
)),
@@ -2675,14 +2843,15 @@ impl protobuf::PhysicalPlanNode {
{
let predicate = conf
.filter()
- .map(|pred| serialize_physical_expr(&pred, extension_codec))
+ .map(|pred| proto_converter.physical_expr_to_proto(&pred, codec))
.transpose()?;
return Ok(Some(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::ParquetScan(
protobuf::ParquetScanExecNode {
base_conf: Some(serialize_file_scan_config(
maybe_parquet,
- extension_codec,
+ codec,
+ proto_converter,
)?),
predicate,
parquet_options: Some(conf.table_parquet_options().try_into()?),
@@ -2700,7 +2869,8 @@ impl protobuf::PhysicalPlanNode {
protobuf::AvroScanExecNode {
base_conf: Some(serialize_file_scan_config(
maybe_avro,
- extension_codec,
+ codec,
+ proto_converter,
)?),
},
)),
@@ -2733,7 +2903,8 @@ impl protobuf::PhysicalPlanNode {
.map(|ordering| {
let sort_exprs = serialize_physical_sort_exprs(
ordering.to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok::<_, DataFusionError>(protobuf::PhysicalSortExprNodeCollection {
physical_sort_expr_nodes: sort_exprs,
@@ -2760,11 +2931,13 @@ impl protobuf::PhysicalPlanNode {
fn try_from_coalesce_partitions_exec(
exec: &CoalescePartitionsExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Merge(Box::new(
@@ -2778,15 +2951,17 @@ impl protobuf::PhysicalPlanNode {
fn try_from_repartition_exec(
exec: &RepartitionExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let pb_partitioning =
- serialize_partitioning(exec.partitioning(), extension_codec)?;
+ serialize_partitioning(exec.partitioning(), codec, proto_converter)?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Repartition(Box::new(
@@ -2800,21 +2975,18 @@ impl protobuf::PhysicalPlanNode {
fn try_from_sort_exec(
exec: &SortExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
- exec.input().to_owned(),
- extension_codec,
- )?;
+ let input = proto_converter.execution_plan_to_proto(exec.input(), codec)?;
let expr = exec
.expr()
.iter()
.map(|expr| {
let sort_expr = Box::new(protobuf::PhysicalSortExprNode {
- expr: Some(Box::new(serialize_physical_expr(
- &expr.expr,
- extension_codec,
- )?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(&expr.expr, codec)?,
+ )),
asc: !expr.options.descending,
nulls_first: expr.options.nulls_first,
});
@@ -2840,14 +3012,18 @@ impl protobuf::PhysicalPlanNode {
fn try_from_union_exec(
union: &UnionExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];
for input in union.inputs() {
- inputs.push(protobuf::PhysicalPlanNode::try_from_physical_plan(
- input.to_owned(),
- extension_codec,
- )?);
+ inputs.push(
+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
+ input.to_owned(),
+ codec,
+ proto_converter,
+ )?,
+ );
}
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Union(protobuf::UnionExecNode {
@@ -2858,14 +3034,18 @@ impl protobuf::PhysicalPlanNode {
fn try_from_interleave_exec(
interleave: &InterleaveExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];
for input in interleave.inputs() {
- inputs.push(protobuf::PhysicalPlanNode::try_from_physical_plan(
- input.to_owned(),
- extension_codec,
- )?);
+ inputs.push(
+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
+ input.to_owned(),
+ codec,
+ proto_converter,
+ )?,
+ );
}
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Interleave(
@@ -2876,21 +3056,22 @@ impl protobuf::PhysicalPlanNode {
fn try_from_sort_preserving_merge_exec(
exec: &SortPreservingMergeExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let expr = exec
.expr()
.iter()
.map(|expr| {
let sort_expr = Box::new(protobuf::PhysicalSortExprNode {
- expr: Some(Box::new(serialize_physical_expr(
- &expr.expr,
- extension_codec,
- )?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(&expr.expr, codec)?,
+ )),
asc: !expr.options.descending,
nulls_first: expr.options.nulls_first,
});
@@ -2912,15 +3093,18 @@ impl protobuf::PhysicalPlanNode {
fn try_from_nested_loop_join_exec(
exec: &NestedLoopJoinExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.left().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.right().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let join_type: protobuf::JoinType = exec.join_type().to_owned().into();
@@ -2929,7 +3113,7 @@ impl protobuf::PhysicalPlanNode {
.as_ref()
.map(|f| {
let expression =
- serialize_physical_expr(f.expression(), extension_codec)?;
+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;
let column_indices = f
.column_indices()
.iter()
@@ -2967,23 +3151,25 @@ impl protobuf::PhysicalPlanNode {
fn try_from_window_agg_exec(
exec: &WindowAggExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let window_expr = exec
.window_expr()
.iter()
- .map(|e| serialize_physical_window_expr(e, extension_codec))
+ .map(|e| serialize_physical_window_expr(e, codec, proto_converter))
.collect::<Result<Vec<protobuf::PhysicalWindowExprNode>>>()?;
let partition_keys = exec
.partition_keys()
.iter()
- .map(|e| serialize_physical_expr(e, extension_codec))
+ .map(|e| proto_converter.physical_expr_to_proto(e, codec))
.collect::<Result<Vec<protobuf::PhysicalExprNode>>>()?;
Ok(protobuf::PhysicalPlanNode {
@@ -3000,23 +3186,25 @@ impl protobuf::PhysicalPlanNode {
fn try_from_bounded_window_agg_exec(
exec: &BoundedWindowAggExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let window_expr = exec
.window_expr()
.iter()
- .map(|e| serialize_physical_window_expr(e, extension_codec))
+ .map(|e| serialize_physical_window_expr(e, codec, proto_converter))
.collect::<Result<Vec<protobuf::PhysicalWindowExprNode>>>()?;
let partition_keys = exec
.partition_keys()
.iter()
- .map(|e| serialize_physical_expr(e, extension_codec))
+ .map(|e| proto_converter.physical_expr_to_proto(e, codec))
.collect::<Result<Vec<protobuf::PhysicalExprNode>>>()?;
let input_order_mode = match &exec.input_order_mode {
@@ -3049,12 +3237,14 @@ impl protobuf::PhysicalPlanNode {
fn try_from_data_sink_exec(
exec: &DataSinkExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Option<Self>> {
let input: protobuf::PhysicalPlanNode =
- protobuf::PhysicalPlanNode::try_from_physical_plan(
+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let sort_order = match exec.sort_order() {
Some(requirements) => {
@@ -3063,10 +3253,10 @@ impl protobuf::PhysicalPlanNode {
.map(|requirement| {
let expr: PhysicalSortExpr = requirement.to_owned().into();
let sort_expr = protobuf::PhysicalSortExprNode {
- expr: Some(Box::new(serialize_physical_expr(
- &expr.expr,
- extension_codec,
- )?)),
+ expr: Some(Box::new(
+ proto_converter
+ .physical_expr_to_proto(&expr.expr, codec)?,
+ )),
asc: !expr.options.descending,
nulls_first: expr.options.nulls_first,
};
@@ -3126,11 +3316,13 @@ impl protobuf::PhysicalPlanNode {
fn try_from_unnest_exec(
exec: &UnnestExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
@@ -3159,11 +3351,13 @@ impl protobuf::PhysicalPlanNode {
fn try_from_cooperative_exec(
exec: &CooperativeExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
exec.input().to_owned(),
- extension_codec,
+ codec,
+ proto_converter,
)?;
Ok(protobuf::PhysicalPlanNode {
@@ -3292,18 +3486,21 @@ impl protobuf::PhysicalPlanNode {
fn try_from_async_func_exec(
exec: &AsyncFuncExec,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Self> {
- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
Arc::clone(exec.input()),
- extension_codec,
+ codec,
+ proto_converter,
)?;
let mut async_exprs = vec![];
let mut async_expr_names = vec![];
for async_expr in exec.async_exprs() {
- async_exprs.push(serialize_physical_expr(&async_expr.func, extension_codec)?);
+ async_exprs
+ .push(proto_converter.physical_expr_to_proto(&async_expr.func, codec)?);
async_expr_names.push(async_expr.name.clone())
}
@@ -3333,12 +3530,12 @@ pub trait AsExecutionPlan: Debug + Send + Sync + Clone {
&self,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
) -> Result<Arc<dyn ExecutionPlan>>;
fn try_from_physical_plan(
plan: Arc<dyn ExecutionPlan>,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
) -> Result<Self>
where
Self: Sized;
@@ -3419,6 +3616,38 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {
}
}
+/// Controls the conversion of physical plans and expressions to and from their
+/// Protobuf variants. Using this trait, users can perform optimizations on the
+/// conversion process or collect performance metrics.
+pub trait PhysicalProtoConverterExtension {
+ fn proto_to_execution_plan(
+ &self,
+ ctx: &TaskContext,
+ codec: &dyn PhysicalExtensionCodec,
+ proto: &protobuf::PhysicalPlanNode,
+ ) -> Result<Arc<dyn ExecutionPlan>>;
+
+ fn execution_plan_to_proto(
+ &self,
+ plan: &Arc<dyn ExecutionPlan>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<protobuf::PhysicalPlanNode>;
+
+ fn proto_to_physical_expr(
+ &self,
+ proto: &protobuf::PhysicalExprNode,
+ ctx: &TaskContext,
+ input_schema: &Schema,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<Arc<dyn PhysicalExpr>>;
+
+ fn physical_expr_to_proto(
+ &self,
+ expr: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<protobuf::PhysicalExprNode>;
+}
+
/// DataEncoderTuple captures the position of the encoder
/// in the codec list that was used to encode the data and actual encoded data
#[derive(Clone, PartialEq, prost::Message)]
@@ -3432,6 +3661,55 @@ struct DataEncoderTuple {
pub blob: Vec<u8>,
}
+pub struct DefaultPhysicalProtoConverter;
+impl PhysicalProtoConverterExtension for DefaultPhysicalProtoConverter {
+ fn proto_to_execution_plan(
+ &self,
+ ctx: &TaskContext,
+ codec: &dyn PhysicalExtensionCodec,
+ proto: &protobuf::PhysicalPlanNode,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ proto.try_into_physical_plan_with_converter(ctx, codec, self)
+ }
+
+ fn execution_plan_to_proto(
+ &self,
+ plan: &Arc<dyn ExecutionPlan>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<protobuf::PhysicalPlanNode>
+ where
+ Self: Sized,
+ {
+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
+ Arc::clone(plan),
+ codec,
+ self,
+ )
+ }
+
+ fn proto_to_physical_expr(
+ &self,
+ proto: &protobuf::PhysicalExprNode,
+ ctx: &TaskContext,
+ input_schema: &Schema,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<Arc<dyn PhysicalExpr>>
+ where
+ Self: Sized,
+ {
+ // Default implementation calls the free function
+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)
+ }
+
+ fn physical_expr_to_proto(
+ &self,
+ expr: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<protobuf::PhysicalExprNode> {
+ serialize_physical_expr_with_converter(expr, codec, self)
+ }
+}
+
/// A PhysicalExtensionCodec that tries one of multiple inner codecs
/// until one works
#[derive(Debug)]
@@ -3534,10 +3812,11 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {
fn into_physical_plan(
node: &Option<Box<protobuf::PhysicalPlanNode>>,
ctx: &TaskContext,
- extension_codec: &dyn PhysicalExtensionCodec,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
if let Some(field) = node {
- field.try_into_physical_plan(ctx, extension_codec)
+ proto_converter.proto_to_execution_plan(ctx, codec, field)
} else {
Err(proto_error("Missing required field in protobuf"))
}
diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs
index 9558effb8a..f85b1d1e12 100644
--- a/datafusion/proto/src/physical_plan/to_proto.rs
+++ b/datafusion/proto/src/physical_plan/to_proto.rs
@@ -24,8 +24,7 @@ use datafusion_common::{
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
};
use datafusion_datasource::file_scan_config::FileScanConfig;
-use datafusion_datasource::file_sink_config::FileSink;
-use datafusion_datasource::file_sink_config::FileSinkConfig;
+use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};
use datafusion_datasource::{FileRange, PartitionedFile};
use datafusion_datasource_csv::file_format::CsvSink;
use datafusion_datasource_json::file_format::JsonSink;
@@ -36,31 +35,37 @@ use datafusion_physical_expr::ScalarFunctionExpr;
use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr;
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
-use datafusion_physical_plan::expressions::LikeExpr;
use datafusion_physical_plan::expressions::{
BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,
- Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,
+ LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,
};
use datafusion_physical_plan::joins::{HashExpr, HashTableLookupExpr};
use datafusion_physical_plan::udaf::AggregateFunctionExpr;
use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};
use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};
+use super::{
+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
+};
use crate::protobuf::{
self, PhysicalSortExprNode, PhysicalSortExprNodeCollection,
physical_aggregate_expr_node, physical_window_expr_node,
};
-use super::PhysicalExtensionCodec;
-
#[expect(clippy::needless_pass_by_value)]
pub fn serialize_physical_aggr_expr(
aggr_expr: Arc<AggregateFunctionExpr>,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::PhysicalExprNode> {
- let expressions = serialize_physical_exprs(&aggr_expr.expressions(), codec)?;
- let order_bys =
- serialize_physical_sort_exprs(aggr_expr.order_bys().iter().cloned(), codec)?;
+ let expressions =
+ serialize_physical_exprs(&aggr_expr.expressions(), codec, proto_converter)?;
+ let order_bys = serialize_physical_sort_exprs(
+ aggr_expr.order_bys().iter().cloned(),
+ codec,
+ proto_converter,
+ )?;
let name = aggr_expr.fun().name().to_string();
let mut buf = Vec::new();
@@ -100,6 +105,7 @@ fn serialize_physical_window_aggr_expr(
pub fn serialize_physical_window_expr(
window_expr: &Arc<dyn WindowExpr>,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::PhysicalWindowExprNode> {
let expr = window_expr.as_any();
let args = window_expr.expressions().to_vec();
@@ -155,9 +161,14 @@ pub fn serialize_physical_window_expr(
return not_impl_err!("WindowExpr not supported: {window_expr:?}");
};
- let args = serialize_physical_exprs(&args, codec)?;
- let partition_by = serialize_physical_exprs(window_expr.partition_by(), codec)?;
- let order_by = serialize_physical_sort_exprs(window_expr.order_by().to_vec(), codec)?;
+ let args = serialize_physical_exprs(&args, codec, proto_converter)?;
+ let partition_by =
+ serialize_physical_exprs(window_expr.partition_by(), codec, proto_converter)?;
+ let order_by = serialize_physical_sort_exprs(
+ window_expr.order_by().to_vec(),
+ codec,
+ proto_converter,
+ )?;
let window_frame: protobuf::WindowFrame = window_frame
.as_ref()
.try_into()
@@ -179,22 +190,24 @@ pub fn serialize_physical_window_expr(
pub fn serialize_physical_sort_exprs<I>(
sort_exprs: I,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Vec<PhysicalSortExprNode>>
where
I: IntoIterator<Item = PhysicalSortExpr>,
{
sort_exprs
.into_iter()
- .map(|sort_expr| serialize_physical_sort_expr(sort_expr, codec))
+ .map(|sort_expr| serialize_physical_sort_expr(sort_expr, codec, proto_converter))
.collect()
}
pub fn serialize_physical_sort_expr(
sort_expr: PhysicalSortExpr,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<PhysicalSortExprNode> {
let PhysicalSortExpr { expr, options } = sort_expr;
- let expr = serialize_physical_expr(&expr, codec)?;
+ let expr = proto_converter.physical_expr_to_proto(&expr, codec)?;
Ok(PhysicalSortExprNode {
expr: Some(Box::new(expr)),
asc: !options.descending,
@@ -205,13 +218,14 @@ pub fn serialize_physical_sort_expr(
pub fn serialize_physical_exprs<'a, I>(
values: I,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Vec<protobuf::PhysicalExprNode>>
where
I: IntoIterator<Item = &'a Arc<dyn PhysicalExpr>>,
{
values
.into_iter()
- .map(|value| serialize_physical_expr(value, codec))
+ .map(|value| proto_converter.physical_expr_to_proto(value, codec))
.collect()
}
@@ -222,6 +236,24 @@ where
pub fn serialize_physical_expr(
value: &Arc<dyn PhysicalExpr>,
codec: &dyn PhysicalExtensionCodec,
+) -> Result<protobuf::PhysicalExprNode> {
+ serialize_physical_expr_with_converter(
+ value,
+ codec,
+ &DefaultPhysicalProtoConverter {},
+ )
+}
+
+/// Serialize a `PhysicalExpr` to default protobuf representation.
+///
+/// If required, a [`PhysicalExtensionCodec`] can be provided which can handle
+/// serialization of udfs requiring specialized serialization (see [`PhysicalExtensionCodec::try_encode_udf`]).
+/// A [`PhysicalProtoConverterExtension`] can be provided to handle the
+/// conversion process (see [`PhysicalProtoConverterExtension::physical_expr_to_proto`]).
+pub fn serialize_physical_expr_with_converter(
+ value: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::PhysicalExprNode> {
// Snapshot the expr in case it has dynamic predicate state so
// it can be serialized
@@ -271,8 +303,12 @@ pub fn serialize_physical_expr(
})
} else if let Some(expr) = expr.downcast_ref::<BinaryExpr>() {
let binary_expr = Box::new(protobuf::PhysicalBinaryExprNode {
- l: Some(Box::new(serialize_physical_expr(expr.left(), codec)?)),
- r: Some(Box::new(serialize_physical_expr(expr.right(), codec)?)),
+ l: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.left(), codec)?,
+ )),
+ r: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.right(), codec)?,
+ )),
op: format!("{:?}", expr.op()),
});
@@ -290,14 +326,21 @@ pub fn serialize_physical_expr(
expr: expr
.expr()
.map(|exp| {
- serialize_physical_expr(exp, codec).map(Box::new)
+ proto_converter
+ .physical_expr_to_proto(exp, codec)
+ .map(Box::new)
})
.transpose()?,
when_then_expr: expr
.when_then_expr()
.iter()
.map(|(when_expr, then_expr)| {
- serialize_when_then_expr(when_expr, then_expr, codec)
+ serialize_when_then_expr(
+ when_expr,
+ then_expr,
+ codec,
+ proto_converter,
+ )
})
.collect::<Result<
Vec<protobuf::PhysicalWhenThen>,
@@ -305,7 +348,11 @@ pub fn serialize_physical_expr(
>>()?,
else_expr: expr
.else_expr()
- .map(|a| serialize_physical_expr(a, codec).map(Box::new))
+ .map(|a| {
+ proto_converter
+ .physical_expr_to_proto(a, codec)
+ .map(Box::new)
+ })
.transpose()?,
},
),
@@ -316,7 +363,9 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::NotExpr(Box::new(
protobuf::PhysicalNot {
- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,
+ )),
},
))),
})
@@ -324,7 +373,9 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::IsNullExpr(
Box::new(protobuf::PhysicalIsNull {
- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,
+ )),
}),
)),
})
@@ -332,7 +383,9 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::IsNotNullExpr(
Box::new(protobuf::PhysicalIsNotNull {
- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,
+ )),
}),
)),
})
@@ -340,8 +393,10 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::InList(Box::new(
protobuf::PhysicalInListNode {
- expr: Some(Box::new(serialize_physical_expr(expr.expr(), codec)?)),
- list: serialize_physical_exprs(expr.list(), codec)?,
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.expr(), codec)?,
+ )),
+ list: serialize_physical_exprs(expr.list(), codec, proto_converter)?,
negated: expr.negated(),
},
))),
@@ -350,7 +405,9 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::Negative(Box::new(
protobuf::PhysicalNegativeNode {
- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,
+ )),
},
))),
})
@@ -364,7 +421,9 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::Cast(Box::new(
protobuf::PhysicalCastNode {
- expr: Some(Box::new(serialize_physical_expr(cast.expr(), codec)?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(cast.expr(), codec)?,
+ )),
arrow_type: Some(cast.cast_type().try_into()?),
},
))),
@@ -373,7 +432,9 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::TryCast(Box::new(
protobuf::PhysicalTryCastNode {
- expr: Some(Box::new(serialize_physical_expr(cast.expr(), codec)?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(cast.expr(), codec)?,
+ )),
arrow_type: Some(cast.cast_type().try_into()?),
},
))),
@@ -385,7 +446,7 @@ pub fn serialize_physical_expr(
expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarUdf(
protobuf::PhysicalScalarUdfNode {
name: expr.name().to_string(),
- args: serialize_physical_exprs(expr.args(), codec)?,
+ args: serialize_physical_exprs(expr.args(), codec, proto_converter)?,
fun_definition: (!buf.is_empty()).then_some(buf),
return_type: Some(expr.return_type().try_into()?),
nullable: expr.nullable(),
@@ -402,11 +463,12 @@ pub fn serialize_physical_expr(
protobuf::PhysicalLikeExprNode {
negated: expr.negated(),
case_insensitive: expr.case_insensitive(),
- expr: Some(Box::new(serialize_physical_expr(expr.expr(), codec)?)),
- pattern: Some(Box::new(serialize_physical_expr(
- expr.pattern(),
- codec,
- )?)),
+ expr: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.expr(), codec)?,
+ )),
+ pattern: Some(Box::new(
+ proto_converter.physical_expr_to_proto(expr.pattern(), codec)?,
+ )),
},
))),
})
@@ -415,7 +477,11 @@ pub fn serialize_physical_expr(
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(
protobuf::PhysicalHashExprNode {
- on_columns: serialize_physical_exprs(expr.on_columns(), codec)?,
+ on_columns: serialize_physical_exprs(
+ expr.on_columns(),
+ codec,
+ proto_converter,
+ )?,
seed0: s0,
seed1: s1,
seed2: s2,
@@ -431,7 +497,7 @@ pub fn serialize_physical_expr(
let inputs: Vec<protobuf::PhysicalExprNode> = value
.children()
.into_iter()
- .map(|e| serialize_physical_expr(e, codec))
+ .map(|e| proto_converter.physical_expr_to_proto(e, codec))
.collect::<Result<_>>()?;
Ok(protobuf::PhysicalExprNode {
expr_type: Some(protobuf::physical_expr_node::ExprType::Extension(
@@ -449,6 +515,7 @@ pub fn serialize_physical_expr(
pub fn serialize_partitioning(
partitioning: &Partitioning,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::Partitioning> {
let serialized_partitioning = match partitioning {
Partitioning::RoundRobinBatch(partition_count) => protobuf::Partitioning {
@@ -457,7 +524,8 @@ pub fn serialize_partitioning(
)),
},
Partitioning::Hash(exprs, partition_count) => {
- let serialized_exprs = serialize_physical_exprs(exprs, codec)?;
+ let serialized_exprs =
+ serialize_physical_exprs(exprs, codec, proto_converter)?;
protobuf::Partitioning {
partition_method: Some(protobuf::partitioning::PartitionMethod::Hash(
protobuf::PhysicalHashRepartition {
@@ -480,10 +548,11 @@ fn serialize_when_then_expr(
when_expr: &Arc<dyn PhysicalExpr>,
then_expr: &Arc<dyn PhysicalExpr>,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::PhysicalWhenThen> {
Ok(protobuf::PhysicalWhenThen {
- when_expr: Some(serialize_physical_expr(when_expr, codec)?),
- then_expr: Some(serialize_physical_expr(then_expr, codec)?),
+ when_expr: Some(proto_converter.physical_expr_to_proto(when_expr, codec)?),
+ then_expr: Some(proto_converter.physical_expr_to_proto(then_expr, codec)?),
})
}
@@ -539,6 +608,7 @@ impl TryFrom<&[PartitionedFile]> for protobuf::FileGroup {
pub fn serialize_file_scan_config(
conf: &FileScanConfig,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::FileScanExecConf> {
let file_groups = conf
.file_groups
@@ -548,7 +618,8 @@ pub fn serialize_file_scan_config(
let mut output_orderings = vec![];
for order in &conf.output_ordering {
- let ordering = serialize_physical_sort_exprs(order.to_vec(), codec)?;
+ let ordering =
+ serialize_physical_sort_exprs(order.to_vec(), codec, proto_converter)?;
output_orderings.push(ordering)
}
@@ -563,8 +634,7 @@ pub fn serialize_file_scan_config(
fields.extend(conf.table_partition_cols().iter().cloned());
let schema = Arc::new(
- arrow::datatypes::Schema::new(fields.clone())
- .with_metadata(conf.file_schema().metadata.clone()),
+ Schema::new(fields.clone()).with_metadata(conf.file_schema().metadata.clone()),
);
let projection_exprs = conf
@@ -579,7 +649,10 @@ pub fn serialize_file_scan_config(
.map(|expr| {
Ok(protobuf::ProjectionExpr {
alias: expr.alias.to_string(),
- expr: Some(serialize_physical_expr(&expr.expr, codec)?),
+ expr: Some(
+ proto_converter
+ .physical_expr_to_proto(&expr.expr, codec)?,
+ ),
})
})
.collect::<Result<Vec<_>>>()?,
@@ -614,11 +687,12 @@ pub fn serialize_file_scan_config(
pub fn serialize_maybe_filter(
expr: Option<Arc<dyn PhysicalExpr>>,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::MaybeFilter> {
match expr {
None => Ok(protobuf::MaybeFilter { expr: None }),
Some(expr) => Ok(protobuf::MaybeFilter {
- expr: Some(serialize_physical_expr(&expr, codec)?),
+ expr: Some(proto_converter.physical_expr_to_proto(&expr, codec)?),
}),
}
}
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index b54b7030fc..f262020ab8 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -18,29 +18,12 @@
use std::any::Any;
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
-
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
use std::vec;
-use crate::cases::{
- CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,
- MyRegexUdfNode,
-};
-
use arrow::array::RecordBatch;
use arrow::csv::WriterBuilder;
use arrow::datatypes::{Fields, TimeUnit};
-use datafusion::physical_expr::aggregate::AggregateExprBuilder;
-#[expect(deprecated)]
-use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
-use datafusion::physical_plan::metrics::MetricType;
-use datafusion_datasource::TableSchema;
-use datafusion_expr::dml::InsertOp;
-use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
-use datafusion_functions_aggregate::array_agg::array_agg_udaf;
-use datafusion_functions_aggregate::min_max::max_udaf;
-use prost::Message;
-
use datafusion::arrow::array::ArrayRef;
use datafusion::arrow::compute::kernels::sort::SortOptions;
use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};
@@ -64,6 +47,7 @@ use datafusion::functions_aggregate::sum::sum_udaf;
use datafusion::functions_window::nth_value::nth_value_udwf;
use datafusion::functions_window::row_number::row_number_udwf;
use datafusion::logical_expr::{JoinType, Operator, Volatility, create_udf};
+use datafusion::physical_expr::aggregate::AggregateExprBuilder;
use datafusion::physical_expr::expressions::Literal;
use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
use datafusion::physical_expr::{
@@ -73,6 +57,8 @@ use datafusion::physical_plan::aggregates::{
AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,
};
use datafusion::physical_plan::analyze::AnalyzeExec;
+#[expect(deprecated)]
+use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::physical_plan::empty::EmptyExec;
use datafusion::physical_plan::expressions::{
@@ -84,6 +70,7 @@ use datafusion::physical_plan::joins::{
StreamJoinPartitionMode, SymmetricHashJoinExec,
};
use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
+use datafusion::physical_plan::metrics::MetricType;
use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion::physical_plan::repartition::RepartitionExec;
@@ -105,22 +92,41 @@ use datafusion_common::file_options::json_writer::JsonWriterOptions;
use datafusion_common::parsers::CompressionTypeVariant;
use datafusion_common::stats::Precision;
use datafusion_common::{
- DataFusionError, NullEquality, Result, UnnestOptions, internal_datafusion_err,
- internal_err, not_impl_err,
+ DataFusionError, NullEquality, Result, UnnestOptions, exec_datafusion_err,
+ internal_datafusion_err, internal_err, not_impl_err,
};
+use datafusion_datasource::TableSchema;
use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
+use datafusion_expr::dml::InsertOp;
use datafusion_expr::{
Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue,
ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,
WindowFrame, WindowFrameBound, WindowUDF,
};
+use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
+use datafusion_functions_aggregate::array_agg::array_agg_udaf;
use datafusion_functions_aggregate::average::avg_udaf;
+use datafusion_functions_aggregate::min_max::max_udaf;
use datafusion_functions_aggregate::nth_value::nth_value_udaf;
use datafusion_functions_aggregate::string_agg::string_agg_udaf;
+use datafusion_proto::bytes::{
+ physical_plan_from_bytes_with_proto_converter,
+ physical_plan_to_bytes_with_proto_converter,
+};
+use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;
+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
use datafusion_proto::physical_plan::{
- AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,
+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+ PhysicalExtensionCodec, PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf;
+use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};
+use prost::Message;
+
+use crate::cases::{
+ CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,
+ MyRegexUdfNode,
};
-use datafusion_proto::protobuf::{self, PhysicalPlanNode};
/// Perform a serde roundtrip and assert that the string representation of the before and after plans
/// are identical. Note that this often isn't sufficient to guarantee that no information is
@@ -128,7 +134,8 @@ use datafusion_proto::protobuf::{self, PhysicalPlanNode};
fn roundtrip_test(exec_plan: Arc<dyn ExecutionPlan>) -> Result<()> {
let ctx = SessionContext::new();
let codec = DefaultPhysicalExtensionCodec {};
- roundtrip_test_and_return(exec_plan, &ctx, &codec)?;
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
Ok(())
}
@@ -142,13 +149,19 @@ fn roundtrip_test_and_return(
exec_plan: Arc<dyn ExecutionPlan>,
ctx: &SessionContext,
codec: &dyn PhysicalExtensionCodec,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- let proto: protobuf::PhysicalPlanNode =
- protobuf::PhysicalPlanNode::try_from_physical_plan(exec_plan.clone(), codec)
- .expect("to proto");
- let result_exec_plan: Arc<dyn ExecutionPlan> = proto
- .try_into_physical_plan(&ctx.task_ctx(), codec)
- .expect("from proto");
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&exec_plan),
+ codec,
+ proto_converter,
+ )?;
+ let result_exec_plan = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ codec,
+ proto_converter,
+ )?;
pretty_assertions::assert_eq!(
format!("{exec_plan:?}"),
@@ -168,7 +181,8 @@ fn roundtrip_test_with_context(
ctx: &SessionContext,
) -> Result<()> {
let codec = DefaultPhysicalExtensionCodec {};
- roundtrip_test_and_return(exec_plan, ctx, &codec)?;
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(exec_plan, ctx, &codec, &proto_converter)?;
Ok(())
}
@@ -176,9 +190,10 @@ fn roundtrip_test_with_context(
/// query results are identical.
async fn roundtrip_test_sql_with_context(sql: &str, ctx: &SessionContext) -> Result<()> {
let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;
- roundtrip_test_and_return(initial_plan, ctx, &codec)?;
+ roundtrip_test_and_return(initial_plan, ctx, &codec, &proto_converter)?;
Ok(())
}
@@ -988,7 +1003,7 @@ fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {
}
impl Display for CustomPredicateExpr {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "CustomPredicateExpr")
}
}
@@ -1081,7 +1096,12 @@ fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {
let exec_plan = DataSourceExec::from_data_source(scan_config);
let ctx = SessionContext::new();
- roundtrip_test_and_return(exec_plan, &ctx, &CustomPhysicalExtensionCodec {})?;
+ roundtrip_test_and_return(
+ exec_plan,
+ &ctx,
+ &CustomPhysicalExtensionCodec {},
+ &DefaultPhysicalProtoConverter {},
+ )?;
Ok(())
}
@@ -1287,7 +1307,8 @@ fn roundtrip_scalar_udf_extension_codec() -> Result<()> {
)?);
let ctx = SessionContext::new();
- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec)?;
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;
Ok(())
}
@@ -1334,7 +1355,8 @@ fn roundtrip_udwf_extension_codec() -> Result<()> {
)?);
let ctx = SessionContext::new();
- roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec)?;
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec, &proto_converter)?;
Ok(())
}
@@ -1405,7 +1427,8 @@ fn roundtrip_aggregate_udf_extension_codec() -> Result<()> {
)?);
let ctx = SessionContext::new();
- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec)?;
+ let proto_converter = DefaultPhysicalProtoConverter {};
+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;
Ok(())
}
@@ -1529,12 +1552,14 @@ fn roundtrip_csv_sink() -> Result<()> {
let ctx = SessionContext::new();
let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = DefaultPhysicalProtoConverter {};
+
let roundtrip_plan = roundtrip_test_and_return(
Arc::new(DataSinkExec::new(input, data_sink, Some(sort_order))),
&ctx,
&codec,
- )
- .unwrap();
+ &proto_converter,
+ )?;
let roundtrip_plan = roundtrip_plan
.as_any()
@@ -1976,6 +2001,7 @@ async fn test_serialize_deserialize_tpch_queries() -> Result<()> {
// serialize the physical plan
let codec = DefaultPhysicalExtensionCodec {};
+
let proto =
PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;
@@ -2097,6 +2123,7 @@ async fn test_tpch_part_in_list_query_with_real_parquet_data() -> Result<()> {
// Serialize the physical plan - bug may happen here already but not necessarily manifests
let codec = DefaultPhysicalExtensionCodec {};
+
let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;
// This will fail with the bug, but should succeed when fixed
@@ -2338,9 +2365,8 @@ async fn roundtrip_async_func_exec() -> Result<()> {
/// it's a performance optimization filter, not a correctness requirement.
#[test]
fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {
- use datafusion::physical_plan::joins::HashTableLookupExpr;
- use datafusion::physical_plan::joins::Map;
use datafusion::physical_plan::joins::join_hash_map::JoinHashMapU32;
+ use datafusion::physical_plan::joins::{HashTableLookupExpr, Map};
// Create a simple schema and input plan
let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));
@@ -2362,8 +2388,9 @@ fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {
// Serialize
let ctx = SessionContext::new();
let codec = DefaultPhysicalExtensionCodec {};
- let proto: protobuf::PhysicalPlanNode =
- protobuf::PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec)
+
+ let proto: PhysicalPlanNode =
+ PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec)
.expect("serialization should succeed");
// Deserialize
@@ -2413,3 +2440,124 @@ fn roundtrip_hash_expr() -> Result<()> {
);
roundtrip_test(filter)
}
+
+#[test]
+fn custom_proto_converter_intercepts() -> Result<()> {
+ #[derive(Default)]
+ struct CustomConverterInterceptor {
+ num_proto_plans: RwLock<usize>,
+ num_physical_plans: RwLock<usize>,
+ num_proto_exprs: RwLock<usize>,
+ num_physical_exprs: RwLock<usize>,
+ }
+
+ impl PhysicalProtoConverterExtension for CustomConverterInterceptor {
+ fn proto_to_execution_plan(
+ &self,
+ ctx: &TaskContext,
+ codec: &dyn PhysicalExtensionCodec,
+ proto: &protobuf::PhysicalPlanNode,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ {
+ let mut counter = self
+ .num_proto_plans
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ proto.try_into_physical_plan_with_converter(ctx, codec, self)
+ }
+
+ fn execution_plan_to_proto(
+ &self,
+ plan: &Arc<dyn ExecutionPlan>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<protobuf::PhysicalPlanNode>
+ where
+ Self: Sized,
+ {
+ {
+ let mut counter = self
+ .num_physical_plans
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ PhysicalPlanNode::try_from_physical_plan_with_converter(
+ Arc::clone(plan),
+ codec,
+ self,
+ )
+ }
+
+ fn proto_to_physical_expr(
+ &self,
+ proto: &PhysicalExprNode,
+ ctx: &TaskContext,
+ input_schema: &Schema,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<Arc<dyn PhysicalExpr>>
+ where
+ Self: Sized,
+ {
+ {
+ let mut counter = self
+ .num_proto_exprs
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)
+ }
+
+ fn physical_expr_to_proto(
+ &self,
+ expr: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+ ) -> Result<PhysicalExprNode> {
+ {
+ let mut counter = self
+ .num_physical_exprs
+ .write()
+ .map_err(|err| exec_datafusion_err!("{err}"))?;
+ *counter += 1;
+ }
+ serialize_physical_expr_with_converter(expr, codec, self)
+ }
+ }
+
+ let field_a = Field::new("a", DataType::Boolean, false);
+ let field_b = Field::new("b", DataType::Int64, false);
+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));
+ let sort_exprs = [
+ PhysicalSortExpr {
+ expr: col("a", &schema)?,
+ options: SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ },
+ PhysicalSortExpr {
+ expr: col("b", &schema)?,
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ },
+ ]
+ .into();
+
+ let exec_plan = Arc::new(SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema))));
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let proto_converter = CustomConverterInterceptor::default();
+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;
+
+ assert_eq!(*proto_converter.num_proto_exprs.read().unwrap(), 2);
+ assert_eq!(*proto_converter.num_physical_exprs.read().unwrap(), 2);
+ assert_eq!(*proto_converter.num_proto_plans.read().unwrap(), 2);
+ assert_eq!(*proto_converter.num_physical_plans.read().unwrap(), 2);
+
+ Ok(())
+}
diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md
index 6043c81d14..447a773eee 100644
--- a/docs/source/library-user-guide/upgrading.md
+++ b/docs/source/library-user-guide/upgrading.md
@@ -154,6 +154,66 @@ The builder pattern is more efficient as it computes properties once during `bui
Note: `with_default_selectivity()` is not deprecated as it simply updates a field value and does not require the overhead of the builder pattern.
+### Protobuf conversion trait added
+
+A new trait, `PhysicalProtoConverterExtension`, has been added to the `datafusion-proto`
+crate. This is used for controlling the process of conversion of physical plans and
+expressions to and from their protobuf equivalents. The methods for conversion now
+require an additional parameter.
+
+The primary APIs for interacting with this crate have not been modified, so most users
+should not need to make any changes. If you do require this trait, you can use the
+`DefaultPhysicalProtoConverter` implementation.
+
+For example, to convert a sort expression protobuf node you can make the following
+updates:
+
+**Before:**
+
+```rust,ignore
+let sort_expr = parse_physical_sort_expr(
+ sort_proto,
+ ctx,
+ input_schema,
+ codec,
+);
+```
+
+**After:**
+
+```rust,ignore
+let converter = DefaultPhysicalProtoConverter {};
+let sort_expr = parse_physical_sort_expr(
+ sort_proto,
+ ctx,
+ input_schema,
+ codec,
+ &converter
+);
+```
+
+Similarly to convert from a physical sort expression into a protobuf node:
+
+**Before:**
+
+```rust,ignore
+let sort_proto = serialize_physical_sort_expr(
+ sort_expr,
+ codec,
+);
+```
+
+**After:**
+
+```rust,ignore
+let converter = DefaultPhysicalProtoConverter {};
+let sort_proto = serialize_physical_sort_expr(
+ sort_expr,
+ codec,
+ &converter,
+);
+```
+
### `generate_series` and `range` table functions changed
The `generate_series` and `range` table functions now return an empty set when the interval is invalid, instead of an error.