Apache DataFusion SQL Query Engine
feat: implement protobuf converter trait to allow control over serialization and deserialization processes (#19437)
Tim Saucer41d48b3f6c8a
diff --git a/Cargo.lock b/Cargo.lockindex 9b87670734..b219b83cd7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2100,6 +2100,7 @@ dependencies = ["object_store","prost","rand 0.9.2",+ "serde","serde_json","strum","strum_macros",diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.tomlindex 368cc67641..4cbec17272 100644--- a/datafusion-examples/Cargo.toml+++ b/datafusion-examples/Cargo.toml@@ -62,6 +62,7 @@ mimalloc = { version = "0.1", default-features = false }object_store = { workspace = true, features = ["aws", "http"] }prost = { workspace = true }rand = { workspace = true }+serde = { version = "1", features = ["derive"] }serde_json = { workspace = true }strum = { workspace = true }strum_macros = { workspace = true }diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rsnew file mode 100644index 0000000000..f72c9d2b61--- /dev/null+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs@@ -0,0 +1,544 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! See `main.rs` for how to run it.+//!+//! This example demonstrates how to use the `PhysicalExtensionCodec` trait's+//! interception methods (`serialize_physical_plan` and `deserialize_physical_plan`)+//! to implement custom serialization logic.+//!+//! The key insight is that `FileScanConfig::expr_adapter_factory` is NOT serialized by+//! default. This example shows how to:+//! 1. Detect plans with custom adapters during serialization+//! 2. Wrap them as Extension nodes with JSON-serialized adapter metadata+//! 3. Unwrap and restore the adapter during deserialization+//!+//! This demonstrates nested serialization (protobuf outer, JSON inner) and the power+//! of the `PhysicalExtensionCodec` interception pattern. Both plan and expression+//! serialization route through the codec, enabling interception at every node in the tree.++use std::fmt::Debug;+use std::sync::Arc;++use arrow::array::record_batch;+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion::assert_batches_eq;+use datafusion::common::{Result, not_impl_err};+use datafusion::datasource::listing::{+ ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,+};+use datafusion::datasource::physical_plan::{FileScanConfig, FileScanConfigBuilder};+use datafusion::datasource::source::DataSourceExec;+use datafusion::execution::TaskContext;+use datafusion::execution::context::SessionContext;+use datafusion::execution::object_store::ObjectStoreUrl;+use datafusion::parquet::arrow::ArrowWriter;+use datafusion::physical_expr::PhysicalExpr;+use datafusion::physical_plan::ExecutionPlan;+use datafusion::prelude::SessionConfig;+use datafusion_physical_expr_adapter::{+ DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,+};+use datafusion_proto::bytes::{+ physical_plan_from_bytes_with_proto_converter,+ physical_plan_to_bytes_with_proto_converter,+};+use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;+use datafusion_proto::physical_plan::{+ PhysicalExtensionCodec, PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;+use datafusion_proto::protobuf::{+ PhysicalExprNode, PhysicalExtensionNode, PhysicalPlanNode,+};+use object_store::memory::InMemory;+use object_store::path::Path;+use object_store::{ObjectStore, PutPayload};+use prost::Message;+use serde::{Deserialize, Serialize};++/// Example showing how to preserve custom adapter information during plan serialization.+///+/// This demonstrates:+/// 1. Creating a custom PhysicalExprAdapter with metadata+/// 2. Using PhysicalExtensionCodec to intercept serialization+/// 3. Wrapping adapter info as Extension nodes+/// 4. Restoring adapters during deserialization+pub async fn adapter_serialization() -> Result<()> {+ println!("=== PhysicalExprAdapter Serialization Example ===\n");++ // Step 1: Create sample Parquet data in memory+ println!("Step 1: Creating sample Parquet data...");+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ let batch = record_batch!(("id", Int32, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))?;+ let path = Path::from("data.parquet");+ write_parquet(&store, &path, &batch).await?;++ // Step 2: Set up session with custom adapter+ println!("Step 2: Setting up session with custom adapter...");+ let logical_schema =+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));++ let mut cfg = SessionConfig::new();+ cfg.options_mut().execution.parquet.pushdown_filters = true;+ let ctx = SessionContext::new_with_config(cfg);+ ctx.runtime_env().register_object_store(+ ObjectStoreUrl::parse("memory://")?.as_ref(),+ Arc::clone(&store),+ );++ // Create a table with our custom MetadataAdapterFactory+ let adapter_factory = Arc::new(MetadataAdapterFactory::new("v1"));+ let listing_config =+ ListingTableConfig::new(ListingTableUrl::parse("memory:///data.parquet")?)+ .infer_options(&ctx.state())+ .await?+ .with_schema(logical_schema)+ .with_expr_adapter_factory(+ Arc::clone(&adapter_factory) as Arc<dyn PhysicalExprAdapterFactory>+ );+ let table = ListingTable::try_new(listing_config)?;+ ctx.register_table("my_table", Arc::new(table))?;++ // Step 3: Create physical plan with filter+ println!("Step 3: Creating physical plan with filter...");+ let df = ctx.sql("SELECT * FROM my_table WHERE id > 5").await?;+ let original_plan = df.create_physical_plan().await?;++ // Verify adapter is present in original plan+ let has_adapter_before = verify_adapter_in_plan(&original_plan, "original");+ println!(" Original plan has adapter: {has_adapter_before}");++ // Step 4: Serialize with our custom codec+ println!("\nStep 4: Serializing plan with AdapterPreservingCodec...");+ let codec = AdapterPreservingCodec;+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&original_plan),+ &codec,+ &codec,+ )?;+ println!(" Serialized {} bytes", bytes.len());+ println!(" (DataSourceExec with adapter was wrapped as PhysicalExtensionNode)");++ // Step 5: Deserialize with our custom codec+ println!("\nStep 5: Deserializing plan with AdapterPreservingCodec...");+ let task_ctx = ctx.task_ctx();+ let restored_plan =+ physical_plan_from_bytes_with_proto_converter(&bytes, &task_ctx, &codec, &codec)?;++ // Verify adapter is restored+ let has_adapter_after = verify_adapter_in_plan(&restored_plan, "restored");+ println!(" Restored plan has adapter: {has_adapter_after}");++ // Step 6: Execute and compare results+ println!("\nStep 6: Executing plans and comparing results...");+ let original_results =+ datafusion::physical_plan::collect(Arc::clone(&original_plan), task_ctx.clone())+ .await?;+ let restored_results =+ datafusion::physical_plan::collect(restored_plan, task_ctx).await?;++ #[rustfmt::skip]+ let expected = [+ "+----+",+ "| id |",+ "+----+",+ "| 6 |",+ "| 7 |",+ "| 8 |",+ "| 9 |",+ "| 10 |",+ "+----+",+ ];++ println!("\n Original plan results:");+ arrow::util::pretty::print_batches(&original_results)?;+ assert_batches_eq!(expected, &original_results);++ println!("\n Restored plan results:");+ arrow::util::pretty::print_batches(&restored_results)?;+ assert_batches_eq!(expected, &restored_results);++ println!("\n=== Example Complete! ===");+ println!("Key takeaways:");+ println!(+ " 1. PhysicalExtensionCodec provides serialize_physical_plan/deserialize_physical_plan hooks"+ );+ println!(" 2. Custom metadata can be wrapped as PhysicalExtensionNode");+ println!(" 3. Nested serialization (protobuf + JSON) works seamlessly");+ println!(+ " 4. Both plans produce identical results despite serialization round-trip"+ );+ println!(" 5. Adapters are fully preserved through the serialization round-trip");++ Ok(())+}++// ============================================================================+// MetadataAdapter - A simple custom adapter with a tag+// ============================================================================++/// A custom PhysicalExprAdapter that wraps another adapter.+/// The tag metadata is stored in the factory, not the adapter itself.+#[derive(Debug)]+struct MetadataAdapter {+ inner: Arc<dyn PhysicalExprAdapter>,+}++impl PhysicalExprAdapter for MetadataAdapter {+ fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {+ // Simply delegate to inner adapter+ self.inner.rewrite(expr)+ }+}++// ============================================================================+// MetadataAdapterFactory - Factory for creating MetadataAdapter instances+// ============================================================================++/// Factory for creating MetadataAdapter instances.+/// The tag is stored in the factory and extracted via Debug formatting in `extract_adapter_tag`.+#[derive(Debug)]+struct MetadataAdapterFactory {+ // Note: This field is read via Debug formatting in `extract_adapter_tag`.+ // Rust's dead code analysis doesn't recognize Debug-based field access.+ // In PR #19234, this field is used by `with_partition_values`, but that method+ // doesn't exist in upstream DataFusion's PhysicalExprAdapter trait.+ #[allow(dead_code)]+ tag: String,+}++impl MetadataAdapterFactory {+ fn new(tag: impl Into<String>) -> Self {+ Self { tag: tag.into() }+ }+}++impl PhysicalExprAdapterFactory for MetadataAdapterFactory {+ fn create(+ &self,+ logical_file_schema: SchemaRef,+ physical_file_schema: SchemaRef,+ ) -> Arc<dyn PhysicalExprAdapter> {+ let inner = DefaultPhysicalExprAdapterFactory+ .create(logical_file_schema, physical_file_schema);+ Arc::new(MetadataAdapter { inner })+ }+}++// ============================================================================+// AdapterPreservingCodec - Custom codec that preserves adapters+// ============================================================================++/// Extension payload structure for serializing adapter info+#[derive(Serialize, Deserialize)]+struct ExtensionPayload {+ /// Marker to identify this is our custom extension+ marker: String,+ /// JSON-serialized adapter metadata+ adapter_metadata: AdapterMetadata,+ /// Protobuf-serialized inner DataSourceExec (without adapter)+ inner_plan_bytes: Vec<u8>,+}++/// Metadata about the adapter to recreate it during deserialization+#[derive(Serialize, Deserialize)]+struct AdapterMetadata {+ /// The adapter tag (e.g., "v1")+ tag: String,+}++const EXTENSION_MARKER: &str = "adapter_preserving_extension_v1";++/// A codec that intercepts serialization to preserve adapter information.+#[derive(Debug)]+struct AdapterPreservingCodec;++impl PhysicalExtensionCodec for AdapterPreservingCodec {+ // Required method: decode custom extension nodes+ fn try_decode(+ &self,+ buf: &[u8],+ _inputs: &[Arc<dyn ExecutionPlan>],+ ctx: &TaskContext,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // Try to parse as our extension payload+ if let Ok(payload) = serde_json::from_slice::<ExtensionPayload>(buf)+ && payload.marker == EXTENSION_MARKER+ {+ // Decode the inner plan+ let inner_proto = PhysicalPlanNode::decode(&payload.inner_plan_bytes[..])+ .map_err(|e| {+ datafusion::error::DataFusionError::Plan(format!(+ "Failed to decode inner plan: {e}"+ ))+ })?;++ // Deserialize the inner plan using default implementation+ let inner_plan =+ inner_proto.try_into_physical_plan_with_converter(ctx, self, self)?;++ // Recreate the adapter factory+ let adapter_factory = create_adapter_factory(&payload.adapter_metadata.tag);++ // Inject adapter into the plan+ return inject_adapter_into_plan(inner_plan, adapter_factory);+ }++ not_impl_err!("Unknown extension type")+ }++ // Required method: encode custom execution plans+ fn try_encode(+ &self,+ _node: Arc<dyn ExecutionPlan>,+ _buf: &mut Vec<u8>,+ ) -> Result<()> {+ // We don't need this for the example - we use serialize_physical_plan instead+ not_impl_err!(+ "try_encode not used - adapter wrapping happens in serialize_physical_plan"+ )+ }+}++impl PhysicalProtoConverterExtension for AdapterPreservingCodec {+ fn execution_plan_to_proto(+ &self,+ plan: &Arc<dyn ExecutionPlan>,+ extension_codec: &dyn PhysicalExtensionCodec,+ ) -> Result<PhysicalPlanNode> {+ // Check if this is a DataSourceExec with adapter+ if let Some(exec) = plan.as_any().downcast_ref::<DataSourceExec>()+ && let Some(config) =+ exec.data_source().as_any().downcast_ref::<FileScanConfig>()+ && let Some(adapter_factory) = &config.expr_adapter_factory+ && let Some(tag) = extract_adapter_tag(adapter_factory.as_ref())+ {+ // Try to extract our MetadataAdapterFactory's tag+ println!(" [Serialize] Found DataSourceExec with adapter tag: {tag}");++ // 1. Create adapter metadata+ let adapter_metadata = AdapterMetadata { tag };++ // 2. Create a copy of the config without the adapter+ let config_without_adapter = rebuild_config_without_adapter(config);++ // 3. Create a new DataSourceExec without adapter+ let plan_without_adapter: Arc<dyn ExecutionPlan> =+ DataSourceExec::from_data_source(config_without_adapter);++ // 4. Serialize the inner plan to protobuf bytes+ let inner_proto = PhysicalPlanNode::try_from_physical_plan_with_converter(+ plan_without_adapter,+ extension_codec,+ self,+ )?;++ let mut inner_bytes = Vec::new();+ inner_proto.encode(&mut inner_bytes).map_err(|e| {+ datafusion::error::DataFusionError::Plan(format!(+ "Failed to encode inner plan: {e}"+ ))+ })?;++ // 5. Create extension payload+ let payload = ExtensionPayload {+ marker: EXTENSION_MARKER.to_string(),+ adapter_metadata,+ inner_plan_bytes: inner_bytes,+ };+ let payload_bytes = serde_json::to_vec(&payload).map_err(|e| {+ datafusion::error::DataFusionError::Plan(format!(+ "Failed to serialize payload: {e}"+ ))+ })?;++ // 6. Return as PhysicalExtensionNode+ return Ok(PhysicalPlanNode {+ physical_plan_type: Some(PhysicalPlanType::Extension(+ PhysicalExtensionNode {+ node: payload_bytes,+ inputs: vec![], // Leaf node+ },+ )),+ });+ }++ // No adapter found - use default serialization+ PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::clone(plan),+ extension_codec,+ self,+ )+ }++ // Interception point: override deserialization to unwrap adapters+ fn proto_to_execution_plan(+ &self,+ ctx: &TaskContext,+ extension_codec: &dyn PhysicalExtensionCodec,+ proto: &PhysicalPlanNode,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // Check if this is our custom extension wrapper+ if let Some(PhysicalPlanType::Extension(extension)) = &proto.physical_plan_type+ && let Ok(payload) =+ serde_json::from_slice::<ExtensionPayload>(&extension.node)+ && payload.marker == EXTENSION_MARKER+ {+ println!(+ " [Deserialize] Found adapter extension with tag: {}",+ payload.adapter_metadata.tag+ );++ // Decode the inner plan+ let inner_proto = PhysicalPlanNode::decode(&payload.inner_plan_bytes[..])+ .map_err(|e| {+ datafusion::error::DataFusionError::Plan(format!(+ "Failed to decode inner plan: {e}"+ ))+ })?;++ // Deserialize the inner plan using default implementation+ let inner_plan = inner_proto.try_into_physical_plan_with_converter(+ ctx,+ extension_codec,+ self,+ )?;++ // Recreate the adapter factory+ let adapter_factory = create_adapter_factory(&payload.adapter_metadata.tag);++ // Inject adapter into the plan+ return inject_adapter_into_plan(inner_plan, adapter_factory);+ }++ // Not our extension - use default deserialization+ proto.try_into_physical_plan_with_converter(ctx, extension_codec, self)+ }++ fn proto_to_physical_expr(+ &self,+ proto: &PhysicalExprNode,+ ctx: &TaskContext,+ input_schema: &Schema,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)+ }++ fn physical_expr_to_proto(+ &self,+ expr: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<PhysicalExprNode> {+ serialize_physical_expr_with_converter(expr, codec, self)+ }+}++// ============================================================================+// Helper functions+// ============================================================================++/// Write a RecordBatch to Parquet in the object store+async fn write_parquet(+ store: &dyn ObjectStore,+ path: &Path,+ batch: &arrow::record_batch::RecordBatch,+) -> Result<()> {+ let mut buf = vec![];+ let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), None)?;+ writer.write(batch)?;+ writer.close()?;++ let payload = PutPayload::from_bytes(buf.into());+ store.put(path, payload).await?;+ Ok(())+}++/// Extract the tag from a MetadataAdapterFactory.+///+/// Note: Since `PhysicalExprAdapterFactory` doesn't provide `as_any()` for downcasting,+/// we parse the Debug output. In a production system, you might add a dedicated trait+/// method for metadata extraction.+fn extract_adapter_tag(factory: &dyn PhysicalExprAdapterFactory) -> Option<String> {+ let debug_str = format!("{factory:?}");+ if debug_str.contains("MetadataAdapterFactory") {+ // Extract tag from debug output: MetadataAdapterFactory { tag: "v1" }+ if let Some(start) = debug_str.find("tag: \"") {+ let after_tag = &debug_str[start + 6..];+ if let Some(end) = after_tag.find('"') {+ return Some(after_tag[..end].to_string());+ }+ }+ }+ None+}++/// Create an adapter factory from a tag+fn create_adapter_factory(tag: &str) -> Arc<dyn PhysicalExprAdapterFactory> {+ Arc::new(MetadataAdapterFactory::new(tag))+}++/// Rebuild a FileScanConfig without the adapter+fn rebuild_config_without_adapter(config: &FileScanConfig) -> FileScanConfig {+ FileScanConfigBuilder::from(config.clone())+ .with_expr_adapter(None)+ .build()+}++/// Inject an adapter into a plan (assumes plan is a DataSourceExec with FileScanConfig)+fn inject_adapter_into_plan(+ plan: Arc<dyn ExecutionPlan>,+ adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,+) -> Result<Arc<dyn ExecutionPlan>> {+ if let Some(exec) = plan.as_any().downcast_ref::<DataSourceExec>()+ && let Some(config) = exec.data_source().as_any().downcast_ref::<FileScanConfig>()+ {+ let new_config = FileScanConfigBuilder::from(config.clone())+ .with_expr_adapter(Some(adapter_factory))+ .build();+ return Ok(DataSourceExec::from_data_source(new_config));+ }+ // If not a DataSourceExec with FileScanConfig, return as-is+ Ok(plan)+}++/// Helper to verify if a plan has an adapter (for testing/validation)+fn verify_adapter_in_plan(plan: &Arc<dyn ExecutionPlan>, label: &str) -> bool {+ // Walk the plan tree to find DataSourceExec with adapter+ fn check_plan(plan: &dyn ExecutionPlan) -> bool {+ if let Some(exec) = plan.as_any().downcast_ref::<DataSourceExec>()+ && let Some(config) =+ exec.data_source().as_any().downcast_ref::<FileScanConfig>()+ && config.expr_adapter_factory.is_some()+ {+ return true;+ }+ // Check children+ for child in plan.children() {+ if check_plan(child.as_ref()) {+ return true;+ }+ }+ false+ }++ let has_adapter = check_plan(plan.as_ref());+ println!(" [Verify] {label} plan adapter check: {has_adapter}");+ has_adapter+}diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rsindex 5846626d81..73e65182a0 100644--- a/datafusion-examples/examples/custom_data_source/main.rs+++ b/datafusion-examples/examples/custom_data_source/main.rs@@ -26,6 +26,7 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module+//! - `adapter_serialization` — preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception//! - `csv_json_opener` — use low level FileOpener APIs to read CSV/JSON into Arrow RecordBatches//! - `csv_sql_streaming` — build and run a streaming query plan from a SQL statement against a local CSV file//! - `custom_datasource` — run queries against a custom datasource (TableProvider)@@ -34,6 +35,7 @@//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter//! - `file_stream_provider` — run a query on FileStreamProvider which implements StreamProvider for reading and writing to arbitrary stream sources/sinks+mod adapter_serialization;mod csv_json_opener;mod csv_sql_streaming;mod custom_datasource;@@ -50,6 +52,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};#[strum(serialize_all = "snake_case")]enum ExampleKind {All,+ AdapterSerialization,CsvJsonOpener,CsvSqlStreaming,CustomDatasource,@@ -74,6 +77,9 @@ impl ExampleKind {Box::pin(example.run()).await?;}}+ ExampleKind::AdapterSerialization => {+ adapter_serialization::adapter_serialization().await?+ }ExampleKind::CsvJsonOpener => csv_json_opener::csv_json_opener().await?,ExampleKind::CsvSqlStreaming => {csv_sql_streaming::csv_sql_streaming().await?diff --git a/datafusion-examples/examples/proto/expression_deduplication.rs b/datafusion-examples/examples/proto/expression_deduplication.rsnew file mode 100644index 0000000000..0dec807f80--- /dev/null+++ b/datafusion-examples/examples/proto/expression_deduplication.rs@@ -0,0 +1,275 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! See `main.rs` for how to run it.+//!+//! This example demonstrates how to use the `PhysicalExtensionCodec` trait's+//! interception methods to implement expression deduplication during deserialization.+//!+//! This pattern is inspired by PR #18192, which introduces expression caching+//! to reduce memory usage when deserializing plans with duplicate expressions.+//!+//! The key insight is that identical expressions serialize to identical protobuf bytes.+//! By caching deserialized expressions keyed by their protobuf bytes, we can:+//! 1. Return the same Arc for duplicate expressions+//! 2. Reduce memory allocation during deserialization+//! 3. Enable downstream optimizations that rely on Arc pointer equality+//!+//! This demonstrates the decorator pattern enabled by the `PhysicalExtensionCodec` trait,+//! where all expression serialization/deserialization routes through the codec methods.++use std::collections::HashMap;+use std::fmt::Debug;+use std::sync::{Arc, RwLock};++use arrow::datatypes::{DataType, Field, Schema};+use datafusion::common::Result;+use datafusion::execution::TaskContext;+use datafusion::logical_expr::Operator;+use datafusion::physical_expr::PhysicalExpr;+use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::expressions::{BinaryExpr, col};+use datafusion::physical_plan::filter::FilterExec;+use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;+use datafusion::prelude::SessionContext;+use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;+use datafusion_proto::physical_plan::{+ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,+ PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};+use prost::Message;++/// Example showing how to implement expression deduplication using the codec decorator pattern.+///+/// This demonstrates:+/// 1. Creating a CachingCodec that caches expressions by their protobuf bytes+/// 2. Intercepting deserialization to return cached Arcs for duplicate expressions+/// 3. Verifying that duplicate expressions share the same Arc after deserialization+///+/// Deduplication is keyed by the protobuf bytes representing the expression,+/// in reality deduplication could be done based on e.g. the pointer address of the+/// serialized expression in memory, but this is simpler to demonstrate.+///+/// In this case our expression is trivial and just for demonstration purposes.+/// In real scenarios, expressions can be much more complex, e.g. a large InList+/// expression could be megabytes in size, so deduplication can save significant memory+/// in addition to more correctly representing the original plan structure.+pub async fn expression_deduplication() -> Result<()> {+ println!("=== Expression Deduplication Example ===\n");++ // Create a schema for our test expressions+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));++ // Step 1: Create expressions with duplicates+ println!("Step 1: Creating expressions with duplicates...");++ // Create expression: col("a")+ let a = col("a", &schema)?;++ // Create a clone to show duplicates+ let a_clone = Arc::clone(&a);++ // Combine: a OR a_clone+ let combined_expr =+ Arc::new(BinaryExpr::new(a, Operator::Or, a_clone)) as Arc<dyn PhysicalExpr>;+ println!(" Created expression: a OR a with duplicates");+ println!(" Note: a appears twice in the expression tree\n");+ // Step 2: Create a filter plan with this expression+ println!("Step 2: Creating physical plan with the expression...");++ let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema)));+ let filter_plan: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(combined_expr, input)?);++ println!(" Created FilterExec with duplicate sub-expressions\n");++ // Step 3: Serialize with the caching codec+ println!("Step 3: Serializing plan...");++ let extension_codec = DefaultPhysicalExtensionCodec {};+ let caching_converter = CachingCodec::new();+ let proto =+ caching_converter.execution_plan_to_proto(&filter_plan, &extension_codec)?;++ // Serialize to bytes+ let mut bytes = Vec::new();+ proto.encode(&mut bytes).unwrap();+ println!(" Serialized plan to {} bytes\n", bytes.len());++ // Step 4: Deserialize with the caching codec+ println!("Step 4: Deserializing plan with CachingCodec...");++ let ctx = SessionContext::new();+ let deserialized_plan = proto.try_into_physical_plan_with_converter(+ &ctx.task_ctx(),+ &extension_codec,+ &caching_converter,+ )?;++ // Step 5: check that we deduplicated expressions+ println!("Step 5: Checking for deduplicated expressions...");+ let Some(filter_exec) = deserialized_plan.as_any().downcast_ref::<FilterExec>()+ else {+ panic!("Deserialized plan is not a FilterExec");+ };+ let predicate = Arc::clone(filter_exec.predicate());+ let binary_expr = predicate+ .as_any()+ .downcast_ref::<BinaryExpr>()+ .expect("Predicate is not a BinaryExpr");+ let left = &binary_expr.left();+ let right = &binary_expr.right();+ // Check if left and right point to the same Arc+ let deduplicated = Arc::ptr_eq(left, right);+ if deduplicated {+ println!(" Success: Duplicate expressions were deduplicated!");+ println!(+ " Cache Stats: hits={}, misses={}",+ caching_converter.stats.read().unwrap().cache_hits,+ caching_converter.stats.read().unwrap().cache_misses,+ );+ } else {+ println!(" Failure: Duplicate expressions were NOT deduplicated.");+ }++ Ok(())+}++// ============================================================================+// CachingCodec - Implements expression deduplication+// ============================================================================++/// Statistics for cache performance monitoring+#[derive(Debug, Default)]+struct CacheStats {+ cache_hits: usize,+ cache_misses: usize,+}++/// A codec that caches deserialized expressions to enable deduplication.+///+/// When deserializing, if we've already seen the same protobuf bytes,+/// we return the cached Arc instead of creating a new allocation.+#[derive(Debug, Default)]+struct CachingCodec {+ /// Cache mapping protobuf bytes -> deserialized expression+ expr_cache: RwLock<HashMap<Vec<u8>, Arc<dyn PhysicalExpr>>>,+ /// Statistics for demonstration+ stats: RwLock<CacheStats>,+}++impl CachingCodec {+ fn new() -> Self {+ Self::default()+ }+}++impl PhysicalExtensionCodec for CachingCodec {+ // Required: decode custom extension nodes+ fn try_decode(+ &self,+ _buf: &[u8],+ _inputs: &[Arc<dyn ExecutionPlan>],+ _ctx: &TaskContext,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ datafusion::common::not_impl_err!("No custom extension nodes")+ }++ // Required: encode custom execution plans+ fn try_encode(+ &self,+ _node: Arc<dyn ExecutionPlan>,+ _buf: &mut Vec<u8>,+ ) -> Result<()> {+ datafusion::common::not_impl_err!("No custom extension nodes")+ }+}++impl PhysicalProtoConverterExtension for CachingCodec {+ fn proto_to_execution_plan(+ &self,+ ctx: &TaskContext,+ extension_codec: &dyn PhysicalExtensionCodec,+ proto: &PhysicalPlanNode,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ proto.try_into_physical_plan_with_converter(ctx, extension_codec, self)+ }++ fn execution_plan_to_proto(+ &self,+ plan: &Arc<dyn ExecutionPlan>,+ extension_codec: &dyn PhysicalExtensionCodec,+ ) -> Result<PhysicalPlanNode> {+ PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::clone(plan),+ extension_codec,+ self,+ )+ }++ // CACHING IMPLEMENTATION: Intercept expression deserialization+ fn proto_to_physical_expr(+ &self,+ proto: &PhysicalExprNode,+ ctx: &TaskContext,+ input_schema: &Schema,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ // Create cache key from protobuf bytes+ let mut key = Vec::new();+ proto.encode(&mut key).map_err(|e| {+ datafusion::error::DataFusionError::Internal(format!(+ "Failed to encode proto for cache key: {e}"+ ))+ })?;++ // Check cache first+ {+ let cache = self.expr_cache.read().unwrap();+ if let Some(cached) = cache.get(&key) {+ // Cache hit! Update stats and return cached Arc+ let mut stats = self.stats.write().unwrap();+ stats.cache_hits += 1;+ return Ok(Arc::clone(cached));+ }+ }++ // Cache miss - deserialize and store+ let expr =+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)?;++ // Store in cache+ {+ let mut cache = self.expr_cache.write().unwrap();+ cache.insert(key, Arc::clone(&expr));+ let mut stats = self.stats.write().unwrap();+ stats.cache_misses += 1;+ }++ Ok(expr)+ }++ fn physical_expr_to_proto(+ &self,+ expr: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<PhysicalExprNode> {+ serialize_physical_expr_with_converter(expr, codec, self)+ }+}diff --git a/datafusion-examples/examples/proto/main.rs b/datafusion-examples/examples/proto/main.rsindex f56078b319..16fcd70f34 100644--- a/datafusion-examples/examples/proto/main.rs+++ b/datafusion-examples/examples/proto/main.rs@@ -27,8 +27,10 @@//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module//! - `composed_extension_codec` — example of using multiple extension codecs for serialization / deserialization+//! - `expression_deduplication` — example of expression caching/deduplication using the codec decorator patternmod composed_extension_codec;+mod expression_deduplication;use datafusion::error::{DataFusionError, Result};use strum::{IntoEnumIterator, VariantNames};@@ -39,6 +41,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};enum ExampleKind {All,ComposedExtensionCodec,+ ExpressionDeduplication,}impl ExampleKind {@@ -59,6 +62,9 @@ impl ExampleKind {ExampleKind::ComposedExtensionCodec => {composed_extension_codec::composed_extension_codec().await?}+ ExampleKind::ExpressionDeduplication => {+ expression_deduplication::expression_deduplication().await?+ }}Ok(())}diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rsindex d95bdd3886..84b15ea9a8 100644--- a/datafusion/proto/src/bytes/mod.rs+++ b/datafusion/proto/src/bytes/mod.rs@@ -21,7 +21,8 @@ use crate::logical_plan::{self, AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec,};use crate::physical_plan::{- AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ PhysicalProtoConverterExtension,};use crate::protobuf;use datafusion_common::{Result, plan_datafusion_err};@@ -276,16 +277,18 @@ pub fn logical_plan_from_json_with_extension_codec(/// Serialize a PhysicalPlan as bytespub fn physical_plan_to_bytes(plan: Arc<dyn ExecutionPlan>) -> Result<Bytes> {let extension_codec = DefaultPhysicalExtensionCodec {};- physical_plan_to_bytes_with_extension_codec(plan, &extension_codec)+ let proto_converter = DefaultPhysicalProtoConverter {};+ physical_plan_to_bytes_with_proto_converter(plan, &extension_codec, &proto_converter)}/// Serialize a PhysicalPlan as JSON#[cfg(feature = "json")]pub fn physical_plan_to_json(plan: Arc<dyn ExecutionPlan>) -> Result<String> {let extension_codec = DefaultPhysicalExtensionCodec {};- let protobuf =- protobuf::PhysicalPlanNode::try_from_physical_plan(plan, &extension_codec)- .map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;+ let proto_converter = DefaultPhysicalProtoConverter {};+ let protobuf = proto_converter+ .execution_plan_to_proto(&plan, &extension_codec)+ .map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;serde_json::to_string(&protobuf).map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))}@@ -295,8 +298,18 @@ pub fn physical_plan_to_bytes_with_extension_codec(plan: Arc<dyn ExecutionPlan>,extension_codec: &dyn PhysicalExtensionCodec,) -> Result<Bytes> {- let protobuf =- protobuf::PhysicalPlanNode::try_from_physical_plan(plan, extension_codec)?;+ let proto_converter = DefaultPhysicalProtoConverter {};+ physical_plan_to_bytes_with_proto_converter(plan, extension_codec, &proto_converter)+}++/// Serialize a PhysicalPlan as bytes, using the provided extension codec+/// and protobuf converter.+pub fn physical_plan_to_bytes_with_proto_converter(+ plan: Arc<dyn ExecutionPlan>,+ extension_codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,+) -> Result<Bytes> {+ let protobuf = proto_converter.execution_plan_to_proto(&plan, extension_codec)?;let mut buffer = BytesMut::new();protobuf.encode(&mut buffer)@@ -313,7 +326,8 @@ pub fn physical_plan_from_json(let back: protobuf::PhysicalPlanNode = serde_json::from_str(json).map_err(|e| plan_datafusion_err!("Error serializing plan: {e}"))?;let extension_codec = DefaultPhysicalExtensionCodec {};- back.try_into_physical_plan(ctx, &extension_codec)+ let proto_converter = DefaultPhysicalProtoConverter {};+ proto_converter.proto_to_execution_plan(ctx, &extension_codec, &back)}/// Deserialize a PhysicalPlan from bytes@@ -322,7 +336,13 @@ pub fn physical_plan_from_bytes(ctx: &TaskContext,) -> Result<Arc<dyn ExecutionPlan>> {let extension_codec = DefaultPhysicalExtensionCodec {};- physical_plan_from_bytes_with_extension_codec(bytes, ctx, &extension_codec)+ let proto_converter = DefaultPhysicalProtoConverter {};+ physical_plan_from_bytes_with_proto_converter(+ bytes,+ ctx,+ &extension_codec,+ &proto_converter,+ )}/// Deserialize a PhysicalPlan from bytes@@ -330,8 +350,24 @@ pub fn physical_plan_from_bytes_with_extension_codec(bytes: &[u8],ctx: &TaskContext,extension_codec: &dyn PhysicalExtensionCodec,+) -> Result<Arc<dyn ExecutionPlan>> {+ let proto_converter = DefaultPhysicalProtoConverter {};+ physical_plan_from_bytes_with_proto_converter(+ bytes,+ ctx,+ extension_codec,+ &proto_converter,+ )+}++/// Deserialize a PhysicalPlan from bytes+pub fn physical_plan_from_bytes_with_proto_converter(+ bytes: &[u8],+ ctx: &TaskContext,+ extension_codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let protobuf = protobuf::PhysicalPlanNode::decode(bytes).map_err(|e| plan_datafusion_err!("Error decoding expr as protobuf: {e}"))?;- protobuf.try_into_physical_plan(ctx, extension_codec)+ proto_converter.proto_to_execution_plan(ctx, extension_codec, &protobuf)}diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 3cfc796700..fc8eba12c5 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -21,14 +21,9 @@ use std::sync::Arc;use arrow::array::RecordBatch;use arrow::compute::SortOptions;-use arrow::datatypes::Field;+use arrow::datatypes::{Field, Schema};use arrow::ipc::reader::StreamReader;use chrono::{TimeZone, Utc};-use datafusion_expr::dml::InsertOp;-use object_store::ObjectMeta;-use object_store::path::Path;--use arrow::datatypes::Schema;use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;@@ -42,6 +37,7 @@ use datafusion_datasource_parquet::file_format::ParquetSink;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{FunctionRegistry, TaskContext};use datafusion_expr::WindowFunctionDefinition;+use datafusion_expr::dml::InsertOp;use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, ScalarFunctionExpr};use datafusion_physical_plan::expressions::{@@ -52,13 +48,16 @@ use datafusion_physical_plan::joins::{HashExpr, SeededRandomState};use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};use datafusion_proto_common::common::proto_error;+use object_store::ObjectMeta;+use object_store::path::Path;-use crate::convert_required;+use super::{+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ PhysicalProtoConverterExtension,+};use crate::logical_plan::{self};-use crate::protobuf;use crate::protobuf::physical_expr_node::ExprType;--use super::PhysicalExtensionCodec;+use crate::{convert_required, protobuf};impl From<&protobuf::PhysicalColumn> for Column {fn from(c: &protobuf::PhysicalColumn) -> Column {@@ -80,9 +79,15 @@ pub fn parse_physical_sort_expr(ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<PhysicalSortExpr> {if let Some(expr) = &proto.expr {- let expr = parse_physical_expr(expr.as_ref(), ctx, input_schema, codec)?;+ let expr = proto_converter.proto_to_physical_expr(+ expr.as_ref(),+ ctx,+ input_schema,+ codec,+ )?;let options = SortOptions {descending: !proto.asc,nulls_first: proto.nulls_first,@@ -107,10 +112,13 @@ pub fn parse_physical_sort_exprs(ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Vec<PhysicalSortExpr>> {proto.iter()- .map(|sort_expr| parse_physical_sort_expr(sort_expr, ctx, input_schema, codec))+ .map(|sort_expr| {+ parse_physical_sort_expr(sort_expr, ctx, input_schema, codec, proto_converter)+ }).collect()}@@ -129,12 +137,25 @@ pub fn parse_physical_window_expr(ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn WindowExpr>> {- let window_node_expr = parse_physical_exprs(&proto.args, ctx, input_schema, codec)?;- let partition_by =- parse_physical_exprs(&proto.partition_by, ctx, input_schema, codec)?;-- let order_by = parse_physical_sort_exprs(&proto.order_by, ctx, input_schema, codec)?;+ let window_node_expr =+ parse_physical_exprs(&proto.args, ctx, input_schema, codec, proto_converter)?;+ let partition_by = parse_physical_exprs(+ &proto.partition_by,+ ctx,+ input_schema,+ codec,+ proto_converter,+ )?;++ let order_by = parse_physical_sort_exprs(+ &proto.order_by,+ ctx,+ input_schema,+ codec,+ proto_converter,+ )?;let window_frame = proto.window_frame@@ -188,13 +209,14 @@ pub fn parse_physical_exprs<'a, I>(ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Vec<Arc<dyn PhysicalExpr>>>whereI: IntoIterator<Item = &'a protobuf::PhysicalExprNode>,{protos.into_iter()- .map(|p| parse_physical_expr(p, ctx, input_schema, codec))+ .map(|p| proto_converter.proto_to_physical_expr(p, ctx, input_schema, codec)).collect::<Result<Vec<_>>>()}@@ -212,6 +234,32 @@ pub fn parse_physical_expr(ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+) -> Result<Arc<dyn PhysicalExpr>> {+ parse_physical_expr_with_converter(+ proto,+ ctx,+ input_schema,+ codec,+ &DefaultPhysicalProtoConverter {},+ )+}++/// Parses a physical expression from a protobuf.+///+/// # Arguments+///+/// * `proto` - Input proto with physical expression node+/// * `registry` - A registry knows how to build logical expressions out of user-defined function names+/// * `input_schema` - The Arrow schema for the input, used for determining expression data types+/// when performing type coercion.+/// * `codec` - An extension codec used to decode custom UDFs.+/// * `proto_converter` - Conversion functions for physical plans and expressions+pub fn parse_physical_expr_with_converter(+ proto: &protobuf::PhysicalExprNode,+ ctx: &TaskContext,+ input_schema: &Schema,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn PhysicalExpr>> {let expr_type = proto.expr_type@@ -232,6 +280,7 @@ pub fn parse_physical_expr("left",input_schema,codec,+ proto_converter,)?,logical_plan::from_proto::from_proto_binary_op(&binary_expr.op)?,parse_required_physical_expr(@@ -240,6 +289,7 @@ pub fn parse_physical_expr("right",input_schema,codec,+ proto_converter,)?,)),ExprType::AggregateExpr(_) => {@@ -262,6 +312,7 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?))}ExprType::IsNotNullExpr(e) => {@@ -271,6 +322,7 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?))}ExprType::NotExpr(e) => Arc::new(NotExpr::new(parse_required_physical_expr(@@ -279,6 +331,7 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?)),ExprType::Negative(e) => {Arc::new(NegativeExpr::new(parse_required_physical_expr(@@ -287,6 +340,7 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?))}ExprType::InList(e) => in_list(@@ -296,15 +350,23 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?,- parse_physical_exprs(&e.list, ctx, input_schema, codec)?,+ parse_physical_exprs(&e.list, ctx, input_schema, codec, proto_converter)?,&e.negated,input_schema,)?,ExprType::Case(e) => Arc::new(CaseExpr::try_new(e.expr.as_ref()- .map(|e| parse_physical_expr(e.as_ref(), ctx, input_schema, codec))+ .map(|e| {+ proto_converter.proto_to_physical_expr(+ e.as_ref(),+ ctx,+ input_schema,+ codec,+ )+ }).transpose()?,e.when_then_expr.iter()@@ -316,6 +378,7 @@ pub fn parse_physical_expr("when_expr",input_schema,codec,+ proto_converter,)?,parse_required_physical_expr(e.then_expr.as_ref(),@@ -323,13 +386,21 @@ pub fn parse_physical_expr("then_expr",input_schema,codec,+ proto_converter,)?,))}).collect::<Result<Vec<_>>>()?,e.else_expr.as_ref()- .map(|e| parse_physical_expr(e.as_ref(), ctx, input_schema, codec))+ .map(|e| {+ proto_converter.proto_to_physical_expr(+ e.as_ref(),+ ctx,+ input_schema,+ codec,+ )+ }).transpose()?,)?),ExprType::Cast(e) => Arc::new(CastExpr::new(@@ -339,6 +410,7 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?,convert_required!(e.arrow_type)?,None,@@ -350,6 +422,7 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?,convert_required!(e.arrow_type)?,)),@@ -362,7 +435,8 @@ pub fn parse_physical_expr(};let scalar_fun_def = Arc::clone(&udf);- let args = parse_physical_exprs(&e.args, ctx, input_schema, codec)?;+ let args =+ parse_physical_exprs(&e.args, ctx, input_schema, codec, proto_converter)?;let config_options = Arc::clone(ctx.session_config().options());@@ -391,6 +465,7 @@ pub fn parse_physical_expr("expr",input_schema,codec,+ proto_converter,)?,parse_required_physical_expr(like_expr.pattern.as_deref(),@@ -398,11 +473,17 @@ pub fn parse_physical_expr("pattern",input_schema,codec,+ proto_converter,)?,)),ExprType::HashExpr(hash_expr) => {- let on_columns =- parse_physical_exprs(&hash_expr.on_columns, ctx, input_schema, codec)?;+ let on_columns = parse_physical_exprs(+ &hash_expr.on_columns,+ ctx,+ input_schema,+ codec,+ proto_converter,+ )?;Arc::new(HashExpr::new(on_columns,SeededRandomState::with_seeds(@@ -418,9 +499,11 @@ pub fn parse_physical_expr(let inputs: Vec<Arc<dyn PhysicalExpr>> = extension.inputs.iter()- .map(|e| parse_physical_expr(e, ctx, input_schema, codec))+ .map(|e| {+ proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec)+ }).collect::<Result<_>>()?;- (codec.try_decode_expr(extension.expr.as_slice(), &inputs)?) as _+ codec.try_decode_expr(extension.expr.as_slice(), &inputs)? as _}};@@ -433,8 +516,9 @@ fn parse_required_physical_expr(field: &str,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn PhysicalExpr>> {- expr.map(|e| parse_physical_expr(e, ctx, input_schema, codec))+ expr.map(|e| proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec)).transpose()?.ok_or_else(|| internal_datafusion_err!("Missing required field {field:?}"))}@@ -444,11 +528,17 @@ pub fn parse_protobuf_hash_partitioning(ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Partitioning>> {match partitioning {Some(hash_part) => {- let expr =- parse_physical_exprs(&hash_part.hash_expr, ctx, input_schema, codec)?;+ let expr = parse_physical_exprs(+ &hash_part.hash_expr,+ ctx,+ input_schema,+ codec,+ proto_converter,+ )?;Ok(Some(Partitioning::Hash(expr,@@ -464,6 +554,7 @@ pub fn parse_protobuf_partitioning(ctx: &TaskContext,input_schema: &Schema,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Partitioning>> {match partitioning {Some(protobuf::Partitioning { partition_method }) => match partition_method {@@ -478,6 +569,7 @@ pub fn parse_protobuf_partitioning(ctx,input_schema,codec,+ proto_converter,)}Some(protobuf::partitioning::PartitionMethod::Unknown(partition_count)) => {@@ -532,6 +624,7 @@ pub fn parse_protobuf_file_scan_config(proto: &protobuf::FileScanExecConf,ctx: &TaskContext,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,file_source: Arc<dyn FileSource>,) -> Result<FileScanConfig> {let schema: Arc<Schema> = parse_protobuf_file_scan_schema(proto)?;@@ -557,6 +650,7 @@ pub fn parse_protobuf_file_scan_config(ctx,&schema,codec,+ proto_converter,)?;output_ordering.extend(LexOrdering::new(sort_exprs));}@@ -567,7 +661,7 @@ pub fn parse_protobuf_file_scan_config(.projections.iter().map(|proto_expr| {- let expr = parse_physical_expr(+ let expr = proto_converter.proto_to_physical_expr(proto_expr.expr.as_ref().ok_or_else(|| {internal_datafusion_err!("ProjectionExpr missing expr field")})?,@@ -743,12 +837,13 @@ impl TryFrom<&protobuf::FileSinkConfig> for FileSinkConfig {#[cfg(test)]mod tests {- use super::*;use chrono::{TimeZone, Utc};use datafusion_datasource::PartitionedFile;use object_store::ObjectMeta;use object_store::path::Path;+ use super::*;+#[test]fn partitioned_file_path_roundtrip_percent_encoded() {let path_str = "foo/foo%2Fbar/baz%252Fqux";diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 54892597b5..ca213bc722 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -18,30 +18,8 @@use std::fmt::Debug;use std::sync::Arc;-use self::from_proto::parse_protobuf_partitioning;-use self::to_proto::{serialize_partitioning, serialize_physical_expr};-use crate::common::{byte_to_string, str_to_byte};-use crate::physical_plan::from_proto::{- parse_physical_expr, parse_physical_sort_expr, parse_physical_sort_exprs,- parse_physical_window_expr, parse_protobuf_file_scan_config, parse_record_batches,- parse_table_schema_from_proto,-};-use crate::physical_plan::to_proto::{- serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr,- serialize_physical_sort_exprs, serialize_physical_window_expr,- serialize_record_batches,-};-use crate::protobuf::physical_aggregate_expr_node::AggregateFunction;-use crate::protobuf::physical_expr_node::ExprType;-use crate::protobuf::physical_plan_node::PhysicalPlanType;-use crate::protobuf::{- self, ListUnnest as ProtoListUnnest, SortExprNode, SortMergeJoinExecNode,- proto_error, window_agg_exec_node,-};-use crate::{convert_required, into_required};-use arrow::compute::SortOptions;-use arrow::datatypes::{IntervalMonthDayNanoType, SchemaRef};+use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};use datafusion_catalog::memory::MemorySourceConfig;use datafusion_common::config::CsvOptions;use datafusion_common::{@@ -68,12 +46,14 @@ use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};use datafusion_functions_table::generate_series::{Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,};-use datafusion_physical_expr::aggregate::AggregateExprBuilder;-use datafusion_physical_expr::aggregate::AggregateFunctionExpr;+use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};+use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;use datafusion_physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};-use datafusion_physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};-use datafusion_physical_plan::aggregates::{AggregateMode, LimitOptions};+use datafusion_physical_plan::aggregates::{+ AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,+};use datafusion_physical_plan::analyze::AnalyzeExec;+use datafusion_physical_plan::async_func::AsyncFuncExec;#[expect(deprecated)]use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;@@ -84,10 +64,9 @@ use datafusion_physical_plan::expressions::PhysicalSortExpr;use datafusion_physical_plan::filter::{FilterExec, FilterExecBuilder};use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};use datafusion_physical_plan::joins::{- CrossJoinExec, NestedLoopJoinExec, SortMergeJoinExec, StreamJoinPartitionMode,- SymmetricHashJoinExec,+ CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec,+ StreamJoinPartitionMode, SymmetricHashJoinExec,};-use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::memory::LazyMemoryExec;use datafusion_physical_plan::metrics::MetricType;@@ -100,12 +79,31 @@ use datafusion_physical_plan::union::{InterleaveExec, UnionExec};use datafusion_physical_plan::unnest::{ListUnnest, UnnestExec};use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr};--use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;-use datafusion_physical_plan::async_func::AsyncFuncExec;use prost::Message;use prost::bytes::BufMut;+use self::from_proto::parse_protobuf_partitioning;+use self::to_proto::serialize_partitioning;+use crate::common::{byte_to_string, str_to_byte};+use crate::physical_plan::from_proto::{+ parse_physical_expr_with_converter, parse_physical_sort_expr,+ parse_physical_sort_exprs, parse_physical_window_expr,+ parse_protobuf_file_scan_config, parse_record_batches, parse_table_schema_from_proto,+};+use crate::physical_plan::to_proto::{+ serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr,+ serialize_physical_expr_with_converter, serialize_physical_sort_exprs,+ serialize_physical_window_expr, serialize_record_batches,+};+use crate::protobuf::physical_aggregate_expr_node::AggregateFunction;+use crate::protobuf::physical_expr_node::ExprType;+use crate::protobuf::physical_plan_node::PhysicalPlanType;+use crate::protobuf::{+ self, ListUnnest as ProtoListUnnest, SortExprNode, SortMergeJoinExecNode,+ proto_error, window_agg_exec_node,+};+use crate::{convert_required, into_required};+pub mod from_proto;pub mod to_proto;@@ -132,8 +130,37 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {fn try_into_physical_plan(&self,ctx: &TaskContext,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.try_into_physical_plan_with_converter(+ ctx,+ codec,+ &DefaultPhysicalProtoConverter {},+ )+ }- extension_codec: &dyn PhysicalExtensionCodec,+ fn try_from_physical_plan(+ plan: Arc<dyn ExecutionPlan>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Self>+ where+ Self: Sized,+ {+ Self::try_from_physical_plan_with_converter(+ plan,+ codec,+ &DefaultPhysicalProtoConverter {},+ )+ }+}++impl protobuf::PhysicalPlanNode {+ pub fn try_into_physical_plan_with_converter(+ &self,+ ctx: &TaskContext,++ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let plan = self.physical_plan_type.as_ref().ok_or_else(|| {proto_error(format!(@@ -142,125 +169,149 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {})?;match plan {PhysicalPlanType::Explain(explain) => {- self.try_into_explain_physical_plan(explain, ctx, extension_codec)- }- PhysicalPlanType::Projection(projection) => {- self.try_into_projection_physical_plan(projection, ctx, extension_codec)+ self.try_into_explain_physical_plan(explain, ctx, codec, proto_converter)}+ PhysicalPlanType::Projection(projection) => self+ .try_into_projection_physical_plan(+ projection,+ ctx,+ codec,+ proto_converter,+ ),PhysicalPlanType::Filter(filter) => {- self.try_into_filter_physical_plan(filter, ctx, extension_codec)+ self.try_into_filter_physical_plan(filter, ctx, codec, proto_converter)}PhysicalPlanType::CsvScan(scan) => {- self.try_into_csv_scan_physical_plan(scan, ctx, extension_codec)+ self.try_into_csv_scan_physical_plan(scan, ctx, codec, proto_converter)}PhysicalPlanType::JsonScan(scan) => {- self.try_into_json_scan_physical_plan(scan, ctx, extension_codec)- }- PhysicalPlanType::ParquetScan(scan) => {- self.try_into_parquet_scan_physical_plan(scan, ctx, extension_codec)+ self.try_into_json_scan_physical_plan(scan, ctx, codec, proto_converter)}+ PhysicalPlanType::ParquetScan(scan) => self+ .try_into_parquet_scan_physical_plan(scan, ctx, codec, proto_converter),PhysicalPlanType::AvroScan(scan) => {- self.try_into_avro_scan_physical_plan(scan, ctx, extension_codec)+ self.try_into_avro_scan_physical_plan(scan, ctx, codec, proto_converter)}PhysicalPlanType::MemoryScan(scan) => {- self.try_into_memory_scan_physical_plan(scan, ctx, extension_codec)+ self.try_into_memory_scan_physical_plan(scan, ctx, codec, proto_converter)}PhysicalPlanType::CoalesceBatches(coalesce_batches) => self.try_into_coalesce_batches_physical_plan(coalesce_batches,ctx,- extension_codec,+ codec,+ proto_converter,),PhysicalPlanType::Merge(merge) => {- self.try_into_merge_physical_plan(merge, ctx, extension_codec)- }- PhysicalPlanType::Repartition(repart) => {- self.try_into_repartition_physical_plan(repart, ctx, extension_codec)- }- PhysicalPlanType::GlobalLimit(limit) => {- self.try_into_global_limit_physical_plan(limit, ctx, extension_codec)- }- PhysicalPlanType::LocalLimit(limit) => {- self.try_into_local_limit_physical_plan(limit, ctx, extension_codec)- }- PhysicalPlanType::Window(window_agg) => {- self.try_into_window_physical_plan(window_agg, ctx, extension_codec)- }- PhysicalPlanType::Aggregate(hash_agg) => {- self.try_into_aggregate_physical_plan(hash_agg, ctx, extension_codec)- }- PhysicalPlanType::HashJoin(hashjoin) => {- self.try_into_hash_join_physical_plan(hashjoin, ctx, extension_codec)+ self.try_into_merge_physical_plan(merge, ctx, codec, proto_converter)}+ PhysicalPlanType::Repartition(repart) => self+ .try_into_repartition_physical_plan(repart, ctx, codec, proto_converter),+ PhysicalPlanType::GlobalLimit(limit) => self+ .try_into_global_limit_physical_plan(limit, ctx, codec, proto_converter),+ PhysicalPlanType::LocalLimit(limit) => self+ .try_into_local_limit_physical_plan(limit, ctx, codec, proto_converter),+ PhysicalPlanType::Window(window_agg) => self.try_into_window_physical_plan(+ window_agg,+ ctx,+ codec,+ proto_converter,+ ),+ PhysicalPlanType::Aggregate(hash_agg) => self+ .try_into_aggregate_physical_plan(hash_agg, ctx, codec, proto_converter),+ PhysicalPlanType::HashJoin(hashjoin) => self+ .try_into_hash_join_physical_plan(hashjoin, ctx, codec, proto_converter),PhysicalPlanType::SymmetricHashJoin(sym_join) => self.try_into_symmetric_hash_join_physical_plan(sym_join,ctx,- extension_codec,+ codec,+ proto_converter,),PhysicalPlanType::Union(union) => {- self.try_into_union_physical_plan(union, ctx, extension_codec)- }- PhysicalPlanType::Interleave(interleave) => {- self.try_into_interleave_physical_plan(interleave, ctx, extension_codec)+ self.try_into_union_physical_plan(union, ctx, codec, proto_converter)}- PhysicalPlanType::CrossJoin(crossjoin) => {- self.try_into_cross_join_physical_plan(crossjoin, ctx, extension_codec)- }- PhysicalPlanType::Empty(empty) => {- self.try_into_empty_physical_plan(empty, ctx, extension_codec)- }- PhysicalPlanType::PlaceholderRow(placeholder) => self- .try_into_placeholder_row_physical_plan(- placeholder,+ PhysicalPlanType::Interleave(interleave) => self+ .try_into_interleave_physical_plan(+ interleave,ctx,- extension_codec,+ codec,+ proto_converter,),- PhysicalPlanType::Sort(sort) => {- self.try_into_sort_physical_plan(sort, ctx, extension_codec)+ PhysicalPlanType::CrossJoin(crossjoin) => self+ .try_into_cross_join_physical_plan(+ crossjoin,+ ctx,+ codec,+ proto_converter,+ ),+ PhysicalPlanType::Empty(empty) => {+ self.try_into_empty_physical_plan(empty, ctx, codec, proto_converter)}- PhysicalPlanType::SortPreservingMerge(sort) => self- .try_into_sort_preserving_merge_physical_plan(sort, ctx, extension_codec),- PhysicalPlanType::Extension(extension) => {- self.try_into_extension_physical_plan(extension, ctx, extension_codec)+ PhysicalPlanType::PlaceholderRow(placeholder) => {+ self.try_into_placeholder_row_physical_plan(placeholder, ctx, codec)}- PhysicalPlanType::NestedLoopJoin(join) => {- self.try_into_nested_loop_join_physical_plan(join, ctx, extension_codec)+ PhysicalPlanType::Sort(sort) => {+ self.try_into_sort_physical_plan(sort, ctx, codec, proto_converter)}+ PhysicalPlanType::SortPreservingMerge(sort) => self+ .try_into_sort_preserving_merge_physical_plan(+ sort,+ ctx,+ codec,+ proto_converter,+ ),+ PhysicalPlanType::Extension(extension) => self+ .try_into_extension_physical_plan(extension, ctx, codec, proto_converter),+ PhysicalPlanType::NestedLoopJoin(join) => self+ .try_into_nested_loop_join_physical_plan(+ join,+ ctx,+ codec,+ proto_converter,+ ),PhysicalPlanType::Analyze(analyze) => {- self.try_into_analyze_physical_plan(analyze, ctx, extension_codec)+ self.try_into_analyze_physical_plan(analyze, ctx, codec, proto_converter)}PhysicalPlanType::JsonSink(sink) => {- self.try_into_json_sink_physical_plan(sink, ctx, extension_codec)+ self.try_into_json_sink_physical_plan(sink, ctx, codec, proto_converter)}PhysicalPlanType::CsvSink(sink) => {- self.try_into_csv_sink_physical_plan(sink, ctx, extension_codec)+ self.try_into_csv_sink_physical_plan(sink, ctx, codec, proto_converter)}#[cfg_attr(not(feature = "parquet"), allow(unused_variables))]- PhysicalPlanType::ParquetSink(sink) => {- self.try_into_parquet_sink_physical_plan(sink, ctx, extension_codec)- }+ PhysicalPlanType::ParquetSink(sink) => self+ .try_into_parquet_sink_physical_plan(sink, ctx, codec, proto_converter),PhysicalPlanType::Unnest(unnest) => {- self.try_into_unnest_physical_plan(unnest, ctx, extension_codec)- }- PhysicalPlanType::Cooperative(cooperative) => {- self.try_into_cooperative_physical_plan(cooperative, ctx, extension_codec)+ self.try_into_unnest_physical_plan(unnest, ctx, codec, proto_converter)}+ PhysicalPlanType::Cooperative(cooperative) => self+ .try_into_cooperative_physical_plan(+ cooperative,+ ctx,+ codec,+ proto_converter,+ ),PhysicalPlanType::GenerateSeries(generate_series) => {self.try_into_generate_series_physical_plan(generate_series)}PhysicalPlanType::SortMergeJoin(sort_join) => {- self.try_into_sort_join(sort_join, ctx, extension_codec)- }- PhysicalPlanType::AsyncFunc(async_func) => {- self.try_into_async_func_physical_plan(async_func, ctx, extension_codec)+ self.try_into_sort_join(sort_join, ctx, codec, proto_converter)}+ PhysicalPlanType::AsyncFunc(async_func) => self+ .try_into_async_func_physical_plan(+ async_func,+ ctx,+ codec,+ proto_converter,+ ),}}- fn try_from_physical_plan(+ pub fn try_from_physical_plan_with_converter(plan: Arc<dyn ExecutionPlan>,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self>whereSelf: Sized,@@ -269,93 +320,96 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {let plan = plan.as_any();if let Some(exec) = plan.downcast_ref::<ExplainExec>() {- return protobuf::PhysicalPlanNode::try_from_explain_exec(- exec,- extension_codec,- );+ return protobuf::PhysicalPlanNode::try_from_explain_exec(exec, codec);}if let Some(exec) = plan.downcast_ref::<ProjectionExec>() {return protobuf::PhysicalPlanNode::try_from_projection_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {return protobuf::PhysicalPlanNode::try_from_analyze_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<FilterExec>() {return protobuf::PhysicalPlanNode::try_from_filter_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(limit) = plan.downcast_ref::<GlobalLimitExec>() {return protobuf::PhysicalPlanNode::try_from_global_limit_exec(limit,- extension_codec,+ codec,+ proto_converter,);}if let Some(limit) = plan.downcast_ref::<LocalLimitExec>() {return protobuf::PhysicalPlanNode::try_from_local_limit_exec(limit,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<HashJoinExec>() {return protobuf::PhysicalPlanNode::try_from_hash_join_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<SymmetricHashJoinExec>() {return protobuf::PhysicalPlanNode::try_from_symmetric_hash_join_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<SortMergeJoinExec>() {return protobuf::PhysicalPlanNode::try_from_sort_merge_join_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<CrossJoinExec>() {return protobuf::PhysicalPlanNode::try_from_cross_join_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<AggregateExec>() {return protobuf::PhysicalPlanNode::try_from_aggregate_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(empty) = plan.downcast_ref::<EmptyExec>() {- return protobuf::PhysicalPlanNode::try_from_empty_exec(- empty,- extension_codec,- );+ return protobuf::PhysicalPlanNode::try_from_empty_exec(empty, codec);}if let Some(empty) = plan.downcast_ref::<PlaceholderRowExec>() {return protobuf::PhysicalPlanNode::try_from_placeholder_row_exec(- empty,- extension_codec,+ empty, codec,);}@@ -363,14 +417,16 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {if let Some(coalesce_batches) = plan.downcast_ref::<CoalesceBatchesExec>() {return protobuf::PhysicalPlanNode::try_from_coalesce_batches_exec(coalesce_batches,- extension_codec,+ codec,+ proto_converter,);}if let Some(data_source_exec) = plan.downcast_ref::<DataSourceExec>()&& let Some(node) = protobuf::PhysicalPlanNode::try_from_data_source_exec(data_source_exec,- extension_codec,+ codec,+ proto_converter,)?{return Ok(node);@@ -379,67 +435,80 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {if let Some(exec) = plan.downcast_ref::<CoalescePartitionsExec>() {return protobuf::PhysicalPlanNode::try_from_coalesce_partitions_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<RepartitionExec>() {return protobuf::PhysicalPlanNode::try_from_repartition_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<SortExec>() {- return protobuf::PhysicalPlanNode::try_from_sort_exec(exec, extension_codec);+ return protobuf::PhysicalPlanNode::try_from_sort_exec(+ exec,+ codec,+ proto_converter,+ );}if let Some(union) = plan.downcast_ref::<UnionExec>() {return protobuf::PhysicalPlanNode::try_from_union_exec(union,- extension_codec,+ codec,+ proto_converter,);}if let Some(interleave) = plan.downcast_ref::<InterleaveExec>() {return protobuf::PhysicalPlanNode::try_from_interleave_exec(interleave,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<SortPreservingMergeExec>() {return protobuf::PhysicalPlanNode::try_from_sort_preserving_merge_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<NestedLoopJoinExec>() {return protobuf::PhysicalPlanNode::try_from_nested_loop_join_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<WindowAggExec>() {return protobuf::PhysicalPlanNode::try_from_window_agg_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<BoundedWindowAggExec>() {return protobuf::PhysicalPlanNode::try_from_bounded_window_agg_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<DataSinkExec>()&& let Some(node) = protobuf::PhysicalPlanNode::try_from_data_sink_exec(exec,- extension_codec,+ codec,+ proto_converter,)?{return Ok(node);@@ -448,14 +517,16 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {if let Some(exec) = plan.downcast_ref::<UnnestExec>() {return protobuf::PhysicalPlanNode::try_from_unnest_exec(exec,- extension_codec,+ codec,+ proto_converter,);}if let Some(exec) = plan.downcast_ref::<CooperativeExec>() {return protobuf::PhysicalPlanNode::try_from_cooperative_exec(exec,- extension_codec,+ codec,+ proto_converter,);}@@ -469,21 +540,23 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {if let Some(exec) = plan.downcast_ref::<AsyncFuncExec>() {return protobuf::PhysicalPlanNode::try_from_async_func_exec(exec,- extension_codec,+ codec,+ proto_converter,);}let mut buf: Vec<u8> = vec![];- match extension_codec.try_encode(Arc::clone(&plan_clone), &mut buf) {+ match codec.try_encode(Arc::clone(&plan_clone), &mut buf) {Ok(_) => {let inputs: Vec<protobuf::PhysicalPlanNode> = plan_clone.children().into_iter().cloned().map(|i| {- protobuf::PhysicalPlanNode::try_from_physical_plan(+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(i,- extension_codec,+ codec,+ proto_converter,)}).collect::<Result<_>>()?;@@ -507,7 +580,8 @@ impl protobuf::PhysicalPlanNode {explain: &protobuf::ExplainExecNode,_ctx: &TaskContext,- _extension_codec: &dyn PhysicalExtensionCodec,+ _codec: &dyn PhysicalExtensionCodec,+ _proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(ExplainExec::new(Arc::new(explain.schema.as_ref().unwrap().try_into()?),@@ -525,21 +599,22 @@ impl protobuf::PhysicalPlanNode {projection: &protobuf::ProjectionExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&projection.input, ctx, extension_codec)?;+ into_physical_plan(&projection.input, ctx, codec, proto_converter)?;let exprs = projection.expr.iter().zip(projection.expr_name.iter()).map(|(expr, name)| {Ok((- parse_physical_expr(+ proto_converter.proto_to_physical_expr(expr,ctx,input.schema().as_ref(),- extension_codec,+ codec,)?,name.to_string(),))@@ -557,16 +632,22 @@ impl protobuf::PhysicalPlanNode {filter: &protobuf::FilterExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&filter.input, ctx, extension_codec)?;+ into_physical_plan(&filter.input, ctx, codec, proto_converter)?;let predicate = filter.expr.as_ref().map(|expr| {- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)+ proto_converter.proto_to_physical_expr(+ expr,+ ctx,+ input.schema().as_ref(),+ codec,+ )}).transpose()?.ok_or_else(|| {@@ -607,7 +688,8 @@ impl protobuf::PhysicalPlanNode {scan: &protobuf::CsvScanExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let escape =if let Some(protobuf::csv_scan_exec_node::OptionalEscape::Escape(escape)) =@@ -648,7 +730,8 @@ impl protobuf::PhysicalPlanNode {let conf = FileScanConfigBuilder::from(parse_protobuf_file_scan_config(scan.base_conf.as_ref().unwrap(),ctx,- extension_codec,+ codec,+ proto_converter,source,)?).with_file_compression_type(FileCompressionType::UNCOMPRESSED)@@ -661,14 +744,16 @@ impl protobuf::PhysicalPlanNode {scan: &protobuf::JsonScanExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let base_conf = scan.base_conf.as_ref().unwrap();let table_schema = parse_table_schema_from_proto(base_conf)?;let scan_conf = parse_protobuf_file_scan_config(base_conf,ctx,- extension_codec,+ codec,+ proto_converter,Arc::new(JsonSource::new(table_schema)),)?;Ok(DataSourceExec::from_data_source(scan_conf))@@ -679,7 +764,8 @@ impl protobuf::PhysicalPlanNode {&self,scan: &protobuf::ParquetScanExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "parquet")]{@@ -696,7 +782,7 @@ impl protobuf::PhysicalPlanNode {.iter().map(|&i| schema.field(i as usize).clone()).collect();- Arc::new(arrow::datatypes::Schema::new(projected_fields))+ Arc::new(Schema::new(projected_fields))} else {schema};@@ -705,11 +791,11 @@ impl protobuf::PhysicalPlanNode {.predicate.as_ref().map(|expr| {- parse_physical_expr(+ proto_converter.proto_to_physical_expr(expr,ctx,predicate_schema.as_ref(),- extension_codec,+ codec,)}).transpose()?;@@ -731,7 +817,8 @@ impl protobuf::PhysicalPlanNode {let base_config = parse_protobuf_file_scan_config(base_conf,ctx,- extension_codec,+ codec,+ proto_converter,Arc::new(source),)?;Ok(DataSourceExec::from_data_source(base_config))@@ -747,7 +834,8 @@ impl protobuf::PhysicalPlanNode {&self,scan: &protobuf::AvroScanExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "avro")]{@@ -756,7 +844,8 @@ impl protobuf::PhysicalPlanNode {let conf = parse_protobuf_file_scan_config(scan.base_conf.as_ref().unwrap(),ctx,- extension_codec,+ codec,+ proto_converter,Arc::new(AvroSource::new(table_schema)),)?;Ok(DataSourceExec::from_data_source(conf))@@ -771,7 +860,8 @@ impl protobuf::PhysicalPlanNode {scan: &protobuf::MemoryScanExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let partitions = scan.partitions@@ -801,7 +891,8 @@ impl protobuf::PhysicalPlanNode {&ordering.physical_sort_expr_nodes,ctx,&schema,- extension_codec,+ codec,+ proto_converter,)?;sort_information.extend(LexOrdering::new(sort_exprs));}@@ -820,10 +911,11 @@ impl protobuf::PhysicalPlanNode {coalesce_batches: &protobuf::CoalesceBatchesExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&coalesce_batches.input, ctx, extension_codec)?;+ into_physical_plan(&coalesce_batches.input, ctx, codec, proto_converter)?;Ok(Arc::new(#[expect(deprecated)]CoalesceBatchesExec::new(input, coalesce_batches.target_batch_size as usize)@@ -836,10 +928,11 @@ impl protobuf::PhysicalPlanNode {merge: &protobuf::CoalescePartitionsExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&merge.input, ctx, extension_codec)?;+ into_physical_plan(&merge.input, ctx, codec, proto_converter)?;Ok(Arc::new(CoalescePartitionsExec::new(input).with_fetch(merge.fetch.map(|f| f as usize)),@@ -851,15 +944,17 @@ impl protobuf::PhysicalPlanNode {repart: &protobuf::RepartitionExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&repart.input, ctx, extension_codec)?;+ into_physical_plan(&repart.input, ctx, codec, proto_converter)?;let partitioning = parse_protobuf_partitioning(repart.partitioning.as_ref(),ctx,input.schema().as_ref(),- extension_codec,+ codec,+ proto_converter,)?;Ok(Arc::new(RepartitionExec::try_new(input,@@ -872,10 +967,11 @@ impl protobuf::PhysicalPlanNode {limit: &protobuf::GlobalLimitExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&limit.input, ctx, extension_codec)?;+ into_physical_plan(&limit.input, ctx, codec, proto_converter)?;let fetch = if limit.fetch >= 0 {Some(limit.fetch as usize)} else {@@ -893,10 +989,11 @@ impl protobuf::PhysicalPlanNode {limit: &protobuf::LocalLimitExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&limit.input, ctx, extension_codec)?;+ into_physical_plan(&limit.input, ctx, codec, proto_converter)?;Ok(Arc::new(LocalLimitExec::new(input, limit.fetch as usize)))}@@ -905,10 +1002,11 @@ impl protobuf::PhysicalPlanNode {window_agg: &protobuf::WindowAggExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&window_agg.input, ctx, extension_codec)?;+ into_physical_plan(&window_agg.input, ctx, codec, proto_converter)?;let input_schema = input.schema();let physical_window_expr: Vec<Arc<dyn WindowExpr>> = window_agg@@ -919,7 +1017,8 @@ impl protobuf::PhysicalPlanNode {window_expr,ctx,input_schema.as_ref(),- extension_codec,+ codec,+ proto_converter,)}).collect::<Result<Vec<_>, _>>()?;@@ -928,7 +1027,12 @@ impl protobuf::PhysicalPlanNode {.partition_keys.iter().map(|expr| {- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)+ proto_converter.proto_to_physical_expr(+ expr,+ ctx,+ input.schema().as_ref(),+ codec,+ )}).collect::<Result<Vec<Arc<dyn PhysicalExpr>>>>()?;@@ -963,10 +1067,11 @@ impl protobuf::PhysicalPlanNode {hash_agg: &protobuf::AggregateExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&hash_agg.input, ctx, extension_codec)?;+ into_physical_plan(&hash_agg.input, ctx, codec, proto_converter)?;let mode = protobuf::AggregateMode::try_from(hash_agg.mode).map_err(|_| {proto_error(format!("Received a AggregateNode message with unknown AggregateMode {}",@@ -990,7 +1095,8 @@ impl protobuf::PhysicalPlanNode {.iter().zip(hash_agg.group_expr_name.iter()).map(|(expr, name)| {- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)+ proto_converter+ .proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec).map(|expr| (expr, name.to_string()))}).collect::<Result<Vec<_>, _>>()?;@@ -1000,7 +1106,8 @@ impl protobuf::PhysicalPlanNode {.iter().zip(hash_agg.group_expr_name.iter()).map(|(expr, name)| {- parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)+ proto_converter+ .proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec).map(|expr| (expr, name.to_string()))}).collect::<Result<Vec<_>, _>>()?;@@ -1029,7 +1136,12 @@ impl protobuf::PhysicalPlanNode {expr.expr.as_ref().map(|e| {- parse_physical_expr(e, ctx, &physical_schema, extension_codec)+ proto_converter.proto_to_physical_expr(+ e,+ ctx,+ &physical_schema,+ codec,+ )}).transpose()})@@ -1050,11 +1162,11 @@ impl protobuf::PhysicalPlanNode {.expr.iter().map(|e| {- parse_physical_expr(+ proto_converter.proto_to_physical_expr(e,ctx,&physical_schema,- extension_codec,+ codec,)}).collect::<Result<Vec<_>>>()?;@@ -1066,7 +1178,8 @@ impl protobuf::PhysicalPlanNode {e,ctx,&physical_schema,- extension_codec,+ codec,+ proto_converter,)}).collect::<Result<_>>()?;@@ -1076,11 +1189,11 @@ impl protobuf::PhysicalPlanNode {.map(|func| match func {AggregateFunction::UserDefinedAggrFunction(udaf_name) => {let agg_udf = match &agg_node.fun_definition {- Some(buf) => extension_codec- .try_decode_udaf(udaf_name, buf)?,+ Some(buf) => {+ codec.try_decode_udaf(udaf_name, buf)?+ }None => ctx.udaf(udaf_name).or_else(|_| {- extension_codec- .try_decode_udaf(udaf_name, &[])+ codec.try_decode_udaf(udaf_name, &[])})?,};@@ -1135,29 +1248,30 @@ impl protobuf::PhysicalPlanNode {hashjoin: &protobuf::HashJoinExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let left: Arc<dyn ExecutionPlan> =- into_physical_plan(&hashjoin.left, ctx, extension_codec)?;+ into_physical_plan(&hashjoin.left, ctx, codec, proto_converter)?;let right: Arc<dyn ExecutionPlan> =- into_physical_plan(&hashjoin.right, ctx, extension_codec)?;+ into_physical_plan(&hashjoin.right, ctx, codec, proto_converter)?;let left_schema = left.schema();let right_schema = right.schema();let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = hashjoin.on.iter().map(|col| {- let left = parse_physical_expr(+ let left = proto_converter.proto_to_physical_expr(&col.left.clone().unwrap(),ctx,left_schema.as_ref(),- extension_codec,+ codec,)?;- let right = parse_physical_expr(+ let right = proto_converter.proto_to_physical_expr(&col.right.clone().unwrap(),ctx,right_schema.as_ref(),- extension_codec,+ codec,)?;Ok((left, right))})@@ -1186,12 +1300,12 @@ impl protobuf::PhysicalPlanNode {.ok_or_else(|| proto_error("Missing JoinFilter schema"))?.try_into()?;- let expression = parse_physical_expr(+ let expression = proto_converter.proto_to_physical_expr(f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,ctx, &schema,- extension_codec,+ codec,)?;let column_indices = f.column_indices.iter()@@ -1254,27 +1368,28 @@ impl protobuf::PhysicalPlanNode {sym_join: &protobuf::SymmetricHashJoinExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let left = into_physical_plan(&sym_join.left, ctx, extension_codec)?;- let right = into_physical_plan(&sym_join.right, ctx, extension_codec)?;+ let left = into_physical_plan(&sym_join.left, ctx, codec, proto_converter)?;+ let right = into_physical_plan(&sym_join.right, ctx, codec, proto_converter)?;let left_schema = left.schema();let right_schema = right.schema();let on = sym_join.on.iter().map(|col| {- let left = parse_physical_expr(+ let left = proto_converter.proto_to_physical_expr(&col.left.clone().unwrap(),ctx,left_schema.as_ref(),- extension_codec,+ codec,)?;- let right = parse_physical_expr(+ let right = proto_converter.proto_to_physical_expr(&col.right.clone().unwrap(),ctx,right_schema.as_ref(),- extension_codec,+ codec,)?;Ok((left, right))})@@ -1303,12 +1418,12 @@ impl protobuf::PhysicalPlanNode {.ok_or_else(|| proto_error("Missing JoinFilter schema"))?.try_into()?;- let expression = parse_physical_expr(+ let expression = proto_converter.proto_to_physical_expr(f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,ctx, &schema,- extension_codec,+ codec,)?;let column_indices = f.column_indices.iter()@@ -1334,7 +1449,8 @@ impl protobuf::PhysicalPlanNode {&sym_join.left_sort_exprs,ctx,&left_schema,- extension_codec,+ codec,+ proto_converter,)?;let left_sort_exprs = LexOrdering::new(left_sort_exprs);@@ -1342,7 +1458,8 @@ impl protobuf::PhysicalPlanNode {&sym_join.right_sort_exprs,ctx,&right_schema,- extension_codec,+ codec,+ proto_converter,)?;let right_sort_exprs = LexOrdering::new(right_sort_exprs);@@ -1382,11 +1499,12 @@ impl protobuf::PhysicalPlanNode {union: &protobuf::UnionExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];for input in &union.inputs {- inputs.push(input.try_into_physical_plan(ctx, extension_codec)?);+ inputs.push(proto_converter.proto_to_execution_plan(ctx, codec, input)?);}UnionExec::try_new(inputs)}@@ -1396,11 +1514,12 @@ impl protobuf::PhysicalPlanNode {interleave: &protobuf::InterleaveExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let mut inputs: Vec<Arc<dyn ExecutionPlan>> = vec![];for input in &interleave.inputs {- inputs.push(input.try_into_physical_plan(ctx, extension_codec)?);+ inputs.push(proto_converter.proto_to_execution_plan(ctx, codec, input)?);}Ok(Arc::new(InterleaveExec::try_new(inputs)?))}@@ -1410,12 +1529,13 @@ impl protobuf::PhysicalPlanNode {crossjoin: &protobuf::CrossJoinExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let left: Arc<dyn ExecutionPlan> =- into_physical_plan(&crossjoin.left, ctx, extension_codec)?;+ into_physical_plan(&crossjoin.left, ctx, codec, proto_converter)?;let right: Arc<dyn ExecutionPlan> =- into_physical_plan(&crossjoin.right, ctx, extension_codec)?;+ into_physical_plan(&crossjoin.right, ctx, codec, proto_converter)?;Ok(Arc::new(CrossJoinExec::new(left, right)))}@@ -1424,7 +1544,8 @@ impl protobuf::PhysicalPlanNode {empty: &protobuf::EmptyExecNode,_ctx: &TaskContext,- _extension_codec: &dyn PhysicalExtensionCodec,+ _codec: &dyn PhysicalExtensionCodec,+ _proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let schema = Arc::new(convert_required!(empty.schema)?);Ok(Arc::new(EmptyExec::new(schema)))@@ -1435,7 +1556,7 @@ impl protobuf::PhysicalPlanNode {placeholder: &protobuf::PlaceholderRowExecNode,_ctx: &TaskContext,- _extension_codec: &dyn PhysicalExtensionCodec,+ _codec: &dyn PhysicalExtensionCodec,) -> Result<Arc<dyn ExecutionPlan>> {let schema = Arc::new(convert_required!(placeholder.schema)?);Ok(Arc::new(PlaceholderRowExec::new(schema)))@@ -1446,9 +1567,10 @@ impl protobuf::PhysicalPlanNode {sort: &protobuf::SortExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sort.input, ctx, extension_codec)?;+ let input = into_physical_plan(&sort.input, ctx, codec, proto_converter)?;let exprs = sort.expr.iter()@@ -1469,7 +1591,7 @@ impl protobuf::PhysicalPlanNode {})?.as_ref();Ok(PhysicalSortExpr {- expr: parse_physical_expr(expr, ctx, input.schema().as_ref(), extension_codec)?,+ expr: proto_converter.proto_to_physical_expr(expr, ctx, input.schema().as_ref(), codec)?,options: SortOptions {descending: !sort_expr.asc,nulls_first: sort_expr.nulls_first,@@ -1498,9 +1620,10 @@ impl protobuf::PhysicalPlanNode {sort: &protobuf::SortPreservingMergeExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sort.input, ctx, extension_codec)?;+ let input = into_physical_plan(&sort.input, ctx, codec, proto_converter)?;let exprs = sort.expr.iter()@@ -1521,11 +1644,11 @@ impl protobuf::PhysicalPlanNode {})?.as_ref();Ok(PhysicalSortExpr {- expr: parse_physical_expr(+ expr: proto_converter.proto_to_physical_expr(expr,ctx,input.schema().as_ref(),- extension_codec,+ codec,)?,options: SortOptions {descending: !sort_expr.asc,@@ -1551,16 +1674,16 @@ impl protobuf::PhysicalPlanNode {extension: &protobuf::PhysicalExtensionNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let inputs: Vec<Arc<dyn ExecutionPlan>> = extension.inputs.iter()- .map(|i| i.try_into_physical_plan(ctx, extension_codec))+ .map(|i| proto_converter.proto_to_execution_plan(ctx, codec, i)).collect::<Result<_>>()?;- let extension_node =- extension_codec.try_decode(extension.node.as_slice(), &inputs, ctx)?;+ let extension_node = codec.try_decode(extension.node.as_slice(), &inputs, ctx)?;Ok(extension_node)}@@ -1570,12 +1693,13 @@ impl protobuf::PhysicalPlanNode {join: &protobuf::NestedLoopJoinExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let left: Arc<dyn ExecutionPlan> =- into_physical_plan(&join.left, ctx, extension_codec)?;+ into_physical_plan(&join.left, ctx, codec, proto_converter)?;let right: Arc<dyn ExecutionPlan> =- into_physical_plan(&join.right, ctx, extension_codec)?;+ into_physical_plan(&join.right, ctx, codec, proto_converter)?;let join_type = protobuf::JoinType::try_from(join.join_type).map_err(|_| {proto_error(format!("Received a NestedLoopJoinExecNode message with unknown JoinType {}",@@ -1592,12 +1716,12 @@ impl protobuf::PhysicalPlanNode {.ok_or_else(|| proto_error("Missing JoinFilter schema"))?.try_into()?;- let expression = parse_physical_expr(+ let expression = proto_converter.proto_to_physical_expr(f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,ctx, &schema,- extension_codec,+ codec,)?;let column_indices = f.column_indices.iter()@@ -1644,10 +1768,11 @@ impl protobuf::PhysicalPlanNode {analyze: &protobuf::AnalyzeExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&analyze.input, ctx, extension_codec)?;+ into_physical_plan(&analyze.input, ctx, codec, proto_converter)?;Ok(Arc::new(AnalyzeExec::new(analyze.verbose,analyze.show_statistics,@@ -1662,9 +1787,10 @@ impl protobuf::PhysicalPlanNode {sink: &protobuf::JsonSinkExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sink.input, ctx, extension_codec)?;+ let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;let data_sink: JsonSink = sink.sink@@ -1680,7 +1806,8 @@ impl protobuf::PhysicalPlanNode {&collection.physical_sort_expr_nodes,ctx,&sink_schema,- extension_codec,+ codec,+ proto_converter,).map(|sort_exprs| {LexRequirement::new(sort_exprs.into_iter().map(Into::into))@@ -1700,9 +1827,10 @@ impl protobuf::PhysicalPlanNode {sink: &protobuf::CsvSinkExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&sink.input, ctx, extension_codec)?;+ let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;let data_sink: CsvSink = sink.sink@@ -1718,7 +1846,8 @@ impl protobuf::PhysicalPlanNode {&collection.physical_sort_expr_nodes,ctx,&sink_schema,- extension_codec,+ codec,+ proto_converter,).map(|sort_exprs| {LexRequirement::new(sort_exprs.into_iter().map(Into::into))@@ -1739,11 +1868,12 @@ impl protobuf::PhysicalPlanNode {sink: &protobuf::ParquetSinkExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "parquet")]{- let input = into_physical_plan(&sink.input, ctx, extension_codec)?;+ let input = into_physical_plan(&sink.input, ctx, codec, proto_converter)?;let data_sink: ParquetSink = sink.sink@@ -1759,7 +1889,8 @@ impl protobuf::PhysicalPlanNode {&collection.physical_sort_expr_nodes,ctx,&sink_schema,- extension_codec,+ codec,+ proto_converter,).map(|sort_exprs| {LexRequirement::new(sort_exprs.into_iter().map(Into::into))@@ -1782,9 +1913,10 @@ impl protobuf::PhysicalPlanNode {unnest: &protobuf::UnnestExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&unnest.input, ctx, extension_codec)?;+ let input = into_physical_plan(&unnest.input, ctx, codec, proto_converter)?;Ok(Arc::new(UnnestExec::new(input,@@ -1813,11 +1945,12 @@ impl protobuf::PhysicalPlanNode {sort_join: &SortMergeJoinExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let left = into_physical_plan(&sort_join.left, ctx, extension_codec)?;+ let left = into_physical_plan(&sort_join.left, ctx, codec, proto_converter)?;let left_schema = left.schema();- let right = into_physical_plan(&sort_join.right, ctx, extension_codec)?;+ let right = into_physical_plan(&sort_join.right, ctx, codec, proto_converter)?;let right_schema = right.schema();let filter = sort_join@@ -1830,13 +1963,13 @@ impl protobuf::PhysicalPlanNode {.ok_or_else(|| proto_error("Missing JoinFilter schema"))?.try_into()?;- let expression = parse_physical_expr(+ let expression = proto_converter.proto_to_physical_expr(f.expression.as_ref().ok_or_else(|| {proto_error("Unexpected empty filter expression")})?,ctx,&schema,- extension_codec,+ codec,)?;let column_indices = f.column_indices@@ -1893,17 +2026,17 @@ impl protobuf::PhysicalPlanNode {.on.iter().map(|col| {- let left = parse_physical_expr(+ let left = proto_converter.proto_to_physical_expr(&col.left.clone().unwrap(),ctx,left_schema.as_ref(),- extension_codec,+ codec,)?;- let right = parse_physical_expr(+ let right = proto_converter.proto_to_physical_expr(&col.right.clone().unwrap(),ctx,right_schema.as_ref(),- extension_codec,+ codec,)?;Ok((left, right))})@@ -1990,9 +2123,10 @@ impl protobuf::PhysicalPlanNode {field_stream: &protobuf::CooperativeExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let input = into_physical_plan(&field_stream.input, ctx, extension_codec)?;+ let input = into_physical_plan(&field_stream.input, ctx, codec, proto_converter)?;Ok(Arc::new(CooperativeExec::new(input)))}@@ -2000,10 +2134,11 @@ impl protobuf::PhysicalPlanNode {&self,async_func: &protobuf::AsyncFuncExecNode,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {let input: Arc<dyn ExecutionPlan> =- into_physical_plan(&async_func.input, ctx, extension_codec)?;+ into_physical_plan(&async_func.input, ctx, codec, proto_converter)?;if async_func.async_exprs.len() != async_func.async_expr_names.len() {return internal_err!(@@ -2016,11 +2151,11 @@ impl protobuf::PhysicalPlanNode {.iter().zip(async_func.async_expr_names.iter()).map(|(expr, name)| {- let physical_expr = parse_physical_expr(+ let physical_expr = proto_converter.proto_to_physical_expr(expr,ctx,input.schema().as_ref(),- extension_codec,+ codec,)?;Ok(Arc::new(AsyncFuncExpr::try_new(@@ -2036,7 +2171,7 @@ impl protobuf::PhysicalPlanNode {fn try_from_explain_exec(exec: &ExplainExec,- _extension_codec: &dyn PhysicalExtensionCodec,+ _codec: &dyn PhysicalExtensionCodec,) -> Result<Self> {Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Explain(@@ -2055,16 +2190,20 @@ impl protobuf::PhysicalPlanNode {fn try_from_projection_exec(exec: &ProjectionExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let expr = exec.expr().iter()- .map(|proj_expr| serialize_physical_expr(&proj_expr.expr, extension_codec))+ .map(|proj_expr| {+ proto_converter.physical_expr_to_proto(&proj_expr.expr, codec)+ }).collect::<Result<Vec<_>>>()?;let expr_name = exec.expr()@@ -2084,11 +2223,13 @@ impl protobuf::PhysicalPlanNode {fn try_from_analyze_exec(exec: &AnalyzeExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Analyze(Box::new(@@ -2104,20 +2245,22 @@ impl protobuf::PhysicalPlanNode {fn try_from_filter_exec(exec: &FilterExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Filter(Box::new(protobuf::FilterExecNode {input: Some(Box::new(input)),- expr: Some(serialize_physical_expr(- exec.predicate(),- extension_codec,- )?),+ expr: Some(+ proto_converter+ .physical_expr_to_proto(exec.predicate(), codec)?,+ ),default_filter_selectivity: exec.default_selectivity() as u32,projection: exec.projection().as_ref().map_or_else(Vec::new, |v| {v.iter().map(|x| *x as u32).collect::<Vec<u32>>()@@ -2130,11 +2273,13 @@ impl protobuf::PhysicalPlanNode {fn try_from_global_limit_exec(limit: &GlobalLimitExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(limit.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {@@ -2153,11 +2298,13 @@ impl protobuf::PhysicalPlanNode {fn try_from_local_limit_exec(limit: &LocalLimitExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(limit.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::LocalLimit(Box::new(@@ -2171,22 +2318,25 @@ impl protobuf::PhysicalPlanNode {fn try_from_hash_join_exec(exec: &HashJoinExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.right().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let on: Vec<protobuf::JoinOn> = exec.on().iter().map(|tuple| {- let l = serialize_physical_expr(&tuple.0, extension_codec)?;- let r = serialize_physical_expr(&tuple.1, extension_codec)?;+ let l = proto_converter.physical_expr_to_proto(&tuple.0, codec)?;+ let r = proto_converter.physical_expr_to_proto(&tuple.1, codec)?;Ok::<_, DataFusionError>(protobuf::JoinOn {left: Some(l),right: Some(r),@@ -2200,7 +2350,7 @@ impl protobuf::PhysicalPlanNode {.as_ref().map(|f| {let expression =- serialize_physical_expr(f.expression(), extension_codec)?;+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;let column_indices = f.column_indices().iter()@@ -2248,22 +2398,25 @@ impl protobuf::PhysicalPlanNode {fn try_from_symmetric_hash_join_exec(exec: &SymmetricHashJoinExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.right().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let on = exec.on().iter().map(|tuple| {- let l = serialize_physical_expr(&tuple.0, extension_codec)?;- let r = serialize_physical_expr(&tuple.1, extension_codec)?;+ let l = proto_converter.physical_expr_to_proto(&tuple.0, codec)?;+ let r = proto_converter.physical_expr_to_proto(&tuple.1, codec)?;Ok::<_, DataFusionError>(protobuf::JoinOn {left: Some(l),right: Some(r),@@ -2277,7 +2430,7 @@ impl protobuf::PhysicalPlanNode {.as_ref().map(|f| {let expression =- serialize_physical_expr(f.expression(), extension_codec)?;+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;let column_indices = f.column_indices().iter()@@ -2314,10 +2467,10 @@ impl protobuf::PhysicalPlanNode {.iter().map(|expr| {Ok(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(serialize_physical_expr(- &expr.expr,- extension_codec,- )?)),+ expr: Some(Box::new(+ proto_converter+ .physical_expr_to_proto(&expr.expr, codec)?,+ )),asc: !expr.options.descending,nulls_first: expr.options.nulls_first,})@@ -2334,10 +2487,10 @@ impl protobuf::PhysicalPlanNode {.iter().map(|expr| {Ok(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(serialize_physical_expr(- &expr.expr,- extension_codec,- )?)),+ expr: Some(Box::new(+ proto_converter+ .physical_expr_to_proto(&expr.expr, codec)?,+ )),asc: !expr.options.descending,nulls_first: expr.options.nulls_first,})@@ -2366,22 +2519,25 @@ impl protobuf::PhysicalPlanNode {fn try_from_sort_merge_join_exec(exec: &SortMergeJoinExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.right().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let on = exec.on().iter().map(|tuple| {- let l = serialize_physical_expr(&tuple.0, extension_codec)?;- let r = serialize_physical_expr(&tuple.1, extension_codec)?;+ let l = proto_converter.physical_expr_to_proto(&tuple.0, codec)?;+ let r = proto_converter.physical_expr_to_proto(&tuple.1, codec)?;Ok::<_, DataFusionError>(protobuf::JoinOn {left: Some(l),right: Some(r),@@ -2395,7 +2551,7 @@ impl protobuf::PhysicalPlanNode {.as_ref().map(|f| {let expression =- serialize_physical_expr(f.expression(), extension_codec)?;+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;let column_indices = f.column_indices().iter()@@ -2435,7 +2591,7 @@ impl protobuf::PhysicalPlanNode {Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::SortMergeJoin(Box::new(- protobuf::SortMergeJoinExecNode {+ SortMergeJoinExecNode {left: Some(Box::new(left)),right: Some(Box::new(right)),on,@@ -2450,15 +2606,18 @@ impl protobuf::PhysicalPlanNode {fn try_from_cross_join_exec(exec: &CrossJoinExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.right().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::CrossJoin(Box::new(@@ -2472,7 +2631,8 @@ impl protobuf::PhysicalPlanNode {fn try_from_aggregate_exec(exec: &AggregateExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {let groups: Vec<bool> = exec.group_expr()@@ -2492,13 +2652,15 @@ impl protobuf::PhysicalPlanNode {let filter = exec.filter_expr().iter()- .map(|expr| serialize_maybe_filter(expr.to_owned(), extension_codec))+ .map(|expr| serialize_maybe_filter(expr.to_owned(), codec, proto_converter)).collect::<Result<Vec<_>>>()?;let agg = exec.aggr_expr().iter()- .map(|expr| serialize_physical_aggr_expr(expr.to_owned(), extension_codec))+ .map(|expr| {+ serialize_physical_aggr_expr(expr.to_owned(), codec, proto_converter)+ }).collect::<Result<Vec<_>>>()?;let agg_names = exec@@ -2517,23 +2679,24 @@ impl protobuf::PhysicalPlanNode {}};let input_schema = exec.input_schema();- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let null_expr = exec.group_expr().null_expr().iter()- .map(|expr| serialize_physical_expr(&expr.0, extension_codec))+ .map(|expr| proto_converter.physical_expr_to_proto(&expr.0, codec)).collect::<Result<Vec<_>>>()?;let group_expr = exec.group_expr().expr().iter()- .map(|expr| serialize_physical_expr(&expr.0, extension_codec))+ .map(|expr| proto_converter.physical_expr_to_proto(&expr.0, codec)).collect::<Result<Vec<_>>>()?;let limit = exec.limit_options().map(|config| protobuf::AggLimit {@@ -2563,7 +2726,7 @@ impl protobuf::PhysicalPlanNode {fn try_from_empty_exec(empty: &EmptyExec,- _extension_codec: &dyn PhysicalExtensionCodec,+ _codec: &dyn PhysicalExtensionCodec,) -> Result<Self> {let schema = empty.schema().as_ref().try_into()?;Ok(protobuf::PhysicalPlanNode {@@ -2575,7 +2738,7 @@ impl protobuf::PhysicalPlanNode {fn try_from_placeholder_row_exec(empty: &PlaceholderRowExec,- _extension_codec: &dyn PhysicalExtensionCodec,+ _codec: &dyn PhysicalExtensionCodec,) -> Result<Self> {let schema = empty.schema().as_ref().try_into()?;Ok(protobuf::PhysicalPlanNode {@@ -2590,11 +2753,13 @@ impl protobuf::PhysicalPlanNode {#[expect(deprecated)]fn try_from_coalesce_batches_exec(coalesce_batches: &CoalesceBatchesExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(coalesce_batches.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::CoalesceBatches(Box::new(@@ -2609,7 +2774,8 @@ impl protobuf::PhysicalPlanNode {fn try_from_data_source_exec(data_source_exec: &DataSourceExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Self>> {let data_source = data_source_exec.data_source();if let Some(maybe_csv) = data_source.as_any().downcast_ref::<FileScanConfig>() {@@ -2620,7 +2786,8 @@ impl protobuf::PhysicalPlanNode {protobuf::CsvScanExecNode {base_conf: Some(serialize_file_scan_config(maybe_csv,- extension_codec,+ codec,+ proto_converter,)?),has_header: csv_config.has_header(),delimiter: byte_to_string(@@ -2661,7 +2828,8 @@ impl protobuf::PhysicalPlanNode {protobuf::JsonScanExecNode {base_conf: Some(serialize_file_scan_config(scan_conf,- extension_codec,+ codec,+ proto_converter,)?),},)),@@ -2675,14 +2843,15 @@ impl protobuf::PhysicalPlanNode {{let predicate = conf.filter()- .map(|pred| serialize_physical_expr(&pred, extension_codec))+ .map(|pred| proto_converter.physical_expr_to_proto(&pred, codec)).transpose()?;return Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::ParquetScan(protobuf::ParquetScanExecNode {base_conf: Some(serialize_file_scan_config(maybe_parquet,- extension_codec,+ codec,+ proto_converter,)?),predicate,parquet_options: Some(conf.table_parquet_options().try_into()?),@@ -2700,7 +2869,8 @@ impl protobuf::PhysicalPlanNode {protobuf::AvroScanExecNode {base_conf: Some(serialize_file_scan_config(maybe_avro,- extension_codec,+ codec,+ proto_converter,)?),},)),@@ -2733,7 +2903,8 @@ impl protobuf::PhysicalPlanNode {.map(|ordering| {let sort_exprs = serialize_physical_sort_exprs(ordering.to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok::<_, DataFusionError>(protobuf::PhysicalSortExprNodeCollection {physical_sort_expr_nodes: sort_exprs,@@ -2760,11 +2931,13 @@ impl protobuf::PhysicalPlanNode {fn try_from_coalesce_partitions_exec(exec: &CoalescePartitionsExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Merge(Box::new(@@ -2778,15 +2951,17 @@ impl protobuf::PhysicalPlanNode {fn try_from_repartition_exec(exec: &RepartitionExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let pb_partitioning =- serialize_partitioning(exec.partitioning(), extension_codec)?;+ serialize_partitioning(exec.partitioning(), codec, proto_converter)?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Repartition(Box::new(@@ -2800,21 +2975,18 @@ impl protobuf::PhysicalPlanNode {fn try_from_sort_exec(exec: &SortExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(- exec.input().to_owned(),- extension_codec,- )?;+ let input = proto_converter.execution_plan_to_proto(exec.input(), codec)?;let expr = exec.expr().iter().map(|expr| {let sort_expr = Box::new(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(serialize_physical_expr(- &expr.expr,- extension_codec,- )?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(&expr.expr, codec)?,+ )),asc: !expr.options.descending,nulls_first: expr.options.nulls_first,});@@ -2840,14 +3012,18 @@ impl protobuf::PhysicalPlanNode {fn try_from_union_exec(union: &UnionExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];for input in union.inputs() {- inputs.push(protobuf::PhysicalPlanNode::try_from_physical_plan(- input.to_owned(),- extension_codec,- )?);+ inputs.push(+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(+ input.to_owned(),+ codec,+ proto_converter,+ )?,+ );}Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Union(protobuf::UnionExecNode {@@ -2858,14 +3034,18 @@ impl protobuf::PhysicalPlanNode {fn try_from_interleave_exec(interleave: &InterleaveExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];for input in interleave.inputs() {- inputs.push(protobuf::PhysicalPlanNode::try_from_physical_plan(- input.to_owned(),- extension_codec,- )?);+ inputs.push(+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(+ input.to_owned(),+ codec,+ proto_converter,+ )?,+ );}Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Interleave(@@ -2876,21 +3056,22 @@ impl protobuf::PhysicalPlanNode {fn try_from_sort_preserving_merge_exec(exec: &SortPreservingMergeExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let expr = exec.expr().iter().map(|expr| {let sort_expr = Box::new(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(serialize_physical_expr(- &expr.expr,- extension_codec,- )?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(&expr.expr, codec)?,+ )),asc: !expr.options.descending,nulls_first: expr.options.nulls_first,});@@ -2912,15 +3093,18 @@ impl protobuf::PhysicalPlanNode {fn try_from_nested_loop_join_exec(exec: &NestedLoopJoinExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let left = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;- let right = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.right().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let join_type: protobuf::JoinType = exec.join_type().to_owned().into();@@ -2929,7 +3113,7 @@ impl protobuf::PhysicalPlanNode {.as_ref().map(|f| {let expression =- serialize_physical_expr(f.expression(), extension_codec)?;+ proto_converter.physical_expr_to_proto(f.expression(), codec)?;let column_indices = f.column_indices().iter()@@ -2967,23 +3151,25 @@ impl protobuf::PhysicalPlanNode {fn try_from_window_agg_exec(exec: &WindowAggExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let window_expr = exec.window_expr().iter()- .map(|e| serialize_physical_window_expr(e, extension_codec))+ .map(|e| serialize_physical_window_expr(e, codec, proto_converter)).collect::<Result<Vec<protobuf::PhysicalWindowExprNode>>>()?;let partition_keys = exec.partition_keys().iter()- .map(|e| serialize_physical_expr(e, extension_codec))+ .map(|e| proto_converter.physical_expr_to_proto(e, codec)).collect::<Result<Vec<protobuf::PhysicalExprNode>>>()?;Ok(protobuf::PhysicalPlanNode {@@ -3000,23 +3186,25 @@ impl protobuf::PhysicalPlanNode {fn try_from_bounded_window_agg_exec(exec: &BoundedWindowAggExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let window_expr = exec.window_expr().iter()- .map(|e| serialize_physical_window_expr(e, extension_codec))+ .map(|e| serialize_physical_window_expr(e, codec, proto_converter)).collect::<Result<Vec<protobuf::PhysicalWindowExprNode>>>()?;let partition_keys = exec.partition_keys().iter()- .map(|e| serialize_physical_expr(e, extension_codec))+ .map(|e| proto_converter.physical_expr_to_proto(e, codec)).collect::<Result<Vec<protobuf::PhysicalExprNode>>>()?;let input_order_mode = match &exec.input_order_mode {@@ -3049,12 +3237,14 @@ impl protobuf::PhysicalPlanNode {fn try_from_data_sink_exec(exec: &DataSinkExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Self>> {let input: protobuf::PhysicalPlanNode =- protobuf::PhysicalPlanNode::try_from_physical_plan(+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;let sort_order = match exec.sort_order() {Some(requirements) => {@@ -3063,10 +3253,10 @@ impl protobuf::PhysicalPlanNode {.map(|requirement| {let expr: PhysicalSortExpr = requirement.to_owned().into();let sort_expr = protobuf::PhysicalSortExprNode {- expr: Some(Box::new(serialize_physical_expr(- &expr.expr,- extension_codec,- )?)),+ expr: Some(Box::new(+ proto_converter+ .physical_expr_to_proto(&expr.expr, codec)?,+ )),asc: !expr.options.descending,nulls_first: expr.options.nulls_first,};@@ -3126,11 +3316,13 @@ impl protobuf::PhysicalPlanNode {fn try_from_unnest_exec(exec: &UnnestExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {@@ -3159,11 +3351,13 @@ impl protobuf::PhysicalPlanNode {fn try_from_cooperative_exec(exec: &CooperativeExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),- extension_codec,+ codec,+ proto_converter,)?;Ok(protobuf::PhysicalPlanNode {@@ -3292,18 +3486,21 @@ impl protobuf::PhysicalPlanNode {fn try_from_async_func_exec(exec: &AsyncFuncExec,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Self> {- let input = protobuf::PhysicalPlanNode::try_from_physical_plan(+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(Arc::clone(exec.input()),- extension_codec,+ codec,+ proto_converter,)?;let mut async_exprs = vec![];let mut async_expr_names = vec![];for async_expr in exec.async_exprs() {- async_exprs.push(serialize_physical_expr(&async_expr.func, extension_codec)?);+ async_exprs+ .push(proto_converter.physical_expr_to_proto(&async_expr.func, codec)?);async_expr_names.push(async_expr.name.clone())}@@ -3333,12 +3530,12 @@ pub trait AsExecutionPlan: Debug + Send + Sync + Clone {&self,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,) -> Result<Arc<dyn ExecutionPlan>>;fn try_from_physical_plan(plan: Arc<dyn ExecutionPlan>,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,) -> Result<Self>whereSelf: Sized;@@ -3419,6 +3616,38 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {}}+/// Controls the conversion of physical plans and expressions to and from their+/// Protobuf variants. Using this trait, users can perform optimizations on the+/// conversion process or collect performance metrics.+pub trait PhysicalProtoConverterExtension {+ fn proto_to_execution_plan(+ &self,+ ctx: &TaskContext,+ codec: &dyn PhysicalExtensionCodec,+ proto: &protobuf::PhysicalPlanNode,+ ) -> Result<Arc<dyn ExecutionPlan>>;++ fn execution_plan_to_proto(+ &self,+ plan: &Arc<dyn ExecutionPlan>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<protobuf::PhysicalPlanNode>;++ fn proto_to_physical_expr(+ &self,+ proto: &protobuf::PhysicalExprNode,+ ctx: &TaskContext,+ input_schema: &Schema,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Arc<dyn PhysicalExpr>>;++ fn physical_expr_to_proto(+ &self,+ expr: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<protobuf::PhysicalExprNode>;+}+/// DataEncoderTuple captures the position of the encoder/// in the codec list that was used to encode the data and actual encoded data#[derive(Clone, PartialEq, prost::Message)]@@ -3432,6 +3661,55 @@ struct DataEncoderTuple {pub blob: Vec<u8>,}+pub struct DefaultPhysicalProtoConverter;+impl PhysicalProtoConverterExtension for DefaultPhysicalProtoConverter {+ fn proto_to_execution_plan(+ &self,+ ctx: &TaskContext,+ codec: &dyn PhysicalExtensionCodec,+ proto: &protobuf::PhysicalPlanNode,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ proto.try_into_physical_plan_with_converter(ctx, codec, self)+ }++ fn execution_plan_to_proto(+ &self,+ plan: &Arc<dyn ExecutionPlan>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<protobuf::PhysicalPlanNode>+ where+ Self: Sized,+ {+ protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::clone(plan),+ codec,+ self,+ )+ }++ fn proto_to_physical_expr(+ &self,+ proto: &protobuf::PhysicalExprNode,+ ctx: &TaskContext,+ input_schema: &Schema,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Arc<dyn PhysicalExpr>>+ where+ Self: Sized,+ {+ // Default implementation calls the free function+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)+ }++ fn physical_expr_to_proto(+ &self,+ expr: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<protobuf::PhysicalExprNode> {+ serialize_physical_expr_with_converter(expr, codec, self)+ }+}+/// A PhysicalExtensionCodec that tries one of multiple inner codecs/// until one works#[derive(Debug)]@@ -3534,10 +3812,11 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {fn into_physical_plan(node: &Option<Box<protobuf::PhysicalPlanNode>>,ctx: &TaskContext,- extension_codec: &dyn PhysicalExtensionCodec,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {if let Some(field) = node {- field.try_into_physical_plan(ctx, extension_codec)+ proto_converter.proto_to_execution_plan(ctx, codec, field)} else {Err(proto_error("Missing required field in protobuf"))}diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 9558effb8a..f85b1d1e12 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -24,8 +24,7 @@ use datafusion_common::{DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,};use datafusion_datasource::file_scan_config::FileScanConfig;-use datafusion_datasource::file_sink_config::FileSink;-use datafusion_datasource::file_sink_config::FileSinkConfig;+use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};use datafusion_datasource::{FileRange, PartitionedFile};use datafusion_datasource_csv::file_format::CsvSink;use datafusion_datasource_json::file_format::JsonSink;@@ -36,31 +35,37 @@ use datafusion_physical_expr::ScalarFunctionExpr;use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;-use datafusion_physical_plan::expressions::LikeExpr;use datafusion_physical_plan::expressions::{BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,- Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,+ LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,};use datafusion_physical_plan::joins::{HashExpr, HashTableLookupExpr};use datafusion_physical_plan::udaf::AggregateFunctionExpr;use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};+use super::{+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ PhysicalProtoConverterExtension,+};use crate::protobuf::{self, PhysicalSortExprNode, PhysicalSortExprNodeCollection,physical_aggregate_expr_node, physical_window_expr_node,};-use super::PhysicalExtensionCodec;-#[expect(clippy::needless_pass_by_value)]pub fn serialize_physical_aggr_expr(aggr_expr: Arc<AggregateFunctionExpr>,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::PhysicalExprNode> {- let expressions = serialize_physical_exprs(&aggr_expr.expressions(), codec)?;- let order_bys =- serialize_physical_sort_exprs(aggr_expr.order_bys().iter().cloned(), codec)?;+ let expressions =+ serialize_physical_exprs(&aggr_expr.expressions(), codec, proto_converter)?;+ let order_bys = serialize_physical_sort_exprs(+ aggr_expr.order_bys().iter().cloned(),+ codec,+ proto_converter,+ )?;let name = aggr_expr.fun().name().to_string();let mut buf = Vec::new();@@ -100,6 +105,7 @@ fn serialize_physical_window_aggr_expr(pub fn serialize_physical_window_expr(window_expr: &Arc<dyn WindowExpr>,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::PhysicalWindowExprNode> {let expr = window_expr.as_any();let args = window_expr.expressions().to_vec();@@ -155,9 +161,14 @@ pub fn serialize_physical_window_expr(return not_impl_err!("WindowExpr not supported: {window_expr:?}");};- let args = serialize_physical_exprs(&args, codec)?;- let partition_by = serialize_physical_exprs(window_expr.partition_by(), codec)?;- let order_by = serialize_physical_sort_exprs(window_expr.order_by().to_vec(), codec)?;+ let args = serialize_physical_exprs(&args, codec, proto_converter)?;+ let partition_by =+ serialize_physical_exprs(window_expr.partition_by(), codec, proto_converter)?;+ let order_by = serialize_physical_sort_exprs(+ window_expr.order_by().to_vec(),+ codec,+ proto_converter,+ )?;let window_frame: protobuf::WindowFrame = window_frame.as_ref().try_into()@@ -179,22 +190,24 @@ pub fn serialize_physical_window_expr(pub fn serialize_physical_sort_exprs<I>(sort_exprs: I,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Vec<PhysicalSortExprNode>>whereI: IntoIterator<Item = PhysicalSortExpr>,{sort_exprs.into_iter()- .map(|sort_expr| serialize_physical_sort_expr(sort_expr, codec))+ .map(|sort_expr| serialize_physical_sort_expr(sort_expr, codec, proto_converter)).collect()}pub fn serialize_physical_sort_expr(sort_expr: PhysicalSortExpr,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<PhysicalSortExprNode> {let PhysicalSortExpr { expr, options } = sort_expr;- let expr = serialize_physical_expr(&expr, codec)?;+ let expr = proto_converter.physical_expr_to_proto(&expr, codec)?;Ok(PhysicalSortExprNode {expr: Some(Box::new(expr)),asc: !options.descending,@@ -205,13 +218,14 @@ pub fn serialize_physical_sort_expr(pub fn serialize_physical_exprs<'a, I>(values: I,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Vec<protobuf::PhysicalExprNode>>whereI: IntoIterator<Item = &'a Arc<dyn PhysicalExpr>>,{values.into_iter()- .map(|value| serialize_physical_expr(value, codec))+ .map(|value| proto_converter.physical_expr_to_proto(value, codec)).collect()}@@ -222,6 +236,24 @@ wherepub fn serialize_physical_expr(value: &Arc<dyn PhysicalExpr>,codec: &dyn PhysicalExtensionCodec,+) -> Result<protobuf::PhysicalExprNode> {+ serialize_physical_expr_with_converter(+ value,+ codec,+ &DefaultPhysicalProtoConverter {},+ )+}++/// Serialize a `PhysicalExpr` to default protobuf representation.+///+/// If required, a [`PhysicalExtensionCodec`] can be provided which can handle+/// serialization of udfs requiring specialized serialization (see [`PhysicalExtensionCodec::try_encode_udf`]).+/// A [`PhysicalProtoConverterExtension`] can be provided to handle the+/// conversion process (see [`PhysicalProtoConverterExtension::physical_expr_to_proto`]).+pub fn serialize_physical_expr_with_converter(+ value: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::PhysicalExprNode> {// Snapshot the expr in case it has dynamic predicate state so// it can be serialized@@ -271,8 +303,12 @@ pub fn serialize_physical_expr(})} else if let Some(expr) = expr.downcast_ref::<BinaryExpr>() {let binary_expr = Box::new(protobuf::PhysicalBinaryExprNode {- l: Some(Box::new(serialize_physical_expr(expr.left(), codec)?)),- r: Some(Box::new(serialize_physical_expr(expr.right(), codec)?)),+ l: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.left(), codec)?,+ )),+ r: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.right(), codec)?,+ )),op: format!("{:?}", expr.op()),});@@ -290,14 +326,21 @@ pub fn serialize_physical_expr(expr: expr.expr().map(|exp| {- serialize_physical_expr(exp, codec).map(Box::new)+ proto_converter+ .physical_expr_to_proto(exp, codec)+ .map(Box::new)}).transpose()?,when_then_expr: expr.when_then_expr().iter().map(|(when_expr, then_expr)| {- serialize_when_then_expr(when_expr, then_expr, codec)+ serialize_when_then_expr(+ when_expr,+ then_expr,+ codec,+ proto_converter,+ )}).collect::<Result<Vec<protobuf::PhysicalWhenThen>,@@ -305,7 +348,11 @@ pub fn serialize_physical_expr(>>()?,else_expr: expr.else_expr()- .map(|a| serialize_physical_expr(a, codec).map(Box::new))+ .map(|a| {+ proto_converter+ .physical_expr_to_proto(a, codec)+ .map(Box::new)+ }).transpose()?,},),@@ -316,7 +363,9 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::NotExpr(Box::new(protobuf::PhysicalNot {- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,+ )),},))),})@@ -324,7 +373,9 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::IsNullExpr(Box::new(protobuf::PhysicalIsNull {- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,+ )),}),)),})@@ -332,7 +383,9 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::IsNotNullExpr(Box::new(protobuf::PhysicalIsNotNull {- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,+ )),}),)),})@@ -340,8 +393,10 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::InList(Box::new(protobuf::PhysicalInListNode {- expr: Some(Box::new(serialize_physical_expr(expr.expr(), codec)?)),- list: serialize_physical_exprs(expr.list(), codec)?,+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.expr(), codec)?,+ )),+ list: serialize_physical_exprs(expr.list(), codec, proto_converter)?,negated: expr.negated(),},))),@@ -350,7 +405,9 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::Negative(Box::new(protobuf::PhysicalNegativeNode {- expr: Some(Box::new(serialize_physical_expr(expr.arg(), codec)?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.arg(), codec)?,+ )),},))),})@@ -364,7 +421,9 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::Cast(Box::new(protobuf::PhysicalCastNode {- expr: Some(Box::new(serialize_physical_expr(cast.expr(), codec)?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(cast.expr(), codec)?,+ )),arrow_type: Some(cast.cast_type().try_into()?),},))),@@ -373,7 +432,9 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::TryCast(Box::new(protobuf::PhysicalTryCastNode {- expr: Some(Box::new(serialize_physical_expr(cast.expr(), codec)?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(cast.expr(), codec)?,+ )),arrow_type: Some(cast.cast_type().try_into()?),},))),@@ -385,7 +446,7 @@ pub fn serialize_physical_expr(expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarUdf(protobuf::PhysicalScalarUdfNode {name: expr.name().to_string(),- args: serialize_physical_exprs(expr.args(), codec)?,+ args: serialize_physical_exprs(expr.args(), codec, proto_converter)?,fun_definition: (!buf.is_empty()).then_some(buf),return_type: Some(expr.return_type().try_into()?),nullable: expr.nullable(),@@ -402,11 +463,12 @@ pub fn serialize_physical_expr(protobuf::PhysicalLikeExprNode {negated: expr.negated(),case_insensitive: expr.case_insensitive(),- expr: Some(Box::new(serialize_physical_expr(expr.expr(), codec)?)),- pattern: Some(Box::new(serialize_physical_expr(- expr.pattern(),- codec,- )?)),+ expr: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.expr(), codec)?,+ )),+ pattern: Some(Box::new(+ proto_converter.physical_expr_to_proto(expr.pattern(), codec)?,+ )),},))),})@@ -415,7 +477,11 @@ pub fn serialize_physical_expr(Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(protobuf::PhysicalHashExprNode {- on_columns: serialize_physical_exprs(expr.on_columns(), codec)?,+ on_columns: serialize_physical_exprs(+ expr.on_columns(),+ codec,+ proto_converter,+ )?,seed0: s0,seed1: s1,seed2: s2,@@ -431,7 +497,7 @@ pub fn serialize_physical_expr(let inputs: Vec<protobuf::PhysicalExprNode> = value.children().into_iter()- .map(|e| serialize_physical_expr(e, codec))+ .map(|e| proto_converter.physical_expr_to_proto(e, codec)).collect::<Result<_>>()?;Ok(protobuf::PhysicalExprNode {expr_type: Some(protobuf::physical_expr_node::ExprType::Extension(@@ -449,6 +515,7 @@ pub fn serialize_physical_expr(pub fn serialize_partitioning(partitioning: &Partitioning,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::Partitioning> {let serialized_partitioning = match partitioning {Partitioning::RoundRobinBatch(partition_count) => protobuf::Partitioning {@@ -457,7 +524,8 @@ pub fn serialize_partitioning()),},Partitioning::Hash(exprs, partition_count) => {- let serialized_exprs = serialize_physical_exprs(exprs, codec)?;+ let serialized_exprs =+ serialize_physical_exprs(exprs, codec, proto_converter)?;protobuf::Partitioning {partition_method: Some(protobuf::partitioning::PartitionMethod::Hash(protobuf::PhysicalHashRepartition {@@ -480,10 +548,11 @@ fn serialize_when_then_expr(when_expr: &Arc<dyn PhysicalExpr>,then_expr: &Arc<dyn PhysicalExpr>,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::PhysicalWhenThen> {Ok(protobuf::PhysicalWhenThen {- when_expr: Some(serialize_physical_expr(when_expr, codec)?),- then_expr: Some(serialize_physical_expr(then_expr, codec)?),+ when_expr: Some(proto_converter.physical_expr_to_proto(when_expr, codec)?),+ then_expr: Some(proto_converter.physical_expr_to_proto(then_expr, codec)?),})}@@ -539,6 +608,7 @@ impl TryFrom<&[PartitionedFile]> for protobuf::FileGroup {pub fn serialize_file_scan_config(conf: &FileScanConfig,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::FileScanExecConf> {let file_groups = conf.file_groups@@ -548,7 +618,8 @@ pub fn serialize_file_scan_config(let mut output_orderings = vec![];for order in &conf.output_ordering {- let ordering = serialize_physical_sort_exprs(order.to_vec(), codec)?;+ let ordering =+ serialize_physical_sort_exprs(order.to_vec(), codec, proto_converter)?;output_orderings.push(ordering)}@@ -563,8 +634,7 @@ pub fn serialize_file_scan_config(fields.extend(conf.table_partition_cols().iter().cloned());let schema = Arc::new(- arrow::datatypes::Schema::new(fields.clone())- .with_metadata(conf.file_schema().metadata.clone()),+ Schema::new(fields.clone()).with_metadata(conf.file_schema().metadata.clone()),);let projection_exprs = conf@@ -579,7 +649,10 @@ pub fn serialize_file_scan_config(.map(|expr| {Ok(protobuf::ProjectionExpr {alias: expr.alias.to_string(),- expr: Some(serialize_physical_expr(&expr.expr, codec)?),+ expr: Some(+ proto_converter+ .physical_expr_to_proto(&expr.expr, codec)?,+ ),})}).collect::<Result<Vec<_>>>()?,@@ -614,11 +687,12 @@ pub fn serialize_file_scan_config(pub fn serialize_maybe_filter(expr: Option<Arc<dyn PhysicalExpr>>,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::MaybeFilter> {match expr {None => Ok(protobuf::MaybeFilter { expr: None }),Some(expr) => Ok(protobuf::MaybeFilter {- expr: Some(serialize_physical_expr(&expr, codec)?),+ expr: Some(proto_converter.physical_expr_to_proto(&expr, codec)?),}),}}diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex b54b7030fc..f262020ab8 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -18,29 +18,12 @@use std::any::Any;use std::collections::HashMap;use std::fmt::{Display, Formatter};--use std::sync::Arc;+use std::sync::{Arc, RwLock};use std::vec;-use crate::cases::{- CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,- MyRegexUdfNode,-};-use arrow::array::RecordBatch;use arrow::csv::WriterBuilder;use arrow::datatypes::{Fields, TimeUnit};-use datafusion::physical_expr::aggregate::AggregateExprBuilder;-#[expect(deprecated)]-use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;-use datafusion::physical_plan::metrics::MetricType;-use datafusion_datasource::TableSchema;-use datafusion_expr::dml::InsertOp;-use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;-use datafusion_functions_aggregate::array_agg::array_agg_udaf;-use datafusion_functions_aggregate::min_max::max_udaf;-use prost::Message;-use datafusion::arrow::array::ArrayRef;use datafusion::arrow::compute::kernels::sort::SortOptions;use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};@@ -64,6 +47,7 @@ use datafusion::functions_aggregate::sum::sum_udaf;use datafusion::functions_window::nth_value::nth_value_udwf;use datafusion::functions_window::row_number::row_number_udwf;use datafusion::logical_expr::{JoinType, Operator, Volatility, create_udf};+use datafusion::physical_expr::aggregate::AggregateExprBuilder;use datafusion::physical_expr::expressions::Literal;use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};use datafusion::physical_expr::{@@ -73,6 +57,8 @@ use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,};use datafusion::physical_plan::analyze::AnalyzeExec;+#[expect(deprecated)]+use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion::physical_plan::empty::EmptyExec;use datafusion::physical_plan::expressions::{@@ -84,6 +70,7 @@ use datafusion::physical_plan::joins::{StreamJoinPartitionMode, SymmetricHashJoinExec,};use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};+use datafusion::physical_plan::metrics::MetricType;use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion::physical_plan::repartition::RepartitionExec;@@ -105,22 +92,41 @@ use datafusion_common::file_options::json_writer::JsonWriterOptions;use datafusion_common::parsers::CompressionTypeVariant;use datafusion_common::stats::Precision;use datafusion_common::{- DataFusionError, NullEquality, Result, UnnestOptions, internal_datafusion_err,- internal_err, not_impl_err,+ DataFusionError, NullEquality, Result, UnnestOptions, exec_datafusion_err,+ internal_datafusion_err, internal_err, not_impl_err,};+use datafusion_datasource::TableSchema;use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};+use datafusion_expr::dml::InsertOp;use datafusion_expr::{Accumulator, AccumulatorFactoryFunction, AggregateUDF, ColumnarValue,ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,WindowFrame, WindowFrameBound, WindowUDF,};+use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;+use datafusion_functions_aggregate::array_agg::array_agg_udaf;use datafusion_functions_aggregate::average::avg_udaf;+use datafusion_functions_aggregate::min_max::max_udaf;use datafusion_functions_aggregate::nth_value::nth_value_udaf;use datafusion_functions_aggregate::string_agg::string_agg_udaf;+use datafusion_proto::bytes::{+ physical_plan_from_bytes_with_proto_converter,+ physical_plan_to_bytes_with_proto_converter,+};+use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;+use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;use datafusion_proto::physical_plan::{- AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec,+ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,+ PhysicalExtensionCodec, PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf;+use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};+use prost::Message;++use crate::cases::{+ CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,+ MyRegexUdfNode,};-use datafusion_proto::protobuf::{self, PhysicalPlanNode};/// Perform a serde roundtrip and assert that the string representation of the before and after plans/// are identical. Note that this often isn't sufficient to guarantee that no information is@@ -128,7 +134,8 @@ use datafusion_proto::protobuf::{self, PhysicalPlanNode};fn roundtrip_test(exec_plan: Arc<dyn ExecutionPlan>) -> Result<()> {let ctx = SessionContext::new();let codec = DefaultPhysicalExtensionCodec {};- roundtrip_test_and_return(exec_plan, &ctx, &codec)?;+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;Ok(())}@@ -142,13 +149,19 @@ fn roundtrip_test_and_return(exec_plan: Arc<dyn ExecutionPlan>,ctx: &SessionContext,codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let proto: protobuf::PhysicalPlanNode =- protobuf::PhysicalPlanNode::try_from_physical_plan(exec_plan.clone(), codec)- .expect("to proto");- let result_exec_plan: Arc<dyn ExecutionPlan> = proto- .try_into_physical_plan(&ctx.task_ctx(), codec)- .expect("from proto");+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&exec_plan),+ codec,+ proto_converter,+ )?;+ let result_exec_plan = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ codec,+ proto_converter,+ )?;pretty_assertions::assert_eq!(format!("{exec_plan:?}"),@@ -168,7 +181,8 @@ fn roundtrip_test_with_context(ctx: &SessionContext,) -> Result<()> {let codec = DefaultPhysicalExtensionCodec {};- roundtrip_test_and_return(exec_plan, ctx, &codec)?;+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(exec_plan, ctx, &codec, &proto_converter)?;Ok(())}@@ -176,9 +190,10 @@ fn roundtrip_test_with_context(/// query results are identical.async fn roundtrip_test_sql_with_context(sql: &str, ctx: &SessionContext) -> Result<()> {let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};let initial_plan = ctx.sql(sql).await?.create_physical_plan().await?;- roundtrip_test_and_return(initial_plan, ctx, &codec)?;+ roundtrip_test_and_return(initial_plan, ctx, &codec, &proto_converter)?;Ok(())}@@ -988,7 +1003,7 @@ fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {}impl Display for CustomPredicateExpr {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {write!(f, "CustomPredicateExpr")}}@@ -1081,7 +1096,12 @@ fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {let exec_plan = DataSourceExec::from_data_source(scan_config);let ctx = SessionContext::new();- roundtrip_test_and_return(exec_plan, &ctx, &CustomPhysicalExtensionCodec {})?;+ roundtrip_test_and_return(+ exec_plan,+ &ctx,+ &CustomPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;Ok(())}@@ -1287,7 +1307,8 @@ fn roundtrip_scalar_udf_extension_codec() -> Result<()> {)?);let ctx = SessionContext::new();- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec)?;+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;Ok(())}@@ -1334,7 +1355,8 @@ fn roundtrip_udwf_extension_codec() -> Result<()> {)?);let ctx = SessionContext::new();- roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec)?;+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(window, &ctx, &UDFExtensionCodec, &proto_converter)?;Ok(())}@@ -1405,7 +1427,8 @@ fn roundtrip_aggregate_udf_extension_codec() -> Result<()> {)?);let ctx = SessionContext::new();- roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec)?;+ let proto_converter = DefaultPhysicalProtoConverter {};+ roundtrip_test_and_return(aggregate, &ctx, &UDFExtensionCodec, &proto_converter)?;Ok(())}@@ -1529,12 +1552,14 @@ fn roundtrip_csv_sink() -> Result<()> {let ctx = SessionContext::new();let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+let roundtrip_plan = roundtrip_test_and_return(Arc::new(DataSinkExec::new(input, data_sink, Some(sort_order))),&ctx,&codec,- )- .unwrap();+ &proto_converter,+ )?;let roundtrip_plan = roundtrip_plan.as_any()@@ -1976,6 +2001,7 @@ async fn test_serialize_deserialize_tpch_queries() -> Result<()> {// serialize the physical planlet codec = DefaultPhysicalExtensionCodec {};+let proto =PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;@@ -2097,6 +2123,7 @@ async fn test_tpch_part_in_list_query_with_real_parquet_data() -> Result<()> {// Serialize the physical plan - bug may happen here already but not necessarily manifestslet codec = DefaultPhysicalExtensionCodec {};+let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?;// This will fail with the bug, but should succeed when fixed@@ -2338,9 +2365,8 @@ async fn roundtrip_async_func_exec() -> Result<()> {/// it's a performance optimization filter, not a correctness requirement.#[test]fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {- use datafusion::physical_plan::joins::HashTableLookupExpr;- use datafusion::physical_plan::joins::Map;use datafusion::physical_plan::joins::join_hash_map::JoinHashMapU32;+ use datafusion::physical_plan::joins::{HashTableLookupExpr, Map};// Create a simple schema and input planlet schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));@@ -2362,8 +2388,9 @@ fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {// Serializelet ctx = SessionContext::new();let codec = DefaultPhysicalExtensionCodec {};- let proto: protobuf::PhysicalPlanNode =- protobuf::PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec)++ let proto: PhysicalPlanNode =+ PhysicalPlanNode::try_from_physical_plan(filter.clone(), &codec).expect("serialization should succeed");// Deserialize@@ -2413,3 +2440,124 @@ fn roundtrip_hash_expr() -> Result<()> {);roundtrip_test(filter)}++#[test]+fn custom_proto_converter_intercepts() -> Result<()> {+ #[derive(Default)]+ struct CustomConverterInterceptor {+ num_proto_plans: RwLock<usize>,+ num_physical_plans: RwLock<usize>,+ num_proto_exprs: RwLock<usize>,+ num_physical_exprs: RwLock<usize>,+ }++ impl PhysicalProtoConverterExtension for CustomConverterInterceptor {+ fn proto_to_execution_plan(+ &self,+ ctx: &TaskContext,+ codec: &dyn PhysicalExtensionCodec,+ proto: &protobuf::PhysicalPlanNode,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ {+ let mut counter = self+ .num_proto_plans+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ proto.try_into_physical_plan_with_converter(ctx, codec, self)+ }++ fn execution_plan_to_proto(+ &self,+ plan: &Arc<dyn ExecutionPlan>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<protobuf::PhysicalPlanNode>+ where+ Self: Sized,+ {+ {+ let mut counter = self+ .num_physical_plans+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ PhysicalPlanNode::try_from_physical_plan_with_converter(+ Arc::clone(plan),+ codec,+ self,+ )+ }++ fn proto_to_physical_expr(+ &self,+ proto: &PhysicalExprNode,+ ctx: &TaskContext,+ input_schema: &Schema,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<Arc<dyn PhysicalExpr>>+ where+ Self: Sized,+ {+ {+ let mut counter = self+ .num_proto_exprs+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ parse_physical_expr_with_converter(proto, ctx, input_schema, codec, self)+ }++ fn physical_expr_to_proto(+ &self,+ expr: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+ ) -> Result<PhysicalExprNode> {+ {+ let mut counter = self+ .num_physical_exprs+ .write()+ .map_err(|err| exec_datafusion_err!("{err}"))?;+ *counter += 1;+ }+ serialize_physical_expr_with_converter(expr, codec, self)+ }+ }++ let field_a = Field::new("a", DataType::Boolean, false);+ let field_b = Field::new("b", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a, field_b]));+ let sort_exprs = [+ PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions {+ descending: true,+ nulls_first: false,+ },+ },+ PhysicalSortExpr {+ expr: col("b", &schema)?,+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ },+ ]+ .into();++ let exec_plan = Arc::new(SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema))));++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = CustomConverterInterceptor::default();+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;++ assert_eq!(*proto_converter.num_proto_exprs.read().unwrap(), 2);+ assert_eq!(*proto_converter.num_physical_exprs.read().unwrap(), 2);+ assert_eq!(*proto_converter.num_proto_plans.read().unwrap(), 2);+ assert_eq!(*proto_converter.num_physical_plans.read().unwrap(), 2);++ Ok(())+}diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex 6043c81d14..447a773eee 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -154,6 +154,66 @@ The builder pattern is more efficient as it computes properties once during `buiNote: `with_default_selectivity()` is not deprecated as it simply updates a field value and does not require the overhead of the builder pattern.+### Protobuf conversion trait added++A new trait, `PhysicalProtoConverterExtension`, has been added to the `datafusion-proto`+crate. This is used for controlling the process of conversion of physical plans and+expressions to and from their protobuf equivalents. The methods for conversion now+require an additional parameter.++The primary APIs for interacting with this crate have not been modified, so most users+should not need to make any changes. If you do require this trait, you can use the+`DefaultPhysicalProtoConverter` implementation.++For example, to convert a sort expression protobuf node you can make the following+updates:++**Before:**++```rust,ignore+let sort_expr = parse_physical_sort_expr(+ sort_proto,+ ctx,+ input_schema,+ codec,+);+```++**After:**++```rust,ignore+let converter = DefaultPhysicalProtoConverter {};+let sort_expr = parse_physical_sort_expr(+ sort_proto,+ ctx,+ input_schema,+ codec,+ &converter+);+```++Similarly to convert from a physical sort expression into a protobuf node:++**Before:**++```rust,ignore+let sort_proto = serialize_physical_sort_expr(+ sort_expr,+ codec,+);+```++**After:**++```rust,ignore+let converter = DefaultPhysicalProtoConverter {};+let sort_proto = serialize_physical_sort_expr(+ sort_expr,+ codec,+ &converter,+);+```+### `generate_series` and `range` table functions changedThe `generate_series` and `range` table functions now return an empty set when the interval is invalid, instead of an error.