Apache DataFusion SQL Query Engine
Tweak `adapter serialization` example (#20035)
Adrian Garcia Badaraccof819061833d0
diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rsindex f72c9d2b61..4f76d105a5 100644--- a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs@@ -25,7 +25,8 @@//! default. This example shows how to://! 1. Detect plans with custom adapters during serialization//! 2. Wrap them as Extension nodes with JSON-serialized adapter metadata-//! 3. Unwrap and restore the adapter during deserialization+//! 3. Store the inner DataSourceExec (without adapter) as a child in the extension's inputs field+//! 4. Unwrap and restore the adapter during deserialization//!//! This demonstrates nested serialization (protobuf outer, JSON inner) and the power//! of the `PhysicalExtensionCodec` interception pattern. Both plan and expression@@ -69,7 +70,6 @@ use datafusion_proto::protobuf::{use object_store::memory::InMemory;use object_store::path::Path;use object_store::{ObjectStore, PutPayload};-use prost::Message;use serde::{Deserialize, Serialize};/// Example showing how to preserve custom adapter information during plan serialization.@@ -234,10 +234,10 @@ impl PhysicalExprAdapterFactory for MetadataAdapterFactory {&self,logical_file_schema: SchemaRef,physical_file_schema: SchemaRef,- ) -> Arc<dyn PhysicalExprAdapter> {+ ) -> Result<Arc<dyn PhysicalExprAdapter>> {let inner = DefaultPhysicalExprAdapterFactory- .create(logical_file_schema, physical_file_schema);- Arc::new(MetadataAdapter { inner })+ .create(logical_file_schema, physical_file_schema)?;+ Ok(Arc::new(MetadataAdapter { inner }))}}@@ -252,8 +252,6 @@ struct ExtensionPayload {marker: String,/// JSON-serialized adapter metadataadapter_metadata: AdapterMetadata,- /// Protobuf-serialized inner DataSourceExec (without adapter)- inner_plan_bytes: Vec<u8>,}/// Metadata about the adapter to recreate it during deserialization@@ -274,24 +272,20 @@ impl PhysicalExtensionCodec for AdapterPreservingCodec {fn try_decode(&self,buf: &[u8],- _inputs: &[Arc<dyn ExecutionPlan>],- ctx: &TaskContext,+ inputs: &[Arc<dyn ExecutionPlan>],+ _ctx: &TaskContext,) -> Result<Arc<dyn ExecutionPlan>> {// Try to parse as our extension payloadif let Ok(payload) = serde_json::from_slice::<ExtensionPayload>(buf)&& payload.marker == EXTENSION_MARKER{- // Decode the inner plan- let inner_proto = PhysicalPlanNode::decode(&payload.inner_plan_bytes[..])- .map_err(|e| {- datafusion::error::DataFusionError::Plan(format!(- "Failed to decode inner plan: {e}"- ))- })?;-- // Deserialize the inner plan using default implementation- let inner_plan =- inner_proto.try_into_physical_plan_with_converter(ctx, self, self)?;+ if inputs.len() != 1 {+ return Err(datafusion::error::DataFusionError::Plan(format!(+ "Extension node expected exactly 1 child, got {}",+ inputs.len()+ )));+ }+ let inner_plan = inputs[0].clone();// Recreate the adapter factorylet adapter_factory = create_adapter_factory(&payload.adapter_metadata.tag);@@ -335,32 +329,20 @@ impl PhysicalProtoConverterExtension for AdapterPreservingCodec {// 1. Create adapter metadatalet adapter_metadata = AdapterMetadata { tag };- // 2. Create a copy of the config without the adapter- let config_without_adapter = rebuild_config_without_adapter(config);-- // 3. Create a new DataSourceExec without adapter- let plan_without_adapter: Arc<dyn ExecutionPlan> =- DataSourceExec::from_data_source(config_without_adapter);-- // 4. Serialize the inner plan to protobuf bytes+ // 2. Serialize the inner plan to protobuf+ // Note that this will drop the custom adapter since the default serialization cannot handle itlet inner_proto = PhysicalPlanNode::try_from_physical_plan_with_converter(- plan_without_adapter,+ Arc::clone(plan),extension_codec,self,)?;- let mut inner_bytes = Vec::new();- inner_proto.encode(&mut inner_bytes).map_err(|e| {- datafusion::error::DataFusionError::Plan(format!(- "Failed to encode inner plan: {e}"- ))- })?;-- // 5. Create extension payload+ // 3. Create extension payload to wrap the plan+ // so that the custom adapter gets re-attached during deserialization+ // The choice of JSON is arbitrary; other formats could be used.let payload = ExtensionPayload {marker: EXTENSION_MARKER.to_string(),adapter_metadata,- inner_plan_bytes: inner_bytes,};let payload_bytes = serde_json::to_vec(&payload).map_err(|e| {datafusion::error::DataFusionError::Plan(format!(@@ -368,18 +350,18 @@ impl PhysicalProtoConverterExtension for AdapterPreservingCodec {))})?;- // 6. Return as PhysicalExtensionNode+ // 4. Return as PhysicalExtensionNode with child plan in inputsreturn Ok(PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Extension(PhysicalExtensionNode {node: payload_bytes,- inputs: vec![], // Leaf node+ inputs: vec![inner_proto],},)),});}- // No adapter found - use default serialization+ // No adapter found, not a DataSourceExec, etc. - use default serializationPhysicalPlanNode::try_from_physical_plan_with_converter(Arc::clone(plan),extension_codec,@@ -405,15 +387,15 @@ impl PhysicalProtoConverterExtension for AdapterPreservingCodec {payload.adapter_metadata.tag);- // Decode the inner plan- let inner_proto = PhysicalPlanNode::decode(&payload.inner_plan_bytes[..])- .map_err(|e| {- datafusion::error::DataFusionError::Plan(format!(- "Failed to decode inner plan: {e}"- ))- })?;+ // Get the inner plan proto from inputs field+ if extension.inputs.is_empty() {+ return Err(datafusion::error::DataFusionError::Plan(+ "Extension node missing child plan in inputs".to_string(),+ ));+ }+ let inner_proto = &extension.inputs[0];- // Deserialize the inner plan using default implementation+ // Deserialize the inner planlet inner_plan = inner_proto.try_into_physical_plan_with_converter(ctx,extension_codec,@@ -494,13 +476,6 @@ fn create_adapter_factory(tag: &str) -> Arc<dyn PhysicalExprAdapterFactory> {Arc::new(MetadataAdapterFactory::new(tag))}-/// Rebuild a FileScanConfig without the adapter-fn rebuild_config_without_adapter(config: &FileScanConfig) -> FileScanConfig {- FileScanConfigBuilder::from(config.clone())- .with_expr_adapter(None)- .build()-}-/// Inject an adapter into a plan (assumes plan is a DataSourceExec with FileScanConfig)fn inject_adapter_into_plan(plan: Arc<dyn ExecutionPlan>,