Apache DataFusion SQL Query Engine
feat: allow extension `PhysicalExpr`s to decode via a per-type registry (#24628)
Adrian Garcia Badaracco9c626cd7d466
diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rsindex 3c35f88eea..a3e8e08b10 100644--- a/datafusion/physical-expr-common/src/physical_expr.rs+++ b/datafusion/physical-expr-common/src/physical_expr.rs@@ -630,6 +630,7 @@ pub mod proto_encode {/// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode#[cfg(feature = "proto")]pub mod proto_decode {+ use std::any::Any;use std::sync::Arc;use arrow::datatypes::Schema;@@ -704,6 +705,17 @@ pub mod proto_decode {self.schema}+ /// The session this decode runs under, if the decoder supplied one+ /// and it is a `T`.+ ///+ /// `datafusion-proto` supplies its `TaskContext`, so an expression that+ /// resolves a UDF by name or reads configuration calls+ /// `ctx.session::<TaskContext>()`. The session is type-erased because+ /// `TaskContext` is defined in a crate that depends on this one.+ pub fn session<T: 'static>(&self) -> Option<&T> {+ self.decoder.session()?.downcast_ref()+ }+/// Decode an expression node, recursing into child sub-expressions.////// Routes built-in `ExprType` variants through `datafusion-proto`'s@@ -793,6 +805,12 @@ pub mod proto_decode {node: &PhysicalExprNode,schema: &Schema,) -> Result<Arc<dyn PhysicalExpr>>;++ /// The session the decode runs under, read through+ /// [`PhysicalExprDecodeCtx::session`]. Defaults to `None`.+ fn session(&self) -> Option<&dyn Any> {+ None+ }}}diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rsindex 80e9f88b51..6015d98843 100644--- a/datafusion/physical-expr/src/lib.rs+++ b/datafusion/physical-expr/src/lib.rs@@ -40,6 +40,8 @@ mod partitioning;mod physical_expr;pub mod planner;pub mod projection;+#[cfg(feature = "proto")]+pub mod proto;/// Shared test helpers for the `try_to_proto` / `try_from_proto` unit tests#[cfg(all(test, feature = "proto"))]pub(crate) mod proto_test_util;diff --git a/datafusion/physical-expr/src/proto.rs b/datafusion/physical-expr/src/proto.rsnew file mode 100644index 0000000000..12ed487e93--- /dev/null+++ b/datafusion/physical-expr/src/proto.rs@@ -0,0 +1,238 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! The `PhysicalExpr` face of the session-scoped [`ProtoDecoderRegistry`],+//! which routes extension expressions to their decoders by name:+//! [`register_physical_expr`] and [`physical_expr_names`], all keyed on `dyn PhysicalExpr`. The store itself+//! is shared with every other extension kind, so a session carries one+//! registry.++use std::sync::Arc;++use datafusion_common::Result;+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+pub use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+pub use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+pub use datafusion_proto_models::ProtoDecoderRegistry;+use datafusion_proto_models::protobuf::PhysicalExprNode;+use datafusion_proto_models::protobuf::physical_expr_node::ExprType;++/// The wire name of an extension [`PhysicalExpr`], and the constructor that+/// rebuilds it.+///+/// Only extension expressions implement this. A built-in expression has an+/// `ExprType` variant of its own, keeps the inherent `try_from_proto`+/// dispatched from that variant, and has no wire name to be registered under —+/// so [`register_physical_expr`] will not accept one.+///+/// One impl block says what the expression is called and how it decodes; its+/// `try_to_proto` on [`PhysicalExpr`] writes the matching node with+/// [`extension_expr_node`]:+///+/// ```ignore+/// impl PhysicalExpr for MyExpr {+/// // ...+/// fn try_to_proto(+/// &self,+/// ctx: &PhysicalExprEncodeCtx<'_>,+/// ) -> Result<Option<PhysicalExprNode>> {+/// // Stamps `Self::NAME`, so the encoded name and the registry key+/// // cannot drift apart.+/// Ok(Some(extension_expr_node::<Self>(+/// ctx,+/// self.state.to_bytes()?,+/// self.children(),+/// )?))+/// }+/// }+///+/// impl ExtensionExprFromProto for MyExpr {+/// // Namespace the name with the owning crate, so a collision with+/// // another crate is an error at registration and not a wrong decode.+/// const NAME: &'static str = "my_crate.MyExpr";+///+/// fn try_from_proto(+/// node: &PhysicalExprNode,+/// ctx: &PhysicalExprDecodeCtx<'_>,+/// ) -> Result<Arc<dyn PhysicalExpr>> {+/// let extension = expect_expr_variant!(+/// node,+/// physical_expr_node::ExprType::Extension,+/// "Extension",+/// );+/// let state = MyState::from_bytes(&extension.expr)?;+/// let children = ctx.decode_children_expressions(&extension.inputs)?;+/// Ok(Arc::new(MyExpr::new(state, children)))+/// }+/// }+///+/// let mut registry = ProtoDecoderRegistry::new();+/// register_physical_expr::<MyExpr>(&mut registry)?;+/// let config = SessionConfig::new().with_extension(Arc::new(registry));+/// ```+pub trait ExtensionExprFromProto: PhysicalExpr + Sized {+ /// The name this expression type is written and registered under.+ ///+ /// Namespace it with the owning crate — `"my_crate.MyExpr"`, not+ /// `"MyExpr"` — so that two independent crates registering into the same+ /// session collide at registration time instead of silently decoding each+ /// other's nodes. Never write it by hand on the wire:+ /// [`extension_expr_node`] stamps it for you.+ const NAME: &'static str;++ /// Rebuild the expression from the `PhysicalExprNode` its+ /// [`PhysicalExpr::try_to_proto`] wrote.+ ///+ /// Takes the whole node — the exact inverse of what `try_to_proto`+ /// returns — so the constructor can also see outer-node fields such as+ /// `expr_id`. `ctx.session::<TaskContext>()` gives the session, for+ /// resolving UDFs by name or reading configuration.+ fn try_from_proto(+ node: &PhysicalExprNode,+ ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>>;+}++/// Build the `PhysicalExprNode` for an extension expression `T`: an+/// `Extension` variant carrying `payload`, the encoded `children`, and `T`'s+/// [`NAME`](ExtensionExprFromProto::NAME).+///+/// The one supported way for an extension expression to write itself. It is a+/// free function rather than a method on [`PhysicalExprEncodeCtx`] because the+/// context lives in `datafusion-physical-expr-common`, below the crate that+/// can name [`ExtensionExprFromProto`].+pub fn extension_expr_node<T: ExtensionExprFromProto>(+ ctx: &PhysicalExprEncodeCtx<'_>,+ payload: Vec<u8>,+ children: Vec<&Arc<dyn PhysicalExpr>>,+) -> Result<PhysicalExprNode> {+ Ok(PhysicalExprNode {+ expr_type: Some(ExprType::Extension(+ datafusion_proto_models::protobuf::PhysicalExtensionExprNode {+ expr: payload,+ inputs: ctx.encode_children_expressions(children)?,+ expr_name: Some(T::NAME.to_string()),+ },+ )),+ ..Default::default()+ })+}++/// How this facade stores a decoder in the shared registry: a function pointer+/// to the monomorphized [`ExtensionExprFromProto::try_from_proto`].+///+/// Deliberately private, and the same type on both the+/// [`register_physical_expr`] and the [`decode_physical_expr`] side.+/// [`ExtensionExprFromProto`] is the public contract and+/// [`decode_physical_expr`] (hidden, internal to DataFusion) is the only way+/// to invoke one, so the storage+/// can change later without a breaking change.+type PhysicalExprDecoder =+ fn(&PhysicalExprNode, &PhysicalExprDecodeCtx<'_>) -> Result<Arc<dyn PhysicalExpr>>;++/// Register `T` in `registry` under its [`ExtensionExprFromProto::NAME`].+///+/// Registering the same type twice is a no-op. Registering a *different* type+/// under a name already taken is an error, so collisions surface here rather+/// than as a wrong decode later. The name is scoped to `dyn PhysicalExpr`, so a+/// plan or a data source may use the same name in the same registry.+///+/// # Who builds the registry+///+/// The application that owns the session builds it. A session carries at most+/// one registry, and `SessionConfig::with_extension` replaces what is there, so+/// a library must never attach a registry of its own: it would silently discard+/// another library's. A library exposes a function that fills a registry it is+/// handed, and the application composes them. One registry holds every kind, so+/// a library registers its plans and its expressions into the same object.+///+/// # A built-in expression cannot be registered+///+/// The bound is [`ExtensionExprFromProto`], which only extension expressions+/// implement. A built-in keeps an inherent `try_from_proto`, dispatched from+/// its own `ExprType` variant:+///+/// ```+/// use datafusion_physical_expr::expressions::Column;+///+/// let _ = Column::try_from_proto;+/// ```+///+/// but it is not an extension, and registering it does not compile:+///+/// ```compile_fail+/// use datafusion_physical_expr::expressions::Column;+/// use datafusion_physical_expr::proto::register_physical_expr;+/// use datafusion_proto_models::ProtoDecoderRegistry;+///+/// let mut registry = ProtoDecoderRegistry::new();+/// register_physical_expr::<Column>(&mut registry).unwrap();+/// ```+pub fn register_physical_expr<T: ExtensionExprFromProto>(+ registry: &mut ProtoDecoderRegistry,+) -> Result<()> {+ registry.register_decoder::<dyn PhysicalExpr, T, PhysicalExprDecoder>(+ T::NAME,+ T::try_from_proto,+ )+}++/// Decode `node` with the extension expression decoder registered under the+/// name `node` carries.+///+/// The name is read from the node's `Extension` variant rather than passed in,+/// so a caller cannot pair a node with a name it does not carry.+///+/// `None` means "this node names no extension expression decoder of ours": it+/// is not an extension node, it carries no name, or no registered name matches.+/// The caller then falls back to the `PhysicalExtensionCodec` chain.+/// `Some(Err(..))` means the decoder that *does* own the name failed, which is+/// fatal: falling back there would let a codec decode the payload wrongly, the+/// very thing the name exists to prevent.+///+/// Not intended as public API: `datafusion-proto` drives decoding and is the+/// only intended caller. It is `pub` only because that call crosses a crate+/// boundary, so it is hidden from the docs and may change without notice.+#[doc(hidden)]+pub fn decode_physical_expr(+ registry: &ProtoDecoderRegistry,+ node: &PhysicalExprNode,+ ctx: &PhysicalExprDecodeCtx<'_>,+) -> Option<Result<Arc<dyn PhysicalExpr>>> {+ let name = expr_name(node)?;+ let decoder = registry.decoder::<dyn PhysicalExpr, PhysicalExprDecoder>(name)?;+ Some(decoder(node, ctx))+}++/// Every extension expression name registered in `registry`, in arbitrary+/// order.+///+/// Names registered for another kind are not included.+pub fn physical_expr_names(+ registry: &ProtoDecoderRegistry,+) -> impl Iterator<Item = &str> {+ registry.names::<dyn PhysicalExpr>()+}++/// The registry name `node` was written with, if it is an extension node that+/// carries one.+fn expr_name(node: &PhysicalExprNode) -> Option<&str> {+ match node.expr_type.as_ref()? {+ ExprType::Extension(extension) => extension.expr_name.as_deref(),+ _ => None,+ }+}diff --git a/datafusion/physical-plan/src/proto/mod.rs b/datafusion/physical-plan/src/proto/mod.rsindex b4b15deae9..afef282bfa 100644--- a/datafusion/physical-plan/src/proto/mod.rs+++ b/datafusion/physical-plan/src/proto/mod.rs@@ -79,12 +79,14 @@ use datafusion_execution::TaskContext;use datafusion_expr::physical_planning_context::ScalarSubqueryResults;use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};use datafusion_physical_expr::PhysicalExpr;-use datafusion_physical_expr_common::physical_expr::proto_decode::{- PhysicalExprDecode, PhysicalExprDecodeCtx,-};-use datafusion_physical_expr_common::physical_expr::proto_encode::{- PhysicalExprEncode, PhysicalExprEncodeCtx,+pub use datafusion_physical_expr::proto::{+ ExtensionExprFromProto, decode_physical_expr, extension_expr_node,+ physical_expr_names, register_physical_expr,};+use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecode;+pub use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncode;+pub use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;use datafusion_proto_models::protobuf::physical_plan_node::PhysicalPlanType;use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalExtensionNode, PhysicalPlanNode,@@ -435,6 +437,10 @@ impl PhysicalExprDecode for ExecutionPlanDecodeCtx<'_> {) -> Result<Arc<dyn PhysicalExpr>> {self.decode_expr(node, schema)}++ fn session(&self) -> Option<&dyn std::any::Any> {+ Some(self.task_ctx())+ }}/// Assert that a [`PhysicalPlanNode`] carries the expected `PhysicalPlanType`diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex c1c7e34bd8..6c4ef8f0a4 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1302,6 +1302,11 @@ message PhysicalNegativeNode {message PhysicalExtensionExprNode {bytes expr = 1;repeated PhysicalExprNode inputs = 2;+ // Optional type discriminator naming the extension expression, e.g.+ // "my_crate.MyExpr". When present and registered in the session's+ // ProtoDecoderRegistry, decoding is routed to that registered+ // decoder; otherwise it falls back to PhysicalExtensionCodec::try_decode_expr.+ optional string expr_name = 3;}message PhysicalHashExprNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 2e600bd542..725cceeb75 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -20296,6 +20296,9 @@ impl serde::Serialize for PhysicalExtensionExprNode {if !self.inputs.is_empty() {len += 1;}+ if self.expr_name.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalExtensionExprNode", len)?;if !self.expr.is_empty() {#[allow(clippy::needless_borrow)]@@ -20305,6 +20308,9 @@ impl serde::Serialize for PhysicalExtensionExprNode {if !self.inputs.is_empty() {struct_ser.serialize_field("inputs", &self.inputs)?;}+ if let Some(v) = self.expr_name.as_ref() {+ struct_ser.serialize_field("exprName", v)?;+ }struct_ser.end()}}@@ -20317,12 +20323,15 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {const FIELDS: &[&str] = &["expr","inputs",+ "expr_name",+ "exprName",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Expr,Inputs,+ ExprName,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -20346,6 +20355,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {match value {"expr" => Ok(GeneratedField::Expr),"inputs" => Ok(GeneratedField::Inputs),+ "exprName" | "expr_name" => Ok(GeneratedField::ExprName),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -20367,6 +20377,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {{let mut expr__ = None;let mut inputs__ = None;+ let mut expr_name__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Expr => {@@ -20383,11 +20394,18 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {}inputs__ = Some(map_.next_value()?);}+ GeneratedField::ExprName => {+ if expr_name__.is_some() {+ return Err(serde::de::Error::duplicate_field("exprName"));+ }+ expr_name__ = map_.next_value()?;+ }}}Ok(PhysicalExtensionExprNode {expr: expr__.unwrap_or_default(),inputs: inputs__.unwrap_or_default(),+ expr_name: expr_name__,})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 119d465281..1034d0443d 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -2004,6 +2004,12 @@ pub struct PhysicalExtensionExprNode {pub expr: ::prost::alloc::vec::Vec<u8>,#[prost(message, repeated, tag = "2")]pub inputs: ::prost::alloc::vec::Vec<PhysicalExprNode>,+ /// Optional type discriminator naming the extension expression, e.g.+ /// "my_crate.MyExpr". When present and registered in the session's+ /// ProtoDecoderRegistry, decoding is routed to that registered+ /// decoder; otherwise it falls back to PhysicalExtensionCodec::try_decode_expr.+ #[prost(string, optional, tag = "3")]+ pub expr_name: ::core::option::Option<::prost::alloc::string::String>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct PhysicalHashExprNode {diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 8c3416ec98..e47c54f2b7 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -23,7 +23,7 @@ use arrow::array::RecordBatch;use arrow::compute::SortOptions;use arrow::datatypes::{Field, Schema};use arrow::ipc::reader::StreamReader;-use datafusion_common::{Result, internal_datafusion_err, not_impl_err};+use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -41,6 +41,9 @@ use datafusion_physical_plan::expressions::{};use datafusion_physical_plan::joins::HashExpr;use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx;+use datafusion_physical_plan::proto::{+ ProtoDecoderRegistry, decode_physical_expr, physical_expr_names,+};use datafusion_physical_plan::repartition::RangeExpr;use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};@@ -368,16 +371,42 @@ pub fn parse_physical_expr_with_converter(SqlSimilarToPattern::try_from_proto(proto, &decode_ctx)?}ExprType::Extension(extension) => {+ // Lookup-order policy, the same one extension plans use: an+ // expression that named itself on the wire and is registered on+ // this session decodes itself, no codec involved. Anything else —+ // no name, or a name this session has no decoder for — takes the+ // codec path, which is also every node written before `expr_name`+ // existed. `None` from the registry means no decoder claims the+ // name; a decode *failure* is returned as-is rather than falling+ // through to the codec.+ let registry = ctx+ .task_ctx()+ .session_config()+ .get_extension::<ProtoDecoderRegistry>();+ if let Some(registry) = registry.as_ref()+ && let Some(decoded) = decode_physical_expr(registry, proto, &decode_ctx)+ {+ return decoded;+ }+let inputs: Vec<Arc<dyn PhysicalExpr>> = extension.inputs.iter().map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx)).collect::<Result<_>>()?;- ctx.codec().try_decode_expr(- extension.expr.as_slice(),- &inputs,- &decode_ctx,- )? as _+ ctx.codec()+ .try_decode_expr(extension.expr.as_slice(), &inputs, &decode_ctx)+ .map_err(|e| match extension.expr_name.as_deref() {+ // The writer named the expression but this session has no+ // decoder for it: say so, rather than leaving only the+ // codec's error to explain a missing registration.+ Some(expr_name) => unregistered_extension_expr_context(+ expr_name,+ registry.as_deref(),+ e,+ ),+ None => e,+ })?}ExprType::Lambda(_) => LambdaExpr::try_from_proto(proto, &decode_ctx)?,ExprType::LambdaVariable(_) => {@@ -490,6 +519,36 @@ struct ConverterDecoder<'a, 'b> {proto_converter: &'a dyn PhysicalProtoConverterExtension,}+/// Error for a `PhysicalExtensionExprNode` that names its expression type but+/// finds no decoder registered for that name.+///+/// Lists what *is* registered, because the usual cause is a session configured+/// on the writing side but not on the reading one. The codec's own error is+/// kept as the cause rather than reworded: the codec may have failed for a+/// reason that has nothing to do with the missing registration, and a caller+/// matching on the error kind must still see the kind the codec chose.+fn unregistered_extension_expr_context(+ expr_name: &str,+ registry: Option<&ProtoDecoderRegistry>,+ codec_error: DataFusionError,+) -> DataFusionError {+ let mut registered: Vec<&str> = registry+ .map(|registry| physical_expr_names(registry).collect())+ .unwrap_or_default();+ registered.sort_unstable();+ let registered = if registered.is_empty() {+ "none".to_string()+ } else {+ registered.join(", ")+ };+ codec_error.context(format!(+ "No decoder is registered for the extension PhysicalExpr '{expr_name}'. Register \+ the expression in the ProtoDecoderRegistry attached to the decoding session's \+ SessionConfig, or supply a PhysicalExtensionCodec that handles it. Registered \+ extension expressions: {registered}"+ ))+}+impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodefor ConverterDecoder<'_, '_>{@@ -501,4 +560,8 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDself.proto_converter.proto_to_physical_expr(node, schema, self.ctx)}++ fn session(&self) -> Option<&dyn std::any::Any> {+ Some(self.ctx.task_ctx())+ }}diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 596c856c0c..f72a9d8d06 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -1561,6 +1561,13 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {/// Decode a custom extension expression from `buf`.///+ /// Expressions can instead name themselves on the wire and be decoded by a+ /// per-type decoder registered on the session — see+ /// [`ExtensionExprFromProto`] and [`register_physical_expr`]. That path+ /// resolves by name rather than by codec registration order, so two crates+ /// claiming the same name collide at registration. This method stays the+ /// fallback for everything unnamed or unregistered.+ ////// `inputs` holds the already-decoded children carried in the/// `PhysicalExtensionExprNode.inputs` field. If the codec instead embeds/// nested `PhysicalExprNode`s *inside* `buf`, decode them through@@ -1573,6 +1580,8 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {/// cache-hits on its `expr_id` and re-shares one `Arc<dyn PhysicalExpr>`.////// [`parse_physical_expr`]: crate::physical_plan::from_proto::parse_physical_expr+ /// [`ExtensionExprFromProto`]: datafusion_physical_plan::proto::ExtensionExprFromProto+ /// [`register_physical_expr`]: datafusion_physical_plan::proto::register_physical_exprfn try_decode_expr(&self,_buf: &[u8],diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 5ae57752de..97101db471 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -284,7 +284,15 @@ pub fn serialize_physical_expr_with_converter(proto_converter,};let ctx = datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx::new(&encoder);- if let Some(node) = expr.try_to_proto(&ctx)? {+ if let Some(mut node) = expr.try_to_proto(&ctx)? {+ // Stamp the expression's identity here, at the common boundary, so+ // a hook that builds its node with a helper such as+ // `extension_expr_node` (which cannot see the id) keeps the same+ // dedup behaviour as the downcast path below. A hook that set its+ // own `expr_id` keeps it.+ if node.expr_id.is_none() {+ node.expr_id = expr_id;+ }return Ok(node);}@@ -332,7 +340,15 @@ pub fn serialize_physical_expr_with_converter(Ok(protobuf::PhysicalExprNode {expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Extension(- protobuf::PhysicalExtensionExprNode { expr: buf, inputs },+ protobuf::PhysicalExtensionExprNode {+ expr: buf,+ inputs,+ // The codec *is* the discriminator on this path,+ // so no name is written. Expressions that want+ // registry decoding write their own node from+ // `try_to_proto`.+ expr_name: None,+ },)),})}diff --git a/datafusion/proto/tests/cases/plans/expr_registry.rs b/datafusion/proto/tests/cases/plans/expr_registry.rsnew file mode 100644index 0000000000..87da5b88ae--- /dev/null+++ b/datafusion/proto/tests/cases/plans/expr_registry.rs@@ -0,0 +1,982 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Extension `PhysicalExpr`s decoded by name through a+//! [`ProtoDecoderRegistry`] instead of a [`PhysicalExtensionCodec`].++use std::fmt::{Debug, Display, Formatter};+use std::marker::PhantomData;+use std::sync::Arc;++use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use arrow::record_batch::RecordBatch;+use datafusion::execution::TaskContext;+use datafusion::physical_plan::ExecutionPlan;+use datafusion::physical_plan::empty::EmptyExec;+use datafusion::physical_plan::expressions::{BinaryExpr, col, lit};+use datafusion::physical_plan::filter::FilterExec;+use datafusion::physical_plan::proto::PhysicalExprDecodeCtx;+use datafusion::physical_plan::proto::PhysicalExprEncodeCtx;+use datafusion::physical_plan::proto::{+ ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionExprFromProto,+ ExtensionPlanFromProto, ProtoDecoderRegistry, extension_expr_node,+ physical_expr_names, register_execution_plan, register_physical_expr,+};+use datafusion::physical_plan::{+ DisplayAs, DisplayFormatType, PlanProperties, SendableRecordBatchStream,+};+use datafusion::prelude::{SessionConfig, SessionContext};+use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::{+ DataFusionError, Result, internal_datafusion_err, internal_err, resources_err,+};+use datafusion_expr::registry::FunctionRegistry;+use datafusion_expr::{+ AggregateUDF, ColumnarValue, Operator, ScalarUDF, Volatility, create_udf,+};+use datafusion_physical_expr_common::expect_expr_variant;+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+use datafusion_proto::bytes::{+ physical_plan_from_bytes_with_extension_codec,+ physical_plan_to_bytes_with_extension_codec,+};+use datafusion_proto::physical_plan::{+ DeduplicatingProtoConverter, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,+ PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,+};+use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;+use datafusion_proto::protobuf::{+ PhysicalExprNode, PhysicalPlanNode, physical_expr_node,+};+use prost::Message;++/// The wire name of a [`TaggedExpr`], and whether its decoder fails on+/// purpose. Each kind is a distinct Rust type, which the registry's identity+/// rules need.+trait TagKind: Debug + Clone + Send + Sync + 'static {+ const NAME: &'static str;+ const DECODE_FAILS: bool = false;+}++#[derive(Debug, Clone)]+struct Tag;+impl TagKind for Tag {+ const NAME: &'static str = "datafusion.tests.TagExpr";+}++/// A different type that claims `Tag`'s name.+#[derive(Debug, Clone)]+struct TagClone;+impl TagKind for TagClone {+ const NAME: &'static str = Tag::NAME;+}++#[derive(Debug, Clone)]+struct Other;+impl TagKind for Other {+ const NAME: &'static str = "other_crate.OtherExpr";+}++#[derive(Debug, Clone)]+struct Unnamed;+impl TagKind for Unnamed {+ const NAME: &'static str = "";+}++/// Registered under `Tag`'s name, with a decoder that fails.+#[derive(Debug, Clone)]+struct Failing;+impl TagKind for Failing {+ const NAME: &'static str = Tag::NAME;+ const DECODE_FAILS: bool = true;+}++/// An extension expression with a payload of its own (`tag`) and one child.+#[derive(Debug, Clone)]+struct TaggedExpr<K: TagKind> {+ tag: String,+ child: Arc<dyn PhysicalExpr>,+ /// The session's batch size, read at decode time. Only the registry+ /// decoder sets it, so a test can tell the registry path from the codec+ /// path.+ decoded_batch_size: Option<usize>,+ kind: PhantomData<K>,+}++type TagExpr = TaggedExpr<Tag>;++impl<K: TagKind> TaggedExpr<K> {+ fn new(tag: &str, child: Arc<dyn PhysicalExpr>) -> Self {+ Self {+ tag: tag.to_string(),+ child,+ decoded_batch_size: None,+ kind: PhantomData,+ }+ }+}++impl<K: TagKind> Display for TaggedExpr<K> {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "TaggedExpr({}, {})", self.tag, self.child)+ }+}++impl<K: TagKind> PartialEq for TaggedExpr<K> {+ fn eq(&self, other: &Self) -> bool {+ self.tag == other.tag && self.child.eq(&other.child)+ }+}++impl<K: TagKind> Eq for TaggedExpr<K> {}++impl<K: TagKind> std::hash::Hash for TaggedExpr<K> {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.tag.hash(state);+ self.child.hash(state);+ }+}++impl<K: TagKind> PhysicalExpr for TaggedExpr<K> {+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {+ self.child.data_type(input_schema)+ }++ fn nullable(&self, input_schema: &Schema) -> Result<bool> {+ self.child.nullable(input_schema)+ }++ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {+ self.child.evaluate(batch)+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![&self.child]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(Arc::new(Self {+ child: Arc::clone(&children[0]),+ ..(*self).clone()+ }))+ }++ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ Display::fmt(self, f)+ }++ fn try_to_proto(+ &self,+ ctx: &PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<PhysicalExprNode>> {+ Ok(Some(extension_expr_node::<Self>(+ ctx,+ name_payload(&self.tag),+ self.children(),+ )?))+ }+}++impl<K: TagKind> ExtensionExprFromProto for TaggedExpr<K> {+ const NAME: &'static str = K::NAME;++ fn try_from_proto(+ node: &PhysicalExprNode,+ ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ if K::DECODE_FAILS {+ return internal_err!("registry decoder failed on purpose");+ }+ let extension = expect_expr_variant!(+ node,+ physical_expr_node::ExprType::Extension,+ "Extension"+ );+ let [child] =+ <[_; 1]>::try_from(ctx.decode_children_expressions(&extension.inputs)?)+ .map_err(|_| {+ internal_datafusion_err!("TaggedExpr expects exactly one child")+ })?;+ Ok(Arc::new(Self {+ tag: decode_name_payload(&extension.expr)?,+ child,+ decoded_batch_size: Some(task_ctx(ctx)?.session_config().batch_size()),+ kind: PhantomData,+ }))+ }+}++/// A function the decoder resolves by name from the session.+#[derive(Debug, Clone)]+enum FunctionRef {+ Scalar(Arc<ScalarUDF>),+ Aggregate(Arc<AggregateUDF>),+}++/// An extension expression that holds a UDF, as an expression that wraps a+/// user-defined scalar or aggregate function would. Only the function name is+/// written. The decoder finds the function in the session's registry, so a+/// user-defined function decodes to the instance registered on the reading+/// session.+#[derive(Debug, Clone)]+struct FunctionRefExpr {+ function: FunctionRef,+ child: Arc<dyn PhysicalExpr>,+}++impl FunctionRefExpr {+ fn name(&self) -> &str {+ match &self.function {+ FunctionRef::Scalar(udf) => udf.name(),+ FunctionRef::Aggregate(udaf) => udaf.name(),+ }+ }+}++impl Display for FunctionRefExpr {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "FunctionRefExpr({}, {})", self.name(), self.child)+ }+}++impl PartialEq for FunctionRefExpr {+ fn eq(&self, other: &Self) -> bool {+ self.name() == other.name() && self.child.eq(&other.child)+ }+}++impl Eq for FunctionRefExpr {}++impl std::hash::Hash for FunctionRefExpr {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.name().hash(state);+ self.child.hash(state);+ }+}++impl PhysicalExpr for FunctionRefExpr {+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {+ self.child.data_type(input_schema)+ }++ fn nullable(&self, input_schema: &Schema) -> Result<bool> {+ self.child.nullable(input_schema)+ }++ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {+ self.child.evaluate(batch)+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![&self.child]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(Arc::new(Self {+ function: self.function.clone(),+ child: Arc::clone(&children[0]),+ }))+ }++ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ Display::fmt(self, f)+ }++ fn try_to_proto(+ &self,+ ctx: &PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<PhysicalExprNode>> {+ let payload = match &self.function {+ FunctionRef::Scalar(udf) => format!("scalar:{}", udf.name()),+ FunctionRef::Aggregate(udaf) => format!("aggregate:{}", udaf.name()),+ };+ Ok(Some(extension_expr_node::<Self>(+ ctx,+ name_payload(&payload),+ self.children(),+ )?))+ }+}++impl ExtensionExprFromProto for FunctionRefExpr {+ const NAME: &'static str = "datafusion.tests.FunctionRefExpr";++ fn try_from_proto(+ node: &PhysicalExprNode,+ ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ let extension = expect_expr_variant!(+ node,+ physical_expr_node::ExprType::Extension,+ "Extension"+ );+ let payload = decode_name_payload(&extension.expr)?;+ let functions = task_ctx(ctx)?;+ let function = match payload.split_once(':') {+ Some(("scalar", name)) => FunctionRef::Scalar(functions.udf(name)?),+ Some(("aggregate", name)) => FunctionRef::Aggregate(functions.udaf(name)?),+ _ => return internal_err!("bad FunctionRefExpr payload: {payload}"),+ };+ let [child] =+ <[_; 1]>::try_from(ctx.decode_children_expressions(&extension.inputs)?)+ .map_err(|_| {+ internal_datafusion_err!("FunctionRefExpr expects exactly one child")+ })?;+ Ok(Arc::new(Self { function, child }))+ }+}++fn task_ctx<'a>(ctx: &'a PhysicalExprDecodeCtx<'_>) -> Result<&'a TaskContext> {+ ctx.session::<TaskContext>()+ .ok_or_else(|| internal_datafusion_err!("decode has no TaskContext"))+}++/// Wire layout for the test expressions' own state. Children are written in+/// the extension node's `inputs`, not here.+#[derive(Clone, PartialEq, prost::Message)]+struct NamePayload {+ #[prost(string, tag = "1")]+ name: String,+}++fn name_payload(name: &str) -> Vec<u8> {+ NamePayload {+ name: name.to_string(),+ }+ .encode_to_vec()+}++fn decode_name_payload(buf: &[u8]) -> Result<String> {+ NamePayload::decode(buf)+ .map(|proto| proto.name)+ .map_err(|e| internal_datafusion_err!("decode NamePayload: {e}"))+}++/// A codec for the expression tests. `decode` is what `try_decode_expr` does.+/// `try_encode_expr` always fails: the test expressions write themselves, so+/// the codec must not be asked to encode them.+#[derive(Debug)]+enum TestCodec {+ /// Fails with an internal error. A successful decode through it proves+ /// the codec was not used.+ Refusing,+ /// Fails with an error kind of its own.+ Exhausted,+ /// Decodes a `TagExpr`, as a codec did before the registry existed.+ Tag,+}++impl PhysicalExtensionCodec for TestCodec {+ fn try_decode(+ &self,+ _buf: &[u8],+ _inputs: &[Arc<dyn ExecutionPlan>],+ _ctx: &TaskContext,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ internal_err!("TestCodec::try_decode must not be reached")+ }++ fn try_encode(+ &self,+ _node: Arc<dyn ExecutionPlan>,+ _buf: &mut Vec<u8>,+ _proto_converter: &dyn PhysicalProtoConverterExtension,+ ) -> Result<()> {+ internal_err!("TestCodec::try_encode must not be reached")+ }++ fn try_decode_expr(+ &self,+ buf: &[u8],+ inputs: &[Arc<dyn PhysicalExpr>],+ _ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ match self {+ Self::Refusing => internal_err!("RefusingCodec must not be reached"),+ Self::Exhausted => resources_err!("the decode buffer pool is empty"),+ Self::Tag => {+ let [child] = inputs else {+ return internal_err!("TagExpr expects exactly one child");+ };+ Ok(Arc::new(TagExpr::new(+ &decode_name_payload(buf)?,+ Arc::clone(child),+ )))+ }+ }+ }++ fn try_encode_expr(+ &self,+ _node: &Arc<dyn PhysicalExpr>,+ _buf: &mut Vec<u8>,+ _ctx: &PhysicalExprEncodeCtx<'_>,+ ) -> Result<()> {+ internal_err!("TestCodec::try_encode_expr must not be reached")+ }+}++/// A session whose registry is filled by `register`.+fn session_with(+ config: SessionConfig,+ register: impl FnOnce(&mut ProtoDecoderRegistry) -> Result<()>,+) -> Result<SessionContext> {+ let mut registry = ProtoDecoderRegistry::new();+ register(&mut registry)?;+ Ok(SessionContext::new_with_config(+ config.with_extension(Arc::new(registry)),+ ))+}++fn test_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]))+}++/// Serialize `expr` and decode the prost bytes, as the wire would.+fn to_wire(+ expr: &Arc<dyn PhysicalExpr>,+ codec: &dyn PhysicalExtensionCodec,+) -> Result<PhysicalExprNode> {+ let proto = DefaultPhysicalProtoConverter {}.physical_expr_to_proto(expr, codec)?;+ PhysicalExprNode::decode(proto.encode_to_vec().as_slice())+ .map_err(|e| internal_datafusion_err!("decode PhysicalExprNode: {e}"))+}++fn from_wire(+ node: &PhysicalExprNode,+ session: &SessionContext,+ codec: &dyn PhysicalExtensionCodec,+) -> Result<Arc<dyn PhysicalExpr>> {+ let task_ctx = session.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), codec);+ DefaultPhysicalProtoConverter {}.proto_to_physical_expr(+ node,+ &test_schema(),+ &decode_ctx,+ )+}++fn tag_expr() -> Result<Arc<dyn PhysicalExpr>> {+ Ok(Arc::new(TagExpr::new("hello", col("a", &test_schema())?)))+}++fn set_expr_name(node: &mut PhysicalExprNode, name: Option<&str>) {+ match node.expr_type.as_mut().unwrap() {+ physical_expr_node::ExprType::Extension(extension) => {+ extension.expr_name = name.map(str::to_string);+ }+ other => panic!("expected an Extension node, got {other:?}"),+ }+}++/// An extension expression writes its name, and the session's registry+/// decodes it with no codec on either side. The decoder can read the session.+#[test]+fn registry_decodes_named_extension_expr() -> Result<()> {+ let node = to_wire(&tag_expr()?, &TestCodec::Refusing)?;+ match node.expr_type.as_ref().unwrap() {+ physical_expr_node::ExprType::Extension(extension) => {+ assert_eq!(extension.expr_name.as_deref(), Some(TagExpr::NAME))+ }+ other => panic!("expected an Extension node, got {other:?}"),+ }++ let session = session_with(+ SessionConfig::new().with_batch_size(1234),+ register_physical_expr::<TagExpr>,+ )?;+ let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;+ let decoded = decoded.downcast_ref::<TagExpr>().unwrap();+ assert_eq!(decoded.tag, "hello");+ assert_eq!(decoded.child.to_string(), "a@0");+ assert_eq!(decoded.decoded_batch_size, Some(1234));+ Ok(())+}++/// A node with no name, or with a name the session does not know, decodes+/// through the codec as before.+#[test]+fn unnamed_or_unregistered_falls_back_to_codec() -> Result<()> {+ let named = to_wire(&tag_expr()?, &TestCodec::Tag)?;+ let mut unnamed = named.clone();+ set_expr_name(&mut unnamed, None);++ let empty = SessionContext::new();+ let registered =+ session_with(SessionConfig::new(), register_physical_expr::<TagExpr>)?;+ for (node, session) in [(&named, &empty), (&unnamed, ®istered)] {+ let decoded = from_wire(node, session, &TestCodec::Tag)?;+ let decoded = decoded.downcast_ref::<TagExpr>().unwrap();+ assert_eq!(decoded.tag, "hello");+ assert_eq!(decoded.decoded_batch_size, None, "must come from the codec");+ }+ Ok(())+}++/// When the codec also fails, the error keeps the codec's own error kind and+/// message. If the node had a name, the error also says that no decoder is+/// registered for it and lists the names the session knows.+#[test]+fn codec_failure_keeps_its_error_and_names_the_missing_registration() -> Result<()> {+ let named = to_wire(&tag_expr()?, &TestCodec::Refusing)?;+ let session = session_with(+ SessionConfig::new(),+ register_physical_expr::<TaggedExpr<Other>>,+ )?;++ let err = from_wire(&named, &session, &TestCodec::Exhausted).unwrap_err();+ assert!(+ matches!(err.find_root(), DataFusionError::ResourcesExhausted(_)),+ "the codec's error kind must survive, got: {err:?}"+ );+ let text = err.to_string();+ for expected in [+ "the decode buffer pool is empty",+ "No decoder is registered",+ TagExpr::NAME,+ "ProtoDecoderRegistry",+ Other::NAME,+ ] {+ assert!(text.contains(expected), "missing {expected:?} in: {text}");+ }++ let mut unnamed = named;+ set_expr_name(&mut unnamed, None);+ let text = from_wire(&unnamed, &session, &TestCodec::Refusing)+ .unwrap_err()+ .to_string();+ assert!(text.contains("RefusingCodec must not be reached"), "{text}");+ assert!(!text.contains("No decoder is registered"), "{text}");+ Ok(())+}++/// Registering one type twice is a no-op. Two types with the same name, or a+/// type with an empty name, are errors at registration.+#[test]+fn registration_rejects_collisions_and_empty_names() -> Result<()> {+ let mut registry = ProtoDecoderRegistry::new();+ register_physical_expr::<TagExpr>(&mut registry)?;+ register_physical_expr::<TagExpr>(&mut registry)?;++ let err = register_physical_expr::<TaggedExpr<TagClone>>(&mut registry)+ .unwrap_err()+ .to_string();+ assert!(err.contains("already registered"), "{err}");++ let err = register_physical_expr::<TaggedExpr<Unnamed>>(&mut registry)+ .unwrap_err()+ .to_string();+ assert!(err.contains("empty name"), "{err}");++ assert_eq!(+ physical_expr_names(®istry).collect::<Vec<_>>(),+ [Tag::NAME]+ );+ Ok(())+}++/// The children of a registry-decoded expression also decode through the+/// registry.+#[test]+fn nested_extension_exprs_both_decode_through_the_registry() -> Result<()> {+ let inner: Arc<dyn PhysicalExpr> =+ Arc::new(TagExpr::new("inner", col("a", &test_schema())?));+ let outer: Arc<dyn PhysicalExpr> = Arc::new(TagExpr::new("outer", inner));+ let node = to_wire(&outer, &TestCodec::Refusing)?;+ let session = session_with(SessionConfig::new(), register_physical_expr::<TagExpr>)?;++ let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;+ let outer = decoded.downcast_ref::<TagExpr>().unwrap();+ let inner = outer.child.downcast_ref::<TagExpr>().unwrap();+ assert_eq!((outer.tag.as_str(), inner.tag.as_str()), ("outer", "inner"));+ assert!(inner.decoded_batch_size.is_some());+ Ok(())+}++/// If the registered decoder fails, the decode fails. The codec is not tried,+/// even though it could decode this node.+#[test]+fn a_failing_registry_decoder_does_not_fall_through_to_the_codec() -> Result<()> {+ let node = to_wire(&tag_expr()?, &TestCodec::Tag)?;+ let session = session_with(+ SessionConfig::new(),+ register_physical_expr::<TaggedExpr<Failing>>,+ )?;+ let err = from_wire(&node, &session, &TestCodec::Tag).unwrap_err();+ assert!(+ err.to_string()+ .contains("registry decoder failed on purpose"),+ "{err}"+ );+ Ok(())+}++/// A user-defined scalar function and an aggregate function, held by an+/// extension expression, decode to the instances registered on the reading+/// session.+#[test]+fn extension_expr_resolves_scalar_and_aggregate_udfs_from_the_session() -> Result<()> {+ let session = session_with(+ SessionConfig::new(),+ register_physical_expr::<FunctionRefExpr>,+ )?;+ session.register_udf(create_udf(+ "my_add_one",+ vec![DataType::Int64],+ DataType::Int64,+ Volatility::Immutable,+ Arc::new(|args| Ok(args[0].clone())),+ ));+ let state = session.state();+ let scalar = state.udf("my_add_one")?;+ let aggregate = state.udaf("sum")?;++ for function in [+ FunctionRef::Scalar(Arc::clone(&scalar)),+ FunctionRef::Aggregate(Arc::clone(&aggregate)),+ ] {+ let expr: Arc<dyn PhysicalExpr> = Arc::new(FunctionRefExpr {+ function,+ child: col("a", &test_schema())?,+ });+ let node = to_wire(&expr, &TestCodec::Refusing)?;+ let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;+ match &decoded.downcast_ref::<FunctionRefExpr>().unwrap().function {+ FunctionRef::Scalar(udf) => assert!(Arc::ptr_eq(udf, &scalar)),+ FunctionRef::Aggregate(udaf) => assert!(Arc::ptr_eq(udaf, &aggregate)),+ }+ }++ // A reader that does not have the UDF gets the registry's error.+ let expr: Arc<dyn PhysicalExpr> = Arc::new(FunctionRefExpr {+ function: FunctionRef::Scalar(scalar),+ child: col("a", &test_schema())?,+ });+ let node = to_wire(&expr, &TestCodec::Refusing)?;+ let reader = session_with(+ SessionConfig::new(),+ register_physical_expr::<FunctionRefExpr>,+ )?;+ let err = from_wire(&node, &reader, &TestCodec::Refusing).unwrap_err();+ assert!(err.to_string().contains("my_add_one"), "{err}");+ Ok(())+}++/// A whole plan, through `physical_plan_to_bytes` and+/// `physical_plan_from_bytes`.+#[test]+fn registry_expr_survives_a_whole_plan_roundtrip() -> Result<()> {+ let schema = test_schema();+ let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(TagExpr::new("in_plan", col("a", &schema)?)),+ Operator::Eq,+ lit(1i64),+ ));+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let plan: Arc<dyn ExecutionPlan> = Arc::new(FilterExec::try_new(predicate, input)?);+ let bytes = physical_plan_to_bytes_with_extension_codec(plan, &TestCodec::Refusing)?;++ let session = session_with(+ SessionConfig::new().with_batch_size(4096),+ register_physical_expr::<TagExpr>,+ )?;+ let decoded = physical_plan_from_bytes_with_extension_codec(+ &bytes,+ session.task_ctx().as_ref(),+ &TestCodec::Refusing,+ )?;+ let filter = decoded.downcast_ref::<FilterExec>().unwrap();+ let binary = filter.predicate().downcast_ref::<BinaryExpr>().unwrap();+ let tag = binary.left().downcast_ref::<TagExpr>().unwrap();+ assert_eq!(tag.tag, "in_plan");+ assert_eq!(tag.decoded_batch_size, Some(4096));+ Ok(())+}++/// An extension expression with an identity: two references with the same+/// `expression_id` share `state`, as two pushed-down copies of one dynamic+/// filter would. `with_new_children` keeps `state`, so a deduplicating decoder+/// that re-shares the first decode keeps the two copies linked.+#[derive(Debug, Clone)]+struct IdentityExpr {+ id: u64,+ state: Arc<String>,+ child: Arc<dyn PhysicalExpr>,+}++impl Display for IdentityExpr {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "IdentityExpr({}, {})", self.id, self.child)+ }+}++impl PartialEq for IdentityExpr {+ fn eq(&self, other: &Self) -> bool {+ self.id == other.id && self.child.eq(&other.child)+ }+}++impl Eq for IdentityExpr {}++impl std::hash::Hash for IdentityExpr {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.id.hash(state);+ self.child.hash(state);+ }+}++impl PhysicalExpr for IdentityExpr {+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {+ self.child.data_type(input_schema)+ }++ fn nullable(&self, input_schema: &Schema) -> Result<bool> {+ self.child.nullable(input_schema)+ }++ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {+ self.child.evaluate(batch)+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![&self.child]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(Arc::new(Self {+ child: Arc::clone(&children[0]),+ ..(*self).clone()+ }))+ }++ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ Display::fmt(self, f)+ }++ fn expression_id(&self) -> Option<u64> {+ Some(self.id)+ }++ fn try_to_proto(+ &self,+ ctx: &PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<PhysicalExprNode>> {+ // The helper cannot see `expression_id`; the serializer stamps it.+ Ok(Some(extension_expr_node::<Self>(+ ctx,+ name_payload(&self.state),+ self.children(),+ )?))+ }+}++impl ExtensionExprFromProto for IdentityExpr {+ const NAME: &'static str = "datafusion-test.IdentityExpr";++ fn try_from_proto(+ node: &PhysicalExprNode,+ ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ let extension = expect_expr_variant!(+ node,+ physical_expr_node::ExprType::Extension,+ "Extension"+ );+ let Some(id) = node.expr_id else {+ return internal_err!("IdentityExpr needs an expr_id");+ };+ let mut children = ctx.decode_children_expressions(&extension.inputs)?;+ Ok(Arc::new(Self {+ id,+ state: Arc::new(decode_name_payload(&extension.expr)?),+ child: children.remove(0),+ }))+ }+}++/// An extension expression built with `extension_expr_node` keeps its+/// `expression_id` on the wire. Two references to it, with different+/// children, decode through the deduplicating converter to two expressions+/// that share state, each with its own child.+#[test]+fn registry_expr_keeps_its_expression_id_for_deduplication() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Int64, false),+ ]));+ let state = Arc::new("shared".to_string());+ let left: Arc<dyn PhysicalExpr> = Arc::new(IdentityExpr {+ id: 42,+ state: Arc::clone(&state),+ child: col("a", &schema)?,+ });+ let right: Arc<dyn PhysicalExpr> = Arc::new(IdentityExpr {+ id: 42,+ state,+ child: col("b", &schema)?,+ });+ let pair: Arc<dyn PhysicalExpr> =+ Arc::new(BinaryExpr::new(Arc::clone(&left), Operator::And, right));++ let codec = TestCodec::Refusing;+ let converter = DeduplicatingProtoConverter {};+ let node = converter.physical_expr_to_proto(&pair, &codec)?;+ assert_eq!(+ to_wire(&left, &codec)?.expr_id,+ Some(42),+ "the serializer must stamp the expression id"+ );++ let session =+ session_with(SessionConfig::new(), register_physical_expr::<IdentityExpr>)?;+ let task_ctx = session.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ let decoded = converter.proto_to_physical_expr(&node, &schema, &decode_ctx)?;+ let binary = decoded.downcast_ref::<BinaryExpr>().unwrap();+ let left = binary.left().downcast_ref::<IdentityExpr>().unwrap();+ let right = binary.right().downcast_ref::<IdentityExpr>().unwrap();+ assert!(+ Arc::ptr_eq(&left.state, &right.state),+ "state must be shared"+ );+ assert_eq!(left.child.to_string(), "a@0");+ assert_eq!(right.child.to_string(), "b@1");+ Ok(())+}++/// An extension plan whose wire name equals [`TagExpr::NAME`]. The registry+/// scopes names per kind, so the plan and the expression do not collide.+#[derive(Debug)]+struct SameNameExec {+ child: Arc<dyn ExecutionPlan>,+}++impl DisplayAs for SameNameExec {+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {+ write!(f, "SameNameExec")+ }+}++impl ExecutionPlan for SameNameExec {+ fn name(&self) -> &str {+ "SameNameExec"+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ self.child.properties()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![&self.child]+ }++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }++ fn with_new_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(Self {+ child: children.remove(0),+ }))+ }++ fn execute(+ &self,+ _partition: usize,+ _context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ internal_err!("SameNameExec is a serde-only test plan")+ }++ fn try_to_proto(+ &self,+ ctx: &ExecutionPlanEncodeCtx<'_>,+ ) -> Result<Option<PhysicalPlanNode>> {+ Ok(Some(ctx.extension_node::<Self>(vec![], self.children())?))+ }+}++impl ExtensionPlanFromProto for SameNameExec {+ const NAME: &'static str = Tag::NAME;++ fn try_from_proto(+ node: &PhysicalPlanNode,+ ctx: &ExecutionPlanDecodeCtx<'_>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let Some(PhysicalPlanType::Extension(extension)) = &node.physical_plan_type+ else {+ return internal_err!("expected an extension node");+ };+ let mut children = ctx.decode_children(&extension.inputs)?;+ Ok(Arc::new(Self {+ child: children.remove(0),+ }))+ }+}++/// A plan that holds a registered extension plan and a registered extension+/// expression with the same wire name. Both decode through the registry, each+/// to its own type.+#[test]+fn registry_plan_and_expr_with_the_same_name_roundtrip_together() -> Result<()> {+ let schema = test_schema();+ let extension_plan: Arc<dyn ExecutionPlan> = Arc::new(SameNameExec {+ child: Arc::new(EmptyExec::new(Arc::clone(&schema))),+ });+ let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(TagExpr::new("same_name", col("a", &schema)?)),+ Operator::Eq,+ lit(1i64),+ ));+ let plan: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(predicate, extension_plan)?);+ let bytes = physical_plan_to_bytes_with_extension_codec(plan, &TestCodec::Refusing)?;++ let session = session_with(SessionConfig::new().with_batch_size(2048), |registry| {+ register_execution_plan::<SameNameExec>(registry)?;+ register_physical_expr::<TagExpr>(registry)+ })?;+ let decoded = physical_plan_from_bytes_with_extension_codec(+ &bytes,+ session.task_ctx().as_ref(),+ &TestCodec::Refusing,+ )?;+ let filter = decoded.downcast_ref::<FilterExec>().unwrap();+ let binary = filter.predicate().downcast_ref::<BinaryExpr>().unwrap();+ let tag = binary.left().downcast_ref::<TagExpr>().unwrap();+ assert_eq!(tag.tag, "same_name");+ assert_eq!(tag.decoded_batch_size, Some(2048));+ let extension = filter.input().downcast_ref::<SameNameExec>().unwrap();+ assert_eq!(extension.child.name(), "EmptyExec");+ Ok(())+}diff --git a/datafusion/proto/tests/cases/plans/mod.rs b/datafusion/proto/tests/cases/plans/mod.rsindex 0f00f35581..69a6256af9 100644--- a/datafusion/proto/tests/cases/plans/mod.rs+++ b/datafusion/proto/tests/cases/plans/mod.rs@@ -36,6 +36,7 @@ use std::sync::Arc;mod aggregates;mod dispatch;mod dynamic_filters;+mod expr_registry;mod exprs;mod extensions;mod filters;diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 8f5604a5cd..1e8d9e2daa 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -1209,6 +1209,63 @@ A registered plan needs no `PhysicalExtensionCodec` at all, and two crates' planSee [issue #24625](https://github.com/apache/datafusion/issues/24625) for details.+### Extension `PhysicalExpr`s can decode by name instead of through a codec++`PhysicalExtensionExprNode` has a new optional `expr_name` field. An extension expression that sets it is decoded by a decoder registered on the session, not by a `PhysicalExtensionCodec`. This is the same mechanism extension `ExecutionPlan`s use (see above), and it uses the same `ProtoDecoderRegistry`.++**Who is affected:** nobody, unless they opt in. A node with no name, or with a name the session has no decoder for, decodes through `PhysicalExtensionCodec::try_decode_expr` as before. Binary protobuf readers skip the new field. Readers that use the optional JSON serde reject unknown fields, so they must be built after this change to read a node that has `exprName`.++**Opting in:**++```rust,ignore+use datafusion::execution::TaskContext;+use datafusion::physical_plan::proto::{+ ExtensionExprFromProto, PhysicalExprDecodeCtx, PhysicalExprEncodeCtx,+ ProtoDecoderRegistry, extension_expr_node, register_physical_expr,+};++impl PhysicalExpr for MyExpr {+ // ...+ fn try_to_proto(+ &self,+ ctx: &PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<PhysicalExprNode>> {+ // Writes `Self::NAME` on the node.+ Ok(Some(extension_expr_node::<Self>(+ ctx,+ my_payload_bytes(self)?,+ self.children(),+ )?))+ }+}++impl ExtensionExprFromProto for MyExpr {+ // Put the crate name in the name, so that a collision is an error at registration.+ const NAME: &'static str = "my_crate.MyExpr";++ fn try_from_proto(+ node: &PhysicalExprNode,+ ctx: &PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ // The session, for example to find a UDF by name.+ let task_ctx = ctx.session::<TaskContext>();+ // ...+ }+}++let mut registry = ProtoDecoderRegistry::new();+register_physical_expr::<MyExpr>(&mut registry)?;+let config = SessionConfig::new().with_extension(Arc::new(registry));+```++Only extension expressions can be registered. `register_physical_expr::<Column>(..)` does not compile.++If a registered decoder fails, the decode fails. It does not fall back to the codec, because the codec could decode a payload that it did not write. Changing an expression to write its own payload is therefore a wire-format change for that expression: register the type on every reader before writers start to use it.++`PhysicalExprDecodeCtx::session::<T>()` is new. It gives the session the decode runs under. `datafusion-proto` supplies its `TaskContext`. A custom `PhysicalExprDecode` implementation returns `None` by default.++See [issue #24626](https://github.com/apache/datafusion/issues/24626) for details.+### Order-preservation hooks for limit pushdown are deprecatedA file scan that advertises an output ordering now always reads its files in