Skip to content

Apache DataFusion SQL Query Engine

feat: allow extension `PhysicalExpr`s to decode via a per-type registry (#24628)

Adrian Garcia Badaracco9c626cd7d466
diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rs
index 3c35f88eea..a3e8e08b10 100644
--- a/datafusion/physical-expr-common/src/physical_expr.rs
+++ b/datafusion/physical-expr-common/src/physical_expr.rs
@@ -630,6 +630,7 @@ pub mod proto_encode {
/// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode
#[cfg(feature = "proto")]
pub mod proto_decode {
+ use std::any::Any;
use std::sync::Arc;
use arrow::datatypes::Schema;
@@ -704,6 +705,17 @@ pub mod proto_decode {
self.schema
}
+ /// The session this decode runs under, if the decoder supplied one
+ /// and it is a `T`.
+ ///
+ /// `datafusion-proto` supplies its `TaskContext`, so an expression that
+ /// resolves a UDF by name or reads configuration calls
+ /// `ctx.session::<TaskContext>()`. The session is type-erased because
+ /// `TaskContext` is defined in a crate that depends on this one.
+ pub fn session<T: 'static>(&self) -> Option<&T> {
+ self.decoder.session()?.downcast_ref()
+ }
+
/// Decode an expression node, recursing into child sub-expressions.
///
/// Routes built-in `ExprType` variants through `datafusion-proto`'s
@@ -793,6 +805,12 @@ pub mod proto_decode {
node: &PhysicalExprNode,
schema: &Schema,
) -> Result<Arc<dyn PhysicalExpr>>;
+
+ /// The session the decode runs under, read through
+ /// [`PhysicalExprDecodeCtx::session`]. Defaults to `None`.
+ fn session(&self) -> Option<&dyn Any> {
+ None
+ }
}
}
diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rs
index 80e9f88b51..6015d98843 100644
--- a/datafusion/physical-expr/src/lib.rs
+++ b/datafusion/physical-expr/src/lib.rs
@@ -40,6 +40,8 @@ mod partitioning;
mod physical_expr;
pub mod planner;
pub mod projection;
+#[cfg(feature = "proto")]
+pub mod proto;
/// Shared test helpers for the `try_to_proto` / `try_from_proto` unit tests
#[cfg(all(test, feature = "proto"))]
pub(crate) mod proto_test_util;
diff --git a/datafusion/physical-expr/src/proto.rs b/datafusion/physical-expr/src/proto.rs
new file mode 100644
index 0000000000..12ed487e93
--- /dev/null
+++ b/datafusion/physical-expr/src/proto.rs
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! The `PhysicalExpr` face of the session-scoped [`ProtoDecoderRegistry`],
+//! which routes extension expressions to their decoders by name:
+//! [`register_physical_expr`] and [`physical_expr_names`], all keyed on `dyn PhysicalExpr`. The store itself
+//! is shared with every other extension kind, so a session carries one
+//! registry.
+
+use std::sync::Arc;
+
+use datafusion_common::Result;
+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
+pub use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+pub use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
+pub use datafusion_proto_models::ProtoDecoderRegistry;
+use datafusion_proto_models::protobuf::PhysicalExprNode;
+use datafusion_proto_models::protobuf::physical_expr_node::ExprType;
+
+/// The wire name of an extension [`PhysicalExpr`], and the constructor that
+/// rebuilds it.
+///
+/// Only extension expressions implement this. A built-in expression has an
+/// `ExprType` variant of its own, keeps the inherent `try_from_proto`
+/// dispatched from that variant, and has no wire name to be registered under —
+/// so [`register_physical_expr`] will not accept one.
+///
+/// One impl block says what the expression is called and how it decodes; its
+/// `try_to_proto` on [`PhysicalExpr`] writes the matching node with
+/// [`extension_expr_node`]:
+///
+/// ```ignore
+/// impl PhysicalExpr for MyExpr {
+/// // ...
+/// fn try_to_proto(
+/// &self,
+/// ctx: &PhysicalExprEncodeCtx<'_>,
+/// ) -> Result<Option<PhysicalExprNode>> {
+/// // Stamps `Self::NAME`, so the encoded name and the registry key
+/// // cannot drift apart.
+/// Ok(Some(extension_expr_node::<Self>(
+/// ctx,
+/// self.state.to_bytes()?,
+/// self.children(),
+/// )?))
+/// }
+/// }
+///
+/// impl ExtensionExprFromProto for MyExpr {
+/// // Namespace the name with the owning crate, so a collision with
+/// // another crate is an error at registration and not a wrong decode.
+/// const NAME: &'static str = "my_crate.MyExpr";
+///
+/// fn try_from_proto(
+/// node: &PhysicalExprNode,
+/// ctx: &PhysicalExprDecodeCtx<'_>,
+/// ) -> Result<Arc<dyn PhysicalExpr>> {
+/// let extension = expect_expr_variant!(
+/// node,
+/// physical_expr_node::ExprType::Extension,
+/// "Extension",
+/// );
+/// let state = MyState::from_bytes(&extension.expr)?;
+/// let children = ctx.decode_children_expressions(&extension.inputs)?;
+/// Ok(Arc::new(MyExpr::new(state, children)))
+/// }
+/// }
+///
+/// let mut registry = ProtoDecoderRegistry::new();
+/// register_physical_expr::<MyExpr>(&mut registry)?;
+/// let config = SessionConfig::new().with_extension(Arc::new(registry));
+/// ```
+pub trait ExtensionExprFromProto: PhysicalExpr + Sized {
+ /// The name this expression type is written and registered under.
+ ///
+ /// Namespace it with the owning crate — `"my_crate.MyExpr"`, not
+ /// `"MyExpr"` — so that two independent crates registering into the same
+ /// session collide at registration time instead of silently decoding each
+ /// other's nodes. Never write it by hand on the wire:
+ /// [`extension_expr_node`] stamps it for you.
+ const NAME: &'static str;
+
+ /// Rebuild the expression from the `PhysicalExprNode` its
+ /// [`PhysicalExpr::try_to_proto`] wrote.
+ ///
+ /// Takes the whole node — the exact inverse of what `try_to_proto`
+ /// returns — so the constructor can also see outer-node fields such as
+ /// `expr_id`. `ctx.session::<TaskContext>()` gives the session, for
+ /// resolving UDFs by name or reading configuration.
+ fn try_from_proto(
+ node: &PhysicalExprNode,
+ ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>>;
+}
+
+/// Build the `PhysicalExprNode` for an extension expression `T`: an
+/// `Extension` variant carrying `payload`, the encoded `children`, and `T`'s
+/// [`NAME`](ExtensionExprFromProto::NAME).
+///
+/// The one supported way for an extension expression to write itself. It is a
+/// free function rather than a method on [`PhysicalExprEncodeCtx`] because the
+/// context lives in `datafusion-physical-expr-common`, below the crate that
+/// can name [`ExtensionExprFromProto`].
+pub fn extension_expr_node<T: ExtensionExprFromProto>(
+ ctx: &PhysicalExprEncodeCtx<'_>,
+ payload: Vec<u8>,
+ children: Vec<&Arc<dyn PhysicalExpr>>,
+) -> Result<PhysicalExprNode> {
+ Ok(PhysicalExprNode {
+ expr_type: Some(ExprType::Extension(
+ datafusion_proto_models::protobuf::PhysicalExtensionExprNode {
+ expr: payload,
+ inputs: ctx.encode_children_expressions(children)?,
+ expr_name: Some(T::NAME.to_string()),
+ },
+ )),
+ ..Default::default()
+ })
+}
+
+/// How this facade stores a decoder in the shared registry: a function pointer
+/// to the monomorphized [`ExtensionExprFromProto::try_from_proto`].
+///
+/// Deliberately private, and the same type on both the
+/// [`register_physical_expr`] and the [`decode_physical_expr`] side.
+/// [`ExtensionExprFromProto`] is the public contract and
+/// [`decode_physical_expr`] (hidden, internal to DataFusion) is the only way
+/// to invoke one, so the storage
+/// can change later without a breaking change.
+type PhysicalExprDecoder =
+ fn(&PhysicalExprNode, &PhysicalExprDecodeCtx<'_>) -> Result<Arc<dyn PhysicalExpr>>;
+
+/// Register `T` in `registry` under its [`ExtensionExprFromProto::NAME`].
+///
+/// Registering the same type twice is a no-op. Registering a *different* type
+/// under a name already taken is an error, so collisions surface here rather
+/// than as a wrong decode later. The name is scoped to `dyn PhysicalExpr`, so a
+/// plan or a data source may use the same name in the same registry.
+///
+/// # Who builds the registry
+///
+/// The application that owns the session builds it. A session carries at most
+/// one registry, and `SessionConfig::with_extension` replaces what is there, so
+/// a library must never attach a registry of its own: it would silently discard
+/// another library's. A library exposes a function that fills a registry it is
+/// handed, and the application composes them. One registry holds every kind, so
+/// a library registers its plans and its expressions into the same object.
+///
+/// # A built-in expression cannot be registered
+///
+/// The bound is [`ExtensionExprFromProto`], which only extension expressions
+/// implement. A built-in keeps an inherent `try_from_proto`, dispatched from
+/// its own `ExprType` variant:
+///
+/// ```
+/// use datafusion_physical_expr::expressions::Column;
+///
+/// let _ = Column::try_from_proto;
+/// ```
+///
+/// but it is not an extension, and registering it does not compile:
+///
+/// ```compile_fail
+/// use datafusion_physical_expr::expressions::Column;
+/// use datafusion_physical_expr::proto::register_physical_expr;
+/// use datafusion_proto_models::ProtoDecoderRegistry;
+///
+/// let mut registry = ProtoDecoderRegistry::new();
+/// register_physical_expr::<Column>(&mut registry).unwrap();
+/// ```
+pub fn register_physical_expr<T: ExtensionExprFromProto>(
+ registry: &mut ProtoDecoderRegistry,
+) -> Result<()> {
+ registry.register_decoder::<dyn PhysicalExpr, T, PhysicalExprDecoder>(
+ T::NAME,
+ T::try_from_proto,
+ )
+}
+
+/// Decode `node` with the extension expression decoder registered under the
+/// name `node` carries.
+///
+/// The name is read from the node's `Extension` variant rather than passed in,
+/// so a caller cannot pair a node with a name it does not carry.
+///
+/// `None` means "this node names no extension expression decoder of ours": it
+/// is not an extension node, it carries no name, or no registered name matches.
+/// The caller then falls back to the `PhysicalExtensionCodec` chain.
+/// `Some(Err(..))` means the decoder that *does* own the name failed, which is
+/// fatal: falling back there would let a codec decode the payload wrongly, the
+/// very thing the name exists to prevent.
+///
+/// Not intended as public API: `datafusion-proto` drives decoding and is the
+/// only intended caller. It is `pub` only because that call crosses a crate
+/// boundary, so it is hidden from the docs and may change without notice.
+#[doc(hidden)]
+pub fn decode_physical_expr(
+ registry: &ProtoDecoderRegistry,
+ node: &PhysicalExprNode,
+ ctx: &PhysicalExprDecodeCtx<'_>,
+) -> Option<Result<Arc<dyn PhysicalExpr>>> {
+ let name = expr_name(node)?;
+ let decoder = registry.decoder::<dyn PhysicalExpr, PhysicalExprDecoder>(name)?;
+ Some(decoder(node, ctx))
+}
+
+/// Every extension expression name registered in `registry`, in arbitrary
+/// order.
+///
+/// Names registered for another kind are not included.
+pub fn physical_expr_names(
+ registry: &ProtoDecoderRegistry,
+) -> impl Iterator<Item = &str> {
+ registry.names::<dyn PhysicalExpr>()
+}
+
+/// The registry name `node` was written with, if it is an extension node that
+/// carries one.
+fn expr_name(node: &PhysicalExprNode) -> Option<&str> {
+ match node.expr_type.as_ref()? {
+ ExprType::Extension(extension) => extension.expr_name.as_deref(),
+ _ => None,
+ }
+}
diff --git a/datafusion/physical-plan/src/proto/mod.rs b/datafusion/physical-plan/src/proto/mod.rs
index b4b15deae9..afef282bfa 100644
--- a/datafusion/physical-plan/src/proto/mod.rs
+++ b/datafusion/physical-plan/src/proto/mod.rs
@@ -79,12 +79,14 @@ use datafusion_execution::TaskContext;
use datafusion_expr::physical_planning_context::ScalarSubqueryResults;
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
use datafusion_physical_expr::PhysicalExpr;
-use datafusion_physical_expr_common::physical_expr::proto_decode::{
- PhysicalExprDecode, PhysicalExprDecodeCtx,
-};
-use datafusion_physical_expr_common::physical_expr::proto_encode::{
- PhysicalExprEncode, PhysicalExprEncodeCtx,
+pub use datafusion_physical_expr::proto::{
+ ExtensionExprFromProto, decode_physical_expr, extension_expr_node,
+ physical_expr_names, register_physical_expr,
};
+use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecode;
+pub use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncode;
+pub use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
use datafusion_proto_models::protobuf::physical_plan_node::PhysicalPlanType;
use datafusion_proto_models::protobuf::{
PhysicalExprNode, PhysicalExtensionNode, PhysicalPlanNode,
@@ -435,6 +437,10 @@ impl PhysicalExprDecode for ExecutionPlanDecodeCtx<'_> {
) -> Result<Arc<dyn PhysicalExpr>> {
self.decode_expr(node, schema)
}
+
+ fn session(&self) -> Option<&dyn std::any::Any> {
+ Some(self.task_ctx())
+ }
}
/// Assert that a [`PhysicalPlanNode`] carries the expected `PhysicalPlanType`
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index c1c7e34bd8..6c4ef8f0a4 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -1302,6 +1302,11 @@ message PhysicalNegativeNode {
message PhysicalExtensionExprNode {
bytes expr = 1;
repeated PhysicalExprNode inputs = 2;
+ // Optional type discriminator naming the extension expression, e.g.
+ // "my_crate.MyExpr". When present and registered in the session's
+ // ProtoDecoderRegistry, decoding is routed to that registered
+ // decoder; otherwise it falls back to PhysicalExtensionCodec::try_decode_expr.
+ optional string expr_name = 3;
}
message PhysicalHashExprNode {
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 2e600bd542..725cceeb75 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -20296,6 +20296,9 @@ impl serde::Serialize for PhysicalExtensionExprNode {
if !self.inputs.is_empty() {
len += 1;
}
+ if self.expr_name.is_some() {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalExtensionExprNode", len)?;
if !self.expr.is_empty() {
#[allow(clippy::needless_borrow)]
@@ -20305,6 +20308,9 @@ impl serde::Serialize for PhysicalExtensionExprNode {
if !self.inputs.is_empty() {
struct_ser.serialize_field("inputs", &self.inputs)?;
}
+ if let Some(v) = self.expr_name.as_ref() {
+ struct_ser.serialize_field("exprName", v)?;
+ }
struct_ser.end()
}
}
@@ -20317,12 +20323,15 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {
const FIELDS: &[&str] = &[
"expr",
"inputs",
+ "expr_name",
+ "exprName",
];
#[allow(clippy::enum_variant_names)]
enum GeneratedField {
Expr,
Inputs,
+ ExprName,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -20346,6 +20355,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {
match value {
"expr" => Ok(GeneratedField::Expr),
"inputs" => Ok(GeneratedField::Inputs),
+ "exprName" | "expr_name" => Ok(GeneratedField::ExprName),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -20367,6 +20377,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {
{
let mut expr__ = None;
let mut inputs__ = None;
+ let mut expr_name__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Expr => {
@@ -20383,11 +20394,18 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionExprNode {
}
inputs__ = Some(map_.next_value()?);
}
+ GeneratedField::ExprName => {
+ if expr_name__.is_some() {
+ return Err(serde::de::Error::duplicate_field("exprName"));
+ }
+ expr_name__ = map_.next_value()?;
+ }
}
}
Ok(PhysicalExtensionExprNode {
expr: expr__.unwrap_or_default(),
inputs: inputs__.unwrap_or_default(),
+ expr_name: expr_name__,
})
}
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index 119d465281..1034d0443d 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -2004,6 +2004,12 @@ pub struct PhysicalExtensionExprNode {
pub expr: ::prost::alloc::vec::Vec<u8>,
#[prost(message, repeated, tag = "2")]
pub inputs: ::prost::alloc::vec::Vec<PhysicalExprNode>,
+ /// Optional type discriminator naming the extension expression, e.g.
+ /// "my_crate.MyExpr". When present and registered in the session's
+ /// ProtoDecoderRegistry, decoding is routed to that registered
+ /// decoder; otherwise it falls back to PhysicalExtensionCodec::try_decode_expr.
+ #[prost(string, optional, tag = "3")]
+ pub expr_name: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct PhysicalHashExprNode {
diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs
index 8c3416ec98..e47c54f2b7 100644
--- a/datafusion/proto/src/physical_plan/from_proto.rs
+++ b/datafusion/proto/src/physical_plan/from_proto.rs
@@ -23,7 +23,7 @@ use arrow::array::RecordBatch;
use arrow::compute::SortOptions;
use arrow::datatypes::{Field, Schema};
use arrow::ipc::reader::StreamReader;
-use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
+use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};
use datafusion_datasource::TableSchema;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -41,6 +41,9 @@ use datafusion_physical_plan::expressions::{
};
use datafusion_physical_plan::joins::HashExpr;
use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx;
+use datafusion_physical_plan::proto::{
+ ProtoDecoderRegistry, decode_physical_expr, physical_expr_names,
+};
use datafusion_physical_plan::repartition::RangeExpr;
use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};
use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};
@@ -368,16 +371,42 @@ pub fn parse_physical_expr_with_converter(
SqlSimilarToPattern::try_from_proto(proto, &decode_ctx)?
}
ExprType::Extension(extension) => {
+ // Lookup-order policy, the same one extension plans use: an
+ // expression that named itself on the wire and is registered on
+ // this session decodes itself, no codec involved. Anything else —
+ // no name, or a name this session has no decoder for — takes the
+ // codec path, which is also every node written before `expr_name`
+ // existed. `None` from the registry means no decoder claims the
+ // name; a decode *failure* is returned as-is rather than falling
+ // through to the codec.
+ let registry = ctx
+ .task_ctx()
+ .session_config()
+ .get_extension::<ProtoDecoderRegistry>();
+ if let Some(registry) = registry.as_ref()
+ && let Some(decoded) = decode_physical_expr(registry, proto, &decode_ctx)
+ {
+ return decoded;
+ }
+
let inputs: Vec<Arc<dyn PhysicalExpr>> = extension
.inputs
.iter()
.map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx))
.collect::<Result<_>>()?;
- ctx.codec().try_decode_expr(
- extension.expr.as_slice(),
- &inputs,
- &decode_ctx,
- )? as _
+ ctx.codec()
+ .try_decode_expr(extension.expr.as_slice(), &inputs, &decode_ctx)
+ .map_err(|e| match extension.expr_name.as_deref() {
+ // The writer named the expression but this session has no
+ // decoder for it: say so, rather than leaving only the
+ // codec's error to explain a missing registration.
+ Some(expr_name) => unregistered_extension_expr_context(
+ expr_name,
+ registry.as_deref(),
+ e,
+ ),
+ None => e,
+ })?
}
ExprType::Lambda(_) => LambdaExpr::try_from_proto(proto, &decode_ctx)?,
ExprType::LambdaVariable(_) => {
@@ -490,6 +519,36 @@ struct ConverterDecoder<'a, 'b> {
proto_converter: &'a dyn PhysicalProtoConverterExtension,
}
+/// Error for a `PhysicalExtensionExprNode` that names its expression type but
+/// finds no decoder registered for that name.
+///
+/// Lists what *is* registered, because the usual cause is a session configured
+/// on the writing side but not on the reading one. The codec's own error is
+/// kept as the cause rather than reworded: the codec may have failed for a
+/// reason that has nothing to do with the missing registration, and a caller
+/// matching on the error kind must still see the kind the codec chose.
+fn unregistered_extension_expr_context(
+ expr_name: &str,
+ registry: Option<&ProtoDecoderRegistry>,
+ codec_error: DataFusionError,
+) -> DataFusionError {
+ let mut registered: Vec<&str> = registry
+ .map(|registry| physical_expr_names(registry).collect())
+ .unwrap_or_default();
+ registered.sort_unstable();
+ let registered = if registered.is_empty() {
+ "none".to_string()
+ } else {
+ registered.join(", ")
+ };
+ codec_error.context(format!(
+ "No decoder is registered for the extension PhysicalExpr '{expr_name}'. Register \
+ the expression in the ProtoDecoderRegistry attached to the decoding session's \
+ SessionConfig, or supply a PhysicalExtensionCodec that handles it. Registered \
+ extension expressions: {registered}"
+ ))
+}
+
impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecode
for ConverterDecoder<'_, '_>
{
@@ -501,4 +560,8 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD
self.proto_converter
.proto_to_physical_expr(node, schema, self.ctx)
}
+
+ fn session(&self) -> Option<&dyn std::any::Any> {
+ Some(self.ctx.task_ctx())
+ }
}
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 596c856c0c..f72a9d8d06 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -1561,6 +1561,13 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {
/// Decode a custom extension expression from `buf`.
///
+ /// Expressions can instead name themselves on the wire and be decoded by a
+ /// per-type decoder registered on the session — see
+ /// [`ExtensionExprFromProto`] and [`register_physical_expr`]. That path
+ /// resolves by name rather than by codec registration order, so two crates
+ /// claiming the same name collide at registration. This method stays the
+ /// fallback for everything unnamed or unregistered.
+ ///
/// `inputs` holds the already-decoded children carried in the
/// `PhysicalExtensionExprNode.inputs` field. If the codec instead embeds
/// nested `PhysicalExprNode`s *inside* `buf`, decode them through
@@ -1573,6 +1580,8 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {
/// cache-hits on its `expr_id` and re-shares one `Arc<dyn PhysicalExpr>`.
///
/// [`parse_physical_expr`]: crate::physical_plan::from_proto::parse_physical_expr
+ /// [`ExtensionExprFromProto`]: datafusion_physical_plan::proto::ExtensionExprFromProto
+ /// [`register_physical_expr`]: datafusion_physical_plan::proto::register_physical_expr
fn try_decode_expr(
&self,
_buf: &[u8],
diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs
index 5ae57752de..97101db471 100644
--- a/datafusion/proto/src/physical_plan/to_proto.rs
+++ b/datafusion/proto/src/physical_plan/to_proto.rs
@@ -284,7 +284,15 @@ pub fn serialize_physical_expr_with_converter(
proto_converter,
};
let ctx = datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx::new(&encoder);
- if let Some(node) = expr.try_to_proto(&ctx)? {
+ if let Some(mut node) = expr.try_to_proto(&ctx)? {
+ // Stamp the expression's identity here, at the common boundary, so
+ // a hook that builds its node with a helper such as
+ // `extension_expr_node` (which cannot see the id) keeps the same
+ // dedup behaviour as the downcast path below. A hook that set its
+ // own `expr_id` keeps it.
+ if node.expr_id.is_none() {
+ node.expr_id = expr_id;
+ }
return Ok(node);
}
@@ -332,7 +340,15 @@ pub fn serialize_physical_expr_with_converter(
Ok(protobuf::PhysicalExprNode {
expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::Extension(
- protobuf::PhysicalExtensionExprNode { expr: buf, inputs },
+ protobuf::PhysicalExtensionExprNode {
+ expr: buf,
+ inputs,
+ // The codec *is* the discriminator on this path,
+ // so no name is written. Expressions that want
+ // registry decoding write their own node from
+ // `try_to_proto`.
+ expr_name: None,
+ },
)),
})
}
diff --git a/datafusion/proto/tests/cases/plans/expr_registry.rs b/datafusion/proto/tests/cases/plans/expr_registry.rs
new file mode 100644
index 0000000000..87da5b88ae
--- /dev/null
+++ b/datafusion/proto/tests/cases/plans/expr_registry.rs
@@ -0,0 +1,982 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Extension `PhysicalExpr`s decoded by name through a
+//! [`ProtoDecoderRegistry`] instead of a [`PhysicalExtensionCodec`].
+
+use std::fmt::{Debug, Display, Formatter};
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use arrow::record_batch::RecordBatch;
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::empty::EmptyExec;
+use datafusion::physical_plan::expressions::{BinaryExpr, col, lit};
+use datafusion::physical_plan::filter::FilterExec;
+use datafusion::physical_plan::proto::PhysicalExprDecodeCtx;
+use datafusion::physical_plan::proto::PhysicalExprEncodeCtx;
+use datafusion::physical_plan::proto::{
+ ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionExprFromProto,
+ ExtensionPlanFromProto, ProtoDecoderRegistry, extension_expr_node,
+ physical_expr_names, register_execution_plan, register_physical_expr,
+};
+use datafusion::physical_plan::{
+ DisplayAs, DisplayFormatType, PlanProperties, SendableRecordBatchStream,
+};
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_common::tree_node::TreeNodeRecursion;
+use datafusion_common::{
+ DataFusionError, Result, internal_datafusion_err, internal_err, resources_err,
+};
+use datafusion_expr::registry::FunctionRegistry;
+use datafusion_expr::{
+ AggregateUDF, ColumnarValue, Operator, ScalarUDF, Volatility, create_udf,
+};
+use datafusion_physical_expr_common::expect_expr_variant;
+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
+use datafusion_proto::bytes::{
+ physical_plan_from_bytes_with_extension_codec,
+ physical_plan_to_bytes_with_extension_codec,
+};
+use datafusion_proto::physical_plan::{
+ DeduplicatingProtoConverter, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,
+};
+use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;
+use datafusion_proto::protobuf::{
+ PhysicalExprNode, PhysicalPlanNode, physical_expr_node,
+};
+use prost::Message;
+
+/// The wire name of a [`TaggedExpr`], and whether its decoder fails on
+/// purpose. Each kind is a distinct Rust type, which the registry's identity
+/// rules need.
+trait TagKind: Debug + Clone + Send + Sync + 'static {
+ const NAME: &'static str;
+ const DECODE_FAILS: bool = false;
+}
+
+#[derive(Debug, Clone)]
+struct Tag;
+impl TagKind for Tag {
+ const NAME: &'static str = "datafusion.tests.TagExpr";
+}
+
+/// A different type that claims `Tag`'s name.
+#[derive(Debug, Clone)]
+struct TagClone;
+impl TagKind for TagClone {
+ const NAME: &'static str = Tag::NAME;
+}
+
+#[derive(Debug, Clone)]
+struct Other;
+impl TagKind for Other {
+ const NAME: &'static str = "other_crate.OtherExpr";
+}
+
+#[derive(Debug, Clone)]
+struct Unnamed;
+impl TagKind for Unnamed {
+ const NAME: &'static str = "";
+}
+
+/// Registered under `Tag`'s name, with a decoder that fails.
+#[derive(Debug, Clone)]
+struct Failing;
+impl TagKind for Failing {
+ const NAME: &'static str = Tag::NAME;
+ const DECODE_FAILS: bool = true;
+}
+
+/// An extension expression with a payload of its own (`tag`) and one child.
+#[derive(Debug, Clone)]
+struct TaggedExpr<K: TagKind> {
+ tag: String,
+ child: Arc<dyn PhysicalExpr>,
+ /// The session's batch size, read at decode time. Only the registry
+ /// decoder sets it, so a test can tell the registry path from the codec
+ /// path.
+ decoded_batch_size: Option<usize>,
+ kind: PhantomData<K>,
+}
+
+type TagExpr = TaggedExpr<Tag>;
+
+impl<K: TagKind> TaggedExpr<K> {
+ fn new(tag: &str, child: Arc<dyn PhysicalExpr>) -> Self {
+ Self {
+ tag: tag.to_string(),
+ child,
+ decoded_batch_size: None,
+ kind: PhantomData,
+ }
+ }
+}
+
+impl<K: TagKind> Display for TaggedExpr<K> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "TaggedExpr({}, {})", self.tag, self.child)
+ }
+}
+
+impl<K: TagKind> PartialEq for TaggedExpr<K> {
+ fn eq(&self, other: &Self) -> bool {
+ self.tag == other.tag && self.child.eq(&other.child)
+ }
+}
+
+impl<K: TagKind> Eq for TaggedExpr<K> {}
+
+impl<K: TagKind> std::hash::Hash for TaggedExpr<K> {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.tag.hash(state);
+ self.child.hash(state);
+ }
+}
+
+impl<K: TagKind> PhysicalExpr for TaggedExpr<K> {
+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
+ self.child.data_type(input_schema)
+ }
+
+ fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+ self.child.nullable(input_schema)
+ }
+
+ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
+ self.child.evaluate(batch)
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![&self.child]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(Arc::new(Self {
+ child: Arc::clone(&children[0]),
+ ..(*self).clone()
+ }))
+ }
+
+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ Display::fmt(self, f)
+ }
+
+ fn try_to_proto(
+ &self,
+ ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<Option<PhysicalExprNode>> {
+ Ok(Some(extension_expr_node::<Self>(
+ ctx,
+ name_payload(&self.tag),
+ self.children(),
+ )?))
+ }
+}
+
+impl<K: TagKind> ExtensionExprFromProto for TaggedExpr<K> {
+ const NAME: &'static str = K::NAME;
+
+ fn try_from_proto(
+ node: &PhysicalExprNode,
+ ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ if K::DECODE_FAILS {
+ return internal_err!("registry decoder failed on purpose");
+ }
+ let extension = expect_expr_variant!(
+ node,
+ physical_expr_node::ExprType::Extension,
+ "Extension"
+ );
+ let [child] =
+ <[_; 1]>::try_from(ctx.decode_children_expressions(&extension.inputs)?)
+ .map_err(|_| {
+ internal_datafusion_err!("TaggedExpr expects exactly one child")
+ })?;
+ Ok(Arc::new(Self {
+ tag: decode_name_payload(&extension.expr)?,
+ child,
+ decoded_batch_size: Some(task_ctx(ctx)?.session_config().batch_size()),
+ kind: PhantomData,
+ }))
+ }
+}
+
+/// A function the decoder resolves by name from the session.
+#[derive(Debug, Clone)]
+enum FunctionRef {
+ Scalar(Arc<ScalarUDF>),
+ Aggregate(Arc<AggregateUDF>),
+}
+
+/// An extension expression that holds a UDF, as an expression that wraps a
+/// user-defined scalar or aggregate function would. Only the function name is
+/// written. The decoder finds the function in the session's registry, so a
+/// user-defined function decodes to the instance registered on the reading
+/// session.
+#[derive(Debug, Clone)]
+struct FunctionRefExpr {
+ function: FunctionRef,
+ child: Arc<dyn PhysicalExpr>,
+}
+
+impl FunctionRefExpr {
+ fn name(&self) -> &str {
+ match &self.function {
+ FunctionRef::Scalar(udf) => udf.name(),
+ FunctionRef::Aggregate(udaf) => udaf.name(),
+ }
+ }
+}
+
+impl Display for FunctionRefExpr {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "FunctionRefExpr({}, {})", self.name(), self.child)
+ }
+}
+
+impl PartialEq for FunctionRefExpr {
+ fn eq(&self, other: &Self) -> bool {
+ self.name() == other.name() && self.child.eq(&other.child)
+ }
+}
+
+impl Eq for FunctionRefExpr {}
+
+impl std::hash::Hash for FunctionRefExpr {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.name().hash(state);
+ self.child.hash(state);
+ }
+}
+
+impl PhysicalExpr for FunctionRefExpr {
+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
+ self.child.data_type(input_schema)
+ }
+
+ fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+ self.child.nullable(input_schema)
+ }
+
+ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
+ self.child.evaluate(batch)
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![&self.child]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(Arc::new(Self {
+ function: self.function.clone(),
+ child: Arc::clone(&children[0]),
+ }))
+ }
+
+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ Display::fmt(self, f)
+ }
+
+ fn try_to_proto(
+ &self,
+ ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<Option<PhysicalExprNode>> {
+ let payload = match &self.function {
+ FunctionRef::Scalar(udf) => format!("scalar:{}", udf.name()),
+ FunctionRef::Aggregate(udaf) => format!("aggregate:{}", udaf.name()),
+ };
+ Ok(Some(extension_expr_node::<Self>(
+ ctx,
+ name_payload(&payload),
+ self.children(),
+ )?))
+ }
+}
+
+impl ExtensionExprFromProto for FunctionRefExpr {
+ const NAME: &'static str = "datafusion.tests.FunctionRefExpr";
+
+ fn try_from_proto(
+ node: &PhysicalExprNode,
+ ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ let extension = expect_expr_variant!(
+ node,
+ physical_expr_node::ExprType::Extension,
+ "Extension"
+ );
+ let payload = decode_name_payload(&extension.expr)?;
+ let functions = task_ctx(ctx)?;
+ let function = match payload.split_once(':') {
+ Some(("scalar", name)) => FunctionRef::Scalar(functions.udf(name)?),
+ Some(("aggregate", name)) => FunctionRef::Aggregate(functions.udaf(name)?),
+ _ => return internal_err!("bad FunctionRefExpr payload: {payload}"),
+ };
+ let [child] =
+ <[_; 1]>::try_from(ctx.decode_children_expressions(&extension.inputs)?)
+ .map_err(|_| {
+ internal_datafusion_err!("FunctionRefExpr expects exactly one child")
+ })?;
+ Ok(Arc::new(Self { function, child }))
+ }
+}
+
+fn task_ctx<'a>(ctx: &'a PhysicalExprDecodeCtx<'_>) -> Result<&'a TaskContext> {
+ ctx.session::<TaskContext>()
+ .ok_or_else(|| internal_datafusion_err!("decode has no TaskContext"))
+}
+
+/// Wire layout for the test expressions' own state. Children are written in
+/// the extension node's `inputs`, not here.
+#[derive(Clone, PartialEq, prost::Message)]
+struct NamePayload {
+ #[prost(string, tag = "1")]
+ name: String,
+}
+
+fn name_payload(name: &str) -> Vec<u8> {
+ NamePayload {
+ name: name.to_string(),
+ }
+ .encode_to_vec()
+}
+
+fn decode_name_payload(buf: &[u8]) -> Result<String> {
+ NamePayload::decode(buf)
+ .map(|proto| proto.name)
+ .map_err(|e| internal_datafusion_err!("decode NamePayload: {e}"))
+}
+
+/// A codec for the expression tests. `decode` is what `try_decode_expr` does.
+/// `try_encode_expr` always fails: the test expressions write themselves, so
+/// the codec must not be asked to encode them.
+#[derive(Debug)]
+enum TestCodec {
+ /// Fails with an internal error. A successful decode through it proves
+ /// the codec was not used.
+ Refusing,
+ /// Fails with an error kind of its own.
+ Exhausted,
+ /// Decodes a `TagExpr`, as a codec did before the registry existed.
+ Tag,
+}
+
+impl PhysicalExtensionCodec for TestCodec {
+ fn try_decode(
+ &self,
+ _buf: &[u8],
+ _inputs: &[Arc<dyn ExecutionPlan>],
+ _ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ internal_err!("TestCodec::try_decode must not be reached")
+ }
+
+ fn try_encode(
+ &self,
+ _node: Arc<dyn ExecutionPlan>,
+ _buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ internal_err!("TestCodec::try_encode must not be reached")
+ }
+
+ fn try_decode_expr(
+ &self,
+ buf: &[u8],
+ inputs: &[Arc<dyn PhysicalExpr>],
+ _ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ match self {
+ Self::Refusing => internal_err!("RefusingCodec must not be reached"),
+ Self::Exhausted => resources_err!("the decode buffer pool is empty"),
+ Self::Tag => {
+ let [child] = inputs else {
+ return internal_err!("TagExpr expects exactly one child");
+ };
+ Ok(Arc::new(TagExpr::new(
+ &decode_name_payload(buf)?,
+ Arc::clone(child),
+ )))
+ }
+ }
+ }
+
+ fn try_encode_expr(
+ &self,
+ _node: &Arc<dyn PhysicalExpr>,
+ _buf: &mut Vec<u8>,
+ _ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<()> {
+ internal_err!("TestCodec::try_encode_expr must not be reached")
+ }
+}
+
+/// A session whose registry is filled by `register`.
+fn session_with(
+ config: SessionConfig,
+ register: impl FnOnce(&mut ProtoDecoderRegistry) -> Result<()>,
+) -> Result<SessionContext> {
+ let mut registry = ProtoDecoderRegistry::new();
+ register(&mut registry)?;
+ Ok(SessionContext::new_with_config(
+ config.with_extension(Arc::new(registry)),
+ ))
+}
+
+fn test_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]))
+}
+
+/// Serialize `expr` and decode the prost bytes, as the wire would.
+fn to_wire(
+ expr: &Arc<dyn PhysicalExpr>,
+ codec: &dyn PhysicalExtensionCodec,
+) -> Result<PhysicalExprNode> {
+ let proto = DefaultPhysicalProtoConverter {}.physical_expr_to_proto(expr, codec)?;
+ PhysicalExprNode::decode(proto.encode_to_vec().as_slice())
+ .map_err(|e| internal_datafusion_err!("decode PhysicalExprNode: {e}"))
+}
+
+fn from_wire(
+ node: &PhysicalExprNode,
+ session: &SessionContext,
+ codec: &dyn PhysicalExtensionCodec,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let task_ctx = session.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), codec);
+ DefaultPhysicalProtoConverter {}.proto_to_physical_expr(
+ node,
+ &test_schema(),
+ &decode_ctx,
+ )
+}
+
+fn tag_expr() -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(Arc::new(TagExpr::new("hello", col("a", &test_schema())?)))
+}
+
+fn set_expr_name(node: &mut PhysicalExprNode, name: Option<&str>) {
+ match node.expr_type.as_mut().unwrap() {
+ physical_expr_node::ExprType::Extension(extension) => {
+ extension.expr_name = name.map(str::to_string);
+ }
+ other => panic!("expected an Extension node, got {other:?}"),
+ }
+}
+
+/// An extension expression writes its name, and the session's registry
+/// decodes it with no codec on either side. The decoder can read the session.
+#[test]
+fn registry_decodes_named_extension_expr() -> Result<()> {
+ let node = to_wire(&tag_expr()?, &TestCodec::Refusing)?;
+ match node.expr_type.as_ref().unwrap() {
+ physical_expr_node::ExprType::Extension(extension) => {
+ assert_eq!(extension.expr_name.as_deref(), Some(TagExpr::NAME))
+ }
+ other => panic!("expected an Extension node, got {other:?}"),
+ }
+
+ let session = session_with(
+ SessionConfig::new().with_batch_size(1234),
+ register_physical_expr::<TagExpr>,
+ )?;
+ let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;
+ let decoded = decoded.downcast_ref::<TagExpr>().unwrap();
+ assert_eq!(decoded.tag, "hello");
+ assert_eq!(decoded.child.to_string(), "a@0");
+ assert_eq!(decoded.decoded_batch_size, Some(1234));
+ Ok(())
+}
+
+/// A node with no name, or with a name the session does not know, decodes
+/// through the codec as before.
+#[test]
+fn unnamed_or_unregistered_falls_back_to_codec() -> Result<()> {
+ let named = to_wire(&tag_expr()?, &TestCodec::Tag)?;
+ let mut unnamed = named.clone();
+ set_expr_name(&mut unnamed, None);
+
+ let empty = SessionContext::new();
+ let registered =
+ session_with(SessionConfig::new(), register_physical_expr::<TagExpr>)?;
+ for (node, session) in [(&named, &empty), (&unnamed, &registered)] {
+ let decoded = from_wire(node, session, &TestCodec::Tag)?;
+ let decoded = decoded.downcast_ref::<TagExpr>().unwrap();
+ assert_eq!(decoded.tag, "hello");
+ assert_eq!(decoded.decoded_batch_size, None, "must come from the codec");
+ }
+ Ok(())
+}
+
+/// When the codec also fails, the error keeps the codec's own error kind and
+/// message. If the node had a name, the error also says that no decoder is
+/// registered for it and lists the names the session knows.
+#[test]
+fn codec_failure_keeps_its_error_and_names_the_missing_registration() -> Result<()> {
+ let named = to_wire(&tag_expr()?, &TestCodec::Refusing)?;
+ let session = session_with(
+ SessionConfig::new(),
+ register_physical_expr::<TaggedExpr<Other>>,
+ )?;
+
+ let err = from_wire(&named, &session, &TestCodec::Exhausted).unwrap_err();
+ assert!(
+ matches!(err.find_root(), DataFusionError::ResourcesExhausted(_)),
+ "the codec's error kind must survive, got: {err:?}"
+ );
+ let text = err.to_string();
+ for expected in [
+ "the decode buffer pool is empty",
+ "No decoder is registered",
+ TagExpr::NAME,
+ "ProtoDecoderRegistry",
+ Other::NAME,
+ ] {
+ assert!(text.contains(expected), "missing {expected:?} in: {text}");
+ }
+
+ let mut unnamed = named;
+ set_expr_name(&mut unnamed, None);
+ let text = from_wire(&unnamed, &session, &TestCodec::Refusing)
+ .unwrap_err()
+ .to_string();
+ assert!(text.contains("RefusingCodec must not be reached"), "{text}");
+ assert!(!text.contains("No decoder is registered"), "{text}");
+ Ok(())
+}
+
+/// Registering one type twice is a no-op. Two types with the same name, or a
+/// type with an empty name, are errors at registration.
+#[test]
+fn registration_rejects_collisions_and_empty_names() -> Result<()> {
+ let mut registry = ProtoDecoderRegistry::new();
+ register_physical_expr::<TagExpr>(&mut registry)?;
+ register_physical_expr::<TagExpr>(&mut registry)?;
+
+ let err = register_physical_expr::<TaggedExpr<TagClone>>(&mut registry)
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("already registered"), "{err}");
+
+ let err = register_physical_expr::<TaggedExpr<Unnamed>>(&mut registry)
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("empty name"), "{err}");
+
+ assert_eq!(
+ physical_expr_names(&registry).collect::<Vec<_>>(),
+ [Tag::NAME]
+ );
+ Ok(())
+}
+
+/// The children of a registry-decoded expression also decode through the
+/// registry.
+#[test]
+fn nested_extension_exprs_both_decode_through_the_registry() -> Result<()> {
+ let inner: Arc<dyn PhysicalExpr> =
+ Arc::new(TagExpr::new("inner", col("a", &test_schema())?));
+ let outer: Arc<dyn PhysicalExpr> = Arc::new(TagExpr::new("outer", inner));
+ let node = to_wire(&outer, &TestCodec::Refusing)?;
+ let session = session_with(SessionConfig::new(), register_physical_expr::<TagExpr>)?;
+
+ let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;
+ let outer = decoded.downcast_ref::<TagExpr>().unwrap();
+ let inner = outer.child.downcast_ref::<TagExpr>().unwrap();
+ assert_eq!((outer.tag.as_str(), inner.tag.as_str()), ("outer", "inner"));
+ assert!(inner.decoded_batch_size.is_some());
+ Ok(())
+}
+
+/// If the registered decoder fails, the decode fails. The codec is not tried,
+/// even though it could decode this node.
+#[test]
+fn a_failing_registry_decoder_does_not_fall_through_to_the_codec() -> Result<()> {
+ let node = to_wire(&tag_expr()?, &TestCodec::Tag)?;
+ let session = session_with(
+ SessionConfig::new(),
+ register_physical_expr::<TaggedExpr<Failing>>,
+ )?;
+ let err = from_wire(&node, &session, &TestCodec::Tag).unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("registry decoder failed on purpose"),
+ "{err}"
+ );
+ Ok(())
+}
+
+/// A user-defined scalar function and an aggregate function, held by an
+/// extension expression, decode to the instances registered on the reading
+/// session.
+#[test]
+fn extension_expr_resolves_scalar_and_aggregate_udfs_from_the_session() -> Result<()> {
+ let session = session_with(
+ SessionConfig::new(),
+ register_physical_expr::<FunctionRefExpr>,
+ )?;
+ session.register_udf(create_udf(
+ "my_add_one",
+ vec![DataType::Int64],
+ DataType::Int64,
+ Volatility::Immutable,
+ Arc::new(|args| Ok(args[0].clone())),
+ ));
+ let state = session.state();
+ let scalar = state.udf("my_add_one")?;
+ let aggregate = state.udaf("sum")?;
+
+ for function in [
+ FunctionRef::Scalar(Arc::clone(&scalar)),
+ FunctionRef::Aggregate(Arc::clone(&aggregate)),
+ ] {
+ let expr: Arc<dyn PhysicalExpr> = Arc::new(FunctionRefExpr {
+ function,
+ child: col("a", &test_schema())?,
+ });
+ let node = to_wire(&expr, &TestCodec::Refusing)?;
+ let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;
+ match &decoded.downcast_ref::<FunctionRefExpr>().unwrap().function {
+ FunctionRef::Scalar(udf) => assert!(Arc::ptr_eq(udf, &scalar)),
+ FunctionRef::Aggregate(udaf) => assert!(Arc::ptr_eq(udaf, &aggregate)),
+ }
+ }
+
+ // A reader that does not have the UDF gets the registry's error.
+ let expr: Arc<dyn PhysicalExpr> = Arc::new(FunctionRefExpr {
+ function: FunctionRef::Scalar(scalar),
+ child: col("a", &test_schema())?,
+ });
+ let node = to_wire(&expr, &TestCodec::Refusing)?;
+ let reader = session_with(
+ SessionConfig::new(),
+ register_physical_expr::<FunctionRefExpr>,
+ )?;
+ let err = from_wire(&node, &reader, &TestCodec::Refusing).unwrap_err();
+ assert!(err.to_string().contains("my_add_one"), "{err}");
+ Ok(())
+}
+
+/// A whole plan, through `physical_plan_to_bytes` and
+/// `physical_plan_from_bytes`.
+#[test]
+fn registry_expr_survives_a_whole_plan_roundtrip() -> Result<()> {
+ let schema = test_schema();
+ let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(TagExpr::new("in_plan", col("a", &schema)?)),
+ Operator::Eq,
+ lit(1i64),
+ ));
+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(FilterExec::try_new(predicate, input)?);
+ let bytes = physical_plan_to_bytes_with_extension_codec(plan, &TestCodec::Refusing)?;
+
+ let session = session_with(
+ SessionConfig::new().with_batch_size(4096),
+ register_physical_expr::<TagExpr>,
+ )?;
+ let decoded = physical_plan_from_bytes_with_extension_codec(
+ &bytes,
+ session.task_ctx().as_ref(),
+ &TestCodec::Refusing,
+ )?;
+ let filter = decoded.downcast_ref::<FilterExec>().unwrap();
+ let binary = filter.predicate().downcast_ref::<BinaryExpr>().unwrap();
+ let tag = binary.left().downcast_ref::<TagExpr>().unwrap();
+ assert_eq!(tag.tag, "in_plan");
+ assert_eq!(tag.decoded_batch_size, Some(4096));
+ Ok(())
+}
+
+/// An extension expression with an identity: two references with the same
+/// `expression_id` share `state`, as two pushed-down copies of one dynamic
+/// filter would. `with_new_children` keeps `state`, so a deduplicating decoder
+/// that re-shares the first decode keeps the two copies linked.
+#[derive(Debug, Clone)]
+struct IdentityExpr {
+ id: u64,
+ state: Arc<String>,
+ child: Arc<dyn PhysicalExpr>,
+}
+
+impl Display for IdentityExpr {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "IdentityExpr({}, {})", self.id, self.child)
+ }
+}
+
+impl PartialEq for IdentityExpr {
+ fn eq(&self, other: &Self) -> bool {
+ self.id == other.id && self.child.eq(&other.child)
+ }
+}
+
+impl Eq for IdentityExpr {}
+
+impl std::hash::Hash for IdentityExpr {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.id.hash(state);
+ self.child.hash(state);
+ }
+}
+
+impl PhysicalExpr for IdentityExpr {
+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
+ self.child.data_type(input_schema)
+ }
+
+ fn nullable(&self, input_schema: &Schema) -> Result<bool> {
+ self.child.nullable(input_schema)
+ }
+
+ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
+ self.child.evaluate(batch)
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![&self.child]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(Arc::new(Self {
+ child: Arc::clone(&children[0]),
+ ..(*self).clone()
+ }))
+ }
+
+ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ Display::fmt(self, f)
+ }
+
+ fn expression_id(&self) -> Option<u64> {
+ Some(self.id)
+ }
+
+ fn try_to_proto(
+ &self,
+ ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<Option<PhysicalExprNode>> {
+ // The helper cannot see `expression_id`; the serializer stamps it.
+ Ok(Some(extension_expr_node::<Self>(
+ ctx,
+ name_payload(&self.state),
+ self.children(),
+ )?))
+ }
+}
+
+impl ExtensionExprFromProto for IdentityExpr {
+ const NAME: &'static str = "datafusion-test.IdentityExpr";
+
+ fn try_from_proto(
+ node: &PhysicalExprNode,
+ ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ let extension = expect_expr_variant!(
+ node,
+ physical_expr_node::ExprType::Extension,
+ "Extension"
+ );
+ let Some(id) = node.expr_id else {
+ return internal_err!("IdentityExpr needs an expr_id");
+ };
+ let mut children = ctx.decode_children_expressions(&extension.inputs)?;
+ Ok(Arc::new(Self {
+ id,
+ state: Arc::new(decode_name_payload(&extension.expr)?),
+ child: children.remove(0),
+ }))
+ }
+}
+
+/// An extension expression built with `extension_expr_node` keeps its
+/// `expression_id` on the wire. Two references to it, with different
+/// children, decode through the deduplicating converter to two expressions
+/// that share state, each with its own child.
+#[test]
+fn registry_expr_keeps_its_expression_id_for_deduplication() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int64, false),
+ Field::new("b", DataType::Int64, false),
+ ]));
+ let state = Arc::new("shared".to_string());
+ let left: Arc<dyn PhysicalExpr> = Arc::new(IdentityExpr {
+ id: 42,
+ state: Arc::clone(&state),
+ child: col("a", &schema)?,
+ });
+ let right: Arc<dyn PhysicalExpr> = Arc::new(IdentityExpr {
+ id: 42,
+ state,
+ child: col("b", &schema)?,
+ });
+ let pair: Arc<dyn PhysicalExpr> =
+ Arc::new(BinaryExpr::new(Arc::clone(&left), Operator::And, right));
+
+ let codec = TestCodec::Refusing;
+ let converter = DeduplicatingProtoConverter {};
+ let node = converter.physical_expr_to_proto(&pair, &codec)?;
+ assert_eq!(
+ to_wire(&left, &codec)?.expr_id,
+ Some(42),
+ "the serializer must stamp the expression id"
+ );
+
+ let session =
+ session_with(SessionConfig::new(), register_physical_expr::<IdentityExpr>)?;
+ let task_ctx = session.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ let decoded = converter.proto_to_physical_expr(&node, &schema, &decode_ctx)?;
+ let binary = decoded.downcast_ref::<BinaryExpr>().unwrap();
+ let left = binary.left().downcast_ref::<IdentityExpr>().unwrap();
+ let right = binary.right().downcast_ref::<IdentityExpr>().unwrap();
+ assert!(
+ Arc::ptr_eq(&left.state, &right.state),
+ "state must be shared"
+ );
+ assert_eq!(left.child.to_string(), "a@0");
+ assert_eq!(right.child.to_string(), "b@1");
+ Ok(())
+}
+
+/// An extension plan whose wire name equals [`TagExpr::NAME`]. The registry
+/// scopes names per kind, so the plan and the expression do not collide.
+#[derive(Debug)]
+struct SameNameExec {
+ child: Arc<dyn ExecutionPlan>,
+}
+
+impl DisplayAs for SameNameExec {
+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+ write!(f, "SameNameExec")
+ }
+}
+
+impl ExecutionPlan for SameNameExec {
+ fn name(&self) -> &str {
+ "SameNameExec"
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ self.child.properties()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![&self.child]
+ }
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ mut children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ Ok(Arc::new(Self {
+ child: children.remove(0),
+ }))
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ internal_err!("SameNameExec is a serde-only test plan")
+ }
+
+ fn try_to_proto(
+ &self,
+ ctx: &ExecutionPlanEncodeCtx<'_>,
+ ) -> Result<Option<PhysicalPlanNode>> {
+ Ok(Some(ctx.extension_node::<Self>(vec![], self.children())?))
+ }
+}
+
+impl ExtensionPlanFromProto for SameNameExec {
+ const NAME: &'static str = Tag::NAME;
+
+ fn try_from_proto(
+ node: &PhysicalPlanNode,
+ ctx: &ExecutionPlanDecodeCtx<'_>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let Some(PhysicalPlanType::Extension(extension)) = &node.physical_plan_type
+ else {
+ return internal_err!("expected an extension node");
+ };
+ let mut children = ctx.decode_children(&extension.inputs)?;
+ Ok(Arc::new(Self {
+ child: children.remove(0),
+ }))
+ }
+}
+
+/// A plan that holds a registered extension plan and a registered extension
+/// expression with the same wire name. Both decode through the registry, each
+/// to its own type.
+#[test]
+fn registry_plan_and_expr_with_the_same_name_roundtrip_together() -> Result<()> {
+ let schema = test_schema();
+ let extension_plan: Arc<dyn ExecutionPlan> = Arc::new(SameNameExec {
+ child: Arc::new(EmptyExec::new(Arc::clone(&schema))),
+ });
+ let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(TagExpr::new("same_name", col("a", &schema)?)),
+ Operator::Eq,
+ lit(1i64),
+ ));
+ let plan: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(predicate, extension_plan)?);
+ let bytes = physical_plan_to_bytes_with_extension_codec(plan, &TestCodec::Refusing)?;
+
+ let session = session_with(SessionConfig::new().with_batch_size(2048), |registry| {
+ register_execution_plan::<SameNameExec>(registry)?;
+ register_physical_expr::<TagExpr>(registry)
+ })?;
+ let decoded = physical_plan_from_bytes_with_extension_codec(
+ &bytes,
+ session.task_ctx().as_ref(),
+ &TestCodec::Refusing,
+ )?;
+ let filter = decoded.downcast_ref::<FilterExec>().unwrap();
+ let binary = filter.predicate().downcast_ref::<BinaryExpr>().unwrap();
+ let tag = binary.left().downcast_ref::<TagExpr>().unwrap();
+ assert_eq!(tag.tag, "same_name");
+ assert_eq!(tag.decoded_batch_size, Some(2048));
+ let extension = filter.input().downcast_ref::<SameNameExec>().unwrap();
+ assert_eq!(extension.child.name(), "EmptyExec");
+ Ok(())
+}
diff --git a/datafusion/proto/tests/cases/plans/mod.rs b/datafusion/proto/tests/cases/plans/mod.rs
index 0f00f35581..69a6256af9 100644
--- a/datafusion/proto/tests/cases/plans/mod.rs
+++ b/datafusion/proto/tests/cases/plans/mod.rs
@@ -36,6 +36,7 @@ use std::sync::Arc;
mod aggregates;
mod dispatch;
mod dynamic_filters;
+mod expr_registry;
mod exprs;
mod extensions;
mod filters;
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 8f5604a5cd..1e8d9e2daa 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1209,6 +1209,63 @@ A registered plan needs no `PhysicalExtensionCodec` at all, and two crates' plan
See [issue #24625](https://github.com/apache/datafusion/issues/24625) for details.
+### Extension `PhysicalExpr`s can decode by name instead of through a codec
+
+`PhysicalExtensionExprNode` has a new optional `expr_name` field. An extension expression that sets it is decoded by a decoder registered on the session, not by a `PhysicalExtensionCodec`. This is the same mechanism extension `ExecutionPlan`s use (see above), and it uses the same `ProtoDecoderRegistry`.
+
+**Who is affected:** nobody, unless they opt in. A node with no name, or with a name the session has no decoder for, decodes through `PhysicalExtensionCodec::try_decode_expr` as before. Binary protobuf readers skip the new field. Readers that use the optional JSON serde reject unknown fields, so they must be built after this change to read a node that has `exprName`.
+
+**Opting in:**
+
+```rust,ignore
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::proto::{
+ ExtensionExprFromProto, PhysicalExprDecodeCtx, PhysicalExprEncodeCtx,
+ ProtoDecoderRegistry, extension_expr_node, register_physical_expr,
+};
+
+impl PhysicalExpr for MyExpr {
+ // ...
+ fn try_to_proto(
+ &self,
+ ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<Option<PhysicalExprNode>> {
+ // Writes `Self::NAME` on the node.
+ Ok(Some(extension_expr_node::<Self>(
+ ctx,
+ my_payload_bytes(self)?,
+ self.children(),
+ )?))
+ }
+}
+
+impl ExtensionExprFromProto for MyExpr {
+ // Put the crate name in the name, so that a collision is an error at registration.
+ const NAME: &'static str = "my_crate.MyExpr";
+
+ fn try_from_proto(
+ node: &PhysicalExprNode,
+ ctx: &PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ // The session, for example to find a UDF by name.
+ let task_ctx = ctx.session::<TaskContext>();
+ // ...
+ }
+}
+
+let mut registry = ProtoDecoderRegistry::new();
+register_physical_expr::<MyExpr>(&mut registry)?;
+let config = SessionConfig::new().with_extension(Arc::new(registry));
+```
+
+Only extension expressions can be registered. `register_physical_expr::<Column>(..)` does not compile.
+
+If a registered decoder fails, the decode fails. It does not fall back to the codec, because the codec could decode a payload that it did not write. Changing an expression to write its own payload is therefore a wire-format change for that expression: register the type on every reader before writers start to use it.
+
+`PhysicalExprDecodeCtx::session::<T>()` is new. It gives the session the decode runs under. `datafusion-proto` supplies its `TaskContext`. A custom `PhysicalExprDecode` implementation returns `None` by default.
+
+See [issue #24626](https://github.com/apache/datafusion/issues/24626) for details.
+
### Order-preservation hooks for limit pushdown are deprecated
A file scan that advertises an output ordering now always reads its files in