Skip to content

Apache DataFusion SQL Query Engine

refactor: register proto decoders with `T::register_proto` instead of a free function (#26565)

Adrian Garcia Badaracco569d085f2719
diff --git a/datafusion-examples/examples/proto/extension_plan_registry.rs b/datafusion-examples/examples/proto/extension_plan_registry.rs
index c50823be20..fd76872252 100644
--- a/datafusion-examples/examples/proto/extension_plan_registry.rs
+++ b/datafusion-examples/examples/proto/extension_plan_registry.rs
@@ -40,7 +40,6 @@ use datafusion::execution::TaskContext;
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_plan::proto::{
ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionPlanFromProto,
- ProtoDecoderRegistry, register_execution_plan,
};
use datafusion::physical_plan::{
ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
@@ -56,16 +55,13 @@ pub fn extension_plan_registry() -> Result<()> {
input: Arc::new(ChildExec {}),
});
- // Each crate exposes a function that fills a registry it is handed; the
- // application that owns the session composes them. The two crates need to
- // know nothing about each other, and the call order does not matter.
- let mut registry = ProtoDecoderRegistry::new();
- crate_a::register(&mut registry)?;
- crate_b::register(&mut registry)?;
-
- let ctx = SessionContext::new_with_config(
- SessionConfig::new().with_extension(Arc::new(registry)),
- );
+ // Each crate exposes a function that adds its decoders to the session's
+ // registry. The two crates need to know nothing about each other, and the
+ // call order does not matter.
+ let mut config = SessionConfig::new();
+ crate_a::register(&mut config)?;
+ crate_b::register(&mut config)?;
+ let ctx = SessionContext::new_with_config(config);
// `DefaultPhysicalExtensionCodec` decodes nothing at all, so whatever
// survives the round trip came from the registry.
@@ -98,8 +94,8 @@ pub fn extension_plan_registry() -> Result<()> {
mod crate_a {
use super::*;
- pub fn register(registry: &mut ProtoDecoderRegistry) -> Result<()> {
- register_execution_plan::<ParentExec>(registry)
+ pub fn register(config: &mut SessionConfig) -> Result<()> {
+ config.update_extension(ParentExec::register_proto)
}
}
@@ -107,8 +103,8 @@ mod crate_a {
mod crate_b {
use super::*;
- pub fn register(registry: &mut ProtoDecoderRegistry) -> Result<()> {
- register_execution_plan::<ChildExec>(registry)
+ pub fn register(config: &mut SessionConfig) -> Result<()> {
+ config.update_extension(ChildExec::register_proto)
}
}
diff --git a/datafusion/physical-expr/src/proto.rs b/datafusion/physical-expr/src/proto.rs
index 12ed487e93..6f48994232 100644
--- a/datafusion/physical-expr/src/proto.rs
+++ b/datafusion/physical-expr/src/proto.rs
@@ -17,7 +17,7 @@
//! The `PhysicalExpr` face of the session-scoped [`ProtoDecoderRegistry`],
//! which routes extension expressions to their decoders by name:
-//! [`register_physical_expr`] and [`physical_expr_names`], all keyed on `dyn PhysicalExpr`. The store itself
+//! [`ExtensionExprFromProto::register_proto`] and [`physical_expr_names`], all keyed on `dyn PhysicalExpr`. The store itself
//! is shared with every other extension kind, so a session carries one
//! registry.
@@ -37,7 +37,7 @@ use datafusion_proto_models::protobuf::physical_expr_node::ExprType;
/// Only extension expressions implement this. A built-in expression has an
/// `ExprType` variant of its own, keeps the inherent `try_from_proto`
/// dispatched from that variant, and has no wire name to be registered under —
-/// so [`register_physical_expr`] will not accept one.
+/// so it has no [`register`](Self::register_proto) method.
///
/// One impl block says what the expression is called and how it decodes; its
/// `try_to_proto` on [`PhysicalExpr`] writes the matching node with
@@ -80,9 +80,8 @@ use datafusion_proto_models::protobuf::physical_expr_node::ExprType;
/// }
/// }
///
-/// let mut registry = ProtoDecoderRegistry::new();
-/// register_physical_expr::<MyExpr>(&mut registry)?;
-/// let config = SessionConfig::new().with_extension(Arc::new(registry));
+/// let mut config = SessionConfig::new();
+/// config.update_extension(MyExpr::register_proto)?;
/// ```
pub trait ExtensionExprFromProto: PhysicalExpr + Sized {
/// The name this expression type is written and registered under.
@@ -105,6 +104,46 @@ pub trait ExtensionExprFromProto: PhysicalExpr + Sized {
node: &PhysicalExprNode,
ctx: &PhysicalExprDecodeCtx<'_>,
) -> Result<Arc<dyn PhysicalExpr>>;
+
+ /// Register this expression's decoder in `registry`, under [`Self::NAME`].
+ ///
+ /// Registering the same type twice is a no-op. A *different* type under a
+ /// name that is already taken is an error, so a collision shows here and
+ /// not as a wrong decode. The name is scoped to `dyn PhysicalExpr`, so a
+ /// plan or a data source can use the same name in the same registry.
+ ///
+ /// Add it to the session's registry with
+ /// `config.update_extension(MyExpr::register_proto)?`, which keeps the decoders
+ /// that other libraries added. One registry holds every kind, so a library
+ /// registers its plans and its expressions into the same registry.
+ ///
+ /// A built-in expression keeps an inherent `try_from_proto`, dispatched
+ /// from its own `ExprType` variant:
+ ///
+ /// ```
+ /// use datafusion_physical_expr::expressions::Column;
+ /// #[allow(unused_imports)]
+ /// use datafusion_physical_expr::proto::ExtensionExprFromProto;
+ ///
+ /// let _ = Column::try_from_proto;
+ /// ```
+ ///
+ /// but it does not implement this trait, so it cannot be registered:
+ ///
+ /// ```compile_fail
+ /// use datafusion_physical_expr::expressions::Column;
+ /// use datafusion_physical_expr::proto::ExtensionExprFromProto;
+ /// use datafusion_proto_models::ProtoDecoderRegistry;
+ ///
+ /// let mut registry = ProtoDecoderRegistry::new();
+ /// Column::register_proto(&mut registry).unwrap();
+ /// ```
+ fn register_proto(registry: &mut ProtoDecoderRegistry) -> Result<()> {
+ registry.register_decoder::<dyn PhysicalExpr, Self, PhysicalExprDecoder>(
+ Self::NAME,
+ Self::try_from_proto,
+ )
+ }
}
/// Build the `PhysicalExprNode` for an extension expression `T`: an
@@ -136,7 +175,7 @@ pub fn extension_expr_node<T: ExtensionExprFromProto>(
/// to the monomorphized [`ExtensionExprFromProto::try_from_proto`].
///
/// Deliberately private, and the same type on both the
-/// [`register_physical_expr`] and the [`decode_physical_expr`] side.
+/// [`ExtensionExprFromProto::register_proto`] and the [`decode_physical_expr`] side.
/// [`ExtensionExprFromProto`] is the public contract and
/// [`decode_physical_expr`] (hidden, internal to DataFusion) is the only way
/// to invoke one, so the storage
@@ -144,53 +183,6 @@ pub fn extension_expr_node<T: ExtensionExprFromProto>(
type PhysicalExprDecoder =
fn(&PhysicalExprNode, &PhysicalExprDecodeCtx<'_>) -> Result<Arc<dyn PhysicalExpr>>;
-/// Register `T` in `registry` under its [`ExtensionExprFromProto::NAME`].
-///
-/// Registering the same type twice is a no-op. Registering a *different* type
-/// under a name already taken is an error, so collisions surface here rather
-/// than as a wrong decode later. The name is scoped to `dyn PhysicalExpr`, so a
-/// plan or a data source may use the same name in the same registry.
-///
-/// # Who builds the registry
-///
-/// The application that owns the session builds it. A session carries at most
-/// one registry, and `SessionConfig::with_extension` replaces what is there, so
-/// a library must never attach a registry of its own: it would silently discard
-/// another library's. A library exposes a function that fills a registry it is
-/// handed, and the application composes them. One registry holds every kind, so
-/// a library registers its plans and its expressions into the same object.
-///
-/// # A built-in expression cannot be registered
-///
-/// The bound is [`ExtensionExprFromProto`], which only extension expressions
-/// implement. A built-in keeps an inherent `try_from_proto`, dispatched from
-/// its own `ExprType` variant:
-///
-/// ```
-/// use datafusion_physical_expr::expressions::Column;
-///
-/// let _ = Column::try_from_proto;
-/// ```
-///
-/// but it is not an extension, and registering it does not compile:
-///
-/// ```compile_fail
-/// use datafusion_physical_expr::expressions::Column;
-/// use datafusion_physical_expr::proto::register_physical_expr;
-/// use datafusion_proto_models::ProtoDecoderRegistry;
-///
-/// let mut registry = ProtoDecoderRegistry::new();
-/// register_physical_expr::<Column>(&mut registry).unwrap();
-/// ```
-pub fn register_physical_expr<T: ExtensionExprFromProto>(
- registry: &mut ProtoDecoderRegistry,
-) -> Result<()> {
- registry.register_decoder::<dyn PhysicalExpr, T, PhysicalExprDecoder>(
- T::NAME,
- T::try_from_proto,
- )
-}
-
/// Decode `node` with the extension expression decoder registered under the
/// name `node` carries.
///
diff --git a/datafusion/physical-plan/src/proto/mod.rs b/datafusion/physical-plan/src/proto/mod.rs
index afef282bfa..7ed88aacff 100644
--- a/datafusion/physical-plan/src/proto/mod.rs
+++ b/datafusion/physical-plan/src/proto/mod.rs
@@ -81,7 +81,7 @@ use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
use datafusion_physical_expr::PhysicalExpr;
pub use datafusion_physical_expr::proto::{
ExtensionExprFromProto, decode_physical_expr, extension_expr_node,
- physical_expr_names, register_physical_expr,
+ physical_expr_names,
};
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecode;
pub use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
@@ -98,9 +98,7 @@ pub use datafusion_proto_models::ProtoDecoderRegistry;
// Not public API: see `decode_execution_plan`.
#[doc(hidden)]
pub use registry::decode_execution_plan;
-pub use registry::{
- ExtensionPlanFromProto, execution_plan_names, register_execution_plan,
-};
+pub use registry::{ExtensionPlanFromProto, execution_plan_names};
/// Internal dispatch trait backing [`ExecutionPlanEncodeCtx`].
///
diff --git a/datafusion/physical-plan/src/proto/registry.rs b/datafusion/physical-plan/src/proto/registry.rs
index 4fa9ddda2e..0be6615231 100644
--- a/datafusion/physical-plan/src/proto/registry.rs
+++ b/datafusion/physical-plan/src/proto/registry.rs
@@ -24,7 +24,7 @@
//! registration order, which two independent crates cannot agree on.
//!
//! An extension plan implements [`ExtensionPlanFromProto`] instead, and
-//! [`register_execution_plan`] puts its decoder in the
+//! [`ExtensionPlanFromProto::register_proto`] puts its decoder in the
//! [`ProtoDecoderRegistry`] the decoding session carries. Lookup is by name,
//! so registration order does not matter. Built-ins cannot be registered;
//! anything unnamed or unregistered still takes the codec chain, unchanged.
@@ -99,13 +99,53 @@ pub trait ExtensionPlanFromProto: ExecutionPlan + Sized {
node: &PhysicalPlanNode,
ctx: &ExecutionPlanDecodeCtx<'_>,
) -> Result<Arc<dyn ExecutionPlan>>;
+
+ /// Register this plan's decoder in `registry`, under [`Self::NAME`].
+ ///
+ /// Registering the same type twice is a no-op. A *different* type under a
+ /// name that is already taken is an error, so a collision shows here and
+ /// not as a wrong decode. The name is scoped to `dyn ExecutionPlan`, so an
+ /// expression or a data source can use the same name in the same registry.
+ ///
+ /// Add it to the session's registry with
+ /// `config.update_extension(MyExec::register_proto)?`, which keeps the decoders
+ /// that other libraries added. See [`ProtoDecoderRegistry`].
+ ///
+ /// A built-in plan does not implement this trait, so it cannot be
+ /// registered:
+ ///
+ /// ```compile_fail
+ /// use datafusion_physical_plan::filter::FilterExec;
+ /// use datafusion_physical_plan::proto::{ExtensionPlanFromProto, ProtoDecoderRegistry};
+ ///
+ /// let mut registry = ProtoDecoderRegistry::new();
+ /// FilterExec::register_proto(&mut registry).unwrap();
+ /// ```
+ ///
+ /// The same imports without that call compile, so the failure above is
+ /// the missing implementation and not a bad path:
+ ///
+ /// ```
+ /// use datafusion_physical_plan::filter::FilterExec;
+ /// #[allow(unused_imports)]
+ /// use datafusion_physical_plan::proto::{ExtensionPlanFromProto, ProtoDecoderRegistry};
+ ///
+ /// let mut registry = ProtoDecoderRegistry::new();
+ /// let _ = (FilterExec::try_from_proto, &mut registry);
+ /// ```
+ fn register_proto(registry: &mut ProtoDecoderRegistry) -> Result<()> {
+ registry.register_decoder::<dyn ExecutionPlan, Self, ExecutionPlanDecoder>(
+ Self::NAME,
+ Self::try_from_proto,
+ )
+ }
}
/// How this facade stores a decoder in the shared registry: a function pointer
/// to the monomorphized [`ExtensionPlanFromProto::try_from_proto`].
///
/// Deliberately private, and the same type on both the
-/// [`register_execution_plan`] and the [`decode_execution_plan`] side.
+/// [`ExtensionPlanFromProto::register_proto`] and the [`decode_execution_plan`] side.
/// [`ExtensionPlanFromProto`] is the public contract and
/// [`decode_execution_plan`] (hidden, internal to DataFusion) is the only way
/// to invoke one, so this can
@@ -115,47 +155,6 @@ pub trait ExtensionPlanFromProto: ExecutionPlan + Sized {
type ExecutionPlanDecoder =
fn(&PhysicalPlanNode, &ExecutionPlanDecodeCtx<'_>) -> Result<Arc<dyn ExecutionPlan>>;
-/// Register `T` in `registry` under its [`ExtensionPlanFromProto::NAME`].
-///
-/// Registering the same type twice is a no-op; a *different* type under a
-/// taken name is an error, so collisions surface here rather than as a wrong
-/// decode. The name is scoped to `dyn ExecutionPlan`, so an expression or a
-/// data source may use the same name in the same registry.
-///
-/// Add to the session's registry with `SessionConfig::update_extension`, so
-/// that the decoders other libraries added stay registered. See
-/// [`ProtoDecoderRegistry`].
-///
-/// A built-in plan is not an extension and cannot be registered:
-///
-/// ```compile_fail
-/// use datafusion_physical_plan::filter::FilterExec;
-/// use datafusion_physical_plan::proto::{ProtoDecoderRegistry, register_execution_plan};
-///
-/// let mut registry = ProtoDecoderRegistry::new();
-/// register_execution_plan::<FilterExec>(&mut registry).unwrap();
-/// ```
-///
-/// The same imports without that call compile, so the failure above is the
-/// missing bound and not a bad path:
-///
-/// ```
-/// use datafusion_physical_plan::filter::FilterExec;
-/// #[allow(unused_imports)]
-/// use datafusion_physical_plan::proto::{ProtoDecoderRegistry, register_execution_plan};
-///
-/// let mut registry = ProtoDecoderRegistry::new();
-/// let _ = (FilterExec::try_from_proto, &mut registry);
-/// ```
-pub fn register_execution_plan<T: ExtensionPlanFromProto>(
- registry: &mut ProtoDecoderRegistry,
-) -> Result<()> {
- registry.register_decoder::<dyn ExecutionPlan, T, ExecutionPlanDecoder>(
- T::NAME,
- T::try_from_proto,
- )
-}
-
/// Decode `node` with the extension plan decoder registered under the name
/// `node` carries.
///
@@ -216,13 +215,9 @@ mod tests {
use datafusion_execution::config::SessionConfig;
let mut config = SessionConfig::new();
- config.update_extension(|registry: &mut ProtoDecoderRegistry| {
- register_execution_plan::<RegisteredExec>(registry)
- })?;
+ config.update_extension(RegisteredExec::register_proto)?;
let before = config.clone();
- config.update_extension(|registry: &mut ProtoDecoderRegistry| {
- register_execution_plan::<OtherRegisteredExec>(registry)
- })?;
+ config.update_extension(OtherRegisteredExec::register_proto)?;
let names = |config: &SessionConfig| {
let registry = config.get_extension::<ProtoDecoderRegistry>().unwrap();
@@ -245,8 +240,8 @@ mod tests {
#[test]
fn register_is_idempotent_for_the_same_type() -> Result<()> {
let mut registry = ProtoDecoderRegistry::new();
- register_execution_plan::<RegisteredExec>(&mut registry)?;
- register_execution_plan::<RegisteredExec>(&mut registry)?;
+ RegisteredExec::register_proto(&mut registry)?;
+ RegisteredExec::register_proto(&mut registry)?;
assert_eq!(execution_plan_names(&registry).count(), 1);
assert!(execution_plan_names(&registry).any(|name| name == RegisteredExec::NAME));
@@ -256,11 +251,11 @@ mod tests {
#[test]
fn register_rejects_a_name_collision_between_types() -> Result<()> {
let mut registry = ProtoDecoderRegistry::new();
- register_execution_plan::<RegisteredExec>(&mut registry)?;
+ RegisteredExec::register_proto(&mut registry)?;
// A distinct type declaring the same `NAME`: the collision two
// independent crates could hit, caught at registration.
- let err = register_execution_plan::<RegisteredExecClone>(&mut registry)
+ let err = RegisteredExecClone::register_proto(&mut registry)
.expect_err("colliding registration must fail");
assert!(
err.to_string().contains(RegisteredExec::NAME)
@@ -273,8 +268,8 @@ mod tests {
#[test]
fn register_keeps_distinct_names_apart() -> Result<()> {
let mut registry = ProtoDecoderRegistry::new();
- register_execution_plan::<RegisteredExec>(&mut registry)?;
- register_execution_plan::<OtherRegisteredExec>(&mut registry)?;
+ RegisteredExec::register_proto(&mut registry)?;
+ OtherRegisteredExec::register_proto(&mut registry)?;
assert_eq!(execution_plan_names(&registry).count(), 2);
let mut names: Vec<&str> = execution_plan_names(&registry).collect();
@@ -287,7 +282,7 @@ mod tests {
#[test]
fn register_rejects_an_empty_plan_name() {
let mut registry = ProtoDecoderRegistry::new();
- let err = register_execution_plan::<UnnamedExec>(&mut registry)
+ let err = UnnamedExec::register_proto(&mut registry)
.expect_err("an empty NAME must fail");
assert!(
err.to_string().contains("empty name"),
@@ -305,7 +300,7 @@ mod tests {
registry.register_decoder::<dyn OtherKind, u8, u32>(RegisteredExec::NAME, 7)?;
assert!(execution_plan_names(&registry).next().is_none());
- register_execution_plan::<RegisteredExec>(&mut registry)?;
+ RegisteredExec::register_proto(&mut registry)?;
assert_eq!(
execution_plan_names(&registry).collect::<Vec<_>>(),
vec![RegisteredExec::NAME]
diff --git a/datafusion/proto-models/src/registry.rs b/datafusion/proto-models/src/registry.rs
index 8db237bd70..aaea5541bb 100644
--- a/datafusion/proto-models/src/registry.rs
+++ b/datafusion/proto-models/src/registry.rs
@@ -64,9 +64,8 @@ struct Registered {
/// ```rust,ignore
/// // in each library
/// pub fn register(config: &mut SessionConfig) -> Result<()> {
-/// config.update_extension(|registry: &mut ProtoDecoderRegistry| {
-/// register_execution_plan::<MyExec>(registry)
-/// })
+/// config.update_extension(MyExec::register_proto)?;
+/// config.update_extension(MyExpr::register_proto)
/// }
///
/// // in the application that owns the session
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index f72a9d8d06..06e8a42d6f 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -1563,7 +1563,7 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {
///
/// Expressions can instead name themselves on the wire and be decoded by a
/// per-type decoder registered on the session — see
- /// [`ExtensionExprFromProto`] and [`register_physical_expr`]. That path
+ /// [`ExtensionExprFromProto`] and [`ExtensionExprFromProto::register_proto`]. That path
/// resolves by name rather than by codec registration order, so two crates
/// claiming the same name collide at registration. This method stays the
/// fallback for everything unnamed or unregistered.
@@ -1581,7 +1581,7 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {
///
/// [`parse_physical_expr`]: crate::physical_plan::from_proto::parse_physical_expr
/// [`ExtensionExprFromProto`]: datafusion_physical_plan::proto::ExtensionExprFromProto
- /// [`register_physical_expr`]: datafusion_physical_plan::proto::register_physical_expr
+ /// [`ExtensionExprFromProto::register_proto`]: datafusion_physical_plan::proto::ExtensionExprFromProto::register_proto
fn try_decode_expr(
&self,
_buf: &[u8],
diff --git a/datafusion/proto/tests/cases/plans/expr_registry.rs b/datafusion/proto/tests/cases/plans/expr_registry.rs
index 87da5b88ae..3b411a85b5 100644
--- a/datafusion/proto/tests/cases/plans/expr_registry.rs
+++ b/datafusion/proto/tests/cases/plans/expr_registry.rs
@@ -34,7 +34,7 @@ use datafusion::physical_plan::proto::PhysicalExprEncodeCtx;
use datafusion::physical_plan::proto::{
ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionExprFromProto,
ExtensionPlanFromProto, ProtoDecoderRegistry, extension_expr_node,
- physical_expr_names, register_execution_plan, register_physical_expr,
+ physical_expr_names,
};
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, PlanProperties, SendableRecordBatchStream,
@@ -504,7 +504,7 @@ fn registry_decodes_named_extension_expr() -> Result<()> {
let session = session_with(
SessionConfig::new().with_batch_size(1234),
- register_physical_expr::<TagExpr>,
+ TagExpr::register_proto,
)?;
let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;
let decoded = decoded.downcast_ref::<TagExpr>().unwrap();
@@ -523,8 +523,7 @@ fn unnamed_or_unregistered_falls_back_to_codec() -> Result<()> {
set_expr_name(&mut unnamed, None);
let empty = SessionContext::new();
- let registered =
- session_with(SessionConfig::new(), register_physical_expr::<TagExpr>)?;
+ let registered = session_with(SessionConfig::new(), TagExpr::register_proto)?;
for (node, session) in [(&named, &empty), (&unnamed, &registered)] {
let decoded = from_wire(node, session, &TestCodec::Tag)?;
let decoded = decoded.downcast_ref::<TagExpr>().unwrap();
@@ -540,10 +539,8 @@ fn unnamed_or_unregistered_falls_back_to_codec() -> Result<()> {
#[test]
fn codec_failure_keeps_its_error_and_names_the_missing_registration() -> Result<()> {
let named = to_wire(&tag_expr()?, &TestCodec::Refusing)?;
- let session = session_with(
- SessionConfig::new(),
- register_physical_expr::<TaggedExpr<Other>>,
- )?;
+ let session =
+ session_with(SessionConfig::new(), <TaggedExpr<Other>>::register_proto)?;
let err = from_wire(&named, &session, &TestCodec::Exhausted).unwrap_err();
assert!(
@@ -576,17 +573,19 @@ fn codec_failure_keeps_its_error_and_names_the_missing_registration() -> Result<
#[test]
fn registration_rejects_collisions_and_empty_names() -> Result<()> {
let mut registry = ProtoDecoderRegistry::new();
- register_physical_expr::<TagExpr>(&mut registry)?;
- register_physical_expr::<TagExpr>(&mut registry)?;
+ TagExpr::register_proto(&mut registry)?;
+ TagExpr::register_proto(&mut registry)?;
- let err = register_physical_expr::<TaggedExpr<TagClone>>(&mut registry)
- .unwrap_err()
- .to_string();
+ let err =
+ <TaggedExpr<TagClone> as ExtensionExprFromProto>::register_proto(&mut registry)
+ .unwrap_err()
+ .to_string();
assert!(err.contains("already registered"), "{err}");
- let err = register_physical_expr::<TaggedExpr<Unnamed>>(&mut registry)
- .unwrap_err()
- .to_string();
+ let err =
+ <TaggedExpr<Unnamed> as ExtensionExprFromProto>::register_proto(&mut registry)
+ .unwrap_err()
+ .to_string();
assert!(err.contains("empty name"), "{err}");
assert_eq!(
@@ -604,7 +603,7 @@ fn nested_extension_exprs_both_decode_through_the_registry() -> Result<()> {
Arc::new(TagExpr::new("inner", col("a", &test_schema())?));
let outer: Arc<dyn PhysicalExpr> = Arc::new(TagExpr::new("outer", inner));
let node = to_wire(&outer, &TestCodec::Refusing)?;
- let session = session_with(SessionConfig::new(), register_physical_expr::<TagExpr>)?;
+ let session = session_with(SessionConfig::new(), TagExpr::register_proto)?;
let decoded = from_wire(&node, &session, &TestCodec::Refusing)?;
let outer = decoded.downcast_ref::<TagExpr>().unwrap();
@@ -619,10 +618,8 @@ fn nested_extension_exprs_both_decode_through_the_registry() -> Result<()> {
#[test]
fn a_failing_registry_decoder_does_not_fall_through_to_the_codec() -> Result<()> {
let node = to_wire(&tag_expr()?, &TestCodec::Tag)?;
- let session = session_with(
- SessionConfig::new(),
- register_physical_expr::<TaggedExpr<Failing>>,
- )?;
+ let session =
+ session_with(SessionConfig::new(), <TaggedExpr<Failing>>::register_proto)?;
let err = from_wire(&node, &session, &TestCodec::Tag).unwrap_err();
assert!(
err.to_string()
@@ -637,10 +634,7 @@ fn a_failing_registry_decoder_does_not_fall_through_to_the_codec() -> Result<()>
/// session.
#[test]
fn extension_expr_resolves_scalar_and_aggregate_udfs_from_the_session() -> Result<()> {
- let session = session_with(
- SessionConfig::new(),
- register_physical_expr::<FunctionRefExpr>,
- )?;
+ let session = session_with(SessionConfig::new(), FunctionRefExpr::register_proto)?;
session.register_udf(create_udf(
"my_add_one",
vec![DataType::Int64],
@@ -674,10 +668,7 @@ fn extension_expr_resolves_scalar_and_aggregate_udfs_from_the_session() -> Resul
child: col("a", &test_schema())?,
});
let node = to_wire(&expr, &TestCodec::Refusing)?;
- let reader = session_with(
- SessionConfig::new(),
- register_physical_expr::<FunctionRefExpr>,
- )?;
+ let reader = session_with(SessionConfig::new(), FunctionRefExpr::register_proto)?;
let err = from_wire(&node, &reader, &TestCodec::Refusing).unwrap_err();
assert!(err.to_string().contains("my_add_one"), "{err}");
Ok(())
@@ -699,7 +690,7 @@ fn registry_expr_survives_a_whole_plan_roundtrip() -> Result<()> {
let session = session_with(
SessionConfig::new().with_batch_size(4096),
- register_physical_expr::<TagExpr>,
+ TagExpr::register_proto,
)?;
let decoded = physical_plan_from_bytes_with_extension_codec(
&bytes,
@@ -851,8 +842,7 @@ fn registry_expr_keeps_its_expression_id_for_deduplication() -> Result<()> {
"the serializer must stamp the expression id"
);
- let session =
- session_with(SessionConfig::new(), register_physical_expr::<IdentityExpr>)?;
+ let session = session_with(SessionConfig::new(), IdentityExpr::register_proto)?;
let task_ctx = session.task_ctx();
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
let decoded = converter.proto_to_physical_expr(&node, &schema, &decode_ctx)?;
@@ -963,8 +953,8 @@ fn registry_plan_and_expr_with_the_same_name_roundtrip_together() -> Result<()>
let bytes = physical_plan_to_bytes_with_extension_codec(plan, &TestCodec::Refusing)?;
let session = session_with(SessionConfig::new().with_batch_size(2048), |registry| {
- register_execution_plan::<SameNameExec>(registry)?;
- register_physical_expr::<TagExpr>(registry)
+ SameNameExec::register_proto(registry)?;
+ TagExpr::register_proto(registry)
})?;
let decoded = physical_plan_from_bytes_with_extension_codec(
&bytes,
diff --git a/datafusion/proto/tests/cases/plans/extensions.rs b/datafusion/proto/tests/cases/plans/extensions.rs
index ded9cf4d0d..eaf94b238d 100644
--- a/datafusion/proto/tests/cases/plans/extensions.rs
+++ b/datafusion/proto/tests/cases/plans/extensions.rs
@@ -32,7 +32,7 @@ use datafusion::execution::TaskContext;
use datafusion::physical_plan::empty::EmptyExec;
use datafusion::physical_plan::proto::{
ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionPlanFromProto,
- ProtoDecoderRegistry, register_execution_plan,
+ ProtoDecoderRegistry,
};
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,
@@ -436,10 +436,8 @@ fn extension_of(node: &PhysicalPlanNode) -> &PhysicalExtensionNode {
#[test]
fn extension_plan_round_trips_with_no_codec_at_all() -> Result<()> {
- let config =
- session_with(&["worker-1", "worker-2"]).with_extension(registry(|registry| {
- register_execution_plan::<ShuffleExec>(registry)
- })?);
+ let config = session_with(&["worker-1", "worker-2"])
+ .with_extension(registry(ShuffleExec::register_proto)?);
let ctx = SessionContext::new_with_config(config);
// The default codec decodes nothing: whatever survives came from the
// registry.
@@ -474,7 +472,7 @@ fn extension_plan_decode_reads_session_state() -> Result<()> {
let codec = DefaultPhysicalExtensionCodec {};
let writer = SessionContext::new_with_config(
session_with(&["writer-only"])
- .with_extension(registry(register_execution_plan::<ShuffleExec>)?),
+ .with_extension(registry(ShuffleExec::register_proto)?),
);
let node = encode(
Arc::new(ShuffleExec {
@@ -493,9 +491,8 @@ fn extension_plan_decode_reads_session_state() -> Result<()> {
// Decoding on a *different* session rebuilds the pool from that session,
// proving the pool never rode the wire.
let reader = SessionContext::new_with_config(
- session_with(&["reader-a", "reader-b"]).with_extension(registry(|registry| {
- register_execution_plan::<ShuffleExec>(registry)
- })?),
+ session_with(&["reader-a", "reader-b"])
+ .with_extension(registry(ShuffleExec::register_proto)?),
);
let decoded = decode(&node, &reader, &codec)?;
let decoded = decoded
@@ -508,8 +505,8 @@ fn extension_plan_decode_reads_session_state() -> Result<()> {
#[test]
fn independent_extension_plans_coexist_without_a_composed_codec() -> Result<()> {
let config = session_with(&["worker-1"]).with_extension(registry(|registry| {
- register_execution_plan::<ShuffleExec>(registry)?;
- register_execution_plan::<SamplerExec>(registry)
+ ShuffleExec::register_proto(registry)?;
+ SamplerExec::register_proto(registry)
})?);
let ctx = SessionContext::new_with_config(config);
let codec = DefaultPhysicalExtensionCodec {};
@@ -545,8 +542,7 @@ fn codec_encoded_extension_plans_are_unaffected() -> Result<()> {
// as before the registry existed. The session registers an unrelated plan
// to prove the registry does not interfere.
let ctx = SessionContext::new_with_config(
- SessionConfig::new()
- .with_extension(registry(register_execution_plan::<SamplerExec>)?),
+ SessionConfig::new().with_extension(registry(SamplerExec::register_proto)?),
);
let codec = LegacyCodec {};
@@ -564,9 +560,8 @@ fn codec_encoded_extension_plans_are_unaffected() -> Result<()> {
#[test]
fn a_registered_name_wins_over_a_codec_that_claims_everything() -> Result<()> {
- let config = session_with(&["worker-1"]).with_extension(registry(|registry| {
- register_execution_plan::<ShuffleExec>(registry)
- })?);
+ let config = session_with(&["worker-1"])
+ .with_extension(registry(ShuffleExec::register_proto)?);
let ctx = SessionContext::new_with_config(config);
// This codec would decode *any* payload into a `LegacyExec`; the name must
// route around it.
@@ -594,7 +589,7 @@ fn a_named_plan_the_session_does_not_know_reports_the_missing_registration() ->
let codec = DefaultPhysicalExtensionCodec {};
let writer = SessionContext::new_with_config(
session_with(&["worker-1"])
- .with_extension(registry(register_execution_plan::<ShuffleExec>)?),
+ .with_extension(registry(ShuffleExec::register_proto)?),
);
let node = encode(
Arc::new(ShuffleExec {
@@ -614,8 +609,7 @@ fn a_named_plan_the_session_does_not_know_reports_the_missing_registration() ->
// and the error has to point at the missing registration rather than at the
// codec's generic complaint.
let reader = SessionContext::new_with_config(
- SessionConfig::new()
- .with_extension(registry(register_execution_plan::<SamplerExec>)?),
+ SessionConfig::new().with_extension(registry(SamplerExec::register_proto)?),
);
let err = decode(&node, &reader, &codec)
.expect_err("an unregistered plan name must fail to decode");
@@ -639,9 +633,8 @@ fn a_registered_decoder_failing_does_not_fall_back_to_the_codec() -> Result<()>
// the codec after a registered decoder failed would resurrect exactly the
// wrong decode this replaces: `GreedyCodec` would happily turn a corrupt
// ShuffleExec into a LegacyExec.
- let config = session_with(&["worker-1"]).with_extension(registry(|registry| {
- register_execution_plan::<ShuffleExec>(registry)
- })?);
+ let config = session_with(&["worker-1"])
+ .with_extension(registry(ShuffleExec::register_proto)?);
let ctx = SessionContext::new_with_config(config);
let node = PhysicalPlanNode {
@@ -665,9 +658,8 @@ fn a_registered_decoder_failing_does_not_fall_back_to_the_codec() -> Result<()>
#[test]
fn the_plan_name_survives_the_wire_and_is_absent_for_old_writers() -> Result<()> {
- let config = session_with(&["worker-1"]).with_extension(registry(|registry| {
- register_execution_plan::<ShuffleExec>(registry)
- })?);
+ let config = session_with(&["worker-1"])
+ .with_extension(registry(ShuffleExec::register_proto)?);
let ctx = SessionContext::new_with_config(config);
let codec = DefaultPhysicalExtensionCodec {};
@@ -720,9 +712,8 @@ fn a_named_plan_can_still_be_decoded_by_a_codec() -> Result<()> {
// when the session has a registry with other plans in it: a codec that
// understands the payload still decodes it, which is what keeps a mixed
// fleet (some nodes upgraded, some not) working.
- let config = SessionConfig::new().with_extension(registry(|registry| {
- register_execution_plan::<SamplerExec>(registry)
- })?);
+ let config =
+ SessionConfig::new().with_extension(registry(SamplerExec::register_proto)?);
let ctx = SessionContext::new_with_config(config);
let node = PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Extension(PhysicalExtensionNode {
@@ -744,8 +735,7 @@ fn the_codecs_own_error_survives_the_missing_registration_hint() -> Result<()> {
// registration, and a caller matching on the error kind must still see the
// kind the codec chose.
let ctx = SessionContext::new_with_config(
- SessionConfig::new()
- .with_extension(registry(register_execution_plan::<SamplerExec>)?),
+ SessionConfig::new().with_extension(registry(SamplerExec::register_proto)?),
);
let node = PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Extension(PhysicalExtensionNode {
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index e927d3ec4a..ba87076b3d 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1154,7 +1154,6 @@ let plan = provider
```rust,ignore
use datafusion::physical_plan::proto::{
ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionPlanFromProto,
- ProtoDecoderRegistry, register_execution_plan,
};
impl ExecutionPlan for MyExec {
@@ -1193,12 +1192,10 @@ impl ExtensionPlanFromProto for MyExec {
// On every session that decodes the plan — for a distributed engine, the
// workers as well as the coordinator.
let mut config = SessionConfig::new();
-config.update_extension(|registry: &mut ProtoDecoderRegistry| {
- register_execution_plan::<MyExec>(registry)
-})?;
+config.update_extension(MyExec::register_proto)?;
```
-Only extension plans implement `ExtensionPlanFromProto`, and `register_execution_plan` takes that trait as its bound. A built-in plan has a wire variant of its own and keeps its inherent `try_from_proto`, so `register_execution_plan::<FilterExec>(&mut registry)` does not compile.
+Only extension plans implement `ExtensionPlanFromProto`, and registration is the trait's provided `register_proto` method. A built-in plan has a wire variant of its own and keeps its inherent `try_from_proto`, so `FilterExec::register_proto(&mut registry)` does not compile.
`ProtoDecoderRegistry` is shared by every kind of extension DataFusion can decode, not only plans. Its key is the trait plus the name, so a plan and an expression may use the same name, and one session carries one registry rather than one per kind.
@@ -1207,9 +1204,7 @@ A session carries one registry. Each library adds its decoders to it with `Sessi
```rust,ignore
// In each library that ships extension types:
pub fn register(config: &mut SessionConfig) -> Result<()> {
- config.update_extension(|registry: &mut ProtoDecoderRegistry| {
- register_execution_plan::<MyExec>(registry)
- })
+ config.update_extension(MyExec::register_proto)
}
// In the application that owns the session:
@@ -1218,7 +1213,7 @@ shuffle_lib::register(&mut config)?;
sampling_lib::register(&mut config)?;
```
-The libraries need to know nothing about each other, and the order they are called in does not matter. Dispatch is by name, and a real name collision is an error from `register_execution_plan` rather than a wrong decode later.
+The libraries need to know nothing about each other, and the order they are called in does not matter. Dispatch is by name, and a real name collision is an error from `register` rather than a wrong decode later.
A registered plan needs no `PhysicalExtensionCodec` at all, and two crates' plans coexist on one session without a `ComposedPhysicalExtensionCodec`.
@@ -1259,7 +1254,7 @@ See [issue #24625](https://github.com/apache/datafusion/issues/24625) for detail
use datafusion::execution::TaskContext;
use datafusion::physical_plan::proto::{
ExtensionExprFromProto, PhysicalExprDecodeCtx, PhysicalExprEncodeCtx,
- ProtoDecoderRegistry, extension_expr_node, register_physical_expr,
+ extension_expr_node,
};
impl PhysicalExpr for MyExpr {
@@ -1291,12 +1286,11 @@ impl ExtensionExprFromProto for MyExpr {
}
}
-let mut registry = ProtoDecoderRegistry::new();
-register_physical_expr::<MyExpr>(&mut registry)?;
-let config = SessionConfig::new().with_extension(Arc::new(registry));
+let mut config = SessionConfig::new();
+config.update_extension(MyExpr::register_proto)?;
```
-Only extension expressions can be registered. `register_physical_expr::<Column>(..)` does not compile.
+Only extension expressions can be registered. `Column::register_proto(..)` does not compile.
If a registered decoder fails, the decode fails. It does not fall back to the codec, because the codec could decode a payload that it did not write. Changing an expression to write its own payload is therefore a wire-format change for that expression: register the type on every reader before writers start to use it.