Apache DataFusion SQL Query Engine
feat: add `SessionConfig::update_extension` so libraries can share one `ProtoDecoderRegistry` (#26353)
Adrian Garcia Badaracco3bc3d0bb822f
diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rsindex dbfd9bf1d8..a157ae2175 100644--- a/datafusion/execution/src/config.rs+++ b/datafusion/execution/src/config.rs@@ -681,6 +681,39 @@ impl SessionConfig {{self.extensions.get_arc::<T>()}++ /// Change the extension of type `T` in place, and return what `f` returns.+ ///+ /// If there is no extension of type `T`, `f` gets `T::default()`, which is+ /// then attached. If the extension is shared with another config, it is+ /// cloned first, so the other config does not change.+ ///+ /// Use this, not [`with_extension`](Self::with_extension), when more than+ /// one part of an application adds to the same extension. `with_extension`+ /// replaces the extension, so it discards what other parts added.+ ///+ /// # Example+ /// ```+ /// use datafusion_execution::config::SessionConfig;+ ///+ /// #[derive(Clone, Default)]+ /// struct Names(Vec<&'static str>);+ ///+ /// let mut cfg = SessionConfig::default();+ /// cfg.update_extension(|names: &mut Names| names.0.push("a"));+ /// cfg.update_extension(|names: &mut Names| names.0.push("b"));+ ///+ /// assert_eq!(cfg.get_extension::<Names>().unwrap().0, ["a", "b"]);+ /// ```+ pub fn update_extension<T, R>(&mut self, f: impl FnOnce(&mut T) -> R) -> R+ where+ T: Clone + Default + Send + Sync + 'static,+ {+ let mut ext = self.get_extension::<T>().unwrap_or_default();+ let result = f(Arc::make_mut(&mut ext));+ self.set_extension(ext);+ result+ }}impl From<ConfigOptions> for SessionConfig {diff --git a/datafusion/physical-plan/src/proto/registry.rs b/datafusion/physical-plan/src/proto/registry.rsindex 4f12f89e6f..4fa9ddda2e 100644--- a/datafusion/physical-plan/src/proto/registry.rs+++ b/datafusion/physical-plan/src/proto/registry.rs@@ -122,9 +122,9 @@ type ExecutionPlanDecoder =/// decode. The name is scoped to `dyn ExecutionPlan`, so an expression or a/// data source may use the same name in the same registry.///-/// The application that owns the session builds the registry and attaches it-/// once with `SessionConfig::with_extension`; a library exposes a function-/// that fills a registry it is handed. See [`ProtoDecoderRegistry`].+/// Add to the session's registry with `SessionConfig::update_extension`, so+/// that the decoders other libraries added stay registered. See+/// [`ProtoDecoderRegistry`].////// A built-in plan is not an extension and cannot be registered:///@@ -208,6 +208,40 @@ mod tests {OtherRegisteredExec, RegisteredExec, RegisteredExecClone, UnnamedExec,};+ /// Two libraries add to one session's registry, and neither discards the+ /// other's decoders. A config cloned before the second one adds is not+ /// changed.+ #[test]+ fn libraries_add_to_one_session_registry() -> Result<()> {+ use datafusion_execution::config::SessionConfig;++ let mut config = SessionConfig::new();+ config.update_extension(|registry: &mut ProtoDecoderRegistry| {+ register_execution_plan::<RegisteredExec>(registry)+ })?;+ let before = config.clone();+ config.update_extension(|registry: &mut ProtoDecoderRegistry| {+ register_execution_plan::<OtherRegisteredExec>(registry)+ })?;++ let names = |config: &SessionConfig| {+ let registry = config.get_extension::<ProtoDecoderRegistry>().unwrap();+ let mut names: Vec<String> = execution_plan_names(®istry)+ .map(str::to_string)+ .collect();+ names.sort();+ names+ };+ let mut expected = vec![+ RegisteredExec::NAME.to_string(),+ OtherRegisteredExec::NAME.to_string(),+ ];+ expected.sort();+ assert_eq!(names(&config), expected);+ assert_eq!(names(&before), [RegisteredExec::NAME]);+ Ok(())+ }+#[test]fn register_is_idempotent_for_the_same_type() -> Result<()> {let mut registry = ProtoDecoderRegistry::new();diff --git a/datafusion/proto-models/src/registry.rs b/datafusion/proto-models/src/registry.rsindex e2852e57c6..8db237bd70 100644--- a/datafusion/proto-models/src/registry.rs+++ b/datafusion/proto-models/src/registry.rs@@ -56,26 +56,28 @@ struct Registered {/// A store of extension decoders, keyed by the trait they produce and the name/// the encoder wrote on the wire.///-/// Build one, let every library that ships extension types register into it,-/// and attach it to the decoding session with `SessionConfig::with_extension`.-/// A session carries at most one, and attaching another replaces it, so the-/// application that owns the session is the one that builds it. A library-/// exposes a function that fills a registry it is handed:+/// A session carries one registry, as a `SessionConfig` extension. Every+/// library that ships extension types adds its decoders to that registry with+/// `SessionConfig::update_extension`, which creates the registry if there is+/// none:////// ```rust,ignore/// // in each library-/// pub fn register(registry: &mut ProtoDecoderRegistry) -> Result<()> {-/// register_execution_plan::<MyExec>(registry)?;-/// register_physical_expr::<MyExpr>(registry)+/// pub fn register(config: &mut SessionConfig) -> Result<()> {+/// config.update_extension(|registry: &mut ProtoDecoderRegistry| {+/// register_execution_plan::<MyExec>(registry)+/// })/// }////// // in the application that owns the session-/// let mut registry = ProtoDecoderRegistry::new();-/// lib_one::register(&mut registry)?;-/// lib_two::register(&mut registry)?;-/// let config = SessionConfig::new().with_extension(Arc::new(registry));+/// let mut config = SessionConfig::new();+/// lib_one::register(&mut config)?;+/// lib_two::register(&mut config)?;/// ```///+/// Do not attach a registry with `SessionConfig::with_extension`: it replaces+/// the registry, so it discards the decoders that other libraries added.+////// The libraries need to know nothing about each other, and the order they are/// called in does not matter: lookup is by name, and a real collision is an/// error from [`register_decoder`](Self::register_decoder).diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 1e8d9e2daa..2ec01c9947 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -1170,28 +1170,30 @@ impl ExtensionPlanFromProto for MyExec {// On every session that decodes the plan — for a distributed engine, the// workers as well as the coordinator.-let mut registry = ProtoDecoderRegistry::new();-register_execution_plan::<MyExec>(&mut registry)?;-let config = SessionConfig::new().with_extension(Arc::new(registry));+let mut config = SessionConfig::new();+config.update_extension(|registry: &mut ProtoDecoderRegistry| {+ register_execution_plan::<MyExec>(registry)+})?;```Only extension plans implement `ExtensionPlanFromProto`, and `register_execution_plan` takes that trait as its bound. A built-in plan has a wire variant of its own and keeps its inherent `try_from_proto`, so `register_execution_plan::<FilterExec>(&mut registry)` does not compile.`ProtoDecoderRegistry` is shared by every kind of extension DataFusion can decode, not only plans. Its key is the trait plus the name, so a plan and an expression may use the same name, and one session carries one registry rather than one per kind.-The application that owns the session builds the registry. A session carries at most one, and `with_extension` replaces what is there, so a library must not attach a registry of its own: it would silently discard another library's. A library exposes a function that fills a registry it is handed, and the application composes them:+A session carries one registry. Each library adds its decoders to it with `SessionConfig::update_extension`, which creates the registry if there is none. Do not attach a registry with `with_extension`: it replaces the registry, so it discards the decoders that other libraries added.```rust,ignore// In each library that ships extension types:-pub fn register(registry: &mut ProtoDecoderRegistry) -> Result<()> {- register_execution_plan::<MyExec>(registry)+pub fn register(config: &mut SessionConfig) -> Result<()> {+ config.update_extension(|registry: &mut ProtoDecoderRegistry| {+ register_execution_plan::<MyExec>(registry)+ })}// In the application that owns the session:-let mut registry = ProtoDecoderRegistry::new();-shuffle_lib::register(&mut registry)?;-sampling_lib::register(&mut registry)?;-let config = SessionConfig::new().with_extension(Arc::new(registry));+let mut config = SessionConfig::new();+shuffle_lib::register(&mut config)?;+sampling_lib::register(&mut config)?;```The libraries need to know nothing about each other, and the order they are called in does not matter. Dispatch is by name, and a real name collision is an error from `register_execution_plan` rather than a wrong decode later.