Apache DataFusion SQL Query Engine
feat: Add `deregister_object_store` (#17999)
Jonathan Chen531af8e43ae3
diff --git a/datafusion-cli/src/object_storage/instrumented.rs b/datafusion-cli/src/object_storage/instrumented.rsindex 4f8626888e..ebf9899f26 100644--- a/datafusion-cli/src/object_storage/instrumented.rs+++ b/datafusion-cli/src/object_storage/instrumented.rs@@ -694,6 +694,13 @@ impl ObjectStoreRegistry for InstrumentedObjectStoreRegistry {self.inner.register_store(url, instrumented)}+ fn deregister_store(+ &self,+ url: &Url,+ ) -> datafusion::common::Result<Arc<dyn ObjectStore>> {+ self.inner.deregister_store(url)+ }+fn get_store(&self, url: &Url) -> datafusion::common::Result<Arc<dyn ObjectStore>> {self.inner.get_store(url)}diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex a8148b8049..eace5610e1 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -505,6 +505,13 @@ impl SessionContext {self.runtime_env().register_object_store(url, object_store)}+ /// Deregisters an [`ObjectStore`] associated with the specific URL prefix.+ ///+ /// See [`RuntimeEnv::deregister_object_store`] for more details.+ pub fn deregister_object_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {+ self.runtime_env().deregister_object_store(url)+ }+/// Registers the [`RecordBatch`] as the specified table namepub fn register_batch(&self,diff --git a/datafusion/execution/src/object_store.rs b/datafusion/execution/src/object_store.rsindex ef83128ac6..aedee7d444 100644--- a/datafusion/execution/src/object_store.rs+++ b/datafusion/execution/src/object_store.rs@@ -20,7 +20,9 @@//! and query data inside these systems.use dashmap::DashMap;-use datafusion_common::{exec_err, internal_datafusion_err, DataFusionError, Result};+use datafusion_common::{+ exec_err, internal_datafusion_err, not_impl_err, DataFusionError, Result,+};#[cfg(not(target_arch = "wasm32"))]use object_store::local::LocalFileSystem;use object_store::ObjectStore;@@ -154,6 +156,13 @@ pub trait ObjectStoreRegistry: Send + Sync + std::fmt::Debug + 'static {store: Arc<dyn ObjectStore>,) -> Option<Arc<dyn ObjectStore>>;+ /// Deregister the store previously registered with the same key. Returns the+ /// deregistered store if it existed.+ #[allow(unused_variables)]+ fn deregister_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {+ not_impl_err!("ObjectStoreRegistry::deregister_store is not implemented for this ObjectStoreRegistry")+ }+/// Get a suitable store for the provided URL. For example:////// - URL with scheme `file:///` or no scheme will return the default LocalFS store@@ -230,6 +239,17 @@ impl ObjectStoreRegistry for DefaultObjectStoreRegistry {self.object_stores.insert(s, store)}+ fn deregister_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {+ let s = get_url_key(url);+ let (_, object_store) = self.object_stores+ .remove(&s)+ .ok_or_else(|| {+ internal_datafusion_err!("Failed to deregister object store. No suitable object store found for {url}. See `RuntimeEnv::register_object_store`")+ })?;++ Ok(object_store)+ }+fn get_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {let s = get_url_key(url);self.object_storesdiff --git a/datafusion/execution/src/runtime_env.rs b/datafusion/execution/src/runtime_env.rsindex db045a8b7e..b0d0a966b7 100644--- a/datafusion/execution/src/runtime_env.rs+++ b/datafusion/execution/src/runtime_env.rs@@ -114,8 +114,6 @@ impl RuntimeEnv {/// ```////// # Example: Register remote URL object store like [Github](https://github.com)- ///- ////// ```/// # use std::sync::Arc;/// # use url::Url;@@ -141,6 +139,12 @@ impl RuntimeEnv {self.object_store_registry.register_store(url, object_store)}+ /// Deregisters a custom `ObjectStore` previously registered for a specific url.+ /// See [`ObjectStoreRegistry::deregister_store`] for more details.+ pub fn deregister_object_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {+ self.object_store_registry.deregister_store(url)+ }+/// Retrieves a `ObjectStore` instance for a url by consulting the/// registry. See [`ObjectStoreRegistry::get_store`] for more/// details.