Skip to content

Apache DataFusion SQL Query Engine

feat: Add `deregister_object_store` (#17999)

Jonathan Chen531af8e43ae3
diff --git a/datafusion-cli/src/object_storage/instrumented.rs b/datafusion-cli/src/object_storage/instrumented.rs
index 4f8626888e..ebf9899f26 100644
--- a/datafusion-cli/src/object_storage/instrumented.rs
+++ b/datafusion-cli/src/object_storage/instrumented.rs
@@ -694,6 +694,13 @@ impl ObjectStoreRegistry for InstrumentedObjectStoreRegistry {
self.inner.register_store(url, instrumented)
}
+ fn deregister_store(
+ &self,
+ url: &Url,
+ ) -> datafusion::common::Result<Arc<dyn ObjectStore>> {
+ self.inner.deregister_store(url)
+ }
+
fn get_store(&self, url: &Url) -> datafusion::common::Result<Arc<dyn ObjectStore>> {
self.inner.get_store(url)
}
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs
index a8148b8049..eace5610e1 100644
--- a/datafusion/core/src/execution/context/mod.rs
+++ b/datafusion/core/src/execution/context/mod.rs
@@ -505,6 +505,13 @@ impl SessionContext {
self.runtime_env().register_object_store(url, object_store)
}
+ /// Deregisters an [`ObjectStore`] associated with the specific URL prefix.
+ ///
+ /// See [`RuntimeEnv::deregister_object_store`] for more details.
+ pub fn deregister_object_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {
+ self.runtime_env().deregister_object_store(url)
+ }
+
/// Registers the [`RecordBatch`] as the specified table name
pub fn register_batch(
&self,
diff --git a/datafusion/execution/src/object_store.rs b/datafusion/execution/src/object_store.rs
index ef83128ac6..aedee7d444 100644
--- a/datafusion/execution/src/object_store.rs
+++ b/datafusion/execution/src/object_store.rs
@@ -20,7 +20,9 @@
//! and query data inside these systems.
use dashmap::DashMap;
-use datafusion_common::{exec_err, internal_datafusion_err, DataFusionError, Result};
+use datafusion_common::{
+ exec_err, internal_datafusion_err, not_impl_err, DataFusionError, Result,
+};
#[cfg(not(target_arch = "wasm32"))]
use object_store::local::LocalFileSystem;
use object_store::ObjectStore;
@@ -154,6 +156,13 @@ pub trait ObjectStoreRegistry: Send + Sync + std::fmt::Debug + 'static {
store: Arc<dyn ObjectStore>,
) -> Option<Arc<dyn ObjectStore>>;
+ /// Deregister the store previously registered with the same key. Returns the
+ /// deregistered store if it existed.
+ #[allow(unused_variables)]
+ fn deregister_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {
+ not_impl_err!("ObjectStoreRegistry::deregister_store is not implemented for this ObjectStoreRegistry")
+ }
+
/// Get a suitable store for the provided URL. For example:
///
/// - URL with scheme `file:///` or no scheme will return the default LocalFS store
@@ -230,6 +239,17 @@ impl ObjectStoreRegistry for DefaultObjectStoreRegistry {
self.object_stores.insert(s, store)
}
+ fn deregister_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {
+ let s = get_url_key(url);
+ let (_, object_store) = self.object_stores
+ .remove(&s)
+ .ok_or_else(|| {
+ internal_datafusion_err!("Failed to deregister object store. No suitable object store found for {url}. See `RuntimeEnv::register_object_store`")
+ })?;
+
+ Ok(object_store)
+ }
+
fn get_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {
let s = get_url_key(url);
self.object_stores
diff --git a/datafusion/execution/src/runtime_env.rs b/datafusion/execution/src/runtime_env.rs
index db045a8b7e..b0d0a966b7 100644
--- a/datafusion/execution/src/runtime_env.rs
+++ b/datafusion/execution/src/runtime_env.rs
@@ -114,8 +114,6 @@ impl RuntimeEnv {
/// ```
///
/// # Example: Register remote URL object store like [Github](https://github.com)
- ///
- ///
/// ```
/// # use std::sync::Arc;
/// # use url::Url;
@@ -141,6 +139,12 @@ impl RuntimeEnv {
self.object_store_registry.register_store(url, object_store)
}
+ /// Deregisters a custom `ObjectStore` previously registered for a specific url.
+ /// See [`ObjectStoreRegistry::deregister_store`] for more details.
+ pub fn deregister_object_store(&self, url: &Url) -> Result<Arc<dyn ObjectStore>> {
+ self.object_store_registry.deregister_store(url)
+ }
+
/// Retrieves a `ObjectStore` instance for a url by consulting the
/// registry. See [`ObjectStoreRegistry::get_store`] for more
/// details.