Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

storage inspector backend logic and models (#8233)

Shahmir Varqha2dfb689cd890
diff --git a/frontend/src/core/islands/main.ts b/frontend/src/core/islands/main.ts
index 39a66793d8..ad750b12d6 100644
--- a/frontend/src/core/islands/main.ts
+++ b/frontend/src/core/islands/main.ts
@@ -126,6 +126,7 @@ export async function initialize() {
case "datasets":
case "data-source-connections":
case "validate-sql-result":
+ case "storage-namespaces":
case "secret-keys-result":
case "startup-logs":
// Unsupported
diff --git a/frontend/src/core/websocket/useMarimoKernelConnection.tsx b/frontend/src/core/websocket/useMarimoKernelConnection.tsx
index 89e1dff101..92078b18f5 100644
--- a/frontend/src/core/websocket/useMarimoKernelConnection.tsx
+++ b/frontend/src/core/websocket/useMarimoKernelConnection.tsx
@@ -284,6 +284,9 @@ export function useMarimoKernelConnection(opts: {
})),
});
return;
+ case "storage-namespaces":
+ // TODO: Handle storage namespaces
+ return;
case "reconnected":
return;
diff --git a/marimo/_ai/_types.py b/marimo/_ai/_types.py
index 2445c8c678..e5cb868ca5 100644
--- a/marimo/_ai/_types.py
+++ b/marimo/_ai/_types.py
@@ -19,7 +19,7 @@ import msgspec
from marimo import _loggers
from marimo._dependencies.dependencies import DependencyManager
-from marimo._plugins.utils import remove_none_values
+from marimo._utils.dicts import remove_none_values
from marimo._utils.parse_dataclass import parse_raw
LOGGER = _loggers.marimo_logger()
diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.py
index f91064a7d1..2405a569a4 100644
--- a/marimo/_ai/llm/_impl.py
+++ b/marimo/_ai/llm/_impl.py
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, cast
from marimo import _loggers
from marimo._ai._pydantic_ai_utils import generate_id
from marimo._plugins.ui._impl.chat.chat import AI_SDK_VERSION, DONE_CHUNK
-from marimo._plugins.utils import remove_none_values
+from marimo._utils.dicts import remove_none_values
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator
diff --git a/marimo/_cli/development/commands.py b/marimo/_cli/development/commands.py
index 787ce0d8f9..6baed3a6ce 100644
--- a/marimo/_cli/development/commands.py
+++ b/marimo/_cli/development/commands.py
@@ -26,6 +26,7 @@ def _generate_server_api_schema() -> dict[str, Any]:
from starlette.schemas import SchemaGenerator
import marimo._config.config as config
+ import marimo._data._external_storage.models as storage
import marimo._data.models as data
import marimo._messaging.errors as errors
import marimo._messaging.notification as notifications
@@ -90,6 +91,9 @@ def _generate_server_api_schema() -> dict[str, Any]:
data.DataSourceConnection,
data.Schema,
data.Database,
+ # Storage
+ storage.StorageEntry,
+ storage.StorageNamespace,
# Secrets
secrets_models.SecretKeysWithProvider,
secrets.CreateSecretRequest,
@@ -118,6 +122,7 @@ def _generate_server_api_schema() -> dict[str, Any]:
notifications.SQLTablePreviewNotification,
notifications.SQLTableListPreviewNotification,
notifications.DataSourceConnectionsNotification,
+ notifications.StorageNamespacesNotification,
notifications.SecretKeysResultNotification,
notifications.CacheClearedNotification,
notifications.CacheInfoNotification,
diff --git a/marimo/_data/_external_storage/get_storage.py b/marimo/_data/_external_storage/get_storage.py
new file mode 100644
index 0000000000..bec2529429
--- /dev/null
+++ b/marimo/_data/_external_storage/get_storage.py
@@ -0,0 +1,44 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import Any
+
+from marimo._data._external_storage.models import (
+ StorageBackend,
+ StorageNamespace,
+)
+from marimo._data._external_storage.storage import FsspecFilesystem, Obstore
+from marimo._types.ids import VariableName
+
+STORAGE_BACKENDS: list[type[StorageBackend[Any]]] = [Obstore, FsspecFilesystem]
+
+
+def get_storage_backends_from_variables(
+ variables: list[tuple[VariableName, object]],
+) -> list[tuple[VariableName, StorageBackend[Any]]]:
+ storage_backends: list[tuple[VariableName, StorageBackend[Any]]] = []
+ for variable_name, value in variables:
+ for storage_backend in STORAGE_BACKENDS:
+ if storage_backend.is_compatible(value):
+ storage_backends.append(
+ (variable_name, storage_backend(value, variable_name))
+ )
+ break
+ return storage_backends
+
+
+def storage_backend_to_storage_namespace(
+ storage_backend: StorageBackend[Any],
+) -> StorageNamespace:
+ """
+ Convert a storage backend to a storage namespace.
+ Currently, this is called as a synchronous post-execution hook which can block the kernel.
+ So, we don't fetch the entries here.
+ """
+ return StorageNamespace(
+ name=storage_backend.variable_name,
+ display_name=storage_backend.variable_name or "",
+ protocol=storage_backend.protocol,
+ root_path=storage_backend.root_path or "",
+ storage_entries=[],
+ )
diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.py
new file mode 100644
index 0000000000..5f05df9e54
--- /dev/null
+++ b/marimo/_data/_external_storage/models.py
@@ -0,0 +1,98 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import abc
+from typing import Any, Generic, Literal, TypeVar
+
+import msgspec
+
+from marimo._types.ids import VariableName
+
+KNOWN_STORAGE_TYPES = Literal[
+ "s3", "gcs", "azure", "http", "file", "in-memory"
+]
+
+
+# Note: We may want to consolidate with FileInfo from _server/models/files.py
+class StorageEntry(msgspec.Struct, rename="camel"):
+ """A storage entry is a file, directory, or object for external storage systems
+
+ Attributes:
+ path: The path of the storage entry.
+ kind: The kind of the storage entry.
+ size: The size of the storage entry.
+ last_modified: The last modified time of the storage entry.
+ metadata: The metadata of the storage entry.
+ """
+
+ path: str
+ kind: Literal["file", "directory", "object"]
+ size: int
+ last_modified: float | None
+ metadata: dict[str, Any] = msgspec.field(default_factory=dict)
+
+
+class StorageNamespace(msgspec.Struct, rename="camel"):
+ """Represents external storage systems (filesystems and object storage)
+
+ Attributes:
+ name: The variable name of the storage namespace.
+ display_name: The display name of the storage namespace.
+ protocol: The protocol of the storage namespace. E.g. s3, gcs, azure, http, file, in-memory.
+ root_path: The root path of the storage namespace.
+ storage_entries: The storage entries in the storage namespace.
+ """
+
+ name: VariableName | None
+ display_name: str
+ protocol: str
+ root_path: str
+ storage_entries: list[StorageEntry]
+
+
+DEFAULT_FETCH_LIMIT = 100
+
+Backend = TypeVar("Backend")
+
+
+class StorageBackend(abc.ABC, Generic[Backend]):
+ def __init__(
+ self, store: Backend, variable_name: VariableName | None
+ ) -> None:
+ self.store = store
+ self.variable_name = variable_name
+
+ # TODO: We can make this async, but currently post_execution_hooks are synchronous.
+ @abc.abstractmethod
+ def list_entries(
+ self,
+ prefix: str | None,
+ *,
+ limit: int = DEFAULT_FETCH_LIMIT,
+ ) -> list[StorageEntry]:
+ """
+ List the entries at the given prefix. If no prefix is provided, list the root entries.
+ """
+
+ @abc.abstractmethod
+ async def get_entry(self, path: str) -> StorageEntry:
+ """Get the entry at the given path."""
+
+ @abc.abstractmethod
+ async def download(self, path: str) -> bytes:
+ """Download the file at the given path."""
+
+ @property
+ @abc.abstractmethod
+ def protocol(self) -> KNOWN_STORAGE_TYPES | str:
+ """Return the protocol of the storage backend."""
+
+ @property
+ @abc.abstractmethod
+ def root_path(self) -> str | None:
+ """Return the root path of the storage backend. None if in-memory or cannot be found"""
+
+ @staticmethod
+ @abc.abstractmethod
+ def is_compatible(var: Any) -> bool:
+ """Check if the backend is compatible with the given variable."""
diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.py
new file mode 100644
index 0000000000..be35ae2862
--- /dev/null
+++ b/marimo/_data/_external_storage/storage.py
@@ -0,0 +1,286 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING, Any, Literal, cast
+
+from marimo import _loggers
+from marimo._data._external_storage.models import (
+ DEFAULT_FETCH_LIMIT,
+ KNOWN_STORAGE_TYPES,
+ StorageBackend,
+ StorageEntry,
+)
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._utils.assert_never import log_never
+from marimo._utils.dicts import remove_none_values
+
+if TYPE_CHECKING:
+ from fsspec import ( # type: ignore[import-untyped]
+ AbstractFileSystem, # noqa: F401
+ )
+ from obstore import ObjectMeta
+ from obstore.store import ObjectStore # noqa: F401
+
+LOGGER = _loggers.marimo_logger()
+
+
+class Obstore(StorageBackend["ObjectStore"]):
+ def list_entries(
+ self,
+ prefix: str | None,
+ *,
+ limit: int = DEFAULT_FETCH_LIMIT,
+ ) -> list[StorageEntry]:
+ result = self.store.list_with_delimiter(prefix=prefix)
+
+ storage_entries: list[StorageEntry] = []
+
+ # Common prefixes are virtual directories (e.g., "folder/")
+ # We can't identify the size / last modified time unless we recursively list the entries
+ for common_prefix in result["common_prefixes"]:
+ storage_entries.append(
+ StorageEntry(
+ path=common_prefix,
+ kind="directory",
+ size=0,
+ last_modified=None,
+ )
+ )
+
+ # Objects are actual files/objects at this level
+ for entry in result["objects"]:
+ storage_entries.append(self._create_storage_entry(entry))
+
+ if len(storage_entries) > limit:
+ LOGGER.debug(
+ "Fetched %s entries, but limiting to %s",
+ len(storage_entries),
+ limit,
+ )
+ storage_entries = storage_entries[:limit]
+
+ return storage_entries
+
+ async def get_entry(self, path: str) -> StorageEntry:
+ entry = await self.store.head_async(path)
+ return self._create_storage_entry(entry)
+
+ def _create_storage_entry(self, entry: ObjectMeta) -> StorageEntry:
+ path, size = entry.get("path"), entry.get("size")
+ if path is None or size is None:
+ LOGGER.debug(
+ "Entry is missing required fields: path=%s, size=%s",
+ path,
+ size,
+ )
+ entry_meta = remove_none_values(
+ {"e_tag": entry.get("e_tag"), "version": entry.get("version")}
+ )
+ last_modified = entry.get("last_modified")
+ return StorageEntry(
+ path=path or "",
+ size=size or 0,
+ last_modified=last_modified.timestamp() if last_modified else None,
+ kind="object",
+ metadata=entry_meta,
+ )
+
+ async def download(self, path: str) -> bytes:
+ result = await self.store.get_async(path)
+ bytes_data = await result.bytes_async()
+ return bytes(bytes_data)
+
+ @property
+ def protocol(self) -> KNOWN_STORAGE_TYPES | str:
+ from obstore.store import (
+ AzureStore,
+ GCSStore,
+ HTTPStore,
+ LocalStore,
+ MemoryStore,
+ S3Store,
+ )
+
+ if isinstance(self.store, MemoryStore):
+ return "in-memory"
+ elif isinstance(self.store, HTTPStore):
+ return "http"
+ elif isinstance(self.store, LocalStore):
+ return "file"
+ elif isinstance(self.store, S3Store):
+ return "s3"
+ elif isinstance(self.store, AzureStore):
+ return "azure"
+ elif isinstance(self.store, GCSStore):
+ return "gcs"
+ else:
+ log_never(self.store)
+ return "unknown"
+
+ @property
+ def root_path(self) -> str | None:
+ from obstore.store import HTTPStore, LocalStore, MemoryStore
+
+ if isinstance(self.store, MemoryStore):
+ return None
+ elif isinstance(self.store, HTTPStore):
+ return self.store.url
+ prefix = self.store.prefix
+
+ if prefix is None:
+ if isinstance(self.store, LocalStore):
+ return None # root
+
+ config = self.store.config
+ bucket = config.get("bucket")
+ if bucket is None:
+ LOGGER.debug(
+ "No bucket found for storage backend. Config %s", config
+ )
+ elif not isinstance(bucket, str):
+ LOGGER.debug("Bucket is not a string: %s", bucket)
+ return str(bucket)
+ return bucket
+
+ return str(prefix)
+
+ @staticmethod
+ def is_compatible(var: Any) -> bool:
+ if not DependencyManager.obstore.imported():
+ return False
+
+ from obstore.store import ObjectStore
+
+ return isinstance(var, ObjectStore) # type: ignore[misc,arg-type]
+
+
+# The async implementations has a few unimplemented methods (like ls), so it's better to use synchronous versions and
+# wrap them in asyncio.to_thread
+class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
+ def list_entries(
+ self,
+ prefix: str | None,
+ *,
+ limit: int = DEFAULT_FETCH_LIMIT,
+ ) -> list[StorageEntry]:
+ # If no prefix provided, we use empty string to list root entries
+ # Else, an error is raised
+ if prefix is None:
+ prefix = ""
+
+ files = self.store.ls(path=prefix, detail=True)
+ if not isinstance(files, list):
+ raise ValueError(f"Files is not a list: {files}")
+ total_files = len(files)
+ if total_files > limit:
+ LOGGER.debug(
+ "Fetched %s files, but limiting to %s",
+ total_files,
+ limit,
+ )
+ files = files[:limit]
+
+ storage_entries = []
+ for file in files:
+ if isinstance(file, dict):
+ storage_entry = self._create_storage_entry(file)
+ storage_entries.append(storage_entry)
+
+ return storage_entries
+
+ def _identify_kind(self, entry_type: str) -> Literal["file", "directory"]:
+ entry_type = entry_type.strip().lower()
+ if entry_type == "file":
+ return "file"
+ elif entry_type == "directory":
+ return "directory"
+ else:
+ LOGGER.debug("Unknown entry type: %s", entry_type)
+ return "file"
+
+ async def get_entry(self, path: str) -> StorageEntry:
+ entry = await asyncio.to_thread(self.store.info, path)
+ if not isinstance(entry, dict):
+ raise ValueError(f"Entry at {path} is not a dictionary")
+ return self._create_storage_entry(entry)
+
+ def _create_storage_entry(self, file: dict[str, Any]) -> StorageEntry:
+ name, size = file.get("name"), file.get("size")
+ if name is None or size is None:
+ LOGGER.debug(
+ "File is missing required fields: name=%s, size=%s",
+ name,
+ size,
+ )
+ entry_meta = remove_none_values(
+ {
+ "e_tag": file.get("ETag"),
+ "is_link": file.get("islink"),
+ "mode": file.get("mode"),
+ "n_link": file.get("nlink"),
+ "created": file.get("created"),
+ }
+ )
+ resolved_kind: Literal["file", "directory"] = "file"
+ entry_type = file.get("type")
+ if entry_type is None:
+ LOGGER.debug(
+ "File is missing required fields: type=%s", entry_type
+ )
+ else:
+ resolved_kind = self._identify_kind(entry_type)
+
+ return StorageEntry(
+ path=name or "",
+ size=size or 0,
+ last_modified=file.get("mtime"),
+ kind=resolved_kind,
+ metadata=entry_meta,
+ )
+
+ async def download(self, path: str) -> bytes:
+ # There is no async version of open, so we wrap the synchronous open method
+ def _read() -> str | bytes:
+ return self.store.open(path).read() # type: ignore[no-any-return]
+
+ file = await asyncio.to_thread(_read)
+ if isinstance(file, str):
+ return file.encode("utf-8")
+ return file
+
+ @property
+ def protocol(self) -> KNOWN_STORAGE_TYPES | str:
+ if isinstance(self.store.protocol, tuple):
+ return normalize_protocol("-".join(self.store.protocol))
+ return normalize_protocol(self.store.protocol)
+
+ @property
+ def root_path(self) -> str | None:
+ return cast(str, self.store.root_marker)
+
+ @staticmethod
+ def is_compatible(var: Any) -> bool:
+ if not DependencyManager.fsspec.imported():
+ return False
+
+ from fsspec import AbstractFileSystem
+
+ return isinstance(var, AbstractFileSystem)
+
+
+def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | str:
+ """Normalize the protocol to a known storage type."""
+ protocol = protocol.strip().lower()
+ if "s3" in protocol:
+ return "s3"
+ elif "gcs" in protocol or "gs" in protocol:
+ return "gcs"
+ elif "azure" in protocol:
+ return "azure"
+ elif "http" in protocol:
+ return "http"
+ elif "file" in protocol:
+ return "file"
+ else:
+ return protocol
diff --git a/marimo/_dependencies/dependencies.py b/marimo/_dependencies/dependencies.py
index f9763bcb37..8c33416d2b 100644
--- a/marimo/_dependencies/dependencies.py
+++ b/marimo/_dependencies/dependencies.py
@@ -254,6 +254,9 @@ class DependencyManager:
pydantic = Dependency("pydantic")
zmq = Dependency("zmq") # pyzmq for sandbox IPC kernels
weave = Dependency("weave")
+ # Storage
+ obstore = Dependency("obstore")
+ fsspec = Dependency("fsspec")
# Version requirements to properly support the new superfences introduced in
# pymdown#2470
diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.py
index bb28cdaffe..74e5860754 100644
--- a/marimo/_messaging/notification.py
+++ b/marimo/_messaging/notification.py
@@ -17,6 +17,7 @@ import msgspec
from marimo import _loggers as loggers
from marimo._ast.app_config import _AppConfig
from marimo._ast.cell import CellConfig, RuntimeStateType
+from marimo._data._external_storage.models import StorageNamespace
from marimo._data.models import (
ColumnStats,
DataSourceConnection,
@@ -577,6 +578,17 @@ class DataSourceConnectionsNotification(
connections: list[DataSourceConnection]
+class StorageNamespacesNotification(Notification, tag="storage-namespaces"):
+ """Available storage namespaces for storage inspector.
+
+ Attributes:
+ namespaces: Available storage namespaces.
+ """
+
+ name: ClassVar[str] = "storage-namespaces"
+ namespaces: list[StorageNamespace]
+
+
class ValidateSQLResultNotification(Notification, tag="validate-sql-result"):
"""SQL query validation result.
@@ -756,6 +768,8 @@ NotificationMessage = Union[
SQLTableListPreviewNotification,
DataSourceConnectionsNotification,
ValidateSQLResultNotification,
+ # Storage
+ StorageNamespacesNotification,
# Secrets
SecretKeysResultNotification,
# Cache
diff --git a/marimo/_plugins/stateless/stat.py b/marimo/_plugins/stateless/stat.py
index 1660842ead..e0412b6e10 100644
--- a/marimo/_plugins/stateless/stat.py
+++ b/marimo/_plugins/stateless/stat.py
@@ -8,7 +8,7 @@ from marimo._output.formatting import as_html
from marimo._output.hypertext import Html
from marimo._output.rich_help import mddoc
from marimo._plugins.core.web_component import build_stateless_plugin
-from marimo._plugins.utils import remove_none_values
+from marimo._utils.dicts import remove_none_values
Logger = marimo_logger()
diff --git a/marimo/_pyodide/pyodide_session.py b/marimo/_pyodide/pyodide_session.py
index 32f027d791..21c166baa5 100644
--- a/marimo/_pyodide/pyodide_session.py
+++ b/marimo/_pyodide/pyodide_session.py
@@ -457,6 +457,7 @@ def _launch_pyodide_kernel(
# Create hooks with mode-specific configuration
from marimo._runtime.runner.hooks_post_execution import (
attempt_pytest,
+ broadcast_storage_backends,
render_toplevel_defs,
)
@@ -465,6 +466,8 @@ def _launch_pyodide_kernel(
hooks.add_post_execution(attempt_pytest, Priority.LATE)
if is_edit_mode:
hooks.add_post_execution(render_toplevel_defs, Priority.LATE)
+ if user_config.get("experimental", {}).get("storage_inspector", False):
+ hooks.add_post_execution(broadcast_storage_backends, Priority.LATE)
kernel = Kernel(
cell_configs=configs,
diff --git a/marimo/_runtime/runner/hooks_post_execution.py b/marimo/_runtime/runner/hooks_post_execution.py
index c555394b03..6b2c5c423b 100644
--- a/marimo/_runtime/runner/hooks_post_execution.py
+++ b/marimo/_runtime/runner/hooks_post_execution.py
@@ -6,6 +6,10 @@ import sys
from marimo import _loggers
from marimo._ast.cell import CellImpl
from marimo._ast.toplevel import TopLevelExtraction
+from marimo._data._external_storage.get_storage import (
+ get_storage_backends_from_variables,
+ storage_backend_to_storage_namespace,
+)
from marimo._data.get_datasets import (
get_datasets_from_variables,
has_updates_to_datasource,
@@ -21,6 +25,7 @@ from marimo._messaging.errors import (
from marimo._messaging.notification import (
DatasetsNotification,
DataSourceConnectionsNotification,
+ StorageNamespacesNotification,
VariableValuesNotification,
)
from marimo._messaging.notification_utils import (
@@ -220,6 +225,38 @@ def _broadcast_duckdb_datasource(
return
+@kernel_tracer.start_as_current_span("broadcast_storage_backends")
+def broadcast_storage_backends(
+ cell: CellImpl,
+ ctx: PostExecutionHookContext,
+ run_result: cell_runner.RunResult,
+) -> None:
+ del run_result
+
+ if not ctx.should_broadcast_data:
+ return
+
+ try:
+ storage_backends = get_storage_backends_from_variables(
+ [
+ (VariableName(variable), ctx.glbls[variable])
+ for variable in cell.defs
+ if variable in ctx.glbls
+ ]
+ )
+ if storage_backends:
+ LOGGER.debug("Broadcasting storage namespaces")
+ namespaces = [
+ storage_backend_to_storage_namespace(storage_backend)
+ for _, storage_backend in storage_backends
+ ]
+ broadcast_notification(
+ StorageNamespacesNotification(namespaces=namespaces)
+ )
+ except Exception:
+ LOGGER.debug("Error getting storage backends", exc_info=True)
+
+
@kernel_tracer.start_as_current_span("store_reference_to_output")
def _store_reference_to_output(
cell: CellImpl,
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index fc5ff21ee0..8af62b014f 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -3234,6 +3234,7 @@ def launch_kernel(
# Create hooks with mode-specific configuration
from marimo._runtime.runner.hooks_post_execution import (
attempt_pytest,
+ broadcast_storage_backends,
render_toplevel_defs,
)
@@ -3242,6 +3243,8 @@ def launch_kernel(
hooks.add_post_execution(attempt_pytest, Priority.LATE)
if is_edit_mode:
hooks.add_post_execution(render_toplevel_defs, Priority.LATE)
+ if user_config.get("experimental", {}).get("storage_inspector", False):
+ hooks.add_post_execution(broadcast_storage_backends, Priority.LATE)
kernel = Kernel(
cell_configs=configs,
diff --git a/marimo/_utils/assert_never.py b/marimo/_utils/assert_never.py
index ab2889573c..ab77a8ec6b 100644
--- a/marimo/_utils/assert_never.py
+++ b/marimo/_utils/assert_never.py
@@ -1,8 +1,20 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
-from typing import NoReturn
+from typing import Any, NoReturn
+
+from marimo import _loggers
+
+LOGGER = _loggers.marimo_logger()
+
+
+def _form_error_message(value: Any) -> str:
+ return f"Unhandled value: {value} ({type(value).__name__})"
def assert_never(value: NoReturn) -> NoReturn:
- raise AssertionError(f"Unhandled value: {value} ({type(value).__name__})")
+ raise AssertionError(_form_error_message(value))
+
+
+def log_never(value: NoReturn) -> None:
+ LOGGER.warning(_form_error_message(value))
diff --git a/marimo/_plugins/utils.py b/marimo/_utils/dicts.py
similarity index 100%
rename from marimo/_plugins/utils.py
rename to marimo/_utils/dicts.py
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index ad5d430219..26808828d3 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -2446,6 +2446,7 @@ components:
- $ref: '#/components/schemas/SQLTableListPreviewNotification'
- $ref: '#/components/schemas/DataSourceConnectionsNotification'
- $ref: '#/components/schemas/ValidateSQLResultNotification'
+ - $ref: '#/components/schemas/StorageNamespacesNotification'
- $ref: '#/components/schemas/SecretKeysResultNotification'
- $ref: '#/components/schemas/CacheClearedNotification'
- $ref: '#/components/schemas/CacheInfoNotification'
@@ -2484,6 +2485,7 @@ components:
sql-table-list-preview: '#/components/schemas/SQLTableListPreviewNotification'
sql-table-preview: '#/components/schemas/SQLTablePreviewNotification'
startup-logs: '#/components/schemas/StartupLogsNotification'
+ storage-namespaces: '#/components/schemas/StorageNamespacesNotification'
update-cell-codes: '#/components/schemas/UpdateCellCodesNotification'
update-cell-ids: '#/components/schemas/UpdateCellIdsNotification'
validate-sql-result: '#/components/schemas/ValidateSQLResultNotification'
@@ -4244,6 +4246,82 @@ components:
- type
title: StopKernelCommand
type: object
+ StorageEntry:
+ description: "A storage entry is a file, directory, or object for external storage\
+ \ systems\n\n Attributes:\n path: The path of the storage entry.\n\
+ \ kind: The kind of the storage entry.\n size: The size of the\
+ \ storage entry.\n last_modified: The last modified time of the storage\
+ \ entry.\n metadata: The metadata of the storage entry."
+ properties:
+ kind:
+ enum:
+ - directory
+ - file
+ - object
+ lastModified:
+ anyOf:
+ - type: number
+ - type: 'null'
+ metadata:
+ default: {}
+ type: object
+ path:
+ type: string
+ size:
+ type: integer
+ required:
+ - path
+ - kind
+ - size
+ - lastModified
+ title: StorageEntry
+ type: object
+ StorageNamespace:
+ description: "Represents external storage systems (filesystems and object storage)\n\
+ \n Attributes:\n name: The variable name of the storage namespace.\n\
+ \ display_name: The display name of the storage namespace.\n \
+ \ protocol: The protocol of the storage namespace. E.g. s3, gcs, azure, http,\
+ \ file, in-memory.\n root_path: The root path of the storage namespace.\n\
+ \ storage_entries: The storage entries in the storage namespace."
+ properties:
+ displayName:
+ type: string
+ name:
+ anyOf:
+ - type: string
+ - type: 'null'
+ protocol:
+ type: string
+ rootPath:
+ type: string
+ storageEntries:
+ items:
+ $ref: '#/components/schemas/StorageEntry'
+ type: array
+ required:
+ - name
+ - displayName
+ - protocol
+ - rootPath
+ - storageEntries
+ title: StorageNamespace
+ type: object
+ StorageNamespacesNotification:
+ description: "Available storage namespaces for storage inspector.\n\n Attributes:\n\
+ \ namespaces: Available storage namespaces."
+ properties:
+ namespaces:
+ items:
+ $ref: '#/components/schemas/StorageNamespace'
+ type: array
+ op:
+ enum:
+ - storage-namespaces
+ required:
+ - op
+ - namespaces
+ title: StorageNamespacesNotification
+ type: object
StoreConfig:
description: Configuration for cache stores.
properties:
@@ -6233,4 +6311,3 @@ paths:
summary: Submit login form
tags:
- auth
-
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index 8351bffa12..867900cfbe 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -4549,6 +4549,7 @@ export interface components {
| components["schemas"]["SQLTableListPreviewNotification"]
| components["schemas"]["DataSourceConnectionsNotification"]
| components["schemas"]["ValidateSQLResultNotification"]
+ | components["schemas"]["StorageNamespacesNotification"]
| components["schemas"]["SecretKeysResultNotification"]
| components["schemas"]["CacheClearedNotification"]
| components["schemas"]["CacheInfoNotification"]
@@ -5615,6 +5616,56 @@ export interface components {
/** @enum {unknown} */
type: "stop-kernel";
};
+ /**
+ * StorageEntry
+ * @description A storage entry is a file, directory, or object for external storage systems
+ *
+ * Attributes:
+ * path: The path of the storage entry.
+ * kind: The kind of the storage entry.
+ * size: The size of the storage entry.
+ * last_modified: The last modified time of the storage entry.
+ * metadata: The metadata of the storage entry.
+ */
+ StorageEntry: {
+ /** @enum {unknown} */
+ kind: "directory" | "file" | "object";
+ lastModified: number | null;
+ /** @default {} */
+ metadata?: Record<string, any>;
+ path: string;
+ size: number;
+ };
+ /**
+ * StorageNamespace
+ * @description Represents external storage systems (filesystems and object storage)
+ *
+ * Attributes:
+ * name: The variable name of the storage namespace.
+ * display_name: The display name of the storage namespace.
+ * protocol: The protocol of the storage namespace. E.g. s3, gcs, azure, http, file, in-memory.
+ * root_path: The root path of the storage namespace.
+ * storage_entries: The storage entries in the storage namespace.
+ */
+ StorageNamespace: {
+ displayName: string;
+ name: string | null;
+ protocol: string;
+ rootPath: string;
+ storageEntries: components["schemas"]["StorageEntry"][];
+ };
+ /**
+ * StorageNamespacesNotification
+ * @description Available storage namespaces for storage inspector.
+ *
+ * Attributes:
+ * namespaces: Available storage namespaces.
+ */
+ StorageNamespacesNotification: {
+ namespaces: components["schemas"]["StorageNamespace"][];
+ /** @enum {unknown} */
+ op: "storage-namespaces";
+ };
/**
* StoreConfig
* @description Configuration for cache stores.
diff --git a/pyproject.toml b/pyproject.toml
index 0cd1b64782..e88aedc9bf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -170,6 +170,8 @@ test-optional = [
"polars>=1.32.2",
"sqlglot[rs]>=26.2.0,<28.7.0", # <28.7.0 due to ibis compatibility issue
"sqlalchemy>=2.0.40",
+ "obstore>=0.8.2",
+ "fsspec>=2026.2.0",
"pyiceberg>=0.9.0",
"cloudpathlib>=0.23.0",
# For testing clickhouse
@@ -246,6 +248,8 @@ dependencies = [
"matplotlib>=3.8.0",
"sqlglot[rs]>=26.2.0",
"sqlalchemy>=2.0.40",
+ "obstore>=0.8.2",
+ "fsspec>=2026.2.0",
"pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.52.0",
"loro>=1.5.0",
"pandas-stubs>=1.5.3.230321",
@@ -485,6 +489,8 @@ banned-module-level-imports = [
"pyarrow",
"redis",
"sqlalchemy",
+ "obstore",
+ "fsspec",
"sqlglot",
"tomlkit",
"typing_extensions",
@@ -569,6 +575,7 @@ format_on_save = true
[tool.marimo.experimental]
multi_column = true
cache_panel = true
+storage_inspector = true
[tool.marimo.display]
dataframes = "rich"
@@ -605,4 +612,3 @@ pip = ">=25.0.1,<26"
[tool.pixi.pypi-dependencies]
marimo = {path = ".", editable = true}
-
diff --git a/tests/_data/_external_storage/test_get_storage.py b/tests/_data/_external_storage/test_get_storage.py
new file mode 100644
index 0000000000..33d9941863
--- /dev/null
+++ b/tests/_data/_external_storage/test_get_storage.py
@@ -0,0 +1,196 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+from inline_snapshot import snapshot
+
+from marimo._data._external_storage.get_storage import (
+ get_storage_backends_from_variables,
+ storage_backend_to_storage_namespace,
+)
+from marimo._data._external_storage.models import StorageNamespace
+from marimo._data._external_storage.storage import FsspecFilesystem, Obstore
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._types.ids import VariableName
+
+HAS_OBSTORE = DependencyManager.obstore.has()
+HAS_FSSPEC = DependencyManager.fsspec.has()
+
+
+class TestGetStorageBackendsFromVariables:
+ def test_empty_variables(self) -> None:
+ result = get_storage_backends_from_variables([])
+ assert result == []
+
+ def test_no_compatible_variables(self) -> None:
+ variables: list[tuple[VariableName, object]] = [
+ (VariableName("x"), "just a string"),
+ (VariableName("y"), 42),
+ (VariableName("z"), [1, 2, 3]),
+ ]
+ result = get_storage_backends_from_variables(variables)
+ assert result == []
+
+ @pytest.mark.skipif(not HAS_OBSTORE, reason="obstore not installed")
+ def test_detects_obstore(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ variables: list[tuple[VariableName, object]] = [
+ (VariableName("my_store"), store),
+ ]
+ result = get_storage_backends_from_variables(variables)
+ assert len(result) == 1
+ var_name, backend = result[0]
+ assert var_name == "my_store"
+ assert isinstance(backend, Obstore)
+
+ @pytest.mark.skipif(not HAS_FSSPEC, reason="fsspec not installed")
+ def test_detects_fsspec(self) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+
+ fs = MemoryFileSystem()
+ variables: list[tuple[VariableName, object]] = [
+ (VariableName("mem_fs"), fs),
+ ]
+ result = get_storage_backends_from_variables(variables)
+ assert len(result) == 1
+ var_name, backend = result[0]
+ assert var_name == "mem_fs"
+ assert isinstance(backend, FsspecFilesystem)
+
+ @pytest.mark.skipif(
+ not (HAS_OBSTORE and HAS_FSSPEC),
+ reason="obstore and fsspec both required",
+ )
+ def test_detects_multiple_backends(self) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+ from obstore.store import MemoryStore
+
+ ob_store = MemoryStore()
+ fs = MemoryFileSystem()
+
+ variables: list[tuple[VariableName, object]] = [
+ (VariableName("ob"), ob_store),
+ (VariableName("not_storage"), "hello"),
+ (VariableName("fs"), fs),
+ ]
+ result = get_storage_backends_from_variables(variables)
+ assert len(result) == 2
+
+ names = [name for name, _ in result]
+ assert VariableName("ob") in names
+ assert VariableName("fs") in names
+
+ types = [type(backend) for _, backend in result]
+ assert Obstore in types
+ assert FsspecFilesystem in types
+
+ def test_mixed_with_none_values(self) -> None:
+ variables: list[tuple[VariableName, object]] = [
+ (VariableName("n"), None),
+ (VariableName("d"), {"key": "value"}),
+ ]
+ result = get_storage_backends_from_variables(variables)
+ assert result == []
+
+
+class TestStorageBackendToStorageNamespace:
+ def test_converts_backend_to_namespace(self) -> None:
+ mock_backend: Any = MagicMock()
+ mock_backend.variable_name = VariableName("test_store")
+ mock_backend.protocol = "s3"
+ mock_backend.root_path = "my-bucket"
+
+ result = storage_backend_to_storage_namespace(mock_backend)
+
+ assert result == snapshot(
+ StorageNamespace(
+ name=VariableName("test_store"),
+ display_name="test_store",
+ protocol="s3",
+ root_path="my-bucket",
+ storage_entries=[],
+ )
+ )
+ mock_backend.list_entries.assert_not_called()
+
+ def test_handles_none_variable_name(self) -> None:
+ mock_backend: Any = MagicMock()
+ mock_backend.variable_name = None
+ mock_backend.protocol = "file"
+ mock_backend.root_path = "/tmp"
+ mock_backend.list_entries.return_value = []
+
+ result = storage_backend_to_storage_namespace(mock_backend)
+
+ assert result == snapshot(
+ StorageNamespace(
+ name=None,
+ display_name="",
+ protocol="file",
+ root_path="/tmp",
+ storage_entries=[],
+ )
+ )
+
+ def test_handles_none_root_path(self) -> None:
+ mock_backend: Any = MagicMock()
+ mock_backend.variable_name = VariableName("mem")
+ mock_backend.protocol = "in-memory"
+ mock_backend.root_path = None
+ mock_backend.list_entries.return_value = []
+
+ result = storage_backend_to_storage_namespace(mock_backend)
+
+ assert result == snapshot(
+ StorageNamespace(
+ name=VariableName("mem"),
+ display_name="mem",
+ protocol="in-memory",
+ root_path="",
+ storage_entries=[],
+ )
+ )
+
+ @pytest.mark.skipif(not HAS_FSSPEC, reason="fsspec not installed")
+ def test_with_real_fsspec_backend(self) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+
+ fs = MemoryFileSystem()
+ fs.pipe("/hello.txt", b"hi")
+
+ backend = FsspecFilesystem(fs, VariableName("mem_fs"))
+ result = storage_backend_to_storage_namespace(backend)
+
+ assert result == snapshot(
+ StorageNamespace(
+ name=VariableName("mem_fs"),
+ display_name="mem_fs",
+ protocol="memory",
+ root_path="/",
+ storage_entries=[],
+ )
+ )
+
+ @pytest.mark.skipif(not HAS_OBSTORE, reason="obstore not installed")
+ def test_with_real_obstore_backend(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+
+ backend = Obstore(store, VariableName("mem_store"))
+ result = storage_backend_to_storage_namespace(backend)
+
+ assert result == snapshot(
+ StorageNamespace(
+ name=VariableName("mem_store"),
+ display_name="mem_store",
+ protocol="in-memory",
+ root_path="",
+ storage_entries=[],
+ )
+ )
diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.py
new file mode 100644
index 0000000000..277f7326b1
--- /dev/null
+++ b/tests/_data/_external_storage/test_storage_models.py
@@ -0,0 +1,717 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+from dirty_equals import IsDatetime, IsPositiveFloat
+from inline_snapshot import snapshot
+
+from marimo._data._external_storage.models import StorageEntry
+from marimo._data._external_storage.storage import (
+ FsspecFilesystem,
+ Obstore,
+ normalize_protocol,
+)
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._types.ids import VariableName
+
+HAS_OBSTORE = DependencyManager.obstore.has()
+HAS_FSSPEC = DependencyManager.fsspec.has()
+
+
[email protected](not HAS_OBSTORE, reason="obstore not installed")
+class TestObstore:
+ def _make_backend(self, store: Any, name: str = "my_store") -> Obstore:
+ return Obstore(store, VariableName(name))
+
+ def test_list_entries(self) -> None:
+ now = datetime.now(tz=timezone.utc)
+ mock_store = MagicMock()
+ mock_store.list_with_delimiter.return_value = {
+ "common_prefixes": ["subdir/"],
+ "objects": [
+ {
+ "path": "file1.txt",
+ "size": 100,
+ "last_modified": now,
+ "e_tag": "abc",
+ "version": None,
+ },
+ {
+ "path": "dir/file2.txt",
+ "size": 200,
+ "last_modified": now,
+ "e_tag": None,
+ "version": "v1",
+ },
+ ],
+ }
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix="some/prefix", limit=10)
+
+ mock_store.list_with_delimiter.assert_called_once_with(
+ prefix="some/prefix",
+ )
+ assert result == snapshot(
+ [
+ StorageEntry(
+ path="subdir/",
+ kind="directory",
+ size=0,
+ last_modified=None,
+ metadata={},
+ ),
+ StorageEntry(
+ path="file1.txt",
+ kind="object",
+ size=100,
+ last_modified=now.timestamp(),
+ metadata={"e_tag": "abc"},
+ ),
+ StorageEntry(
+ path="dir/file2.txt",
+ kind="object",
+ size=200,
+ last_modified=now.timestamp(),
+ metadata={"version": "v1"},
+ ),
+ ]
+ )
+
+ def test_list_entries_empty(self) -> None:
+ mock_store = MagicMock()
+ mock_store.list_with_delimiter.return_value = {
+ "common_prefixes": [],
+ "objects": [],
+ }
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix=None)
+ assert result == []
+
+ def test_create_storage_entry_missing_fields(self) -> None:
+ mock_store = MagicMock()
+ backend = self._make_backend(mock_store)
+
+ entry = backend._create_storage_entry(
+ { # pyright: ignore[reportArgumentType]
+ "path": None,
+ "size": None,
+ "last_modified": None,
+ "e_tag": None,
+ "version": None,
+ }
+ )
+ assert entry == snapshot(
+ StorageEntry(
+ path="",
+ size=0,
+ last_modified=None,
+ kind="object",
+ metadata={},
+ )
+ )
+
+ def test_create_storage_entry_with_all_metadata(self) -> None:
+ now = datetime(2025, 6, 15, 12, 0, 0, tzinfo=timezone.utc)
+ mock_store = MagicMock()
+ backend = self._make_backend(mock_store)
+
+ entry = backend._create_storage_entry(
+ {
+ "path": "test.csv",
+ "size": 500,
+ "last_modified": now,
+ "e_tag": "etag123",
+ "version": "v2",
+ }
+ )
+ assert entry == snapshot(
+ StorageEntry(
+ path="test.csv",
+ size=500,
+ last_modified=now.timestamp(),
+ kind="object",
+ metadata={"e_tag": "etag123", "version": "v2"},
+ )
+ )
+
+ async def test_get_entry(self) -> None:
+ now = datetime(2025, 6, 15, 12, 0, 0, tzinfo=timezone.utc)
+ mock_store = MagicMock()
+ head_result = {
+ "path": "test.txt",
+ "size": 42,
+ "last_modified": now,
+ "e_tag": "e1",
+ "version": None,
+ }
+ mock_store.head_async = MagicMock(
+ return_value=_async_return(head_result)
+ )
+
+ backend = self._make_backend(mock_store)
+ result = await backend.get_entry("test.txt")
+ assert result == snapshot(
+ StorageEntry(
+ path="test.txt",
+ size=42,
+ last_modified=now.timestamp(),
+ kind="object",
+ metadata={"e_tag": "e1"},
+ )
+ )
+ mock_store.head_async.assert_called_once_with("test.txt")
+
+ async def test_download(self) -> None:
+ mock_store = MagicMock()
+ mock_bytes_result = MagicMock()
+ mock_bytes_result.bytes_async = MagicMock(
+ return_value=_async_return(b"hello world")
+ )
+ mock_store.get_async = MagicMock(
+ return_value=_async_return(mock_bytes_result)
+ )
+
+ backend = self._make_backend(mock_store)
+ result = await backend.download("some/path.txt")
+ assert result == b"hello world"
+ mock_store.get_async.assert_called_once_with("some/path.txt")
+
+ def test_protocol_memory(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ backend = self._make_backend(store)
+ assert backend.protocol == "in-memory"
+
+ def test_protocol_local(self) -> None:
+ import tempfile
+
+ from obstore.store import LocalStore
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ store = LocalStore(tmpdir)
+ backend = self._make_backend(store)
+ assert backend.protocol == "file"
+
+ def test_root_path_memory(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ backend = self._make_backend(store)
+ assert backend.root_path is None
+
+ def test_root_path_local(self) -> None:
+ import tempfile
+
+ from obstore.store import LocalStore
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ store = LocalStore(tmpdir)
+ backend = self._make_backend(store)
+ # LocalStore without a prefix should return None
+ root = backend.root_path
+ # This depends on whether there's a prefix; for a bare LocalStore it's None
+ assert root is None or isinstance(root, str)
+
+ def test_root_path_s3_with_prefix(self) -> None:
+ from obstore.store import S3Store
+
+ store = S3Store("test-bucket", prefix="my/prefix", skip_signature=True)
+ backend = self._make_backend(store)
+ root = backend.root_path
+ assert root is not None
+ assert "my/prefix" in str(root)
+
+ def test_root_path_s3_without_prefix(self) -> None:
+ from obstore.store import S3Store
+
+ store = S3Store("test-bucket", skip_signature=True)
+ backend = self._make_backend(store)
+ root = backend.root_path
+ assert root == "test-bucket"
+
+ def test_is_compatible_with_obstore(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ assert Obstore.is_compatible(store) is True
+
+ def test_is_compatible_with_non_obstore(self) -> None:
+ assert Obstore.is_compatible("not a store") is False
+ assert Obstore.is_compatible(42) is False
+ assert Obstore.is_compatible(None) is False
+
+
[email protected](not HAS_FSSPEC, reason="fsspec not installed")
+class TestFsspecFilesystem:
+ def _make_backend(
+ self, store: Any, name: str = "my_fs"
+ ) -> FsspecFilesystem:
+ return FsspecFilesystem(store, VariableName(name))
+
+ def test_list_entries(self) -> None:
+ mock_store = MagicMock()
+ files = [
+ {
+ "name": "file1.txt",
+ "size": 100,
+ "type": "file",
+ "mtime": 1234567890.0,
+ },
+ {
+ "name": "subdir",
+ "size": 0,
+ "type": "directory",
+ "mtime": 1234567891.0,
+ },
+ ]
+ mock_store.ls.return_value = files
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix="some/path")
+
+ mock_store.ls.assert_called_once_with(path="some/path", detail=True)
+ assert result == snapshot(
+ [
+ StorageEntry(
+ path="file1.txt",
+ size=100,
+ last_modified=1234567890.0,
+ kind="file",
+ metadata={},
+ ),
+ StorageEntry(
+ path="subdir",
+ size=0,
+ last_modified=1234567891.0,
+ kind="directory",
+ metadata={},
+ ),
+ ]
+ )
+
+ def test_list_entries_none_prefix_uses_empty_string(self) -> None:
+ mock_store = MagicMock()
+ mock_store.ls.return_value = []
+
+ backend = self._make_backend(mock_store)
+ backend.list_entries(prefix=None)
+
+ mock_store.ls.assert_called_once_with(path="", detail=True)
+
+ def test_list_entries_respects_limit(self) -> None:
+ mock_store = MagicMock()
+ files = [
+ {
+ "name": f"file{i}.txt",
+ "size": i * 10,
+ "type": "file",
+ "mtime": None,
+ }
+ for i in range(10)
+ ]
+ mock_store.ls.return_value = files
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix="", limit=3)
+
+ assert result == snapshot(
+ [
+ StorageEntry(
+ path="file0.txt",
+ size=0,
+ last_modified=None,
+ kind="file",
+ metadata={},
+ ),
+ StorageEntry(
+ path="file1.txt",
+ kind="file",
+ size=10,
+ last_modified=None,
+ metadata={},
+ ),
+ StorageEntry(
+ path="file2.txt",
+ kind="file",
+ size=20,
+ last_modified=None,
+ metadata={},
+ ),
+ ]
+ )
+
+ def test_list_entries_raises_on_non_list(self) -> None:
+ mock_store = MagicMock()
+ mock_store.ls.return_value = "not_a_list"
+
+ backend = self._make_backend(mock_store)
+ with pytest.raises(ValueError, match="Files is not a list"):
+ backend.list_entries(prefix="")
+
+ def test_list_entries_skips_non_dict_entries(self) -> None:
+ mock_store = MagicMock()
+ mock_store.ls.return_value = [
+ {"name": "good.txt", "size": 10, "type": "file"},
+ "bad_entry",
+ {"name": "also_good.txt", "size": 20, "type": "file"},
+ ]
+
+ backend = self._make_backend(mock_store)
+ result = backend.list_entries(prefix="")
+
+ assert result == snapshot(
+ [
+ StorageEntry(
+ path="good.txt",
+ size=10,
+ last_modified=None,
+ kind="file",
+ metadata={},
+ ),
+ StorageEntry(
+ path="also_good.txt",
+ size=20,
+ last_modified=None,
+ kind="file",
+ metadata={},
+ ),
+ ]
+ )
+
+ def test_identify_kind(self) -> None:
+ mock_store = MagicMock()
+ backend = self._make_backend(mock_store)
+
+ assert backend._identify_kind("file") == "file"
+ assert backend._identify_kind("FILE") == "file"
+ assert backend._identify_kind(" file ") == "file"
+ assert backend._identify_kind("directory") == "directory"
+ assert backend._identify_kind("DIRECTORY") == "directory"
+ assert backend._identify_kind(" directory ") == "directory"
+ # Unknown types default to "file"
+ assert backend._identify_kind("unknown") == "file"
+ assert backend._identify_kind("symlink") == "file"
+
+ def test_create_storage_entry_full(self) -> None:
+ mock_store = MagicMock()
+ backend = self._make_backend(mock_store)
+
+ entry = backend._create_storage_entry(
+ {
+ "name": "data.csv",
+ "size": 1024,
+ "type": "file",
+ "mtime": 1700000000.0,
+ "ETag": "abc123",
+ "islink": False,
+ "mode": 0o644,
+ "nlink": 1,
+ "created": 1699000000.0,
+ }
+ )
+ assert entry == snapshot(
+ StorageEntry(
+ path="data.csv",
+ size=1024,
+ last_modified=1700000000.0,
+ kind="file",
+ metadata={
+ "e_tag": "abc123",
+ "is_link": False,
+ "mode": 420,
+ "n_link": 1,
+ "created": 1699000000.0,
+ },
+ )
+ )
+
+ def test_create_storage_entry_missing_fields(self) -> None:
+ mock_store = MagicMock()
+ backend = self._make_backend(mock_store)
+
+ entry = backend._create_storage_entry(
+ {"name": None, "size": None, "type": None}
+ )
+ assert entry == snapshot(
+ StorageEntry(
+ path="",
+ size=0,
+ last_modified=None,
+ kind="file",
+ metadata={},
+ )
+ )
+
+ def test_create_storage_entry_directory(self) -> None:
+ mock_store = MagicMock()
+ backend = self._make_backend(mock_store)
+
+ entry = backend._create_storage_entry(
+ {"name": "my_dir/", "size": 0, "type": "directory", "mtime": None}
+ )
+ assert entry == snapshot(
+ StorageEntry(
+ path="my_dir/",
+ size=0,
+ last_modified=None,
+ kind="directory",
+ metadata={},
+ )
+ )
+
+ async def test_get_entry(self) -> None:
+ mock_store = MagicMock()
+ mock_store.info.return_value = {
+ "name": "test.txt",
+ "size": 42,
+ "type": "file",
+ "mtime": 1700000000.0,
+ }
+
+ backend = self._make_backend(mock_store)
+ result = await backend.get_entry("test.txt")
+ assert result == snapshot(
+ StorageEntry(
+ path="test.txt",
+ size=42,
+ last_modified=1700000000.0,
+ kind="file",
+ metadata={},
+ )
+ )
+
+ async def test_get_entry_raises_on_non_dict(self) -> None:
+ mock_store = MagicMock()
+ mock_store.info.return_value = "not_a_dict"
+
+ backend = self._make_backend(mock_store)
+ with pytest.raises(ValueError, match="is not a dictionary"):
+ await backend.get_entry("test.txt")
+
+ async def test_download_bytes(self) -> None:
+ mock_store = MagicMock()
+ mock_file = MagicMock()
+ mock_file.read.return_value = b"binary content"
+ mock_store.open.return_value = mock_file
+
+ backend = self._make_backend(mock_store)
+ result = await backend.download("path/to/file.bin")
+ assert result == b"binary content"
+ mock_store.open.assert_called_once_with("path/to/file.bin")
+
+ async def test_download_string_encoded_to_bytes(self) -> None:
+ mock_store = MagicMock()
+ mock_file = MagicMock()
+ mock_file.read.return_value = "text content"
+ mock_store.open.return_value = mock_file
+
+ backend = self._make_backend(mock_store)
+ result = await backend.download("path/to/file.txt")
+ assert result == b"text content"
+
+ def test_protocol_tuple(self) -> None:
+ mock_store = MagicMock()
+ mock_store.protocol = ("gcs", "gs")
+
+ backend = self._make_backend(mock_store)
+ assert backend.protocol == "gcs"
+
+ def test_root_path(self) -> None:
+ mock_store = MagicMock()
+ mock_store.root_marker = "/some/root"
+
+ backend = self._make_backend(mock_store)
+ assert backend.root_path == "/some/root"
+
+ def test_root_path_empty(self) -> None:
+ mock_store = MagicMock()
+ mock_store.root_marker = ""
+
+ backend = self._make_backend(mock_store)
+ assert backend.root_path == ""
+
+ def test_is_compatible_with_fsspec(self) -> None:
+ from fsspec import AbstractFileSystem
+
+ mock_fs = MagicMock(spec=AbstractFileSystem)
+ assert FsspecFilesystem.is_compatible(mock_fs) is True
+
+ def test_is_compatible_with_non_fsspec(self) -> None:
+ assert FsspecFilesystem.is_compatible("not a fs") is False
+ assert FsspecFilesystem.is_compatible(42) is False
+ assert FsspecFilesystem.is_compatible(None) is False
+
+ def test_is_compatible_with_concrete_filesystem(self) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+
+ fs = MemoryFileSystem()
+ assert FsspecFilesystem.is_compatible(fs) is True
+
+
[email protected](not HAS_FSSPEC, reason="fsspec not installed")
+class TestFsspecFilesystemIntegration:
+ """Integration tests using a real fsspec MemoryFileSystem."""
+
+ async def test_list_and_download_with_memory_fs(self) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+
+ fs = MemoryFileSystem()
+ fs.mkdir("/test")
+ fs.pipe("/test/hello.txt", b"hello world")
+ fs.pipe("/test/data.csv", b"a,b,c\n1,2,3")
+
+ backend = FsspecFilesystem(fs, VariableName("mem_fs"))
+ entries = backend.list_entries(prefix="/test")
+
+ assert entries == snapshot(
+ [
+ StorageEntry(
+ path="/test/hello.txt",
+ kind="file",
+ size=11,
+ last_modified=None,
+ metadata={"created": IsPositiveFloat()},
+ ),
+ StorageEntry(
+ path="/test/data.csv",
+ kind="file",
+ size=11,
+ last_modified=None,
+ metadata={"created": IsPositiveFloat()},
+ ),
+ ]
+ )
+
+ result = await backend.download("/test/hello.txt")
+ assert result == b"hello world"
+
+ async def test_get_entry_with_memory_fs(self) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+
+ fs = MemoryFileSystem()
+ fs.pipe("/myfile.txt", b"content here")
+
+ backend = FsspecFilesystem(fs, VariableName("mem_fs"))
+ entry = await backend.get_entry("/myfile.txt")
+ assert entry == snapshot(
+ StorageEntry(
+ path="/myfile.txt",
+ kind="file",
+ size=12,
+ last_modified=None,
+ metadata={"created": IsDatetime()},
+ )
+ )
+
+ def test_protocol_memory_filesystem(self) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+
+ fs = MemoryFileSystem()
+ backend = FsspecFilesystem(fs, VariableName("mem_fs"))
+ # MemoryFileSystem protocol is "memory", which doesn't match known types
+ assert backend.protocol == "memory"
+
+
[email protected](not HAS_OBSTORE, reason="obstore not installed")
+class TestObstoreIntegration:
+ """Integration tests using a real obstore MemoryStore."""
+
+ async def test_list_entries_with_memory_store(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ # Put some data
+ await store.put_async("test/file1.txt", b"hello")
+ await store.put_async("test/file2.txt", b"world!")
+
+ backend = Obstore(store, VariableName("mem_store"))
+ entries = backend.list_entries(prefix="test/")
+ assert entries == snapshot(
+ [
+ StorageEntry(
+ path="test/file1.txt",
+ kind="object",
+ size=5,
+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]
+ metadata={"e_tag": "0"},
+ ),
+ StorageEntry(
+ path="test/file2.txt",
+ kind="object",
+ size=6,
+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]
+ metadata={"e_tag": "1"},
+ ),
+ ]
+ )
+
+ async def test_download_with_memory_store(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ await store.put_async("data.bin", b"binary data")
+
+ backend = Obstore(store, VariableName("mem_store"))
+ result = await backend.download("data.bin")
+ assert result == b"binary data"
+
+ async def test_get_entry_with_memory_store(self) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ await store.put_async("info.txt", b"some content")
+
+ backend = Obstore(store, VariableName("mem_store"))
+ entry = await backend.get_entry("info.txt")
+ assert entry == snapshot(
+ StorageEntry(
+ path="info.txt",
+ kind="object",
+ size=12,
+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]
+ metadata={"e_tag": "0"},
+ )
+ )
+
+
+class TestNormalizeProtocol:
+ @pytest.mark.parametrize(
+ ("protocol", "expected"),
+ [
+ ("s3", "s3"),
+ ("s3a", "s3"),
+ ("S3", "s3"),
+ ("gcs", "gcs"),
+ ("GCS", "gcs"),
+ ("gs-gcs", "gcs"),
+ ("azure", "azure"),
+ ("Azure", "azure"),
+ ("abfs-azure", "azure"),
+ ("http", "http"),
+ ("HTTP", "http"),
+ ("https-http", "http"),
+ ("file", "file"),
+ ("FILE", "file"),
+ ("local-file", "file"),
+ ("ftp", "ftp"),
+ ("custom", "custom"),
+ (" s3 ", "s3"),
+ (" gcs ", "gcs"),
+ ],
+ )
+ def test_normalize_protocol(self, protocol: str, expected: str) -> None:
+ assert normalize_protocol(protocol) == expected
+
+
+# --- Helpers ---
+
+
+async def _async_return(value: Any) -> Any:
+ return value