A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
storage inspector backend logic and models (#8233)
Shahmir Varqha2dfb689cd890
diff --git a/frontend/src/core/islands/main.ts b/frontend/src/core/islands/main.tsindex 39a66793d8..ad750b12d6 100644--- a/frontend/src/core/islands/main.ts+++ b/frontend/src/core/islands/main.ts@@ -126,6 +126,7 @@ export async function initialize() {case "datasets":case "data-source-connections":case "validate-sql-result":+ case "storage-namespaces":case "secret-keys-result":case "startup-logs":// Unsupporteddiff --git a/frontend/src/core/websocket/useMarimoKernelConnection.tsx b/frontend/src/core/websocket/useMarimoKernelConnection.tsxindex 89e1dff101..92078b18f5 100644--- a/frontend/src/core/websocket/useMarimoKernelConnection.tsx+++ b/frontend/src/core/websocket/useMarimoKernelConnection.tsx@@ -284,6 +284,9 @@ export function useMarimoKernelConnection(opts: {})),});return;+ case "storage-namespaces":+ // TODO: Handle storage namespaces+ return;case "reconnected":return;diff --git a/marimo/_ai/_types.py b/marimo/_ai/_types.pyindex 2445c8c678..e5cb868ca5 100644--- a/marimo/_ai/_types.py+++ b/marimo/_ai/_types.py@@ -19,7 +19,7 @@ import msgspecfrom marimo import _loggersfrom marimo._dependencies.dependencies import DependencyManager-from marimo._plugins.utils import remove_none_values+from marimo._utils.dicts import remove_none_valuesfrom marimo._utils.parse_dataclass import parse_rawLOGGER = _loggers.marimo_logger()diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.pyindex f91064a7d1..2405a569a4 100644--- a/marimo/_ai/llm/_impl.py+++ b/marimo/_ai/llm/_impl.py@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, castfrom marimo import _loggersfrom marimo._ai._pydantic_ai_utils import generate_idfrom marimo._plugins.ui._impl.chat.chat import AI_SDK_VERSION, DONE_CHUNK-from marimo._plugins.utils import remove_none_values+from marimo._utils.dicts import remove_none_valuesif TYPE_CHECKING:from collections.abc import AsyncGenerator, Generatordiff --git a/marimo/_cli/development/commands.py b/marimo/_cli/development/commands.pyindex 787ce0d8f9..6baed3a6ce 100644--- a/marimo/_cli/development/commands.py+++ b/marimo/_cli/development/commands.py@@ -26,6 +26,7 @@ def _generate_server_api_schema() -> dict[str, Any]:from starlette.schemas import SchemaGeneratorimport marimo._config.config as config+ import marimo._data._external_storage.models as storageimport marimo._data.models as dataimport marimo._messaging.errors as errorsimport marimo._messaging.notification as notifications@@ -90,6 +91,9 @@ def _generate_server_api_schema() -> dict[str, Any]:data.DataSourceConnection,data.Schema,data.Database,+ # Storage+ storage.StorageEntry,+ storage.StorageNamespace,# Secretssecrets_models.SecretKeysWithProvider,secrets.CreateSecretRequest,@@ -118,6 +122,7 @@ def _generate_server_api_schema() -> dict[str, Any]:notifications.SQLTablePreviewNotification,notifications.SQLTableListPreviewNotification,notifications.DataSourceConnectionsNotification,+ notifications.StorageNamespacesNotification,notifications.SecretKeysResultNotification,notifications.CacheClearedNotification,notifications.CacheInfoNotification,diff --git a/marimo/_data/_external_storage/get_storage.py b/marimo/_data/_external_storage/get_storage.pynew file mode 100644index 0000000000..bec2529429--- /dev/null+++ b/marimo/_data/_external_storage/get_storage.py@@ -0,0 +1,44 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++from typing import Any++from marimo._data._external_storage.models import (+ StorageBackend,+ StorageNamespace,+)+from marimo._data._external_storage.storage import FsspecFilesystem, Obstore+from marimo._types.ids import VariableName++STORAGE_BACKENDS: list[type[StorageBackend[Any]]] = [Obstore, FsspecFilesystem]+++def get_storage_backends_from_variables(+ variables: list[tuple[VariableName, object]],+) -> list[tuple[VariableName, StorageBackend[Any]]]:+ storage_backends: list[tuple[VariableName, StorageBackend[Any]]] = []+ for variable_name, value in variables:+ for storage_backend in STORAGE_BACKENDS:+ if storage_backend.is_compatible(value):+ storage_backends.append(+ (variable_name, storage_backend(value, variable_name))+ )+ break+ return storage_backends+++def storage_backend_to_storage_namespace(+ storage_backend: StorageBackend[Any],+) -> StorageNamespace:+ """+ Convert a storage backend to a storage namespace.+ Currently, this is called as a synchronous post-execution hook which can block the kernel.+ So, we don't fetch the entries here.+ """+ return StorageNamespace(+ name=storage_backend.variable_name,+ display_name=storage_backend.variable_name or "",+ protocol=storage_backend.protocol,+ root_path=storage_backend.root_path or "",+ storage_entries=[],+ )diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.pynew file mode 100644index 0000000000..5f05df9e54--- /dev/null+++ b/marimo/_data/_external_storage/models.py@@ -0,0 +1,98 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import abc+from typing import Any, Generic, Literal, TypeVar++import msgspec++from marimo._types.ids import VariableName++KNOWN_STORAGE_TYPES = Literal[+ "s3", "gcs", "azure", "http", "file", "in-memory"+]+++# Note: We may want to consolidate with FileInfo from _server/models/files.py+class StorageEntry(msgspec.Struct, rename="camel"):+ """A storage entry is a file, directory, or object for external storage systems++ Attributes:+ path: The path of the storage entry.+ kind: The kind of the storage entry.+ size: The size of the storage entry.+ last_modified: The last modified time of the storage entry.+ metadata: The metadata of the storage entry.+ """++ path: str+ kind: Literal["file", "directory", "object"]+ size: int+ last_modified: float | None+ metadata: dict[str, Any] = msgspec.field(default_factory=dict)+++class StorageNamespace(msgspec.Struct, rename="camel"):+ """Represents external storage systems (filesystems and object storage)++ Attributes:+ name: The variable name of the storage namespace.+ display_name: The display name of the storage namespace.+ protocol: The protocol of the storage namespace. E.g. s3, gcs, azure, http, file, in-memory.+ root_path: The root path of the storage namespace.+ storage_entries: The storage entries in the storage namespace.+ """++ name: VariableName | None+ display_name: str+ protocol: str+ root_path: str+ storage_entries: list[StorageEntry]+++DEFAULT_FETCH_LIMIT = 100++Backend = TypeVar("Backend")+++class StorageBackend(abc.ABC, Generic[Backend]):+ def __init__(+ self, store: Backend, variable_name: VariableName | None+ ) -> None:+ self.store = store+ self.variable_name = variable_name++ # TODO: We can make this async, but currently post_execution_hooks are synchronous.+ @abc.abstractmethod+ def list_entries(+ self,+ prefix: str | None,+ *,+ limit: int = DEFAULT_FETCH_LIMIT,+ ) -> list[StorageEntry]:+ """+ List the entries at the given prefix. If no prefix is provided, list the root entries.+ """++ @abc.abstractmethod+ async def get_entry(self, path: str) -> StorageEntry:+ """Get the entry at the given path."""++ @abc.abstractmethod+ async def download(self, path: str) -> bytes:+ """Download the file at the given path."""++ @property+ @abc.abstractmethod+ def protocol(self) -> KNOWN_STORAGE_TYPES | str:+ """Return the protocol of the storage backend."""++ @property+ @abc.abstractmethod+ def root_path(self) -> str | None:+ """Return the root path of the storage backend. None if in-memory or cannot be found"""++ @staticmethod+ @abc.abstractmethod+ def is_compatible(var: Any) -> bool:+ """Check if the backend is compatible with the given variable."""diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.pynew file mode 100644index 0000000000..be35ae2862--- /dev/null+++ b/marimo/_data/_external_storage/storage.py@@ -0,0 +1,286 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import asyncio+from typing import TYPE_CHECKING, Any, Literal, cast++from marimo import _loggers+from marimo._data._external_storage.models import (+ DEFAULT_FETCH_LIMIT,+ KNOWN_STORAGE_TYPES,+ StorageBackend,+ StorageEntry,+)+from marimo._dependencies.dependencies import DependencyManager+from marimo._utils.assert_never import log_never+from marimo._utils.dicts import remove_none_values++if TYPE_CHECKING:+ from fsspec import ( # type: ignore[import-untyped]+ AbstractFileSystem, # noqa: F401+ )+ from obstore import ObjectMeta+ from obstore.store import ObjectStore # noqa: F401++LOGGER = _loggers.marimo_logger()+++class Obstore(StorageBackend["ObjectStore"]):+ def list_entries(+ self,+ prefix: str | None,+ *,+ limit: int = DEFAULT_FETCH_LIMIT,+ ) -> list[StorageEntry]:+ result = self.store.list_with_delimiter(prefix=prefix)++ storage_entries: list[StorageEntry] = []++ # Common prefixes are virtual directories (e.g., "folder/")+ # We can't identify the size / last modified time unless we recursively list the entries+ for common_prefix in result["common_prefixes"]:+ storage_entries.append(+ StorageEntry(+ path=common_prefix,+ kind="directory",+ size=0,+ last_modified=None,+ )+ )++ # Objects are actual files/objects at this level+ for entry in result["objects"]:+ storage_entries.append(self._create_storage_entry(entry))++ if len(storage_entries) > limit:+ LOGGER.debug(+ "Fetched %s entries, but limiting to %s",+ len(storage_entries),+ limit,+ )+ storage_entries = storage_entries[:limit]++ return storage_entries++ async def get_entry(self, path: str) -> StorageEntry:+ entry = await self.store.head_async(path)+ return self._create_storage_entry(entry)++ def _create_storage_entry(self, entry: ObjectMeta) -> StorageEntry:+ path, size = entry.get("path"), entry.get("size")+ if path is None or size is None:+ LOGGER.debug(+ "Entry is missing required fields: path=%s, size=%s",+ path,+ size,+ )+ entry_meta = remove_none_values(+ {"e_tag": entry.get("e_tag"), "version": entry.get("version")}+ )+ last_modified = entry.get("last_modified")+ return StorageEntry(+ path=path or "",+ size=size or 0,+ last_modified=last_modified.timestamp() if last_modified else None,+ kind="object",+ metadata=entry_meta,+ )++ async def download(self, path: str) -> bytes:+ result = await self.store.get_async(path)+ bytes_data = await result.bytes_async()+ return bytes(bytes_data)++ @property+ def protocol(self) -> KNOWN_STORAGE_TYPES | str:+ from obstore.store import (+ AzureStore,+ GCSStore,+ HTTPStore,+ LocalStore,+ MemoryStore,+ S3Store,+ )++ if isinstance(self.store, MemoryStore):+ return "in-memory"+ elif isinstance(self.store, HTTPStore):+ return "http"+ elif isinstance(self.store, LocalStore):+ return "file"+ elif isinstance(self.store, S3Store):+ return "s3"+ elif isinstance(self.store, AzureStore):+ return "azure"+ elif isinstance(self.store, GCSStore):+ return "gcs"+ else:+ log_never(self.store)+ return "unknown"++ @property+ def root_path(self) -> str | None:+ from obstore.store import HTTPStore, LocalStore, MemoryStore++ if isinstance(self.store, MemoryStore):+ return None+ elif isinstance(self.store, HTTPStore):+ return self.store.url+ prefix = self.store.prefix++ if prefix is None:+ if isinstance(self.store, LocalStore):+ return None # root++ config = self.store.config+ bucket = config.get("bucket")+ if bucket is None:+ LOGGER.debug(+ "No bucket found for storage backend. Config %s", config+ )+ elif not isinstance(bucket, str):+ LOGGER.debug("Bucket is not a string: %s", bucket)+ return str(bucket)+ return bucket++ return str(prefix)++ @staticmethod+ def is_compatible(var: Any) -> bool:+ if not DependencyManager.obstore.imported():+ return False++ from obstore.store import ObjectStore++ return isinstance(var, ObjectStore) # type: ignore[misc,arg-type]+++# The async implementations has a few unimplemented methods (like ls), so it's better to use synchronous versions and+# wrap them in asyncio.to_thread+class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):+ def list_entries(+ self,+ prefix: str | None,+ *,+ limit: int = DEFAULT_FETCH_LIMIT,+ ) -> list[StorageEntry]:+ # If no prefix provided, we use empty string to list root entries+ # Else, an error is raised+ if prefix is None:+ prefix = ""++ files = self.store.ls(path=prefix, detail=True)+ if not isinstance(files, list):+ raise ValueError(f"Files is not a list: {files}")+ total_files = len(files)+ if total_files > limit:+ LOGGER.debug(+ "Fetched %s files, but limiting to %s",+ total_files,+ limit,+ )+ files = files[:limit]++ storage_entries = []+ for file in files:+ if isinstance(file, dict):+ storage_entry = self._create_storage_entry(file)+ storage_entries.append(storage_entry)++ return storage_entries++ def _identify_kind(self, entry_type: str) -> Literal["file", "directory"]:+ entry_type = entry_type.strip().lower()+ if entry_type == "file":+ return "file"+ elif entry_type == "directory":+ return "directory"+ else:+ LOGGER.debug("Unknown entry type: %s", entry_type)+ return "file"++ async def get_entry(self, path: str) -> StorageEntry:+ entry = await asyncio.to_thread(self.store.info, path)+ if not isinstance(entry, dict):+ raise ValueError(f"Entry at {path} is not a dictionary")+ return self._create_storage_entry(entry)++ def _create_storage_entry(self, file: dict[str, Any]) -> StorageEntry:+ name, size = file.get("name"), file.get("size")+ if name is None or size is None:+ LOGGER.debug(+ "File is missing required fields: name=%s, size=%s",+ name,+ size,+ )+ entry_meta = remove_none_values(+ {+ "e_tag": file.get("ETag"),+ "is_link": file.get("islink"),+ "mode": file.get("mode"),+ "n_link": file.get("nlink"),+ "created": file.get("created"),+ }+ )+ resolved_kind: Literal["file", "directory"] = "file"+ entry_type = file.get("type")+ if entry_type is None:+ LOGGER.debug(+ "File is missing required fields: type=%s", entry_type+ )+ else:+ resolved_kind = self._identify_kind(entry_type)++ return StorageEntry(+ path=name or "",+ size=size or 0,+ last_modified=file.get("mtime"),+ kind=resolved_kind,+ metadata=entry_meta,+ )++ async def download(self, path: str) -> bytes:+ # There is no async version of open, so we wrap the synchronous open method+ def _read() -> str | bytes:+ return self.store.open(path).read() # type: ignore[no-any-return]++ file = await asyncio.to_thread(_read)+ if isinstance(file, str):+ return file.encode("utf-8")+ return file++ @property+ def protocol(self) -> KNOWN_STORAGE_TYPES | str:+ if isinstance(self.store.protocol, tuple):+ return normalize_protocol("-".join(self.store.protocol))+ return normalize_protocol(self.store.protocol)++ @property+ def root_path(self) -> str | None:+ return cast(str, self.store.root_marker)++ @staticmethod+ def is_compatible(var: Any) -> bool:+ if not DependencyManager.fsspec.imported():+ return False++ from fsspec import AbstractFileSystem++ return isinstance(var, AbstractFileSystem)+++def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | str:+ """Normalize the protocol to a known storage type."""+ protocol = protocol.strip().lower()+ if "s3" in protocol:+ return "s3"+ elif "gcs" in protocol or "gs" in protocol:+ return "gcs"+ elif "azure" in protocol:+ return "azure"+ elif "http" in protocol:+ return "http"+ elif "file" in protocol:+ return "file"+ else:+ return protocoldiff --git a/marimo/_dependencies/dependencies.py b/marimo/_dependencies/dependencies.pyindex f9763bcb37..8c33416d2b 100644--- a/marimo/_dependencies/dependencies.py+++ b/marimo/_dependencies/dependencies.py@@ -254,6 +254,9 @@ class DependencyManager:pydantic = Dependency("pydantic")zmq = Dependency("zmq") # pyzmq for sandbox IPC kernelsweave = Dependency("weave")+ # Storage+ obstore = Dependency("obstore")+ fsspec = Dependency("fsspec")# Version requirements to properly support the new superfences introduced in# pymdown#2470diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.pyindex bb28cdaffe..74e5860754 100644--- a/marimo/_messaging/notification.py+++ b/marimo/_messaging/notification.py@@ -17,6 +17,7 @@ import msgspecfrom marimo import _loggers as loggersfrom marimo._ast.app_config import _AppConfigfrom marimo._ast.cell import CellConfig, RuntimeStateType+from marimo._data._external_storage.models import StorageNamespacefrom marimo._data.models import (ColumnStats,DataSourceConnection,@@ -577,6 +578,17 @@ class DataSourceConnectionsNotification(connections: list[DataSourceConnection]+class StorageNamespacesNotification(Notification, tag="storage-namespaces"):+ """Available storage namespaces for storage inspector.++ Attributes:+ namespaces: Available storage namespaces.+ """++ name: ClassVar[str] = "storage-namespaces"+ namespaces: list[StorageNamespace]++class ValidateSQLResultNotification(Notification, tag="validate-sql-result"):"""SQL query validation result.@@ -756,6 +768,8 @@ NotificationMessage = Union[SQLTableListPreviewNotification,DataSourceConnectionsNotification,ValidateSQLResultNotification,+ # Storage+ StorageNamespacesNotification,# SecretsSecretKeysResultNotification,# Cachediff --git a/marimo/_plugins/stateless/stat.py b/marimo/_plugins/stateless/stat.pyindex 1660842ead..e0412b6e10 100644--- a/marimo/_plugins/stateless/stat.py+++ b/marimo/_plugins/stateless/stat.py@@ -8,7 +8,7 @@ from marimo._output.formatting import as_htmlfrom marimo._output.hypertext import Htmlfrom marimo._output.rich_help import mddocfrom marimo._plugins.core.web_component import build_stateless_plugin-from marimo._plugins.utils import remove_none_values+from marimo._utils.dicts import remove_none_valuesLogger = marimo_logger()diff --git a/marimo/_pyodide/pyodide_session.py b/marimo/_pyodide/pyodide_session.pyindex 32f027d791..21c166baa5 100644--- a/marimo/_pyodide/pyodide_session.py+++ b/marimo/_pyodide/pyodide_session.py@@ -457,6 +457,7 @@ def _launch_pyodide_kernel(# Create hooks with mode-specific configurationfrom marimo._runtime.runner.hooks_post_execution import (attempt_pytest,+ broadcast_storage_backends,render_toplevel_defs,)@@ -465,6 +466,8 @@ def _launch_pyodide_kernel(hooks.add_post_execution(attempt_pytest, Priority.LATE)if is_edit_mode:hooks.add_post_execution(render_toplevel_defs, Priority.LATE)+ if user_config.get("experimental", {}).get("storage_inspector", False):+ hooks.add_post_execution(broadcast_storage_backends, Priority.LATE)kernel = Kernel(cell_configs=configs,diff --git a/marimo/_runtime/runner/hooks_post_execution.py b/marimo/_runtime/runner/hooks_post_execution.pyindex c555394b03..6b2c5c423b 100644--- a/marimo/_runtime/runner/hooks_post_execution.py+++ b/marimo/_runtime/runner/hooks_post_execution.py@@ -6,6 +6,10 @@ import sysfrom marimo import _loggersfrom marimo._ast.cell import CellImplfrom marimo._ast.toplevel import TopLevelExtraction+from marimo._data._external_storage.get_storage import (+ get_storage_backends_from_variables,+ storage_backend_to_storage_namespace,+)from marimo._data.get_datasets import (get_datasets_from_variables,has_updates_to_datasource,@@ -21,6 +25,7 @@ from marimo._messaging.errors import (from marimo._messaging.notification import (DatasetsNotification,DataSourceConnectionsNotification,+ StorageNamespacesNotification,VariableValuesNotification,)from marimo._messaging.notification_utils import (@@ -220,6 +225,38 @@ def _broadcast_duckdb_datasource(return+@kernel_tracer.start_as_current_span("broadcast_storage_backends")+def broadcast_storage_backends(+ cell: CellImpl,+ ctx: PostExecutionHookContext,+ run_result: cell_runner.RunResult,+) -> None:+ del run_result++ if not ctx.should_broadcast_data:+ return++ try:+ storage_backends = get_storage_backends_from_variables(+ [+ (VariableName(variable), ctx.glbls[variable])+ for variable in cell.defs+ if variable in ctx.glbls+ ]+ )+ if storage_backends:+ LOGGER.debug("Broadcasting storage namespaces")+ namespaces = [+ storage_backend_to_storage_namespace(storage_backend)+ for _, storage_backend in storage_backends+ ]+ broadcast_notification(+ StorageNamespacesNotification(namespaces=namespaces)+ )+ except Exception:+ LOGGER.debug("Error getting storage backends", exc_info=True)++@kernel_tracer.start_as_current_span("store_reference_to_output")def _store_reference_to_output(cell: CellImpl,diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex fc5ff21ee0..8af62b014f 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -3234,6 +3234,7 @@ def launch_kernel(# Create hooks with mode-specific configurationfrom marimo._runtime.runner.hooks_post_execution import (attempt_pytest,+ broadcast_storage_backends,render_toplevel_defs,)@@ -3242,6 +3243,8 @@ def launch_kernel(hooks.add_post_execution(attempt_pytest, Priority.LATE)if is_edit_mode:hooks.add_post_execution(render_toplevel_defs, Priority.LATE)+ if user_config.get("experimental", {}).get("storage_inspector", False):+ hooks.add_post_execution(broadcast_storage_backends, Priority.LATE)kernel = Kernel(cell_configs=configs,diff --git a/marimo/_utils/assert_never.py b/marimo/_utils/assert_never.pyindex ab2889573c..ab77a8ec6b 100644--- a/marimo/_utils/assert_never.py+++ b/marimo/_utils/assert_never.py@@ -1,8 +1,20 @@# Copyright 2026 Marimo. All rights reserved.from __future__ import annotations-from typing import NoReturn+from typing import Any, NoReturn++from marimo import _loggers++LOGGER = _loggers.marimo_logger()+++def _form_error_message(value: Any) -> str:+ return f"Unhandled value: {value} ({type(value).__name__})"def assert_never(value: NoReturn) -> NoReturn:- raise AssertionError(f"Unhandled value: {value} ({type(value).__name__})")+ raise AssertionError(_form_error_message(value))+++def log_never(value: NoReturn) -> None:+ LOGGER.warning(_form_error_message(value))diff --git a/marimo/_plugins/utils.py b/marimo/_utils/dicts.pysimilarity index 100%rename from marimo/_plugins/utils.pyrename to marimo/_utils/dicts.pydiff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex ad5d430219..26808828d3 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -2446,6 +2446,7 @@ components:- $ref: '#/components/schemas/SQLTableListPreviewNotification'- $ref: '#/components/schemas/DataSourceConnectionsNotification'- $ref: '#/components/schemas/ValidateSQLResultNotification'+ - $ref: '#/components/schemas/StorageNamespacesNotification'- $ref: '#/components/schemas/SecretKeysResultNotification'- $ref: '#/components/schemas/CacheClearedNotification'- $ref: '#/components/schemas/CacheInfoNotification'@@ -2484,6 +2485,7 @@ components:sql-table-list-preview: '#/components/schemas/SQLTableListPreviewNotification'sql-table-preview: '#/components/schemas/SQLTablePreviewNotification'startup-logs: '#/components/schemas/StartupLogsNotification'+ storage-namespaces: '#/components/schemas/StorageNamespacesNotification'update-cell-codes: '#/components/schemas/UpdateCellCodesNotification'update-cell-ids: '#/components/schemas/UpdateCellIdsNotification'validate-sql-result: '#/components/schemas/ValidateSQLResultNotification'@@ -4244,6 +4246,82 @@ components:- typetitle: StopKernelCommandtype: object+ StorageEntry:+ description: "A storage entry is a file, directory, or object for external storage\+ \ systems\n\n Attributes:\n path: The path of the storage entry.\n\+ \ kind: The kind of the storage entry.\n size: The size of the\+ \ storage entry.\n last_modified: The last modified time of the storage\+ \ entry.\n metadata: The metadata of the storage entry."+ properties:+ kind:+ enum:+ - directory+ - file+ - object+ lastModified:+ anyOf:+ - type: number+ - type: 'null'+ metadata:+ default: {}+ type: object+ path:+ type: string+ size:+ type: integer+ required:+ - path+ - kind+ - size+ - lastModified+ title: StorageEntry+ type: object+ StorageNamespace:+ description: "Represents external storage systems (filesystems and object storage)\n\+ \n Attributes:\n name: The variable name of the storage namespace.\n\+ \ display_name: The display name of the storage namespace.\n \+ \ protocol: The protocol of the storage namespace. E.g. s3, gcs, azure, http,\+ \ file, in-memory.\n root_path: The root path of the storage namespace.\n\+ \ storage_entries: The storage entries in the storage namespace."+ properties:+ displayName:+ type: string+ name:+ anyOf:+ - type: string+ - type: 'null'+ protocol:+ type: string+ rootPath:+ type: string+ storageEntries:+ items:+ $ref: '#/components/schemas/StorageEntry'+ type: array+ required:+ - name+ - displayName+ - protocol+ - rootPath+ - storageEntries+ title: StorageNamespace+ type: object+ StorageNamespacesNotification:+ description: "Available storage namespaces for storage inspector.\n\n Attributes:\n\+ \ namespaces: Available storage namespaces."+ properties:+ namespaces:+ items:+ $ref: '#/components/schemas/StorageNamespace'+ type: array+ op:+ enum:+ - storage-namespaces+ required:+ - op+ - namespaces+ title: StorageNamespacesNotification+ type: objectStoreConfig:description: Configuration for cache stores.properties:@@ -6233,4 +6311,3 @@ paths:summary: Submit login formtags:- auth-diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex 8351bffa12..867900cfbe 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -4549,6 +4549,7 @@ export interface components {| components["schemas"]["SQLTableListPreviewNotification"]| components["schemas"]["DataSourceConnectionsNotification"]| components["schemas"]["ValidateSQLResultNotification"]+ | components["schemas"]["StorageNamespacesNotification"]| components["schemas"]["SecretKeysResultNotification"]| components["schemas"]["CacheClearedNotification"]| components["schemas"]["CacheInfoNotification"]@@ -5615,6 +5616,56 @@ export interface components {/** @enum {unknown} */type: "stop-kernel";};+ /**+ * StorageEntry+ * @description A storage entry is a file, directory, or object for external storage systems+ *+ * Attributes:+ * path: The path of the storage entry.+ * kind: The kind of the storage entry.+ * size: The size of the storage entry.+ * last_modified: The last modified time of the storage entry.+ * metadata: The metadata of the storage entry.+ */+ StorageEntry: {+ /** @enum {unknown} */+ kind: "directory" | "file" | "object";+ lastModified: number | null;+ /** @default {} */+ metadata?: Record<string, any>;+ path: string;+ size: number;+ };+ /**+ * StorageNamespace+ * @description Represents external storage systems (filesystems and object storage)+ *+ * Attributes:+ * name: The variable name of the storage namespace.+ * display_name: The display name of the storage namespace.+ * protocol: The protocol of the storage namespace. E.g. s3, gcs, azure, http, file, in-memory.+ * root_path: The root path of the storage namespace.+ * storage_entries: The storage entries in the storage namespace.+ */+ StorageNamespace: {+ displayName: string;+ name: string | null;+ protocol: string;+ rootPath: string;+ storageEntries: components["schemas"]["StorageEntry"][];+ };+ /**+ * StorageNamespacesNotification+ * @description Available storage namespaces for storage inspector.+ *+ * Attributes:+ * namespaces: Available storage namespaces.+ */+ StorageNamespacesNotification: {+ namespaces: components["schemas"]["StorageNamespace"][];+ /** @enum {unknown} */+ op: "storage-namespaces";+ };/*** StoreConfig* @description Configuration for cache stores.diff --git a/pyproject.toml b/pyproject.tomlindex 0cd1b64782..e88aedc9bf 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -170,6 +170,8 @@ test-optional = ["polars>=1.32.2","sqlglot[rs]>=26.2.0,<28.7.0", # <28.7.0 due to ibis compatibility issue"sqlalchemy>=2.0.40",+ "obstore>=0.8.2",+ "fsspec>=2026.2.0","pyiceberg>=0.9.0","cloudpathlib>=0.23.0",# For testing clickhouse@@ -246,6 +248,8 @@ dependencies = ["matplotlib>=3.8.0","sqlglot[rs]>=26.2.0","sqlalchemy>=2.0.40",+ "obstore>=0.8.2",+ "fsspec>=2026.2.0","pydantic-ai-slim[google,anthropic,bedrock,openai]>=1.52.0","loro>=1.5.0","pandas-stubs>=1.5.3.230321",@@ -485,6 +489,8 @@ banned-module-level-imports = ["pyarrow","redis","sqlalchemy",+ "obstore",+ "fsspec","sqlglot","tomlkit","typing_extensions",@@ -569,6 +575,7 @@ format_on_save = true[tool.marimo.experimental]multi_column = truecache_panel = true+storage_inspector = true[tool.marimo.display]dataframes = "rich"@@ -605,4 +612,3 @@ pip = ">=25.0.1,<26"[tool.pixi.pypi-dependencies]marimo = {path = ".", editable = true}-diff --git a/tests/_data/_external_storage/test_get_storage.py b/tests/_data/_external_storage/test_get_storage.pynew file mode 100644index 0000000000..33d9941863--- /dev/null+++ b/tests/_data/_external_storage/test_get_storage.py@@ -0,0 +1,196 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++from typing import Any+from unittest.mock import MagicMock++import pytest+from inline_snapshot import snapshot++from marimo._data._external_storage.get_storage import (+ get_storage_backends_from_variables,+ storage_backend_to_storage_namespace,+)+from marimo._data._external_storage.models import StorageNamespace+from marimo._data._external_storage.storage import FsspecFilesystem, Obstore+from marimo._dependencies.dependencies import DependencyManager+from marimo._types.ids import VariableName++HAS_OBSTORE = DependencyManager.obstore.has()+HAS_FSSPEC = DependencyManager.fsspec.has()+++class TestGetStorageBackendsFromVariables:+ def test_empty_variables(self) -> None:+ result = get_storage_backends_from_variables([])+ assert result == []++ def test_no_compatible_variables(self) -> None:+ variables: list[tuple[VariableName, object]] = [+ (VariableName("x"), "just a string"),+ (VariableName("y"), 42),+ (VariableName("z"), [1, 2, 3]),+ ]+ result = get_storage_backends_from_variables(variables)+ assert result == []++ @pytest.mark.skipif(not HAS_OBSTORE, reason="obstore not installed")+ def test_detects_obstore(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ variables: list[tuple[VariableName, object]] = [+ (VariableName("my_store"), store),+ ]+ result = get_storage_backends_from_variables(variables)+ assert len(result) == 1+ var_name, backend = result[0]+ assert var_name == "my_store"+ assert isinstance(backend, Obstore)++ @pytest.mark.skipif(not HAS_FSSPEC, reason="fsspec not installed")+ def test_detects_fsspec(self) -> None:+ from fsspec.implementations.memory import MemoryFileSystem++ fs = MemoryFileSystem()+ variables: list[tuple[VariableName, object]] = [+ (VariableName("mem_fs"), fs),+ ]+ result = get_storage_backends_from_variables(variables)+ assert len(result) == 1+ var_name, backend = result[0]+ assert var_name == "mem_fs"+ assert isinstance(backend, FsspecFilesystem)++ @pytest.mark.skipif(+ not (HAS_OBSTORE and HAS_FSSPEC),+ reason="obstore and fsspec both required",+ )+ def test_detects_multiple_backends(self) -> None:+ from fsspec.implementations.memory import MemoryFileSystem+ from obstore.store import MemoryStore++ ob_store = MemoryStore()+ fs = MemoryFileSystem()++ variables: list[tuple[VariableName, object]] = [+ (VariableName("ob"), ob_store),+ (VariableName("not_storage"), "hello"),+ (VariableName("fs"), fs),+ ]+ result = get_storage_backends_from_variables(variables)+ assert len(result) == 2++ names = [name for name, _ in result]+ assert VariableName("ob") in names+ assert VariableName("fs") in names++ types = [type(backend) for _, backend in result]+ assert Obstore in types+ assert FsspecFilesystem in types++ def test_mixed_with_none_values(self) -> None:+ variables: list[tuple[VariableName, object]] = [+ (VariableName("n"), None),+ (VariableName("d"), {"key": "value"}),+ ]+ result = get_storage_backends_from_variables(variables)+ assert result == []+++class TestStorageBackendToStorageNamespace:+ def test_converts_backend_to_namespace(self) -> None:+ mock_backend: Any = MagicMock()+ mock_backend.variable_name = VariableName("test_store")+ mock_backend.protocol = "s3"+ mock_backend.root_path = "my-bucket"++ result = storage_backend_to_storage_namespace(mock_backend)++ assert result == snapshot(+ StorageNamespace(+ name=VariableName("test_store"),+ display_name="test_store",+ protocol="s3",+ root_path="my-bucket",+ storage_entries=[],+ )+ )+ mock_backend.list_entries.assert_not_called()++ def test_handles_none_variable_name(self) -> None:+ mock_backend: Any = MagicMock()+ mock_backend.variable_name = None+ mock_backend.protocol = "file"+ mock_backend.root_path = "/tmp"+ mock_backend.list_entries.return_value = []++ result = storage_backend_to_storage_namespace(mock_backend)++ assert result == snapshot(+ StorageNamespace(+ name=None,+ display_name="",+ protocol="file",+ root_path="/tmp",+ storage_entries=[],+ )+ )++ def test_handles_none_root_path(self) -> None:+ mock_backend: Any = MagicMock()+ mock_backend.variable_name = VariableName("mem")+ mock_backend.protocol = "in-memory"+ mock_backend.root_path = None+ mock_backend.list_entries.return_value = []++ result = storage_backend_to_storage_namespace(mock_backend)++ assert result == snapshot(+ StorageNamespace(+ name=VariableName("mem"),+ display_name="mem",+ protocol="in-memory",+ root_path="",+ storage_entries=[],+ )+ )++ @pytest.mark.skipif(not HAS_FSSPEC, reason="fsspec not installed")+ def test_with_real_fsspec_backend(self) -> None:+ from fsspec.implementations.memory import MemoryFileSystem++ fs = MemoryFileSystem()+ fs.pipe("/hello.txt", b"hi")++ backend = FsspecFilesystem(fs, VariableName("mem_fs"))+ result = storage_backend_to_storage_namespace(backend)++ assert result == snapshot(+ StorageNamespace(+ name=VariableName("mem_fs"),+ display_name="mem_fs",+ protocol="memory",+ root_path="/",+ storage_entries=[],+ )+ )++ @pytest.mark.skipif(not HAS_OBSTORE, reason="obstore not installed")+ def test_with_real_obstore_backend(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()++ backend = Obstore(store, VariableName("mem_store"))+ result = storage_backend_to_storage_namespace(backend)++ assert result == snapshot(+ StorageNamespace(+ name=VariableName("mem_store"),+ display_name="mem_store",+ protocol="in-memory",+ root_path="",+ storage_entries=[],+ )+ )diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.pynew file mode 100644index 0000000000..277f7326b1--- /dev/null+++ b/tests/_data/_external_storage/test_storage_models.py@@ -0,0 +1,717 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++from datetime import datetime, timezone+from typing import Any+from unittest.mock import MagicMock++import pytest+from dirty_equals import IsDatetime, IsPositiveFloat+from inline_snapshot import snapshot++from marimo._data._external_storage.models import StorageEntry+from marimo._data._external_storage.storage import (+ FsspecFilesystem,+ Obstore,+ normalize_protocol,+)+from marimo._dependencies.dependencies import DependencyManager+from marimo._types.ids import VariableName++HAS_OBSTORE = DependencyManager.obstore.has()+HAS_FSSPEC = DependencyManager.fsspec.has()++[email protected](not HAS_OBSTORE, reason="obstore not installed")+class TestObstore:+ def _make_backend(self, store: Any, name: str = "my_store") -> Obstore:+ return Obstore(store, VariableName(name))++ def test_list_entries(self) -> None:+ now = datetime.now(tz=timezone.utc)+ mock_store = MagicMock()+ mock_store.list_with_delimiter.return_value = {+ "common_prefixes": ["subdir/"],+ "objects": [+ {+ "path": "file1.txt",+ "size": 100,+ "last_modified": now,+ "e_tag": "abc",+ "version": None,+ },+ {+ "path": "dir/file2.txt",+ "size": 200,+ "last_modified": now,+ "e_tag": None,+ "version": "v1",+ },+ ],+ }++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix="some/prefix", limit=10)++ mock_store.list_with_delimiter.assert_called_once_with(+ prefix="some/prefix",+ )+ assert result == snapshot(+ [+ StorageEntry(+ path="subdir/",+ kind="directory",+ size=0,+ last_modified=None,+ metadata={},+ ),+ StorageEntry(+ path="file1.txt",+ kind="object",+ size=100,+ last_modified=now.timestamp(),+ metadata={"e_tag": "abc"},+ ),+ StorageEntry(+ path="dir/file2.txt",+ kind="object",+ size=200,+ last_modified=now.timestamp(),+ metadata={"version": "v1"},+ ),+ ]+ )++ def test_list_entries_empty(self) -> None:+ mock_store = MagicMock()+ mock_store.list_with_delimiter.return_value = {+ "common_prefixes": [],+ "objects": [],+ }++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix=None)+ assert result == []++ def test_create_storage_entry_missing_fields(self) -> None:+ mock_store = MagicMock()+ backend = self._make_backend(mock_store)++ entry = backend._create_storage_entry(+ { # pyright: ignore[reportArgumentType]+ "path": None,+ "size": None,+ "last_modified": None,+ "e_tag": None,+ "version": None,+ }+ )+ assert entry == snapshot(+ StorageEntry(+ path="",+ size=0,+ last_modified=None,+ kind="object",+ metadata={},+ )+ )++ def test_create_storage_entry_with_all_metadata(self) -> None:+ now = datetime(2025, 6, 15, 12, 0, 0, tzinfo=timezone.utc)+ mock_store = MagicMock()+ backend = self._make_backend(mock_store)++ entry = backend._create_storage_entry(+ {+ "path": "test.csv",+ "size": 500,+ "last_modified": now,+ "e_tag": "etag123",+ "version": "v2",+ }+ )+ assert entry == snapshot(+ StorageEntry(+ path="test.csv",+ size=500,+ last_modified=now.timestamp(),+ kind="object",+ metadata={"e_tag": "etag123", "version": "v2"},+ )+ )++ async def test_get_entry(self) -> None:+ now = datetime(2025, 6, 15, 12, 0, 0, tzinfo=timezone.utc)+ mock_store = MagicMock()+ head_result = {+ "path": "test.txt",+ "size": 42,+ "last_modified": now,+ "e_tag": "e1",+ "version": None,+ }+ mock_store.head_async = MagicMock(+ return_value=_async_return(head_result)+ )++ backend = self._make_backend(mock_store)+ result = await backend.get_entry("test.txt")+ assert result == snapshot(+ StorageEntry(+ path="test.txt",+ size=42,+ last_modified=now.timestamp(),+ kind="object",+ metadata={"e_tag": "e1"},+ )+ )+ mock_store.head_async.assert_called_once_with("test.txt")++ async def test_download(self) -> None:+ mock_store = MagicMock()+ mock_bytes_result = MagicMock()+ mock_bytes_result.bytes_async = MagicMock(+ return_value=_async_return(b"hello world")+ )+ mock_store.get_async = MagicMock(+ return_value=_async_return(mock_bytes_result)+ )++ backend = self._make_backend(mock_store)+ result = await backend.download("some/path.txt")+ assert result == b"hello world"+ mock_store.get_async.assert_called_once_with("some/path.txt")++ def test_protocol_memory(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ backend = self._make_backend(store)+ assert backend.protocol == "in-memory"++ def test_protocol_local(self) -> None:+ import tempfile++ from obstore.store import LocalStore++ with tempfile.TemporaryDirectory() as tmpdir:+ store = LocalStore(tmpdir)+ backend = self._make_backend(store)+ assert backend.protocol == "file"++ def test_root_path_memory(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ backend = self._make_backend(store)+ assert backend.root_path is None++ def test_root_path_local(self) -> None:+ import tempfile++ from obstore.store import LocalStore++ with tempfile.TemporaryDirectory() as tmpdir:+ store = LocalStore(tmpdir)+ backend = self._make_backend(store)+ # LocalStore without a prefix should return None+ root = backend.root_path+ # This depends on whether there's a prefix; for a bare LocalStore it's None+ assert root is None or isinstance(root, str)++ def test_root_path_s3_with_prefix(self) -> None:+ from obstore.store import S3Store++ store = S3Store("test-bucket", prefix="my/prefix", skip_signature=True)+ backend = self._make_backend(store)+ root = backend.root_path+ assert root is not None+ assert "my/prefix" in str(root)++ def test_root_path_s3_without_prefix(self) -> None:+ from obstore.store import S3Store++ store = S3Store("test-bucket", skip_signature=True)+ backend = self._make_backend(store)+ root = backend.root_path+ assert root == "test-bucket"++ def test_is_compatible_with_obstore(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ assert Obstore.is_compatible(store) is True++ def test_is_compatible_with_non_obstore(self) -> None:+ assert Obstore.is_compatible("not a store") is False+ assert Obstore.is_compatible(42) is False+ assert Obstore.is_compatible(None) is False++[email protected](not HAS_FSSPEC, reason="fsspec not installed")+class TestFsspecFilesystem:+ def _make_backend(+ self, store: Any, name: str = "my_fs"+ ) -> FsspecFilesystem:+ return FsspecFilesystem(store, VariableName(name))++ def test_list_entries(self) -> None:+ mock_store = MagicMock()+ files = [+ {+ "name": "file1.txt",+ "size": 100,+ "type": "file",+ "mtime": 1234567890.0,+ },+ {+ "name": "subdir",+ "size": 0,+ "type": "directory",+ "mtime": 1234567891.0,+ },+ ]+ mock_store.ls.return_value = files++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix="some/path")++ mock_store.ls.assert_called_once_with(path="some/path", detail=True)+ assert result == snapshot(+ [+ StorageEntry(+ path="file1.txt",+ size=100,+ last_modified=1234567890.0,+ kind="file",+ metadata={},+ ),+ StorageEntry(+ path="subdir",+ size=0,+ last_modified=1234567891.0,+ kind="directory",+ metadata={},+ ),+ ]+ )++ def test_list_entries_none_prefix_uses_empty_string(self) -> None:+ mock_store = MagicMock()+ mock_store.ls.return_value = []++ backend = self._make_backend(mock_store)+ backend.list_entries(prefix=None)++ mock_store.ls.assert_called_once_with(path="", detail=True)++ def test_list_entries_respects_limit(self) -> None:+ mock_store = MagicMock()+ files = [+ {+ "name": f"file{i}.txt",+ "size": i * 10,+ "type": "file",+ "mtime": None,+ }+ for i in range(10)+ ]+ mock_store.ls.return_value = files++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix="", limit=3)++ assert result == snapshot(+ [+ StorageEntry(+ path="file0.txt",+ size=0,+ last_modified=None,+ kind="file",+ metadata={},+ ),+ StorageEntry(+ path="file1.txt",+ kind="file",+ size=10,+ last_modified=None,+ metadata={},+ ),+ StorageEntry(+ path="file2.txt",+ kind="file",+ size=20,+ last_modified=None,+ metadata={},+ ),+ ]+ )++ def test_list_entries_raises_on_non_list(self) -> None:+ mock_store = MagicMock()+ mock_store.ls.return_value = "not_a_list"++ backend = self._make_backend(mock_store)+ with pytest.raises(ValueError, match="Files is not a list"):+ backend.list_entries(prefix="")++ def test_list_entries_skips_non_dict_entries(self) -> None:+ mock_store = MagicMock()+ mock_store.ls.return_value = [+ {"name": "good.txt", "size": 10, "type": "file"},+ "bad_entry",+ {"name": "also_good.txt", "size": 20, "type": "file"},+ ]++ backend = self._make_backend(mock_store)+ result = backend.list_entries(prefix="")++ assert result == snapshot(+ [+ StorageEntry(+ path="good.txt",+ size=10,+ last_modified=None,+ kind="file",+ metadata={},+ ),+ StorageEntry(+ path="also_good.txt",+ size=20,+ last_modified=None,+ kind="file",+ metadata={},+ ),+ ]+ )++ def test_identify_kind(self) -> None:+ mock_store = MagicMock()+ backend = self._make_backend(mock_store)++ assert backend._identify_kind("file") == "file"+ assert backend._identify_kind("FILE") == "file"+ assert backend._identify_kind(" file ") == "file"+ assert backend._identify_kind("directory") == "directory"+ assert backend._identify_kind("DIRECTORY") == "directory"+ assert backend._identify_kind(" directory ") == "directory"+ # Unknown types default to "file"+ assert backend._identify_kind("unknown") == "file"+ assert backend._identify_kind("symlink") == "file"++ def test_create_storage_entry_full(self) -> None:+ mock_store = MagicMock()+ backend = self._make_backend(mock_store)++ entry = backend._create_storage_entry(+ {+ "name": "data.csv",+ "size": 1024,+ "type": "file",+ "mtime": 1700000000.0,+ "ETag": "abc123",+ "islink": False,+ "mode": 0o644,+ "nlink": 1,+ "created": 1699000000.0,+ }+ )+ assert entry == snapshot(+ StorageEntry(+ path="data.csv",+ size=1024,+ last_modified=1700000000.0,+ kind="file",+ metadata={+ "e_tag": "abc123",+ "is_link": False,+ "mode": 420,+ "n_link": 1,+ "created": 1699000000.0,+ },+ )+ )++ def test_create_storage_entry_missing_fields(self) -> None:+ mock_store = MagicMock()+ backend = self._make_backend(mock_store)++ entry = backend._create_storage_entry(+ {"name": None, "size": None, "type": None}+ )+ assert entry == snapshot(+ StorageEntry(+ path="",+ size=0,+ last_modified=None,+ kind="file",+ metadata={},+ )+ )++ def test_create_storage_entry_directory(self) -> None:+ mock_store = MagicMock()+ backend = self._make_backend(mock_store)++ entry = backend._create_storage_entry(+ {"name": "my_dir/", "size": 0, "type": "directory", "mtime": None}+ )+ assert entry == snapshot(+ StorageEntry(+ path="my_dir/",+ size=0,+ last_modified=None,+ kind="directory",+ metadata={},+ )+ )++ async def test_get_entry(self) -> None:+ mock_store = MagicMock()+ mock_store.info.return_value = {+ "name": "test.txt",+ "size": 42,+ "type": "file",+ "mtime": 1700000000.0,+ }++ backend = self._make_backend(mock_store)+ result = await backend.get_entry("test.txt")+ assert result == snapshot(+ StorageEntry(+ path="test.txt",+ size=42,+ last_modified=1700000000.0,+ kind="file",+ metadata={},+ )+ )++ async def test_get_entry_raises_on_non_dict(self) -> None:+ mock_store = MagicMock()+ mock_store.info.return_value = "not_a_dict"++ backend = self._make_backend(mock_store)+ with pytest.raises(ValueError, match="is not a dictionary"):+ await backend.get_entry("test.txt")++ async def test_download_bytes(self) -> None:+ mock_store = MagicMock()+ mock_file = MagicMock()+ mock_file.read.return_value = b"binary content"+ mock_store.open.return_value = mock_file++ backend = self._make_backend(mock_store)+ result = await backend.download("path/to/file.bin")+ assert result == b"binary content"+ mock_store.open.assert_called_once_with("path/to/file.bin")++ async def test_download_string_encoded_to_bytes(self) -> None:+ mock_store = MagicMock()+ mock_file = MagicMock()+ mock_file.read.return_value = "text content"+ mock_store.open.return_value = mock_file++ backend = self._make_backend(mock_store)+ result = await backend.download("path/to/file.txt")+ assert result == b"text content"++ def test_protocol_tuple(self) -> None:+ mock_store = MagicMock()+ mock_store.protocol = ("gcs", "gs")++ backend = self._make_backend(mock_store)+ assert backend.protocol == "gcs"++ def test_root_path(self) -> None:+ mock_store = MagicMock()+ mock_store.root_marker = "/some/root"++ backend = self._make_backend(mock_store)+ assert backend.root_path == "/some/root"++ def test_root_path_empty(self) -> None:+ mock_store = MagicMock()+ mock_store.root_marker = ""++ backend = self._make_backend(mock_store)+ assert backend.root_path == ""++ def test_is_compatible_with_fsspec(self) -> None:+ from fsspec import AbstractFileSystem++ mock_fs = MagicMock(spec=AbstractFileSystem)+ assert FsspecFilesystem.is_compatible(mock_fs) is True++ def test_is_compatible_with_non_fsspec(self) -> None:+ assert FsspecFilesystem.is_compatible("not a fs") is False+ assert FsspecFilesystem.is_compatible(42) is False+ assert FsspecFilesystem.is_compatible(None) is False++ def test_is_compatible_with_concrete_filesystem(self) -> None:+ from fsspec.implementations.memory import MemoryFileSystem++ fs = MemoryFileSystem()+ assert FsspecFilesystem.is_compatible(fs) is True++[email protected](not HAS_FSSPEC, reason="fsspec not installed")+class TestFsspecFilesystemIntegration:+ """Integration tests using a real fsspec MemoryFileSystem."""++ async def test_list_and_download_with_memory_fs(self) -> None:+ from fsspec.implementations.memory import MemoryFileSystem++ fs = MemoryFileSystem()+ fs.mkdir("/test")+ fs.pipe("/test/hello.txt", b"hello world")+ fs.pipe("/test/data.csv", b"a,b,c\n1,2,3")++ backend = FsspecFilesystem(fs, VariableName("mem_fs"))+ entries = backend.list_entries(prefix="/test")++ assert entries == snapshot(+ [+ StorageEntry(+ path="/test/hello.txt",+ kind="file",+ size=11,+ last_modified=None,+ metadata={"created": IsPositiveFloat()},+ ),+ StorageEntry(+ path="/test/data.csv",+ kind="file",+ size=11,+ last_modified=None,+ metadata={"created": IsPositiveFloat()},+ ),+ ]+ )++ result = await backend.download("/test/hello.txt")+ assert result == b"hello world"++ async def test_get_entry_with_memory_fs(self) -> None:+ from fsspec.implementations.memory import MemoryFileSystem++ fs = MemoryFileSystem()+ fs.pipe("/myfile.txt", b"content here")++ backend = FsspecFilesystem(fs, VariableName("mem_fs"))+ entry = await backend.get_entry("/myfile.txt")+ assert entry == snapshot(+ StorageEntry(+ path="/myfile.txt",+ kind="file",+ size=12,+ last_modified=None,+ metadata={"created": IsDatetime()},+ )+ )++ def test_protocol_memory_filesystem(self) -> None:+ from fsspec.implementations.memory import MemoryFileSystem++ fs = MemoryFileSystem()+ backend = FsspecFilesystem(fs, VariableName("mem_fs"))+ # MemoryFileSystem protocol is "memory", which doesn't match known types+ assert backend.protocol == "memory"++[email protected](not HAS_OBSTORE, reason="obstore not installed")+class TestObstoreIntegration:+ """Integration tests using a real obstore MemoryStore."""++ async def test_list_entries_with_memory_store(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ # Put some data+ await store.put_async("test/file1.txt", b"hello")+ await store.put_async("test/file2.txt", b"world!")++ backend = Obstore(store, VariableName("mem_store"))+ entries = backend.list_entries(prefix="test/")+ assert entries == snapshot(+ [+ StorageEntry(+ path="test/file1.txt",+ kind="object",+ size=5,+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]+ metadata={"e_tag": "0"},+ ),+ StorageEntry(+ path="test/file2.txt",+ kind="object",+ size=6,+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]+ metadata={"e_tag": "1"},+ ),+ ]+ )++ async def test_download_with_memory_store(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ await store.put_async("data.bin", b"binary data")++ backend = Obstore(store, VariableName("mem_store"))+ result = await backend.download("data.bin")+ assert result == b"binary data"++ async def test_get_entry_with_memory_store(self) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ await store.put_async("info.txt", b"some content")++ backend = Obstore(store, VariableName("mem_store"))+ entry = await backend.get_entry("info.txt")+ assert entry == snapshot(+ StorageEntry(+ path="info.txt",+ kind="object",+ size=12,+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]+ metadata={"e_tag": "0"},+ )+ )+++class TestNormalizeProtocol:+ @pytest.mark.parametrize(+ ("protocol", "expected"),+ [+ ("s3", "s3"),+ ("s3a", "s3"),+ ("S3", "s3"),+ ("gcs", "gcs"),+ ("GCS", "gcs"),+ ("gs-gcs", "gcs"),+ ("azure", "azure"),+ ("Azure", "azure"),+ ("abfs-azure", "azure"),+ ("http", "http"),+ ("HTTP", "http"),+ ("https-http", "http"),+ ("file", "file"),+ ("FILE", "file"),+ ("local-file", "file"),+ ("ftp", "ftp"),+ ("custom", "custom"),+ (" s3 ", "s3"),+ (" gcs ", "gcs"),+ ],+ )+ def test_normalize_protocol(self, protocol: str, expected: str) -> None:+ assert normalize_protocol(protocol) == expected+++# --- Helpers ---+++async def _async_return(value: Any) -> Any:+ return value