A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
storage download with signed url and vfile cleanup (#8441)
Shahmir Varqhae3cad9ed1152
diff --git a/frontend/src/components/datasources/components.tsx b/frontend/src/components/datasources/components.tsxindex 65f287f88b..11b1da9e1c 100644--- a/frontend/src/components/datasources/components.tsx+++ b/frontend/src/components/datasources/components.tsx@@ -1,6 +1,7 @@/* Copyright 2026 Marimo. All rights reserved. */import { ChevronRightIcon, LoaderCircle, XIcon } from "lucide-react";+import type { CSSProperties } from "react";import type { DataType } from "@/core/kernel/messages";import { cn } from "@/utils/cn";import { DATA_TYPE_ICON, getDataTypeColor } from "../datasets/icons";@@ -42,16 +43,19 @@ export const EmptyState: React.FC<{ content: string; className?: string }> = ({export const ErrorState: React.FC<{error: Error;+ style?: CSSProperties;className?: string;-}> = ({ error, className }) => {+ showIcon?: boolean;+}> = ({ error, style, className, showIcon = true }) => {return (<divclassName={cn("text-sm bg-red-50 dark:bg-red-900 text-red-600 dark:text-red-50 flex items-center gap-2 p-2 h-8",className,)}+ style={style}>- <XIcon className="h-4 w-4 mt-0.5" />+ {showIcon && <XIcon className="h-4 w-4 mt-0.5" />}{error.message}</div>);diff --git a/frontend/src/components/storage/storage-inspector.tsx b/frontend/src/components/storage/storage-inspector.tsxindex 46cc4257ae..31851703cf 100644--- a/frontend/src/components/storage/storage-inspector.tsx+++ b/frontend/src/components/storage/storage-inspector.tsx@@ -44,6 +44,7 @@ import { copyToClipboard } from "@/utils/copy";import { downloadByURL } from "@/utils/download";import { formatBytes } from "@/utils/formatting";import { Logger } from "@/utils/Logger";+import { ErrorState } from "../datasources/components";import { Button } from "../ui/button";import { renderFileIcon, renderProtocolIcon } from "./components";@@ -445,12 +446,12 @@ const StorageNamespaceSection: React.FC<{</div>)}{error && entries.length === 0 && (- <div- className="py-1 text-xs text-destructive"+ <ErrorState+ error={error}style={indentStyle(1)}- >- Failed to load entries: {error.message}- </div>+ className="py-1 text-xs h-auto"+ showIcon={false}+ />)}{!isPending && entries.length === 0 && !error && (<divdiff --git a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsxindex 29e60fdafb..8cd37b3434 100644--- a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx+++ b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx@@ -139,6 +139,25 @@ describe("useStorageEntries", () => {expect(result.current.isPending).toBe(false);});+ it("should surface result.error as a thrown error", async () => {+ mockRequest.mockResolvedValue({+ entries: [],+ error: "access denied",+ });++ const { result } = renderHook(() => useStorageEntries("ns", "pfx/"), {+ wrapper,+ });++ await waitFor(() => {+ expect(result.current.error).toBeDefined();+ });++ expect(result.current.error?.message).toBe("access denied");+ expect(result.current.entries).toEqual([]);+ expect(result.current.isPending).toBe(false);+ });+it("should suppress error when entries are cached", async () => {const entries = [makeEntry({ path: "ok.txt" })];setStorageState({diff --git a/frontend/src/core/storage/state.ts b/frontend/src/core/storage/state.tsindex 780c63183b..240f309f8b 100644--- a/frontend/src/core/storage/state.ts+++ b/frontend/src/core/storage/state.ts@@ -111,6 +111,9 @@ export function useStorageEntries(namespace: string, prefix?: string) {prefix: prefix ?? ROOT_PATH,limit: DEFAULT_FETCH_LIMIT,});+ if (result.error) {+ throw new Error(result.error);+ }setEntries({ namespace, prefix, entries: result.entries });}, [namespace, prefix, cached === undefined]);diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.pyindex 7f84cbf1a8..058aec532f 100644--- a/marimo/_data/_external_storage/models.py+++ b/marimo/_data/_external_storage/models.py@@ -13,6 +13,7 @@ from marimo._utils.assert_never import log_neverKNOWN_STORAGE_TYPES = Literal["s3", "gcs", "azure", "http", "file", "in-memory"]+SIGNED_URL_EXPIRATION = 60# Note: We may want to consolidate with FileInfo from _server/models/files.py@@ -98,6 +99,12 @@ class StorageBackend(abc.ABC, Generic[Backend]):async def download(self, path: str) -> bytes:"""Download the file at the given path."""+ @abc.abstractmethod+ async def sign_download_url(+ self, path: str, expiration: int = SIGNED_URL_EXPIRATION+ ) -> str | None:+ """Return a signed URL for direct browser download, or None if unsupported."""+async def download_file(self, path: str) -> DownloadResult:"""Download the file at the given path with extracted metadata.diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.pyindex be35ae2862..56dec4d98d 100644--- a/marimo/_data/_external_storage/storage.py+++ b/marimo/_data/_external_storage/storage.py@@ -2,12 +2,14 @@from __future__ import annotationsimport asyncio+from datetime import timedeltafrom typing import TYPE_CHECKING, Any, Literal, castfrom marimo import _loggersfrom marimo._data._external_storage.models import (DEFAULT_FETCH_LIMIT,KNOWN_STORAGE_TYPES,+ SIGNED_URL_EXPIRATION,StorageBackend,StorageEntry,)@@ -91,6 +93,25 @@ class Obstore(StorageBackend["ObjectStore"]):bytes_data = await result.bytes_async()return bytes(bytes_data)+ async def sign_download_url(+ self, path: str, expiration: int = SIGNED_URL_EXPIRATION+ ) -> str | None:+ from obstore import sign_async+ from obstore.store import AzureStore, GCSStore, S3Store++ if not isinstance(self.store, (S3Store, GCSStore, AzureStore)):+ return None+ try:+ return await sign_async(+ self.store,+ "GET",+ path,+ expires_in=timedelta(seconds=expiration),+ )+ except Exception:+ LOGGER.info("Failed to sign URL for %s", path)+ return None+@propertydef protocol(self) -> KNOWN_STORAGE_TYPES | str:from obstore.store import (@@ -249,6 +270,20 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):return file.encode("utf-8")return file+ async def sign_download_url(+ self, path: str, expiration: int = SIGNED_URL_EXPIRATION+ ) -> str | None:+ try:+ url = await asyncio.to_thread(+ self.store.sign, path, expiration=expiration+ )+ return str(url)+ except NotImplementedError:+ return None+ except Exception:+ LOGGER.info("Failed to sign URL for %s", path)+ return None+@propertydef protocol(self) -> KNOWN_STORAGE_TYPES | str:if isinstance(self.store.protocol, tuple):diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.pyindex 6fbe61cc97..5694dced23 100644--- a/marimo/_messaging/notification.py+++ b/marimo/_messaging/notification.py@@ -616,11 +616,14 @@ class StorageEntriesNotification(Notification, tag="storage-entries"):class StorageDownloadReadyNotification(Notification, tag="storage-download-ready"):- """Signals that a storage file download is ready as a virtual file.+ """Signals that a storage file download is ready.++ The url may be a signed cloud URL (preferred) or a virtual file URL+ (fallback for backends that don't support signing).Attributes:request_id: Request ID this responds to.- url: Virtual file URL to download from.+ url: Signed or virtual-file URL to download from.filename: Suggested filename for the download.error: Error message if the download failed."""diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex 06d8be6fb1..54c00e5025 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -212,6 +212,7 @@ if TYPE_CHECKING:from types import ModuleTypefrom marimo._plugins.ui._core.ui_element import UIElement+ from marimo._runtime.virtual_file.virtual_file import VirtualFileLOGGER = _loggers.marimo_logger()@@ -2689,6 +2690,24 @@ class ExternalStorageCallbacks:"storage backend (expected obstore or fsspec)")+ _VFILE_TTL_SECONDS = 60++ def _schedule_vfile_cleanup(self, vfile: VirtualFile) -> None:+ """Best-effort cleanup of a virtual file after a TTL."""+ import asyncio++ from marimo._runtime.context import get_context++ try:+ registry = get_context().virtual_file_registry+ loop = asyncio.get_running_loop()+ loop.call_later(self._VFILE_TTL_SECONDS, registry.remove, vfile)+ except Exception:+ LOGGER.debug(+ "Could not schedule virtual file cleanup for %s",+ vfile.filename,+ )+@kernel_tracer.start_as_current_span("storage_list_entries")async def list_entries(self, request: StorageListEntriesCommand) -> None:"""List storage entries at a given prefix."""@@ -2739,7 +2758,7 @@ class ExternalStorageCallbacks:@kernel_tracer.start_as_current_span("storage_download")async def download(self, request: StorageDownloadCommand) -> None:- """Download a storage entry and create a virtual file."""+ """Download a storage entry, preferring a signed URL."""backend, error = self._get_storage_backend(request.namespace)if error is not None or backend is None:broadcast_notification(@@ -2752,13 +2771,29 @@ class ExternalStorageCallbacks:)return+ filename = request.path.rsplit("/", 1)[-1] or "download"+try:+ signed_url = await backend.sign_download_url(request.path)+ if signed_url is not None:+ broadcast_notification(+ StorageDownloadReadyNotification(+ request_id=request.request_id,+ url=signed_url,+ filename=filename,+ ),+ )+ return++ # Signing not supported; fall back to virtual file with TTLfrom marimo._runtime.virtual_file.virtual_file import VirtualFileresult = await backend.download_file(request.path)vfile = VirtualFile.create_and_register(- result.file_bytes, result.ext+ result.file_bytes,+ result.ext,)+ self._schedule_vfile_cleanup(vfile)broadcast_notification(StorageDownloadReadyNotification(diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex 21df7d7195..89a5b83ef2 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -4280,11 +4280,12 @@ components:title: StorageDownloadCommandtype: objectStorageDownloadReadyNotification:- description: "Signals that a storage file download is ready as a virtual file.\n\- \n Attributes:\n request_id: Request ID this responds to.\n \- \ url: Virtual file URL to download from.\n filename: Suggested\- \ filename for the download.\n error: Error message if the download\- \ failed."+ description: "Signals that a storage file download is ready.\n\n The url\+ \ may be a signed cloud URL (preferred) or a virtual file URL\n (fallback\+ \ for backends that don't support signing).\n\n Attributes:\n request_id:\+ \ Request ID this responds to.\n url: Signed or virtual-file URL to\+ \ download from.\n filename: Suggested filename for the download.\n\+ \ error: Error message if the download failed."properties:error:anyOf:@@ -5105,7 +5106,7 @@ components:type: objectinfo:title: marimo API- version: 0.20.1+ version: 0.20.2openapi: 3.1.0paths:/@file/{filename_and_length}:@@ -6518,4 +6519,3 @@ paths:summary: Submit login formtags:- auth-diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex 608f12cb54..242b858547 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -5724,11 +5724,14 @@ export interface components {};/*** StorageDownloadReadyNotification- * @description Signals that a storage file download is ready as a virtual file.+ * @description Signals that a storage file download is ready.+ *+ * The url may be a signed cloud URL (preferred) or a virtual file URL+ * (fallback for backends that don't support signing).** Attributes:* request_id: Request ID this responds to.- * url: Virtual file URL to download from.+ * url: Signed or virtual-file URL to download from.* filename: Suggested filename for the download.* error: Error message if the download failed.*/diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.pyindex 75b7261147..8660eb80d2 100644--- a/tests/_data/_external_storage/test_storage_models.py+++ b/tests/_data/_external_storage/test_storage_models.py@@ -1,9 +1,10 @@# Copyright 2026 Marimo. All rights reserved.from __future__ import annotations-from datetime import datetime, timezone+import tempfile+from datetime import datetime, timedelta, timezonefrom typing import Any-from unittest.mock import MagicMock+from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom dirty_equals import IsDatetime, IsPositiveFloat@@ -342,6 +343,63 @@ class TestObstore:assert Obstore.is_compatible(42) is Falseassert Obstore.is_compatible(None) is False+ async def test_sign_download_url_returns_none_for_non_cloud_store(+ self,+ ) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ backend = self._make_backend(store)+ result = await backend.sign_download_url("some/path.txt")+ assert result is None++ async def test_sign_download_url_returns_none_for_local_store(+ self,+ ) -> None:+ from obstore.store import LocalStore++ with tempfile.TemporaryDirectory() as tmpdir:+ store = LocalStore(tmpdir)+ backend = self._make_backend(store)+ result = await backend.sign_download_url("some/path.txt")+ assert result is None++ async def test_sign_download_url_calls_sign_async_for_s3(self) -> None:+ from obstore.store import S3Store++ store = S3Store("test-bucket", skip_signature=True)+ backend = self._make_backend(store)++ with patch(+ "obstore.sign_async",+ new_callable=AsyncMock,+ return_value="https://signed.example.com/file",+ ) as mock_sign:+ result = await backend.sign_download_url(+ "data/file.csv", expiration=600+ )++ assert result == "https://signed.example.com/file"+ mock_sign.assert_called_once()+ args, kwargs = mock_sign.call_args+ assert args == (store, "GET", "data/file.csv")+ assert kwargs["expires_in"] == timedelta(seconds=600)++ async def test_sign_download_url_returns_none_on_exception(self) -> None:+ from obstore.store import S3Store++ store = S3Store("test-bucket", skip_signature=True)+ backend = self._make_backend(store)++ with patch(+ "obstore.sign_async",+ new_callable=AsyncMock,+ side_effect=RuntimeError("signing failed"),+ ):+ result = await backend.sign_download_url("data/file.csv")++ assert result is None+def test_display_name_known_protocol(self) -> None:from obstore.store import MemoryStore@@ -677,6 +735,49 @@ class TestFsspecFilesystem:backend = self._make_backend(mock_store)assert backend.display_name == "Amazon S3"+ async def test_sign_download_url_returns_signed_url(self) -> None:+ mock_store = MagicMock()+ mock_store.sign.return_value = "https://signed.example.com/path"++ backend = self._make_backend(mock_store)+ result = await backend.sign_download_url(+ "bucket/file.csv", expiration=900+ )++ assert result == "https://signed.example.com/path"+ mock_store.sign.assert_called_once_with(+ "bucket/file.csv", expiration=900+ )++ async def test_sign_download_url_returns_none_on_not_implemented(+ self,+ ) -> None:+ mock_store = MagicMock()+ mock_store.sign.side_effect = NotImplementedError++ backend = self._make_backend(mock_store)+ result = await backend.sign_download_url("bucket/file.csv")++ assert result is None++ async def test_sign_download_url_returns_none_on_exception(self) -> None:+ mock_store = MagicMock()+ mock_store.sign.side_effect = RuntimeError("unexpected error")++ backend = self._make_backend(mock_store)+ result = await backend.sign_download_url("bucket/file.csv")++ assert result is None++ async def test_sign_download_url_converts_result_to_str(self) -> None:+ mock_store = MagicMock()+ mock_store.sign.return_value = 12345++ backend = self._make_backend(mock_store)+ result = await backend.sign_download_url("path")++ assert result == "12345"+def test_display_name_unknown_protocol(self) -> None:mock_store = MagicMock()mock_store.protocol = "custom-proto"@@ -739,6 +840,18 @@ class TestFsspecFilesystemIntegration:))+ async def test_sign_download_url_not_implemented_by_memory_fs(+ self,+ ) -> None:+ from fsspec.implementations.memory import MemoryFileSystem++ fs = MemoryFileSystem()+ fs.pipe("/test/file.txt", b"hello")++ backend = FsspecFilesystem(fs, VariableName("mem_fs"))+ result = await backend.sign_download_url("/test/file.txt")+ assert result is None+def test_protocol_memory_filesystem(self) -> None:from fsspec.implementations.memory import MemoryFileSystem@@ -809,6 +922,18 @@ class TestObstoreIntegration:))+ async def test_sign_download_url_returns_none_for_memory_store(+ self,+ ) -> None:+ from obstore.store import MemoryStore++ store = MemoryStore()+ await store.put_async("data.txt", b"test")++ backend = Obstore(store, VariableName("mem_store"))+ result = await backend.sign_download_url("data.txt")+ assert result is None+class TestNormalizeProtocol:@pytest.mark.parametrize(diff --git a/tests/_runtime/test_runtime_external_storage.py b/tests/_runtime/test_runtime_external_storage.pyindex 6de995da94..ad8e6349bd 100644--- a/tests/_runtime/test_runtime_external_storage.py+++ b/tests/_runtime/test_runtime_external_storage.py@@ -489,6 +489,149 @@ class TestExternalStorageCallbacks:)]+ async def test_download_uses_signed_url_when_available(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(+ [+ ExecuteCellCommand(+ cell_id=CellId_t("0"),+ code=(+ "from obstore.store import MemoryStore\n"+ f"{STORAGE_VAR} = MemoryStore()"+ ),+ ),+ ]+ )++ store = k.globals[VariableName(STORAGE_VAR)]+ backend = Obstore(store, VariableName(STORAGE_VAR))+ backend.sign_download_url = AsyncMock( # type: ignore[method-assign]+ return_value="https://signed.example.com/data.csv?token=abc",+ )+ # download_file should NOT be called when signing succeeds+ backend.download_file = AsyncMock( # type: ignore[method-assign]+ side_effect=AssertionError("should not be called"),+ )++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ k.external_storage_callbacks,+ "_get_storage_backend",+ lambda _: (backend, None),+ )+ request = StorageDownloadCommand(+ request_id=RequestId("req-22"),+ namespace=STORAGE_VAR,+ path="reports/data.csv",+ )+ await k.handle_message(request)++ results = [+ op+ for op in stream.operations+ if isinstance(op, StorageDownloadReadyNotification)+ ]+ assert results == [+ StorageDownloadReadyNotification(+ request_id=RequestId("req-22"),+ url="https://signed.example.com/data.csv?token=abc",+ filename="data.csv",+ )+ ]++ async def test_download_falls_back_when_signing_returns_none(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(+ [+ ExecuteCellCommand(+ cell_id=CellId_t("0"),+ code=(+ "from obstore.store import MemoryStore\n"+ f"{STORAGE_VAR} = MemoryStore()"+ ),+ ),+ ]+ )++ store = k.globals[VariableName(STORAGE_VAR)]+ await store.put_async("local/readme.txt", b"hello world")++ # MemoryStore can't sign, so sign_download_url returns None+ # and the handler falls back to virtual file+ request = StorageDownloadCommand(+ request_id=RequestId("req-23"),+ namespace=STORAGE_VAR,+ path="local/readme.txt",+ )+ await k.handle_message(request)++ results = [+ op+ for op in stream.operations+ if isinstance(op, StorageDownloadReadyNotification)+ ]+ assert results == [+ StorageDownloadReadyNotification(+ request_id=RequestId("req-23"),+ url=IsStr(), # pyright: ignore[reportArgumentType]+ filename="readme.txt",+ )+ ]++ async def test_download_fallback_schedules_cleanup(+ self, mocked_kernel: MockedKernel+ ) -> None:+ import asyncio++ k = mocked_kernel.k++ await k.run(+ [+ ExecuteCellCommand(+ cell_id=CellId_t("0"),+ code=(+ "from obstore.store import MemoryStore\n"+ f"{STORAGE_VAR} = MemoryStore()"+ ),+ ),+ ]+ )++ store = k.globals[VariableName(STORAGE_VAR)]+ await store.put_async("tmp/file.bin", b"data")++ request = StorageDownloadCommand(+ request_id=RequestId("req-24"),+ namespace=STORAGE_VAR,+ path="tmp/file.bin",+ )++ loop = asyncio.get_running_loop()+ scheduled: list[tuple[float, object]] = []+ original_call_later = loop.call_later++ def spy_call_later(+ delay: float, callback: object, *args: object+ ) -> asyncio.TimerHandle:+ scheduled.append((delay, callback))+ return original_call_later(delay, callback, *args)++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(loop, "call_later", spy_call_later)+ await k.handle_message(request)++ assert len(scheduled) == 1+ delay, _callback = scheduled[0]+ assert delay == 60+async def test_download_backend_exception(self, mocked_kernel: MockedKernel) -> None: