Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

storage download with signed url and vfile cleanup (#8441)

Shahmir Varqhae3cad9ed1152
diff --git a/frontend/src/components/datasources/components.tsx b/frontend/src/components/datasources/components.tsx
index 65f287f88b..11b1da9e1c 100644
--- a/frontend/src/components/datasources/components.tsx
+++ b/frontend/src/components/datasources/components.tsx
@@ -1,6 +1,7 @@
/* Copyright 2026 Marimo. All rights reserved. */
import { ChevronRightIcon, LoaderCircle, XIcon } from "lucide-react";
+import type { CSSProperties } from "react";
import type { DataType } from "@/core/kernel/messages";
import { cn } from "@/utils/cn";
import { DATA_TYPE_ICON, getDataTypeColor } from "../datasets/icons";
@@ -42,16 +43,19 @@ export const EmptyState: React.FC<{ content: string; className?: string }> = ({
export const ErrorState: React.FC<{
error: Error;
+ style?: CSSProperties;
className?: string;
-}> = ({ error, className }) => {
+ showIcon?: boolean;
+}> = ({ error, style, className, showIcon = true }) => {
return (
<div
className={cn(
"text-sm bg-red-50 dark:bg-red-900 text-red-600 dark:text-red-50 flex items-center gap-2 p-2 h-8",
className,
)}
+ style={style}
>
- <XIcon className="h-4 w-4 mt-0.5" />
+ {showIcon && <XIcon className="h-4 w-4 mt-0.5" />}
{error.message}
</div>
);
diff --git a/frontend/src/components/storage/storage-inspector.tsx b/frontend/src/components/storage/storage-inspector.tsx
index 46cc4257ae..31851703cf 100644
--- a/frontend/src/components/storage/storage-inspector.tsx
+++ b/frontend/src/components/storage/storage-inspector.tsx
@@ -44,6 +44,7 @@ import { copyToClipboard } from "@/utils/copy";
import { downloadByURL } from "@/utils/download";
import { formatBytes } from "@/utils/formatting";
import { Logger } from "@/utils/Logger";
+import { ErrorState } from "../datasources/components";
import { Button } from "../ui/button";
import { renderFileIcon, renderProtocolIcon } from "./components";
@@ -445,12 +446,12 @@ const StorageNamespaceSection: React.FC<{
</div>
)}
{error && entries.length === 0 && (
- <div
- className="py-1 text-xs text-destructive"
+ <ErrorState
+ error={error}
style={indentStyle(1)}
- >
- Failed to load entries: {error.message}
- </div>
+ className="py-1 text-xs h-auto"
+ showIcon={false}
+ />
)}
{!isPending && entries.length === 0 && !error && (
<div
diff --git a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx
index 29e60fdafb..8cd37b3434 100644
--- a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx
+++ b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx
@@ -139,6 +139,25 @@ describe("useStorageEntries", () => {
expect(result.current.isPending).toBe(false);
});
+ it("should surface result.error as a thrown error", async () => {
+ mockRequest.mockResolvedValue({
+ entries: [],
+ error: "access denied",
+ });
+
+ const { result } = renderHook(() => useStorageEntries("ns", "pfx/"), {
+ wrapper,
+ });
+
+ await waitFor(() => {
+ expect(result.current.error).toBeDefined();
+ });
+
+ expect(result.current.error?.message).toBe("access denied");
+ expect(result.current.entries).toEqual([]);
+ expect(result.current.isPending).toBe(false);
+ });
+
it("should suppress error when entries are cached", async () => {
const entries = [makeEntry({ path: "ok.txt" })];
setStorageState({
diff --git a/frontend/src/core/storage/state.ts b/frontend/src/core/storage/state.ts
index 780c63183b..240f309f8b 100644
--- a/frontend/src/core/storage/state.ts
+++ b/frontend/src/core/storage/state.ts
@@ -111,6 +111,9 @@ export function useStorageEntries(namespace: string, prefix?: string) {
prefix: prefix ?? ROOT_PATH,
limit: DEFAULT_FETCH_LIMIT,
});
+ if (result.error) {
+ throw new Error(result.error);
+ }
setEntries({ namespace, prefix, entries: result.entries });
}, [namespace, prefix, cached === undefined]);
diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.py
index 7f84cbf1a8..058aec532f 100644
--- a/marimo/_data/_external_storage/models.py
+++ b/marimo/_data/_external_storage/models.py
@@ -13,6 +13,7 @@ from marimo._utils.assert_never import log_never
KNOWN_STORAGE_TYPES = Literal[
"s3", "gcs", "azure", "http", "file", "in-memory"
]
+SIGNED_URL_EXPIRATION = 60
# Note: We may want to consolidate with FileInfo from _server/models/files.py
@@ -98,6 +99,12 @@ class StorageBackend(abc.ABC, Generic[Backend]):
async def download(self, path: str) -> bytes:
"""Download the file at the given path."""
+ @abc.abstractmethod
+ async def sign_download_url(
+ self, path: str, expiration: int = SIGNED_URL_EXPIRATION
+ ) -> str | None:
+ """Return a signed URL for direct browser download, or None if unsupported."""
+
async def download_file(self, path: str) -> DownloadResult:
"""Download the file at the given path with extracted metadata.
diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.py
index be35ae2862..56dec4d98d 100644
--- a/marimo/_data/_external_storage/storage.py
+++ b/marimo/_data/_external_storage/storage.py
@@ -2,12 +2,14 @@
from __future__ import annotations
import asyncio
+from datetime import timedelta
from typing import TYPE_CHECKING, Any, Literal, cast
from marimo import _loggers
from marimo._data._external_storage.models import (
DEFAULT_FETCH_LIMIT,
KNOWN_STORAGE_TYPES,
+ SIGNED_URL_EXPIRATION,
StorageBackend,
StorageEntry,
)
@@ -91,6 +93,25 @@ class Obstore(StorageBackend["ObjectStore"]):
bytes_data = await result.bytes_async()
return bytes(bytes_data)
+ async def sign_download_url(
+ self, path: str, expiration: int = SIGNED_URL_EXPIRATION
+ ) -> str | None:
+ from obstore import sign_async
+ from obstore.store import AzureStore, GCSStore, S3Store
+
+ if not isinstance(self.store, (S3Store, GCSStore, AzureStore)):
+ return None
+ try:
+ return await sign_async(
+ self.store,
+ "GET",
+ path,
+ expires_in=timedelta(seconds=expiration),
+ )
+ except Exception:
+ LOGGER.info("Failed to sign URL for %s", path)
+ return None
+
@property
def protocol(self) -> KNOWN_STORAGE_TYPES | str:
from obstore.store import (
@@ -249,6 +270,20 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
return file.encode("utf-8")
return file
+ async def sign_download_url(
+ self, path: str, expiration: int = SIGNED_URL_EXPIRATION
+ ) -> str | None:
+ try:
+ url = await asyncio.to_thread(
+ self.store.sign, path, expiration=expiration
+ )
+ return str(url)
+ except NotImplementedError:
+ return None
+ except Exception:
+ LOGGER.info("Failed to sign URL for %s", path)
+ return None
+
@property
def protocol(self) -> KNOWN_STORAGE_TYPES | str:
if isinstance(self.store.protocol, tuple):
diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.py
index 6fbe61cc97..5694dced23 100644
--- a/marimo/_messaging/notification.py
+++ b/marimo/_messaging/notification.py
@@ -616,11 +616,14 @@ class StorageEntriesNotification(Notification, tag="storage-entries"):
class StorageDownloadReadyNotification(
Notification, tag="storage-download-ready"
):
- """Signals that a storage file download is ready as a virtual file.
+ """Signals that a storage file download is ready.
+
+ The url may be a signed cloud URL (preferred) or a virtual file URL
+ (fallback for backends that don't support signing).
Attributes:
request_id: Request ID this responds to.
- url: Virtual file URL to download from.
+ url: Signed or virtual-file URL to download from.
filename: Suggested filename for the download.
error: Error message if the download failed.
"""
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index 06d8be6fb1..54c00e5025 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -212,6 +212,7 @@ if TYPE_CHECKING:
from types import ModuleType
from marimo._plugins.ui._core.ui_element import UIElement
+ from marimo._runtime.virtual_file.virtual_file import VirtualFile
LOGGER = _loggers.marimo_logger()
@@ -2689,6 +2690,24 @@ class ExternalStorageCallbacks:
"storage backend (expected obstore or fsspec)"
)
+ _VFILE_TTL_SECONDS = 60
+
+ def _schedule_vfile_cleanup(self, vfile: VirtualFile) -> None:
+ """Best-effort cleanup of a virtual file after a TTL."""
+ import asyncio
+
+ from marimo._runtime.context import get_context
+
+ try:
+ registry = get_context().virtual_file_registry
+ loop = asyncio.get_running_loop()
+ loop.call_later(self._VFILE_TTL_SECONDS, registry.remove, vfile)
+ except Exception:
+ LOGGER.debug(
+ "Could not schedule virtual file cleanup for %s",
+ vfile.filename,
+ )
+
@kernel_tracer.start_as_current_span("storage_list_entries")
async def list_entries(self, request: StorageListEntriesCommand) -> None:
"""List storage entries at a given prefix."""
@@ -2739,7 +2758,7 @@ class ExternalStorageCallbacks:
@kernel_tracer.start_as_current_span("storage_download")
async def download(self, request: StorageDownloadCommand) -> None:
- """Download a storage entry and create a virtual file."""
+ """Download a storage entry, preferring a signed URL."""
backend, error = self._get_storage_backend(request.namespace)
if error is not None or backend is None:
broadcast_notification(
@@ -2752,13 +2771,29 @@ class ExternalStorageCallbacks:
)
return
+ filename = request.path.rsplit("/", 1)[-1] or "download"
+
try:
+ signed_url = await backend.sign_download_url(request.path)
+ if signed_url is not None:
+ broadcast_notification(
+ StorageDownloadReadyNotification(
+ request_id=request.request_id,
+ url=signed_url,
+ filename=filename,
+ ),
+ )
+ return
+
+ # Signing not supported; fall back to virtual file with TTL
from marimo._runtime.virtual_file.virtual_file import VirtualFile
result = await backend.download_file(request.path)
vfile = VirtualFile.create_and_register(
- result.file_bytes, result.ext
+ result.file_bytes,
+ result.ext,
)
+ self._schedule_vfile_cleanup(vfile)
broadcast_notification(
StorageDownloadReadyNotification(
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 21df7d7195..89a5b83ef2 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -4280,11 +4280,12 @@ components:
title: StorageDownloadCommand
type: object
StorageDownloadReadyNotification:
- description: "Signals that a storage file download is ready as a virtual file.\n\
- \n Attributes:\n request_id: Request ID this responds to.\n \
- \ url: Virtual file URL to download from.\n filename: Suggested\
- \ filename for the download.\n error: Error message if the download\
- \ failed."
+ description: "Signals that a storage file download is ready.\n\n The url\
+ \ may be a signed cloud URL (preferred) or a virtual file URL\n (fallback\
+ \ for backends that don't support signing).\n\n Attributes:\n request_id:\
+ \ Request ID this responds to.\n url: Signed or virtual-file URL to\
+ \ download from.\n filename: Suggested filename for the download.\n\
+ \ error: Error message if the download failed."
properties:
error:
anyOf:
@@ -5105,7 +5106,7 @@ components:
type: object
info:
title: marimo API
- version: 0.20.1
+ version: 0.20.2
openapi: 3.1.0
paths:
/@file/{filename_and_length}:
@@ -6518,4 +6519,3 @@ paths:
summary: Submit login form
tags:
- auth
-
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index 608f12cb54..242b858547 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -5724,11 +5724,14 @@ export interface components {
};
/**
* StorageDownloadReadyNotification
- * @description Signals that a storage file download is ready as a virtual file.
+ * @description Signals that a storage file download is ready.
+ *
+ * The url may be a signed cloud URL (preferred) or a virtual file URL
+ * (fallback for backends that don't support signing).
*
* Attributes:
* request_id: Request ID this responds to.
- * url: Virtual file URL to download from.
+ * url: Signed or virtual-file URL to download from.
* filename: Suggested filename for the download.
* error: Error message if the download failed.
*/
diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.py
index 75b7261147..8660eb80d2 100644
--- a/tests/_data/_external_storage/test_storage_models.py
+++ b/tests/_data/_external_storage/test_storage_models.py
@@ -1,9 +1,10 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
-from datetime import datetime, timezone
+import tempfile
+from datetime import datetime, timedelta, timezone
from typing import Any
-from unittest.mock import MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from dirty_equals import IsDatetime, IsPositiveFloat
@@ -342,6 +343,63 @@ class TestObstore:
assert Obstore.is_compatible(42) is False
assert Obstore.is_compatible(None) is False
+ async def test_sign_download_url_returns_none_for_non_cloud_store(
+ self,
+ ) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ backend = self._make_backend(store)
+ result = await backend.sign_download_url("some/path.txt")
+ assert result is None
+
+ async def test_sign_download_url_returns_none_for_local_store(
+ self,
+ ) -> None:
+ from obstore.store import LocalStore
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ store = LocalStore(tmpdir)
+ backend = self._make_backend(store)
+ result = await backend.sign_download_url("some/path.txt")
+ assert result is None
+
+ async def test_sign_download_url_calls_sign_async_for_s3(self) -> None:
+ from obstore.store import S3Store
+
+ store = S3Store("test-bucket", skip_signature=True)
+ backend = self._make_backend(store)
+
+ with patch(
+ "obstore.sign_async",
+ new_callable=AsyncMock,
+ return_value="https://signed.example.com/file",
+ ) as mock_sign:
+ result = await backend.sign_download_url(
+ "data/file.csv", expiration=600
+ )
+
+ assert result == "https://signed.example.com/file"
+ mock_sign.assert_called_once()
+ args, kwargs = mock_sign.call_args
+ assert args == (store, "GET", "data/file.csv")
+ assert kwargs["expires_in"] == timedelta(seconds=600)
+
+ async def test_sign_download_url_returns_none_on_exception(self) -> None:
+ from obstore.store import S3Store
+
+ store = S3Store("test-bucket", skip_signature=True)
+ backend = self._make_backend(store)
+
+ with patch(
+ "obstore.sign_async",
+ new_callable=AsyncMock,
+ side_effect=RuntimeError("signing failed"),
+ ):
+ result = await backend.sign_download_url("data/file.csv")
+
+ assert result is None
+
def test_display_name_known_protocol(self) -> None:
from obstore.store import MemoryStore
@@ -677,6 +735,49 @@ class TestFsspecFilesystem:
backend = self._make_backend(mock_store)
assert backend.display_name == "Amazon S3"
+ async def test_sign_download_url_returns_signed_url(self) -> None:
+ mock_store = MagicMock()
+ mock_store.sign.return_value = "https://signed.example.com/path"
+
+ backend = self._make_backend(mock_store)
+ result = await backend.sign_download_url(
+ "bucket/file.csv", expiration=900
+ )
+
+ assert result == "https://signed.example.com/path"
+ mock_store.sign.assert_called_once_with(
+ "bucket/file.csv", expiration=900
+ )
+
+ async def test_sign_download_url_returns_none_on_not_implemented(
+ self,
+ ) -> None:
+ mock_store = MagicMock()
+ mock_store.sign.side_effect = NotImplementedError
+
+ backend = self._make_backend(mock_store)
+ result = await backend.sign_download_url("bucket/file.csv")
+
+ assert result is None
+
+ async def test_sign_download_url_returns_none_on_exception(self) -> None:
+ mock_store = MagicMock()
+ mock_store.sign.side_effect = RuntimeError("unexpected error")
+
+ backend = self._make_backend(mock_store)
+ result = await backend.sign_download_url("bucket/file.csv")
+
+ assert result is None
+
+ async def test_sign_download_url_converts_result_to_str(self) -> None:
+ mock_store = MagicMock()
+ mock_store.sign.return_value = 12345
+
+ backend = self._make_backend(mock_store)
+ result = await backend.sign_download_url("path")
+
+ assert result == "12345"
+
def test_display_name_unknown_protocol(self) -> None:
mock_store = MagicMock()
mock_store.protocol = "custom-proto"
@@ -739,6 +840,18 @@ class TestFsspecFilesystemIntegration:
)
)
+ async def test_sign_download_url_not_implemented_by_memory_fs(
+ self,
+ ) -> None:
+ from fsspec.implementations.memory import MemoryFileSystem
+
+ fs = MemoryFileSystem()
+ fs.pipe("/test/file.txt", b"hello")
+
+ backend = FsspecFilesystem(fs, VariableName("mem_fs"))
+ result = await backend.sign_download_url("/test/file.txt")
+ assert result is None
+
def test_protocol_memory_filesystem(self) -> None:
from fsspec.implementations.memory import MemoryFileSystem
@@ -809,6 +922,18 @@ class TestObstoreIntegration:
)
)
+ async def test_sign_download_url_returns_none_for_memory_store(
+ self,
+ ) -> None:
+ from obstore.store import MemoryStore
+
+ store = MemoryStore()
+ await store.put_async("data.txt", b"test")
+
+ backend = Obstore(store, VariableName("mem_store"))
+ result = await backend.sign_download_url("data.txt")
+ assert result is None
+
class TestNormalizeProtocol:
@pytest.mark.parametrize(
diff --git a/tests/_runtime/test_runtime_external_storage.py b/tests/_runtime/test_runtime_external_storage.py
index 6de995da94..ad8e6349bd 100644
--- a/tests/_runtime/test_runtime_external_storage.py
+++ b/tests/_runtime/test_runtime_external_storage.py
@@ -489,6 +489,149 @@ class TestExternalStorageCallbacks:
)
]
+ async def test_download_uses_signed_url_when_available(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(
+ [
+ ExecuteCellCommand(
+ cell_id=CellId_t("0"),
+ code=(
+ "from obstore.store import MemoryStore\n"
+ f"{STORAGE_VAR} = MemoryStore()"
+ ),
+ ),
+ ]
+ )
+
+ store = k.globals[VariableName(STORAGE_VAR)]
+ backend = Obstore(store, VariableName(STORAGE_VAR))
+ backend.sign_download_url = AsyncMock( # type: ignore[method-assign]
+ return_value="https://signed.example.com/data.csv?token=abc",
+ )
+ # download_file should NOT be called when signing succeeds
+ backend.download_file = AsyncMock( # type: ignore[method-assign]
+ side_effect=AssertionError("should not be called"),
+ )
+
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(
+ k.external_storage_callbacks,
+ "_get_storage_backend",
+ lambda _: (backend, None),
+ )
+ request = StorageDownloadCommand(
+ request_id=RequestId("req-22"),
+ namespace=STORAGE_VAR,
+ path="reports/data.csv",
+ )
+ await k.handle_message(request)
+
+ results = [
+ op
+ for op in stream.operations
+ if isinstance(op, StorageDownloadReadyNotification)
+ ]
+ assert results == [
+ StorageDownloadReadyNotification(
+ request_id=RequestId("req-22"),
+ url="https://signed.example.com/data.csv?token=abc",
+ filename="data.csv",
+ )
+ ]
+
+ async def test_download_falls_back_when_signing_returns_none(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(
+ [
+ ExecuteCellCommand(
+ cell_id=CellId_t("0"),
+ code=(
+ "from obstore.store import MemoryStore\n"
+ f"{STORAGE_VAR} = MemoryStore()"
+ ),
+ ),
+ ]
+ )
+
+ store = k.globals[VariableName(STORAGE_VAR)]
+ await store.put_async("local/readme.txt", b"hello world")
+
+ # MemoryStore can't sign, so sign_download_url returns None
+ # and the handler falls back to virtual file
+ request = StorageDownloadCommand(
+ request_id=RequestId("req-23"),
+ namespace=STORAGE_VAR,
+ path="local/readme.txt",
+ )
+ await k.handle_message(request)
+
+ results = [
+ op
+ for op in stream.operations
+ if isinstance(op, StorageDownloadReadyNotification)
+ ]
+ assert results == [
+ StorageDownloadReadyNotification(
+ request_id=RequestId("req-23"),
+ url=IsStr(), # pyright: ignore[reportArgumentType]
+ filename="readme.txt",
+ )
+ ]
+
+ async def test_download_fallback_schedules_cleanup(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ import asyncio
+
+ k = mocked_kernel.k
+
+ await k.run(
+ [
+ ExecuteCellCommand(
+ cell_id=CellId_t("0"),
+ code=(
+ "from obstore.store import MemoryStore\n"
+ f"{STORAGE_VAR} = MemoryStore()"
+ ),
+ ),
+ ]
+ )
+
+ store = k.globals[VariableName(STORAGE_VAR)]
+ await store.put_async("tmp/file.bin", b"data")
+
+ request = StorageDownloadCommand(
+ request_id=RequestId("req-24"),
+ namespace=STORAGE_VAR,
+ path="tmp/file.bin",
+ )
+
+ loop = asyncio.get_running_loop()
+ scheduled: list[tuple[float, object]] = []
+ original_call_later = loop.call_later
+
+ def spy_call_later(
+ delay: float, callback: object, *args: object
+ ) -> asyncio.TimerHandle:
+ scheduled.append((delay, callback))
+ return original_call_later(delay, callback, *args)
+
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(loop, "call_later", spy_call_later)
+ await k.handle_message(request)
+
+ assert len(scheduled) == 1
+ delay, _callback = scheduled[0]
+ assert delay == 60
+
async def test_download_backend_exception(
self, mocked_kernel: MockedKernel
) -> None: