A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat(storage): paginate remote storage listings with a "Load more" button (#9834)
Shahmir Varqhaefd920ddf08f
diff --git a/frontend/src/components/storage/storage-inspector.tsx b/frontend/src/components/storage/storage-inspector.tsxindex c40ccd04e1..e91b0043b8 100644--- a/frontend/src/components/storage/storage-inspector.tsx+++ b/frontend/src/components/storage/storage-inspector.tsx@@ -153,6 +153,33 @@ function filterEntries(return entries.filter((entry) => entryMatchesSearch(entry, context));}+const LoadMoreStorageEntries: React.FC<{+ depth: number;+ isLoading: boolean;+ error?: Error;+ onLoadMore: () => void;+}> = ({ depth, isLoading, error, onLoadMore }) => {+ return (+ <div className="py-px text-xs" style={indentStyle(depth)}>+ <Button+ variant="text"+ size="xs"+ className="h-6 px-0 hover:text-blue-600"+ disabled={isLoading}+ onClick={onLoadMore}+ >+ {isLoading && <LoaderCircle className="h-3 w-3 mr-1 animate-spin" />}+ {isLoading ? "Loading..." : "Load more"}+ </Button>+ {error && (+ <span className="ml-2 text-destructive">+ Failed to load: {error.message}+ </span>+ )}+ </div>+ );+};+/*** Lazily loaded children of a directory entry.* Caches fetched entries in the Jotai store so re-expanding doesn't re-fetch.@@ -183,6 +210,10 @@ const StorageEntryChildren: React.FC<{entries: children,isPending,error,+ hasMore,+ loadMore,+ isLoadingMore,+ loadMoreError,} = useStorageEntries(namespace, prefix);if (isPending) {@@ -242,6 +273,14 @@ const StorageEntryChildren: React.FC<{/>);})}+ {hasMore && (+ <LoadMoreStorageEntries+ depth={depth}+ isLoading={isLoadingMore}+ error={loadMoreError}+ onLoadMore={loadMore}+ />+ )}</>);};@@ -461,6 +500,10 @@ const StorageNamespaceSection: React.FC<{entries: fetchedEntries,isPending,error,+ hasMore,+ loadMore,+ isLoadingMore,+ loadMoreError,refetch,} = useStorageEntries(namespaceName);@@ -559,6 +602,14 @@ const StorageNamespaceSection: React.FC<{/>);})}+ {hasMore && (+ <LoadMoreStorageEntries+ depth={1}+ isLoading={isLoadingMore}+ error={loadMoreError}+ onLoadMore={loadMore}+ />+ )}</>)}</>diff --git a/frontend/src/core/storage/__tests__/state.test.ts b/frontend/src/core/storage/__tests__/state.test.tsindex 997e316789..851481a91f 100644--- a/frontend/src/core/storage/__tests__/state.test.ts+++ b/frontend/src/core/storage/__tests__/state.test.ts@@ -48,6 +48,7 @@ describe("storage state", () => {expect(state).toEqual({namespaces: [],entriesByPath: new Map(),+ pageMetadataByPath: new Map(),});});});@@ -134,6 +135,49 @@ describe("storage state", () => {expect(state.entriesByPath.get("my_s3::data/")).toEqual(entries);});+ it("should store next page tokens keyed by namespace and prefix", () => {+ const entries = [makeEntry({ path: "a.txt" })];++ actions.setEntries({+ namespace: "my_s3",+ prefix: "data/",+ entries,+ nextPageToken: "150",+ });++ expect(state.entriesByPath.get("my_s3::data/")).toEqual(entries);+ expect(state.pageMetadataByPath.get("my_s3::data/")?.nextPageToken).toBe(+ "150",+ );+ });++ it("should append entries when requested", () => {+ const firstPage = [makeEntry({ path: "a.txt" })];+ const secondPage = [makeEntry({ path: "b.txt" })];++ actions.setEntries({+ namespace: "my_s3",+ prefix: "data/",+ entries: firstPage,+ nextPageToken: "150",+ });+ actions.setEntries({+ namespace: "my_s3",+ prefix: "data/",+ entries: secondPage,+ nextPageToken: null,+ append: true,+ });++ expect(state.entriesByPath.get("my_s3::data/")).toEqual([+ ...firstPage,+ ...secondPage,+ ]);+ expect(+ state.pageMetadataByPath.get("my_s3::data/")?.nextPageToken,+ ).toBeNull();+ });+it("should use empty string for null prefix", () => {const entries = [makeEntry({ path: "root.txt" })];@@ -251,6 +295,11 @@ describe("storage state", () => {expect(state.entriesByPath.get("my_s3::")).toBeUndefined();expect(state.entriesByPath.get("my_s3::data/")).toBeUndefined();expect(state.entriesByPath.get("my_s3::data/nested/")).toBeUndefined();+ expect(state.pageMetadataByPath.get("my_s3::")).toBeUndefined();+ expect(state.pageMetadataByPath.get("my_s3::data/")).toBeUndefined();+ expect(+ state.pageMetadataByPath.get("my_s3::data/nested/"),+ ).toBeUndefined();});it("should not affect entries from other namespaces", () => {diff --git a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsxindex 8cd37b3434..abd577be36 100644--- a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx+++ b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx@@ -1,11 +1,15 @@/* Copyright 2026 Marimo. All rights reserved. */-import { renderHook, waitFor } from "@testing-library/react";+import { act, renderHook, waitFor } from "@testing-library/react";import { createStore, Provider } from "jotai";import type { ReactNode } from "react";import * as React from "react";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { storageAtom, useStorageEntries } from "../state";+import {+ storageAtom,+ useStorageEntries,+ useStoragePageFetcher,+} from "../state";import type { StorageEntry, StorageState } from "../types";const mockRequest = vi.fn();@@ -175,13 +179,205 @@ describe("useStorageEntries", () => {it("should store fetched entries in the atom", async () => {const entries = [makeEntry({ path: "new.txt" })];- mockRequest.mockResolvedValue({ entries });+ mockRequest.mockResolvedValue({+ entries,+ next_page_token: "150",+ });renderHook(() => useStorageEntries("ns", "sub/"), { wrapper });await waitFor(() => {const state = store.get(storageAtom);expect(state.entriesByPath.get("ns::sub/")).toEqual(entries);+ expect(state.pageMetadataByPath.get("ns::sub/")?.nextPageToken).toBe(+ "150",+ );+ });+ });++ it("should load more entries when a next page token exists", async () => {+ const firstPage = [makeEntry({ path: "a.txt" })];+ const secondPage = [makeEntry({ path: "b.txt" })];+ mockRequest+ .mockResolvedValueOnce({+ entries: firstPage,+ next_page_token: "150",+ })+ .mockResolvedValueOnce({+ entries: secondPage,+ next_page_token: null,+ });++ const { result } = renderHook(() => useStorageEntries("ns", "sub/"), {+ wrapper,+ });++ await waitFor(() => {+ expect(result.current.entries).toEqual(firstPage);+ });+ expect(result.current.hasMore).toBe(true);++ await act(async () => {+ await result.current.loadMore();+ });++ expect(result.current.entries).toEqual([...firstPage, ...secondPage]);+ expect(result.current.hasMore).toBe(false);+ expect(mockRequest).toHaveBeenLastCalledWith({+ namespace: "ns",+ prefix: "sub/",+ limit: 150,+ pageToken: "150",+ });+ });++ it("should load more entries when the next page token is an empty string", async () => {+ const firstPage = [makeEntry({ path: "a.txt" })];+ const secondPage = [makeEntry({ path: "b.txt" })];+ mockRequest+ .mockResolvedValueOnce({+ entries: firstPage,+ next_page_token: "",+ })+ .mockResolvedValueOnce({+ entries: secondPage,+ next_page_token: null,+ });++ const { result } = renderHook(() => useStorageEntries("ns", "sub/"), {+ wrapper,+ });++ await waitFor(() => {+ expect(result.current.entries).toEqual(firstPage);+ });+ expect(result.current.hasMore).toBe(true);++ await act(async () => {+ await result.current.loadMore();+ });++ expect(result.current.entries).toEqual([...firstPage, ...secondPage]);+ expect(result.current.hasMore).toBe(false);+ expect(mockRequest).toHaveBeenLastCalledWith({+ namespace: "ns",+ prefix: "sub/",+ limit: 150,+ pageToken: "",+ });+ });++ it("should ignore duplicate load more calls while a page is loading", async () => {+ const firstPage = [makeEntry({ path: "a.txt" })];+ const secondPage = [makeEntry({ path: "b.txt" })];+ let resolveLoadMore!: (value: {+ entries: StorageEntry[];+ next_page_token: string | null;+ }) => void;+ const loadMorePromise = new Promise<{+ entries: StorageEntry[];+ next_page_token: string | null;+ }>((resolve) => {+ resolveLoadMore = resolve;+ });+ mockRequest+ .mockResolvedValueOnce({+ entries: firstPage,+ next_page_token: "150",+ })+ .mockReturnValueOnce(loadMorePromise);++ const { result } = renderHook(() => useStorageEntries("ns", "sub/"), {+ wrapper,+ });++ await waitFor(() => {+ expect(result.current.entries).toEqual(firstPage);+ });++ await act(async () => {+ const firstLoad = result.current.loadMore();+ const secondLoad = result.current.loadMore();++ expect(mockRequest).toHaveBeenCalledTimes(2);+ resolveLoadMore({+ entries: secondPage,+ next_page_token: null,+ });+ await Promise.all([firstLoad, secondLoad]);+ });++ expect(result.current.entries).toEqual([...firstPage, ...secondPage]);+ expect(mockRequest).toHaveBeenCalledTimes(2);+ });++ it("should fetch arbitrary storage pages", async () => {+ const firstPage = [makeEntry({ path: "folder/a.txt" })];+ const secondPage = [makeEntry({ path: "folder/b.txt" })];+ mockRequest+ .mockResolvedValueOnce({+ entries: firstPage,+ next_page_token: "150",+ })+ .mockResolvedValueOnce({+ entries: secondPage,+ next_page_token: null,+ });++ const { result } = renderHook(() => useStoragePageFetcher(), {+ wrapper,+ });++ await act(async () => {+ await result.current({+ namespace: "ns",+ prefix: "folder/",+ });+ });+ await act(async () => {+ await result.current({+ namespace: "ns",+ prefix: "folder/",+ pageToken: "150",+ append: true,+ });+ });++ expect(store.get(storageAtom).entriesByPath.get("ns::folder/")).toEqual([+ ...firstPage,+ ...secondPage,+ ]);+ expect(mockRequest).toHaveBeenLastCalledWith({+ namespace: "ns",+ prefix: "folder/",+ limit: 150,+ pageToken: "150",+ });+ });++ it("should forward an empty string page token", async () => {+ mockRequest.mockResolvedValue({+ entries: [],+ next_page_token: null,+ });++ const { result } = renderHook(() => useStoragePageFetcher(), {+ wrapper,+ });++ await act(async () => {+ await result.current({+ namespace: "ns",+ prefix: "folder/",+ pageToken: "",+ });+ });++ expect(mockRequest).toHaveBeenLastCalledWith({+ namespace: "ns",+ prefix: "folder/",+ limit: 150,+ pageToken: "",});});});diff --git a/frontend/src/core/storage/state.ts b/frontend/src/core/storage/state.tsindex 1a49431b3d..4aed690ee7 100644--- a/frontend/src/core/storage/state.ts+++ b/frontend/src/core/storage/state.ts@@ -1,11 +1,15 @@/* Copyright 2026 Marimo. All rights reserved. */import { atom, useAtomValue } from "jotai";+import { useCallback, useRef, useState } from "react";import { useAsyncData } from "@/hooks/useAsyncData";import { createReducerAndAtoms } from "@/utils/createReducer";import type { NotificationMessageData } from "../kernel/messages";import type { VariableName } from "../variables/types";-import { ListStorageEntries } from "./request-registry";+import {+ ListStorageEntries,+ type StorageEntriesResult,+} from "./request-registry";import type { StorageEntry, StorageState } from "./types";import {DEFAULT_FETCH_LIMIT,@@ -18,6 +22,7 @@ function initialState(): StorageState {return {namespaces: [],entriesByPath: new Map(),+ pageMetadataByPath: new Map(),};}@@ -48,23 +53,48 @@ const {namespace: string;prefix: string | null | undefined;entries: StorageEntry[];+ nextPageToken?: string | null;+ append?: boolean;},) => {const key = storagePathKey(opts.namespace, opts.prefix);const entriesByPath = new Map(state.entriesByPath);- entriesByPath.set(key, opts.entries);- return { ...state, entriesByPath };+ const entries =+ opts.append && entriesByPath.has(key)+ ? [...(entriesByPath.get(key) ?? []), ...opts.entries]+ : opts.entries;+ entriesByPath.set(key, entries);++ const pageMetadataByPath = new Map(state.pageMetadataByPath);+ pageMetadataByPath.set(key, {+ nextPageToken: opts.nextPageToken ?? null,+ });+ return {+ ...state,+ entriesByPath,+ pageMetadataByPath,+ };},clearNamespaceCache: (state, namespace: string) => {const entriesByPath = new Map(state.entriesByPath);+ const pageMetadataByPath = new Map(state.pageMetadataByPath);const prefix = storageNamespacePrefix(namespace);for (const key of entriesByPath.keys()) {if (key.startsWith(prefix)) {entriesByPath.delete(key);}}- return { ...state, entriesByPath };+ for (const key of pageMetadataByPath.keys()) {+ if (key.startsWith(prefix)) {+ pageMetadataByPath.delete(key);+ }+ }+ return {+ ...state,+ entriesByPath,+ pageMetadataByPath,+ };},filterFromVariables: (state, variableNames: VariableName[]) => {@@ -93,38 +123,119 @@ export function useStorageActions() {export { storageAtom };+async function fetchStorageEntriesPage({+ namespace,+ prefix,+ pageToken,+ append,+ setEntries,+}: {+ namespace: string;+ prefix: string | null | undefined;+ pageToken?: string | null;+ append?: boolean;+ setEntries: ReturnType<typeof useStorageActions>["setEntries"];+}): Promise<StorageEntriesResult> {+ const result = await ListStorageEntries.request({+ namespace,+ prefix: prefix ?? ROOT_PATH,+ limit: DEFAULT_FETCH_LIMIT,+ ...(pageToken != null ? { pageToken } : {}),+ });+ if (result.error) {+ throw new Error(result.error);+ }+ setEntries({+ namespace,+ prefix,+ entries: result.entries,+ nextPageToken: result.next_page_token,+ append,+ });+ return result;+}+/*** Hook that fetches and caches storage entries for a given namespace/prefix.* Entries are fetched on first access and cached in the store for subsequent renders.*/export function useStorageEntries(namespace: string, prefix?: string) {- const { entriesByPath } = useStorage();+ const { entriesByPath, pageMetadataByPath } = useStorage();const { setEntries } = useStorageActions();- const cached = entriesByPath.get(storagePathKey(namespace, prefix));+ const key = storagePathKey(namespace, prefix);+ const cached = entriesByPath.get(key);+ const metadata = pageMetadataByPath.get(key);+ const nextPageToken = metadata?.nextPageToken ?? null;+ const isLoadingMoreRef = useRef(false);+ const [isLoadingMore, setIsLoadingMore] = useState(false);+ const [loadMoreError, setLoadMoreError] = useState<Error | undefined>();const { isPending, error, refetch } = useAsyncData(async () => {if (cached) {return;}- const result = await ListStorageEntries.request({+ await fetchStorageEntriesPage({namespace,- prefix: prefix ?? ROOT_PATH,- limit: DEFAULT_FETCH_LIMIT,+ prefix,+ setEntries,});- if (result.error) {- throw new Error(result.error);- }- setEntries({ namespace, prefix, entries: result.entries });}, [namespace, prefix, cached === undefined]);+ const loadMore = useCallback(async () => {+ if (nextPageToken == null || isLoadingMoreRef.current) {+ return;+ }++ isLoadingMoreRef.current = true;+ setIsLoadingMore(true);+ setLoadMoreError(undefined);+ try {+ await fetchStorageEntriesPage({+ namespace,+ prefix,+ pageToken: nextPageToken,+ append: true,+ setEntries,+ });+ } catch (error) {+ setLoadMoreError(+ error instanceof Error ? error : new Error(String(error)),+ );+ } finally {+ isLoadingMoreRef.current = false;+ setIsLoadingMore(false);+ }+ }, [namespace, prefix, nextPageToken, setEntries]);+return {entries: cached ?? [],isPending: isPending && !cached,error: cached ? undefined : error,+ hasMore: nextPageToken !== null,+ loadMore,+ isLoadingMore,+ loadMoreError,refetch,};}+export function useStoragePageFetcher() {+ const { setEntries } = useStorageActions();+ return useCallback(+ (opts: {+ namespace: string;+ prefix: string | null | undefined;+ pageToken?: string | null;+ append?: boolean;+ }) =>+ fetchStorageEntriesPage({+ ...opts,+ setEntries,+ }),+ [setEntries],+ );+}+export const exportedForTesting = {reducer,createActions,diff --git a/frontend/src/core/storage/types.ts b/frontend/src/core/storage/types.tsindex f8514c947a..3dd5cb3682 100644--- a/frontend/src/core/storage/types.ts+++ b/frontend/src/core/storage/types.ts@@ -30,6 +30,12 @@ export interface StorageState {namespaces: StorageNamespace[];/** Lazy-loaded entries keyed by "namespace::prefix" */entriesByPath: ReadonlyMap<StoragePathKey, StorageEntry[]>;+ /** Pagination metadata keyed by "namespace::prefix" */+ pageMetadataByPath: ReadonlyMap<StoragePathKey, StoragePageMetadata>;+}++export interface StoragePageMetadata {+ nextPageToken: string | null;}export function storageUrl(diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.pyindex 9d3257fccc..78b021550e 100644--- a/marimo/_data/_external_storage/models.py+++ b/marimo/_data/_external_storage/models.py@@ -60,6 +60,14 @@ class StorageNamespace(msgspec.Struct, rename="camel"):DEFAULT_FETCH_LIMIT = 100+@dataclass+class StorageListResult:+ """Result of listing entries from a storage backend."""++ entries: list[StorageEntry]+ next_page_token: str | None = None++@dataclassclass DownloadResult:"""Result of downloading a file from external storage.@@ -90,9 +98,11 @@ class StorageBackend(abc.ABC, Generic[Backend]):prefix: str | None,*,limit: int = DEFAULT_FETCH_LIMIT,- ) -> list[StorageEntry]:+ page_token: str | None = None,+ ) -> StorageListResult:"""- List the entries at the given prefix. If no prefix is provided, list the root entries.+ List one page of entries at the given prefix. If no prefix is provided,+ list the root entries."""@abc.abstractmethoddiff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.pyindex 5074a1c64f..bc8e7cdf2e 100644--- a/marimo/_data/_external_storage/storage.py+++ b/marimo/_data/_external_storage/storage.py@@ -15,6 +15,7 @@ from marimo._data._external_storage.models import (BackendType,StorageBackend,StorageEntry,+ StorageListResult,)from marimo._dependencies.dependencies import DependencyManagerfrom marimo._utils.assert_never import log_never@@ -44,7 +45,17 @@ class Obstore(StorageBackend["ObjectStore"]):prefix: str | None,*,limit: int = DEFAULT_FETCH_LIMIT,- ) -> list[StorageEntry]:+ page_token: str | None = None,+ ) -> StorageListResult:+ offset = _parse_page_offset(page_token)+ storage_entries = self._list_storage_entries(prefix)+ return _paginate_entries(storage_entries, offset=offset, limit=limit)++ def _list_storage_entries(self, prefix: str | None) -> list[StorageEntry]:+ # Object stores commonly cap a single delimiter listing at ~1000+ # entries, and we don't page beyond one listing, so directories with+ # more entries than that will be silently truncated here.+ # https://docs.rs/object_store/latest/object_store/struct.ListResult.htmlresult = self.store.list_with_delimiter(prefix=prefix)storage_entries: list[StorageEntry] = []@@ -71,14 +82,6 @@ class Obstore(StorageBackend["ObjectStore"]):continuestorage_entries.append(self._create_storage_entry(entry))- if len(storage_entries) > limit:- LOGGER.debug(- "Fetched %s entries, but limiting to %s",- len(storage_entries),- limit,- )- storage_entries = storage_entries[:limit]-return storage_entriesasync def get_entry(self, path: str) -> StorageEntry:@@ -247,7 +250,13 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):prefix: str | None,*,limit: int = DEFAULT_FETCH_LIMIT,- ) -> list[StorageEntry]:+ page_token: str | None = None,+ ) -> StorageListResult:+ offset = _parse_page_offset(page_token)+ entries = self._list_storage_entries(prefix)+ return _paginate_entries(entries, offset=offset, limit=limit)++ def _list_storage_entries(self, prefix: str | None) -> list[StorageEntry]:# If no prefix provided, we use empty string to list root entries# Else, an error is raisedif prefix is None:@@ -263,15 +272,6 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):self._invalidate_listing_cache_for_prefix(prefix)files = self._list_files(prefix)- total_files = len(files)- if total_files > limit:- LOGGER.debug(- "Fetched %s files, but limiting to %s",- total_files,- limit,- )- files = files[:limit]-storage_entries = []for file in files:if isinstance(file, dict):@@ -512,3 +512,42 @@ def detect_protocol_from_url(url: str) -> CLOUD_STORAGE_TYPES | None:def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | None:"""Normalize a protocol string (e.g. 's3a', 'gs', 'abfs') to a known storage type."""return _PROTOCOL_MAP.get(protocol.strip().lower())+++def _parse_page_offset(page_token: str | None) -> int:+ if page_token is None:+ return 0+ try:+ offset = int(page_token)+ except ValueError as exc:+ raise ValueError(f"Invalid storage page token: {page_token}") from exc+ if offset < 0:+ raise ValueError(f"Invalid storage page token: {page_token}")+ return offset+++def _paginate_entries(+ entries: list[StorageEntry],+ *,+ offset: int,+ limit: int,+) -> StorageListResult:+ if limit < 1:+ raise ValueError("Storage list limit must be positive")++ total_entries = len(entries)+ if total_entries > limit:+ LOGGER.debug(+ "Fetched %s entries, returning page offset %s with limit %s",+ total_entries,+ offset,+ limit,+ )++ end = offset + limit+ has_next_page = end < total_entries+ next_page_token = str(end) if has_next_page else None+ return StorageListResult(+ entries=entries[offset:end],+ next_page_token=next_page_token,+ )diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.pyindex 155f49e8d4..554f880752 100644--- a/marimo/_messaging/notification.py+++ b/marimo/_messaging/notification.py@@ -695,6 +695,7 @@ class StorageEntriesNotification(Notification, tag="storage-entries"):namespace: Variable name of the storage backend.prefix: The prefix that was listed (set by list_entries).query: The search query that was used (set by search).+ next_page_token: Token for fetching the next page of entries.error: Error message if the operation failed."""@@ -704,6 +705,7 @@ class StorageEntriesNotification(Notification, tag="storage-entries"):namespace: strprefix: str | None = Nonequery: str | None = None+ next_page_token: str | None = Noneerror: str | None = Nonediff --git a/marimo/_runtime/callbacks/external_storage.py b/marimo/_runtime/callbacks/external_storage.pyindex d76347bc13..2bb21f7c2f 100644--- a/marimo/_runtime/callbacks/external_storage.py+++ b/marimo/_runtime/callbacks/external_storage.py@@ -6,7 +6,10 @@ import osfrom typing import TYPE_CHECKING, Anyfrom marimo import _loggers-from marimo._data._external_storage.models import StorageBackend, StorageEntry+from marimo._data._external_storage.models import (+ StorageBackend,+ StorageListResult,+)from marimo._messaging.notification import (StorageDownloadReadyNotification,StorageEntriesNotification,@@ -92,19 +95,22 @@ class ExternalStorageCallbacks:return# list_entries is synchronous, so we wrap it in asyncio.to_thread- def list_entries() -> list[StorageEntry]:+ def list_entries() -> StorageListResult:return backend.list_entries(- prefix=request.prefix, limit=request.limit+ prefix=request.prefix,+ limit=request.limit,+ page_token=request.page_token,)try:- entries = await asyncio.to_thread(list_entries)+ result = await asyncio.to_thread(list_entries)broadcast_notification(StorageEntriesNotification(request_id=request.request_id,- entries=entries,+ entries=result.entries,namespace=request.namespace,prefix=request.prefix,+ next_page_token=result.next_page_token,),)except Exception as e:diff --git a/marimo/_runtime/commands.py b/marimo/_runtime/commands.pyindex 32d6397a7b..0b860176c9 100644--- a/marimo/_runtime/commands.py+++ b/marimo/_runtime/commands.py@@ -757,12 +757,14 @@ class StorageListEntriesCommand(Command):namespace: Variable name identifying the storage backend.limit: Max entries to return.prefix: Path prefix to list (None = root).+ page_token: Token for the next page of entries."""request_id: RequestIdnamespace: strlimit: intprefix: str | None = None+ page_token: str | None = Noneclass StorageDownloadCommand(Command):diff --git a/marimo/_server/models/models.py b/marimo/_server/models/models.pyindex 13811cae38..0862124a8f 100644--- a/marimo/_server/models/models.py+++ b/marimo/_server/models/models.py@@ -155,6 +155,7 @@ class StorageListEntriesRequest(StorageListEntriesCommand, tag=False):namespace=self.namespace,limit=self.limit,prefix=self.prefix,+ page_token=self.page_token,)diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex 7b86d7e5b9..5dc322bb37 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -4946,7 +4946,8 @@ components:\ entries returned by the operation.\n namespace: Variable name of\\ the storage backend.\n prefix: The prefix that was listed (set by\\ list_entries).\n query: The search query that was used (set by search).\n\- \ error: Error message if the operation failed."+ \ next_page_token: Token for fetching the next page of entries.\n \+ \ error: Error message if the operation failed."properties:entries:items:@@ -4959,6 +4960,11 @@ components:default: nullnamespace:type: string+ next_page_token:+ anyOf:+ - type: string+ - type: 'null'+ default: nullop:enum:- storage-entries@@ -5023,12 +5029,18 @@ components:\ and virtual directories at one level.\n\n Attributes:\n request_id:\\ Unique identifier for this request.\n namespace: Variable name identifying\\ the storage backend.\n limit: Max entries to return.\n prefix:\- \ Path prefix to list (None = root)."+ \ Path prefix to list (None = root).\n page_token: Token for the next\+ \ page of entries."properties:limit:type: integernamespace:type: string+ pageToken:+ anyOf:+ - type: string+ - type: 'null'+ default: nullprefix:anyOf:- type: string@@ -5052,6 +5064,11 @@ components:type: integernamespace:type: string+ pageToken:+ anyOf:+ - type: string+ - type: 'null'+ default: nullprefix:anyOf:- type: stringdiff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex 6d57f54158..befb476eba 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -6433,6 +6433,7 @@ export interface components {* namespace: Variable name of the storage backend.* prefix: The prefix that was listed (set by list_entries).* query: The search query that was used (set by search).+ * next_page_token: Token for fetching the next page of entries.* error: Error message if the operation failed.*/StorageEntriesNotification: {@@ -6440,6 +6441,8 @@ export interface components {/** @default null */error?: string | null;namespace: string;+ /** @default null */+ next_page_token?: string | null;/** @enum {unknown} */op: "storage-entries";/** @default null */@@ -6483,11 +6486,14 @@ export interface components {* namespace: Variable name identifying the storage backend.* limit: Max entries to return.* prefix: Path prefix to list (None = root).+ * page_token: Token for the next page of entries.*/StorageListEntriesCommand: {limit: number;namespace: string;/** @default null */+ pageToken?: string | null;+ /** @default null */prefix?: string | null;requestId: string;/** @enum {unknown} */@@ -6498,6 +6504,8 @@ export interface components {limit: number;namespace: string;/** @default null */+ pageToken?: string | null;+ /** @default null */prefix?: string | null;requestId: components["schemas"]["RequestId"];};diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.pyindex f1e902c020..f2612b7975 100644--- a/tests/_data/_external_storage/test_storage_models.py+++ b/tests/_data/_external_storage/test_storage_models.py@@ -10,7 +10,11 @@ import pytestfrom dirty_equals import IsDatetime, IsPositiveFloatfrom inline_snapshot import snapshot-from marimo._data._external_storage.models import DownloadResult, StorageEntry+from marimo._data._external_storage.models import (+ DownloadResult,+ StorageEntry,+ StorageListResult,+)from marimo._data._external_storage.storage import (FsspecFilesystem,Obstore,@@ -58,7 +62,7 @@ class TestObstore:mock_store.list_with_delimiter.assert_called_once_with(prefix="some/prefix",)- assert result == snapshot(+ assert result.entries == snapshot([StorageEntry(path="subdir/",@@ -113,7 +117,7 @@ class TestObstore:backend = self._make_backend(mock_store)result = backend.list_entries(prefix="folder")- assert result == [+ assert result.entries == [StorageEntry(path="folder/order_details.csv",kind="object",@@ -133,7 +137,67 @@ class TestObstore:backend = self._make_backend(mock_store)result = backend.list_entries(prefix=None)- assert result == []+ assert result.entries == []++ def test_list_entries_returns_next_page_token(self) -> None:+ now = datetime.now(tz=timezone.utc)+ mock_store = MagicMock()+ mock_store.list_with_delimiter.return_value = {+ "common_prefixes": ["a/", "b/"],+ "objects": [+ {+ "path": "c.txt",+ "size": 1,+ "last_modified": now,+ "e_tag": None,+ "version": None,+ },+ ],+ }++ backend = self._make_backend(mock_store)++ assert backend.list_entries(prefix=None, limit=2) == snapshot(+ StorageListResult(+ entries=[+ StorageEntry(+ path="a/",+ kind="directory",+ size=0,+ last_modified=None,+ metadata={},+ mime_type=None,+ ),+ StorageEntry(+ path="b/",+ kind="directory",+ size=0,+ last_modified=None,+ metadata={},+ mime_type=None,+ ),+ ],+ next_page_token="2",+ )+ )++ assert backend.list_entries(+ prefix=None, limit=2, page_token="2"+ ) == snapshot(+ StorageListResult(+ entries=[+ StorageEntry(+ path="c.txt",+ kind="object",+ size=1,+ last_modified=now.timestamp(),+ metadata={},+ mime_type="text/plain",+ ),+ ],+ next_page_token=None,+ )+ )def test_create_storage_entry_missing_fields(self) -> None:mock_store = MagicMock()@@ -586,7 +650,7 @@ class TestFsspecFilesystem:result = backend.list_entries(prefix="some/path")mock_store.ls.assert_called_once_with(path="some/path", detail=True)- assert result == snapshot(+ assert result.entries == snapshot([StorageEntry(path="file1.txt",@@ -646,7 +710,7 @@ class TestFsspecFilesystem:assert mock_store.ls.call_count == 2assert "" not in mock_store.dircacheassert "folder" not in mock_store.dircache- assert result == snapshot(+ assert result.entries == snapshot([StorageEntry(path="folder/file.txt",@@ -690,7 +754,7 @@ class TestFsspecFilesystem:result = backend.list_entries(prefix="folder")mock_store.ls.assert_called_once_with(path="folder", detail=True)- assert result == snapshot(+ assert result.entries == snapshot([StorageEntry(path="folder",@@ -735,7 +799,7 @@ class TestFsspecFilesystem:backend = self._make_backend(mock_store)result = backend.list_entries(prefix="", limit=3)- assert result == snapshot(+ assert result.entries == snapshot([StorageEntry(path="file0.txt",@@ -764,6 +828,63 @@ class TestFsspecFilesystem:])+ def test_list_entries_returns_next_page_token(self) -> None:+ mock_store = MagicMock()+ files = [+ {+ "name": f"file{i}.txt",+ "size": i,+ "type": "file",+ "mtime": None,+ }+ for i in range(3)+ ]+ mock_store.ls.return_value = files++ backend = self._make_backend(mock_store)++ assert backend.list_entries(prefix="", limit=2) == snapshot(+ StorageListResult(+ entries=[+ StorageEntry(+ path="file0.txt",+ kind="file",+ size=0,+ last_modified=None,+ metadata={},+ mime_type="text/plain",+ ),+ StorageEntry(+ path="file1.txt",+ kind="file",+ size=1,+ last_modified=None,+ metadata={},+ mime_type="text/plain",+ ),+ ],+ next_page_token="2",+ )+ )++ assert backend.list_entries(+ prefix="", limit=2, page_token="2"+ ) == snapshot(+ StorageListResult(+ entries=[+ StorageEntry(+ path="file2.txt",+ kind="file",+ size=2,+ last_modified=None,+ metadata={},+ mime_type="text/plain",+ ),+ ],+ next_page_token=None,+ )+ )+def test_list_entries_raises_on_non_list(self) -> None:mock_store = MagicMock()mock_store.ls.return_value = "not_a_list"@@ -783,7 +904,7 @@ class TestFsspecFilesystem:backend = self._make_backend(mock_store)result = backend.list_entries(prefix="")- assert result == snapshot(+ assert result.entries == snapshot([StorageEntry(path="good.txt",@@ -1129,7 +1250,7 @@ class TestFsspecFilesystemIntegration:backend = FsspecFilesystem(fs, VariableName("mem_fs"))entries = backend.list_entries(prefix="/test")- assert entries == snapshot(+ assert entries.entries == snapshot([StorageEntry(path="/test/hello.txt",@@ -1253,7 +1374,7 @@ class TestObstoreIntegration:backend = Obstore(store, VariableName("mem_store"))entries = backend.list_entries(prefix="test/")- assert entries == snapshot(+ assert entries.entries == snapshot([StorageEntry(path="test/file1.txt",diff --git a/tests/_runtime/test_runtime_external_storage.py b/tests/_runtime/test_runtime_external_storage.pyindex 7268072cfe..416942d625 100644--- a/tests/_runtime/test_runtime_external_storage.py+++ b/tests/_runtime/test_runtime_external_storage.py@@ -8,7 +8,10 @@ from unittest.mock import AsyncMock, MagicMockimport pytestfrom dirty_equals import IsPositiveFloat, IsStr-from marimo._data._external_storage.models import StorageEntry+from marimo._data._external_storage.models import (+ StorageEntry,+ StorageListResult,+)from marimo._data._external_storage.storage import Obstorefrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._messaging.notification import (@@ -290,6 +293,40 @@ class TestExternalStorageCallbacks:],namespace=STORAGE_VAR,prefix=None,+ next_page_token="2",+ )+ ]++ request = StorageListEntriesCommand(+ request_id=RequestId("req-11-page-2"),+ namespace=STORAGE_VAR,+ limit=2,+ prefix=None,+ page_token="2",+ )+ await k.handle_message(request)++ results = [+ op+ for op in stream.operations+ if isinstance(op, StorageEntriesNotification)+ and op.request_id == RequestId("req-11-page-2")+ ]+ assert results == [+ StorageEntriesNotification(+ request_id=RequestId("req-11-page-2"),+ entries=[+ StorageEntry(+ path="c.txt",+ kind="object",+ size=1,+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]+ metadata={"e_tag": IsStr()},+ mime_type="text/plain",+ ),+ ],+ namespace=STORAGE_VAR,+ prefix=None,)]@@ -413,11 +450,14 @@ class TestExternalStorageCallbacks:original_list = real_backend.list_entriesdef spy_list_entries(- prefix: str | None, *, limit: int = 100- ) -> list[StorageEntry]:+ prefix: str | None,+ *,+ limit: int = 100,+ page_token: str | None = None,+ ) -> StorageListResult:nonlocal call_thread_namecall_thread_name = threading.current_thread().name- return original_list(prefix, limit=limit)+ return original_list(prefix, limit=limit, page_token=page_token)real_backend.list_entries = spy_list_entries # type: ignore[method-assign]