Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(storage): paginate remote storage listings with a "Load more" button (#9834)

Shahmir Varqhaefd920ddf08f
diff --git a/frontend/src/components/storage/storage-inspector.tsx b/frontend/src/components/storage/storage-inspector.tsx
index c40ccd04e1..e91b0043b8 100644
--- a/frontend/src/components/storage/storage-inspector.tsx
+++ b/frontend/src/components/storage/storage-inspector.tsx
@@ -153,6 +153,33 @@ function filterEntries(
return entries.filter((entry) => entryMatchesSearch(entry, context));
}
+const LoadMoreStorageEntries: React.FC<{
+ depth: number;
+ isLoading: boolean;
+ error?: Error;
+ onLoadMore: () => void;
+}> = ({ depth, isLoading, error, onLoadMore }) => {
+ return (
+ <div className="py-px text-xs" style={indentStyle(depth)}>
+ <Button
+ variant="text"
+ size="xs"
+ className="h-6 px-0 hover:text-blue-600"
+ disabled={isLoading}
+ onClick={onLoadMore}
+ >
+ {isLoading && <LoaderCircle className="h-3 w-3 mr-1 animate-spin" />}
+ {isLoading ? "Loading..." : "Load more"}
+ </Button>
+ {error && (
+ <span className="ml-2 text-destructive">
+ Failed to load: {error.message}
+ </span>
+ )}
+ </div>
+ );
+};
+
/**
* Lazily loaded children of a directory entry.
* Caches fetched entries in the Jotai store so re-expanding doesn't re-fetch.
@@ -183,6 +210,10 @@ const StorageEntryChildren: React.FC<{
entries: children,
isPending,
error,
+ hasMore,
+ loadMore,
+ isLoadingMore,
+ loadMoreError,
} = useStorageEntries(namespace, prefix);
if (isPending) {
@@ -242,6 +273,14 @@ const StorageEntryChildren: React.FC<{
/>
);
})}
+ {hasMore && (
+ <LoadMoreStorageEntries
+ depth={depth}
+ isLoading={isLoadingMore}
+ error={loadMoreError}
+ onLoadMore={loadMore}
+ />
+ )}
</>
);
};
@@ -461,6 +500,10 @@ const StorageNamespaceSection: React.FC<{
entries: fetchedEntries,
isPending,
error,
+ hasMore,
+ loadMore,
+ isLoadingMore,
+ loadMoreError,
refetch,
} = useStorageEntries(namespaceName);
@@ -559,6 +602,14 @@ const StorageNamespaceSection: React.FC<{
/>
);
})}
+ {hasMore && (
+ <LoadMoreStorageEntries
+ depth={1}
+ isLoading={isLoadingMore}
+ error={loadMoreError}
+ onLoadMore={loadMore}
+ />
+ )}
</>
)}
</>
diff --git a/frontend/src/core/storage/__tests__/state.test.ts b/frontend/src/core/storage/__tests__/state.test.ts
index 997e316789..851481a91f 100644
--- a/frontend/src/core/storage/__tests__/state.test.ts
+++ b/frontend/src/core/storage/__tests__/state.test.ts
@@ -48,6 +48,7 @@ describe("storage state", () => {
expect(state).toEqual({
namespaces: [],
entriesByPath: new Map(),
+ pageMetadataByPath: new Map(),
});
});
});
@@ -134,6 +135,49 @@ describe("storage state", () => {
expect(state.entriesByPath.get("my_s3::data/")).toEqual(entries);
});
+ it("should store next page tokens keyed by namespace and prefix", () => {
+ const entries = [makeEntry({ path: "a.txt" })];
+
+ actions.setEntries({
+ namespace: "my_s3",
+ prefix: "data/",
+ entries,
+ nextPageToken: "150",
+ });
+
+ expect(state.entriesByPath.get("my_s3::data/")).toEqual(entries);
+ expect(state.pageMetadataByPath.get("my_s3::data/")?.nextPageToken).toBe(
+ "150",
+ );
+ });
+
+ it("should append entries when requested", () => {
+ const firstPage = [makeEntry({ path: "a.txt" })];
+ const secondPage = [makeEntry({ path: "b.txt" })];
+
+ actions.setEntries({
+ namespace: "my_s3",
+ prefix: "data/",
+ entries: firstPage,
+ nextPageToken: "150",
+ });
+ actions.setEntries({
+ namespace: "my_s3",
+ prefix: "data/",
+ entries: secondPage,
+ nextPageToken: null,
+ append: true,
+ });
+
+ expect(state.entriesByPath.get("my_s3::data/")).toEqual([
+ ...firstPage,
+ ...secondPage,
+ ]);
+ expect(
+ state.pageMetadataByPath.get("my_s3::data/")?.nextPageToken,
+ ).toBeNull();
+ });
+
it("should use empty string for null prefix", () => {
const entries = [makeEntry({ path: "root.txt" })];
@@ -251,6 +295,11 @@ describe("storage state", () => {
expect(state.entriesByPath.get("my_s3::")).toBeUndefined();
expect(state.entriesByPath.get("my_s3::data/")).toBeUndefined();
expect(state.entriesByPath.get("my_s3::data/nested/")).toBeUndefined();
+ expect(state.pageMetadataByPath.get("my_s3::")).toBeUndefined();
+ expect(state.pageMetadataByPath.get("my_s3::data/")).toBeUndefined();
+ expect(
+ state.pageMetadataByPath.get("my_s3::data/nested/"),
+ ).toBeUndefined();
});
it("should not affect entries from other namespaces", () => {
diff --git a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx
index 8cd37b3434..abd577be36 100644
--- a/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx
+++ b/frontend/src/core/storage/__tests__/useStorageEntries.test.tsx
@@ -1,11 +1,15 @@
/* Copyright 2026 Marimo. All rights reserved. */
-import { renderHook, waitFor } from "@testing-library/react";
+import { act, renderHook, waitFor } from "@testing-library/react";
import { createStore, Provider } from "jotai";
import type { ReactNode } from "react";
import * as React from "react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { storageAtom, useStorageEntries } from "../state";
+import {
+ storageAtom,
+ useStorageEntries,
+ useStoragePageFetcher,
+} from "../state";
import type { StorageEntry, StorageState } from "../types";
const mockRequest = vi.fn();
@@ -175,13 +179,205 @@ describe("useStorageEntries", () => {
it("should store fetched entries in the atom", async () => {
const entries = [makeEntry({ path: "new.txt" })];
- mockRequest.mockResolvedValue({ entries });
+ mockRequest.mockResolvedValue({
+ entries,
+ next_page_token: "150",
+ });
renderHook(() => useStorageEntries("ns", "sub/"), { wrapper });
await waitFor(() => {
const state = store.get(storageAtom);
expect(state.entriesByPath.get("ns::sub/")).toEqual(entries);
+ expect(state.pageMetadataByPath.get("ns::sub/")?.nextPageToken).toBe(
+ "150",
+ );
+ });
+ });
+
+ it("should load more entries when a next page token exists", async () => {
+ const firstPage = [makeEntry({ path: "a.txt" })];
+ const secondPage = [makeEntry({ path: "b.txt" })];
+ mockRequest
+ .mockResolvedValueOnce({
+ entries: firstPage,
+ next_page_token: "150",
+ })
+ .mockResolvedValueOnce({
+ entries: secondPage,
+ next_page_token: null,
+ });
+
+ const { result } = renderHook(() => useStorageEntries("ns", "sub/"), {
+ wrapper,
+ });
+
+ await waitFor(() => {
+ expect(result.current.entries).toEqual(firstPage);
+ });
+ expect(result.current.hasMore).toBe(true);
+
+ await act(async () => {
+ await result.current.loadMore();
+ });
+
+ expect(result.current.entries).toEqual([...firstPage, ...secondPage]);
+ expect(result.current.hasMore).toBe(false);
+ expect(mockRequest).toHaveBeenLastCalledWith({
+ namespace: "ns",
+ prefix: "sub/",
+ limit: 150,
+ pageToken: "150",
+ });
+ });
+
+ it("should load more entries when the next page token is an empty string", async () => {
+ const firstPage = [makeEntry({ path: "a.txt" })];
+ const secondPage = [makeEntry({ path: "b.txt" })];
+ mockRequest
+ .mockResolvedValueOnce({
+ entries: firstPage,
+ next_page_token: "",
+ })
+ .mockResolvedValueOnce({
+ entries: secondPage,
+ next_page_token: null,
+ });
+
+ const { result } = renderHook(() => useStorageEntries("ns", "sub/"), {
+ wrapper,
+ });
+
+ await waitFor(() => {
+ expect(result.current.entries).toEqual(firstPage);
+ });
+ expect(result.current.hasMore).toBe(true);
+
+ await act(async () => {
+ await result.current.loadMore();
+ });
+
+ expect(result.current.entries).toEqual([...firstPage, ...secondPage]);
+ expect(result.current.hasMore).toBe(false);
+ expect(mockRequest).toHaveBeenLastCalledWith({
+ namespace: "ns",
+ prefix: "sub/",
+ limit: 150,
+ pageToken: "",
+ });
+ });
+
+ it("should ignore duplicate load more calls while a page is loading", async () => {
+ const firstPage = [makeEntry({ path: "a.txt" })];
+ const secondPage = [makeEntry({ path: "b.txt" })];
+ let resolveLoadMore!: (value: {
+ entries: StorageEntry[];
+ next_page_token: string | null;
+ }) => void;
+ const loadMorePromise = new Promise<{
+ entries: StorageEntry[];
+ next_page_token: string | null;
+ }>((resolve) => {
+ resolveLoadMore = resolve;
+ });
+ mockRequest
+ .mockResolvedValueOnce({
+ entries: firstPage,
+ next_page_token: "150",
+ })
+ .mockReturnValueOnce(loadMorePromise);
+
+ const { result } = renderHook(() => useStorageEntries("ns", "sub/"), {
+ wrapper,
+ });
+
+ await waitFor(() => {
+ expect(result.current.entries).toEqual(firstPage);
+ });
+
+ await act(async () => {
+ const firstLoad = result.current.loadMore();
+ const secondLoad = result.current.loadMore();
+
+ expect(mockRequest).toHaveBeenCalledTimes(2);
+ resolveLoadMore({
+ entries: secondPage,
+ next_page_token: null,
+ });
+ await Promise.all([firstLoad, secondLoad]);
+ });
+
+ expect(result.current.entries).toEqual([...firstPage, ...secondPage]);
+ expect(mockRequest).toHaveBeenCalledTimes(2);
+ });
+
+ it("should fetch arbitrary storage pages", async () => {
+ const firstPage = [makeEntry({ path: "folder/a.txt" })];
+ const secondPage = [makeEntry({ path: "folder/b.txt" })];
+ mockRequest
+ .mockResolvedValueOnce({
+ entries: firstPage,
+ next_page_token: "150",
+ })
+ .mockResolvedValueOnce({
+ entries: secondPage,
+ next_page_token: null,
+ });
+
+ const { result } = renderHook(() => useStoragePageFetcher(), {
+ wrapper,
+ });
+
+ await act(async () => {
+ await result.current({
+ namespace: "ns",
+ prefix: "folder/",
+ });
+ });
+ await act(async () => {
+ await result.current({
+ namespace: "ns",
+ prefix: "folder/",
+ pageToken: "150",
+ append: true,
+ });
+ });
+
+ expect(store.get(storageAtom).entriesByPath.get("ns::folder/")).toEqual([
+ ...firstPage,
+ ...secondPage,
+ ]);
+ expect(mockRequest).toHaveBeenLastCalledWith({
+ namespace: "ns",
+ prefix: "folder/",
+ limit: 150,
+ pageToken: "150",
+ });
+ });
+
+ it("should forward an empty string page token", async () => {
+ mockRequest.mockResolvedValue({
+ entries: [],
+ next_page_token: null,
+ });
+
+ const { result } = renderHook(() => useStoragePageFetcher(), {
+ wrapper,
+ });
+
+ await act(async () => {
+ await result.current({
+ namespace: "ns",
+ prefix: "folder/",
+ pageToken: "",
+ });
+ });
+
+ expect(mockRequest).toHaveBeenLastCalledWith({
+ namespace: "ns",
+ prefix: "folder/",
+ limit: 150,
+ pageToken: "",
});
});
});
diff --git a/frontend/src/core/storage/state.ts b/frontend/src/core/storage/state.ts
index 1a49431b3d..4aed690ee7 100644
--- a/frontend/src/core/storage/state.ts
+++ b/frontend/src/core/storage/state.ts
@@ -1,11 +1,15 @@
/* Copyright 2026 Marimo. All rights reserved. */
import { atom, useAtomValue } from "jotai";
+import { useCallback, useRef, useState } from "react";
import { useAsyncData } from "@/hooks/useAsyncData";
import { createReducerAndAtoms } from "@/utils/createReducer";
import type { NotificationMessageData } from "../kernel/messages";
import type { VariableName } from "../variables/types";
-import { ListStorageEntries } from "./request-registry";
+import {
+ ListStorageEntries,
+ type StorageEntriesResult,
+} from "./request-registry";
import type { StorageEntry, StorageState } from "./types";
import {
DEFAULT_FETCH_LIMIT,
@@ -18,6 +22,7 @@ function initialState(): StorageState {
return {
namespaces: [],
entriesByPath: new Map(),
+ pageMetadataByPath: new Map(),
};
}
@@ -48,23 +53,48 @@ const {
namespace: string;
prefix: string | null | undefined;
entries: StorageEntry[];
+ nextPageToken?: string | null;
+ append?: boolean;
},
) => {
const key = storagePathKey(opts.namespace, opts.prefix);
const entriesByPath = new Map(state.entriesByPath);
- entriesByPath.set(key, opts.entries);
- return { ...state, entriesByPath };
+ const entries =
+ opts.append && entriesByPath.has(key)
+ ? [...(entriesByPath.get(key) ?? []), ...opts.entries]
+ : opts.entries;
+ entriesByPath.set(key, entries);
+
+ const pageMetadataByPath = new Map(state.pageMetadataByPath);
+ pageMetadataByPath.set(key, {
+ nextPageToken: opts.nextPageToken ?? null,
+ });
+ return {
+ ...state,
+ entriesByPath,
+ pageMetadataByPath,
+ };
},
clearNamespaceCache: (state, namespace: string) => {
const entriesByPath = new Map(state.entriesByPath);
+ const pageMetadataByPath = new Map(state.pageMetadataByPath);
const prefix = storageNamespacePrefix(namespace);
for (const key of entriesByPath.keys()) {
if (key.startsWith(prefix)) {
entriesByPath.delete(key);
}
}
- return { ...state, entriesByPath };
+ for (const key of pageMetadataByPath.keys()) {
+ if (key.startsWith(prefix)) {
+ pageMetadataByPath.delete(key);
+ }
+ }
+ return {
+ ...state,
+ entriesByPath,
+ pageMetadataByPath,
+ };
},
filterFromVariables: (state, variableNames: VariableName[]) => {
@@ -93,38 +123,119 @@ export function useStorageActions() {
export { storageAtom };
+async function fetchStorageEntriesPage({
+ namespace,
+ prefix,
+ pageToken,
+ append,
+ setEntries,
+}: {
+ namespace: string;
+ prefix: string | null | undefined;
+ pageToken?: string | null;
+ append?: boolean;
+ setEntries: ReturnType<typeof useStorageActions>["setEntries"];
+}): Promise<StorageEntriesResult> {
+ const result = await ListStorageEntries.request({
+ namespace,
+ prefix: prefix ?? ROOT_PATH,
+ limit: DEFAULT_FETCH_LIMIT,
+ ...(pageToken != null ? { pageToken } : {}),
+ });
+ if (result.error) {
+ throw new Error(result.error);
+ }
+ setEntries({
+ namespace,
+ prefix,
+ entries: result.entries,
+ nextPageToken: result.next_page_token,
+ append,
+ });
+ return result;
+}
+
/**
* Hook that fetches and caches storage entries for a given namespace/prefix.
* Entries are fetched on first access and cached in the store for subsequent renders.
*/
export function useStorageEntries(namespace: string, prefix?: string) {
- const { entriesByPath } = useStorage();
+ const { entriesByPath, pageMetadataByPath } = useStorage();
const { setEntries } = useStorageActions();
- const cached = entriesByPath.get(storagePathKey(namespace, prefix));
+ const key = storagePathKey(namespace, prefix);
+ const cached = entriesByPath.get(key);
+ const metadata = pageMetadataByPath.get(key);
+ const nextPageToken = metadata?.nextPageToken ?? null;
+ const isLoadingMoreRef = useRef(false);
+ const [isLoadingMore, setIsLoadingMore] = useState(false);
+ const [loadMoreError, setLoadMoreError] = useState<Error | undefined>();
const { isPending, error, refetch } = useAsyncData(async () => {
if (cached) {
return;
}
- const result = await ListStorageEntries.request({
+ await fetchStorageEntriesPage({
namespace,
- prefix: prefix ?? ROOT_PATH,
- limit: DEFAULT_FETCH_LIMIT,
+ prefix,
+ setEntries,
});
- if (result.error) {
- throw new Error(result.error);
- }
- setEntries({ namespace, prefix, entries: result.entries });
}, [namespace, prefix, cached === undefined]);
+ const loadMore = useCallback(async () => {
+ if (nextPageToken == null || isLoadingMoreRef.current) {
+ return;
+ }
+
+ isLoadingMoreRef.current = true;
+ setIsLoadingMore(true);
+ setLoadMoreError(undefined);
+ try {
+ await fetchStorageEntriesPage({
+ namespace,
+ prefix,
+ pageToken: nextPageToken,
+ append: true,
+ setEntries,
+ });
+ } catch (error) {
+ setLoadMoreError(
+ error instanceof Error ? error : new Error(String(error)),
+ );
+ } finally {
+ isLoadingMoreRef.current = false;
+ setIsLoadingMore(false);
+ }
+ }, [namespace, prefix, nextPageToken, setEntries]);
+
return {
entries: cached ?? [],
isPending: isPending && !cached,
error: cached ? undefined : error,
+ hasMore: nextPageToken !== null,
+ loadMore,
+ isLoadingMore,
+ loadMoreError,
refetch,
};
}
+export function useStoragePageFetcher() {
+ const { setEntries } = useStorageActions();
+ return useCallback(
+ (opts: {
+ namespace: string;
+ prefix: string | null | undefined;
+ pageToken?: string | null;
+ append?: boolean;
+ }) =>
+ fetchStorageEntriesPage({
+ ...opts,
+ setEntries,
+ }),
+ [setEntries],
+ );
+}
+
export const exportedForTesting = {
reducer,
createActions,
diff --git a/frontend/src/core/storage/types.ts b/frontend/src/core/storage/types.ts
index f8514c947a..3dd5cb3682 100644
--- a/frontend/src/core/storage/types.ts
+++ b/frontend/src/core/storage/types.ts
@@ -30,6 +30,12 @@ export interface StorageState {
namespaces: StorageNamespace[];
/** Lazy-loaded entries keyed by "namespace::prefix" */
entriesByPath: ReadonlyMap<StoragePathKey, StorageEntry[]>;
+ /** Pagination metadata keyed by "namespace::prefix" */
+ pageMetadataByPath: ReadonlyMap<StoragePathKey, StoragePageMetadata>;
+}
+
+export interface StoragePageMetadata {
+ nextPageToken: string | null;
}
export function storageUrl(
diff --git a/marimo/_data/_external_storage/models.py b/marimo/_data/_external_storage/models.py
index 9d3257fccc..78b021550e 100644
--- a/marimo/_data/_external_storage/models.py
+++ b/marimo/_data/_external_storage/models.py
@@ -60,6 +60,14 @@ class StorageNamespace(msgspec.Struct, rename="camel"):
DEFAULT_FETCH_LIMIT = 100
+@dataclass
+class StorageListResult:
+ """Result of listing entries from a storage backend."""
+
+ entries: list[StorageEntry]
+ next_page_token: str | None = None
+
+
@dataclass
class DownloadResult:
"""Result of downloading a file from external storage.
@@ -90,9 +98,11 @@ class StorageBackend(abc.ABC, Generic[Backend]):
prefix: str | None,
*,
limit: int = DEFAULT_FETCH_LIMIT,
- ) -> list[StorageEntry]:
+ page_token: str | None = None,
+ ) -> StorageListResult:
"""
- List the entries at the given prefix. If no prefix is provided, list the root entries.
+ List one page of entries at the given prefix. If no prefix is provided,
+ list the root entries.
"""
@abc.abstractmethod
diff --git a/marimo/_data/_external_storage/storage.py b/marimo/_data/_external_storage/storage.py
index 5074a1c64f..bc8e7cdf2e 100644
--- a/marimo/_data/_external_storage/storage.py
+++ b/marimo/_data/_external_storage/storage.py
@@ -15,6 +15,7 @@ from marimo._data._external_storage.models import (
BackendType,
StorageBackend,
StorageEntry,
+ StorageListResult,
)
from marimo._dependencies.dependencies import DependencyManager
from marimo._utils.assert_never import log_never
@@ -44,7 +45,17 @@ class Obstore(StorageBackend["ObjectStore"]):
prefix: str | None,
*,
limit: int = DEFAULT_FETCH_LIMIT,
- ) -> list[StorageEntry]:
+ page_token: str | None = None,
+ ) -> StorageListResult:
+ offset = _parse_page_offset(page_token)
+ storage_entries = self._list_storage_entries(prefix)
+ return _paginate_entries(storage_entries, offset=offset, limit=limit)
+
+ def _list_storage_entries(self, prefix: str | None) -> list[StorageEntry]:
+ # Object stores commonly cap a single delimiter listing at ~1000
+ # entries, and we don't page beyond one listing, so directories with
+ # more entries than that will be silently truncated here.
+ # https://docs.rs/object_store/latest/object_store/struct.ListResult.html
result = self.store.list_with_delimiter(prefix=prefix)
storage_entries: list[StorageEntry] = []
@@ -71,14 +82,6 @@ class Obstore(StorageBackend["ObjectStore"]):
continue
storage_entries.append(self._create_storage_entry(entry))
- if len(storage_entries) > limit:
- LOGGER.debug(
- "Fetched %s entries, but limiting to %s",
- len(storage_entries),
- limit,
- )
- storage_entries = storage_entries[:limit]
-
return storage_entries
async def get_entry(self, path: str) -> StorageEntry:
@@ -247,7 +250,13 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
prefix: str | None,
*,
limit: int = DEFAULT_FETCH_LIMIT,
- ) -> list[StorageEntry]:
+ page_token: str | None = None,
+ ) -> StorageListResult:
+ offset = _parse_page_offset(page_token)
+ entries = self._list_storage_entries(prefix)
+ return _paginate_entries(entries, offset=offset, limit=limit)
+
+ def _list_storage_entries(self, prefix: str | None) -> list[StorageEntry]:
# If no prefix provided, we use empty string to list root entries
# Else, an error is raised
if prefix is None:
@@ -263,15 +272,6 @@ class FsspecFilesystem(StorageBackend["AbstractFileSystem"]):
self._invalidate_listing_cache_for_prefix(prefix)
files = self._list_files(prefix)
- total_files = len(files)
- if total_files > limit:
- LOGGER.debug(
- "Fetched %s files, but limiting to %s",
- total_files,
- limit,
- )
- files = files[:limit]
-
storage_entries = []
for file in files:
if isinstance(file, dict):
@@ -512,3 +512,42 @@ def detect_protocol_from_url(url: str) -> CLOUD_STORAGE_TYPES | None:
def normalize_protocol(protocol: str) -> KNOWN_STORAGE_TYPES | None:
"""Normalize a protocol string (e.g. 's3a', 'gs', 'abfs') to a known storage type."""
return _PROTOCOL_MAP.get(protocol.strip().lower())
+
+
+def _parse_page_offset(page_token: str | None) -> int:
+ if page_token is None:
+ return 0
+ try:
+ offset = int(page_token)
+ except ValueError as exc:
+ raise ValueError(f"Invalid storage page token: {page_token}") from exc
+ if offset < 0:
+ raise ValueError(f"Invalid storage page token: {page_token}")
+ return offset
+
+
+def _paginate_entries(
+ entries: list[StorageEntry],
+ *,
+ offset: int,
+ limit: int,
+) -> StorageListResult:
+ if limit < 1:
+ raise ValueError("Storage list limit must be positive")
+
+ total_entries = len(entries)
+ if total_entries > limit:
+ LOGGER.debug(
+ "Fetched %s entries, returning page offset %s with limit %s",
+ total_entries,
+ offset,
+ limit,
+ )
+
+ end = offset + limit
+ has_next_page = end < total_entries
+ next_page_token = str(end) if has_next_page else None
+ return StorageListResult(
+ entries=entries[offset:end],
+ next_page_token=next_page_token,
+ )
diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.py
index 155f49e8d4..554f880752 100644
--- a/marimo/_messaging/notification.py
+++ b/marimo/_messaging/notification.py
@@ -695,6 +695,7 @@ class StorageEntriesNotification(Notification, tag="storage-entries"):
namespace: Variable name of the storage backend.
prefix: The prefix that was listed (set by list_entries).
query: The search query that was used (set by search).
+ next_page_token: Token for fetching the next page of entries.
error: Error message if the operation failed.
"""
@@ -704,6 +705,7 @@ class StorageEntriesNotification(Notification, tag="storage-entries"):
namespace: str
prefix: str | None = None
query: str | None = None
+ next_page_token: str | None = None
error: str | None = None
diff --git a/marimo/_runtime/callbacks/external_storage.py b/marimo/_runtime/callbacks/external_storage.py
index d76347bc13..2bb21f7c2f 100644
--- a/marimo/_runtime/callbacks/external_storage.py
+++ b/marimo/_runtime/callbacks/external_storage.py
@@ -6,7 +6,10 @@ import os
from typing import TYPE_CHECKING, Any
from marimo import _loggers
-from marimo._data._external_storage.models import StorageBackend, StorageEntry
+from marimo._data._external_storage.models import (
+ StorageBackend,
+ StorageListResult,
+)
from marimo._messaging.notification import (
StorageDownloadReadyNotification,
StorageEntriesNotification,
@@ -92,19 +95,22 @@ class ExternalStorageCallbacks:
return
# list_entries is synchronous, so we wrap it in asyncio.to_thread
- def list_entries() -> list[StorageEntry]:
+ def list_entries() -> StorageListResult:
return backend.list_entries(
- prefix=request.prefix, limit=request.limit
+ prefix=request.prefix,
+ limit=request.limit,
+ page_token=request.page_token,
)
try:
- entries = await asyncio.to_thread(list_entries)
+ result = await asyncio.to_thread(list_entries)
broadcast_notification(
StorageEntriesNotification(
request_id=request.request_id,
- entries=entries,
+ entries=result.entries,
namespace=request.namespace,
prefix=request.prefix,
+ next_page_token=result.next_page_token,
),
)
except Exception as e:
diff --git a/marimo/_runtime/commands.py b/marimo/_runtime/commands.py
index 32d6397a7b..0b860176c9 100644
--- a/marimo/_runtime/commands.py
+++ b/marimo/_runtime/commands.py
@@ -757,12 +757,14 @@ class StorageListEntriesCommand(Command):
namespace: Variable name identifying the storage backend.
limit: Max entries to return.
prefix: Path prefix to list (None = root).
+ page_token: Token for the next page of entries.
"""
request_id: RequestId
namespace: str
limit: int
prefix: str | None = None
+ page_token: str | None = None
class StorageDownloadCommand(Command):
diff --git a/marimo/_server/models/models.py b/marimo/_server/models/models.py
index 13811cae38..0862124a8f 100644
--- a/marimo/_server/models/models.py
+++ b/marimo/_server/models/models.py
@@ -155,6 +155,7 @@ class StorageListEntriesRequest(StorageListEntriesCommand, tag=False):
namespace=self.namespace,
limit=self.limit,
prefix=self.prefix,
+ page_token=self.page_token,
)
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 7b86d7e5b9..5dc322bb37 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -4946,7 +4946,8 @@ components:
\ entries returned by the operation.\n namespace: Variable name of\
\ the storage backend.\n prefix: The prefix that was listed (set by\
\ list_entries).\n query: The search query that was used (set by search).\n\
- \ error: Error message if the operation failed."
+ \ next_page_token: Token for fetching the next page of entries.\n \
+ \ error: Error message if the operation failed."
properties:
entries:
items:
@@ -4959,6 +4960,11 @@ components:
default: null
namespace:
type: string
+ next_page_token:
+ anyOf:
+ - type: string
+ - type: 'null'
+ default: null
op:
enum:
- storage-entries
@@ -5023,12 +5029,18 @@ components:
\ and virtual directories at one level.\n\n Attributes:\n request_id:\
\ Unique identifier for this request.\n namespace: Variable name identifying\
\ the storage backend.\n limit: Max entries to return.\n prefix:\
- \ Path prefix to list (None = root)."
+ \ Path prefix to list (None = root).\n page_token: Token for the next\
+ \ page of entries."
properties:
limit:
type: integer
namespace:
type: string
+ pageToken:
+ anyOf:
+ - type: string
+ - type: 'null'
+ default: null
prefix:
anyOf:
- type: string
@@ -5052,6 +5064,11 @@ components:
type: integer
namespace:
type: string
+ pageToken:
+ anyOf:
+ - type: string
+ - type: 'null'
+ default: null
prefix:
anyOf:
- type: string
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index 6d57f54158..befb476eba 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -6433,6 +6433,7 @@ export interface components {
* namespace: Variable name of the storage backend.
* prefix: The prefix that was listed (set by list_entries).
* query: The search query that was used (set by search).
+ * next_page_token: Token for fetching the next page of entries.
* error: Error message if the operation failed.
*/
StorageEntriesNotification: {
@@ -6440,6 +6441,8 @@ export interface components {
/** @default null */
error?: string | null;
namespace: string;
+ /** @default null */
+ next_page_token?: string | null;
/** @enum {unknown} */
op: "storage-entries";
/** @default null */
@@ -6483,11 +6486,14 @@ export interface components {
* namespace: Variable name identifying the storage backend.
* limit: Max entries to return.
* prefix: Path prefix to list (None = root).
+ * page_token: Token for the next page of entries.
*/
StorageListEntriesCommand: {
limit: number;
namespace: string;
/** @default null */
+ pageToken?: string | null;
+ /** @default null */
prefix?: string | null;
requestId: string;
/** @enum {unknown} */
@@ -6498,6 +6504,8 @@ export interface components {
limit: number;
namespace: string;
/** @default null */
+ pageToken?: string | null;
+ /** @default null */
prefix?: string | null;
requestId: components["schemas"]["RequestId"];
};
diff --git a/tests/_data/_external_storage/test_storage_models.py b/tests/_data/_external_storage/test_storage_models.py
index f1e902c020..f2612b7975 100644
--- a/tests/_data/_external_storage/test_storage_models.py
+++ b/tests/_data/_external_storage/test_storage_models.py
@@ -10,7 +10,11 @@ import pytest
from dirty_equals import IsDatetime, IsPositiveFloat
from inline_snapshot import snapshot
-from marimo._data._external_storage.models import DownloadResult, StorageEntry
+from marimo._data._external_storage.models import (
+ DownloadResult,
+ StorageEntry,
+ StorageListResult,
+)
from marimo._data._external_storage.storage import (
FsspecFilesystem,
Obstore,
@@ -58,7 +62,7 @@ class TestObstore:
mock_store.list_with_delimiter.assert_called_once_with(
prefix="some/prefix",
)
- assert result == snapshot(
+ assert result.entries == snapshot(
[
StorageEntry(
path="subdir/",
@@ -113,7 +117,7 @@ class TestObstore:
backend = self._make_backend(mock_store)
result = backend.list_entries(prefix="folder")
- assert result == [
+ assert result.entries == [
StorageEntry(
path="folder/order_details.csv",
kind="object",
@@ -133,7 +137,67 @@ class TestObstore:
backend = self._make_backend(mock_store)
result = backend.list_entries(prefix=None)
- assert result == []
+ assert result.entries == []
+
+ def test_list_entries_returns_next_page_token(self) -> None:
+ now = datetime.now(tz=timezone.utc)
+ mock_store = MagicMock()
+ mock_store.list_with_delimiter.return_value = {
+ "common_prefixes": ["a/", "b/"],
+ "objects": [
+ {
+ "path": "c.txt",
+ "size": 1,
+ "last_modified": now,
+ "e_tag": None,
+ "version": None,
+ },
+ ],
+ }
+
+ backend = self._make_backend(mock_store)
+
+ assert backend.list_entries(prefix=None, limit=2) == snapshot(
+ StorageListResult(
+ entries=[
+ StorageEntry(
+ path="a/",
+ kind="directory",
+ size=0,
+ last_modified=None,
+ metadata={},
+ mime_type=None,
+ ),
+ StorageEntry(
+ path="b/",
+ kind="directory",
+ size=0,
+ last_modified=None,
+ metadata={},
+ mime_type=None,
+ ),
+ ],
+ next_page_token="2",
+ )
+ )
+
+ assert backend.list_entries(
+ prefix=None, limit=2, page_token="2"
+ ) == snapshot(
+ StorageListResult(
+ entries=[
+ StorageEntry(
+ path="c.txt",
+ kind="object",
+ size=1,
+ last_modified=now.timestamp(),
+ metadata={},
+ mime_type="text/plain",
+ ),
+ ],
+ next_page_token=None,
+ )
+ )
def test_create_storage_entry_missing_fields(self) -> None:
mock_store = MagicMock()
@@ -586,7 +650,7 @@ class TestFsspecFilesystem:
result = backend.list_entries(prefix="some/path")
mock_store.ls.assert_called_once_with(path="some/path", detail=True)
- assert result == snapshot(
+ assert result.entries == snapshot(
[
StorageEntry(
path="file1.txt",
@@ -646,7 +710,7 @@ class TestFsspecFilesystem:
assert mock_store.ls.call_count == 2
assert "" not in mock_store.dircache
assert "folder" not in mock_store.dircache
- assert result == snapshot(
+ assert result.entries == snapshot(
[
StorageEntry(
path="folder/file.txt",
@@ -690,7 +754,7 @@ class TestFsspecFilesystem:
result = backend.list_entries(prefix="folder")
mock_store.ls.assert_called_once_with(path="folder", detail=True)
- assert result == snapshot(
+ assert result.entries == snapshot(
[
StorageEntry(
path="folder",
@@ -735,7 +799,7 @@ class TestFsspecFilesystem:
backend = self._make_backend(mock_store)
result = backend.list_entries(prefix="", limit=3)
- assert result == snapshot(
+ assert result.entries == snapshot(
[
StorageEntry(
path="file0.txt",
@@ -764,6 +828,63 @@ class TestFsspecFilesystem:
]
)
+ def test_list_entries_returns_next_page_token(self) -> None:
+ mock_store = MagicMock()
+ files = [
+ {
+ "name": f"file{i}.txt",
+ "size": i,
+ "type": "file",
+ "mtime": None,
+ }
+ for i in range(3)
+ ]
+ mock_store.ls.return_value = files
+
+ backend = self._make_backend(mock_store)
+
+ assert backend.list_entries(prefix="", limit=2) == snapshot(
+ StorageListResult(
+ entries=[
+ StorageEntry(
+ path="file0.txt",
+ kind="file",
+ size=0,
+ last_modified=None,
+ metadata={},
+ mime_type="text/plain",
+ ),
+ StorageEntry(
+ path="file1.txt",
+ kind="file",
+ size=1,
+ last_modified=None,
+ metadata={},
+ mime_type="text/plain",
+ ),
+ ],
+ next_page_token="2",
+ )
+ )
+
+ assert backend.list_entries(
+ prefix="", limit=2, page_token="2"
+ ) == snapshot(
+ StorageListResult(
+ entries=[
+ StorageEntry(
+ path="file2.txt",
+ kind="file",
+ size=2,
+ last_modified=None,
+ metadata={},
+ mime_type="text/plain",
+ ),
+ ],
+ next_page_token=None,
+ )
+ )
+
def test_list_entries_raises_on_non_list(self) -> None:
mock_store = MagicMock()
mock_store.ls.return_value = "not_a_list"
@@ -783,7 +904,7 @@ class TestFsspecFilesystem:
backend = self._make_backend(mock_store)
result = backend.list_entries(prefix="")
- assert result == snapshot(
+ assert result.entries == snapshot(
[
StorageEntry(
path="good.txt",
@@ -1129,7 +1250,7 @@ class TestFsspecFilesystemIntegration:
backend = FsspecFilesystem(fs, VariableName("mem_fs"))
entries = backend.list_entries(prefix="/test")
- assert entries == snapshot(
+ assert entries.entries == snapshot(
[
StorageEntry(
path="/test/hello.txt",
@@ -1253,7 +1374,7 @@ class TestObstoreIntegration:
backend = Obstore(store, VariableName("mem_store"))
entries = backend.list_entries(prefix="test/")
- assert entries == snapshot(
+ assert entries.entries == snapshot(
[
StorageEntry(
path="test/file1.txt",
diff --git a/tests/_runtime/test_runtime_external_storage.py b/tests/_runtime/test_runtime_external_storage.py
index 7268072cfe..416942d625 100644
--- a/tests/_runtime/test_runtime_external_storage.py
+++ b/tests/_runtime/test_runtime_external_storage.py
@@ -8,7 +8,10 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from dirty_equals import IsPositiveFloat, IsStr
-from marimo._data._external_storage.models import StorageEntry
+from marimo._data._external_storage.models import (
+ StorageEntry,
+ StorageListResult,
+)
from marimo._data._external_storage.storage import Obstore
from marimo._dependencies.dependencies import DependencyManager
from marimo._messaging.notification import (
@@ -290,6 +293,40 @@ class TestExternalStorageCallbacks:
],
namespace=STORAGE_VAR,
prefix=None,
+ next_page_token="2",
+ )
+ ]
+
+ request = StorageListEntriesCommand(
+ request_id=RequestId("req-11-page-2"),
+ namespace=STORAGE_VAR,
+ limit=2,
+ prefix=None,
+ page_token="2",
+ )
+ await k.handle_message(request)
+
+ results = [
+ op
+ for op in stream.operations
+ if isinstance(op, StorageEntriesNotification)
+ and op.request_id == RequestId("req-11-page-2")
+ ]
+ assert results == [
+ StorageEntriesNotification(
+ request_id=RequestId("req-11-page-2"),
+ entries=[
+ StorageEntry(
+ path="c.txt",
+ kind="object",
+ size=1,
+ last_modified=IsPositiveFloat(), # pyright: ignore[reportArgumentType]
+ metadata={"e_tag": IsStr()},
+ mime_type="text/plain",
+ ),
+ ],
+ namespace=STORAGE_VAR,
+ prefix=None,
)
]
@@ -413,11 +450,14 @@ class TestExternalStorageCallbacks:
original_list = real_backend.list_entries
def spy_list_entries(
- prefix: str | None, *, limit: int = 100
- ) -> list[StorageEntry]:
+ prefix: str | None,
+ *,
+ limit: int = 100,
+ page_token: str | None = None,
+ ) -> StorageListResult:
nonlocal call_thread_name
call_thread_name = threading.current_thread().name
- return original_list(prefix, limit=limit)
+ return original_list(prefix, limit=limit, page_token=page_token)
real_backend.list_entries = spy_list_entries # type: ignore[method-assign]