Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(table): browse lazy tables without counting all rows (#11409)

Shahmir Varqhadffef8dc12ce
diff --git a/frontend/plugins.openapi.yaml b/frontend/plugins.openapi.yaml
index 44ef17f724..1069f54910 100644
--- a/frontend/plugins.openapi.yaml
+++ b/frontend/plugins.openapi.yaml
@@ -5638,6 +5638,10 @@ components:
- type: number
- type: string
const: too_many
+ has_next_page:
+ type:
+ - boolean
+ - "null"
cell_styles:
anyOf:
- type: object
diff --git a/frontend/src/components/data-table/TableBottomBar.tsx b/frontend/src/components/data-table/TableBottomBar.tsx
index f47a8468d0..5dfffdf7d2 100644
--- a/frontend/src/components/data-table/TableBottomBar.tsx
+++ b/frontend/src/components/data-table/TableBottomBar.tsx
@@ -17,6 +17,7 @@ import type { DataTableSelection } from "./types";
interface TableBottomBarProps<TData> {
pagination: boolean;
+ hasNextPage?: boolean;
totalColumns: number;
selection?: DataTableSelection;
onRowSelectionChange?: (value: RowSelectionState) => void;
@@ -30,6 +31,7 @@ interface TableBottomBarProps<TData> {
export const TableBottomBar = <TData,>({
pagination,
+ hasNextPage,
totalColumns,
selection,
onRowSelectionChange,
@@ -147,7 +149,7 @@ export const TableBottomBar = <TData,>({
const { totalColumns: effectiveTotalColumns, hiddenColumns } =
getColumnCountForDisplay(table, totalColumns);
const { rowsAndColumns, hiddenSuffix } = prettifyRowColumnCount({
- numRows: table.getRowCount(),
+ numRows: hasNextPage === undefined ? table.getRowCount() : "too_many",
totalColumns: effectiveTotalColumns,
hiddenColumns,
locale,
@@ -178,6 +180,7 @@ export const TableBottomBar = <TData,>({
<DataTablePagination
table={table}
tableLoading={tableLoading}
+ hasNextPage={hasNextPage}
showPageSizeSelector={showPageSizeSelector}
/>
)}
diff --git a/frontend/src/components/data-table/__tests__/pagination.test.tsx b/frontend/src/components/data-table/__tests__/pagination.test.tsx
index d484e96ec9..c4b1d666b2 100644
--- a/frontend/src/components/data-table/__tests__/pagination.test.tsx
+++ b/frontend/src/components/data-table/__tests__/pagination.test.tsx
@@ -9,9 +9,11 @@ import { DataTablePagination, matchingPageRanges } from "../pagination";
function PaginationHarness({
totalPages = 3,
loading = false,
+ hasNextPage,
}: {
totalPages?: number;
loading?: boolean;
+ hasNextPage?: boolean;
}) {
const table = useReactTable({
locale: "en-US",
@@ -23,7 +25,11 @@ function PaginationHarness({
});
return (
<TooltipProvider>
- <DataTablePagination table={table} tableLoading={loading} />
+ <DataTablePagination
+ table={table}
+ tableLoading={loading}
+ hasNextPage={hasNextPage}
+ />
</TooltipProvider>
);
}
@@ -140,3 +146,24 @@ test("prefix 1 with small totalPages", () => {
test("prefix 1 with totalPages=1", () => {
expect(matchingPageRanges("1", 1)).toEqual([[1, 1]]);
});
+
+test("unknown totals use lookahead for navigation without a last page or page picker", () => {
+ const { rerender } = render(
+ <PaginationHarness totalPages={-1} hasNextPage={true} />,
+ );
+ expect(screen.getByText("1")).toBeInTheDocument();
+ expect(
+ screen.queryByRole("button", { name: "Last page" }),
+ ).not.toBeInTheDocument();
+ expect(
+ screen.queryByRole("button", { name: /Choose page/ }),
+ ).not.toBeInTheDocument();
+ expect(screen.getByRole("button", { name: "Previous page" })).toBeDisabled();
+ fireEvent.click(screen.getByRole("button", { name: "Next page" }));
+ expect(screen.getByText("2")).toBeInTheDocument();
+ rerender(<PaginationHarness totalPages={-1} hasNextPage={false} />);
+ expect(screen.getByRole("button", { name: "Next page" })).toBeDisabled();
+ fireEvent.click(screen.getByRole("button", { name: "Previous page" }));
+ expect(screen.getByText("1")).toBeInTheDocument();
+ expect(screen.getByRole("button", { name: "Previous page" })).toBeDisabled();
+});
diff --git a/frontend/src/components/data-table/data-table.tsx b/frontend/src/components/data-table/data-table.tsx
index 599def6478..5818a8c784 100644
--- a/frontend/src/components/data-table/data-table.tsx
+++ b/frontend/src/components/data-table/data-table.tsx
@@ -86,6 +86,7 @@ interface DataTableProps<TData> extends Partial<ExportActionProps> {
setSorting?: OnChangeFn<SortingState>; // controlled sorting
// Pagination
totalRows: number | TooManyRows;
+ hasNextPage?: boolean;
// JSON-serialized size of the currently-rendered data. Forwarded to
// ExportActions so hosts can size-gate the Export button via downloadSizeLimitAtom.
sizeBytes?: number | null;
@@ -143,6 +144,7 @@ const DataTableInternal = <TData,>({
selection,
totalColumns,
totalRows,
+ hasNextPage,
sizeBytes,
sizeBytesIsLoading,
manualSorting = false,
@@ -255,6 +257,7 @@ const DataTableInternal = <TData,>({
getCoreRowModel: getCoreRowModel(),
// pagination
rowCount: totalRows === "too_many" ? undefined : totalRows,
+ pageCount: hasNextPage === undefined ? undefined : -1,
...(setPaginationState
? {
onPaginationChange: setPaginationState,
@@ -432,6 +435,7 @@ const DataTableInternal = <TData,>({
className="pt-1.5 pb-0.5 border-t border-border"
totalColumns={totalColumns}
pagination={pagination}
+ hasNextPage={hasNextPage}
selection={selection}
onRowSelectionChange={onRowSelectionChange}
table={table}
diff --git a/frontend/src/components/data-table/pagination.tsx b/frontend/src/components/data-table/pagination.tsx
index 5be917c8d7..e2c5ce62d5 100644
--- a/frontend/src/components/data-table/pagination.tsx
+++ b/frontend/src/components/data-table/pagination.tsx
@@ -41,21 +41,24 @@ import { PluralWord } from "@/utils/pluralize";
interface DataTablePaginationProps<TData> {
table: Table<TData>;
tableLoading?: boolean;
+ hasNextPage?: boolean;
showPageSizeSelector?: boolean;
}
export const DataTablePagination = <TData,>({
table,
tableLoading,
+ hasNextPage,
showPageSizeSelector,
}: DataTablePaginationProps<TData>) => {
const { locale } = useLocale();
- const currentPage = Math.min(
- table.getState().pagination.pageIndex + 1,
- table.getPageCount(),
- );
const totalPages = table.getPageCount();
+ const hasKnownTotal = totalPages !== -1;
+ const pageNumber = table.getState().pagination.pageIndex + 1;
+ const currentPage = hasKnownTotal
+ ? Math.min(pageNumber, totalPages)
+ : pageNumber;
const pageSize = table.getState().pagination.pageSize;
@@ -142,16 +145,22 @@ export const DataTablePagination = <TData,>({
</Tooltip>
<div className="flex items-center justify-center text-xs font-medium gap-1">
<span>Page</span>
- <PageSelector
- currentPage={currentPage}
- totalPages={totalPages}
- onPageChange={(page) =>
- handlePageChange(() => table.setPageIndex(page))
- }
- />
- <span className="shrink-0">
- of {prettyNumber(totalPages, locale)}
- </span>
+ {hasKnownTotal ? (
+ <>
+ <PageSelector
+ currentPage={currentPage}
+ totalPages={totalPages}
+ onPageChange={(page) =>
+ handlePageChange(() => table.setPageIndex(page))
+ }
+ />
+ <span className="shrink-0">
+ of {prettyNumber(totalPages, locale)}
+ </span>
+ </>
+ ) : (
+ <span>{prettyNumber(currentPage, locale)}</span>
+ )}
</div>
<Tooltip content="Next page">
<Button
@@ -162,29 +171,31 @@ export const DataTablePagination = <TData,>({
className="h-6 w-5 p-0"
onClick={() => handlePageChange(() => table.nextPage())}
onMouseDown={Events.preventFocus}
- disabled={!table.getCanNextPage()}
+ disabled={tableLoading || !(hasNextPage ?? table.getCanNextPage())}
>
<ChevronRight className="h-4 w-4" />
</Button>
</Tooltip>
- <Tooltip content="Last page">
- <Button
- size="xs"
- variant="text"
- data-testid="last-page-button"
- aria-label="Last page"
- className="hidden h-6 w-5 p-0 lg:flex"
- onClick={() =>
- handlePageChange(() =>
- table.setPageIndex(table.getPageCount() - 1),
- )
- }
- onMouseDown={Events.preventFocus}
- disabled={!table.getCanNextPage()}
- >
- <ChevronsRight className="h-4 w-4" />
- </Button>
- </Tooltip>
+ {hasKnownTotal && (
+ <Tooltip content="Last page">
+ <Button
+ size="xs"
+ variant="text"
+ data-testid="last-page-button"
+ aria-label="Last page"
+ className="hidden h-6 w-5 p-0 lg:flex"
+ onClick={() =>
+ handlePageChange(() =>
+ table.setPageIndex(table.getPageCount() - 1),
+ )
+ }
+ onMouseDown={Events.preventFocus}
+ disabled={!table.getCanNextPage()}
+ >
+ <ChevronsRight className="h-4 w-4" />
+ </Button>
+ </Tooltip>
+ )}
</div>
</div>
);
diff --git a/frontend/src/plugins/impl/DataTablePlugin.tsx b/frontend/src/plugins/impl/DataTablePlugin.tsx
index 94000a89e3..51b2616d03 100644
--- a/frontend/src/plugins/impl/DataTablePlugin.tsx
+++ b/frontend/src/plugins/impl/DataTablePlugin.tsx
@@ -231,6 +231,7 @@ type DataTableFunctions = {
}) => Promise<{
data: TableData<T>;
total_rows: number | TooManyRows;
+ has_next_page?: boolean | null;
cell_styles?: CellStyleState | null;
cell_hover_texts?: Record<string, Record<string, string | null>> | null;
raw_data?: TableData<T> | null;
@@ -343,6 +344,7 @@ export const DataTablePlugin = createPlugin<S>("marimo-table")
z.object({
data: z.union([z.string(), z.array(z.looseObject({}))]),
total_rows: z.union([z.number(), z.literal(TOO_MANY_ROWS)]),
+ has_next_page: z.boolean().nullish(),
cell_styles: z
.record(z.string(), z.record(z.string(), z.looseObject({})))
.nullable(),
@@ -507,6 +509,7 @@ interface DataTableSearchProps {
filters?: ColumnFiltersState;
setFilters?: SetFilters;
hasStableRowId: boolean;
+ hasNextPage?: boolean;
}
export const LoadingDataTableComponent = memo(
@@ -550,11 +553,35 @@ export const LoadingDataTableComponent = memo(
setPaginationState({ pageIndex: 0, pageSize: props.pageSize });
}
+ const resetLazyPage = useEvent(() => {
+ if (props.lazy) {
+ setPaginationState((state) =>
+ state.pageIndex === 0 ? state : { ...state, pageIndex: 0 },
+ );
+ }
+ });
+
+ const handleSortingChange: OnChangeFn<SortingState> = useEvent(
+ (updater) => {
+ setSorting(updater);
+ resetLazyPage();
+ },
+ );
+ const handleFiltersChange: SetFilters = useEvent((updater) => {
+ setFilters(updater);
+ resetLazyPage();
+ });
+ const handleSearchQueryChange = useEvent((query: string) => {
+ setSearchQuery(query);
+ resetLazyPage();
+ });
+
// Data loading
const { data, error, isPending, isFetching } = useAsyncData<{
rows: T[];
rawRows?: T[];
totalRows: number | TooManyRows;
+ hasNextPage?: boolean;
cellStyles: CellStyleState | undefined | null;
cellHoverTexts?: Record<string, Record<string, string | null>> | null;
}>(async () => {
@@ -571,6 +598,7 @@ export const LoadingDataTableComponent = memo(
let tableData = props.data;
let rawTableData: TableData<T> | undefined | null = props.rawData;
let totalRows = props.totalRows;
+ let hasNextPage: boolean | undefined;
let cellStyles = props.cellStyles;
let cellHoverTexts = props.cellHoverTexts;
@@ -614,6 +642,7 @@ export const LoadingDataTableComponent = memo(
tableData = searchResults.data;
rawTableData = searchResults.raw_data;
totalRows = searchResults.total_rows;
+ hasNextPage = searchResults.has_next_page ?? undefined;
cellStyles = searchResults.cell_styles || {};
cellHoverTexts = searchResults.cell_hover_texts || {};
}
@@ -626,6 +655,7 @@ export const LoadingDataTableComponent = memo(
rows: tableData,
rawRows: rawData,
totalRows: totalRows,
+ hasNextPage,
cellStyles,
cellHoverTexts,
};
@@ -775,13 +805,16 @@ export const LoadingDataTableComponent = memo(
rawData={data?.rawRows}
columnSummaries={columnSummaries}
sorting={sorting}
- setSorting={setSorting}
+ setSorting={handleSortingChange}
searchQuery={searchQuery}
- setSearchQuery={setSearchQuery}
+ setSearchQuery={handleSearchQueryChange}
filters={filters}
- setFilters={setFilters}
+ setFilters={handleFiltersChange}
reloading={isFetching && !isPending}
totalRows={data?.totalRows ?? props.totalRows}
+ hasNextPage={
+ props.lazy ? !error && (data?.hasNextPage ?? false) : undefined
+ }
sizeBytes={sizeBytes}
sizeBytesIsLoading={sizeBytesIsLoading}
paginationState={paginationState}
@@ -831,6 +864,7 @@ const DataTableComponent = ({
data,
rawData,
totalRows,
+ hasNextPage,
sizeBytes,
sizeBytesIsLoading,
maxColumns,
@@ -1130,7 +1164,8 @@ const DataTableComponent = ({
<>
{/* When the totalRows is "too_many" and the pageSize is the same as the
* number of rows, we are likely displaying all the data (could be more, but we don't know the total). */}
- {totalRows === TOO_MANY_ROWS &&
+ {hasNextPage === undefined &&
+ totalRows === TOO_MANY_ROWS &&
paginationState.pageSize === data.length && (
<Banner className="mb-1 rounded">
Previewing the first {paginationState.pageSize} rows.
@@ -1168,6 +1203,7 @@ const DataTableComponent = ({
maxHeight={maxHeight}
sorting={sorting}
totalRows={totalRows}
+ hasNextPage={hasNextPage}
sizeBytes={sizeBytes}
sizeBytesIsLoading={sizeBytesIsLoading}
totalColumns={totalColumns}
diff --git a/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx b/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
index 65602ab477..c3b877c548 100644
--- a/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
+++ b/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
@@ -10,6 +10,7 @@ import {
render,
screen,
waitFor,
+ within,
} from "@testing-library/react";
import { Provider } from "jotai";
import { useState } from "react";
@@ -540,6 +541,275 @@ describe("table pagination and static notebook control suppression", () => {
mockIsStatic.mockReturnValue(false);
});
+ it("preserves the preview banner for ordinary paginated tables with unknown totals", async () => {
+ const props = makeProps();
+ render(
+ <Wrapper>
+ <LoadingDataTableComponent {...props} totalRows="too_many" />
+ </Wrapper>,
+ );
+ expect(
+ await screen.findByText("Previewing the first 10 rows."),
+ ).toBeInTheDocument();
+ });
+
+ it("pages lazy results with unknown totals and resets search, sorting, and filters to page one", async () => {
+ const props = makeProps();
+ const search = vi
+ .fn()
+ .mockImplementation(
+ async (request: {
+ page_number: number;
+ query?: string;
+ sort?: { by: string; descending: boolean }[];
+ }) => ({
+ data: [
+ { id: request.page_number + 1, name: request.query || "item" },
+ ],
+ total_rows: "too_many",
+ has_next_page: request.page_number === 0,
+ }),
+ );
+ render(
+ <Wrapper>
+ <LoadingDataTableComponent
+ {...props}
+ lazy={true}
+ totalRows="too_many"
+ selection={null}
+ showColumnSummaries={false}
+ search={search}
+ />
+ </Wrapper>,
+ );
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeEnabled(),
+ );
+ expect(screen.getByText("Unknown, 2 columns")).toBeInTheDocument();
+ expect(screen.queryByTestId("last-page-button")).not.toBeInTheDocument();
+ expect(screen.queryByText(/Previewing the first/)).not.toBeInTheDocument();
+ fireEvent.click(screen.getByTestId("next-page-button"));
+ await waitFor(() =>
+ expect(search).toHaveBeenLastCalledWith(
+ expect.objectContaining({ page_number: 1 }),
+ ),
+ );
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeDisabled(),
+ );
+ fireEvent.click(screen.getByTestId("previous-page-button"));
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeEnabled(),
+ );
+ fireEvent.click(screen.getByTestId("next-page-button"));
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeDisabled(),
+ );
+ search.mockClear();
+ fireEvent.change(screen.getByPlaceholderText("Search..."), {
+ target: { value: "new" },
+ });
+ await waitFor(() =>
+ expect(search).toHaveBeenCalledWith(
+ expect.objectContaining({ query: "new", page_number: 0 }),
+ ),
+ );
+ expect(
+ search.mock.calls.every(([request]) => request.page_number === 0),
+ ).toBe(true);
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeEnabled(),
+ );
+ fireEvent.click(screen.getByTestId("next-page-button"));
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeDisabled(),
+ );
+ search.mockClear();
+ fireEvent.click(
+ within(screen.getByRole("columnheader", { name: /name/ })).getByRole(
+ "button",
+ { name: "Sort column ascending" },
+ ),
+ );
+ await waitFor(() =>
+ expect(search).toHaveBeenCalledWith(
+ expect.objectContaining({
+ page_number: 0,
+ sort: [{ by: "name", descending: false }],
+ }),
+ ),
+ );
+ expect(
+ search.mock.calls.every(([request]) => request.page_number === 0),
+ ).toBe(true);
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeEnabled(),
+ );
+ fireEvent.click(screen.getByTestId("next-page-button"));
+ await waitFor(() =>
+ expect(search).toHaveBeenLastCalledWith(
+ expect.objectContaining({ page_number: 1 }),
+ ),
+ );
+ await waitFor(() =>
+ expect(screen.getByTestId("next-page-button")).toBeDisabled(),
+ );
+ search.mockClear();
+ const columnMenu = within(
+ screen.getByRole("columnheader", { name: /name/ }),
+ ).getByRole("button", { name: "Column options" });
+ fireEvent.keyDown(columnMenu, { key: "Enter" });
+ fireEvent.click(await screen.findByRole("menuitem", { name: "Filter" }));
+ fireEvent.change(screen.getByPlaceholderText("Text…"), {
+ target: { value: "item" },
+ });
+ fireEvent.click(screen.getByLabelText("Apply filter"));
+ await waitFor(() =>
+ expect(search).toHaveBeenCalledWith(
+ expect.objectContaining({
+ page_number: 0,
+ filters: expect.objectContaining({ children: expect.any(Array) }),
+ }),
+ ),
+ );
+ expect(
+ search.mock.calls.every(([request]) => request.page_number === 0),
+ ).toBe(true);
+ });
+
+ it.each(["success", "error"] as const)(
+ "ignores a stale lazy page %s after the query changes",
+ async (outcome) => {
+ const props = makeProps();
+ const initial = {
+ data: [{ id: 1, name: "initial-page" }],
+ total_rows: "too_many" as const,
+ has_next_page: true,
+ };
+ const oldPage = new Deferred<typeof initial>();
+ const newQuery = new Deferred<typeof initial>();
+ const search = vi
+ .fn()
+ .mockResolvedValueOnce(initial)
+ .mockReturnValueOnce(oldPage.promise)
+ .mockReturnValueOnce(newQuery.promise);
+ render(
+ <Wrapper>
+ <LoadingDataTableComponent
+ {...props}
+ lazy={true}
+ totalRows="too_many"
+ selection={null}
+ showColumnSummaries={false}
+ search={search}
+ />
+ </Wrapper>,
+ );
+ await screen.findByText("initial-page");
+ fireEvent.click(screen.getByTestId("next-page-button"));
+ await waitFor(() =>
+ expect(search).toHaveBeenLastCalledWith(
+ expect.objectContaining({ page_number: 1 }),
+ ),
+ );
+ expect(screen.getByTestId("next-page-button")).toBeDisabled();
+
+ fireEvent.change(screen.getByPlaceholderText("Search..."), {
+ target: { value: "current" },
+ });
+ await waitFor(() =>
+ expect(search).toHaveBeenLastCalledWith(
+ expect.objectContaining({ query: "current", page_number: 0 }),
+ ),
+ );
+ await act(async () => {
+ newQuery.resolve({
+ data: [{ id: 2, name: "latest-page" }],
+ total_rows: "too_many",
+ has_next_page: false,
+ });
+ });
+ await screen.findByText("latest-page");
+
+ await act(async () => {
+ if (outcome === "success") {
+ oldPage.resolve({
+ ...initial,
+ data: [{ id: 3, name: "stale-result" }],
+ });
+ } else {
+ oldPage.reject(new Error("stale-page-error"));
+ }
+ });
+ expect(screen.getByText("latest-page")).toBeInTheDocument();
+ expect(screen.queryByText("stale-result")).not.toBeInTheDocument();
+ expect(screen.queryByText("stale-page-error")).not.toBeInTheDocument();
+ expect(screen.getByTestId("previous-page-button")).toBeDisabled();
+ expect(screen.getByTestId("next-page-button")).toBeDisabled();
+ },
+ );
+
+ it("recovers from a failed lazy page by navigating back", async () => {
+ const props = makeProps();
+ const initial = {
+ data: [{ id: 1, name: "initial-page" }],
+ total_rows: "too_many" as const,
+ has_next_page: true,
+ };
+ const failedPage = new Deferred<typeof initial>();
+ const search = vi
+ .fn()
+ .mockResolvedValueOnce(initial)
+ .mockReturnValueOnce(failedPage.promise)
+ .mockResolvedValueOnce({
+ ...initial,
+ data: [{ id: 1, name: "recovered-page" }],
+ })
+ .mockResolvedValueOnce({
+ ...initial,
+ data: [{ id: 2, name: "last-page" }],
+ has_next_page: false,
+ });
+ render(
+ <Wrapper>
+ <LoadingDataTableComponent
+ {...props}
+ lazy={true}
+ totalRows="too_many"
+ selection={null}
+ showColumnSummaries={false}
+ search={search}
+ />
+ </Wrapper>,
+ );
+ await screen.findByText("initial-page");
+ fireEvent.click(screen.getByTestId("next-page-button"));
+ await waitFor(() =>
+ expect(search).toHaveBeenLastCalledWith(
+ expect.objectContaining({ page_number: 1 }),
+ ),
+ );
+ expect(screen.getByTestId("next-page-button")).toBeDisabled();
+ expect(screen.getByText("initial-page")).toBeInTheDocument();
+ await act(async () => {
+ failedPage.reject(new Error("page-fetch-failed"));
+ });
+ await screen.findByText("page-fetch-failed");
+ expect(screen.getByTestId("next-page-button")).toBeDisabled();
+ expect(screen.getByTestId("previous-page-button")).toBeEnabled();
+
+ fireEvent.click(screen.getByTestId("previous-page-button"));
+ await screen.findByText("recovered-page");
+ expect(search).toHaveBeenLastCalledWith(
+ expect.objectContaining({ page_number: 0 }),
+ );
+ expect(screen.queryByText("page-fetch-failed")).not.toBeInTheDocument();
+ expect(screen.getByTestId("next-page-button")).toBeEnabled();
+ fireEvent.click(screen.getByTestId("next-page-button"));
+ await screen.findByText("last-page");
+ expect(screen.getByTestId("next-page-button")).toBeDisabled();
+ });
+
it("requests only the first page with the new configured page size", async () => {
const props = makeProps();
const view = (pageSize: number) => (
diff --git a/frontend/src/stories/data-table.stories.tsx b/frontend/src/stories/data-table.stories.tsx
index 213bbb5e14..3d6d95eb6e 100644
--- a/frontend/src/stories/data-table.stories.tsx
+++ b/frontend/src/stories/data-table.stories.tsx
@@ -1,5 +1,6 @@
/* Copyright 2026 Marimo. All rights reserved. */
+import { useState } from "react";
import {
generateColumns,
inferFieldTypes,
@@ -128,3 +129,33 @@ export const Pagination = {
name: "Pagination",
};
+
+export const UnknownTotal = {
+ render: function UnknownTotal() {
+ const [paginationState, setPaginationState] = useState({
+ pageIndex: 0,
+ pageSize: 5,
+ });
+ const rows = Array.from({ length: 12 }, (_, index) => ({
+ value: index + 1,
+ }));
+ const offset = paginationState.pageIndex * paginationState.pageSize;
+ return (
+ <DataTable
+ data={rows.slice(offset, offset + paginationState.pageSize)}
+ columns={generateColumns({
+ fieldTypes: inferFieldTypes(rows),
+ rowHeaders: [],
+ selection: null,
+ })}
+ totalRows="too_many"
+ hasNextPage={offset + paginationState.pageSize < rows.length}
+ totalColumns={1}
+ pagination={true}
+ manualPagination={true}
+ paginationState={paginationState}
+ setPaginationState={setPaginationState}
+ />
+ );
+ },
+};
diff --git a/marimo/_plugins/ui/_impl/table.py b/marimo/_plugins/ui/_impl/table.py
index 07e711f801..7926857ff2 100644
--- a/marimo/_plugins/ui/_impl/table.py
+++ b/marimo/_plugins/ui/_impl/table.py
@@ -49,6 +49,7 @@ from marimo._plugins.ui._impl.tables.geometry_export import (
get_export_metadata,
has_geometry_columns,
)
+from marimo._plugins.ui._impl.tables.narwhals_table import NarwhalsTableManager
from marimo._plugins.ui._impl.tables.selection import (
INDEX_COLUMN_NAME,
add_selection_column,
@@ -253,6 +254,13 @@ class SearchTableResponse:
# Unformatted data mirroring the same shape/page as `data`,
# provided when format_mapping is applied.
raw_data: str | None = None
+ has_next_page: bool | None = None
+
+
+@dataclass(frozen=True)
+class _TablePage:
+ manager: TableManager[Any]
+ has_next_page: bool | None = None
@dataclass(frozen=True)
@@ -607,9 +615,10 @@ class table(
Create a table from a lazy dataframe (e.g., Polars LazyFrame, Ibis Table, DuckDB Relation).
This won't load the data into memory until requested by the user.
- Once requested, only the first 10 rows will be loaded.
+ Pagination is supported; selection is not.
- Pagination and selection are not supported for lazy tables.
+ Pages are queried independently. For consistent results, sort by a unique
+ key and keep the source unchanged.
Args:
data (IntoLazyFrame): The data to display.
@@ -629,7 +638,7 @@ class table(
return table(
data=data,
- pagination=False,
+ pagination=True,
selection=None,
initial_selection=None,
page_size=page_size,
@@ -1045,7 +1054,7 @@ class table(
def _get_banner_text(self) -> str:
if self._lazy:
- return f"Previewing only the first {self._page_size} rows."
+ return ""
total_columns = self._manager.get_num_columns()
if self._max_columns is not None and total_columns > self._max_columns:
@@ -1783,6 +1792,38 @@ class table(
result[row_str] = row_hovers
return result
+ def _clamp_columns(
+ self, manager: TableManager[Any], max_columns: int | None
+ ) -> TableManager[Any]:
+ column_names = manager.get_column_names()
+ if max_columns is None or len(column_names) <= max_columns:
+ return manager
+
+ columns_to_select = column_names[:max_columns]
+ # Selection needs stable row IDs even when columns are clamped.
+ if self._has_stable_row_id:
+ columns_to_select = [INDEX_COLUMN_NAME] + columns_to_select
+ return manager.select_columns(columns_to_select)
+
+ def _prepare_page(
+ self, manager: TableManager[Any], page_size: int
+ ) -> _TablePage:
+ if not self._lazy or not isinstance(manager, NarwhalsTableManager):
+ return _TablePage(manager)
+
+ frame = manager.as_frame()
+ return _TablePage(
+ manager=get_table_manager(frame.head(page_size).to_native()),
+ has_next_page=frame.shape[0] > page_size,
+ )
+
+ def _serialize_page(
+ self, manager: TableManager[Any]
+ ) -> tuple[str, str | None]:
+ formatted = manager.to_json_str(self._format_mapping)
+ raw = manager.to_json_str() if self._format_mapping else None
+ return formatted, raw
+
def _search(self, args: SearchTableArgs) -> SearchTableResponse:
"""Search and filter the table data.
@@ -1815,75 +1856,38 @@ class table(
self._current_filters = args.filters
self._current_query = args.query
- def clamp_rows_and_columns(
- manager: TableManager[Any],
- ) -> tuple[str, str | None]:
- # Limit to page and column clamping for the frontend
- data = manager.take(args.page_size, offset)
- column_names = data.get_column_names()
-
- # Do not clamp if max_columns is None
- if max_columns is not None and len(column_names) > max_columns:
- columns_to_select = column_names[:max_columns]
- # Always include _marimo_row_id so the frontend can use
- # stable row IDs for selection, even when columns are clamped.
- # Without this, filtering + selecting returns wrong rows.
- if self._has_stable_row_id:
- columns_to_select = [INDEX_COLUMN_NAME] + columns_to_select
- data = data.select_columns(columns_to_select)
-
- try:
- formatted = data.to_json_str(self._format_mapping)
- raw = data.to_json_str() if self._format_mapping else None
- return formatted, raw
- except BaseException as e:
- # Catch and re-raise the error as a non-BaseException
- # to avoid crashing the kernel
- raise TableSearchError(str(e)) from e
-
- # If no query or sort, return nothing
- # The frontend will just show the original data
- total_rows: int | Literal["too_many"]
if not args.query and not args.sort and not args.filters:
- self._searched_manager = self._manager
- if self._lazy:
- total_rows = "too_many"
- else:
- total_rows = self._manager.get_num_rows(force=True) or 0
-
- formatted_data, raw_data = clamp_rows_and_columns(self._manager)
- return SearchTableResponse(
- data=formatted_data,
- total_rows=total_rows,
- cell_styles=self._style_cells(
- offset, args.page_size, total_rows
- ),
- cell_hover_texts=self._hover_cells(
- offset, args.page_size, total_rows
- ),
- raw_data=raw_data,
+ result = self._manager
+ else:
+ filter_function = (
+ self._apply_filters_query_sort_cached
+ if is_hashable(args.filters, args.query, args.sort)
+ else self._apply_filters_query_sort
+ )
+ result = filter_function(
+ args.filters,
+ args.query,
+ tuple(args.sort) if args.sort else None, # type: ignore
)
-
- filter_function = (
- self._apply_filters_query_sort_cached
- if is_hashable(args.filters, args.query, args.sort)
- else self._apply_filters_query_sort
- )
- result = filter_function(
- args.filters,
- args.query,
- tuple(args.sort) if args.sort else None, # type: ignore
- )
# Save the manager to be used for selection
self._searched_manager = result
+ total_rows: int | Literal["too_many"]
if self._lazy:
total_rows = "too_many"
else:
total_rows = result.get_num_rows(force=True) or 0
- formatted_data, raw_data = clamp_rows_and_columns(result)
+ count = args.page_size + 1 if self._lazy else args.page_size
+ data = self._clamp_columns(result.take(count, offset), max_columns)
+ try:
+ page = self._prepare_page(data, args.page_size)
+ formatted_data, raw_data = self._serialize_page(page.manager)
+ except BaseException as e:
+ # Translate native panics so they do not crash the kernel.
+ raise TableSearchError(str(e)) from e
+
return SearchTableResponse(
data=formatted_data,
total_rows=total_rows,
@@ -1892,6 +1896,7 @@ class table(
offset, args.page_size, total_rows
),
raw_data=raw_data,
+ has_next_page=page.has_next_page,
)
def _get_row_ids(self, args: EmptyArgs) -> GetRowIdsResponse:
diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
index f733833999..c56308b57b 100644
--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py
+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
@@ -13,6 +13,11 @@ from typing import TYPE_CHECKING, Any, Literal, cast
import msgspec
import narwhals.stable.v2 as nw
+from narwhals.dependencies import (
+ is_duckdb_relation,
+ is_ibis_table,
+ is_polars_lazyframe,
+)
from narwhals.typing import IntoDataFrameT, IntoLazyFrameT
from marimo import _loggers
@@ -443,16 +448,35 @@ class NarwhalsTableManager(
if offset < 0:
raise ValueError("Offset must be a non-negative integer")
+ if is_narwhals_lazyframe(self.data):
+ return self._take_lazy(self.data, count, offset)
+
if offset == 0:
return self.with_new_data(self.data.head(count))
- else:
- if is_narwhals_lazyframe(self.data):
- # Lazyframes do not support slicing, https://github.com/narwhals-dev/narwhals/issues/2389
- # So we collect the first n rows
- data = self.data.head(offset + count).collect()
- return self.with_new_data(data[offset : offset + count])
- else:
- return self.with_new_data(self.data[offset : offset + count])
+ return self.with_new_data(self.data[offset : offset + count])
+
+ def _take_lazy(
+ self, frame: nw.LazyFrame[IntoLazyFrameT], count: int, offset: int
+ ) -> TableManager[Any]:
+ native = frame.to_native()
+ if is_polars_lazyframe(native):
+ return self.with_new_data(
+ nw.from_native(native.slice(offset, count))
+ )
+ if is_duckdb_relation(native):
+ return self.with_new_data(
+ nw.from_native(native.limit(count, offset=offset))
+ )
+ if is_ibis_table(native):
+ return self.with_new_data(
+ nw.from_native(native.limit(count, offset=offset))
+ )
+
+ if offset == 0:
+ return self.with_new_data(frame.head(count))
+ # Narwhals has no portable lazy slicing API for the other backends.
+ data = frame.head(offset + count).collect()
+ return self.with_new_data(data[offset : offset + count])
def search(self, query: str) -> TableManager[Any]:
pattern = f"(?i){regex_literal(query)}"
diff --git a/tests/_plugins/ui/_impl/tables/test_ibis_table.py b/tests/_plugins/ui/_impl/tables/test_ibis_table.py
index 6699e5fe72..b1eff1a98c 100644
--- a/tests/_plugins/ui/_impl/tables/test_ibis_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_ibis_table.py
@@ -166,12 +166,16 @@ class TestIbisTableManagerFactory(unittest.TestCase):
2,
]
- assert as_list(self.manager.take(2, 1).select_columns(["A"]).data) == [
+ assert as_list(
+ self.manager.take(2, 1).select_columns(["A"]).data.collect()
+ ) == [
2,
3,
]
- assert as_list(self.manager.take(2, 2).select_columns(["A"]).data) == [
+ assert as_list(
+ self.manager.take(2, 2).select_columns(["A"]).data.collect()
+ ) == [
3,
]
diff --git a/tests/_plugins/ui/_impl/tables/test_narwhals.py b/tests/_plugins/ui/_impl/tables/test_narwhals.py
index 0ca85ac783..cc150bd90f 100644
--- a/tests/_plugins/ui/_impl/tables/test_narwhals.py
+++ b/tests/_plugins/ui/_impl/tables/test_narwhals.py
@@ -306,9 +306,8 @@ class TestNarwhalsTableManagerFactory(unittest.TestCase):
assert lazy_manager.take(1, 0).data.collect()["A"].to_list() == [1]
assert lazy_manager.take(2, 0).data.collect()["A"].to_list() == [1, 2]
- # Converted to eager frame
- assert lazy_manager.take(2, 1).data["A"].to_list() == [2, 3]
- assert lazy_manager.take(2, 2).data["A"].to_list() == [3]
+ assert lazy_manager.take(2, 1).data.collect()["A"].to_list() == [2, 3]
+ assert lazy_manager.take(2, 2).data.collect()["A"].to_list() == [3]
def test_take_zero(self) -> None:
limited_manager = self.manager.take(0, 0)
@@ -1007,6 +1006,30 @@ def test_empty_dataframe(df: Any) -> None:
assert empty_manager.get_field_types() == []
+ "df",
+ create_dataframes({"value": range(5)}, include=["pandas", "pyarrow"]),
+)
+ ("count", "offset", "expected"),
+ [(2, 0, [0, 1]), (2, 4, [4]), (2, 5, []), (0, 3, [])],
+)
+def test_take_lazy_without_native_slice(
+ df: Any, count: int, offset: int, expected: list[int]
+) -> None:
+ source = nw.from_native(df).lazy()
+ manager = NarwhalsTableManager.from_dataframe(source)
+
+ page = manager.take(count, offset)
+
+ assert json.loads(page.to_json_str()) == [
+ {"value": value} for value in expected
+ ]
+ assert source.collect().to_dict(as_series=False) == {
+ "value": list(range(5))
+ }
+
+
@pytest.mark.parametrize(
"df", create_dataframes({"A": [1, 2, 3]}, exclude=["duckdb"])
)
diff --git a/tests/_plugins/ui/_impl/test_table.py b/tests/_plugins/ui/_impl/test_table.py
index 911a96f991..5ea2c05f2e 100644
--- a/tests/_plugins/ui/_impl/test_table.py
+++ b/tests/_plugins/ui/_impl/test_table.py
@@ -2935,17 +2935,13 @@ def test_lazy_dataframe(df: Any) -> None:
# Verify the lazy flag is set
assert table._lazy is True
- # Check that the banner text indicates lazy loading
- assert (
- table._get_banner_text()
- == f"Previewing only the first {num_rows} rows."
- )
+ assert table._get_banner_text() == ""
# Verify the component args are set
assert table._component_args["lazy"] is True
assert table._component_args["total-rows"] == "too_many"
assert table._component_args["page-size"] == num_rows
- assert table._component_args["pagination"] is False
+ assert table._component_args["pagination"] is True
assert table._component_args["data"] == []
assert table._component_args["total-columns"] == 0
assert table._component_args["max-columns"] == DEFAULT_MAX_COLUMNS
@@ -2961,6 +2957,7 @@ def test_lazy_dataframe(df: Any) -> None:
search_args = SearchTableArgs(page_size=num_rows, page_number=0)
search_response = table._search(search_args)
assert search_response.total_rows == "too_many"
+ assert search_response.has_next_page is True
# Check that only the preview rows are returned
json_data = json.loads(search_response.data)
@@ -3890,3 +3887,102 @@ def test_binary_export_failure_is_contained(
assert data_format == "json"
assert json.loads(from_data_uri(url)[1]) == expected_json
assert manager.collect().equals(source)
+
+
+ "df",
+ create_dataframes({"value": range(6)}, include=NON_EAGER_LIBS),
+)
+ ("page_number", "expected", "has_next_page"),
+ [(0, [5, 4], True), (1, [3, 2], True), (2, [1, 0], False), (3, [], False)],
+)
+def test_lazy_table_pagination(
+ df: Any, page_number: int, expected: list[int], has_next_page: bool
+) -> None:
+ table = ui.table.lazy(df, page_size=2)
+ response = table._search(
+ SearchTableArgs(
+ page_size=2,
+ page_number=page_number,
+ sort=[SortArgs(by="value", descending=True)],
+ )
+ )
+ assert (
+ json.loads(response.data),
+ response.total_rows,
+ response.has_next_page,
+ ) == ([{"value": value} for value in expected], "too_many", has_next_page)
+
+ empty = table._search(
+ SearchTableArgs(
+ page_size=2,
+ page_number=0,
+ filters=FilterGroup(
+ type="group",
+ operator="and",
+ children=[
+ FilterCondition(
+ type="condition",
+ column_id="value",
+ operator=">",
+ value=10,
+ )
+ ],
+ ),
+ )
+ )
+ assert (json.loads(empty.data), empty.total_rows, empty.has_next_page) == (
+ [],
+ "too_many",
+ False,
+ )
+
+
+ "df", create_dataframes({"value": range(1000)}, include=NON_EAGER_LIBS)
+)
[email protected]("wrapped", [False, True])
+def test_lazy_table_pagination_collects_one_bounded_page(
+ df: Any, wrapped: bool
+) -> None:
+ import narwhals.stable.v2 as nw
+
+ collected_shapes: list[tuple[int, int]] = []
+ collect = nw.LazyFrame.collect
+
+ def record_collect(
+ frame: nw.LazyFrame[Any], *args: Any, **kwargs: Any
+ ) -> nw.DataFrame[Any]:
+ result = collect(frame, *args, **kwargs)
+ collected_shapes.append(result.shape)
+ return result
+
+ with patch.object(nw.LazyFrame, "collect", record_collect):
+ table = ui.table.lazy(
+ nw.from_native(df) if wrapped else df, page_size=10
+ )
+ assert collected_shapes == []
+ response = table._search(SearchTableArgs(page_size=10, page_number=50))
+
+ assert (
+ json.loads(response.data),
+ response.has_next_page,
+ collected_shapes,
+ ) == ([{"value": value} for value in range(500, 510)], True, [(11, 1)])
+ assert nw.from_native(df).collect().shape == (1000, 1)
+
+
+ "df", create_dataframes({"value": range(6)}, include=NON_EAGER_LIBS)
+)
+def test_lazy_table_pagination_of_limited_plan(df: Any) -> None:
+ import narwhals.stable.v2 as nw
+
+ source = nw.from_native(df).sort("value", descending=True).head(5)
+ table = ui.table.lazy(source.to_native(), page_size=3)
+ response = table._search(SearchTableArgs(page_size=3, page_number=1))
+ assert (json.loads(response.data), response.has_next_page) == (
+ [{"value": 2}, {"value": 1}],
+ False,
+ )