Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

perf(table): lazy download-size RPC + first-page extrapolation (#9691)

Kiran Gadhaveb65bfb4876ed
diff --git a/frontend/src/components/data-table/TableTopBar.tsx b/frontend/src/components/data-table/TableTopBar.tsx
index ba52e37350..83706cb23f 100644
--- a/frontend/src/components/data-table/TableTopBar.tsx
+++ b/frontend/src/components/data-table/TableTopBar.tsx
@@ -35,6 +35,7 @@ interface TableTopBarProps extends Partial<ExportActionProps> {
togglePanel?: (panelType: PanelType) => void;
isAnyPanelOpen?: boolean;
sizeBytes?: number | null;
+ sizeBytesIsLoading?: boolean;
}
export const TableTopBar: React.FC<TableTopBarProps> = ({
@@ -50,6 +51,7 @@ export const TableTopBar: React.FC<TableTopBarProps> = ({
isAnyPanelOpen,
downloadAs,
sizeBytes,
+ sizeBytesIsLoading,
}) => {
const [internalValue, setInternalValue] = useState(searchQuery || "");
const debouncedSearch = useDebounce(internalValue, 500);
@@ -133,7 +135,11 @@ export const TableTopBar: React.FC<TableTopBarProps> = ({
</Button>
)}
{downloadAs && (
- <ExportMenu downloadAs={downloadAs} sizeBytes={sizeBytes} />
+ <ExportMenu
+ downloadAs={downloadAs}
+ sizeBytes={sizeBytes}
+ sizeBytesIsLoading={sizeBytesIsLoading}
+ />
)}
</div>
</div>
diff --git a/frontend/src/components/data-table/data-table.tsx b/frontend/src/components/data-table/data-table.tsx
index 4d172caba0..aadb5ae113 100644
--- a/frontend/src/components/data-table/data-table.tsx
+++ b/frontend/src/components/data-table/data-table.tsx
@@ -86,6 +86,7 @@ interface DataTableProps<TData> extends Partial<ExportActionProps> {
// JSON-serialized size of the currently-rendered data. Forwarded to
// ExportMenu so hosts can size-gate the Export button via downloadSizeLimitAtom.
sizeBytes?: number | null;
+ sizeBytesIsLoading?: boolean;
totalColumns: number;
pagination?: boolean;
manualPagination?: boolean; // server-side pagination
@@ -139,6 +140,7 @@ const DataTableInternal = <TData,>({
totalColumns,
totalRows,
sizeBytes,
+ sizeBytesIsLoading,
manualSorting = false,
sorting,
setSorting,
@@ -362,6 +364,7 @@ const DataTableInternal = <TData,>({
isAnyPanelOpen={isAnyPanelOpen}
downloadAs={downloadAs}
sizeBytes={sizeBytes}
+ sizeBytesIsLoading={sizeBytesIsLoading}
/>
{allUserColumnsHidden && (
<Banner className="mb-1 mx-2 rounded flex items-center justify-between">
diff --git a/frontend/src/components/data-table/export-actions.tsx b/frontend/src/components/data-table/export-actions.tsx
index a20f0c733c..dd6e389364 100644
--- a/frontend/src/components/data-table/export-actions.tsx
+++ b/frontend/src/components/data-table/export-actions.tsx
@@ -91,6 +91,7 @@ export interface ExportActionProps {
// marimo-lsp inside VS Code) declares a download size cap. Null/undefined
// means "no info" and the gate stays disabled (fail-open).
sizeBytes?: number | null;
+ sizeBytesIsLoading?: boolean;
}
const labelForDownloadFormat = (format: DownloadFormat): string =>
@@ -100,13 +101,19 @@ const labelForCopyFormat = (format: CopyFormat): string =>
export const ExportMenu: React.FC<ExportActionProps> = (props) => {
const { locale } = useLocale();
- const [open, setOpen] = React.useState(false);
+ const [downloadMenuOpen, setDownloadMenuOpen] = React.useState(false);
const policy = useAtomValue(downloadSizeLimitAtom);
- const disabled = !!(
+ const overLimit = !!(
policy &&
props.sizeBytes != null &&
props.sizeBytes > policy.limitBytes
);
+ const disabled = !!(policy && (props.sizeBytesIsLoading || overLimit));
+ const tooltipContent = !disabled
+ ? "Export"
+ : props.sizeBytesIsLoading
+ ? "Checking download size…"
+ : policy?.unavailableMessage;
const button = (
<Button
@@ -116,7 +123,7 @@ export const ExportMenu: React.FC<ExportActionProps> = (props) => {
disabled={disabled}
className={cn(
"print:hidden text-xs gap-1",
- open ? "text-primary" : "text-muted-foreground",
+ downloadMenuOpen ? "text-primary" : "text-muted-foreground",
)}
>
<DownloadIcon className="w-3.5 h-3.5" />
@@ -248,10 +255,14 @@ export const ExportMenu: React.FC<ExportActionProps> = (props) => {
};
return (
- <DropdownMenu modal={false} open={open} onOpenChange={setOpen}>
+ <DropdownMenu
+ modal={false}
+ open={downloadMenuOpen}
+ onOpenChange={setDownloadMenuOpen}
+ >
<Tooltip
- content={disabled ? policy?.unavailableMessage : "Export"}
- open={open ? false : undefined}
+ content={tooltipContent}
+ open={downloadMenuOpen ? false : undefined}
>
<DropdownMenuTrigger asChild={true} disabled={disabled}>
<span tabIndex={disabled ? 0 : -1} className="inline-flex">
diff --git a/frontend/src/plugins/impl/DataTablePlugin.tsx b/frontend/src/plugins/impl/DataTablePlugin.tsx
index 4ed484ab4c..840ccba501 100644
--- a/frontend/src/plugins/impl/DataTablePlugin.tsx
+++ b/frontend/src/plugins/impl/DataTablePlugin.tsx
@@ -9,7 +9,7 @@ import type {
RowSelectionState,
SortingState,
} from "@tanstack/react-table";
-import { Provider } from "jotai";
+import { Provider, useAtomValue } from "jotai";
import { Table2Icon } from "lucide-react";
import type { JSX } from "react";
import React, {
@@ -28,6 +28,7 @@ import { TablePanel } from "@/components/data-table/charts/charts";
import { hasChart } from "@/components/data-table/charts/storage";
import { ColumnChartSpecModel } from "@/components/data-table/column-summary/chart-spec-model";
import { ColumnChartContext } from "@/components/data-table/column-summary/column-summary";
+import { downloadSizeLimitAtom } from "@/components/data-table/download-policy/atoms";
import { filtersToFilterGroup } from "@/components/data-table/filters";
import { usePanelOwnership } from "@/components/data-table/hooks/use-panel-ownership";
import { LoadingTable } from "@/components/data-table/loading-table";
@@ -195,7 +196,6 @@ interface Data<T> {
wrappedColumns?: string[];
headerTooltip?: Record<string, string>;
totalColumns: number;
- sizeBytes?: number | null;
maxColumns: number | "all";
hasStableRowId: boolean;
lazy: boolean;
@@ -222,12 +222,14 @@ type DataTableFunctions = {
cell_styles?: CellStyleState | null;
cell_hover_texts?: Record<string, Record<string, string | null>> | null;
raw_data?: TableData<T> | null;
- size_bytes?: number | null;
}>;
get_data_url?: GetDataUrl;
get_row_ids?: GetRowIds;
calculate_top_k_rows?: CalculateTopKRows;
preview_column?: PreviewColumn;
+ get_size_bytes: (opts: Record<string, never>) => Promise<{
+ size_bytes?: number | null;
+ }>;
};
type S = (number | string | { rowId: string; columnName?: string })[];
@@ -274,7 +276,6 @@ export const DataTablePlugin = createPlugin<S>("marimo-table")
headerTooltip: z.record(z.string(), z.string()).optional(),
fieldTypes: columnToFieldTypesSchema.nullish(),
totalColumns: z.number(),
- sizeBytes: z.number().nullish(),
maxColumns: z.union([z.number(), z.literal("all")]).default("all"),
hasStableRowId: z.boolean().default(false),
maxHeight: z.number().optional(),
@@ -332,7 +333,6 @@ export const DataTablePlugin = createPlugin<S>("marimo-table")
.nullable(),
cell_hover_texts: cellHoverTextSchema.nullable(),
raw_data: z.union([z.string(), z.array(z.looseObject({}))]).nullish(),
- size_bytes: z.number().nullish(),
}),
),
get_row_ids: rpc.input(z.object({}).passthrough()).output(
@@ -364,6 +364,9 @@ export const DataTablePlugin = createPlugin<S>("marimo-table")
stats: columnStats.nullable(),
}),
),
+ get_size_bytes: rpc
+ .input(z.object({}))
+ .output(z.object({ size_bytes: z.number().nullish() })),
})
.renderer((props) => {
return (
@@ -538,7 +541,6 @@ export const LoadingDataTableComponent = memo(
rows: T[];
rawRows?: T[];
totalRows: number | TooManyRows;
- sizeBytes?: number | null;
cellStyles: CellStyleState | undefined | null;
cellHoverTexts?: Record<string, Record<string, string | null>> | null;
}>(async () => {
@@ -555,7 +557,6 @@ export const LoadingDataTableComponent = memo(
let tableData = props.data;
let rawTableData: TableData<T> | undefined | null = props.rawData;
let totalRows = props.totalRows;
- let sizeBytes = props.sizeBytes ?? null;
let cellStyles = props.cellStyles;
let cellHoverTexts = props.cellHoverTexts;
@@ -599,7 +600,6 @@ export const LoadingDataTableComponent = memo(
tableData = searchResults.data;
rawTableData = searchResults.raw_data;
totalRows = searchResults.total_rows;
- sizeBytes = searchResults.size_bytes ?? null;
cellStyles = searchResults.cell_styles || {};
cellHoverTexts = searchResults.cell_hover_texts || {};
}
@@ -612,7 +612,6 @@ export const LoadingDataTableComponent = memo(
rows: tableData,
rawRows: rawData,
totalRows: totalRows,
- sizeBytes,
cellStyles,
cellHoverTexts,
};
@@ -624,7 +623,6 @@ export const LoadingDataTableComponent = memo(
useDeepCompareMemoize(props.fieldTypes),
props.data,
props.totalRows,
- props.sizeBytes,
props.lazy,
props.cellHoverTexts,
props.cellStyles,
@@ -632,6 +630,34 @@ export const LoadingDataTableComponent = memo(
paginationState.pageIndex,
]);
+ const policy = useAtomValue(downloadSizeLimitAtom);
+ const { data: sizeBytesData, isPending: sizeBytesPending } = useAsyncData<
+ number | null
+ >(async () => {
+ if (
+ !policy ||
+ !props.showDownload ||
+ props.lazy ||
+ props.totalRows === 0
+ ) {
+ return null;
+ }
+ const result = await props.get_size_bytes({});
+ return result.size_bytes ?? null;
+ }, [
+ policy,
+ props.showDownload,
+ props.get_size_bytes,
+ props.lazy,
+ props.totalRows,
+ searchQuery,
+ useDeepCompareMemoize(filters),
+ useDeepCompareMemoize(sorting),
+ ]);
+ const sizeBytes = sizeBytesData ?? null;
+ const sizeBytesIsLoading =
+ !!policy && props.showDownload && sizeBytesPending;
+
const getRow = useCallback(
async (rowId: number) => {
const sortArgs =
@@ -739,7 +765,8 @@ export const LoadingDataTableComponent = memo(
setFilters={setFilters}
reloading={isFetching && !isPending}
totalRows={data?.totalRows ?? props.totalRows}
- sizeBytes={data?.sizeBytes ?? props.sizeBytes ?? null}
+ sizeBytes={sizeBytes}
+ sizeBytesIsLoading={sizeBytesIsLoading}
paginationState={paginationState}
setPaginationState={setPaginationState}
cellStyles={data?.cellStyles ?? props.cellStyles}
@@ -787,6 +814,7 @@ const DataTableComponent = ({
rawData,
totalRows,
sizeBytes,
+ sizeBytesIsLoading,
maxColumns,
pagination,
selection,
@@ -838,6 +866,8 @@ const DataTableComponent = ({
rawData?: unknown[];
columnSummaries?: ColumnSummaries;
getRow: (rowIdx: number) => Promise<GetRowResult>;
+ sizeBytes?: number | null;
+ sizeBytesIsLoading?: boolean;
}): JSX.Element => {
const id = useId();
const [viewedRowIdx, setViewedRowIdx] = useState(0);
@@ -1068,6 +1098,7 @@ const DataTableComponent = ({
sorting={sorting}
totalRows={totalRows}
sizeBytes={sizeBytes}
+ sizeBytesIsLoading={sizeBytesIsLoading}
totalColumns={totalColumns}
manualSorting={true}
setSorting={setSorting}
diff --git a/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx b/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
index ceaab93cd7..488a222ee9 100644
--- a/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
+++ b/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
@@ -107,6 +107,7 @@ describe("LoadingDataTableComponent", () => {
}),
get_data_url: vi.fn() as GetDataUrl,
get_row_ids: vi.fn() as GetRowIds,
+ get_size_bytes: vi.fn().mockResolvedValue({ size_bytes: null }),
};
const Wrapper = ({ children }: { children: React.ReactNode }) => (
diff --git a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx
index 18d138e635..c8be4cd045 100644
--- a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx
+++ b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx
@@ -64,7 +64,6 @@ type PluginFunctions = {
column_types_per_step: FieldTypesWithExternalType[];
python_code?: string | null;
sql_code?: string | null;
- size_bytes?: number | null;
}>;
get_column_values: (req: { column: string }) => Promise<{
values: unknown[];
@@ -82,9 +81,11 @@ type PluginFunctions = {
}) => Promise<{
data: TableData<T>;
total_rows: number;
- size_bytes?: number | null;
}>;
download_as: DownloadAsArgs;
+ get_size_bytes: (opts: Record<string, never>) => Promise<{
+ size_bytes?: number | null;
+ }>;
};
// Value is selection, but it is not currently exposed to the user
@@ -120,7 +121,6 @@ export const DataFramePlugin = createPlugin<S>("marimo-dataframe")
column_types_per_step: z.array(columnToFieldTypesSchema),
python_code: z.string().nullish(),
sql_code: z.string().nullish(),
- size_bytes: z.number().nullish(),
}),
),
get_column_values: rpc.input(z.object({ column: z.string() })).output(
@@ -150,10 +150,12 @@ export const DataFramePlugin = createPlugin<S>("marimo-dataframe")
z.object({
data: z.union([z.string(), z.array(z.object({}).passthrough())]),
total_rows: z.number(),
- size_bytes: z.number().nullish(),
}),
),
download_as: DownloadAsSchema,
+ get_size_bytes: rpc
+ .input(z.object({}))
+ .output(z.object({ size_bytes: z.number().nullish() })),
})
.renderer((props) => (
<TableProviders>
@@ -192,6 +194,7 @@ export const DataFrameComponent = memo(
get_column_values,
search,
download_as,
+ get_size_bytes,
host,
}: DataTableProps): JSX.Element => {
const { data, error, isPending } = useAsyncData(
@@ -207,7 +210,6 @@ export const DataFrameComponent = memo(
column_types_per_step,
python_code,
sql_code,
- size_bytes,
} = data || {};
const totalColumns = field_types?.length;
@@ -327,7 +329,6 @@ export const DataFrameComponent = memo(
data={url || ""}
hasStableRowId={false}
totalRows={total_rows ?? 0}
- sizeBytes={size_bytes ?? null}
totalColumns={totalColumns ?? 0}
maxColumns="all"
pageSize={pageSize}
@@ -336,6 +337,7 @@ export const DataFrameComponent = memo(
rowHeaders={row_headers || Arrays.EMPTY}
showDownload={showDownload}
download_as={download_as}
+ get_size_bytes={get_size_bytes}
enableSearch={false}
showFilters={false}
search={search}
diff --git a/frontend/src/stories/dataframe.stories.tsx b/frontend/src/stories/dataframe.stories.tsx
index c109a312bb..6dbb05c533 100644
--- a/frontend/src/stories/dataframe.stories.tsx
+++ b/frontend/src/stories/dataframe.stories.tsx
@@ -42,6 +42,7 @@ export const DataFrame: StoryObj = {
host={document.body}
showDownload={false}
download_as={async () => ({ url: "", filename: "" })}
+ get_size_bytes={async () => ({ size_bytes: null })}
lazy={false}
/>
);
diff --git a/marimo/_plugins/ui/_impl/dataframes/dataframe.py b/marimo/_plugins/ui/_impl/dataframes/dataframe.py
index b2ec97eed5..8fdc41674f 100644
--- a/marimo/_plugins/ui/_impl/dataframes/dataframe.py
+++ b/marimo/_plugins/ui/_impl/dataframes/dataframe.py
@@ -26,6 +26,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
from marimo._plugins.ui._impl.table import (
DownloadAsArgs,
DownloadAsResponse,
+ GetSizeBytesResponse,
SearchTableArgs,
SearchTableResponse,
SortArgs,
@@ -72,9 +73,6 @@ class GetDataFrameResponse:
column_types_per_step: list[FieldTypes]
python_code: str | None = None
sql_code: str | None = None
- # JSON-serialized size of the current (post-transform) data; see
- # ``SearchTableResponse.size_bytes`` for usage.
- size_bytes: int | None = None
@dataclass
@@ -199,7 +197,6 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
"columns": self._get_column_types(),
"dataframe-name": dataframe_name,
"total": rows,
- "size-bytes": self._get_json_size_bytes(self._manager),
"page-size": page_size,
"show-download": show_download,
"download-csv-encoding": download_csv_encoding,
@@ -228,6 +225,11 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
arg_cls=DownloadAsArgs,
function=self._download_as,
),
+ Function(
+ name="get_size_bytes",
+ arg_cls=EmptyArgs,
+ function=self._get_size_bytes,
+ ),
),
)
@@ -271,7 +273,6 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
sql_code=self._handler.as_sql_code(
self._transform_container._snapshot_df
),
- size_bytes=response.size_bytes,
)
def _get_column_values(
@@ -338,7 +339,6 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
return SearchTableResponse(
data=data,
total_rows=result.get_num_rows(force=True) or 0,
- size_bytes=self._get_json_size_bytes(result),
)
def _download_as(self, args: DownloadAsArgs) -> DownloadAsResponse:
@@ -398,10 +398,7 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
tm = tm.take(limit, 0)
return tm
- @memoize_last_value
- def _get_json_size_bytes(self, manager: TableManager[Any]) -> int | None:
- """JSON-serialized size of `manager`; see table._get_json_size_bytes."""
- try:
- return len(manager.to_json(strict_json=True))
- except Exception:
- return None
+ def _get_size_bytes(self, args: EmptyArgs) -> GetSizeBytesResponse:
+ del args
+ manager = self._get_cached_table_manager(self._value, self._limit)
+ return GetSizeBytesResponse(size_bytes=manager.estimate_size_bytes())
diff --git a/marimo/_plugins/ui/_impl/table.py b/marimo/_plugins/ui/_impl/table.py
index 01632e4903..23b9bac746 100644
--- a/marimo/_plugins/ui/_impl/table.py
+++ b/marimo/_plugins/ui/_impl/table.py
@@ -71,7 +71,6 @@ from marimo._runtime.context.types import (
from marimo._runtime.context.utils import get_mode
from marimo._runtime.functions import EmptyArgs, Function
from marimo._utils.hashable import is_hashable
-from marimo._utils.memoize import memoize_last_value
from marimo._utils.methods import getcallable
from marimo._utils.narwhals_utils import (
can_narwhalify_lazyframe,
@@ -166,7 +165,10 @@ class SearchTableResponse:
# Unformatted data mirroring the same shape/page as `data`,
# provided when format_mapping is applied.
raw_data: str | None = None
- # JSON-serialized size of the currently-rendered (searched/filtered) data.
+
+
+@dataclass(frozen=True)
+class GetSizeBytesResponse:
size_bytes: int | None = None
@@ -822,11 +824,6 @@ class table(
"data": search_result_data,
"raw-data": search_result_raw_data,
"total-rows": total_rows,
- "size-bytes": (
- self._get_json_size_bytes(self._manager)
- if not _internal_lazy
- else None
- ),
"total-columns": num_columns,
"max-columns": max_columns_arg,
"banner-text": self._get_banner_text(),
@@ -898,6 +895,11 @@ class table(
arg_cls=PreviewColumnArgs,
function=self._preview_column,
),
+ Function(
+ name="get_size_bytes",
+ arg_cls=EmptyArgs,
+ function=self._get_size_bytes,
+ ),
),
)
@@ -950,19 +952,10 @@ class table(
)
return unwrap_narwhals_dataframe(self._selected_manager.data) # type: ignore[no-any-return]
- @memoize_last_value
- def _get_json_size_bytes(self, manager: TableManager[Any]) -> int | None:
- """Size in bytes of the manager's JSON serialization.
-
- JSON is the largest format we export, so this is a conservative size estimate:
- if the JSON fits under a host's download limit, CSV and Parquet will
- too. Memoized on manager identity — recomputed when filters/search
- produce a new `_searched_manager`.
- """
- try:
- return len(manager.to_json(strict_json=True))
- except Exception:
- return None
+ def _get_size_bytes(self, args: EmptyArgs) -> GetSizeBytesResponse:
+ del args
+ manager = self._searched_manager or self._manager
+ return GetSizeBytesResponse(size_bytes=manager.estimate_size_bytes())
def _download_as(self, args: DownloadAsArgs) -> DownloadAsResponse:
"""Download the table data in the specified format.
@@ -1667,7 +1660,6 @@ class table(
offset, args.page_size, total_rows
),
raw_data=raw_data,
- size_bytes=self._get_json_size_bytes(self._manager),
)
filter_function = (
@@ -1698,7 +1690,6 @@ class table(
offset, args.page_size, total_rows
),
raw_data=raw_data,
- size_bytes=self._get_json_size_bytes(result),
)
def _get_row_ids(self, args: EmptyArgs) -> GetRowIdsResponse:
diff --git a/marimo/_plugins/ui/_impl/tables/table_manager.py b/marimo/_plugins/ui/_impl/tables/table_manager.py
index 7d3ad17d6e..2cb2769f3a 100644
--- a/marimo/_plugins/ui/_impl/tables/table_manager.py
+++ b/marimo/_plugins/ui/_impl/tables/table_manager.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import abc
+import math
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
@@ -24,6 +25,10 @@ if TYPE_CHECKING:
T = TypeVar("T")
+# Rows used when estimating the JSON-serialized size of the rendered data.
+# Bigger samples are more precise but cost more on the kernel control loop.
+SIZE_ESTIMATE_SAMPLE_ROWS = 100
+
ColumnName = str
RowId = str
FieldType = DataType
@@ -136,6 +141,36 @@ class TableManager(abc.ABC, Generic[T]):
ensure_ascii=ensure_ascii,
).encode(resolved_encoding)
+ def estimate_size_bytes(
+ self,
+ sample_rows: int = SIZE_ESTIMATE_SAMPLE_ROWS,
+ ) -> int | None:
+ """Estimate JSON-download byte size by extrapolating from a head sample.
+
+ Always models a JSON download (raw values, `strict_json=True`,
+ `ensure_ascii=True`) since the frontend gates downloads before the
+ user picks a format; JSON with ASCII escaping is the upper-bound
+ proxy.
+
+ Returns the exact size when the table fits within `sample_rows`,
+ otherwise scales the sample size by `total_rows / sample_rows`.
+ Returns `None` if serialization fails.
+ """
+ total_rows = self.get_num_rows(force=True) or 0
+ if total_rows == 0:
+ return 0
+ n = min(total_rows, sample_rows)
+ try:
+ sample = self.take(n, 0).to_json_str(
+ strict_json=True, ensure_ascii=True
+ )
+ except Exception:
+ return None
+ sample_bytes = len(sample.encode("utf-8"))
+ if n == total_rows:
+ return sample_bytes
+ return math.ceil(sample_bytes * total_rows / n)
+
@abc.abstractmethod
def to_parquet(self) -> bytes:
raise NotImplementedError
diff --git a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
index 0703434795..666e3de949 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
@@ -889,37 +889,38 @@ def test_base_exception_handling():
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
-def test_dataframe_render_args_carry_size_bytes() -> None:
+def test_dataframe_get_size_bytes_rpc_extrapolates() -> None:
import pandas as pd
- df = pd.DataFrame({"a": [1, 2, 3]})
+ from marimo._plugins.ui._impl.table import GetSizeBytesResponse
+
+ df = pd.DataFrame({"a": list(range(2000))})
subject = ui.dataframe(df)
- args = subject._component_args # type: ignore[attr-defined]
- assert args["size-bytes"] == len(
- subject._manager.to_json(strict_json=True)
- )
+ resp = subject._get_size_bytes(EmptyArgs())
+ assert isinstance(resp, GetSizeBytesResponse)
+ assert resp.size_bytes is not None
+ exact = len(subject._manager.to_json(strict_json=True))
+ assert abs(resp.size_bytes - exact) / exact < 0.25
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
-def test_dataframe_get_dataframe_response_carries_size_bytes() -> None:
- import pandas as pd
+def test_dataframe_get_size_bytes_rpc_returns_none_on_serialization_failure() -> (
+ None
+):
+ from unittest.mock import patch
- df = pd.DataFrame({"a": [1, 2, 3]})
- subject = ui.dataframe(df)
- response = subject._get_dataframe(EmptyArgs())
- assert response.size_bytes == len(
- subject._manager.to_json(strict_json=True)
- )
+ import pandas as pd
+ from marimo._plugins.ui._impl.table import GetSizeBytesResponse
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
-def test_dataframe_get_json_size_bytes_fails_open() -> None:
- import pandas as pd
+ subject = ui.dataframe(pd.DataFrame({"a": [1, 2, 3]}))
+ manager_cls = type(subject._manager)
- subject = ui.dataframe(pd.DataFrame({"a": [1]}))
+ def _raise(*_args: object, **_kwargs: object) -> str:
+ raise RuntimeError("boom")
- class _Boom:
- def to_json(self, **_: object) -> str:
- raise RuntimeError("boom")
+ with patch.object(manager_cls, "to_json_str", _raise):
+ resp = subject._get_size_bytes(EmptyArgs())
- assert subject._get_json_size_bytes(_Boom()) is None # type: ignore[arg-type]
+ assert isinstance(resp, GetSizeBytesResponse)
+ assert resp.size_bytes is None
diff --git a/tests/_plugins/ui/_impl/test_table.py b/tests/_plugins/ui/_impl/test_table.py
index 0d4fd91bfb..dde9c1c0ce 100644
--- a/tests/_plugins/ui/_impl/test_table.py
+++ b/tests/_plugins/ui/_impl/test_table.py
@@ -1534,70 +1534,55 @@ def test_download_as(df: Any) -> None:
assert selected_nw["cities"][0] == "New York"
-def test_json_size_bytes_matches_payload() -> None:
- data = [{"a": 1, "b": "x"}, {"a": 2, "b": "y"}]
- table = ui.table(data)
- expected = len(table._manager.to_json(strict_json=True))
- assert table._get_json_size_bytes(table._manager) == expected
- assert expected > 0
-
-
-def test_json_size_bytes_cached_per_manager_identity() -> None:
- table = ui.table([{"a": 1}, {"a": 2}])
- # The constructor already primed the cache with table._manager; verify
- # that asking again does not re-serialize, while a fresh manager does.
- call_count = 0
- original_to_json = type(table._manager).to_json
-
- def counting_to_json(self: Any, **kwargs: Any) -> Any:
- nonlocal call_count
- call_count += 1
- return original_to_json(self, **kwargs)
-
- other_manager = ui.table([{"a": 3, "b": 4}])._manager
- # other_manager's construction primed the cache with a different key,
- # so the next call on table._manager will be a fresh miss.
- with patch.object(type(table._manager), "to_json", counting_to_json):
- table._get_json_size_bytes(table._manager)
- table._get_json_size_bytes(table._manager)
- assert call_count == 1, (
- "second call with same manager identity should hit cache"
- )
- table._get_json_size_bytes(other_manager)
- assert call_count == 2, "different manager identity should recompute"
+def test_get_size_bytes_rpc_extrapolates_from_sample() -> None:
+ from marimo._plugins.ui._impl.table import GetSizeBytesResponse
+ from marimo._plugins.ui._impl.tables.table_manager import (
+ SIZE_ESTIMATE_SAMPLE_ROWS,
+ )
+ small = ui.table([{"a": i} for i in range(SIZE_ESTIMATE_SAMPLE_ROWS)])
+ large = ui.table([{"a": i} for i in range(SIZE_ESTIMATE_SAMPLE_ROWS * 10)])
-def test_json_size_bytes_fails_open() -> None:
- table = ui.table([{"a": 1}])
+ resp_small = small._get_size_bytes(EmptyArgs())
+ resp_large = large._get_size_bytes(EmptyArgs())
- class _Boom:
- def to_json(self, **_: Any) -> str:
- raise RuntimeError("boom")
+ assert isinstance(resp_small, GetSizeBytesResponse)
+ assert isinstance(resp_large, GetSizeBytesResponse)
+ assert resp_small.size_bytes is not None
+ assert resp_large.size_bytes is not None
+ ratio = resp_large.size_bytes / resp_small.size_bytes
+ assert 7.5 < ratio < 12.5, f"unexpected ratio {ratio}"
- assert table._get_json_size_bytes(_Boom()) is None # type: ignore[arg-type]
+def test_get_size_bytes_rpc_uses_searched_manager() -> None:
+ data = [{"a": i, "b": "match" if i < 5 else "miss"} for i in range(50)]
+ t = ui.table(data)
-def test_render_args_carry_size_bytes() -> None:
- table = ui.table([{"a": 1}, {"a": 2}])
- args = table._component_args # type: ignore[attr-defined]
- assert args["size-bytes"] == len(table._manager.to_json(strict_json=True))
+ full = t._get_size_bytes(EmptyArgs()).size_bytes
+ t._search(SearchTableArgs(query="match", page_size=5, page_number=0))
+ filtered = t._get_size_bytes(EmptyArgs()).size_bytes
+ assert full is not None
+ assert filtered is not None
+ assert filtered < full
-def test_search_response_carries_size_bytes() -> None:
- data = [{"a": i} for i in range(10)]
- table = ui.table(data)
- # Unfiltered branch — should report _manager's size.
- unfiltered = table._search(SearchTableArgs(page_size=5, page_number=0))
- assert unfiltered.size_bytes == len(
- table._manager.to_json(strict_json=True)
- )
- # Filtered branch — should report the searched manager's size, which
- # differs from the full manager.
- filtered = table._search(
- SearchTableArgs(query="1", page_size=5, page_number=0)
- )
- assert filtered.size_bytes is not None
- assert filtered.size_bytes < unfiltered.size_bytes
+
+def test_get_size_bytes_rpc_returns_none_on_serialization_failure() -> None:
+ from unittest.mock import patch
+
+ from marimo._plugins.ui._impl.table import GetSizeBytesResponse
+
+ t = ui.table([{"a": 1}])
+ manager_cls = type(t._manager)
+
+ def _raise(*_args: object, **_kwargs: object) -> str:
+ raise RuntimeError("boom")
+
+ with patch.object(manager_cls, "to_json_str", _raise):
+ resp = t._get_size_bytes(EmptyArgs())
+
+ assert isinstance(resp, GetSizeBytesResponse)
+ assert resp.size_bytes is None
def test_download_as_ignores_cell_selection() -> None: