A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
add lazy argument to ui.dataframe (#7476)
Shahmir Varqha6a2716b185ef
diff --git a/examples/ui/dataframe.py b/examples/ui/dataframe.pyindex ae720e8b38..4711483348 100644--- a/examples/ui/dataframe.py+++ b/examples/ui/dataframe.py@@ -8,7 +8,7 @@import marimo-__generated_with = "0.15.5"+__generated_with = "0.18.3"app = marimo.App(width="medium")@@ -24,9 +24,16 @@ def _():return (data,)[email protected](hide_code=True)+def _(mo):+ lazy_button = mo.ui.checkbox(label="Lazy Dataframe")+ lazy_button+ return (lazy_button,)++@app.cell-def _(data, mo):- dataframe_transformer = mo.ui.dataframe(data.iris())+def _(data, lazy_button, mo):+ dataframe_transformer = mo.ui.dataframe(data.iris(), lazy=lazy_button.value)dataframe_transformerreturn (dataframe_transformer,)diff --git a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsxindex 8394d9633a..46e2f1be4d 100644--- a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx+++ b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx@@ -2,7 +2,14 @@import { isEqual } from "lodash-es";import { Code2Icon, DatabaseIcon, FunctionSquareIcon } from "lucide-react";-import { type JSX, memo, useEffect, useRef, useState } from "react";+import {+ type JSX,+ memo,+ useCallback,+ useEffect,+ useRef,+ useState,+} from "react";import { z } from "zod";import {type DownloadAsArgs,@@ -20,7 +27,7 @@ import { Functions } from "@/utils/functions";import { ErrorBanner } from "../common/error-banner";import { LoadingDataTableComponent, TableProviders } from "../DataTablePlugin";import type { DataType } from "../vega/vega-loader";-import { TransformPanel } from "./panel";+import { TransformPanel, type TransformPanelHandle } from "./panel";import {ConditionSchema,type ConditionType,@@ -43,6 +50,7 @@ interface Data {columns: ColumnDataTypes;pageSize: number;showDownload: boolean;+ lazy: boolean;}// eslint-disable-next-line @typescript-eslint/consistent-type-definitions@@ -93,6 +101,7 @@ export const DataFramePlugin = createPlugin<S>("marimo-dataframe")});return map;}),+ lazy: z.boolean().default(false),}),).withFunctions<PluginFunctions>({@@ -167,6 +176,7 @@ export const DataFrameComponent = memo(columns,pageSize,showDownload,+ lazy,value,setValue,get_dataframe,@@ -187,6 +197,20 @@ export const DataFrameComponent = memo(value || EMPTY,);+ const transformPanelRef = useRef<TransformPanelHandle>(null);++ const transformTab = "transform";++ // When switching tabs in lazy mode, save any pending changes+ const handleTabChange = useCallback(+ (newTab: string) => {+ if (lazy && newTab !== transformTab) {+ transformPanelRef.current?.submit();+ }+ },+ [lazy],+ );+// If dataframe changes and value.transforms gets reset, then// apply existing transformations (displayed in panel) to new dataconst prevValueRef = useRef(internalValue);@@ -204,10 +228,10 @@ export const DataFrameComponent = memo(return (<div>- <Tabs defaultValue="transform">+ <Tabs defaultValue={transformTab} onValueChange={handleTabChange}><div className="flex items-center gap-2"><TabsList className="h-8">- <TabsTrigger value="transform" className="text-xs py-1">+ <TabsTrigger value={transformTab} className="text-xs py-1"><FunctionSquareIcon className="w-3 h-3 mr-2" />Transform</TabsTrigger>@@ -232,6 +256,7 @@ export const DataFrameComponent = memo(className="mt-1 border rounded-t overflow-hidden"><TransformPanel+ ref={transformPanelRef}initialValue={internalValue}columns={columns}onChange={(newValue) => {@@ -245,6 +270,7 @@ export const DataFrameComponent = memo(}}onInvalidChange={setInternalValue}getColumnValues={get_column_values}+ lazy={lazy}/></TabsContent>{python_code && (diff --git a/frontend/src/plugins/impl/data-frames/panel.tsx b/frontend/src/plugins/impl/data-frames/panel.tsxindex cae50644b3..768e026959 100644--- a/frontend/src/plugins/impl/data-frames/panel.tsx+++ b/frontend/src/plugins/impl/data-frames/panel.tsx@@ -11,6 +11,7 @@ import {FilterIcon,FunctionSquareIcon,GroupIcon,+ InfoIcon,MousePointerSquareDashedIcon,PencilIcon,PlusIcon,@@ -18,7 +19,12 @@ import {SquareMousePointerIcon,Trash2Icon,} from "lucide-react";-import React, { type PropsWithChildren, useEffect, useMemo } from "react";+import React, {+ type PropsWithChildren,+ useEffect,+ useImperativeHandle,+ useMemo,+} from "react";import { useFieldArray, useForm } from "react-hook-form";import useEvent from "react-use-event-hook";import type { z } from "zod";@@ -42,6 +48,7 @@ import {DropdownMenuSeparator,DropdownMenuTrigger,} from "../../../components/ui/dropdown-menu";+import { Tooltip } from "../../../components/ui/tooltip";import { cn } from "../../../utils/cn";import { DATAFRAME_FORM_RENDERERS } from "./forms/renderers";import {@@ -53,6 +60,10 @@ import {import type { ColumnDataTypes } from "./types";import { getUpdatedColumnTypes } from "./utils/getUpdatedColumnTypes";+export interface TransformPanelHandle {+ submit: () => void;+}+interface Props {columns: ColumnDataTypes;initialValue: Transformations;@@ -62,6 +73,8 @@ interface Props {values: unknown[];too_many_values: boolean;}>;+ lazy: boolean;+ ref?: React.Ref<TransformPanelHandle>;}export const TransformPanel: React.FC<Props> = ({@@ -70,6 +83,8 @@ export const TransformPanel: React.FC<Props> = ({onChange,onInvalidChange,getColumnValues,+ lazy,+ ref,}) => {const form = useForm<z.infer<typeof TransformationsSchema>>({resolver: zodResolver(TransformationsSchema),@@ -77,7 +92,7 @@ export const TransformPanel: React.FC<Props> = ({mode: "onChange",reValidateMode: "onChange",});- const { handleSubmit, watch, control } = form;+ const { handleSubmit, watch, control, formState } = form;const onSubmit = useEvent((values: z.infer<typeof TransformationsSchema>) => {onChange(values);@@ -89,14 +104,38 @@ export const TransformPanel: React.FC<Props> = ({},);+ const handleApply = useEvent(() => {+ handleSubmit(+ (values) => {+ onSubmit(values);+ // Reset dirty state by setting current values as new default+ // Use keepValues to avoid re-initializing field arrays+ if (lazy) {+ form.reset(values, { keepValues: true });+ }+ },+ () => {+ onInvalidSubmit(form.getValues());+ },+ )();+ });++ useImperativeHandle(ref, () => {+ return {+ submit: handleApply,+ };+ }, []);+useEffect(() => {+ // If lazy, do not auto-submit on input changes+ if (lazy) {+ return;+ }const subscription = watch(() => {- handleSubmit(onSubmit, () => {- onInvalidSubmit(form.getValues());- })();+ handleApply();});return () => subscription.unsubscribe();- }, [handleSubmit, watch, onInvalidSubmit, onSubmit, form]);+ }, [watch, handleApply, lazy]);const [selectedTransform, setSelectedTransform] = React.useState<number | undefined@@ -136,7 +175,11 @@ export const TransformPanel: React.FC<Props> = ({<ColumnFetchValuesContext value={getColumnValues}><formonSubmit={(e) => e.preventDefault()}- className="flex flex-row max-h-[400px] overflow-hidden bg-background"+ // When lazy, prevent Enter from submitting+ onKeyDown={+ lazy ? (e) => e.key === "Enter" && e.preventDefault() : undefined+ }+ className="relative flex flex-row max-h-[400px] overflow-hidden bg-background"><Sidebaritems={form.watch("transforms")}@@ -176,6 +219,49 @@ export const TransformPanel: React.FC<Props> = ({</div>)}</div>+ {lazy && (+ <div className="absolute bottom-0 right-0 border-l border-t rounded-tl-sm flex flex-row items-center gap-1 p-1.5 pr-1">+ <Button+ data-testid="marimo-plugin-data-frames-apply"+ variant={formState.isDirty ? "warn" : "outline"}+ size="xs"+ onClick={handleApply}+ className="h-6"+ >+ Apply+ </Button>+ <Tooltip+ delayDuration={100}+ content={+ <div className="flex flex-col gap-1.5 text-xs text-muted-foreground max-w-96 text-pretty">+ <p>+ This dataframe is marked lazy to improve performance.{" "}+ <span>+ Click{" "}+ <span className="px-1 mr-1 rounded border border-border">+ Apply+ </span>+ to apply the transforms.+ </span>+ </p>+ <p>+ Pass{" "}+ <code className="bg-muted px-1 py-0.5 rounded-sm">+ lazy=False+ </code>{" "}+ to{" "}+ <code className="bg-muted px-1 py-0.5 rounded-sm">+ mo.ui.dataframe+ </code>{" "}+ to automatically apply transformations.+ </p>+ </div>+ }+ >+ <InfoIcon className="w-3 h-3 text-muted-foreground hover:text-foreground cursor-help" />+ </Tooltip>+ </div>+ )}</form></ColumnFetchValuesContext></ColumnInfoContext>diff --git a/frontend/src/stories/dataframe.stories.tsx b/frontend/src/stories/dataframe.stories.tsxindex 436eba2796..5b851ccb24 100644--- a/frontend/src/stories/dataframe.stories.tsx+++ b/frontend/src/stories/dataframe.stories.tsx@@ -42,6 +42,7 @@ export const DataFrame: StoryObj = {host={document.body}showDownload={false}download_as={async () => ""}+ lazy={false}/>);},diff --git a/marimo/_plugins/ui/_impl/dataframes/dataframe.py b/marimo/_plugins/ui/_impl/dataframes/dataframe.pyindex a16cd5574d..b51412fb0f 100644--- a/marimo/_plugins/ui/_impl/dataframes/dataframe.py+++ b/marimo/_plugins/ui/_impl/dataframes/dataframe.py@@ -44,9 +44,11 @@ from marimo._plugins.validators import ()from marimo._runtime.functions import EmptyArgs, Functionfrom marimo._utils.memoize import memoize_last_value-from marimo._utils.narwhals_utils import make_lazy+from marimo._utils.narwhals_utils import is_narwhals_lazyframe, make_lazyfrom marimo._utils.parse_dataclass import parse_raw+TOO_MANY_ROWS = 100_000+@dataclassclass GetDataFrameResponse:@@ -108,6 +110,8 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):Defaults to True.on_change (Optional[Callable[[DataFrameType], None]], optional): Optional callbackto run when this element's value changes.+ lazy (Optional[bool], optional): When lazy is True, an 'Apply' button will be shown to apply transformations.+ Defaults to None where lazy is inferred from the dataframe and row count."""_name: Final[str] = "marimo-dataframe"@@ -119,6 +123,7 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):page_size: Optional[int] = 5,limit: Optional[int] = None,show_download: bool = True,+ lazy: Optional[bool] = None,) -> None:validate_no_integer_columns(df)# This will raise an error if the dataframe type is not supported.@@ -152,6 +157,12 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):self._last_transforms = Transformations([])self._page_size = page_size or 5 # Default to 5 rows (.head())self._show_download = show_download+ rows = self._manager.get_num_rows(force=False)+ if lazy is None:+ lazy = is_narwhals_lazyframe(df) or (+ rows is None or rows > TOO_MANY_ROWS+ )+ self._lazy = lazyvalidate_page_size(self._page_size)super().__init__(@@ -164,9 +175,10 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):args={"columns": self._get_column_types(),"dataframe-name": dataframe_name,- "total": self._manager.get_num_rows(force=False),+ "total": rows,"page-size": page_size,"show-download": show_download,+ "lazy": self._lazy,},functions=(Function(diff --git a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py b/tests/_plugins/ui/_impl/dataframes/test_dataframe.pyindex 3adcb7d38b..71fce8d49c 100644--- a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py+++ b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py@@ -598,6 +598,68 @@ class TestDataframes:)[email protected](not HAS_DEPS, reason="optional dependencies not installed")+def test_dataframe_lazy_small() -> None:+ """Small dataframes should not be marked as lazy."""+ df = pd.DataFrame({"A": [1, 2, 3], "B": ["a", "b", "c"]})++ subject = ui.dataframe(df)+ assert subject._lazy is False+ assert subject._component_args["lazy"] is False++[email protected](not HAS_POLARS, reason="Polars not installed")+def test_dataframe_lazy_lazyframe() -> None:+ """Polars LazyFrames should be marked as lazy."""+ import polars as pl++ df = pl.DataFrame({"A": [1, 2, 3]}).lazy()+ subject = ui.dataframe(df)+ assert subject._lazy is True+ assert subject._component_args["lazy"] is True++[email protected](not HAS_DEPS, reason="optional dependencies not installed")+def test_dataframe_lazy_large() -> None:+ """DataFrames with more than 100,000 rows should be marked as lazy."""+ from marimo._plugins.ui._impl.dataframes.dataframe import TOO_MANY_ROWS++ df = pd.DataFrame({"A": range(TOO_MANY_ROWS + 1)})+ subject = ui.dataframe(df)+ assert subject._lazy is True+ assert subject._component_args["lazy"] is True++[email protected](not HAS_DEPS, reason="optional dependencies not installed")+def test_dataframe_lazy_explicit_override() -> None:+ """Explicit lazy parameter should override inferred value."""+ df = pd.DataFrame({"A": [1, 2, 3]})++ # Force lazy=True on a small dataframe+ subject = ui.dataframe(df, lazy=True)+ assert subject._lazy is True+ assert subject._component_args["lazy"] is True++ # Force lazy=False on a small dataframe (same as inferred)+ subject = ui.dataframe(df, lazy=False)+ assert subject._lazy is False+ assert subject._component_args["lazy"] is False++[email protected](not HAS_IBIS, reason="Ibis not installed")+def test_ibis_table_lazy() -> None:+ """Ibis tables should be marked as lazy (row count may be unknown)."""+ import ibis++ # Ibis memtables may have unknown row count initially+ data = {"A": [1, 2, 3], "B": ["a", "b", "c"]}+ memtable = ibis.memtable(data)+ subject = ui.dataframe(memtable)+ # Ibis tables are typically treated as lazy since row count+ # may not be immediately available+ assert subject._component_args["lazy"] is True++@pytest.mark.skipif(not HAS_IBIS or not HAS_POLARS,reason="optional dependencies not installed",