Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

add lazy argument to ui.dataframe (#7476)

Shahmir Varqha6a2716b185ef
diff --git a/examples/ui/dataframe.py b/examples/ui/dataframe.py
index ae720e8b38..4711483348 100644
--- a/examples/ui/dataframe.py
+++ b/examples/ui/dataframe.py
@@ -8,7 +8,7 @@
import marimo
-__generated_with = "0.15.5"
+__generated_with = "0.18.3"
app = marimo.App(width="medium")
@@ -24,9 +24,16 @@ def _():
return (data,)
[email protected](hide_code=True)
+def _(mo):
+ lazy_button = mo.ui.checkbox(label="Lazy Dataframe")
+ lazy_button
+ return (lazy_button,)
+
+
@app.cell
-def _(data, mo):
- dataframe_transformer = mo.ui.dataframe(data.iris())
+def _(data, lazy_button, mo):
+ dataframe_transformer = mo.ui.dataframe(data.iris(), lazy=lazy_button.value)
dataframe_transformer
return (dataframe_transformer,)
diff --git a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx
index 8394d9633a..46e2f1be4d 100644
--- a/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx
+++ b/frontend/src/plugins/impl/data-frames/DataFramePlugin.tsx
@@ -2,7 +2,14 @@
import { isEqual } from "lodash-es";
import { Code2Icon, DatabaseIcon, FunctionSquareIcon } from "lucide-react";
-import { type JSX, memo, useEffect, useRef, useState } from "react";
+import {
+ type JSX,
+ memo,
+ useCallback,
+ useEffect,
+ useRef,
+ useState,
+} from "react";
import { z } from "zod";
import {
type DownloadAsArgs,
@@ -20,7 +27,7 @@ import { Functions } from "@/utils/functions";
import { ErrorBanner } from "../common/error-banner";
import { LoadingDataTableComponent, TableProviders } from "../DataTablePlugin";
import type { DataType } from "../vega/vega-loader";
-import { TransformPanel } from "./panel";
+import { TransformPanel, type TransformPanelHandle } from "./panel";
import {
ConditionSchema,
type ConditionType,
@@ -43,6 +50,7 @@ interface Data {
columns: ColumnDataTypes;
pageSize: number;
showDownload: boolean;
+ lazy: boolean;
}
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
@@ -93,6 +101,7 @@ export const DataFramePlugin = createPlugin<S>("marimo-dataframe")
});
return map;
}),
+ lazy: z.boolean().default(false),
}),
)
.withFunctions<PluginFunctions>({
@@ -167,6 +176,7 @@ export const DataFrameComponent = memo(
columns,
pageSize,
showDownload,
+ lazy,
value,
setValue,
get_dataframe,
@@ -187,6 +197,20 @@ export const DataFrameComponent = memo(
value || EMPTY,
);
+ const transformPanelRef = useRef<TransformPanelHandle>(null);
+
+ const transformTab = "transform";
+
+ // When switching tabs in lazy mode, save any pending changes
+ const handleTabChange = useCallback(
+ (newTab: string) => {
+ if (lazy && newTab !== transformTab) {
+ transformPanelRef.current?.submit();
+ }
+ },
+ [lazy],
+ );
+
// If dataframe changes and value.transforms gets reset, then
// apply existing transformations (displayed in panel) to new data
const prevValueRef = useRef(internalValue);
@@ -204,10 +228,10 @@ export const DataFrameComponent = memo(
return (
<div>
- <Tabs defaultValue="transform">
+ <Tabs defaultValue={transformTab} onValueChange={handleTabChange}>
<div className="flex items-center gap-2">
<TabsList className="h-8">
- <TabsTrigger value="transform" className="text-xs py-1">
+ <TabsTrigger value={transformTab} className="text-xs py-1">
<FunctionSquareIcon className="w-3 h-3 mr-2" />
Transform
</TabsTrigger>
@@ -232,6 +256,7 @@ export const DataFrameComponent = memo(
className="mt-1 border rounded-t overflow-hidden"
>
<TransformPanel
+ ref={transformPanelRef}
initialValue={internalValue}
columns={columns}
onChange={(newValue) => {
@@ -245,6 +270,7 @@ export const DataFrameComponent = memo(
}}
onInvalidChange={setInternalValue}
getColumnValues={get_column_values}
+ lazy={lazy}
/>
</TabsContent>
{python_code && (
diff --git a/frontend/src/plugins/impl/data-frames/panel.tsx b/frontend/src/plugins/impl/data-frames/panel.tsx
index cae50644b3..768e026959 100644
--- a/frontend/src/plugins/impl/data-frames/panel.tsx
+++ b/frontend/src/plugins/impl/data-frames/panel.tsx
@@ -11,6 +11,7 @@ import {
FilterIcon,
FunctionSquareIcon,
GroupIcon,
+ InfoIcon,
MousePointerSquareDashedIcon,
PencilIcon,
PlusIcon,
@@ -18,7 +19,12 @@ import {
SquareMousePointerIcon,
Trash2Icon,
} from "lucide-react";
-import React, { type PropsWithChildren, useEffect, useMemo } from "react";
+import React, {
+ type PropsWithChildren,
+ useEffect,
+ useImperativeHandle,
+ useMemo,
+} from "react";
import { useFieldArray, useForm } from "react-hook-form";
import useEvent from "react-use-event-hook";
import type { z } from "zod";
@@ -42,6 +48,7 @@ import {
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "../../../components/ui/dropdown-menu";
+import { Tooltip } from "../../../components/ui/tooltip";
import { cn } from "../../../utils/cn";
import { DATAFRAME_FORM_RENDERERS } from "./forms/renderers";
import {
@@ -53,6 +60,10 @@ import {
import type { ColumnDataTypes } from "./types";
import { getUpdatedColumnTypes } from "./utils/getUpdatedColumnTypes";
+export interface TransformPanelHandle {
+ submit: () => void;
+}
+
interface Props {
columns: ColumnDataTypes;
initialValue: Transformations;
@@ -62,6 +73,8 @@ interface Props {
values: unknown[];
too_many_values: boolean;
}>;
+ lazy: boolean;
+ ref?: React.Ref<TransformPanelHandle>;
}
export const TransformPanel: React.FC<Props> = ({
@@ -70,6 +83,8 @@ export const TransformPanel: React.FC<Props> = ({
onChange,
onInvalidChange,
getColumnValues,
+ lazy,
+ ref,
}) => {
const form = useForm<z.infer<typeof TransformationsSchema>>({
resolver: zodResolver(TransformationsSchema),
@@ -77,7 +92,7 @@ export const TransformPanel: React.FC<Props> = ({
mode: "onChange",
reValidateMode: "onChange",
});
- const { handleSubmit, watch, control } = form;
+ const { handleSubmit, watch, control, formState } = form;
const onSubmit = useEvent((values: z.infer<typeof TransformationsSchema>) => {
onChange(values);
@@ -89,14 +104,38 @@ export const TransformPanel: React.FC<Props> = ({
},
);
+ const handleApply = useEvent(() => {
+ handleSubmit(
+ (values) => {
+ onSubmit(values);
+ // Reset dirty state by setting current values as new default
+ // Use keepValues to avoid re-initializing field arrays
+ if (lazy) {
+ form.reset(values, { keepValues: true });
+ }
+ },
+ () => {
+ onInvalidSubmit(form.getValues());
+ },
+ )();
+ });
+
+ useImperativeHandle(ref, () => {
+ return {
+ submit: handleApply,
+ };
+ }, []);
+
useEffect(() => {
+ // If lazy, do not auto-submit on input changes
+ if (lazy) {
+ return;
+ }
const subscription = watch(() => {
- handleSubmit(onSubmit, () => {
- onInvalidSubmit(form.getValues());
- })();
+ handleApply();
});
return () => subscription.unsubscribe();
- }, [handleSubmit, watch, onInvalidSubmit, onSubmit, form]);
+ }, [watch, handleApply, lazy]);
const [selectedTransform, setSelectedTransform] = React.useState<
number | undefined
@@ -136,7 +175,11 @@ export const TransformPanel: React.FC<Props> = ({
<ColumnFetchValuesContext value={getColumnValues}>
<form
onSubmit={(e) => e.preventDefault()}
- className="flex flex-row max-h-[400px] overflow-hidden bg-background"
+ // When lazy, prevent Enter from submitting
+ onKeyDown={
+ lazy ? (e) => e.key === "Enter" && e.preventDefault() : undefined
+ }
+ className="relative flex flex-row max-h-[400px] overflow-hidden bg-background"
>
<Sidebar
items={form.watch("transforms")}
@@ -176,6 +219,49 @@ export const TransformPanel: React.FC<Props> = ({
</div>
)}
</div>
+ {lazy && (
+ <div className="absolute bottom-0 right-0 border-l border-t rounded-tl-sm flex flex-row items-center gap-1 p-1.5 pr-1">
+ <Button
+ data-testid="marimo-plugin-data-frames-apply"
+ variant={formState.isDirty ? "warn" : "outline"}
+ size="xs"
+ onClick={handleApply}
+ className="h-6"
+ >
+ Apply
+ </Button>
+ <Tooltip
+ delayDuration={100}
+ content={
+ <div className="flex flex-col gap-1.5 text-xs text-muted-foreground max-w-96 text-pretty">
+ <p>
+ This dataframe is marked lazy to improve performance.{" "}
+ <span>
+ Click{" "}
+ <span className="px-1 mr-1 rounded border border-border">
+ Apply
+ </span>
+ to apply the transforms.
+ </span>
+ </p>
+ <p>
+ Pass{" "}
+ <code className="bg-muted px-1 py-0.5 rounded-sm">
+ lazy=False
+ </code>{" "}
+ to{" "}
+ <code className="bg-muted px-1 py-0.5 rounded-sm">
+ mo.ui.dataframe
+ </code>{" "}
+ to automatically apply transformations.
+ </p>
+ </div>
+ }
+ >
+ <InfoIcon className="w-3 h-3 text-muted-foreground hover:text-foreground cursor-help" />
+ </Tooltip>
+ </div>
+ )}
</form>
</ColumnFetchValuesContext>
</ColumnInfoContext>
diff --git a/frontend/src/stories/dataframe.stories.tsx b/frontend/src/stories/dataframe.stories.tsx
index 436eba2796..5b851ccb24 100644
--- a/frontend/src/stories/dataframe.stories.tsx
+++ b/frontend/src/stories/dataframe.stories.tsx
@@ -42,6 +42,7 @@ export const DataFrame: StoryObj = {
host={document.body}
showDownload={false}
download_as={async () => ""}
+ lazy={false}
/>
);
},
diff --git a/marimo/_plugins/ui/_impl/dataframes/dataframe.py b/marimo/_plugins/ui/_impl/dataframes/dataframe.py
index a16cd5574d..b51412fb0f 100644
--- a/marimo/_plugins/ui/_impl/dataframes/dataframe.py
+++ b/marimo/_plugins/ui/_impl/dataframes/dataframe.py
@@ -44,9 +44,11 @@ from marimo._plugins.validators import (
)
from marimo._runtime.functions import EmptyArgs, Function
from marimo._utils.memoize import memoize_last_value
-from marimo._utils.narwhals_utils import make_lazy
+from marimo._utils.narwhals_utils import is_narwhals_lazyframe, make_lazy
from marimo._utils.parse_dataclass import parse_raw
+TOO_MANY_ROWS = 100_000
+
@dataclass
class GetDataFrameResponse:
@@ -108,6 +110,8 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
Defaults to True.
on_change (Optional[Callable[[DataFrameType], None]], optional): Optional callback
to run when this element's value changes.
+ lazy (Optional[bool], optional): When lazy is True, an 'Apply' button will be shown to apply transformations.
+ Defaults to None where lazy is inferred from the dataframe and row count.
"""
_name: Final[str] = "marimo-dataframe"
@@ -119,6 +123,7 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
page_size: Optional[int] = 5,
limit: Optional[int] = None,
show_download: bool = True,
+ lazy: Optional[bool] = None,
) -> None:
validate_no_integer_columns(df)
# This will raise an error if the dataframe type is not supported.
@@ -152,6 +157,12 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
self._last_transforms = Transformations([])
self._page_size = page_size or 5 # Default to 5 rows (.head())
self._show_download = show_download
+ rows = self._manager.get_num_rows(force=False)
+ if lazy is None:
+ lazy = is_narwhals_lazyframe(df) or (
+ rows is None or rows > TOO_MANY_ROWS
+ )
+ self._lazy = lazy
validate_page_size(self._page_size)
super().__init__(
@@ -164,9 +175,10 @@ class dataframe(UIElement[dict[str, Any], DataFrameType]):
args={
"columns": self._get_column_types(),
"dataframe-name": dataframe_name,
- "total": self._manager.get_num_rows(force=False),
+ "total": rows,
"page-size": page_size,
"show-download": show_download,
+ "lazy": self._lazy,
},
functions=(
Function(
diff --git a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
index 3adcb7d38b..71fce8d49c 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py
@@ -598,6 +598,68 @@ class TestDataframes:
)
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+def test_dataframe_lazy_small() -> None:
+ """Small dataframes should not be marked as lazy."""
+ df = pd.DataFrame({"A": [1, 2, 3], "B": ["a", "b", "c"]})
+
+ subject = ui.dataframe(df)
+ assert subject._lazy is False
+ assert subject._component_args["lazy"] is False
+
+
[email protected](not HAS_POLARS, reason="Polars not installed")
+def test_dataframe_lazy_lazyframe() -> None:
+ """Polars LazyFrames should be marked as lazy."""
+ import polars as pl
+
+ df = pl.DataFrame({"A": [1, 2, 3]}).lazy()
+ subject = ui.dataframe(df)
+ assert subject._lazy is True
+ assert subject._component_args["lazy"] is True
+
+
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+def test_dataframe_lazy_large() -> None:
+ """DataFrames with more than 100,000 rows should be marked as lazy."""
+ from marimo._plugins.ui._impl.dataframes.dataframe import TOO_MANY_ROWS
+
+ df = pd.DataFrame({"A": range(TOO_MANY_ROWS + 1)})
+ subject = ui.dataframe(df)
+ assert subject._lazy is True
+ assert subject._component_args["lazy"] is True
+
+
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+def test_dataframe_lazy_explicit_override() -> None:
+ """Explicit lazy parameter should override inferred value."""
+ df = pd.DataFrame({"A": [1, 2, 3]})
+
+ # Force lazy=True on a small dataframe
+ subject = ui.dataframe(df, lazy=True)
+ assert subject._lazy is True
+ assert subject._component_args["lazy"] is True
+
+ # Force lazy=False on a small dataframe (same as inferred)
+ subject = ui.dataframe(df, lazy=False)
+ assert subject._lazy is False
+ assert subject._component_args["lazy"] is False
+
+
[email protected](not HAS_IBIS, reason="Ibis not installed")
+def test_ibis_table_lazy() -> None:
+ """Ibis tables should be marked as lazy (row count may be unknown)."""
+ import ibis
+
+ # Ibis memtables may have unknown row count initially
+ data = {"A": [1, 2, 3], "B": ["a", "b", "c"]}
+ memtable = ibis.memtable(data)
+ subject = ui.dataframe(memtable)
+ # Ibis tables are typically treated as lazy since row count
+ # may not be immediately available
+ assert subject._component_args["lazy"] is True
+
+
@pytest.mark.skipif(
not HAS_IBIS or not HAS_POLARS,
reason="optional dependencies not installed",