Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

perf(table): count lazy rows without loading the dataframe (#11405)

Shahmir Varqha313e65110fe9
diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
index 06f743f9d6..1693142bdc 100644
--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py
+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
@@ -807,15 +807,17 @@ class NarwhalsTableManager(
return list(range(total))
return [round(i * (total - 1) / (size - 1)) for i in range(size)]
+ @cached_property
+ def _lazy_num_rows(self) -> int:
+ return int(self.as_lazy_frame().select(nw.len()).collect().item())
+
def get_num_rows(self, force: bool = True) -> int | None:
- # If force is true, collect the data and get the number of rows
+ if is_narwhals_lazyframe(self.data):
+ return self._lazy_num_rows if force else None
+
if force:
return self.as_frame().shape[0]
- # When lazy, we don't know the number of rows
- if is_narwhals_lazyframe(self.data):
- return None
-
# Otherwise, we can get the number of rows from the shape
try:
return self.data.shape[0]
diff --git a/tests/_plugins/ui/_impl/tables/test_narwhals.py b/tests/_plugins/ui/_impl/tables/test_narwhals.py
index 195dc3161c..696c87af7e 100644
--- a/tests/_plugins/ui/_impl/tables/test_narwhals.py
+++ b/tests/_plugins/ui/_impl/tables/test_narwhals.py
@@ -1007,6 +1007,48 @@ def test_empty_dataframe(df: Any) -> None:
assert empty_manager.get_field_types() == []
+ "df", create_dataframes({"A": [1, 2, 3]}, exclude=["duckdb"])
+)
[email protected](("threshold", "expected"), [(0, 3), (1, 2), (3, 0)])
+def test_filtered_row_count(df: Any, threshold: int, expected: int) -> None:
+ manager = NarwhalsTableManager.from_dataframe(df)
+ filtered = manager.with_new_data(
+ manager.data.filter(nw.col("A") > threshold)
+ )
+ assert filtered.get_num_rows() == expected
+ assert filtered.get_num_rows() == expected
+ assert manager.get_num_rows() == 3
+
+
+def test_lazy_row_count_collects_only_scalar(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import polars as pl
+
+ collected_shapes: list[tuple[int, int]] = []
+ collect = pl.LazyFrame.collect
+
+ def record_collect(
+ self: pl.LazyFrame, *args: Any, **kwargs: Any
+ ) -> pl.DataFrame:
+ result = collect(self, *args, **kwargs)
+ collected_shapes.append(result.shape)
+ return result
+
+ monkeypatch.setattr(pl.LazyFrame, "collect", record_collect)
+ manager = NarwhalsTableManager.from_dataframe(
+ pl.LazyFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+ )
+ assert manager.get_num_rows(force=False) is None
+ assert collected_shapes == []
+ assert manager.get_num_rows() == 3
+ assert manager.get_num_rows() == 3
+ assert manager.get_num_rows(force=False) is None
+ assert collected_shapes == [(1, 1)]
+
+
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
@pytest.mark.parametrize(
"df",