A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
perf(table): count lazy rows without loading the dataframe (#11405)
Shahmir Varqha313e65110fe9
diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.pyindex 06f743f9d6..1693142bdc 100644--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py@@ -807,15 +807,17 @@ class NarwhalsTableManager(return list(range(total))return [round(i * (total - 1) / (size - 1)) for i in range(size)]+ @cached_property+ def _lazy_num_rows(self) -> int:+ return int(self.as_lazy_frame().select(nw.len()).collect().item())+def get_num_rows(self, force: bool = True) -> int | None:- # If force is true, collect the data and get the number of rows+ if is_narwhals_lazyframe(self.data):+ return self._lazy_num_rows if force else None+if force:return self.as_frame().shape[0]- # When lazy, we don't know the number of rows- if is_narwhals_lazyframe(self.data):- return None-# Otherwise, we can get the number of rows from the shapetry:return self.data.shape[0]diff --git a/tests/_plugins/ui/_impl/tables/test_narwhals.py b/tests/_plugins/ui/_impl/tables/test_narwhals.pyindex 195dc3161c..696c87af7e 100644--- a/tests/_plugins/ui/_impl/tables/test_narwhals.py+++ b/tests/_plugins/ui/_impl/tables/test_narwhals.py@@ -1007,6 +1007,48 @@ def test_empty_dataframe(df: Any) -> None:assert empty_manager.get_field_types() == []+ "df", create_dataframes({"A": [1, 2, 3]}, exclude=["duckdb"])+)[email protected](("threshold", "expected"), [(0, 3), (1, 2), (3, 0)])+def test_filtered_row_count(df: Any, threshold: int, expected: int) -> None:+ manager = NarwhalsTableManager.from_dataframe(df)+ filtered = manager.with_new_data(+ manager.data.filter(nw.col("A") > threshold)+ )+ assert filtered.get_num_rows() == expected+ assert filtered.get_num_rows() == expected+ assert manager.get_num_rows() == 3++[email protected]("polars")+def test_lazy_row_count_collects_only_scalar(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ import polars as pl++ collected_shapes: list[tuple[int, int]] = []+ collect = pl.LazyFrame.collect++ def record_collect(+ self: pl.LazyFrame, *args: Any, **kwargs: Any+ ) -> pl.DataFrame:+ result = collect(self, *args, **kwargs)+ collected_shapes.append(result.shape)+ return result++ monkeypatch.setattr(pl.LazyFrame, "collect", record_collect)+ manager = NarwhalsTableManager.from_dataframe(+ pl.LazyFrame({"A": [1, 2, 3], "B": [4, 5, 6]})+ )+ assert manager.get_num_rows(force=False) is None+ assert collected_shapes == []+ assert manager.get_num_rows() == 3+ assert manager.get_num_rows() == 3+ assert manager.get_num_rows(force=False) is None+ assert collected_shapes == [(1, 1)]++@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")@pytest.mark.parametrize("df",