A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: serialize Polars cache values with native IPC (#10775)
Péter Gyarmatib0f23f44e277
diff --git a/marimo/_runtime/primitives.py b/marimo/_runtime/primitives.pyindex 7bd25ce9db..5f8e911044 100644--- a/marimo/_runtime/primitives.py+++ b/marimo/_runtime/primitives.py@@ -76,6 +76,10 @@ def is_data_primitive(value: Any) -> bool:if is_instance_by_name(value, "torch.Tensor"):return str(value.device) == "cpu"+ # Series follow the same numeric-schema policy as dataframes.+ if is_instance_by_name(value, "polars.series.series.Series"):+ return bool(value.dtype.is_numeric())+# If a numpy like array, ensure that it's not an object array.if hasattr(value, "dtype"):return not (diff --git a/marimo/_save/cache.py b/marimo/_save/cache.pyindex 767e5318f5..76eb1ab291 100644--- a/marimo/_save/cache.py+++ b/marimo/_save/cache.py@@ -46,7 +46,7 @@ if TYPE_CHECKING:from marimo._save.stores import Store# NB. Increment on cache breaking changes.-MARIMO_CACHE_VERSION: int = 5+MARIMO_CACHE_VERSION: int = 6CacheType = Literal["ContextExecutionPath",diff --git a/marimo/_save/encode.py b/marimo/_save/encode.pyindex 9953270d91..ebda84d4a6 100644--- a/marimo/_save/encode.py+++ b/marimo/_save/encode.py@@ -139,7 +139,11 @@ def common_container_to_bytes(value: Any) -> bytes:if is_primitive(value):return primitive_to_bytes(value)- return data_to_buffer(value)+ if is_data_primitive(value):+ return data_to_buffer(value)+ raise TypeError(+ f"Expected numeric array data, got {type(value).__name__}"+ )return recurse_container(value)diff --git a/marimo/_save/loaders/lazy.py b/marimo/_save/loaders/lazy.pyindex 5f623defc6..40376f02df 100644--- a/marimo/_save/loaders/lazy.py+++ b/marimo/_save/loaders/lazy.py@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Anyimport msgspecfrom marimo import _loggers+from marimo._dependencies.dependencies import DependencyManagerfrom marimo._runtime.context import safe_get_contextfrom marimo._runtime.primitives import (is_data_primitive,@@ -218,6 +219,13 @@ def maybe_update_lazy_stub(value: Any) -> str:# MRO not that expensive, type hashable for functools lookup.result = mro_lookup(value_type, LAZY_STUB_LOOKUP)loader = result[1] if result else "pickle"+ if (+ loader == "arrow"+ and result is not None+ and result[0].startswith("pandas.")+ and not DependencyManager.pyarrow.has()+ ):+ return "pickle"return loaderdiff --git a/marimo/_save/stubs/lazy_stub.py b/marimo/_save/stubs/lazy_stub.pyindex e6eb71abf5..afce872b85 100644--- a/marimo/_save/stubs/lazy_stub.py+++ b/marimo/_save/stubs/lazy_stub.py@@ -184,12 +184,14 @@ def _arrow_load(data: bytes, type_hint: str | None = None) -> Any:# type_hint is the fq class name written by to_item() at save time.# Using it (rather than schema metadata inspection) is explicit and# version-stable across pyarrow/polars/pandas releases.- DependencyManager.pyarrow.require("to load cached Arrow IPC blobs.")- import pyarrow as pa-- reader = pa.ipc.open_file(io.BytesIO(data))- table = reader.read_all()if type_hint and type_hint.startswith("pandas."):+ DependencyManager.pyarrow.require(+ "to load cached pandas Arrow IPC blobs."+ )+ import pyarrow as pa++ reader = pa.ipc.open_file(io.BytesIO(data))+ table = reader.read_all()df = table.to_pandas()if type_hint in ("pandas.Series", "pandas.core.series.Series"):# Stored as a single-column DataFrame; recover as a Series.@@ -197,12 +199,10 @@ def _arrow_load(data: bytes, type_hint: str | None = None) -> Any:return dfimport polars as pl- result = pl.from_arrow(table)+ result = pl.read_ipc(io.BytesIO(data))if type_hint == "polars.series.series.Series":# Stored as a single-column DataFrame; recover as a Series.- if isinstance(result, pl.DataFrame):- return result.to_series(0)- return result+ return result.to_series(0)return result@@ -265,27 +265,15 @@ def _pandas_to_arrow_ipc(df: Any) -> bytes:def _arrow_dump(obj: Any) -> bytes:- # Duck-type dispatch:- # polars DataFrame → write_ipc()- # pandas DataFrame → Arrow IPC via pyarrow.ipc- # Series (either) → to_frame() first, then the appropriate DataFrame method- # Fall back to pickle when pyarrow is absent so the cache write never fails.- if not DependencyManager.pyarrow.has():- return pickle.dumps(obj)- if hasattr(obj, "write_ipc"): # polars DataFrame- buf = io.BytesIO()- obj.write_ipc(buf)- return buf.getvalue()- if hasattr(obj, "to_feather"): # pandas DataFrame- return _pandas_to_arrow_ipc(obj)- # Series — promote to single-column DataFrame, then detect library- frame = obj.to_frame()- if hasattr(frame, "write_ipc"): # polars Series → polars DataFrame- buf = io.BytesIO()- frame.write_ipc(buf)- return buf.getvalue()- # pandas Series → pandas DataFrame- return _pandas_to_arrow_ipc(frame)+ # Inspect class methods because pandas columns also appear as attributes.+ if not callable(getattr(type(obj), "write_ipc", None)):+ if callable(getattr(type(obj), "to_frame", None)):+ obj = obj.to_frame()+ if not callable(getattr(type(obj), "write_ipc", None)):+ return _pandas_to_arrow_ipc(obj)+ buf = io.BytesIO()+ obj.write_ipc(buf)+ return buf.getvalue()def _pt_dump(obj: Any) -> bytes:diff --git a/tests/_runtime/test_primitives.py b/tests/_runtime/test_primitives.pyindex bd41239a87..83cfad002f 100644--- a/tests/_runtime/test_primitives.py+++ b/tests/_runtime/test_primitives.py@@ -24,6 +24,27 @@ class TestDataPrimitiveClassification:assert is_data_primitive(_ClassLikeArray) is False+ @pytest.mark.skipif(+ not DependencyManager.polars.has(), reason="polars required"+ )+ def test_polars_series_numeric_classification(self) -> None:+ import polars as pl++ values = [+ pl.Series("integer", [1, 2]),+ pl.Series("float", [1.0, 2.0]),+ pl.Series("string", ["a", "b"]),+ pl.Series("list", [[1], [2]]),+ pl.Series("object", [object()], dtype=pl.Object),+ ]+ assert {value.name: is_data_primitive(value) for value in values} == {+ "integer": True,+ "float": True,+ "string": False,+ "list": False,+ "object": False,+ }+class TestWrappedFunctionHandling:"""Test handling of wrapped functions (decorators) in is_pure_function."""diff --git a/tests/_save/loaders/test_loader.py b/tests/_save/loaders/test_loader.pyindex e10fae5672..40d6a0b415 100644--- a/tests/_save/loaders/test_loader.py+++ b/tests/_save/loaders/test_loader.py@@ -542,12 +542,19 @@ class TestLazyLoader(ABCTestLoader):@pytest.mark.skipif(not DependencyManager.has("polars"), reason="polars required")- def test_polars_round_trip(self) -> None:+ @pytest.mark.parametrize("pyarrow_available", [True, False])+ def test_polars_round_trip(+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool+ ) -> None:"""polars DataFrames survive save → flush → load via .arrow format."""import polars as pl+ if not pyarrow_available:+ monkeypatch.setattr(+ DependencyManager.pyarrow, "has", lambda: False+ )loader = self.instance()- df = pl.DataFrame({"a": [1, 2], "b": [3.0, 4.0]})+ df = pl.DataFrame({"a": [1, 2], "b": [3.0, 4.0], "c": ["x", "y"]})cache = Cache(defs={"df": df},hash="pl_hash",@@ -569,12 +576,19 @@ class TestLazyLoader(ABCTestLoader):@pytest.mark.skipif(not DependencyManager.has("polars"), reason="polars required")- def test_polars_series_round_trip(self) -> None:+ @pytest.mark.parametrize("pyarrow_available", [True, False])+ def test_polars_series_round_trip(+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool+ ) -> None:"""polars Series survive save → flush → load via .arrow format."""import polars as pl+ if not pyarrow_available:+ monkeypatch.setattr(+ DependencyManager.pyarrow, "has", lambda: False+ )loader = self.instance()- s = pl.Series("vals", [10, 20, 30])+ s = pl.Series("vals", ["a", "b", "c"])cache = Cache(defs={"s": s},hash="pl_series_hash",@@ -594,40 +608,84 @@ class TestLazyLoader(ABCTestLoader):assert isinstance(loaded.defs["s"], pl.Series)assert loaded.defs["s"].to_list() == s.to_list()+ @pytest.mark.skipif(+ not DependencyManager.has("polars"), reason="polars required"+ )+ @pytest.mark.parametrize("stored_version", [None, 5])+ def test_polars_cache_versions_reuse_values(+ self, stored_version: int | None+ ) -> None:+ import polars as pl++ loader = self.instance()+ frame = pl.DataFrame({"value": [1, 2]})+ cache = Cache(+ defs={"frame": frame},+ hash="versioned_polars",+ cache_type="Pure",+ stateful_refs=set(),+ hit=False,+ meta={} if stored_version is None else {"version": stored_version},+ )+ assert loader.save_cache(cache)+ loader.flush()+ restored = loader.cache_attempt({"frame"}, cache.key, set())+ assert restored.hit+ assert restored.defs["frame"].equals(frame)+ assert restored.meta["version"] == (6 if stored_version is None else 5)+@pytest.mark.skipif(not DependencyManager.has("pandas"), reason="pandas required")- def test_pandas_round_trip(self) -> None:- """pandas DataFrames survive save → flush → load via .arrow format."""+ @pytest.mark.parametrize("pyarrow_available", [True, False])+ def test_pandas_round_trip(+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool+ ) -> None:+ """Persist pandas DataFrames with the available serialization backend."""import pandas as pd+ if pyarrow_available:+ pytest.importorskip("pyarrow")+ else:+ monkeypatch.setattr(+ DependencyManager.pyarrow, "has", lambda: False+ )loader = self.instance()- df = pd.DataFrame({"x": [1, 2], "y": [3.0, 4.0]})+ df = pd.DataFrame({"to_frame": [1, 2], "write_ipc": [3.0, 4.0]})cache = Cache(defs={"df": df},hash="pd_hash",cache_type="Pure",stateful_refs=set(),hit=False,- meta={"version": MARIMO_CACHE_VERSION},+ meta={"version": MARIMO_CACHE_VERSION, "return": df},)assert loader.save_cache(cache)loader.flush()- assert list(Path(self.store.save_path).rglob("*.arrow")), (- "expected .arrow blob, got pickle fallback"- )+ extension = "arrow" if pyarrow_available else "pickle"+ assert list(Path(self.store.save_path).rglob(f"*.{extension}"))loaded = loader.load_cache(key("pd_hash", "Pure"))assert loaded is not Nonepd.testing.assert_frame_equal(loaded.defs["df"], df)+ pd.testing.assert_frame_equal(loaded.meta["return"], df)@pytest.mark.skipif(not DependencyManager.has("pandas"), reason="pandas required")- def test_pandas_series_round_trip(self) -> None:- """pandas Series survive save → flush → load via .arrow format."""+ @pytest.mark.parametrize("pyarrow_available", [True, False])+ def test_pandas_series_round_trip(+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool+ ) -> None:+ """Persist pandas Series with the available serialization backend."""import pandas as pd+ if pyarrow_available:+ pytest.importorskip("pyarrow")+ else:+ monkeypatch.setattr(+ DependencyManager.pyarrow, "has", lambda: False+ )loader = self.instance()s = pd.Series([10, 20, 30], name="vals")cache = Cache(@@ -636,15 +694,15 @@ class TestLazyLoader(ABCTestLoader):cache_type="Pure",stateful_refs=set(),hit=False,- meta={"version": MARIMO_CACHE_VERSION},+ meta={"version": MARIMO_CACHE_VERSION, "return": s},)assert loader.save_cache(cache)loader.flush()- assert list(Path(self.store.save_path).rglob("*.arrow")), (- "expected .arrow blob, got pickle fallback"- )+ extension = "arrow" if pyarrow_available else "pickle"+ assert list(Path(self.store.save_path).rglob(f"*.{extension}"))loaded = loader.load_cache(key("pd_series_hash", "Pure"))assert loaded is not Noneassert isinstance(loaded.defs["s"], pd.Series)pd.testing.assert_series_equal(loaded.defs["s"], s)+ pd.testing.assert_series_equal(loaded.meta["return"], s)diff --git a/tests/_save/test_encode.py b/tests/_save/test_encode.pyindex 362bf4d062..e248a2d1ee 100644--- a/tests/_save/test_encode.py+++ b/tests/_save/test_encode.py@@ -9,7 +9,12 @@ from typing import Anyimport pytestfrom marimo._dependencies.dependencies import DependencyManager-from marimo._save.encode import common_container_to_bytes, deterministic_dumps+from marimo._save.encode import (+ attempt_signed_bytes,+ common_container_to_bytes,+ data_to_buffer,+ deterministic_dumps,+)HAS_PANDAS = DependencyManager.pandas.has()HAS_NUMPY = DependencyManager.numpy.has()@@ -137,3 +142,63 @@ def test_bytearray_does_not_collide_with_equal_bytes() -> None:assert common_container_to_bytes(bytearray(b"abc")) != (common_container_to_bytes(b"abc"))+++ not DependencyManager.polars.has(), reason="polars required"+)+def test_polars_numeric_hash_depends_on_values() -> None:+ import polars as pl++ frame = pl.DataFrame({"x": [1, 2], "y": [3, 4]})+ encoded = data_to_buffer(frame)+ assert encoded == data_to_buffer(frame.rename({"x": "renamed"}))+ assert encoded != data_to_buffer(pl.DataFrame({"x": [1, 2], "y": [3, 5]}))+++ not DependencyManager.polars.has(), reason="polars required"+)[email protected]("as_frame", [False, True])+def test_polars_non_numeric_state_uses_fallback(as_frame: bool) -> None:+ import polars as pl++ series = pl.Series("label", ["a", "b"])+ data = (+ series.to_frame().with_columns(count=pl.Series([1, 2]))+ if as_frame+ else series+ )+ value = {"data": data}+ assert attempt_signed_bytes(value, "state") is value+++ not DependencyManager.polars.has(), reason="polars required"+)[email protected]("as_frame", [False, True])+def test_polars_non_numeric_pickle_fallback_preserves_values(+ as_frame: bool,+) -> None:+ import polars as pl++ series = pl.Series("label", ["a", "b"])+ value = series.to_frame() if as_frame else series+ assert pickle.loads(deterministic_dumps(value, "sha256")).equals(value)+++ not DependencyManager.polars.has(), reason="polars required"+)[email protected]("as_frame", [False, True])[email protected]("in_container", [False, True])+def test_polars_object_hash_uses_fallback(+ as_frame: bool, in_container: bool+) -> None:+ import polars as pl++ series = pl.Series("objects", [object()], dtype=pl.Object)+ value: Any = series.to_frame() if as_frame else series+ value = {"data": value} if in_container else value++ assert attempt_signed_bytes(value, "state") is value