Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: serialize Polars cache values with native IPC (#10775)

Péter Gyarmatib0f23f44e277
diff --git a/marimo/_runtime/primitives.py b/marimo/_runtime/primitives.py
index 7bd25ce9db..5f8e911044 100644
--- a/marimo/_runtime/primitives.py
+++ b/marimo/_runtime/primitives.py
@@ -76,6 +76,10 @@ def is_data_primitive(value: Any) -> bool:
if is_instance_by_name(value, "torch.Tensor"):
return str(value.device) == "cpu"
+ # Series follow the same numeric-schema policy as dataframes.
+ if is_instance_by_name(value, "polars.series.series.Series"):
+ return bool(value.dtype.is_numeric())
+
# If a numpy like array, ensure that it's not an object array.
if hasattr(value, "dtype"):
return not (
diff --git a/marimo/_save/cache.py b/marimo/_save/cache.py
index 767e5318f5..76eb1ab291 100644
--- a/marimo/_save/cache.py
+++ b/marimo/_save/cache.py
@@ -46,7 +46,7 @@ if TYPE_CHECKING:
from marimo._save.stores import Store
# NB. Increment on cache breaking changes.
-MARIMO_CACHE_VERSION: int = 5
+MARIMO_CACHE_VERSION: int = 6
CacheType = Literal[
"ContextExecutionPath",
diff --git a/marimo/_save/encode.py b/marimo/_save/encode.py
index 9953270d91..ebda84d4a6 100644
--- a/marimo/_save/encode.py
+++ b/marimo/_save/encode.py
@@ -139,7 +139,11 @@ def common_container_to_bytes(value: Any) -> bytes:
if is_primitive(value):
return primitive_to_bytes(value)
- return data_to_buffer(value)
+ if is_data_primitive(value):
+ return data_to_buffer(value)
+ raise TypeError(
+ f"Expected numeric array data, got {type(value).__name__}"
+ )
return recurse_container(value)
diff --git a/marimo/_save/loaders/lazy.py b/marimo/_save/loaders/lazy.py
index 5f623defc6..40376f02df 100644
--- a/marimo/_save/loaders/lazy.py
+++ b/marimo/_save/loaders/lazy.py
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any
import msgspec
from marimo import _loggers
+from marimo._dependencies.dependencies import DependencyManager
from marimo._runtime.context import safe_get_context
from marimo._runtime.primitives import (
is_data_primitive,
@@ -218,6 +219,13 @@ def maybe_update_lazy_stub(value: Any) -> str:
# MRO not that expensive, type hashable for functools lookup.
result = mro_lookup(value_type, LAZY_STUB_LOOKUP)
loader = result[1] if result else "pickle"
+ if (
+ loader == "arrow"
+ and result is not None
+ and result[0].startswith("pandas.")
+ and not DependencyManager.pyarrow.has()
+ ):
+ return "pickle"
return loader
diff --git a/marimo/_save/stubs/lazy_stub.py b/marimo/_save/stubs/lazy_stub.py
index e6eb71abf5..afce872b85 100644
--- a/marimo/_save/stubs/lazy_stub.py
+++ b/marimo/_save/stubs/lazy_stub.py
@@ -184,12 +184,14 @@ def _arrow_load(data: bytes, type_hint: str | None = None) -> Any:
# type_hint is the fq class name written by to_item() at save time.
# Using it (rather than schema metadata inspection) is explicit and
# version-stable across pyarrow/polars/pandas releases.
- DependencyManager.pyarrow.require("to load cached Arrow IPC blobs.")
- import pyarrow as pa
-
- reader = pa.ipc.open_file(io.BytesIO(data))
- table = reader.read_all()
if type_hint and type_hint.startswith("pandas."):
+ DependencyManager.pyarrow.require(
+ "to load cached pandas Arrow IPC blobs."
+ )
+ import pyarrow as pa
+
+ reader = pa.ipc.open_file(io.BytesIO(data))
+ table = reader.read_all()
df = table.to_pandas()
if type_hint in ("pandas.Series", "pandas.core.series.Series"):
# Stored as a single-column DataFrame; recover as a Series.
@@ -197,12 +199,10 @@ def _arrow_load(data: bytes, type_hint: str | None = None) -> Any:
return df
import polars as pl
- result = pl.from_arrow(table)
+ result = pl.read_ipc(io.BytesIO(data))
if type_hint == "polars.series.series.Series":
# Stored as a single-column DataFrame; recover as a Series.
- if isinstance(result, pl.DataFrame):
- return result.to_series(0)
- return result
+ return result.to_series(0)
return result
@@ -265,27 +265,15 @@ def _pandas_to_arrow_ipc(df: Any) -> bytes:
def _arrow_dump(obj: Any) -> bytes:
- # Duck-type dispatch:
- # polars DataFrame → write_ipc()
- # pandas DataFrame → Arrow IPC via pyarrow.ipc
- # Series (either) → to_frame() first, then the appropriate DataFrame method
- # Fall back to pickle when pyarrow is absent so the cache write never fails.
- if not DependencyManager.pyarrow.has():
- return pickle.dumps(obj)
- if hasattr(obj, "write_ipc"): # polars DataFrame
- buf = io.BytesIO()
- obj.write_ipc(buf)
- return buf.getvalue()
- if hasattr(obj, "to_feather"): # pandas DataFrame
- return _pandas_to_arrow_ipc(obj)
- # Series — promote to single-column DataFrame, then detect library
- frame = obj.to_frame()
- if hasattr(frame, "write_ipc"): # polars Series → polars DataFrame
- buf = io.BytesIO()
- frame.write_ipc(buf)
- return buf.getvalue()
- # pandas Series → pandas DataFrame
- return _pandas_to_arrow_ipc(frame)
+ # Inspect class methods because pandas columns also appear as attributes.
+ if not callable(getattr(type(obj), "write_ipc", None)):
+ if callable(getattr(type(obj), "to_frame", None)):
+ obj = obj.to_frame()
+ if not callable(getattr(type(obj), "write_ipc", None)):
+ return _pandas_to_arrow_ipc(obj)
+ buf = io.BytesIO()
+ obj.write_ipc(buf)
+ return buf.getvalue()
def _pt_dump(obj: Any) -> bytes:
diff --git a/tests/_runtime/test_primitives.py b/tests/_runtime/test_primitives.py
index bd41239a87..83cfad002f 100644
--- a/tests/_runtime/test_primitives.py
+++ b/tests/_runtime/test_primitives.py
@@ -24,6 +24,27 @@ class TestDataPrimitiveClassification:
assert is_data_primitive(_ClassLikeArray) is False
+ @pytest.mark.skipif(
+ not DependencyManager.polars.has(), reason="polars required"
+ )
+ def test_polars_series_numeric_classification(self) -> None:
+ import polars as pl
+
+ values = [
+ pl.Series("integer", [1, 2]),
+ pl.Series("float", [1.0, 2.0]),
+ pl.Series("string", ["a", "b"]),
+ pl.Series("list", [[1], [2]]),
+ pl.Series("object", [object()], dtype=pl.Object),
+ ]
+ assert {value.name: is_data_primitive(value) for value in values} == {
+ "integer": True,
+ "float": True,
+ "string": False,
+ "list": False,
+ "object": False,
+ }
+
class TestWrappedFunctionHandling:
"""Test handling of wrapped functions (decorators) in is_pure_function."""
diff --git a/tests/_save/loaders/test_loader.py b/tests/_save/loaders/test_loader.py
index e10fae5672..40d6a0b415 100644
--- a/tests/_save/loaders/test_loader.py
+++ b/tests/_save/loaders/test_loader.py
@@ -542,12 +542,19 @@ class TestLazyLoader(ABCTestLoader):
@pytest.mark.skipif(
not DependencyManager.has("polars"), reason="polars required"
)
- def test_polars_round_trip(self) -> None:
+ @pytest.mark.parametrize("pyarrow_available", [True, False])
+ def test_polars_round_trip(
+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool
+ ) -> None:
"""polars DataFrames survive save → flush → load via .arrow format."""
import polars as pl
+ if not pyarrow_available:
+ monkeypatch.setattr(
+ DependencyManager.pyarrow, "has", lambda: False
+ )
loader = self.instance()
- df = pl.DataFrame({"a": [1, 2], "b": [3.0, 4.0]})
+ df = pl.DataFrame({"a": [1, 2], "b": [3.0, 4.0], "c": ["x", "y"]})
cache = Cache(
defs={"df": df},
hash="pl_hash",
@@ -569,12 +576,19 @@ class TestLazyLoader(ABCTestLoader):
@pytest.mark.skipif(
not DependencyManager.has("polars"), reason="polars required"
)
- def test_polars_series_round_trip(self) -> None:
+ @pytest.mark.parametrize("pyarrow_available", [True, False])
+ def test_polars_series_round_trip(
+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool
+ ) -> None:
"""polars Series survive save → flush → load via .arrow format."""
import polars as pl
+ if not pyarrow_available:
+ monkeypatch.setattr(
+ DependencyManager.pyarrow, "has", lambda: False
+ )
loader = self.instance()
- s = pl.Series("vals", [10, 20, 30])
+ s = pl.Series("vals", ["a", "b", "c"])
cache = Cache(
defs={"s": s},
hash="pl_series_hash",
@@ -594,40 +608,84 @@ class TestLazyLoader(ABCTestLoader):
assert isinstance(loaded.defs["s"], pl.Series)
assert loaded.defs["s"].to_list() == s.to_list()
+ @pytest.mark.skipif(
+ not DependencyManager.has("polars"), reason="polars required"
+ )
+ @pytest.mark.parametrize("stored_version", [None, 5])
+ def test_polars_cache_versions_reuse_values(
+ self, stored_version: int | None
+ ) -> None:
+ import polars as pl
+
+ loader = self.instance()
+ frame = pl.DataFrame({"value": [1, 2]})
+ cache = Cache(
+ defs={"frame": frame},
+ hash="versioned_polars",
+ cache_type="Pure",
+ stateful_refs=set(),
+ hit=False,
+ meta={} if stored_version is None else {"version": stored_version},
+ )
+ assert loader.save_cache(cache)
+ loader.flush()
+ restored = loader.cache_attempt({"frame"}, cache.key, set())
+ assert restored.hit
+ assert restored.defs["frame"].equals(frame)
+ assert restored.meta["version"] == (6 if stored_version is None else 5)
+
@pytest.mark.skipif(
not DependencyManager.has("pandas"), reason="pandas required"
)
- def test_pandas_round_trip(self) -> None:
- """pandas DataFrames survive save → flush → load via .arrow format."""
+ @pytest.mark.parametrize("pyarrow_available", [True, False])
+ def test_pandas_round_trip(
+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool
+ ) -> None:
+ """Persist pandas DataFrames with the available serialization backend."""
import pandas as pd
+ if pyarrow_available:
+ pytest.importorskip("pyarrow")
+ else:
+ monkeypatch.setattr(
+ DependencyManager.pyarrow, "has", lambda: False
+ )
loader = self.instance()
- df = pd.DataFrame({"x": [1, 2], "y": [3.0, 4.0]})
+ df = pd.DataFrame({"to_frame": [1, 2], "write_ipc": [3.0, 4.0]})
cache = Cache(
defs={"df": df},
hash="pd_hash",
cache_type="Pure",
stateful_refs=set(),
hit=False,
- meta={"version": MARIMO_CACHE_VERSION},
+ meta={"version": MARIMO_CACHE_VERSION, "return": df},
)
assert loader.save_cache(cache)
loader.flush()
- assert list(Path(self.store.save_path).rglob("*.arrow")), (
- "expected .arrow blob, got pickle fallback"
- )
+ extension = "arrow" if pyarrow_available else "pickle"
+ assert list(Path(self.store.save_path).rglob(f"*.{extension}"))
loaded = loader.load_cache(key("pd_hash", "Pure"))
assert loaded is not None
pd.testing.assert_frame_equal(loaded.defs["df"], df)
+ pd.testing.assert_frame_equal(loaded.meta["return"], df)
@pytest.mark.skipif(
not DependencyManager.has("pandas"), reason="pandas required"
)
- def test_pandas_series_round_trip(self) -> None:
- """pandas Series survive save → flush → load via .arrow format."""
+ @pytest.mark.parametrize("pyarrow_available", [True, False])
+ def test_pandas_series_round_trip(
+ self, monkeypatch: pytest.MonkeyPatch, pyarrow_available: bool
+ ) -> None:
+ """Persist pandas Series with the available serialization backend."""
import pandas as pd
+ if pyarrow_available:
+ pytest.importorskip("pyarrow")
+ else:
+ monkeypatch.setattr(
+ DependencyManager.pyarrow, "has", lambda: False
+ )
loader = self.instance()
s = pd.Series([10, 20, 30], name="vals")
cache = Cache(
@@ -636,15 +694,15 @@ class TestLazyLoader(ABCTestLoader):
cache_type="Pure",
stateful_refs=set(),
hit=False,
- meta={"version": MARIMO_CACHE_VERSION},
+ meta={"version": MARIMO_CACHE_VERSION, "return": s},
)
assert loader.save_cache(cache)
loader.flush()
- assert list(Path(self.store.save_path).rglob("*.arrow")), (
- "expected .arrow blob, got pickle fallback"
- )
+ extension = "arrow" if pyarrow_available else "pickle"
+ assert list(Path(self.store.save_path).rglob(f"*.{extension}"))
loaded = loader.load_cache(key("pd_series_hash", "Pure"))
assert loaded is not None
assert isinstance(loaded.defs["s"], pd.Series)
pd.testing.assert_series_equal(loaded.defs["s"], s)
+ pd.testing.assert_series_equal(loaded.meta["return"], s)
diff --git a/tests/_save/test_encode.py b/tests/_save/test_encode.py
index 362bf4d062..e248a2d1ee 100644
--- a/tests/_save/test_encode.py
+++ b/tests/_save/test_encode.py
@@ -9,7 +9,12 @@ from typing import Any
import pytest
from marimo._dependencies.dependencies import DependencyManager
-from marimo._save.encode import common_container_to_bytes, deterministic_dumps
+from marimo._save.encode import (
+ attempt_signed_bytes,
+ common_container_to_bytes,
+ data_to_buffer,
+ deterministic_dumps,
+)
HAS_PANDAS = DependencyManager.pandas.has()
HAS_NUMPY = DependencyManager.numpy.has()
@@ -137,3 +142,63 @@ def test_bytearray_does_not_collide_with_equal_bytes() -> None:
assert common_container_to_bytes(bytearray(b"abc")) != (
common_container_to_bytes(b"abc")
)
+
+
+ not DependencyManager.polars.has(), reason="polars required"
+)
+def test_polars_numeric_hash_depends_on_values() -> None:
+ import polars as pl
+
+ frame = pl.DataFrame({"x": [1, 2], "y": [3, 4]})
+ encoded = data_to_buffer(frame)
+ assert encoded == data_to_buffer(frame.rename({"x": "renamed"}))
+ assert encoded != data_to_buffer(pl.DataFrame({"x": [1, 2], "y": [3, 5]}))
+
+
+ not DependencyManager.polars.has(), reason="polars required"
+)
[email protected]("as_frame", [False, True])
+def test_polars_non_numeric_state_uses_fallback(as_frame: bool) -> None:
+ import polars as pl
+
+ series = pl.Series("label", ["a", "b"])
+ data = (
+ series.to_frame().with_columns(count=pl.Series([1, 2]))
+ if as_frame
+ else series
+ )
+ value = {"data": data}
+ assert attempt_signed_bytes(value, "state") is value
+
+
+ not DependencyManager.polars.has(), reason="polars required"
+)
[email protected]("as_frame", [False, True])
+def test_polars_non_numeric_pickle_fallback_preserves_values(
+ as_frame: bool,
+) -> None:
+ import polars as pl
+
+ series = pl.Series("label", ["a", "b"])
+ value = series.to_frame() if as_frame else series
+ assert pickle.loads(deterministic_dumps(value, "sha256")).equals(value)
+
+
+ not DependencyManager.polars.has(), reason="polars required"
+)
[email protected]("as_frame", [False, True])
[email protected]("in_container", [False, True])
+def test_polars_object_hash_uses_fallback(
+ as_frame: bool, in_container: bool
+) -> None:
+ import polars as pl
+
+ series = pl.Series("objects", [object()], dtype=pl.Object)
+ value: Any = series.to_frame() if as_frame else series
+ value = {"data": value} if in_container else value
+
+ assert attempt_signed_bytes(value, "state") is value