Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(cache): fall back to pickle when a typed serializer rejects a value (#11368)

Dylan Madisettica983d9b4e86
diff --git a/marimo/_save/loaders/lazy.py b/marimo/_save/loaders/lazy.py
index 6b128843a0..155876c19f 100644
--- a/marimo/_save/loaders/lazy.py
+++ b/marimo/_save/loaders/lazy.py
@@ -1481,7 +1481,7 @@ class LazyLoader(BasePersistenceLoader):
def _put_or_mark_unserializable(
key: str,
value: Any,
- serialize: Callable[[Any], bytes],
+ loader: str,
items: list[Item],
var_name: str = "",
) -> bool:
@@ -1492,8 +1492,25 @@ class LazyLoader(BasePersistenceLoader):
marked unserializable. Records the blob digest for the signed
manifest when the loader is signing.
"""
+ # Serializer defaults can fail, so fall back to pickle in those cases.
+ serializers: dict[str, Callable[[Any], bytes]] = {
+ loader: BLOB_SERIALIZERS.get(loader, pickle.dumps),
+ "pickle": pickle.dumps,
+ }
try:
- data = serialize(value)
+ for suffix, serialize in serializers.items():
+ try:
+ data = serialize(value)
+ except Exception as e:
+ error = e
+ continue
+ if suffix != loader:
+ key = Path(key).with_suffix(f".{suffix}").as_posix()
+ for item in items:
+ item.reference = key
+ break
+ else:
+ raise error
store.put(key, data)
if signing:
blob_hashes[key] = _sha256hex(data)
@@ -1519,13 +1536,10 @@ class LazyLoader(BasePersistenceLoader):
"""Serialize and write all blobs + manifest in background."""
try:
if return_ref:
- serialize = BLOB_SERIALIZERS.get(
- return_loader, pickle.dumps
- )
_put_or_mark_unserializable(
return_ref,
return_value,
- serialize,
+ return_loader,
[return_item],
"return",
)
@@ -1534,7 +1548,7 @@ class LazyLoader(BasePersistenceLoader):
ui_ok = _put_or_mark_unserializable(
ui_key,
ui_vars,
- pickle.dumps,
+ "pickle",
[defs_dict[v] for v in ui_defs_list],
"ui",
)
@@ -1553,12 +1567,11 @@ class LazyLoader(BasePersistenceLoader):
"Failed to clear stale ui blob %s", ui_key
)
for loader, vars_dict in format_vars.items():
- serialize = BLOB_SERIALIZERS.get(loader, pickle.dumps)
for var, obj in vars_dict.items():
_put_or_mark_unserializable(
(path / f"{var}.{loader}").as_posix(),
obj,
- serialize,
+ loader,
[defs_dict[var]],
var,
)
diff --git a/tests/_save/loaders/test_loader.py b/tests/_save/loaders/test_loader.py
index fac492106e..787708a63a 100644
--- a/tests/_save/loaders/test_loader.py
+++ b/tests/_save/loaders/test_loader.py
@@ -708,3 +708,86 @@ class TestLazyLoader(ABCTestLoader):
assert isinstance(loaded.defs["s"], pd.Series)
pd.testing.assert_series_equal(loaded.defs["s"], s)
pd.testing.assert_series_equal(loaded.meta["return"], s)
+
+ @pytest.mark.skipif(
+ not DependencyManager.has("pandas"), reason="pandas required"
+ )
+ def test_pandas_mixed_dtype_falls_back_to_pickle(self) -> None:
+ """An object column that mixes ints and strings fails in Arrow, so
+ the frame is pickled and a cache hit restores it intact."""
+ import pandas as pd
+
+ pytest.importorskip("pyarrow")
+ loader = self.instance()
+ df = pd.DataFrame({"mixed": [1, "two"]})
+ cache = Cache(
+ defs={"df": df},
+ hash="pd_mixed_hash",
+ cache_type="Pure",
+ stateful_refs=set(),
+ hit=False,
+ meta={"version": MARIMO_CACHE_VERSION, "return": df},
+ )
+ assert loader.save_cache(cache)
+ loader.flush()
+
+ blob_names = sorted(
+ p.name
+ for p in Path(self.store.save_path).rglob("*")
+ if p.is_file() and p.suffix != ".jsonl"
+ )
+ assert blob_names == ["df.pickle", "return.pickle"], blob_names
+
+ cache_path = loader.build_path(key("pd_mixed_hash", "Pure"))
+ decoded = msgspec.json.decode(
+ self.store.get(str(cache_path)), type=CacheSchema
+ )
+ assert decoded.defs["df"].unserializable_type is None
+ assert decoded.meta.return_value.unserializable_type is None
+
+ loaded = loader.load_cache(key("pd_mixed_hash", "Pure"))
+ assert loaded is not None
+ pd.testing.assert_frame_equal(loaded.defs["df"], df)
+ pd.testing.assert_frame_equal(loaded.meta["return"], df)
+
+ def test_typed_serializer_failure_falls_back_to_pickle(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """When the typed serializer chosen for a value raises, the value is
+ pickled under a `.pickle` key and restores intact."""
+ import marimo._save.loaders.lazy as lazy_module
+ from marimo._save.stubs.lazy_stub import BLOB_SERIALIZERS
+
+ def _reject(obj: object) -> bytes:
+ raise TypeError(f"cannot encode {type(obj).__name__}")
+
+ monkeypatch.setitem(BLOB_SERIALIZERS, "npy", _reject)
+ monkeypatch.setattr(
+ lazy_module,
+ "maybe_update_lazy_stub",
+ lambda value: "npy" if isinstance(value, list) else "pickle",
+ )
+ loader = self.instance()
+ value = [1, "two"]
+ cache = Cache(
+ defs={"v": value},
+ hash="typed_fallback_hash",
+ cache_type="Pure",
+ stateful_refs=set(),
+ hit=False,
+ meta={"version": MARIMO_CACHE_VERSION, "return": value},
+ )
+ assert loader.save_cache(cache)
+ loader.flush()
+
+ blob_names = sorted(
+ p.name
+ for p in Path(self.store.save_path).rglob("*")
+ if p.is_file() and p.suffix != ".jsonl"
+ )
+ assert blob_names == ["return.pickle", "v.pickle"], blob_names
+
+ loaded = loader.load_cache(key("typed_fallback_hash", "Pure"))
+ assert loaded is not None
+ assert loaded.defs["v"] == value
+ assert loaded.meta["return"] == value