A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(cache): fall back to pickle when a typed serializer rejects a value (#11368)
Dylan Madisettica983d9b4e86
diff --git a/marimo/_save/loaders/lazy.py b/marimo/_save/loaders/lazy.pyindex 6b128843a0..155876c19f 100644--- a/marimo/_save/loaders/lazy.py+++ b/marimo/_save/loaders/lazy.py@@ -1481,7 +1481,7 @@ class LazyLoader(BasePersistenceLoader):def _put_or_mark_unserializable(key: str,value: Any,- serialize: Callable[[Any], bytes],+ loader: str,items: list[Item],var_name: str = "",) -> bool:@@ -1492,8 +1492,25 @@ class LazyLoader(BasePersistenceLoader):marked unserializable. Records the blob digest for the signedmanifest when the loader is signing."""+ # Serializer defaults can fail, so fall back to pickle in those cases.+ serializers: dict[str, Callable[[Any], bytes]] = {+ loader: BLOB_SERIALIZERS.get(loader, pickle.dumps),+ "pickle": pickle.dumps,+ }try:- data = serialize(value)+ for suffix, serialize in serializers.items():+ try:+ data = serialize(value)+ except Exception as e:+ error = e+ continue+ if suffix != loader:+ key = Path(key).with_suffix(f".{suffix}").as_posix()+ for item in items:+ item.reference = key+ break+ else:+ raise errorstore.put(key, data)if signing:blob_hashes[key] = _sha256hex(data)@@ -1519,13 +1536,10 @@ class LazyLoader(BasePersistenceLoader):"""Serialize and write all blobs + manifest in background."""try:if return_ref:- serialize = BLOB_SERIALIZERS.get(- return_loader, pickle.dumps- )_put_or_mark_unserializable(return_ref,return_value,- serialize,+ return_loader,[return_item],"return",)@@ -1534,7 +1548,7 @@ class LazyLoader(BasePersistenceLoader):ui_ok = _put_or_mark_unserializable(ui_key,ui_vars,- pickle.dumps,+ "pickle",[defs_dict[v] for v in ui_defs_list],"ui",)@@ -1553,12 +1567,11 @@ class LazyLoader(BasePersistenceLoader):"Failed to clear stale ui blob %s", ui_key)for loader, vars_dict in format_vars.items():- serialize = BLOB_SERIALIZERS.get(loader, pickle.dumps)for var, obj in vars_dict.items():_put_or_mark_unserializable((path / f"{var}.{loader}").as_posix(),obj,- serialize,+ loader,[defs_dict[var]],var,)diff --git a/tests/_save/loaders/test_loader.py b/tests/_save/loaders/test_loader.pyindex fac492106e..787708a63a 100644--- a/tests/_save/loaders/test_loader.py+++ b/tests/_save/loaders/test_loader.py@@ -708,3 +708,86 @@ class TestLazyLoader(ABCTestLoader):assert isinstance(loaded.defs["s"], pd.Series)pd.testing.assert_series_equal(loaded.defs["s"], s)pd.testing.assert_series_equal(loaded.meta["return"], s)++ @pytest.mark.skipif(+ not DependencyManager.has("pandas"), reason="pandas required"+ )+ def test_pandas_mixed_dtype_falls_back_to_pickle(self) -> None:+ """An object column that mixes ints and strings fails in Arrow, so+ the frame is pickled and a cache hit restores it intact."""+ import pandas as pd++ pytest.importorskip("pyarrow")+ loader = self.instance()+ df = pd.DataFrame({"mixed": [1, "two"]})+ cache = Cache(+ defs={"df": df},+ hash="pd_mixed_hash",+ cache_type="Pure",+ stateful_refs=set(),+ hit=False,+ meta={"version": MARIMO_CACHE_VERSION, "return": df},+ )+ assert loader.save_cache(cache)+ loader.flush()++ blob_names = sorted(+ p.name+ for p in Path(self.store.save_path).rglob("*")+ if p.is_file() and p.suffix != ".jsonl"+ )+ assert blob_names == ["df.pickle", "return.pickle"], blob_names++ cache_path = loader.build_path(key("pd_mixed_hash", "Pure"))+ decoded = msgspec.json.decode(+ self.store.get(str(cache_path)), type=CacheSchema+ )+ assert decoded.defs["df"].unserializable_type is None+ assert decoded.meta.return_value.unserializable_type is None++ loaded = loader.load_cache(key("pd_mixed_hash", "Pure"))+ assert loaded is not None+ pd.testing.assert_frame_equal(loaded.defs["df"], df)+ pd.testing.assert_frame_equal(loaded.meta["return"], df)++ def test_typed_serializer_failure_falls_back_to_pickle(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """When the typed serializer chosen for a value raises, the value is+ pickled under a `.pickle` key and restores intact."""+ import marimo._save.loaders.lazy as lazy_module+ from marimo._save.stubs.lazy_stub import BLOB_SERIALIZERS++ def _reject(obj: object) -> bytes:+ raise TypeError(f"cannot encode {type(obj).__name__}")++ monkeypatch.setitem(BLOB_SERIALIZERS, "npy", _reject)+ monkeypatch.setattr(+ lazy_module,+ "maybe_update_lazy_stub",+ lambda value: "npy" if isinstance(value, list) else "pickle",+ )+ loader = self.instance()+ value = [1, "two"]+ cache = Cache(+ defs={"v": value},+ hash="typed_fallback_hash",+ cache_type="Pure",+ stateful_refs=set(),+ hit=False,+ meta={"version": MARIMO_CACHE_VERSION, "return": value},+ )+ assert loader.save_cache(cache)+ loader.flush()++ blob_names = sorted(+ p.name+ for p in Path(self.store.save_path).rglob("*")+ if p.is_file() and p.suffix != ".jsonl"+ )+ assert blob_names == ["return.pickle", "v.pickle"], blob_names++ loaded = loader.load_cache(key("typed_fallback_hash", "Pure"))+ assert loaded is not None+ assert loaded.defs["v"] == value+ assert loaded.meta["return"] == value