A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(cache): preserve manifest history during flush (#11495)
Dylan Madisettia021077dd1f2
diff --git a/marimo/_save/loaders/lazy.py b/marimo/_save/loaders/lazy.pyindex e964e8e778..5570582345 100644--- a/marimo/_save/loaders/lazy.py+++ b/marimo/_save/loaders/lazy.py@@ -472,6 +472,12 @@ class LazyStore(Store):self._touched_keys.add(key)return result+ def read(self, key: str) -> bytes | None:+ result = self._inner.read(key)+ if result is not None:+ self._touched_keys.add(key)+ return result+def put(self, key: str, value: bytes) -> bool:self._written_keys.add(key)return self._inner.put(key, value)@@ -512,6 +518,12 @@ class WasmLazyStore(LazyStore):return self._http_get(key)return None+ def read(self, key: str) -> bytes | None:+ result = super().read(key)+ if result is None and key not in _cache_state().poisoned_keys:+ return self._http_get(key, strict=True)+ return result+def get_batch(self, keys: Iterable[str]) -> Iterator[tuple[str, bytes | None]]:@@ -545,15 +557,30 @@ class WasmLazyStore(LazyStore):raise ValueError(f"Invalid cache key: {key}")return key- def _http_get(self, key: str) -> bytes | None:+ def _http_get(self, key: str, *, strict: bool = False) -> bytes | None:"""Single sync fetch via pyodide_http-patched urllib."""+ import urllib.errorimport urllib.requesturl = f"{self._base_url()}/{self._sanitize_key(key)}"try:with urllib.request.urlopen(url) as resp:+ if strict and resp.status not in (200, 404):+ raise urllib.error.HTTPError(+ url,+ resp.status,+ "Unexpected status",+ resp.headers,+ None,+ )data = resp.read() if resp.status == 200 else None+ except urllib.error.HTTPError as e:+ if strict and e.code != 404:+ raise+ return Noneexcept Exception:+ if strict:+ raisereturn Noneif data is not None:self._inner.put(key, data)diff --git a/marimo/_save/manifest.py b/marimo/_save/manifest.pyindex 505e972b53..12bf9ab307 100644--- a/marimo/_save/manifest.py+++ b/marimo/_save/manifest.py@@ -14,6 +14,8 @@ too. It is pure and kernel-free, so the CLI can read it without a runtime.from __future__ import annotations+import contextlib+import copyimport hashlibimport jsonimport os@@ -27,7 +29,8 @@ from marimo import _loggersfrom marimo._utils.paths import normalize_pathif TYPE_CHECKING:- from collections.abc import Collection+ from collections.abc import Collection, Iterator+ from contextlib import AbstractContextManagerfrom marimo._runtime.context.types import RuntimeContextfrom marimo._runtime.dataflow.topology import GraphTopology@@ -188,7 +191,7 @@ def load_manifest(store: Store, key: str) -> CacheManifest | None:CorruptManifestError: If the stored bytes are not a manifest.UnsupportedManifestVersionError: If they come from a newer marimo."""- data = store.get(key)+ data = store.read(key)if not data:return Nonereturn CacheManifest.from_bytes(data)@@ -283,24 +286,73 @@ def flush_dirty_manifests(ctx: RuntimeContext) -> None:def flush_cache_manifest(ctx: RuntimeContext, store: Store) -> bool:- """Merge `store`'s records into its on-disk manifest and write it back.-- Returns whether a manifest was written. A session that produced no- persistent entry writes nothing, so reading a cache never creates one.- """+ """Merge records into each tier and return whether no retry is needed."""records = ctx.cache.manifest_records.get(store)if not records or not ctx.filename:return Falsekey = manifest_name(ctx.filename)+ settled = True+ for tier in _manifest_stores(store):+ try:+ with _manifest_lock(tier, key):+ if not _merge_records(records, ctx.filename, key, tier):+ settled = False+ except Exception:+ LOGGER.warning(+ "Could not write cache manifest %s", key, exc_info=True+ )+ settled = False+ return settled+++def _manifest_stores(store: Store) -> Iterator[Store]:+ from marimo._save.loaders.lazy import LazyStore, WasmLazyStore+ from marimo._save.stores.tiered import TieredStore++ if isinstance(store, TieredStore):+ for tier in store.stores:+ yield from _manifest_stores(tier)+ elif isinstance(store, LazyStore) and not isinstance(store, WasmLazyStore):+ for inner in _manifest_stores(store._inner):+ if inner is store._inner:+ yield store+ continue+ # Keep the wrapper's shared sets of keys for export.+ wrapped = copy.copy(store)+ wrapped._inner = inner+ yield wrapped+ else:+ yield store+++def _manifest_lock(store: Store, key: str) -> AbstractContextManager[None]:+ from marimo._save.loaders.lazy import LazyStore+ from marimo._save.stores.file import FileStore+ from marimo._utils.file_lock import file_lock++ while isinstance(store, LazyStore):+ store = store._inner+ if isinstance(store, FileStore):+ path = store.save_path / key+ path.parent.mkdir(parents=True, exist_ok=True)+ return file_lock(path.with_name(f"{path.name}.lock"), blocking=True)+ return contextlib.nullcontext()+++def _merge_records(+ records: dict[str, dict[str, set[str]]],+ notebook: str,+ key: str,+ store: Store,+) -> bool:try:manifest = load_manifest(store, key) or CacheManifest()except UnsupportedManifestVersionError as e:# A rewrite under the older schema drops whatever the newer one# holds, including the marker that tells a reader to stay away.LOGGER.warning("Leaving cache manifest %s alone: %s", key, e)- ctx.cache.manifest_dirty.discard(store)- return False+ return Trueexcept ManifestError as e:# Bytes that parse as no manifest at all record nothing that can be# acted on, so replacing them costs nothing.@@ -308,13 +360,9 @@ def flush_cache_manifest(ctx: RuntimeContext, store: Store) -> bool:manifest = CacheManifest()manifest.merge(CacheManifest(nodes=records))- manifest.notebook = _notebook_ref(ctx.filename, store)+ manifest.notebook = _notebook_ref(notebook, store)manifest.written_at = _utc_now()- try:- written = store.put(key, manifest.to_bytes())- except Exception as e:- LOGGER.warning("Could not write cache manifest %s: %s", key, e)- return False+ written = store.put(key, manifest.to_bytes())if not written:# A store can decline a write by its return value rather than raise.LOGGER.warning("Could not write cache manifest %s", key)diff --git a/marimo/_save/stores/file.py b/marimo/_save/stores/file.pyindex 9a6b5975a2..6e37522205 100644--- a/marimo/_save/stores/file.py+++ b/marimo/_save/stores/file.py@@ -110,6 +110,12 @@ class FileStore(Store):return Nonereturn path.read_bytes()+ def read(self, key: str) -> bytes | None:+ try:+ return (self.save_path / key).read_bytes()+ except FileNotFoundError:+ return None+def put(self, key: str, value: bytes) -> bool:path = self.save_path / keypath.parent.mkdir(parents=True, exist_ok=True)diff --git a/marimo/_save/stores/rest.py b/marimo/_save/stores/rest.pyindex 5f08232d4f..7965d65461 100644--- a/marimo/_save/stores/rest.py+++ b/marimo/_save/stores/rest.py@@ -54,6 +54,25 @@ class RestStore(Store):LOGGER.warning(f"GET {url} - Error: {e}")return None+ def read(self, key: str) -> bytes | None:+ url = self._get_url(key)+ req = urllib.request.Request(url, headers=self.headers)+ try:+ with urllib.request.urlopen(req, context=self.context) as response:+ if response.status == 200:+ return response.read() # type: ignore[no-any-return]+ raise urllib.error.HTTPError(+ url,+ response.status,+ "Unexpected status",+ response.headers,+ None,+ )+ except urllib.error.HTTPError as e:+ if e.code == 404:+ return None+ raise+def put(self, key: str, value: bytes) -> bool:url = self._get_url(key)req = urllib.request.Request(diff --git a/marimo/_save/stores/store.py b/marimo/_save/stores/store.pyindex 6c40ac5b11..8fb68c042d 100644--- a/marimo/_save/stores/store.py+++ b/marimo/_save/stores/store.py@@ -14,6 +14,13 @@ class Store(ABC):def get(self, key: str) -> bytes | None:"""Get the bytes of a cache from the store"""+ def read(self, key: str) -> bytes | None:+ """Read bytes and return `None` only for missing keys.++ Stores that suppress read errors in `get` must override this method.+ """+ return self.get(key)+@abstractmethoddef put(self, key: str, value: bytes) -> bool:"""Put a cache into the store."""diff --git a/marimo/_save/stores/tiered.py b/marimo/_save/stores/tiered.pyindex 36a20d9e7d..95f3ed11d0 100644--- a/marimo/_save/stores/tiered.py+++ b/marimo/_save/stores/tiered.py@@ -43,6 +43,13 @@ class TieredStore(Store):return None+ def read(self, key: str) -> bytes | None:+ for store in self.stores:+ value = store.read(key)+ if value is not None:+ return value+ return None+def put(self, key: str, value: bytes) -> bool:"""Put a value in all stores."""success = Falsediff --git a/tests/_save/test_manifest.py b/tests/_save/test_manifest.pyindex 228da429a2..9fee09fba4 100644--- a/tests/_save/test_manifest.py+++ b/tests/_save/test_manifest.py@@ -8,22 +8,33 @@ import copyimport jsonimport osimport shutil+from concurrent.futures import ThreadPoolExecutorfrom pathlib import Path+from threading import Eventfrom typing import TYPE_CHECKING, Any+from unittest.mock import MagicMock, patch+from urllib.error import HTTPError, URLErrorimport pytestfrom marimo._config.config import DEFAULT_CONFIG+from marimo._runtime.context import get_context+from marimo._save.loaders.lazy import LazyStore, WasmLazyStorefrom marimo._save.manifest import (MANIFEST_VERSION,CacheManifest,CorruptManifestError,UnsupportedManifestVersionError,+ flush_cache_manifest,+ flush_dirty_manifests,is_manifest_name,load_manifest,manifest_name,)+from marimo._save.stores.dict_store import DictStorefrom marimo._save.stores.file import FileStore+from marimo._save.stores.rest import RestStore+from marimo._save.stores.tiered import TieredStorefrom tests._runtime._helpers.factories import default_app_metadatafrom tests._runtime._helpers.session import mocked_kernel_session@@ -826,14 +837,14 @@ class TestRecording:monkeypatch: pytest.MonkeyPatch,) -> None:"""Bookkeeping about a cache must never break the run it happens in."""- original_get = FileStore.get+ original_read = Path.read_bytes- def refuse_manifest(self: FileStore, key: str) -> bytes | None:- if is_manifest_name(key):- raise PermissionError(f"unreadable: {key}")- return original_get(self, key)+ def refuse_manifest(self: Path) -> bytes:+ if is_manifest_name(self.name):+ raise PermissionError(f"unreadable: {self.name}")+ return original_read(self)- monkeypatch.setattr(FileStore, "get", refuse_manifest)+ monkeypatch.setattr(Path, "read_bytes", refuse_manifest)with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):await k.run(@@ -891,3 +902,261 @@ class TestRecording:# The entry is on disk. Nothing in the notebook records it.assert list((cache_dir / "scratch").iterdir())assert not (cache_dir / manifest_name(notebook)).exists()+++class TestManifestFlush:+ @staticmethod+ @pytest.mark.parametrize("wasm", [False, True])+ @pytest.mark.parametrize("status", [200, 404, 403, 503])+ def test_http_manifest_read(wasm: bool, status: int) -> None:+ store: Store = (+ WasmLazyStore(DictStore())+ if wasm+ else RestStore(+ base_url="https://example.invalid", api_key="test-key"+ )+ )+ manifest = CacheManifest(nodes={"node": {"block": {"C_old"}}})+ response = MagicMock()+ response.__enter__.return_value = response+ response.status = 200+ response.read.return_value = manifest.to_bytes()+ key = "nb-0123456789abcdef.json"+ with patch("urllib.request.urlopen", return_value=response) as request:+ if status != 200:+ request.side_effect = HTTPError(+ "https://example.invalid", status, "Unavailable", {}, None+ )+ if status in (200, 404):+ loaded = load_manifest(store, key)+ if status == 404:+ assert loaded is None+ else:+ assert loaded is not None+ assert loaded.nodes == manifest.nodes+ if wasm:+ assert store.export_keys() == [key]+ else:+ with pytest.raises(HTTPError):+ load_manifest(store, key)+ assert store.get(key) is None++ @staticmethod+ @pytest.mark.parametrize("wrapped", [False, True])+ def test_each_tier_retains_its_history(+ tmp_path: Path, wrapped: bool+ ) -> None:+ fast = FileStore(str(tmp_path / "fast"))+ slow = FileStore(str(tmp_path / "slow"))+ store: Store = TieredStore([fast, TieredStore([slow])])+ if wrapped:+ store = LazyStore(LazyStore(store))+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):+ key = manifest_name(notebook)+ for tier, entry in [(fast, "C_fast"), (slow, "C_slow")]:+ tier.put(+ key,+ CacheManifest(+ nodes={"node": {"block": {entry}}}+ ).to_bytes(),+ )+ ctx = get_context()+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}+ ctx.cache.manifest_dirty.add(store)++ flush_dirty_manifests(ctx)++ for tier, entry in [(fast, "C_fast"), (slow, "C_slow")]:+ manifest = load_manifest(tier, key)+ assert manifest is not None+ assert manifest.nodes == {"node": {"block": {entry, "C_new"}}}+ assert not ctx.cache.manifest_dirty+ if wrapped:+ assert store.export_keys() == [key]++ @staticmethod+ @pytest.mark.parametrize("failure", [403, 503, "connection", "write"])+ def test_failed_read_preserves_history_until_retry(+ tmp_path: Path, failure: int | str+ ) -> None:+ remote = RestStore(+ base_url="https://example.invalid", api_key="test-key"+ )+ local = FileStore(str(tmp_path / "local"))+ store = LazyStore(TieredStore([remote, local]))+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):+ key = manifest_name(notebook)+ history = CacheManifest(+ nodes={"node": {"block": {"C_remote"}}}+ ).to_bytes()+ remote_data = {key: history}+ local.put(+ key,+ CacheManifest(+ nodes={"node": {"block": {"C_local"}}}+ ).to_bytes(),+ )+ ctx = get_context()+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}+ ctx.cache.manifest_dirty.add(store)+ failing = True++ def urlopen(request: Any, **_kwargs: Any) -> MagicMock:+ if (+ failing+ and failure == "write"+ and request.get_method() == "PUT"+ ):+ raise URLError("Connection failed")+ if (+ request.get_method() == "GET"+ and failing+ and failure != "write"+ ):+ if isinstance(failure, int):+ raise HTTPError(+ request.full_url, failure, "Unavailable", {}, None+ )+ raise URLError("Connection failed")+ response = MagicMock()+ response.status = 200+ response.__enter__.return_value = response+ if request.get_method() == "PUT":+ remote_data[key] = request.data+ else:+ response.read.return_value = remote_data[key]+ return response++ with patch("urllib.request.urlopen", side_effect=urlopen):+ flush_dirty_manifests(ctx)+ assert remote_data[key] == history+ local_manifest = load_manifest(local, key)+ assert local_manifest is not None+ assert local_manifest.nodes == {+ "node": {"block": {"C_local", "C_new"}}+ }+ assert store in ctx.cache.manifest_dirty++ failing = False+ flush_dirty_manifests(ctx)++ assert CacheManifest.from_bytes(remote_data[key]).nodes == {+ "node": {"block": {"C_remote", "C_new"}}+ }+ assert not ctx.cache.manifest_dirty++ @staticmethod+ def test_newer_tier_is_untouched_while_another_updates(+ tmp_path: Path,+ ) -> None:+ newer = FileStore(str(tmp_path / "newer"))+ current = FileStore(str(tmp_path / "current"))+ store = TieredStore([newer, current])+ payload = json.dumps(+ {+ "version": MANIFEST_VERSION + 1,+ "nodes": {"future": {"block": ["C_old"]}},+ }+ ).encode()+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):+ key = manifest_name(notebook)+ newer.put(key, payload)+ ctx = get_context()+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}+ ctx.cache.manifest_dirty.add(store)++ flush_dirty_manifests(ctx)++ assert newer.read(key) == payload+ manifest = load_manifest(current, key)+ assert manifest is not None+ assert manifest.nodes == {"node": {"block": {"C_new"}}}+ assert not ctx.cache.manifest_dirty++ @staticmethod+ def test_wasm_checks_all_inner_tiers_before_http(tmp_path: Path) -> None:+ first, second = DictStore(), DictStore()+ store = WasmLazyStore(TieredStore([first, second]))+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):+ key = manifest_name(notebook)+ second.put(+ key,+ CacheManifest(nodes={"node": {"block": {"C_old"}}}).to_bytes(),+ )+ ctx = get_context()+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}+ ctx.cache.manifest_dirty.add(store)++ with patch(+ "urllib.request.urlopen", side_effect=URLError("Offline")+ ) as request:+ flush_dirty_manifests(ctx)+ request.assert_not_called()++ for tier in (first, second):+ manifest = load_manifest(tier, key)+ assert manifest is not None+ assert manifest.nodes == {+ "node": {"block": {"C_old", "C_new"}}+ }+ assert not ctx.cache.manifest_dirty+ assert store.export_keys() == [key]++ @staticmethod+ def test_concurrent_flushes_keep_both_writers(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ path = str(tmp_path / "cache")+ first = LazyStore(FileStore(path))+ second = LazyStore(FileStore(path))+ first_read = Event()+ second_started = Event()+ second_read = Event()+ release = Event()+ original_read = FileStore.read++ def read(self: FileStore, key: str) -> bytes | None:+ data = original_read(self, key)+ if self is first._inner:+ first_read.set()+ assert release.wait(5)+ elif self is second._inner:+ second_read.set()+ return data++ with _notebook_kernel(tmp_path) as (_, notebook, _):+ ctx = get_context()+ ctx.cache.manifest_records[first] = {+ "node": {"block": {"C_first"}}+ }+ ctx.cache.manifest_records[second] = {+ "node": {"block": {"C_second"}}+ }+ key = manifest_name(notebook)+ first.put(+ key,+ CacheManifest(nodes={"node": {"block": {"C_old"}}}).to_bytes(),+ )+ monkeypatch.setattr(FileStore, "read", read)++ def flush_second() -> bool:+ second_started.set()+ return flush_cache_manifest(ctx, second)++ with ThreadPoolExecutor(max_workers=2) as pool:+ a = pool.submit(flush_cache_manifest, ctx, first)+ try:+ assert first_read.wait(5)+ b = pool.submit(flush_second)+ assert second_started.wait(5)+ assert not second_read.wait(0.1)+ finally:+ release.set()+ assert a.result(timeout=5)+ assert b.result(timeout=5)++ manifest = load_manifest(FileStore(path), key)+ assert manifest is not None+ assert manifest.nodes == {+ "node": {"block": {"C_old", "C_first", "C_second"}}+ }