Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(cache): preserve manifest history during flush (#11495)

Dylan Madisettia021077dd1f2
diff --git a/marimo/_save/loaders/lazy.py b/marimo/_save/loaders/lazy.py
index e964e8e778..5570582345 100644
--- a/marimo/_save/loaders/lazy.py
+++ b/marimo/_save/loaders/lazy.py
@@ -472,6 +472,12 @@ class LazyStore(Store):
self._touched_keys.add(key)
return result
+ def read(self, key: str) -> bytes | None:
+ result = self._inner.read(key)
+ if result is not None:
+ self._touched_keys.add(key)
+ return result
+
def put(self, key: str, value: bytes) -> bool:
self._written_keys.add(key)
return self._inner.put(key, value)
@@ -512,6 +518,12 @@ class WasmLazyStore(LazyStore):
return self._http_get(key)
return None
+ def read(self, key: str) -> bytes | None:
+ result = super().read(key)
+ if result is None and key not in _cache_state().poisoned_keys:
+ return self._http_get(key, strict=True)
+ return result
+
def get_batch(
self, keys: Iterable[str]
) -> Iterator[tuple[str, bytes | None]]:
@@ -545,15 +557,30 @@ class WasmLazyStore(LazyStore):
raise ValueError(f"Invalid cache key: {key}")
return key
- def _http_get(self, key: str) -> bytes | None:
+ def _http_get(self, key: str, *, strict: bool = False) -> bytes | None:
"""Single sync fetch via pyodide_http-patched urllib."""
+ import urllib.error
import urllib.request
url = f"{self._base_url()}/{self._sanitize_key(key)}"
try:
with urllib.request.urlopen(url) as resp:
+ if strict and resp.status not in (200, 404):
+ raise urllib.error.HTTPError(
+ url,
+ resp.status,
+ "Unexpected status",
+ resp.headers,
+ None,
+ )
data = resp.read() if resp.status == 200 else None
+ except urllib.error.HTTPError as e:
+ if strict and e.code != 404:
+ raise
+ return None
except Exception:
+ if strict:
+ raise
return None
if data is not None:
self._inner.put(key, data)
diff --git a/marimo/_save/manifest.py b/marimo/_save/manifest.py
index 505e972b53..12bf9ab307 100644
--- a/marimo/_save/manifest.py
+++ b/marimo/_save/manifest.py
@@ -14,6 +14,8 @@ too. It is pure and kernel-free, so the CLI can read it without a runtime.
from __future__ import annotations
+import contextlib
+import copy
import hashlib
import json
import os
@@ -27,7 +29,8 @@ from marimo import _loggers
from marimo._utils.paths import normalize_path
if TYPE_CHECKING:
- from collections.abc import Collection
+ from collections.abc import Collection, Iterator
+ from contextlib import AbstractContextManager
from marimo._runtime.context.types import RuntimeContext
from marimo._runtime.dataflow.topology import GraphTopology
@@ -188,7 +191,7 @@ def load_manifest(store: Store, key: str) -> CacheManifest | None:
CorruptManifestError: If the stored bytes are not a manifest.
UnsupportedManifestVersionError: If they come from a newer marimo.
"""
- data = store.get(key)
+ data = store.read(key)
if not data:
return None
return CacheManifest.from_bytes(data)
@@ -283,24 +286,73 @@ def flush_dirty_manifests(ctx: RuntimeContext) -> None:
def flush_cache_manifest(ctx: RuntimeContext, store: Store) -> bool:
- """Merge `store`'s records into its on-disk manifest and write it back.
-
- Returns whether a manifest was written. A session that produced no
- persistent entry writes nothing, so reading a cache never creates one.
- """
+ """Merge records into each tier and return whether no retry is needed."""
records = ctx.cache.manifest_records.get(store)
if not records or not ctx.filename:
return False
key = manifest_name(ctx.filename)
+ settled = True
+ for tier in _manifest_stores(store):
+ try:
+ with _manifest_lock(tier, key):
+ if not _merge_records(records, ctx.filename, key, tier):
+ settled = False
+ except Exception:
+ LOGGER.warning(
+ "Could not write cache manifest %s", key, exc_info=True
+ )
+ settled = False
+ return settled
+
+
+def _manifest_stores(store: Store) -> Iterator[Store]:
+ from marimo._save.loaders.lazy import LazyStore, WasmLazyStore
+ from marimo._save.stores.tiered import TieredStore
+
+ if isinstance(store, TieredStore):
+ for tier in store.stores:
+ yield from _manifest_stores(tier)
+ elif isinstance(store, LazyStore) and not isinstance(store, WasmLazyStore):
+ for inner in _manifest_stores(store._inner):
+ if inner is store._inner:
+ yield store
+ continue
+ # Keep the wrapper's shared sets of keys for export.
+ wrapped = copy.copy(store)
+ wrapped._inner = inner
+ yield wrapped
+ else:
+ yield store
+
+
+def _manifest_lock(store: Store, key: str) -> AbstractContextManager[None]:
+ from marimo._save.loaders.lazy import LazyStore
+ from marimo._save.stores.file import FileStore
+ from marimo._utils.file_lock import file_lock
+
+ while isinstance(store, LazyStore):
+ store = store._inner
+ if isinstance(store, FileStore):
+ path = store.save_path / key
+ path.parent.mkdir(parents=True, exist_ok=True)
+ return file_lock(path.with_name(f"{path.name}.lock"), blocking=True)
+ return contextlib.nullcontext()
+
+
+def _merge_records(
+ records: dict[str, dict[str, set[str]]],
+ notebook: str,
+ key: str,
+ store: Store,
+) -> bool:
try:
manifest = load_manifest(store, key) or CacheManifest()
except UnsupportedManifestVersionError as e:
# A rewrite under the older schema drops whatever the newer one
# holds, including the marker that tells a reader to stay away.
LOGGER.warning("Leaving cache manifest %s alone: %s", key, e)
- ctx.cache.manifest_dirty.discard(store)
- return False
+ return True
except ManifestError as e:
# Bytes that parse as no manifest at all record nothing that can be
# acted on, so replacing them costs nothing.
@@ -308,13 +360,9 @@ def flush_cache_manifest(ctx: RuntimeContext, store: Store) -> bool:
manifest = CacheManifest()
manifest.merge(CacheManifest(nodes=records))
- manifest.notebook = _notebook_ref(ctx.filename, store)
+ manifest.notebook = _notebook_ref(notebook, store)
manifest.written_at = _utc_now()
- try:
- written = store.put(key, manifest.to_bytes())
- except Exception as e:
- LOGGER.warning("Could not write cache manifest %s: %s", key, e)
- return False
+ written = store.put(key, manifest.to_bytes())
if not written:
# A store can decline a write by its return value rather than raise.
LOGGER.warning("Could not write cache manifest %s", key)
diff --git a/marimo/_save/stores/file.py b/marimo/_save/stores/file.py
index 9a6b5975a2..6e37522205 100644
--- a/marimo/_save/stores/file.py
+++ b/marimo/_save/stores/file.py
@@ -110,6 +110,12 @@ class FileStore(Store):
return None
return path.read_bytes()
+ def read(self, key: str) -> bytes | None:
+ try:
+ return (self.save_path / key).read_bytes()
+ except FileNotFoundError:
+ return None
+
def put(self, key: str, value: bytes) -> bool:
path = self.save_path / key
path.parent.mkdir(parents=True, exist_ok=True)
diff --git a/marimo/_save/stores/rest.py b/marimo/_save/stores/rest.py
index 5f08232d4f..7965d65461 100644
--- a/marimo/_save/stores/rest.py
+++ b/marimo/_save/stores/rest.py
@@ -54,6 +54,25 @@ class RestStore(Store):
LOGGER.warning(f"GET {url} - Error: {e}")
return None
+ def read(self, key: str) -> bytes | None:
+ url = self._get_url(key)
+ req = urllib.request.Request(url, headers=self.headers)
+ try:
+ with urllib.request.urlopen(req, context=self.context) as response:
+ if response.status == 200:
+ return response.read() # type: ignore[no-any-return]
+ raise urllib.error.HTTPError(
+ url,
+ response.status,
+ "Unexpected status",
+ response.headers,
+ None,
+ )
+ except urllib.error.HTTPError as e:
+ if e.code == 404:
+ return None
+ raise
+
def put(self, key: str, value: bytes) -> bool:
url = self._get_url(key)
req = urllib.request.Request(
diff --git a/marimo/_save/stores/store.py b/marimo/_save/stores/store.py
index 6c40ac5b11..8fb68c042d 100644
--- a/marimo/_save/stores/store.py
+++ b/marimo/_save/stores/store.py
@@ -14,6 +14,13 @@ class Store(ABC):
def get(self, key: str) -> bytes | None:
"""Get the bytes of a cache from the store"""
+ def read(self, key: str) -> bytes | None:
+ """Read bytes and return `None` only for missing keys.
+
+ Stores that suppress read errors in `get` must override this method.
+ """
+ return self.get(key)
+
@abstractmethod
def put(self, key: str, value: bytes) -> bool:
"""Put a cache into the store."""
diff --git a/marimo/_save/stores/tiered.py b/marimo/_save/stores/tiered.py
index 36a20d9e7d..95f3ed11d0 100644
--- a/marimo/_save/stores/tiered.py
+++ b/marimo/_save/stores/tiered.py
@@ -43,6 +43,13 @@ class TieredStore(Store):
return None
+ def read(self, key: str) -> bytes | None:
+ for store in self.stores:
+ value = store.read(key)
+ if value is not None:
+ return value
+ return None
+
def put(self, key: str, value: bytes) -> bool:
"""Put a value in all stores."""
success = False
diff --git a/tests/_save/test_manifest.py b/tests/_save/test_manifest.py
index 228da429a2..9fee09fba4 100644
--- a/tests/_save/test_manifest.py
+++ b/tests/_save/test_manifest.py
@@ -8,22 +8,33 @@ import copy
import json
import os
import shutil
+from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
+from threading import Event
from typing import TYPE_CHECKING, Any
+from unittest.mock import MagicMock, patch
+from urllib.error import HTTPError, URLError
import pytest
from marimo._config.config import DEFAULT_CONFIG
+from marimo._runtime.context import get_context
+from marimo._save.loaders.lazy import LazyStore, WasmLazyStore
from marimo._save.manifest import (
MANIFEST_VERSION,
CacheManifest,
CorruptManifestError,
UnsupportedManifestVersionError,
+ flush_cache_manifest,
+ flush_dirty_manifests,
is_manifest_name,
load_manifest,
manifest_name,
)
+from marimo._save.stores.dict_store import DictStore
from marimo._save.stores.file import FileStore
+from marimo._save.stores.rest import RestStore
+from marimo._save.stores.tiered import TieredStore
from tests._runtime._helpers.factories import default_app_metadata
from tests._runtime._helpers.session import mocked_kernel_session
@@ -826,14 +837,14 @@ class TestRecording:
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Bookkeeping about a cache must never break the run it happens in."""
- original_get = FileStore.get
+ original_read = Path.read_bytes
- def refuse_manifest(self: FileStore, key: str) -> bytes | None:
- if is_manifest_name(key):
- raise PermissionError(f"unreadable: {key}")
- return original_get(self, key)
+ def refuse_manifest(self: Path) -> bytes:
+ if is_manifest_name(self.name):
+ raise PermissionError(f"unreadable: {self.name}")
+ return original_read(self)
- monkeypatch.setattr(FileStore, "get", refuse_manifest)
+ monkeypatch.setattr(Path, "read_bytes", refuse_manifest)
with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
await k.run(
@@ -891,3 +902,261 @@ class TestRecording:
# The entry is on disk. Nothing in the notebook records it.
assert list((cache_dir / "scratch").iterdir())
assert not (cache_dir / manifest_name(notebook)).exists()
+
+
+class TestManifestFlush:
+ @staticmethod
+ @pytest.mark.parametrize("wasm", [False, True])
+ @pytest.mark.parametrize("status", [200, 404, 403, 503])
+ def test_http_manifest_read(wasm: bool, status: int) -> None:
+ store: Store = (
+ WasmLazyStore(DictStore())
+ if wasm
+ else RestStore(
+ base_url="https://example.invalid", api_key="test-key"
+ )
+ )
+ manifest = CacheManifest(nodes={"node": {"block": {"C_old"}}})
+ response = MagicMock()
+ response.__enter__.return_value = response
+ response.status = 200
+ response.read.return_value = manifest.to_bytes()
+ key = "nb-0123456789abcdef.json"
+ with patch("urllib.request.urlopen", return_value=response) as request:
+ if status != 200:
+ request.side_effect = HTTPError(
+ "https://example.invalid", status, "Unavailable", {}, None
+ )
+ if status in (200, 404):
+ loaded = load_manifest(store, key)
+ if status == 404:
+ assert loaded is None
+ else:
+ assert loaded is not None
+ assert loaded.nodes == manifest.nodes
+ if wasm:
+ assert store.export_keys() == [key]
+ else:
+ with pytest.raises(HTTPError):
+ load_manifest(store, key)
+ assert store.get(key) is None
+
+ @staticmethod
+ @pytest.mark.parametrize("wrapped", [False, True])
+ def test_each_tier_retains_its_history(
+ tmp_path: Path, wrapped: bool
+ ) -> None:
+ fast = FileStore(str(tmp_path / "fast"))
+ slow = FileStore(str(tmp_path / "slow"))
+ store: Store = TieredStore([fast, TieredStore([slow])])
+ if wrapped:
+ store = LazyStore(LazyStore(store))
+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):
+ key = manifest_name(notebook)
+ for tier, entry in [(fast, "C_fast"), (slow, "C_slow")]:
+ tier.put(
+ key,
+ CacheManifest(
+ nodes={"node": {"block": {entry}}}
+ ).to_bytes(),
+ )
+ ctx = get_context()
+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}
+ ctx.cache.manifest_dirty.add(store)
+
+ flush_dirty_manifests(ctx)
+
+ for tier, entry in [(fast, "C_fast"), (slow, "C_slow")]:
+ manifest = load_manifest(tier, key)
+ assert manifest is not None
+ assert manifest.nodes == {"node": {"block": {entry, "C_new"}}}
+ assert not ctx.cache.manifest_dirty
+ if wrapped:
+ assert store.export_keys() == [key]
+
+ @staticmethod
+ @pytest.mark.parametrize("failure", [403, 503, "connection", "write"])
+ def test_failed_read_preserves_history_until_retry(
+ tmp_path: Path, failure: int | str
+ ) -> None:
+ remote = RestStore(
+ base_url="https://example.invalid", api_key="test-key"
+ )
+ local = FileStore(str(tmp_path / "local"))
+ store = LazyStore(TieredStore([remote, local]))
+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):
+ key = manifest_name(notebook)
+ history = CacheManifest(
+ nodes={"node": {"block": {"C_remote"}}}
+ ).to_bytes()
+ remote_data = {key: history}
+ local.put(
+ key,
+ CacheManifest(
+ nodes={"node": {"block": {"C_local"}}}
+ ).to_bytes(),
+ )
+ ctx = get_context()
+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}
+ ctx.cache.manifest_dirty.add(store)
+ failing = True
+
+ def urlopen(request: Any, **_kwargs: Any) -> MagicMock:
+ if (
+ failing
+ and failure == "write"
+ and request.get_method() == "PUT"
+ ):
+ raise URLError("Connection failed")
+ if (
+ request.get_method() == "GET"
+ and failing
+ and failure != "write"
+ ):
+ if isinstance(failure, int):
+ raise HTTPError(
+ request.full_url, failure, "Unavailable", {}, None
+ )
+ raise URLError("Connection failed")
+ response = MagicMock()
+ response.status = 200
+ response.__enter__.return_value = response
+ if request.get_method() == "PUT":
+ remote_data[key] = request.data
+ else:
+ response.read.return_value = remote_data[key]
+ return response
+
+ with patch("urllib.request.urlopen", side_effect=urlopen):
+ flush_dirty_manifests(ctx)
+ assert remote_data[key] == history
+ local_manifest = load_manifest(local, key)
+ assert local_manifest is not None
+ assert local_manifest.nodes == {
+ "node": {"block": {"C_local", "C_new"}}
+ }
+ assert store in ctx.cache.manifest_dirty
+
+ failing = False
+ flush_dirty_manifests(ctx)
+
+ assert CacheManifest.from_bytes(remote_data[key]).nodes == {
+ "node": {"block": {"C_remote", "C_new"}}
+ }
+ assert not ctx.cache.manifest_dirty
+
+ @staticmethod
+ def test_newer_tier_is_untouched_while_another_updates(
+ tmp_path: Path,
+ ) -> None:
+ newer = FileStore(str(tmp_path / "newer"))
+ current = FileStore(str(tmp_path / "current"))
+ store = TieredStore([newer, current])
+ payload = json.dumps(
+ {
+ "version": MANIFEST_VERSION + 1,
+ "nodes": {"future": {"block": ["C_old"]}},
+ }
+ ).encode()
+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):
+ key = manifest_name(notebook)
+ newer.put(key, payload)
+ ctx = get_context()
+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}
+ ctx.cache.manifest_dirty.add(store)
+
+ flush_dirty_manifests(ctx)
+
+ assert newer.read(key) == payload
+ manifest = load_manifest(current, key)
+ assert manifest is not None
+ assert manifest.nodes == {"node": {"block": {"C_new"}}}
+ assert not ctx.cache.manifest_dirty
+
+ @staticmethod
+ def test_wasm_checks_all_inner_tiers_before_http(tmp_path: Path) -> None:
+ first, second = DictStore(), DictStore()
+ store = WasmLazyStore(TieredStore([first, second]))
+ with _notebook_kernel(tmp_path, store=store) as (_, notebook, _):
+ key = manifest_name(notebook)
+ second.put(
+ key,
+ CacheManifest(nodes={"node": {"block": {"C_old"}}}).to_bytes(),
+ )
+ ctx = get_context()
+ ctx.cache.manifest_records[store] = {"node": {"block": {"C_new"}}}
+ ctx.cache.manifest_dirty.add(store)
+
+ with patch(
+ "urllib.request.urlopen", side_effect=URLError("Offline")
+ ) as request:
+ flush_dirty_manifests(ctx)
+ request.assert_not_called()
+
+ for tier in (first, second):
+ manifest = load_manifest(tier, key)
+ assert manifest is not None
+ assert manifest.nodes == {
+ "node": {"block": {"C_old", "C_new"}}
+ }
+ assert not ctx.cache.manifest_dirty
+ assert store.export_keys() == [key]
+
+ @staticmethod
+ def test_concurrent_flushes_keep_both_writers(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ path = str(tmp_path / "cache")
+ first = LazyStore(FileStore(path))
+ second = LazyStore(FileStore(path))
+ first_read = Event()
+ second_started = Event()
+ second_read = Event()
+ release = Event()
+ original_read = FileStore.read
+
+ def read(self: FileStore, key: str) -> bytes | None:
+ data = original_read(self, key)
+ if self is first._inner:
+ first_read.set()
+ assert release.wait(5)
+ elif self is second._inner:
+ second_read.set()
+ return data
+
+ with _notebook_kernel(tmp_path) as (_, notebook, _):
+ ctx = get_context()
+ ctx.cache.manifest_records[first] = {
+ "node": {"block": {"C_first"}}
+ }
+ ctx.cache.manifest_records[second] = {
+ "node": {"block": {"C_second"}}
+ }
+ key = manifest_name(notebook)
+ first.put(
+ key,
+ CacheManifest(nodes={"node": {"block": {"C_old"}}}).to_bytes(),
+ )
+ monkeypatch.setattr(FileStore, "read", read)
+
+ def flush_second() -> bool:
+ second_started.set()
+ return flush_cache_manifest(ctx, second)
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ a = pool.submit(flush_cache_manifest, ctx, first)
+ try:
+ assert first_read.wait(5)
+ b = pool.submit(flush_second)
+ assert second_started.wait(5)
+ assert not second_read.wait(0.1)
+ finally:
+ release.set()
+ assert a.result(timeout=5)
+ assert b.result(timeout=5)
+
+ manifest = load_manifest(FileStore(path), key)
+ assert manifest is not None
+ assert manifest.nodes == {
+ "node": {"block": {"C_old", "C_first", "C_second"}}
+ }