A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(cache): finish manifest lifecycle cleanup (#11497)
Dylan Madisetti109879b025c0
diff --git a/marimo/_runtime/app/script_runner.py b/marimo/_runtime/app/script_runner.pyindex 79c8cfe8a5..1c1246c1af 100644--- a/marimo/_runtime/app/script_runner.py+++ b/marimo/_runtime/app/script_runner.py@@ -340,14 +340,13 @@ class AppScriptRunner:from marimo._save.loaders import flush_active_cachesfrom marimo._save.manifest import flush_dirty_manifests- # NB. joins background blob writes; without this, cache entries- # can drop at interpreter exit.- flush_active_caches()- # A script has no kernel run to finish, so nothing else writes- # the manifest of the entries this run recorded.- if (context := safe_get_context()) is not None:- flush_dirty_manifests(context)- if cache is not None:- cache.hash_memo = outer_hash_memo- if installed_script_context:- teardown_context()+ try:+ # Join background blob writes before flushing the manifest.+ flush_active_caches()+ if (context := safe_get_context()) is not None:+ flush_dirty_manifests(context)+ finally:+ if cache is not None:+ cache.hash_memo = outer_hash_memo+ if installed_script_context:+ teardown_context()diff --git a/marimo/_runtime/executor/lifecycles/cached.py b/marimo/_runtime/executor/lifecycles/cached.pyindex 967c0a4b35..15fec3093d 100644--- a/marimo/_runtime/executor/lifecycles/cached.py+++ b/marimo/_runtime/executor/lifecycles/cached.py@@ -252,7 +252,11 @@ class CachedLifecycle:from marimo._save.manifest import record_cache_eventrecord_cache_event(- self._loader, attempt.key, cell_id, self._graph, saved=True+ self._loader,+ attempt.key,+ cell_id,+ self._graph,+ saved=not attempt.hit,)def invalidate_stub_definers(self, glbls: MutableGlobals) -> set[CellId_t]:diff --git a/tests/_ast/test_app.py b/tests/_ast/test_app.pyindex 45cc419ada..cc25876284 100644--- a/tests/_ast/test_app.py+++ b/tests/_ast/test_app.py@@ -710,6 +710,59 @@ class TestApp:# one back onto the calling thread.assert not runtime_context_installed()+ @staticmethod+ @pytest.mark.parametrize("execution", ["sync", "async", "running-loop"])+ @pytest.mark.parametrize("phase", ["writes", "manifest"])+ def test_run_tears_down_context_after_interrupted_flush(+ monkeypatch: pytest.MonkeyPatch, execution: str, phase: str+ ) -> None:+ import asyncio++ from marimo._runtime.context.types import (+ runtime_context_installed,+ teardown_context,+ )+ from marimo._save import loaders, manifest++ app = App()+ if execution == "sync":++ @app.cell+ def _():+ x = 1+ return (x,)++ else:++ @app.cell+ async def _():+ import asyncio++ await asyncio.sleep(0)+ x = 1+ return (x,)++ def interrupt(*_args: Any) -> None:+ raise KeyboardInterrupt("interrupted flush")++ if phase == "writes":+ monkeypatch.setattr(loaders, "flush_active_caches", interrupt)+ else:+ monkeypatch.setattr(manifest, "flush_dirty_manifests", interrupt)++ async def run() -> None:+ app.run()++ try:+ with pytest.raises(KeyboardInterrupt, match="interrupted flush"):+ if execution == "running-loop":+ asyncio.run(run())+ else:+ app.run()+ assert not runtime_context_installed()+ finally:+ teardown_context()+@staticmethodasync def test_run_async_from_running_event_loop_propagates_error() -> (Nonediff --git a/tests/_save/test_manifest.py b/tests/_save/test_manifest.pyindex 9fee09fba4..78f0bcdc78 100644--- a/tests/_save/test_manifest.py+++ b/tests/_save/test_manifest.py@@ -10,7 +10,7 @@ import osimport shutilfrom concurrent.futures import ThreadPoolExecutorfrom pathlib import Path-from threading import Event+from threading import Barrierfrom typing import TYPE_CHECKING, Anyfrom unittest.mock import MagicMock, patchfrom urllib.error import HTTPError, URLError@@ -587,6 +587,35 @@ class TestRecording:# the hit branch can have recorded its entry.assert all(is_manifest_name(key) for key in keys)+ @staticmethod+ async def test_repeated_cell_hit_does_not_rewrite_manifest(+ tmp_path: Path,+ exec_req: ExecReqProvider,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ request = exec_req.get("x = 42")+ with _notebook_kernel(tmp_path, cache_cells=True) as (k, _, _):+ await k.run([request])++ with _recorded_puts(monkeypatch) as keys:+ with _notebook_kernel(tmp_path, cache_cells=True) as (+ k,+ notebook,+ cache_dir,+ ):+ await k.run([request])+ assert not k.stderr.messages+ assert k.globals["x"] == 42+ assert keys == [manifest_name(notebook)]+ recorded = _manifest(notebook, cache_dir)+ keys.clear()++ await k.run([request])+ assert not k.stderr.messages+ assert k.globals["x"] == 42+ assert keys == []+ assert _manifest(notebook, cache_dir) == recorded+@staticmethodasync def test_teardown_writes_what_a_failed_flush_left_pending(tmp_path: Path,@@ -1106,23 +1135,26 @@ class TestManifestFlush:def test_concurrent_flushes_keep_both_writers(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ from marimo._utils.file_lock import file_lock+path = str(tmp_path / "cache")first = LazyStore(FileStore(path))second = LazyStore(FileStore(path))- first_read = Event()- second_started = Event()- second_read = Event()- release = Event()original_read = FileStore.read+ original_put = FileStore.put++ def assert_locked(key: str) -> None:+ with pytest.raises(BlockingIOError):+ with file_lock(Path(path) / f"{key}.lock", blocking=False):+ passdef read(self: FileStore, key: str) -> bytes | None:- data = original_read(self, key)- if self is first._inner:- first_read.set()- assert release.wait(5)- elif self is second._inner:- second_read.set()- return data+ assert_locked(key)+ return original_read(self, key)++ def put(self: FileStore, key: str, value: bytes) -> bool:+ assert_locked(key)+ return original_put(self, key, value)with _notebook_kernel(tmp_path) as (_, notebook, _):ctx = get_context()@@ -1137,23 +1169,23 @@ class TestManifestFlush:key,CacheManifest(nodes={"node": {"block": {"C_old"}}}).to_bytes(),)- monkeypatch.setattr(FileStore, "read", read)-- def flush_second() -> bool:- second_started.set()- return flush_cache_manifest(ctx, second)-- with ThreadPoolExecutor(max_workers=2) as pool:- a = pool.submit(flush_cache_manifest, ctx, first)- try:- assert first_read.wait(5)- b = pool.submit(flush_second)- assert second_started.wait(5)- assert not second_read.wait(0.1)- finally:- release.set()- assert a.result(timeout=5)- assert b.result(timeout=5)+ start = Barrier(2)++ def flush(store: Store) -> bool:+ start.wait(timeout=30)+ return flush_cache_manifest(ctx, store)++ with (+ monkeypatch.context() as patcher,+ ThreadPoolExecutor(max_workers=2) as pool,+ ):+ patcher.setattr(FileStore, "read", read)+ patcher.setattr(FileStore, "put", put)+ writers = [+ pool.submit(flush, store) for store in (first, second)+ ]+ for writer in writers:+ assert writer.result(timeout=30)manifest = load_manifest(FileStore(path), key)assert manifest is not None