A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: populate the cell cache when a notebook runs as a script (#11369)
Dylan Madisettia43b750476ae
diff --git a/marimo/_runtime/app/script_runner.py b/marimo/_runtime/app/script_runner.pyindex 45c97a13c2..79c8cfe8a5 100644--- a/marimo/_runtime/app/script_runner.py+++ b/marimo/_runtime/app/script_runner.py@@ -12,11 +12,13 @@ from marimo._runtime.context.types import (get_context,initialize_context,runtime_context_installed,+ safe_get_context,teardown_context,)from marimo._runtime.control_flow import MarimoStopErrorfrom marimo._runtime.exceptions import (MarimoMissingRefError,+ MarimoRescheduleError,MarimoRuntimeException,unwrap_user_exception,)@@ -39,6 +41,8 @@ if TYPE_CHECKING:from marimo._ast.app import InternalAppfrom marimo._runtime.context.types import RuntimeContext+ from marimo._runtime.executor.lifecycles.cached import CachedLifecycle+ from marimo._save.cache import CacheStateclass AppScriptRunner:@@ -54,6 +58,15 @@ class AppScriptRunner:self.filename = filenameself._docstring = extract_docstring_from_header(app._app._header)self._glbls = glbls if glbls else {}+ # `glbls` also carries the setup cell's own values; any other name+ # is a `defs` override.+ # The setup cell's id carries a prefix when the notebook is imported+ # from inside another notebook, so it is looked up by its real id.+ setup_cell = self.app.graph.cells.get(+ self.app.cell_manager.setup_cell_id+ )+ setup_defs = setup_cell.defs if setup_cell is not None else set()+ self._has_overrides = bool(self._glbls.keys() - setup_defs)# Setup cell cannot be overridden, and it's possible that some# variables are not defined, so ignore it.@@ -73,6 +86,7 @@ class AppScriptRunner:self._scheduler = SequentialScheduler(cells_to_run, self.app.graph)self._evaluator = Evaluator(executor=resolve_executor(), lifecycles=[])+ self._cache: CachedLifecycle | None = None# _run_synchronous and _run_asynchronous are deliberate near-twins:# the only difference is the await on the cell step. Keeping them@@ -94,7 +108,7 @@ class AppScriptRunner:glbls.update(self._glbls)outputs: dict[CellId_t, Any] = {}- while self._scheduler.pending():+ while self._scheduler_pending(glbls):cid = self._scheduler.pop_cell()if self._scheduler.cancelled(cid):continue@@ -128,7 +142,7 @@ class AppScriptRunner:glbls.update(self._glbls)outputs: dict[CellId_t, Any] = {}- while self._scheduler.pending():+ while self._scheduler_pending(glbls):cid = self._scheduler.pop_cell()if self._scheduler.cancelled(cid):continue@@ -172,6 +186,25 @@ class AppScriptRunner:if installed_context:teardown_context()+ def _scheduler_pending(self, glbls: dict[str, Any]) -> bool:+ """Whether a cell is queued, after requeuing stub definers."""+ if self._scheduler.pending():+ return True+ if self._cache is None:+ return False+ # In a notebook, the cell that reads a stub reruns its producer; the+ # caller of `App.run` reads the definitions only after the run.+ cells = self._cache.invalidate_stub_definers(glbls)+ if not cells:+ return False+ # A rerun producer may define new values, so every cell downstream+ # of it is requeued too. Those whose inputs did not change hit the+ # cache again.+ self._scheduler.requeue_for_rerun(+ dataflow.transitive_closure(self.app.graph, cells)+ )+ return self._scheduler.pending()+def _handle_run_result(self,cid: CellId_t,@@ -183,6 +216,15 @@ class AppScriptRunner:if exc is None:outputs[cid] = result.outputreturn+ if isinstance(exc, MarimoRescheduleError):+ # Stale-cache signal from a lifecycle; `cells_to_rerun` includes+ # this cell, so requeuing (producers first) retries it live.+ # Descendants of the rerun producers are requeued as well, since+ # the values they read may change.+ self._scheduler.requeue_for_rerun(+ dataflow.transitive_closure(self.app.graph, exc.cells_to_rerun)+ )+ returnif not isinstance(exc, BaseException):# Defensive check descendants, since all exceptions are expected to# be wrapper..@@ -222,6 +264,8 @@ class AppScriptRunner:breakinstalled_script_context = False+ cache: CacheState | None = None+ outer_hash_memo: dict[str, bytes] = {}try:if not runtime_context_installed():# script context is ephemeral, only installed while the app is@@ -231,6 +275,13 @@ class AppScriptRunner:)installed_script_context = True+ # Content hashes are memoized by variable name. A run started+ # from inside a running notebook reuses that notebook's context,+ # so without a memo of its own it would read the hashes of+ # another notebook's variables of the same name.+ cache = get_context().cache+ outer_hash_memo, cache.hash_memo = cache.hash_memo, {}+# formatters aren't automatically registered when running as a# scriptfrom marimo._output.formatters.formatters import (@@ -241,6 +292,27 @@ class AppScriptRunner:if not FORMATTERS.is_empty():register_formatters()+ # NB. attached here rather than __init__: config resolution+ # needs the runtime context, and the notebook's inline+ # [tool.marimo.runtime] table merges by filename.+ runtime_config = get_context().marimo_config.get("runtime", {})+ # A cache key can hash the code of the cells that define a ref+ # instead of its value, so with overrides a run would restore,+ # and save, results under the keys of the cells they replace.+ if (+ runtime_config.get("cache_cells", False)+ and not self._has_overrides+ ):+ from marimo._runtime.executor.lifecycles.cached import (+ CachedLifecycle,+ )++ self._cache = CachedLifecycle(+ self.app.graph,+ pin_modules=bool(runtime_config.get("pin_modules", True)),+ )+ self._evaluator.lifecycles = [self._cache]+post_execute_hooks: list[Callable[[], Any]] = []if DependencyManager.matplotlib.has():from marimo._output.mpl import close_figures@@ -265,5 +337,17 @@ class AppScriptRunner:except MarimoRuntimeException as e:raise e.__cause__ from None # type: ignorefinally:+ from marimo._save.loaders import flush_active_caches+ from marimo._save.manifest import flush_dirty_manifests++ # NB. joins background blob writes; without this, cache entries+ # can drop at interpreter exit.+ flush_active_caches()+ # A script has no kernel run to finish, so nothing else writes+ # the manifest of the entries this run recorded.+ if (context := safe_get_context()) is not None:+ flush_dirty_manifests(context)+ if cache is not None:+ cache.hash_memo = outer_hash_memoif installed_script_context:teardown_context()diff --git a/marimo/_runtime/executor/lifecycles/cached.py b/marimo/_runtime/executor/lifecycles/cached.pyindex a802fcc532..967c0a4b35 100644--- a/marimo/_runtime/executor/lifecycles/cached.py+++ b/marimo/_runtime/executor/lifecycles/cached.py@@ -176,7 +176,11 @@ class CachedLifecycle:# TODO(dmadisetti): Attempt to restore UI elements as well.# Currently the UIElement class has UIDs that are session-specific.ui_live = self._restored_ui_defs(attempt, glbls)- if not ui_live:+ # A cell with no definitions and an unpicklable last expression+ # restores a placeholder as its output. The caller would get the+ # placeholder back, so the cell runs live instead.+ stub_output = _is_unhashable_stub(attempt.meta.get("return"))+ if not ui_live and not stub_output:self._restored_keys[cell_id] = str(self._loader.build_path(attempt.key))@@ -188,8 +192,8 @@ class CachedLifecycle:)LOGGER.debug(- "Cache hit for %s defines UI elements; running "- "live to register them with this session",+ "Cache hit for %s defines UI elements or a placeholder "+ "output, so it runs live",cell_id,)self._attempts.pop(cell_id, None)@@ -251,6 +255,22 @@ class CachedLifecycle:self._loader, attempt.key, cell_id, self._graph, saved=True)+ def invalidate_stub_definers(self, glbls: MutableGlobals) -> set[CellId_t]:+ """Invalidate and return the cells that still define a stub.++ A cell already invalidated is left out, so one that keeps restoring+ a stub is not rerun forever.+ """+ stale = {+ cell_id+ for cell_id, cell in self._graph.cells.items()+ if cell_id not in self._invalidated+ and self._defines_stub(cell, glbls)+ }+ for cell_id in stale:+ self._invalidate(cell_id)+ return stale+@staticmethoddef _defines_stub(cell: CellImpl, glbls: MutableGlobals) -> bool:"""True if any name the cell defines is still an UnhashableStub."""diff --git a/tests/_save/test_script_cell_cache.py b/tests/_save/test_script_cell_cache.pynew file mode 100644index 0000000000..7f7201dd54--- /dev/null+++ b/tests/_save/test_script_cell_cache.py@@ -0,0 +1,476 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import subprocess+import sys+from typing import TYPE_CHECKING++import pytest++from marimo._dependencies.dependencies import DependencyManager+from marimo._save.manifest import CacheManifest, is_manifest_name++if TYPE_CHECKING:+ from collections.abc import Callable+ from pathlib import Path++NOTEBOOK_SOURCE = """# /// script+# [tool.marimo.runtime]+# cache_cells = true+# ///+import marimo++app = marimo.App()+++def _():+ value = sum(range(1000))+ print(f"computed value={value}")+ return (value,)+++def _(value):+ doubled = value * 2+ print(f"downstream doubled={doubled}")+ return (doubled,)+++if __name__ == "__main__":+ app.run()+"""+++def test_script_run_populates_and_restores_cell_cache(tmp_path: Path) -> None:+ """`python notebook.py` warms the `cache_cells` cell cache.++ The first run executes cell bodies and saves them to+ `__marimo__/cache`. The second run restores from cache, so the cell+ bodies (and their prints) do not run.+ """+ nb_path = tmp_path / "notebook.py"+ nb_path.write_text(NOTEBOOK_SOURCE, encoding="utf-8")++ first = subprocess.run(+ [sys.executable, str(nb_path)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert first.returncode == 0, first.stderr+ assert "computed value=499500" in first.stdout+ assert "downstream doubled=999000" in first.stdout++ cache_dir = tmp_path / "__marimo__" / "cache"+ assert cache_dir.is_dir(), "script run must create the cell cache"+ assert any(cache_dir.rglob("*")), "script run must write cache entries"++ second = subprocess.run(+ [sys.executable, str(nb_path)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert second.returncode == 0, second.stderr+ assert "computed value" not in second.stdout, (+ "cached cell body ran again instead of restoring"+ )+++def test_script_run_records_its_cache_entries_in_a_manifest(+ tmp_path: Path,+) -> None:+ """A script run writes the manifest that `marimo cache` commands read."""+ nb_path = tmp_path / "notebook.py"+ nb_path.write_text(NOTEBOOK_SOURCE, encoding="utf-8")++ result = subprocess.run(+ [sys.executable, str(nb_path)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert result.returncode == 0, result.stderr++ cache_dir = tmp_path / "__marimo__" / "cache"+ (manifest_path,) = [+ path for path in cache_dir.iterdir() if is_manifest_name(path.name)+ ]+ manifest = CacheManifest.from_bytes(manifest_path.read_bytes())+ entries = manifest.entries()+ assert len(entries) == 2, "one entry per cell"+ assert {block for block, _ in entries} == {"cell_cache"}+++def test_script_run_without_cache_cells_stays_cold(tmp_path: Path) -> None:+ """Without `cache_cells`, script runs execute every cell, every time."""+ source = NOTEBOOK_SOURCE.replace("# cache_cells = true\n", "")+ nb_path = tmp_path / "notebook.py"+ nb_path.write_text(source, encoding="utf-8")++ for _ in range(2):+ result = subprocess.run(+ [sys.executable, str(nb_path)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert result.returncode == 0, result.stderr+ assert "computed value=499500" in result.stdout++ assert not (tmp_path / "__marimo__" / "cache").exists()+++# `x` is not a primitive, so the consumer's cache key hashes the code of+# the cell that defines `x` rather than its value.+OVERRIDE_NOTEBOOK_SOURCE = """# /// script+# [tool.marimo.runtime]+# cache_cells = true+# ///+import marimo++app = marimo.App()++with app.setup:+ from types import SimpleNamespace+++def _():+ x = SimpleNamespace(value=1)+ return (x,)+++def _(x):+ y = x.value+ print(f"computed y={y}")+ return (y,)+"""++OVERRIDE_DRIVER_SOURCE = """import sys+from types import SimpleNamespace++from notebook import app++overrides = {"x": SimpleNamespace(value=2)} if sys.argv[1:] else None+_, defs = app.run(defs=overrides)+print(f"returned y={defs['y']}")+"""+++def test_app_run_with_overrides_bypasses_cell_cache(tmp_path: Path) -> None:+ """`App.run(defs=...)` neither restores nor saves cached cells."""+ (tmp_path / "notebook.py").write_text(+ OVERRIDE_NOTEBOOK_SOURCE, encoding="utf-8"+ )+ driver = tmp_path / "driver.py"+ driver.write_text(OVERRIDE_DRIVER_SOURCE, encoding="utf-8")++ def run(*args: str) -> str:+ result = subprocess.run(+ [sys.executable, str(driver), *args],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert result.returncode == 0, result.stderr+ return result.stdout++ assert "returned y=1" in run()+ assert "returned y=2" in run("--override")+ warm = run()+ assert "returned y=1" in warm+ assert "computed y" not in warm, "the first run's entry should restore"+++ARRAY_NOTEBOOK_SOURCE = """import marimo++app = marimo.App()+++def _():+ import numpy as np++ return (np,)+++def _(np):+ x = np.array([1, 2])+ return (x,)+++def _(x):+ y = int(x.sum())+ return (y,)+"""++# Both notebooks run inside this notebook's cell, so they reuse its runtime+# context, including the `cache_cells` setting.+HOST_NOTEBOOK_SOURCE = """# /// script+# [tool.marimo.runtime]+# cache_cells = true+# ///+import marimo++app = marimo.App()+++def _():+ import ones+ import tens++ _, ones_defs = ones.app.run()+ _, tens_defs = tens.app.run()+ print(f"returned y={ones_defs['y']},{tens_defs['y']}")+ return+++if __name__ == "__main__":+ app.run()+"""+++ not DependencyManager.numpy.has(), reason="numpy not installed"+)+def test_app_runs_in_one_context_hash_their_own_values(tmp_path: Path) -> None:+ """Each `App.run()` hashes its own values, even in a shared context.++ Content hashes are memoized by variable name. A second run that read+ the first run's memo would hash its `x` as `[1, 2]` and restore the+ first run's `y`.+ """+ (tmp_path / "ones.py").write_text(ARRAY_NOTEBOOK_SOURCE, encoding="utf-8")+ (tmp_path / "tens.py").write_text(+ ARRAY_NOTEBOOK_SOURCE.replace("[1, 2]", "[10, 20]"), encoding="utf-8"+ )+ host = tmp_path / "host.py"+ host.write_text(HOST_NOTEBOOK_SOURCE, encoding="utf-8")++ result = subprocess.run(+ [sys.executable, str(host)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert result.returncode == 0, result.stderr+ assert "returned y=3,30" in result.stdout+++UNPICKLABLE_NOTEBOOK_SOURCE = """# /// script+# [tool.marimo.runtime]+# cache_cells = true+# ///+import marimo++app = marimo.App()+++def _():+ increment = lambda value: value + 1+ print("defined increment")+ return (increment,)+"""++UNPICKLABLE_DRIVER_SOURCE = """from notebook import app++_, defs = app.run()+print(f"returned {defs['increment'](1)}")+"""+++def test_app_run_reruns_cells_whose_definitions_did_not_pickle(+ tmp_path: Path,+) -> None:+ """`App.run()` never hands back a cache placeholder.++ A lambda cannot be pickled, so its cache entry holds a placeholder. In+ a notebook the first cell to read the placeholder reruns its producer;+ the caller of `App.run()` reads the definitions only after the run, so+ the script runner reruns the producer itself.+ """+ (tmp_path / "notebook.py").write_text(+ UNPICKLABLE_NOTEBOOK_SOURCE, encoding="utf-8"+ )+ driver = tmp_path / "driver.py"+ driver.write_text(UNPICKLABLE_DRIVER_SOURCE, encoding="utf-8")++ for attempt in ("cold", "warm"):+ result = subprocess.run(+ [sys.executable, str(driver)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert result.returncode == 0, (attempt, result.stderr)+ assert "returned 2" in result.stdout, attempt+ assert "defined increment" in result.stdout, (+ f"the {attempt} run must execute the cell"+ )+++# `n` changes on every run and `f` cannot be pickled.+STUB_DESCENDANT_NOTEBOOK_SOURCE = """# /// script+# [tool.marimo.runtime]+# cache_cells = true+# ///+import marimo++app = marimo.App()+++def _():+ import time++ n = time.time()+ f = lambda: 1+ return f, n+++def _(n):+ y = 2 * n+ return (y,)+"""++# Reads only `f`, so its cache key hashes the code of the defining cell.+STUB_READER_CELL_SOURCE = """++def _(f):+ h = f()+ print("computed h")+ return (h,)+"""++STUB_DESCENDANT_DRIVER_SOURCE = """from notebook import app++_, defs = app.run()+print(f"consistent={defs['y'] == 2 * defs['n']}")+"""+++def _run_driver(tmp_path: Path, notebook: str) -> Callable[[], str]:+ (tmp_path / "notebook.py").write_text(notebook, encoding="utf-8")+ driver = tmp_path / "driver.py"+ driver.write_text(STUB_DESCENDANT_DRIVER_SOURCE, encoding="utf-8")++ def run() -> str:+ result = subprocess.run(+ [sys.executable, str(driver)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert result.returncode == 0, result.stderr+ return result.stdout++ return run+++def test_app_run_reruns_descendants_of_rerun_producers(+ tmp_path: Path,+) -> None:+ """A producer rerun after the run reruns the cells that read it.++ On a warm run the first cell restores a stale `n` next to a placeholder+ for `f`, and the second cell restores the `y` that matched the stale+ `n`. Rerunning the first cell alone would hand back `y != 2 * n`.+ """+ run = _run_driver(tmp_path, STUB_DESCENDANT_NOTEBOOK_SOURCE)+ assert "consistent=True" in run()+ assert "consistent=True" in run(), (+ "the derived value must come from the rerun producer"+ )+++def test_app_run_reruns_descendants_when_a_reader_reschedules(+ tmp_path: Path,+) -> None:+ """A producer rerun mid-run reruns the cells that read it.++ The third cell reads the placeholder for `f` and reschedules the first+ cell before the run ends. The second cell must follow the rerun, while+ the third cell's key did not change, so it still restores.+ """+ run = _run_driver(+ tmp_path, STUB_DESCENDANT_NOTEBOOK_SOURCE + STUB_READER_CELL_SOURCE+ )+ cold = run()+ assert "consistent=True" in cold+ assert "computed h" in cold+ warm = run()+ assert "consistent=True" in warm, (+ "the derived value must come from the rerun producer"+ )+ assert "computed h" not in warm, (+ "a descendant whose inputs did not change should still restore"+ )+++OUTPUT_STUB_NOTEBOOK_SOURCE = """# /// script+# [tool.marimo.runtime]+# cache_cells = true+# ///+import marimo++app = marimo.App()+++def _():+ lambda value: value + 1+ return+"""++OUTPUT_STUB_DRIVER_SOURCE = """from notebook import app++outputs, _ = app.run()+output = outputs[0]+print(f"output kind={type(output).__name__}")+print(f"called {output(1)}")+"""+++def test_app_run_never_returns_a_placeholder_output(tmp_path: Path) -> None:+ """A def-less cell whose last expression did not pickle runs live.++ The cell defines nothing, so no definition reveals the placeholder. The+ restored output itself must be recognized and the cell rerun.+ """+ (tmp_path / "notebook.py").write_text(+ OUTPUT_STUB_NOTEBOOK_SOURCE, encoding="utf-8"+ )+ driver = tmp_path / "driver.py"+ driver.write_text(OUTPUT_STUB_DRIVER_SOURCE, encoding="utf-8")++ for attempt in ("cold", "warm"):+ result = subprocess.run(+ [sys.executable, str(driver)],+ capture_output=True,+ text=True,+ cwd=tmp_path,+ timeout=120,+ )+ assert "output kind=function" in result.stdout, (+ attempt,+ result.stdout,+ )+ assert result.returncode == 0, (attempt, result.stderr)+ assert "called 2" in result.stdout, attempt