Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: populate the cell cache when a notebook runs as a script (#11369)

Dylan Madisettia43b750476ae
diff --git a/marimo/_runtime/app/script_runner.py b/marimo/_runtime/app/script_runner.py
index 45c97a13c2..79c8cfe8a5 100644
--- a/marimo/_runtime/app/script_runner.py
+++ b/marimo/_runtime/app/script_runner.py
@@ -12,11 +12,13 @@ from marimo._runtime.context.types import (
get_context,
initialize_context,
runtime_context_installed,
+ safe_get_context,
teardown_context,
)
from marimo._runtime.control_flow import MarimoStopError
from marimo._runtime.exceptions import (
MarimoMissingRefError,
+ MarimoRescheduleError,
MarimoRuntimeException,
unwrap_user_exception,
)
@@ -39,6 +41,8 @@ if TYPE_CHECKING:
from marimo._ast.app import InternalApp
from marimo._runtime.context.types import RuntimeContext
+ from marimo._runtime.executor.lifecycles.cached import CachedLifecycle
+ from marimo._save.cache import CacheState
class AppScriptRunner:
@@ -54,6 +58,15 @@ class AppScriptRunner:
self.filename = filename
self._docstring = extract_docstring_from_header(app._app._header)
self._glbls = glbls if glbls else {}
+ # `glbls` also carries the setup cell's own values; any other name
+ # is a `defs` override.
+ # The setup cell's id carries a prefix when the notebook is imported
+ # from inside another notebook, so it is looked up by its real id.
+ setup_cell = self.app.graph.cells.get(
+ self.app.cell_manager.setup_cell_id
+ )
+ setup_defs = setup_cell.defs if setup_cell is not None else set()
+ self._has_overrides = bool(self._glbls.keys() - setup_defs)
# Setup cell cannot be overridden, and it's possible that some
# variables are not defined, so ignore it.
@@ -73,6 +86,7 @@ class AppScriptRunner:
self._scheduler = SequentialScheduler(cells_to_run, self.app.graph)
self._evaluator = Evaluator(executor=resolve_executor(), lifecycles=[])
+ self._cache: CachedLifecycle | None = None
# _run_synchronous and _run_asynchronous are deliberate near-twins:
# the only difference is the await on the cell step. Keeping them
@@ -94,7 +108,7 @@ class AppScriptRunner:
glbls.update(self._glbls)
outputs: dict[CellId_t, Any] = {}
- while self._scheduler.pending():
+ while self._scheduler_pending(glbls):
cid = self._scheduler.pop_cell()
if self._scheduler.cancelled(cid):
continue
@@ -128,7 +142,7 @@ class AppScriptRunner:
glbls.update(self._glbls)
outputs: dict[CellId_t, Any] = {}
- while self._scheduler.pending():
+ while self._scheduler_pending(glbls):
cid = self._scheduler.pop_cell()
if self._scheduler.cancelled(cid):
continue
@@ -172,6 +186,25 @@ class AppScriptRunner:
if installed_context:
teardown_context()
+ def _scheduler_pending(self, glbls: dict[str, Any]) -> bool:
+ """Whether a cell is queued, after requeuing stub definers."""
+ if self._scheduler.pending():
+ return True
+ if self._cache is None:
+ return False
+ # In a notebook, the cell that reads a stub reruns its producer; the
+ # caller of `App.run` reads the definitions only after the run.
+ cells = self._cache.invalidate_stub_definers(glbls)
+ if not cells:
+ return False
+ # A rerun producer may define new values, so every cell downstream
+ # of it is requeued too. Those whose inputs did not change hit the
+ # cache again.
+ self._scheduler.requeue_for_rerun(
+ dataflow.transitive_closure(self.app.graph, cells)
+ )
+ return self._scheduler.pending()
+
def _handle_run_result(
self,
cid: CellId_t,
@@ -183,6 +216,15 @@ class AppScriptRunner:
if exc is None:
outputs[cid] = result.output
return
+ if isinstance(exc, MarimoRescheduleError):
+ # Stale-cache signal from a lifecycle; `cells_to_rerun` includes
+ # this cell, so requeuing (producers first) retries it live.
+ # Descendants of the rerun producers are requeued as well, since
+ # the values they read may change.
+ self._scheduler.requeue_for_rerun(
+ dataflow.transitive_closure(self.app.graph, exc.cells_to_rerun)
+ )
+ return
if not isinstance(exc, BaseException):
# Defensive check descendants, since all exceptions are expected to
# be wrapper..
@@ -222,6 +264,8 @@ class AppScriptRunner:
break
installed_script_context = False
+ cache: CacheState | None = None
+ outer_hash_memo: dict[str, bytes] = {}
try:
if not runtime_context_installed():
# script context is ephemeral, only installed while the app is
@@ -231,6 +275,13 @@ class AppScriptRunner:
)
installed_script_context = True
+ # Content hashes are memoized by variable name. A run started
+ # from inside a running notebook reuses that notebook's context,
+ # so without a memo of its own it would read the hashes of
+ # another notebook's variables of the same name.
+ cache = get_context().cache
+ outer_hash_memo, cache.hash_memo = cache.hash_memo, {}
+
# formatters aren't automatically registered when running as a
# script
from marimo._output.formatters.formatters import (
@@ -241,6 +292,27 @@ class AppScriptRunner:
if not FORMATTERS.is_empty():
register_formatters()
+ # NB. attached here rather than __init__: config resolution
+ # needs the runtime context, and the notebook's inline
+ # [tool.marimo.runtime] table merges by filename.
+ runtime_config = get_context().marimo_config.get("runtime", {})
+ # A cache key can hash the code of the cells that define a ref
+ # instead of its value, so with overrides a run would restore,
+ # and save, results under the keys of the cells they replace.
+ if (
+ runtime_config.get("cache_cells", False)
+ and not self._has_overrides
+ ):
+ from marimo._runtime.executor.lifecycles.cached import (
+ CachedLifecycle,
+ )
+
+ self._cache = CachedLifecycle(
+ self.app.graph,
+ pin_modules=bool(runtime_config.get("pin_modules", True)),
+ )
+ self._evaluator.lifecycles = [self._cache]
+
post_execute_hooks: list[Callable[[], Any]] = []
if DependencyManager.matplotlib.has():
from marimo._output.mpl import close_figures
@@ -265,5 +337,17 @@ class AppScriptRunner:
except MarimoRuntimeException as e:
raise e.__cause__ from None # type: ignore
finally:
+ from marimo._save.loaders import flush_active_caches
+ from marimo._save.manifest import flush_dirty_manifests
+
+ # NB. joins background blob writes; without this, cache entries
+ # can drop at interpreter exit.
+ flush_active_caches()
+ # A script has no kernel run to finish, so nothing else writes
+ # the manifest of the entries this run recorded.
+ if (context := safe_get_context()) is not None:
+ flush_dirty_manifests(context)
+ if cache is not None:
+ cache.hash_memo = outer_hash_memo
if installed_script_context:
teardown_context()
diff --git a/marimo/_runtime/executor/lifecycles/cached.py b/marimo/_runtime/executor/lifecycles/cached.py
index a802fcc532..967c0a4b35 100644
--- a/marimo/_runtime/executor/lifecycles/cached.py
+++ b/marimo/_runtime/executor/lifecycles/cached.py
@@ -176,7 +176,11 @@ class CachedLifecycle:
# TODO(dmadisetti): Attempt to restore UI elements as well.
# Currently the UIElement class has UIDs that are session-specific.
ui_live = self._restored_ui_defs(attempt, glbls)
- if not ui_live:
+ # A cell with no definitions and an unpicklable last expression
+ # restores a placeholder as its output. The caller would get the
+ # placeholder back, so the cell runs live instead.
+ stub_output = _is_unhashable_stub(attempt.meta.get("return"))
+ if not ui_live and not stub_output:
self._restored_keys[cell_id] = str(
self._loader.build_path(attempt.key)
)
@@ -188,8 +192,8 @@ class CachedLifecycle:
)
LOGGER.debug(
- "Cache hit for %s defines UI elements; running "
- "live to register them with this session",
+ "Cache hit for %s defines UI elements or a placeholder "
+ "output, so it runs live",
cell_id,
)
self._attempts.pop(cell_id, None)
@@ -251,6 +255,22 @@ class CachedLifecycle:
self._loader, attempt.key, cell_id, self._graph, saved=True
)
+ def invalidate_stub_definers(self, glbls: MutableGlobals) -> set[CellId_t]:
+ """Invalidate and return the cells that still define a stub.
+
+ A cell already invalidated is left out, so one that keeps restoring
+ a stub is not rerun forever.
+ """
+ stale = {
+ cell_id
+ for cell_id, cell in self._graph.cells.items()
+ if cell_id not in self._invalidated
+ and self._defines_stub(cell, glbls)
+ }
+ for cell_id in stale:
+ self._invalidate(cell_id)
+ return stale
+
@staticmethod
def _defines_stub(cell: CellImpl, glbls: MutableGlobals) -> bool:
"""True if any name the cell defines is still an UnhashableStub."""
diff --git a/tests/_save/test_script_cell_cache.py b/tests/_save/test_script_cell_cache.py
new file mode 100644
index 0000000000..7f7201dd54
--- /dev/null
+++ b/tests/_save/test_script_cell_cache.py
@@ -0,0 +1,476 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import subprocess
+import sys
+from typing import TYPE_CHECKING
+
+import pytest
+
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._save.manifest import CacheManifest, is_manifest_name
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+ from pathlib import Path
+
+NOTEBOOK_SOURCE = """# /// script
+# [tool.marimo.runtime]
+# cache_cells = true
+# ///
+import marimo
+
+app = marimo.App()
+
+
+def _():
+ value = sum(range(1000))
+ print(f"computed value={value}")
+ return (value,)
+
+
+def _(value):
+ doubled = value * 2
+ print(f"downstream doubled={doubled}")
+ return (doubled,)
+
+
+if __name__ == "__main__":
+ app.run()
+"""
+
+
+def test_script_run_populates_and_restores_cell_cache(tmp_path: Path) -> None:
+ """`python notebook.py` warms the `cache_cells` cell cache.
+
+ The first run executes cell bodies and saves them to
+ `__marimo__/cache`. The second run restores from cache, so the cell
+ bodies (and their prints) do not run.
+ """
+ nb_path = tmp_path / "notebook.py"
+ nb_path.write_text(NOTEBOOK_SOURCE, encoding="utf-8")
+
+ first = subprocess.run(
+ [sys.executable, str(nb_path)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert first.returncode == 0, first.stderr
+ assert "computed value=499500" in first.stdout
+ assert "downstream doubled=999000" in first.stdout
+
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ assert cache_dir.is_dir(), "script run must create the cell cache"
+ assert any(cache_dir.rglob("*")), "script run must write cache entries"
+
+ second = subprocess.run(
+ [sys.executable, str(nb_path)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert second.returncode == 0, second.stderr
+ assert "computed value" not in second.stdout, (
+ "cached cell body ran again instead of restoring"
+ )
+
+
+def test_script_run_records_its_cache_entries_in_a_manifest(
+ tmp_path: Path,
+) -> None:
+ """A script run writes the manifest that `marimo cache` commands read."""
+ nb_path = tmp_path / "notebook.py"
+ nb_path.write_text(NOTEBOOK_SOURCE, encoding="utf-8")
+
+ result = subprocess.run(
+ [sys.executable, str(nb_path)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert result.returncode == 0, result.stderr
+
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ (manifest_path,) = [
+ path for path in cache_dir.iterdir() if is_manifest_name(path.name)
+ ]
+ manifest = CacheManifest.from_bytes(manifest_path.read_bytes())
+ entries = manifest.entries()
+ assert len(entries) == 2, "one entry per cell"
+ assert {block for block, _ in entries} == {"cell_cache"}
+
+
+def test_script_run_without_cache_cells_stays_cold(tmp_path: Path) -> None:
+ """Without `cache_cells`, script runs execute every cell, every time."""
+ source = NOTEBOOK_SOURCE.replace("# cache_cells = true\n", "")
+ nb_path = tmp_path / "notebook.py"
+ nb_path.write_text(source, encoding="utf-8")
+
+ for _ in range(2):
+ result = subprocess.run(
+ [sys.executable, str(nb_path)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert result.returncode == 0, result.stderr
+ assert "computed value=499500" in result.stdout
+
+ assert not (tmp_path / "__marimo__" / "cache").exists()
+
+
+# `x` is not a primitive, so the consumer's cache key hashes the code of
+# the cell that defines `x` rather than its value.
+OVERRIDE_NOTEBOOK_SOURCE = """# /// script
+# [tool.marimo.runtime]
+# cache_cells = true
+# ///
+import marimo
+
+app = marimo.App()
+
+with app.setup:
+ from types import SimpleNamespace
+
+
+def _():
+ x = SimpleNamespace(value=1)
+ return (x,)
+
+
+def _(x):
+ y = x.value
+ print(f"computed y={y}")
+ return (y,)
+"""
+
+OVERRIDE_DRIVER_SOURCE = """import sys
+from types import SimpleNamespace
+
+from notebook import app
+
+overrides = {"x": SimpleNamespace(value=2)} if sys.argv[1:] else None
+_, defs = app.run(defs=overrides)
+print(f"returned y={defs['y']}")
+"""
+
+
+def test_app_run_with_overrides_bypasses_cell_cache(tmp_path: Path) -> None:
+ """`App.run(defs=...)` neither restores nor saves cached cells."""
+ (tmp_path / "notebook.py").write_text(
+ OVERRIDE_NOTEBOOK_SOURCE, encoding="utf-8"
+ )
+ driver = tmp_path / "driver.py"
+ driver.write_text(OVERRIDE_DRIVER_SOURCE, encoding="utf-8")
+
+ def run(*args: str) -> str:
+ result = subprocess.run(
+ [sys.executable, str(driver), *args],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert result.returncode == 0, result.stderr
+ return result.stdout
+
+ assert "returned y=1" in run()
+ assert "returned y=2" in run("--override")
+ warm = run()
+ assert "returned y=1" in warm
+ assert "computed y" not in warm, "the first run's entry should restore"
+
+
+ARRAY_NOTEBOOK_SOURCE = """import marimo
+
+app = marimo.App()
+
+
+def _():
+ import numpy as np
+
+ return (np,)
+
+
+def _(np):
+ x = np.array([1, 2])
+ return (x,)
+
+
+def _(x):
+ y = int(x.sum())
+ return (y,)
+"""
+
+# Both notebooks run inside this notebook's cell, so they reuse its runtime
+# context, including the `cache_cells` setting.
+HOST_NOTEBOOK_SOURCE = """# /// script
+# [tool.marimo.runtime]
+# cache_cells = true
+# ///
+import marimo
+
+app = marimo.App()
+
+
+def _():
+ import ones
+ import tens
+
+ _, ones_defs = ones.app.run()
+ _, tens_defs = tens.app.run()
+ print(f"returned y={ones_defs['y']},{tens_defs['y']}")
+ return
+
+
+if __name__ == "__main__":
+ app.run()
+"""
+
+
+ not DependencyManager.numpy.has(), reason="numpy not installed"
+)
+def test_app_runs_in_one_context_hash_their_own_values(tmp_path: Path) -> None:
+ """Each `App.run()` hashes its own values, even in a shared context.
+
+ Content hashes are memoized by variable name. A second run that read
+ the first run's memo would hash its `x` as `[1, 2]` and restore the
+ first run's `y`.
+ """
+ (tmp_path / "ones.py").write_text(ARRAY_NOTEBOOK_SOURCE, encoding="utf-8")
+ (tmp_path / "tens.py").write_text(
+ ARRAY_NOTEBOOK_SOURCE.replace("[1, 2]", "[10, 20]"), encoding="utf-8"
+ )
+ host = tmp_path / "host.py"
+ host.write_text(HOST_NOTEBOOK_SOURCE, encoding="utf-8")
+
+ result = subprocess.run(
+ [sys.executable, str(host)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert result.returncode == 0, result.stderr
+ assert "returned y=3,30" in result.stdout
+
+
+UNPICKLABLE_NOTEBOOK_SOURCE = """# /// script
+# [tool.marimo.runtime]
+# cache_cells = true
+# ///
+import marimo
+
+app = marimo.App()
+
+
+def _():
+ increment = lambda value: value + 1
+ print("defined increment")
+ return (increment,)
+"""
+
+UNPICKLABLE_DRIVER_SOURCE = """from notebook import app
+
+_, defs = app.run()
+print(f"returned {defs['increment'](1)}")
+"""
+
+
+def test_app_run_reruns_cells_whose_definitions_did_not_pickle(
+ tmp_path: Path,
+) -> None:
+ """`App.run()` never hands back a cache placeholder.
+
+ A lambda cannot be pickled, so its cache entry holds a placeholder. In
+ a notebook the first cell to read the placeholder reruns its producer;
+ the caller of `App.run()` reads the definitions only after the run, so
+ the script runner reruns the producer itself.
+ """
+ (tmp_path / "notebook.py").write_text(
+ UNPICKLABLE_NOTEBOOK_SOURCE, encoding="utf-8"
+ )
+ driver = tmp_path / "driver.py"
+ driver.write_text(UNPICKLABLE_DRIVER_SOURCE, encoding="utf-8")
+
+ for attempt in ("cold", "warm"):
+ result = subprocess.run(
+ [sys.executable, str(driver)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert result.returncode == 0, (attempt, result.stderr)
+ assert "returned 2" in result.stdout, attempt
+ assert "defined increment" in result.stdout, (
+ f"the {attempt} run must execute the cell"
+ )
+
+
+# `n` changes on every run and `f` cannot be pickled.
+STUB_DESCENDANT_NOTEBOOK_SOURCE = """# /// script
+# [tool.marimo.runtime]
+# cache_cells = true
+# ///
+import marimo
+
+app = marimo.App()
+
+
+def _():
+ import time
+
+ n = time.time()
+ f = lambda: 1
+ return f, n
+
+
+def _(n):
+ y = 2 * n
+ return (y,)
+"""
+
+# Reads only `f`, so its cache key hashes the code of the defining cell.
+STUB_READER_CELL_SOURCE = """
+
+def _(f):
+ h = f()
+ print("computed h")
+ return (h,)
+"""
+
+STUB_DESCENDANT_DRIVER_SOURCE = """from notebook import app
+
+_, defs = app.run()
+print(f"consistent={defs['y'] == 2 * defs['n']}")
+"""
+
+
+def _run_driver(tmp_path: Path, notebook: str) -> Callable[[], str]:
+ (tmp_path / "notebook.py").write_text(notebook, encoding="utf-8")
+ driver = tmp_path / "driver.py"
+ driver.write_text(STUB_DESCENDANT_DRIVER_SOURCE, encoding="utf-8")
+
+ def run() -> str:
+ result = subprocess.run(
+ [sys.executable, str(driver)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert result.returncode == 0, result.stderr
+ return result.stdout
+
+ return run
+
+
+def test_app_run_reruns_descendants_of_rerun_producers(
+ tmp_path: Path,
+) -> None:
+ """A producer rerun after the run reruns the cells that read it.
+
+ On a warm run the first cell restores a stale `n` next to a placeholder
+ for `f`, and the second cell restores the `y` that matched the stale
+ `n`. Rerunning the first cell alone would hand back `y != 2 * n`.
+ """
+ run = _run_driver(tmp_path, STUB_DESCENDANT_NOTEBOOK_SOURCE)
+ assert "consistent=True" in run()
+ assert "consistent=True" in run(), (
+ "the derived value must come from the rerun producer"
+ )
+
+
+def test_app_run_reruns_descendants_when_a_reader_reschedules(
+ tmp_path: Path,
+) -> None:
+ """A producer rerun mid-run reruns the cells that read it.
+
+ The third cell reads the placeholder for `f` and reschedules the first
+ cell before the run ends. The second cell must follow the rerun, while
+ the third cell's key did not change, so it still restores.
+ """
+ run = _run_driver(
+ tmp_path, STUB_DESCENDANT_NOTEBOOK_SOURCE + STUB_READER_CELL_SOURCE
+ )
+ cold = run()
+ assert "consistent=True" in cold
+ assert "computed h" in cold
+ warm = run()
+ assert "consistent=True" in warm, (
+ "the derived value must come from the rerun producer"
+ )
+ assert "computed h" not in warm, (
+ "a descendant whose inputs did not change should still restore"
+ )
+
+
+OUTPUT_STUB_NOTEBOOK_SOURCE = """# /// script
+# [tool.marimo.runtime]
+# cache_cells = true
+# ///
+import marimo
+
+app = marimo.App()
+
+
+def _():
+ lambda value: value + 1
+ return
+"""
+
+OUTPUT_STUB_DRIVER_SOURCE = """from notebook import app
+
+outputs, _ = app.run()
+output = outputs[0]
+print(f"output kind={type(output).__name__}")
+print(f"called {output(1)}")
+"""
+
+
+def test_app_run_never_returns_a_placeholder_output(tmp_path: Path) -> None:
+ """A def-less cell whose last expression did not pickle runs live.
+
+ The cell defines nothing, so no definition reveals the placeholder. The
+ restored output itself must be recognized and the cell rerun.
+ """
+ (tmp_path / "notebook.py").write_text(
+ OUTPUT_STUB_NOTEBOOK_SOURCE, encoding="utf-8"
+ )
+ driver = tmp_path / "driver.py"
+ driver.write_text(OUTPUT_STUB_DRIVER_SOURCE, encoding="utf-8")
+
+ for attempt in ("cold", "warm"):
+ result = subprocess.run(
+ [sys.executable, str(driver)],
+ capture_output=True,
+ text=True,
+ cwd=tmp_path,
+ timeout=120,
+ )
+ assert "output kind=function" in result.stdout, (
+ attempt,
+ result.stdout,
+ )
+ assert result.returncode == 0, (attempt, result.stderr)
+ assert "called 2" in result.stdout, attempt