Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: record cache entries in a per-notebook manifest (#10838)

Dylan Madisettia0335e745858
diff --git a/marimo/_ast/variables.py b/marimo/_ast/variables.py
index 28d04b24c2..5a5c156552 100644
--- a/marimo/_ast/variables.py
+++ b/marimo/_ast/variables.py
@@ -47,7 +47,9 @@ def unmangle_local(
) -> UnmagledLocal:
if not is_mangled_local(name, cell_id):
return UnmagledLocal(name, CellId_t(""))
- private_prefix = r"^_cell_\w+?_"
+ # Cell ids can hold hyphens (a UUID, or a nested app's prefix), which
+ # `\w` excludes.
+ private_prefix = r"^_cell_[\w-]+?_"
if cell_id:
private_prefix = f"^_cell_{cell_id}_"
return UnmagledLocal(
diff --git a/marimo/_runtime/callbacks/cache.py b/marimo/_runtime/callbacks/cache.py
index a272e80500..a120175f61 100644
--- a/marimo/_runtime/callbacks/cache.py
+++ b/marimo/_runtime/callbacks/cache.py
@@ -69,11 +69,21 @@ class CacheCallbacks:
# NB. flush unconditionally (durability); joins background blob writes.
flush_active_caches()
+ self._flush_cache_manifest()
# NB. manifest only when cell caching is on (html-wasm --execute), else
# a normal session would litter cache dirs with manifests.
if self._caching_enabled():
dump_cache_manifests(export_manifest_name(self._notebook_filename))
+ def _flush_cache_manifest(self) -> None:
+ """Persist any cache record that a write during the session missed."""
+ from marimo._runtime.context import safe_get_context
+ from marimo._save.manifest import flush_dirty_manifests
+
+ ctx = safe_get_context()
+ if ctx is not None:
+ flush_dirty_manifests(ctx)
+
async def clear_cache(self, request: ClearCacheCommand) -> None:
del request
from marimo._save.cache import CacheContext
diff --git a/marimo/_runtime/executor/lifecycles/cached.py b/marimo/_runtime/executor/lifecycles/cached.py
index b02c679170..a802fcc532 100644
--- a/marimo/_runtime/executor/lifecycles/cached.py
+++ b/marimo/_runtime/executor/lifecycles/cached.py
@@ -180,6 +180,7 @@ class CachedLifecycle:
self._restored_keys[cell_id] = str(
self._loader.build_path(attempt.key)
)
+ self._record(cell_id, attempt)
return Skip(
result=RunResult(
output=attempt.meta.get("return"), exception=None
@@ -236,11 +237,20 @@ class CachedLifecycle:
self._restored_keys[cell_id] = str(
self._loader.build_path(attempt.key)
)
+ self._record(cell_id, attempt)
except BaseException as e:
# Best-effort: save failures (incl. CacheException, which
# extends BaseException) must never break the teardown chain.
LOGGER.warning("Cache save failed for %s: %s", cell_id, e)
+ def _record(self, cell_id: CellId_t, attempt: Cache) -> None:
+ """Record the entry under the cell it caches."""
+ from marimo._save.manifest import record_cache_event
+
+ record_cache_event(
+ self._loader, attempt.key, cell_id, self._graph, saved=True
+ )
+
@staticmethod
def _defines_stub(cell: CellImpl, glbls: MutableGlobals) -> bool:
"""True if any name the cell defines is still an UnhashableStub."""
diff --git a/marimo/_runtime/runner/hooks_on_finish.py b/marimo/_runtime/runner/hooks_on_finish.py
index bc9ee9c2be..740af0a984 100644
--- a/marimo/_runtime/runner/hooks_on_finish.py
+++ b/marimo/_runtime/runner/hooks_on_finish.py
@@ -86,7 +86,24 @@ def _send_cancellation_errors(ctx: OnFinishHookContext) -> None:
)
+@kernel_tracer.start_as_current_span("flush_cache_manifests")
+def _flush_cache_manifests(ctx: OnFinishHookContext) -> None:
+ """Write out the cache entries this run recorded.
+
+ Once per run rather than once per entry. A manifest names every key the
+ notebook produced, and is rewritten whole.
+ """
+ del ctx
+ from marimo._runtime.context import safe_get_context
+ from marimo._save.manifest import flush_dirty_manifests
+
+ context = safe_get_context()
+ if context is not None:
+ flush_dirty_manifests(context)
+
+
ON_FINISH_HOOKS: list[OnFinishHook] = [
_send_interrupt_errors,
_send_cancellation_errors,
+ _flush_cache_manifests,
]
diff --git a/marimo/_save/cache.py b/marimo/_save/cache.py
index 76eb1ab291..776f63df46 100644
--- a/marimo/_save/cache.py
+++ b/marimo/_save/cache.py
@@ -44,6 +44,7 @@ if TYPE_CHECKING:
from marimo._save.loaders.lazy import LazyLoader
from marimo._save.signing_policy import SigningPolicy
from marimo._save.stores import Store
+ from marimo._types.ids import CellId_t
# NB. Increment on cache breaking changes.
MARIMO_CACHE_VERSION: int = 6
@@ -86,6 +87,16 @@ class CacheState:
# config (user/env only; project/script trust is stripped upstream). A
# `LazyLoader` reads it as the default trust/identity source.
signing_policy: SigningPolicy | None = None
+ # Cache keys this session recorded, grouped by the store they were
+ # written through, then by path hash and block. A manifest lists only
+ # the entries stored beside it.
+ manifest_records: dict[Store, dict[str, dict[str, set[str]]]] = field(
+ default_factory=dict
+ )
+ # The stores with a record not yet written to a manifest.
+ manifest_dirty: set[Store] = field(default_factory=set)
+ # Path hash per cell, cleared with `hash_memo` when a cell re-executes.
+ node_memo: dict[CellId_t, str] = field(default_factory=dict)
# Lazy-store session state; see `loaders/lazy.py:_cache_state`.
active_lazy_loaders: dict[str, LazyLoader] = field(default_factory=dict)
poisoned_keys: set[str] = field(default_factory=set)
@@ -122,6 +133,8 @@ class HashMemoCleanup(CellLifecycleItem):
def dispose(self, context: RuntimeContext, deletion: bool) -> bool: # noqa: ARG002
context.cache.hash_memo.clear()
+ # An edit anywhere in a cell's closure changes its path hash.
+ context.cache.node_memo.clear()
return True
diff --git a/marimo/_save/cache_dirs.py b/marimo/_save/cache_dirs.py
index aefa282839..fd88abd377 100644
--- a/marimo/_save/cache_dirs.py
+++ b/marimo/_save/cache_dirs.py
@@ -7,9 +7,11 @@ Pure and kernel-free so the CLI can import it without starting a runtime.
from __future__ import annotations
import os
+import re
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
+from uuid import uuid4
from marimo import _loggers
from marimo._utils.paths import (
@@ -25,7 +27,14 @@ LOGGER = _loggers.marimo_logger()
CACHE_DIR_NAME = "cache"
-# Statuses that still describe a marimo notebook; "empty" and "invalid" do not.
+# An entry is written to a sibling and renamed into place, so a reader never
+# meets a half-written value. A process killed between the two steps leaves
+# the sibling behind, bytes on disk that hold no cached value.
+PARTIAL_WRITE_INFIX = ".tmp"
+_PARTIAL_WRITE_TAG = re.compile(r"[0-9a-f]{8}")
+
+# Statuses that still describe a marimo notebook, unlike "empty" and
+# "invalid".
_NOTEBOOK_STATUSES = frozenset({"valid", "has_warnings", "has_errors"})
@@ -132,6 +141,17 @@ def cache_dir_stats(cache_dir: Path) -> CacheDirStats:
return stats
+def partial_write_name(name: str) -> str:
+ """Return the sibling name to write `name` to before renaming it in."""
+ return f"{name}{PARTIAL_WRITE_INFIX}{uuid4().hex[:8]}"
+
+
+def is_partial_write(name: str) -> bool:
+ """Whether `name` is a sibling that an interrupted write left behind."""
+ head, _, tag = name.rpartition(PARTIAL_WRITE_INFIX)
+ return bool(head) and _PARTIAL_WRITE_TAG.fullmatch(tag) is not None
+
+
def entry_bytes(entry: Path) -> int:
"""Return the bytes a cache entry occupies.
@@ -157,12 +177,14 @@ def _block_stats(block: Path) -> CacheDirStats:
hashes = {
_entry_hash(child.name)
for child in children
- if not _is_directory(child)
+ if not _is_directory(child) and not is_partial_write(child.name)
}
total_bytes = 0
entries = 0
for child in children:
total_bytes += entry_bytes(child)
+ if is_partial_write(child.name):
+ continue
if not (_is_directory(child) and child.name in hashes):
entries += 1
return CacheDirStats(total_bytes=total_bytes, entries=entries)
diff --git a/marimo/_save/hash.py b/marimo/_save/hash.py
index 7678b95354..a8f07498fe 100644
--- a/marimo/_save/hash.py
+++ b/marimo/_save/hash.py
@@ -18,7 +18,11 @@ from marimo._ast.variables import (
)
from marimo._ast.visitor import ImportData, Name, ScopedVisitor
from marimo._plugins.ui._core.ui_element import UIElement
-from marimo._runtime.context import ContextNotInitializedError, get_context
+from marimo._runtime.context import (
+ ContextNotInitializedError,
+ get_context,
+ safe_get_context,
+)
from marimo._runtime.dataflow import induced_subgraph
from marimo._runtime.dataflow.topology import GraphTopology
from marimo._runtime.primitives import (
@@ -167,6 +171,39 @@ def hash_cell_execution(
return hash_cell_group(ancestors, graph, hash_type)
+def hash_cell_closure(
+ cell_id: CellId_t, graph: GraphTopology, hash_type: str = DEFAULT_HASH
+) -> bytes:
+ """Hash a cell together with every cell it depends on.
+
+ Unlike an execution hash, an edit to the cell itself changes the digest.
+ Compiling a notebook is enough to compute it. Nothing here reads a value.
+ """
+ return hash_cell_group(
+ graph.ancestors(cell_id) | {cell_id}, graph, hash_type
+ )
+
+
+def cell_path_hash(cell_id: CellId_t, graph: GraphTopology) -> str:
+ """Hex closure digest of `cell_id`, memoized for the session.
+
+ Always the default hash, never the hash type a cache block chose. The
+ digest is recorded so that a later process can recompute it from source
+ alone. No block is in reach there to supply a hash type.
+ """
+ ctx = safe_get_context()
+ if ctx is not None and cell_id in ctx.cache.node_memo:
+ return ctx.cache.node_memo[cell_id]
+
+ digest = hash_cell_closure(cell_id, graph).hex()
+ if ctx is not None:
+ ctx.cache.node_memo[cell_id] = digest
+ # An edit to an ancestor reruns this cell as well, and disposal empties
+ # the whole memo, so the hook on this cell covers its closure.
+ ctx.cell_lifecycle_registry.inject(cell_id, HashMemoCleanup())
+ return digest
+
+
def get_and_update_context_from_scope(
scope: dict[str, Any],
scope_refs: set[Name] | None = None,
@@ -413,6 +450,17 @@ class BlockHasher:
)
return self._hash
+ @property
+ def path_hash(self) -> str:
+ """Static digest of the closure of the cell this block sits in.
+
+ A third digest, independent of `hash` and `exe_hash`. Those fold in
+ serialized values and side effects that only a running kernel holds,
+ while this one is recomputable from the notebook source alone. It
+ covers the whole cell, so every block in a cell shares it.
+ """
+ return cell_path_hash(self.cell_id, self.graph)
+
@property
def exe_hash(self) -> str:
if self._exe_hash is None:
diff --git a/marimo/_save/loaders/loader.py b/marimo/_save/loaders/loader.py
index 5850763d17..f326d442e2 100644
--- a/marimo/_save/loaders/loader.py
+++ b/marimo/_save/loaders/loader.py
@@ -302,8 +302,18 @@ class BasePersistenceLoader(Loader):
"""Paths under `root` that `clear()` removes."""
import glob
- pattern = str(root / self.name / f"*.{self.suffix}")
- return [Path(match) for match in glob.glob(pattern)]
+ from marimo._save.cache_dirs import PARTIAL_WRITE_INFIX
+
+ block = root / self.name
+ # The leftovers of an interrupted write hold no value anyone can read,
+ # so clearing the block is the last chance to reclaim their bytes.
+ patterns = (
+ str(block / f"*.{self.suffix}"),
+ str(block / f"*.{self.suffix}{PARTIAL_WRITE_INFIX}*"),
+ )
+ return [
+ Path(match) for pattern in patterns for match in glob.glob(pattern)
+ ]
def clear(self) -> None:
"""Clear all cached items for this loader."""
diff --git a/marimo/_save/manifest.py b/marimo/_save/manifest.py
new file mode 100644
index 0000000000..41242de7af
--- /dev/null
+++ b/marimo/_save/manifest.py
@@ -0,0 +1,369 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Record of the cache entries a notebook produced.
+
+A manifest groups cache keys under path hashes. A path hash is a static
+digest of a cell and everything it depends on, computed from source alone.
+Records accumulate across runs, so an entry stays recorded for as long as
+the code that produced it exists, whatever branch or user-interface state
+the last run took.
+
+The manifest travels through the same `Store` as the entries it describes,
+under a reserved key, so a remote or in-browser cache holds its manifest
+too. It is pure and kernel-free, so the CLI can read it without a runtime.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from marimo import _loggers
+from marimo._utils.paths import normalize_path
+
+if TYPE_CHECKING:
+ from collections.abc import Collection
+
+ from marimo._runtime.context.types import RuntimeContext
+ from marimo._runtime.dataflow.topology import GraphTopology
+ from marimo._save.hash import HashKey
+ from marimo._save.loaders import Loader
+ from marimo._save.stores import Store
+ from marimo._types.ids import CellId_t
+
+LOGGER = _loggers.marimo_logger()
+
+MANIFEST_VERSION = 1
+MANIFEST_SUFFIX = ".json"
+MAX_SLUG_LENGTH = 64
+# A manifest ends in a 16-hex path digest, which nothing else at the root of
+# a cache directory ends in: a block directory never holds a dot, the export
+# manifest ends in `-export.json`, and the leftovers of an interrupted write
+# end in their tag.
+_MANIFEST_NAME = re.compile(r"[0-9A-Za-z._-]+-[0-9a-f]{16}\.json")
+
+
+class ManifestError(Exception):
+ """Base class for manifest failures."""
+
+
+class CorruptManifestError(ManifestError):
+ """Raised when a manifest cannot be read as one."""
+
+
+class UnsupportedManifestVersionError(ManifestError):
+ """Raised when a manifest comes from a newer version of marimo."""
+
+
+def manifest_name(notebook_path: str | Path) -> str:
+ """Return the reserved store key holding `notebook_path`'s manifest.
+
+ The name carries a digest of the notebook's absolute path, so two
+ notebooks that share a cache directory keep separate manifests however
+ their stems collide. The stem is kept only to make the file recognizable.
+ """
+ path = normalize_path(Path(notebook_path))
+ slug = re.sub(r"[^0-9A-Za-z._-]+", "-", path.stem).strip("-") or "notebook"
+ # The digest tells manifests apart. The stem is only a label, and a long
+ # one would push the name past what a filesystem allows.
+ slug = slug[:MAX_SLUG_LENGTH]
+ # Encoded as the filesystem hands it over: a path that is not valid UTF-8
+ # arrives carrying surrogate escapes, which strict encoding rejects.
+ digest = hashlib.sha256(
+ str(path).encode("utf-8", "surrogateescape")
+ ).hexdigest()[:16]
+ return f"{slug}-{digest}{MANIFEST_SUFFIX}"
+
+
+def is_manifest_name(name: str) -> bool:
+ """Whether `name` is the reserved key of a cache manifest.
+
+ A cache directory holds other files that describe rather than hold a
+ cached value, so recognizing a manifest takes more than the suffix.
+ """
+ return _MANIFEST_NAME.fullmatch(name) is not None
+
+
+@dataclass
+class CacheManifest:
+ """The cache keys a notebook produced, grouped by path hash."""
+
+ # Path to the notebook, relative to the cache directory where possible, so
+ # a manifest can be attributed to its notebook after the tree is moved.
+ notebook: str = ""
+ nodes: dict[str, dict[str, set[str]]] = field(default_factory=dict)
+ written_at: str | None = None
+
+ def merge(self, other: CacheManifest) -> None:
+ """Union `other`'s keys into this manifest, per path hash and block."""
+ for node, blocks in other.nodes.items():
+ merged = self.nodes.setdefault(node, {})
+ for block, keys in blocks.items():
+ merged[block] = merged.get(block, set()) | set(keys)
+
+ def evict_dead(self, live_nodes: Collection[str]) -> set[tuple[str, str]]:
+ """Drop the path hashes absent from `live_nodes`.
+
+ Returns the `(block, key)` entries the dropped hashes listed. They
+ are only unreachable through this manifest. Another notebook sharing
+ the cache directory can still list them.
+ """
+ live = set(live_nodes)
+ evicted: set[tuple[str, str]] = set()
+ for node in set(self.nodes) - live:
+ for block, keys in self.nodes.pop(node).items():
+ evicted.update((block, key) for key in keys)
+ return evicted
+
+ def entries(self) -> set[tuple[str, str]]:
+ """Every `(block, key)` this manifest attests."""
+ return {
+ (block, key)
+ for blocks in self.nodes.values()
+ for block, keys in blocks.items()
+ for key in keys
+ }
+
+ def to_bytes(self) -> bytes:
+ """Serialize to the bytes a store holds under the manifest key."""
+ return json.dumps(
+ {
+ "version": MANIFEST_VERSION,
+ "notebook": self.notebook,
+ "written_at": self.written_at or _utc_now(),
+ "nodes": {
+ node: {
+ block: sorted(keys)
+ for block, keys in sorted(blocks.items())
+ }
+ for node, blocks in sorted(self.nodes.items())
+ },
+ },
+ indent=2,
+ ).encode("utf-8")
+
+ @classmethod
+ def from_bytes(cls, data: bytes) -> CacheManifest:
+ """Parse manifest bytes.
+
+ Raises:
+ CorruptManifestError: If `data` is not a manifest.
+ UnsupportedManifestVersionError: If it was written by a newer
+ marimo, whose semantics this one cannot assume.
+ """
+ try:
+ payload = json.loads(data)
+ except (UnicodeDecodeError, json.JSONDecodeError) as e:
+ raise CorruptManifestError(f"Manifest is not JSON: {e}") from e
+ if not isinstance(payload, dict):
+ raise CorruptManifestError("Manifest is not an object.")
+
+ version = payload.get("version")
+ if not isinstance(version, int) or isinstance(version, bool):
+ raise CorruptManifestError("Manifest has no version.")
+ if version > MANIFEST_VERSION:
+ raise UnsupportedManifestVersionError(
+ f"Manifest version {version} was written by a newer version "
+ f"of marimo; this one reads up to {MANIFEST_VERSION}."
+ )
+
+ return cls(
+ notebook=_read_str(payload, "notebook"),
+ nodes=_read_nodes(payload.get("nodes")),
+ written_at=payload.get("written_at")
+ if isinstance(payload.get("written_at"), str)
+ else None,
+ )
+
+
+def load_manifest(store: Store, key: str) -> CacheManifest | None:
+ """Read the manifest `store` holds under `key`, or `None` if it holds none.
+
+ Raises:
+ CorruptManifestError: If the stored bytes are not a manifest.
+ UnsupportedManifestVersionError: If they come from a newer marimo.
+ """
+ data = store.get(key)
+ if not data:
+ return None
+ return CacheManifest.from_bytes(data)
+
+
+def record_cache_event(
+ loader: Loader,
+ key: HashKey,
+ cell_id: CellId_t,
+ graph: GraphTopology,
+ *,
+ saved: bool = False,
+) -> None:
+ """Record `key` against the cell that defines its cached code.
+
+ `saved` says the entry was just written rather than read. A record
+ already made this session is not repeated for a hit, but a write means
+ the entry is new on disk, however often it was seen before: something
+ else, such as `marimo cache clean`, can have taken the entry and its
+ record away in between.
+
+ `cell_id` is the defining cell, which a function cache reaches from
+ another cell. The manifest keys on where the cached code is defined, not
+ on where it ran. Anything that leaves no on-disk entry to clean up
+ records nothing: a session-local cache, a scratchpad cell, a notebook
+ with no file.
+
+ Recording happens inside the user's cell, so it reports errors instead
+ of raising them. A cache that was written and restored must not fail its
+ cell because bookkeeping about it went wrong. The record reaches a store
+ when the run finishes.
+ """
+ try:
+ _record_cache_event(loader, key, cell_id, graph, saved=saved)
+ except Exception:
+ LOGGER.warning("Could not record a cache entry", exc_info=True)
+
+
+def _record_cache_event(
+ loader: Loader,
+ key: HashKey,
+ cell_id: CellId_t,
+ graph: GraphTopology,
+ *,
+ saved: bool,
+) -> None:
+ from marimo._runtime.context import safe_get_context
+ from marimo._save.loaders import BasePersistenceLoader
+
+ if not isinstance(loader, BasePersistenceLoader):
+ return
+ if cell_id not in graph.cells:
+ return
+ ctx = safe_get_context()
+ if ctx is None or not ctx.filename:
+ return
+
+ from marimo._save.hash import cell_path_hash
+
+ path = loader.build_path(key)
+ block, entry = path.parent.name, path.stem
+ node = cell_path_hash(cell_id, graph)
+
+ # Records are kept per store. A manifest lists only the entries stored
+ # beside it, and a notebook can cache to more than one location.
+ store = loader.store
+ blocks = ctx.cache.manifest_records.setdefault(store, {}).setdefault(
+ node, {}
+ )
+ keys = blocks.setdefault(block, set())
+ if entry in keys and not saved:
+ return
+ keys.add(entry)
+ ctx.cache.manifest_dirty.add(store)
+
+
+def flush_dirty_manifests(ctx: RuntimeContext) -> None:
+ """Write a manifest for every store this session recorded an entry in.
+
+ Writes are coalesced rather than issued per entry. Each write replaces
+ the whole manifest, so a per-key flush for a cell that fills a cache
+ with a thousand keys rewrites the growing file a thousand times.
+ """
+ for store in list(ctx.cache.manifest_dirty):
+ try:
+ written = flush_cache_manifest(ctx, store)
+ except Exception:
+ LOGGER.warning("Could not write a cache manifest", exc_info=True)
+ continue
+ if written:
+ ctx.cache.manifest_dirty.discard(store)
+
+
+def flush_cache_manifest(ctx: RuntimeContext, store: Store) -> bool:
+ """Merge `store`'s records into its on-disk manifest and write it back.
+
+ Returns whether a manifest was written. A session that produced no
+ persistent entry writes nothing, so reading a cache never creates one.
+ """
+ records = ctx.cache.manifest_records.get(store)
+ if not records or not ctx.filename:
+ return False
+
+ key = manifest_name(ctx.filename)
+ try:
+ manifest = load_manifest(store, key) or CacheManifest()
+ except UnsupportedManifestVersionError as e:
+ # A rewrite under the older schema drops whatever the newer one
+ # holds, including the marker that tells a reader to stay away.
+ LOGGER.warning("Leaving cache manifest %s alone: %s", key, e)
+ ctx.cache.manifest_dirty.discard(store)
+ return False
+ except ManifestError as e:
+ # Bytes that parse as no manifest at all record nothing that can be
+ # acted on, so replacing them costs nothing.
+ LOGGER.warning("Rewriting unreadable cache manifest %s: %s", key, e)
+ manifest = CacheManifest()
+
+ manifest.merge(CacheManifest(nodes=records))
+ manifest.notebook = _notebook_ref(ctx.filename, store)
+ manifest.written_at = _utc_now()
+ try:
+ written = store.put(key, manifest.to_bytes())
+ except Exception as e:
+ LOGGER.warning("Could not write cache manifest %s: %s", key, e)
+ return False
+ if not written:
+ # A store can decline a write by its return value rather than raise.
+ LOGGER.warning("Could not write cache manifest %s", key)
+ return written
+
+
+def _notebook_ref(notebook_path: str, store: Store) -> str:
+ """Where the manifest records its notebook, relative to the cache dir."""
+ path = normalize_path(Path(notebook_path))
+ directories = store.local_dirs()
+ # A relative reference survives a moved tree, but only reads back as the
+ # notebook from the one directory it was measured against. A store that
+ # writes the same bytes to several directories has no such directory.
+ if len(directories) != 1:
+ return path.as_posix()
+ try:
+ return Path(os.path.relpath(path, directories[0])).as_posix()
+ except ValueError:
+ # Different Windows drives have no path between them.
+ return path.as_posix()
+
+
+def _utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat()
+
+
+def _read_str(payload: dict[str, Any], field_name: str) -> str:
+ value = payload.get(field_name)
+ if value is None:
+ return ""
+ if not isinstance(value, str):
+ raise CorruptManifestError(f"Manifest field {field_name} is not text.")
+ return value
+
+
+def _read_nodes(nodes: Any) -> dict[str, dict[str, set[str]]]:
+ if nodes is None:
+ return {}
+ if not isinstance(nodes, dict):
+ raise CorruptManifestError("Manifest nodes are not an object.")
+ parsed: dict[str, dict[str, set[str]]] = {}
+ for node, blocks in nodes.items():
+ if not isinstance(node, str) or not isinstance(blocks, dict):
+ raise CorruptManifestError("Manifest node is malformed.")
+ parsed[node] = {}
+ for block, keys in blocks.items():
+ if not isinstance(block, str) or not isinstance(keys, list):
+ raise CorruptManifestError("Manifest block is malformed.")
+ if not all(isinstance(key, str) for key in keys):
+ raise CorruptManifestError("Manifest key is not text.")
+ parsed[node][block] = set(keys)
+ return parsed
diff --git a/marimo/_save/save.py b/marimo/_save/save.py
index d498d68c74..43b240f3ca 100644
--- a/marimo/_save/save.py
+++ b/marimo/_save/save.py
@@ -372,7 +372,27 @@ class _cache_call(CacheContext, Generic[P, R]):
# stateful variables may be global
scope = {k: v for k, v in scope.items() if k in attempt.stateful_refs}
attempt.update(scope, meta={"return": response, "runtime": runtime})
- self.loader.save_cache(attempt)
+ if self.loader.save_cache(attempt):
+ self._record(attempt)
+
+ def _record(self, attempt: Cache) -> None:
+ """Record the entry under the cell the function is defined in.
+
+ A call can come from any cell. The definition is what a later prune
+ recompiles.
+ """
+ if self.base_block is None:
+ return
+
+ from marimo._save.manifest import record_cache_event
+
+ record_cache_event(
+ self.loader,
+ attempt.key,
+ self.base_block.cell_id,
+ self.base_block.graph,
+ saved=not attempt.hit,
+ )
@property
def misses(self) -> int:
@@ -495,6 +515,7 @@ class _cache_call(CacheContext, Generic[P, R]):
try:
if attempt.hit:
attempt.restore(scope)
+ self._record(attempt)
return cast(R, attempt.meta.get("return"))
start_time = time.time()
@@ -622,6 +643,7 @@ class _cache_call_async(_cache_call[P, R]):
try:
if attempt.hit:
attempt.restore(scope)
+ self._record(attempt)
return attempt.meta.get("return")
start_time = time.time()
@@ -655,6 +677,9 @@ class _cache_context(SkipContext, CacheContext):
self._cache: Cache | None = None
self._body_start: int = MAXINT
+ # The cell the block was found in, resolved while tracing.
+ self._cell_id: CellId_t | None = None
+ self._graph: DirectedGraph | None = None
# TODO: Consider having a user level setting.
self.pin_modules = pin_modules
self.hash_type = hash_type
@@ -669,6 +694,22 @@ class _cache_context(SkipContext, CacheContext):
def hit(self) -> bool:
return self._cache is not None and self._cache.hit
+ def _record(self) -> None:
+ """Record the entry under the cell holding the block."""
+ if self._cache is None or self._cell_id is None:
+ return
+ assert self._graph is not None, UNEXPECTED_FAILURE_BOILERPLATE
+
+ from marimo._save.manifest import record_cache_event
+
+ record_cache_event(
+ self.loader,
+ self._cache.key,
+ self._cell_id,
+ self._graph,
+ saved=not self._cache.hit,
+ )
+
def trace(self, with_frame: FrameType) -> None:
# General flow is as follows:
# 1) Follow the stack trace backwards to the first instance of a
@@ -749,6 +790,8 @@ class _cache_context(SkipContext, CacheContext):
ast.parse(code).body # type: ignore[arg-type]
)
+ self._cell_id = cell_id
+ self._graph = graph
self._cache = cache_attempt_from_hash(
save_module,
graph,
@@ -813,6 +856,7 @@ class _cache_context(SkipContext, CacheContext):
if self._cache.hit:
assert self._frame is not None, UNEXPECTED_FAILURE_BOILERPLATE
self._cache.restore(self._frame.f_locals)
+ self._record()
# Return true to suppress the SkipWithBlock exception.
return True
@@ -820,8 +864,9 @@ class _cache_context(SkipContext, CacheContext):
runtime = time.time() - self._start_time
self._cache.update(self._frame.f_locals, meta={"runtime": runtime})
+ saved = False
try:
- self.loader.save_cache(self._cache)
+ saved = self.loader.save_cache(self._cache)
except Exception as e:
sys.stderr.write(
"An exception was raised when attempting to cache this code "
@@ -833,6 +878,8 @@ class _cache_context(SkipContext, CacheContext):
traceback.print_exc(file=tmpio)
tmpio.seek(0)
write_traceback(tmpio.read())
+ if saved:
+ self._record()
except Exception:
failed = True
raise
diff --git a/marimo/_save/stores/file.py b/marimo/_save/stores/file.py
index 16c1449c3b..9a6b5975a2 100644
--- a/marimo/_save/stores/file.py
+++ b/marimo/_save/stores/file.py
@@ -1,12 +1,14 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
+import contextlib
import os
import re
from pathlib import Path
from marimo import _loggers
from marimo._runtime.runtime import notebook_dir
+from marimo._save.cache_dirs import partial_write_name
from marimo._save.stores.store import Store
from marimo._utils.paths import MARIMO_DIR_NAME, notebook_output_dir
@@ -112,7 +114,16 @@ class FileStore(Store):
path = self.save_path / key
path.parent.mkdir(parents=True, exist_ok=True)
self._initialized = True
- path.write_bytes(value)
+ # A sibling keeps the rename on one filesystem, where it is atomic.
+ tmp = path.with_name(partial_write_name(path.name))
+ try:
+ tmp.write_bytes(value)
+ os.replace(tmp, path)
+ except BaseException:
+ # The write failing is the error to report, not the cleanup.
+ with contextlib.suppress(OSError):
+ tmp.unlink(missing_ok=True)
+ raise
return True
def hit(self, key: str) -> bool:
diff --git a/marimo/_save/stores/store.py b/marimo/_save/stores/store.py
index ac8424a623..6c40ac5b11 100644
--- a/marimo/_save/stores/store.py
+++ b/marimo/_save/stores/store.py
@@ -16,7 +16,7 @@ class Store(ABC):
@abstractmethod
def put(self, key: str, value: bytes) -> bool:
- """Put a cache into the store"""
+ """Put a cache into the store."""
@abstractmethod
def hit(self, key: str) -> bool:
diff --git a/tests/_ast/test_demangle_locals.py b/tests/_ast/test_demangle_locals.py
index a8a31e4cf9..0b2df43d31 100644
--- a/tests/_ast/test_demangle_locals.py
+++ b/tests/_ast/test_demangle_locals.py
@@ -1,7 +1,7 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
-from marimo._ast.variables import demangle_locals_in_text
+from marimo._ast.variables import demangle_locals_in_text, unmangle_local
def test_demangle_in_name_error_message() -> None:
@@ -42,6 +42,19 @@ def test_demangle_handles_uuid_cell_id() -> None:
)
+def test_unmangle_local_handles_uuid_cell_id() -> None:
+ # A cell hashes to the same digest wherever it is compiled only if the
+ # name survives mangling identically, so hyphenated ids must unmangle.
+ name, cell = unmangle_local("_cell_c9bf9e57-1685-4c89-bafb-1234abcd_a")
+ assert name == "_a"
+ assert cell == "c9bf9e57-1685-4c89-bafb-1234abcd"
+
+
+def test_unmangle_local_leaves_an_unmangled_name_alone() -> None:
+ assert unmangle_local("_a") == ("_a", "")
+ assert unmangle_local("value") == ("value", "")
+
+
def test_demangle_handles_single_underscore_local() -> None:
# `_` is a valid local name (variables.py:is_local); it mangles to
# `_cell_<id>_` with no name suffix.
diff --git a/tests/_save/test_cache_dirs.py b/tests/_save/test_cache_dirs.py
index 502b4ce3a3..44f0d050ab 100644
--- a/tests/_save/test_cache_dirs.py
+++ b/tests/_save/test_cache_dirs.py
@@ -16,6 +16,7 @@ from marimo._save.cache_dirs import (
directory_cache_dir,
entry_bytes,
notebook_cache_dir,
+ partial_write_name,
resolve_cache_dirs,
)
@@ -318,6 +319,21 @@ def test_cache_dir_stats_counts_an_entry_and_its_blobs_once(
)
+def test_cache_dir_stats_does_not_count_an_interrupted_write(
+ tmp_path: Path,
+) -> None:
+ """A killed kernel leaves the sibling of a rename. It is not an entry."""
+ cache_dir = make_cache_dir(tmp_path)
+ populate_cache_dir(cache_dir)
+ leftover = cache_dir / "train" / partial_write_name("C_77e0.pickle")
+ leftover.write_bytes(b"p" * 8)
+
+ # The bytes are on disk and reported as such. The value is not.
+ assert cache_dir_stats(cache_dir) == CacheDirStats(
+ total_bytes=113, entries=2
+ )
+
+
def test_cache_dir_stats_counts_an_orphaned_blob_directory(
tmp_path: Path,
) -> None:
diff --git a/tests/_save/test_hash.py b/tests/_save/test_hash.py
index 5ecbc083bb..788990c88f 100644
--- a/tests/_save/test_hash.py
+++ b/tests/_save/test_hash.py
@@ -2,13 +2,14 @@
from __future__ import annotations
+import ast
import asyncio
import dataclasses
import os
import subprocess
import sys
import textwrap
-from typing import Any
+from typing import TYPE_CHECKING, Any
import pytest
@@ -19,6 +20,11 @@ from marimo._runtime.runtime import Kernel
from marimo._save.hash import BlockHasher
from tests.conftest import ExecReqProvider
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from marimo._runtime.dataflow import DirectedGraph
+
class TestHash:
@staticmethod
@@ -2896,3 +2902,179 @@ def test_signed_stateful_bytes_unpicklable_raises() -> None:
hasher = BlockHasher.__new__(BlockHasher)
with pytest.raises(TypeError, match="neither"):
hasher._signed_stateful_bytes(lambda: None, "ui")
+
+
+NOTEBOOK_WITH_SETUP = """import marimo
+
+app = marimo.App()
+
+with app.setup:
+ import marimo as mo
+
+
+def _():
+ _private = 3
+ base = _private * 2
+ return
+
+
+def _(base):
+ derived = base + len(mo.__name__)
+ return
+"""
+
+
+def _static_graph(notebook: Path) -> DirectedGraph:
+ """The graph a notebook compiles to, without running it."""
+ from marimo._ast.app import InternalApp
+ from marimo._ast.load import load_app
+
+ app = load_app(str(notebook))
+ assert app is not None
+ return InternalApp(app).graph
+
+
+def _path_hashes(graph: DirectedGraph) -> set[str]:
+ from marimo._save.hash import hash_cell_closure
+
+ return {hash_cell_closure(cell_id, graph).hex() for cell_id in graph.cells}
+
+
+def _path_hash_of(graph: DirectedGraph, name: str) -> str:
+ """The digest of the cell defining `name`."""
+ from marimo._save.hash import hash_cell_closure
+
+ (cell_id,) = [
+ cell_id for cell_id, cell in graph.cells.items() if name in cell.defs
+ ]
+ return hash_cell_closure(cell_id, graph).hex()
+
+
+def _write_notebook(path: Path, *cells: str) -> Path:
+ path.write_text(
+ "import marimo\n\napp = marimo.App()\n\n\n" + "\n\n".join(cells),
+ encoding="utf-8",
+ )
+ return path
+
+
+class TestPathHash:
+ """The static digest a manifest records cache entries under."""
+
+ @staticmethod
+ def test_stable_across_loads(tmp_path: Path) -> None:
+ notebook = tmp_path / "nb.py"
+ notebook.write_text(NOTEBOOK_WITH_SETUP, encoding="utf-8")
+ first = _path_hashes(_static_graph(notebook))
+ second = _path_hashes(_static_graph(notebook))
+ assert first == second
+ assert len(first) == 3
+
+ @staticmethod
+ async def test_kernel_and_static_graphs_agree(
+ k: Kernel, exec_req: ExecReqProvider, tmp_path: Path
+ ) -> None:
+ """A running kernel and a compiled file must produce one digest. If
+ they differ, prune reads live code as dead. Private definitions are
+ mangled by cell id, which differs between the two."""
+ notebook = tmp_path / "nb.py"
+ notebook.write_text(NOTEBOOK_WITH_SETUP, encoding="utf-8")
+
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get("_private = 3\nbase = _private * 2"),
+ exec_req.get("derived = base + len(mo.__name__)"),
+ ]
+ )
+ assert not k.stderr.messages
+ assert _path_hashes(k.graph) == _path_hashes(_static_graph(notebook))
+
+ @staticmethod
+ def test_only_an_ancestor_edit_moves_a_digest(tmp_path: Path) -> None:
+ """A stable digest is how prune tells that the code which produced
+ an entry is still there. Only the code the entry depends on moves
+ it. A digest that followed the rest of the notebook lets any edit
+ make entries produced by untouched code look dead."""
+
+ def digests(name: str, *cells: str) -> tuple[str, str]:
+ graph = _static_graph(_write_notebook(tmp_path / name, *cells))
+ return _path_hash_of(graph, "a"), _path_hash_of(graph, "c")
+
+ cells = [
+ "@app.cell\ndef _():\n a = 1\n return\n",
+ "@app.cell\ndef _():\n b = 2\n return\n",
+ "@app.cell\ndef _(a):\n c = a + 1\n return\n",
+ ]
+ base_a, base_c = digests("base.py", *cells)
+
+ unrelated = list(cells)
+ unrelated[1] = "@app.cell\ndef _():\n b = 3\n return\n"
+ assert digests("unrelated.py", *unrelated) == (base_a, base_c)
+
+ # `c` reads `a`, so it is downstream of it and no part of its closure.
+ downstream = list(cells)
+ downstream[2] = "@app.cell\ndef _(a):\n c = a + 2\n return\n"
+ assert digests("downstream.py", *downstream)[0] == base_a
+
+ ancestor = list(cells)
+ ancestor[0] = "@app.cell\ndef _():\n a = 2\n return\n"
+ edited_a, edited_c = digests("ancestor.py", *ancestor)
+ assert edited_a != base_a
+ assert edited_c != base_c
+
+ @staticmethod
+ def test_ancestor_order_does_not_matter(tmp_path: Path) -> None:
+ cells = [
+ "@app.cell\ndef _():\n a = 1\n return\n",
+ "@app.cell\ndef _():\n b = 2\n return\n",
+ "@app.cell\ndef _(a, b):\n c = a + b\n return\n",
+ ]
+ forward = _write_notebook(tmp_path / "forward.py", *cells)
+ swapped = _write_notebook(
+ tmp_path / "swapped.py", cells[1], cells[0], cells[2]
+ )
+ assert _path_hashes(_static_graph(forward)) == _path_hashes(
+ _static_graph(swapped)
+ )
+
+ @staticmethod
+ async def test_property_tracks_the_defining_cell(
+ k: Kernel, exec_req: ExecReqProvider
+ ) -> None:
+ from marimo._save.hash import hash_cell_closure
+
+ base = exec_req.get("value = 1")
+ await k.run([base, exec_req.get("doubled = value * 2")])
+ assert not k.stderr.messages
+
+ hasher = BlockHasher(
+ module=ast.parse("held = value + 1"),
+ graph=k.graph,
+ cell_id=base.cell_id,
+ scope={"value": 1},
+ )
+ assert (
+ hasher.path_hash == hash_cell_closure(base.cell_id, k.graph).hex()
+ )
+ # A third digest: the block's own hashes fold in values and side
+ # effects that no compile-time recomputation could reach.
+ assert hasher.path_hash not in (hasher.hash, hasher.exe_hash)
+
+ @staticmethod
+ async def test_memo_is_dropped_when_a_cell_changes(
+ k: Kernel, exec_req: ExecReqProvider
+ ) -> None:
+ from marimo._runtime.context import get_context
+ from marimo._save.hash import cell_path_hash
+
+ cell = exec_req.get("value = 1")
+ await k.run([cell])
+ before = cell_path_hash(cell.cell_id, k.graph)
+ assert get_context().cache.node_memo
+
+ await k.run([exec_req.get_with_id(cell.cell_id, "value = 2")])
+ assert not k.stderr.messages
+ assert cell_path_hash(cell.cell_id, k.graph) != before
diff --git a/tests/_save/test_manifest.py b/tests/_save/test_manifest.py
new file mode 100644
index 0000000000..4862353857
--- /dev/null
+++ b/tests/_save/test_manifest.py
@@ -0,0 +1,867 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""The manifest format, and the runtime paths that write into it."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import json
+import os
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from marimo._config.config import DEFAULT_CONFIG
+from marimo._save.manifest import (
+ MANIFEST_VERSION,
+ CacheManifest,
+ CorruptManifestError,
+ UnsupportedManifestVersionError,
+ is_manifest_name,
+ load_manifest,
+ manifest_name,
+)
+from marimo._save.stores.file import FileStore
+from tests._runtime._helpers.factories import default_app_metadata
+from tests._runtime._helpers.session import mocked_kernel_session
+
+if TYPE_CHECKING:
+ from collections.abc import Iterator
+
+ from marimo._runtime.runtime import Kernel
+ from marimo._save.stores import Store
+ from tests.conftest import ExecReqProvider
+
+NOTEBOOK_SOURCE = "import marimo\n\napp = marimo.App()\n"
+
+
+class TestManifestFormat:
+ @staticmethod
+ def test_round_trip() -> None:
+ manifest = CacheManifest(
+ notebook="../../nb.py",
+ nodes={"3f9c": {"train": {"C_ab12", "C_77e0"}}},
+ written_at="2026-09-01T00:00:00+00:00",
+ )
+ assert CacheManifest.from_bytes(manifest.to_bytes()) == manifest
+
+ @staticmethod
+ def test_merge_unions_keys() -> None:
+ manifest = CacheManifest(nodes={"3f9c": {"train": {"C_ab12"}}})
+ manifest.merge(
+ CacheManifest(
+ nodes={
+ "3f9c": {"train": {"C_77e0"}, "eval": {"E_0001"}},
+ "8d21": {"train": {"C_e4f9"}},
+ }
+ )
+ )
+ assert manifest.nodes == {
+ "3f9c": {"train": {"C_ab12", "C_77e0"}, "eval": {"E_0001"}},
+ "8d21": {"train": {"C_e4f9"}},
+ }
+
+ @staticmethod
+ def test_merge_does_not_alias_the_other_manifest() -> None:
+ records = {"3f9c": {"train": {"C_ab12"}}}
+ manifest = CacheManifest()
+ manifest.merge(CacheManifest(nodes=records))
+ records["3f9c"]["train"].add("C_77e0")
+ assert manifest.nodes == {"3f9c": {"train": {"C_ab12"}}}
+
+ @staticmethod
+ def test_evict_dead_returns_entries_and_keeps_live() -> None:
+ manifest = CacheManifest(
+ nodes={
+ "dead": {"train": {"C_ab12", "C_77e0"}},
+ "live": {"train": {"C_e4f9"}},
+ }
+ )
+ assert manifest.evict_dead({"live"}) == {
+ ("train", "C_ab12"),
+ ("train", "C_77e0"),
+ }
+ assert manifest.nodes == {"live": {"train": {"C_e4f9"}}}
+ assert manifest.entries() == {("train", "C_e4f9")}
+
+ @staticmethod
+ def test_corrupt_manifest_raises() -> None:
+ with pytest.raises(CorruptManifestError):
+ CacheManifest.from_bytes(b"{not json")
+ with pytest.raises(CorruptManifestError):
+ CacheManifest.from_bytes(b"[]")
+ # A manifest with no version cannot be read under any rules.
+ with pytest.raises(CorruptManifestError):
+ CacheManifest.from_bytes(b'{"nodes": {}}')
+ with pytest.raises(CorruptManifestError):
+ CacheManifest.from_bytes(
+ json.dumps(
+ {"version": 1, "nodes": {"3f9c": {"train": "C_ab12"}}}
+ ).encode()
+ )
+
+ @staticmethod
+ def test_newer_version_raises() -> None:
+ payload = json.dumps(
+ {"version": MANIFEST_VERSION + 1, "nodes": {}}
+ ).encode()
+ with pytest.raises(UnsupportedManifestVersionError):
+ CacheManifest.from_bytes(payload)
+
+ @staticmethod
+ def test_same_stem_notebooks_get_distinct_names(tmp_path: Path) -> None:
+ first = manifest_name(tmp_path / "a" / "nb.py")
+ second = manifest_name(tmp_path / "b" / "nb.py")
+ assert first != second
+ assert first.startswith("nb-")
+ assert first.endswith(".json")
+ # Relative and absolute spellings of one notebook agree.
+ assert manifest_name(Path("nb.py")) == manifest_name(
+ Path.cwd() / "nb.py"
+ )
+
+ @staticmethod
+ def test_a_long_stem_is_cut_to_fit_a_filename() -> None:
+ """The digest tells manifests apart. The stem is only a label."""
+ from marimo._save.manifest import MAX_SLUG_LENGTH
+
+ name = manifest_name(Path("a" * 240 + ".py"))
+ assert is_manifest_name(name)
+ assert len(name) <= MAX_SLUG_LENGTH + len("-") + 16 + len(".json")
+ assert name != manifest_name(Path("a" * 241 + ".py"))
+
+ @staticmethod
+ def test_a_notebook_path_that_is_not_utf8_gets_a_name() -> None:
+ """A filesystem hands over a name it cannot decode as surrogates."""
+ name = manifest_name(Path("/tmp/caf\udce9.py"))
+ assert is_manifest_name(name)
+ assert name != manifest_name(Path("/tmp/caf.py"))
+
+ @staticmethod
+ def test_only_a_cache_manifest_is_read_as_one() -> None:
+ from marimo._save.stores.file import export_manifest_name
+
+ assert is_manifest_name(manifest_name(Path("nb.py")))
+ # Other bookkeeping a cache directory holds, and an entry.
+ assert not is_manifest_name(export_manifest_name("nb.py"))
+ assert not is_manifest_name("nb-0123456789abcdef.json.tmp0a1b2c3d")
+ assert not is_manifest_name("C_ab12.pickle")
+
+ @staticmethod
+ def test_load_manifest_reads_what_put_stored(tmp_path: Path) -> None:
+ store = FileStore(save_path=str(tmp_path))
+ assert load_manifest(store, "absent.json") is None
+ manifest = CacheManifest(nodes={"3f9c": {"train": {"C_ab12"}}})
+ store.put("m.json", manifest.to_bytes())
+ loaded = load_manifest(store, "m.json")
+ assert loaded is not None
+ assert loaded.nodes == manifest.nodes
+
+
+class TestAtomicPut:
+ @staticmethod
+ def test_a_write_that_dies_midway_leaves_the_previous_value(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """The new value is written elsewhere and renamed in, so a write that
+ fails after its bytes are on disk still leaves a readable entry."""
+ store = FileStore(save_path=str(tmp_path))
+ store.put("entry", b"first")
+
+ def explode(*_: object) -> None:
+ raise OSError("no space left on device")
+
+ monkeypatch.setattr(os, "replace", explode)
+ with pytest.raises(OSError):
+ store.put("entry", b"second-and-much-longer")
+
+ assert store.get("entry") == b"first"
+ # The half-written value never appears under a name a reader looks up,
+ # and nothing is left behind.
+ assert [path.name for path in tmp_path.iterdir()] == ["entry"]
+
+ @staticmethod
+ def test_clearing_reclaims_what_a_killed_write_left(
+ tmp_path: Path,
+ ) -> None:
+ """Nothing runs after SIGKILL, so the leftover outlives the process
+ that wrote it; clearing the block is what reclaims its bytes."""
+ from marimo._save.cache_dirs import partial_write_name
+ from marimo._save.loaders import PickleLoader
+
+ store = FileStore(save_path=str(tmp_path))
+ block = tmp_path / "train"
+ block.mkdir()
+ (block / "C_ab12.pickle").write_bytes(b"value")
+ leftover = block / partial_write_name("C_77e0.pickle")
+ leftover.write_bytes(b"half")
+
+ PickleLoader("train", store=store).clear()
+ assert list(block.iterdir()) == []
+
+
+def _notebook_kernel(
+ tmp_path: Path, *, cache_cells: bool = False, store: Store | None = None
+) -> Iterator[tuple[Kernel, Path, Path]]:
+ """A kernel for a notebook on disk, caching beside it."""
+ notebook = tmp_path / "nb.py"
+ notebook.write_text(NOTEBOOK_SOURCE, encoding="utf-8")
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ user_config = None
+ if cache_cells:
+ user_config = copy.deepcopy(DEFAULT_CONFIG)
+ user_config["runtime"]["cache_cells"] = True
+ with mocked_kernel_session(
+ app_metadata=default_app_metadata(filename=str(notebook)),
+ user_config=user_config,
+ ) as tk:
+ # Pin the store. The ambient config can point a developer's cache
+ # somewhere else entirely.
+ tk.ctx.cache.store = store or FileStore(save_path=str(cache_dir))
+ yield tk.kernel, notebook, cache_dir
+
+
+def _manifest(notebook: Path, cache_dir: Path) -> dict[str, Any]:
+ return json.loads(
+ (cache_dir / manifest_name(notebook)).read_text(encoding="utf-8")
+ )
+
+
+def _entry_keys(manifest: dict[str, Any]) -> set[tuple[str, str]]:
+ return {
+ (block, key)
+ for blocks in manifest["nodes"].values()
+ for block, keys in blocks.items()
+ for key in keys
+ }
+
+
+def _recorded_puts(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[str]]:
+ """Collect the keys written through `FileStore` while running."""
+ original_put = FileStore.put
+ keys: list[str] = []
+
+ def record(self: FileStore, key: str, value: bytes) -> bool:
+ keys.append(key)
+ return original_put(self, key, value)
+
+ monkeypatch.setattr(FileStore, "put", record)
+ yield keys
+
+
+class TestRecording:
+ @staticmethod
+ async def test_with_block_records_its_entry(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ ]
+ )
+ assert not k.stderr.messages
+ assert k.globals["x"] == 42
+
+ manifest = _manifest(notebook, cache_dir)
+ assert manifest["version"] == MANIFEST_VERSION
+ assert Path(manifest["notebook"]) == Path("../../nb.py")
+ entries = _entry_keys(manifest)
+ assert len(entries) == 1
+ block, key = entries.pop()
+ assert block == "train"
+ assert (cache_dir / block / f"{key}.pickle").exists()
+
+ @staticmethod
+ async def test_records_accumulate_across_edits(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """A rerun under changed code adds a key. It never replaces one."""
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ setup = exec_req.get("import marimo as mo")
+ block = exec_req.get(
+ "epochs = 2\nwith mo.persistent_cache('train'):\n"
+ " model = epochs * 2\n"
+ )
+ await k.run([setup, block])
+ await k.run(
+ [
+ exec_req.get_with_id(
+ block.cell_id,
+ "epochs = 5\nwith mo.persistent_cache('train'):\n"
+ " model = epochs * 2\n",
+ )
+ ]
+ )
+ assert not k.stderr.messages
+
+ manifest = _manifest(notebook, cache_dir)
+ entries = _entry_keys(manifest)
+ assert len(entries) == 2
+ assert {block_name for block_name, _ in entries} == {"train"}
+
+ @staticmethod
+ async def test_records_under_the_defining_cell(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """A function cached in one cell and called from another records
+ against the cell that defines it, the one prune recompiles."""
+ from marimo._save.hash import hash_cell_closure
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ setup = exec_req.get("import marimo as mo")
+ define = exec_req.get(
+ "@mo.persistent_cache\ndef square(n):\n return n * n\n"
+ )
+ call = exec_req.get("answer = square(7)")
+ await k.run([setup, define, call])
+ assert not k.stderr.messages
+ assert k.globals["answer"] == 49
+
+ manifest = _manifest(notebook, cache_dir)
+ assert set(manifest["nodes"]) == {
+ hash_cell_closure(define.cell_id, k.graph).hex()
+ }
+ assert (
+ hash_cell_closure(call.cell_id, k.graph).hex()
+ not in (manifest["nodes"])
+ )
+
+ @staticmethod
+ async def test_records_merge_with_an_earlier_session(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """A session records what it produced without forgetting what an
+ earlier one, running other cells, left in the same directory."""
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ ]
+ )
+ written = _entry_keys(_manifest(notebook, cache_dir))
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('eval'):\n y = 7\n"
+ ),
+ ]
+ )
+ assert not k.stderr.messages
+ entries = _entry_keys(_manifest(notebook, cache_dir))
+
+ assert written < entries
+ assert {block for block, _ in entries} == {"train", "eval"}
+
+ @staticmethod
+ async def test_a_hit_records_too(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """A key first seen in an earlier session is recorded again on a hit,
+ so a run that only reads the cache still protects what it read."""
+ block = (
+ "with mo.persistent_cache('train'):\n"
+ " print('computed')\n"
+ " x = 42\n"
+ )
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [exec_req.get("import marimo as mo"), exec_req.get(block)]
+ )
+ assert k.stdout.messages.count("computed") == 1
+ written = _manifest(notebook, cache_dir)
+ (cache_dir / manifest_name(notebook)).unlink()
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [exec_req.get("import marimo as mo"), exec_req.get(block)]
+ )
+ assert not k.stderr.messages
+ # The body did not run: only the hit branch can have recorded.
+ assert not k.stdout.messages
+ assert k.globals["x"] == 42
+ assert _entry_keys(_manifest(notebook, cache_dir)) == _entry_keys(
+ written
+ )
+
+ @staticmethod
+ async def test_a_function_hit_records_too(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ source = "@mo.persistent_cache\ndef square(n):\n return n * n\n"
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(source),
+ exec_req.get("answer = square(7)"),
+ ]
+ )
+ assert not k.stderr.messages
+ written = _manifest(notebook, cache_dir)
+ (cache_dir / manifest_name(notebook)).unlink()
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(source),
+ exec_req.get("answer = square(7)"),
+ ]
+ )
+ assert not k.stderr.messages
+ assert k.globals["answer"] == 49
+ assert k.globals["square"].misses == 0
+ assert _entry_keys(_manifest(notebook, cache_dir)) == _entry_keys(
+ written
+ )
+
+ @staticmethod
+ async def test_an_async_hit_records_too(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ source = (
+ "@mo.persistent_cache\nasync def square(n):\n return n * n\n"
+ )
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(source),
+ exec_req.get("answer = await square(7)"),
+ ]
+ )
+ assert not k.stderr.messages
+ assert k.globals["answer"] == 49
+ written = _manifest(notebook, cache_dir)
+ (cache_dir / manifest_name(notebook)).unlink()
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(source),
+ exec_req.get("answer = await square(7)"),
+ ]
+ )
+ assert not k.stderr.messages
+ assert k.globals["square"].misses == 0
+ assert _entry_keys(_manifest(notebook, cache_dir)) == _entry_keys(
+ written
+ )
+
+ @staticmethod
+ async def test_a_manifest_is_written_once_for_a_run_of_new_keys(
+ tmp_path: Path,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """The manifest is rewritten whole, so a cell that fills a cache with
+ many keys must not rewrite it once per key, and a rerun that records
+ nothing new must not rewrite it at all."""
+
+ def manifest_writes(keys: list[str]) -> int:
+ return sum(is_manifest_name(key) for key in keys)
+
+ with _recorded_puts(monkeypatch) as keys:
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ call = exec_req.get(
+ "total = sum(square(n) for n in range(20))"
+ )
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "@mo.persistent_cache\n"
+ "def square(n):\n"
+ " return n * n\n"
+ ),
+ call,
+ ]
+ )
+ assert not k.stderr.messages
+ assert len(_entry_keys(_manifest(notebook, cache_dir))) == 20
+ assert manifest_writes(keys) == 1
+
+ await k.run([call])
+ assert not k.stderr.messages
+ assert k.globals["square"].misses == 20
+ assert manifest_writes(keys) == 1
+
+ @staticmethod
+ async def test_cell_caching_records_its_entries(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """Whole-cell caching writes entries of its own, under one block."""
+ with _notebook_kernel(tmp_path, cache_cells=True) as (
+ k,
+ notebook,
+ cache_dir,
+ ):
+ await k.run([exec_req.get("x = 42")])
+ assert not k.stderr.messages
+ entries = _entry_keys(_manifest(notebook, cache_dir))
+ assert {block for block, _ in entries} == {"cell_cache"}
+
+ @staticmethod
+ async def test_a_restored_cell_records_too(
+ tmp_path: Path,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """A session whose cells all restore from disk still records the
+ entries it restored from."""
+ cell = "x = 42"
+ with _notebook_kernel(tmp_path, cache_cells=True) as (
+ k,
+ notebook,
+ cache_dir,
+ ):
+ await k.run([exec_req.get(cell)])
+ written = _manifest(notebook, cache_dir)
+ (cache_dir / manifest_name(notebook)).unlink()
+
+ with _recorded_puts(monkeypatch) as keys:
+ with _notebook_kernel(tmp_path, cache_cells=True) as (
+ k,
+ notebook,
+ cache_dir,
+ ):
+ await k.run([exec_req.get(cell)])
+ assert not k.stderr.messages
+ assert _entry_keys(
+ _manifest(notebook, cache_dir)
+ ) == _entry_keys(written)
+
+ # Nothing but the manifest was written: the cell was restored, so only
+ # the hit branch can have recorded its entry.
+ assert all(is_manifest_name(key) for key in keys)
+
+ @staticmethod
+ async def test_teardown_writes_what_a_failed_flush_left_pending(
+ tmp_path: Path,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """A store that refuses one write must not lose the record."""
+ original_put = FileStore.put
+ refused: list[str] = []
+
+ def refuse_first_manifest(
+ self: FileStore, key: str, value: bytes
+ ) -> bool:
+ if is_manifest_name(key) and not refused:
+ refused.append(key)
+ raise OSError("no space left on device")
+ return original_put(self, key, value)
+
+ monkeypatch.setattr(FileStore, "put", refuse_first_manifest)
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ ]
+ )
+ assert refused
+ assert not (cache_dir / manifest_name(notebook)).exists()
+
+ assert _entry_keys(_manifest(notebook, cache_dir))
+
+ @staticmethod
+ async def test_a_declined_write_is_a_failed_flush_too(
+ tmp_path: Path,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """A store can decline a write by returning False rather than
+ raising. The record is kept pending all the same."""
+ original_put = FileStore.put
+ declined: list[str] = []
+
+ def decline_first_manifest(
+ self: FileStore, key: str, value: bytes
+ ) -> bool:
+ if is_manifest_name(key) and not declined:
+ declined.append(key)
+ return False
+ return original_put(self, key, value)
+
+ monkeypatch.setattr(FileStore, "put", decline_first_manifest)
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ ]
+ )
+ assert declined
+ assert not (cache_dir / manifest_name(notebook)).exists()
+
+ assert _entry_keys(_manifest(notebook, cache_dir))
+
+ @staticmethod
+ async def test_a_saved_entry_is_recorded_again_after_a_clean(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """`marimo cache clean` can take an entry and its record away in the
+ middle of a session. A rerun that writes the entry again records
+ it again, however often the session saw it before."""
+ cell = "with mo.persistent_cache('train'):\n x = 42\n"
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ cached = exec_req.get(cell)
+ await k.run([exec_req.get("import marimo as mo"), cached])
+ assert _entry_keys(_manifest(notebook, cache_dir))
+ # What `marimo cache clean nb.py` leaves behind.
+ shutil.rmtree(cache_dir / "train")
+ (cache_dir / manifest_name(notebook)).write_bytes(
+ CacheManifest().to_bytes()
+ )
+
+ await k.run([exec_req.get_with_id(cached.cell_id, cell)])
+
+ assert _entry_keys(_manifest(notebook, cache_dir))
+
+ @staticmethod
+ async def test_a_failed_flush_retries_where_the_entry_lives(
+ tmp_path: Path,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """The retry writes the manifest beside the entries it lists, not
+ into whichever cache directory the session happens to default to."""
+ original_put = FileStore.put
+ elsewhere = tmp_path / "elsewhere"
+ refused: list[str] = []
+
+ def refuse_first_manifest(
+ self: FileStore, key: str, value: bytes
+ ) -> bool:
+ if is_manifest_name(key) and not refused:
+ refused.append(key)
+ raise OSError("no space left on device")
+ return original_put(self, key, value)
+
+ monkeypatch.setattr(FileStore, "put", refuse_first_manifest)
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train', save_path="
+ f"{str(elsewhere)!r}):\n x = 42\n"
+ ),
+ ]
+ )
+ assert refused
+
+ assert _entry_keys(_manifest(notebook, elsewhere))
+ assert not (cache_dir / manifest_name(notebook)).exists()
+
+ @staticmethod
+ async def test_each_cache_location_gets_its_own_manifest(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """A manifest lists only the entries stored beside it. A prune reads
+ it as a statement about its own directory."""
+ elsewhere = tmp_path / "elsewhere"
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('here'):\n a = 1\n"
+ ),
+ exec_req.get(
+ "with mo.persistent_cache('there', save_path="
+ f"{str(elsewhere)!r}):\n b = 2\n"
+ ),
+ ]
+ )
+ assert not k.stderr.messages
+
+ here = _entry_keys(_manifest(notebook, cache_dir))
+ there = _entry_keys(_manifest(notebook, elsewhere))
+ assert {block for block, _ in here} == {"here"}
+ assert {block for block, _ in there} == {"there"}
+ for block, key in here | there:
+ directory = cache_dir if block == "here" else elsewhere
+ assert (directory / block / f"{key}.pickle").exists()
+
+ @staticmethod
+ async def test_every_copy_of_a_manifest_finds_the_notebook(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """A tiered store puts each entry, manifest included, in every tier.
+ The notebook a manifest names has to be reachable from whichever
+ directory the copy was read out of."""
+ from marimo._save.stores.tiered import TieredStore
+
+ fast = tmp_path / "fast"
+ with _notebook_kernel(
+ tmp_path,
+ store=TieredStore(
+ [
+ FileStore(save_path=str(fast)),
+ FileStore(
+ save_path=str(tmp_path / "__marimo__" / "cache")
+ ),
+ ]
+ ),
+ ) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ ]
+ )
+ assert not k.stderr.messages
+
+ for directory in (fast, cache_dir):
+ manifest = _manifest(notebook, directory)
+ recorded = directory / manifest["notebook"]
+ assert recorded.resolve() == notebook.resolve()
+
+ @staticmethod
+ async def test_a_corrupt_manifest_is_rewritten(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """Bytes that read as no manifest list nothing to preserve."""
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ cache_dir.mkdir(parents=True, exist_ok=True)
+ (cache_dir / manifest_name(notebook)).write_bytes(b"{not json")
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ ]
+ )
+ assert not k.stderr.messages
+ manifest = _manifest(notebook, cache_dir)
+ assert manifest["version"] == MANIFEST_VERSION
+ assert len(_entry_keys(manifest)) == 1
+
+ @staticmethod
+ async def test_a_newer_manifest_is_left_alone(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ """A manifest from a newer marimo holds records under rules this one
+ does not know. A rewrite drops them silently."""
+ payload = json.dumps(
+ {
+ "version": MANIFEST_VERSION + 1,
+ "nodes": {"3f9c": {"train": ["C_ab12"]}},
+ }
+ ).encode()
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ cache_dir.mkdir(parents=True, exist_ok=True)
+ (cache_dir / manifest_name(notebook)).write_bytes(payload)
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ ]
+ )
+ assert not k.stderr.messages
+ assert k.globals["x"] == 42
+
+ assert (cache_dir / manifest_name(notebook)).read_bytes() == payload
+
+ @staticmethod
+ async def test_an_unreadable_manifest_does_not_fail_the_cell(
+ tmp_path: Path,
+ exec_req: ExecReqProvider,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """Bookkeeping about a cache must never break the run it happens in."""
+ original_get = FileStore.get
+
+ def refuse_manifest(self: FileStore, key: str) -> bytes | None:
+ if is_manifest_name(key):
+ raise PermissionError(f"unreadable: {key}")
+ return original_get(self, key)
+
+ monkeypatch.setattr(FileStore, "get", refuse_manifest)
+
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "with mo.persistent_cache('train'):\n x = 42\n"
+ ),
+ exec_req.get("downstream = x + 1"),
+ ]
+ )
+ assert not k.stderr.messages
+ assert k.globals["x"] == 42
+ assert k.globals["downstream"] == 43
+ assert not (cache_dir / manifest_name(notebook)).exists()
+
+ @staticmethod
+ async def test_session_cache_records_nothing(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run(
+ [
+ exec_req.get("import marimo as mo"),
+ exec_req.get(
+ "@mo.cache\ndef square(n):\n return n * n\n"
+ ),
+ exec_req.get("answer = square(7)"),
+ ]
+ )
+ assert not k.stderr.messages
+ assert k.globals["answer"] == 49
+ assert not (cache_dir / manifest_name(notebook)).exists()
+
+ @staticmethod
+ async def test_no_manifest_without_a_persistent_event(
+ tmp_path: Path, exec_req: ExecReqProvider
+ ) -> None:
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run([exec_req.get("x = 42")])
+ assert not k.stderr.messages
+ assert not (cache_dir / manifest_name(notebook)).exists()
+
+ @staticmethod
+ async def test_scratchpad_records_nothing(tmp_path: Path) -> None:
+ """A scratchpad cell is in no notebook graph, so no manifest can list
+ its entries and prune must never see them."""
+ with _notebook_kernel(tmp_path) as (k, notebook, cache_dir):
+ await k.run_scratchpad(
+ "import marimo as mo\n"
+ "with mo.persistent_cache('scratch'):\n"
+ " x = 42\n"
+ )
+ assert not k.stderr.messages
+ # The entry is on disk. Nothing in the notebook records it.
+ assert list((cache_dir / "scratch").iterdir())
+ assert not (cache_dir / manifest_name(notebook)).exists()