Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: marimo cache size (#10837)

Dylan Madisetti3dbc007e164e
diff --git a/docs/guides/expensive_notebooks.md b/docs/guides/expensive_notebooks.md
index 91f351f56d..b8948bddd7 100644
--- a/docs/guides/expensive_notebooks.md
+++ b/docs/guides/expensive_notebooks.md
@@ -202,6 +202,15 @@ from the given folder.
///
+### Check disk usage
+
+```bash
+marimo cache size -r notebooks/
+```
+
+Run `marimo cache size` to print the disk usage and entry count for each
+cache directory, plus a total when there is more than one.
+
## Lazy-load expensive UIs
Lazily render UI elements that are expensive to compute using
diff --git a/marimo/_cli/cache/commands.py b/marimo/_cli/cache/commands.py
index 22fd8bd3b9..2384fed24e 100644
--- a/marimo/_cli/cache/commands.py
+++ b/marimo/_cli/cache/commands.py
@@ -38,6 +38,27 @@ def format_dir(cache_dir: Path) -> str:
return f"{cache_dir} (does not exist)"
+_BYTE_UNITS = ("B", "KB", "MB", "GB", "TB", "PB")
+
+
+def format_bytes(size: int) -> str:
+ """Render a byte count in binary units, where 1 KB is 1024 B."""
+ value = float(size)
+ unit = 0
+ # Round before comparing, so a value that renders as a full 1024 of its
+ # unit moves up instead of printing as "1024.0".
+ while round(value, 1) >= 1024 and unit < len(_BYTE_UNITS) - 1:
+ value /= 1024
+ unit += 1
+ if unit == 0:
+ return f"{size} B"
+ return f"{value:.1f} {_BYTE_UNITS[unit]}"
+
+
+def format_entries(entries: int) -> str:
+ return f"{entries} entry" if entries == 1 else f"{entries} entries"
+
+
def resolve_cache_dirs_or_error(path: Path, recursive: bool) -> list[Path]:
"""Resolve PATH, reporting a resolution failure as a CLI error."""
from marimo._save.cache_dirs import CacheDirError, resolve_cache_dirs
@@ -123,3 +144,45 @@ def cache_dir(path: Path, recursive: bool) -> None:
for cache_directory in cache_dirs:
click.echo(format_dir(cache_directory))
+
+
+ name="size",
+ help="""Print disk usage and entry counts for the cache.
+
+Prints the disk usage and entry count for each cache directory that
+PATH resolves to, and a total when there is more than one. Accepts
+`-r`/`--recursive` to search PATH recursively.
+
+Example usage:
+
+ marimo cache size
+
+ marimo cache size -r notebooks/
+""",
+)
+@path_argument
+@recursive_option
+def cache_size(path: Path, recursive: bool) -> None:
+ from marimo._save.cache_dirs import CacheDirStats, cache_dir_stats
+
+ cache_dirs = resolve_cache_dirs_or_error(path, recursive)
+ if not cache_dirs:
+ click.echo("No cache directories found.")
+ return
+
+ total = CacheDirStats()
+ for cache_directory in cache_dirs:
+ stats = cache_dir_stats(cache_directory)
+ total += stats
+ label = format_dir(cache_directory)
+ click.echo(
+ f"{label}\t{format_bytes(stats.total_bytes)}"
+ f"\t{format_entries(stats.entries)}"
+ )
+
+ if len(cache_dirs) > 1:
+ click.echo(
+ f"Total\t{format_bytes(total.total_bytes)}"
+ f"\t{format_entries(total.entries)}"
+ )
diff --git a/marimo/_runtime/callbacks/cache.py b/marimo/_runtime/callbacks/cache.py
index 29591a84e5..a272e80500 100644
--- a/marimo/_runtime/callbacks/cache.py
+++ b/marimo/_runtime/callbacks/cache.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, cast
+from marimo import _loggers
from marimo._messaging.notification import (
CacheClearedNotification,
CacheInfoNotification,
@@ -17,6 +18,22 @@ if TYPE_CHECKING:
from marimo._config.config import MarimoConfig
from marimo._runtime.callbacks.protocol import GlobalsView
from marimo._runtime.request_router import RequestRouter
+ from marimo._save.cache import CacheContext
+ from marimo._save.loaders.loader import Loader
+
+LOGGER = _loggers.marimo_logger()
+
+
+def _cache_loader(context: CacheContext) -> Loader | None:
+ """The loader `context` reads and writes through, if one is bound yet.
+
+ A decorator not yet applied to a function is a cache in the notebook's
+ globals with nothing behind it.
+ """
+ try:
+ return context.loader
+ except Exception:
+ return None
def cache_cells_enabled(user_config: MarimoConfig) -> bool:
@@ -68,15 +85,57 @@ class CacheCallbacks:
broadcast_notification(CacheClearedNotification(bytes_freed=saved))
+ def _disk_usage(self, loaders: list[Loader]) -> tuple[int, int]:
+ """Bytes the caches in scope hold, and bytes a purge frees.
+
+ Every number comes from the loaders themselves rather than from a
+ directory derived from the notebook path, so a cache configured to
+ write somewhere else is measured where it actually writes, and a
+ same-named directory it does not use is left out. The two differ
+ because a cache directory also holds caches this session never
+ loaded, which purging leaves alone. Measuring is best-effort. An
+ unreadable or unwritten cache reports zero rather than failing the
+ request.
+ """
+ try:
+ from marimo._save.cache_dirs import cache_dir_stats, entry_bytes
+ from marimo._utils.paths import normalize_path
+
+ # Keyed by normalized path. Two loaders can name one directory
+ # by different spellings, and its bytes are held once.
+ cache_dirs = {
+ normalize_path(directory)
+ for loader in loaders
+ for directory in loader.storage_dirs()
+ }
+ # Two names can land in one block once made path components, as
+ # `train/v2` and `train:v2` do. Each file is promised once.
+ clearable = {
+ normalize_path(path)
+ for loader in loaders
+ for path in loader.clearable_paths()
+ }
+ return (
+ sum(
+ cache_dir_stats(directory).total_bytes
+ for directory in cache_dirs
+ ),
+ sum(entry_bytes(path) for path in clearable),
+ )
+ except Exception:
+ LOGGER.debug("Could not measure cache disk usage", exc_info=True)
+ return 0, 0
+
async def get_cache_info(self, request: GetCacheInfoCommand) -> None:
del request
from marimo._save.cache import CacheContext
total_hits = 0
total_misses = 0
- total_time = 0
- disk_to_free = -1 # TODO: sum up disk usage
- disk_total = -1
+ total_time = 0.0
+ # Keyed by identity. One loader reached through several names is one
+ # cache, and its bytes must not be counted once per name.
+ loaders: dict[int, Loader] = {}
for obj in self._scope.globals.values():
if isinstance(obj, CacheContext):
@@ -84,7 +143,11 @@ class CacheCallbacks:
total_hits += hits
total_misses += misses
total_time += time
- # d2f, dt = obj.loader.disk_usage()
+ loader = _cache_loader(obj)
+ if loader is not None:
+ loaders[id(loader)] = loader
+
+ disk_total, disk_to_free = self._disk_usage(list(loaders.values()))
broadcast_notification(
CacheInfoNotification(
hits=total_hits,
diff --git a/marimo/_save/cache_dirs.py b/marimo/_save/cache_dirs.py
index 81fec0ec9d..aefa282839 100644
--- a/marimo/_save/cache_dirs.py
+++ b/marimo/_save/cache_dirs.py
@@ -7,6 +7,7 @@ Pure and kernel-free so the CLI can import it without starting a runtime.
from __future__ import annotations
import os
+from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
@@ -94,6 +95,114 @@ def resolve_cache_dirs(path: Path, recursive: bool = False) -> list[Path]:
return [notebook_cache_dir(path)]
+@dataclass(frozen=True)
+class CacheDirStats:
+ """Disk usage of a cache directory."""
+
+ total_bytes: int = 0
+ entries: int = 0
+
+ def __add__(self, other: CacheDirStats) -> CacheDirStats:
+ return CacheDirStats(
+ total_bytes=self.total_bytes + other.total_bytes,
+ entries=self.entries + other.entries,
+ )
+
+
+def cache_dir_stats(cache_dir: Path) -> CacheDirStats:
+ """Return the bytes and entry count held by `cache_dir`.
+
+ An entry is one cached value. It is a file in a block directory, together
+ with the directory of blobs that a value too large to inline is split
+ over.
+ Files at the root of the cache directory, such as manifests, hold no
+ cached value and so add bytes without adding entries.
+
+ A directory that does not exist, or that cannot be read, reports zero.
+ """
+ if not cache_dir.is_dir():
+ return CacheDirStats()
+
+ stats = CacheDirStats()
+ for block in _children(cache_dir):
+ if _is_directory(block):
+ stats += _block_stats(block)
+ else:
+ stats += CacheDirStats(total_bytes=_file_bytes(block))
+ return stats
+
+
+def entry_bytes(entry: Path) -> int:
+ """Return the bytes a cache entry occupies.
+
+ An entry is a file, or a directory holding the blobs of one value; either
+ one that cannot be read counts as nothing.
+ """
+ if not _is_directory(entry):
+ return _file_bytes(entry)
+ total = 0
+ for dirpath, _dirnames, filenames in os.walk(
+ entry, onerror=_log_walk_error
+ ):
+ total += sum(_file_bytes(Path(dirpath) / name) for name in filenames)
+ return total
+
+
+def _block_stats(block: Path) -> CacheDirStats:
+ """Return the bytes and entry count of one block directory."""
+ children = _children(block)
+ # A value stored in pieces leaves an entry file next to a directory of
+ # blobs named after the same hash. Counting that pair once keeps an entry
+ # equal to a cached value.
+ hashes = {
+ _entry_hash(child.name)
+ for child in children
+ if not _is_directory(child)
+ }
+ total_bytes = 0
+ entries = 0
+ for child in children:
+ total_bytes += entry_bytes(child)
+ if not (_is_directory(child) and child.name in hashes):
+ entries += 1
+ return CacheDirStats(total_bytes=total_bytes, entries=entries)
+
+
+def _entry_hash(entry_name: str) -> str:
+ """Return the hash an entry file names, without its prefix or suffix."""
+ stem = entry_name.split(".", 1)[0]
+ if len(stem) > 2 and stem[1] == "_":
+ return stem[2:]
+ return stem
+
+
+def _is_directory(path: Path) -> bool:
+ # A symlinked directory is never descended into: it is measured as the
+ # link it is.
+ return path.is_dir() and not path.is_symlink()
+
+
+def _children(directory: Path) -> list[Path]:
+ try:
+ return list(directory.iterdir())
+ except OSError as e:
+ LOGGER.warning("Skipping %s: %s", directory, e)
+ return []
+
+
+def _file_bytes(path: Path) -> int:
+ try:
+ # lstat: a symlink counts as itself, so a link out of the cache is not
+ # reported as space the cache holds.
+ return path.lstat().st_size
+ except FileNotFoundError:
+ # A running kernel writes and replaces entries underneath the walk.
+ return 0
+ except OSError as e:
+ LOGGER.warning("Skipping %s: %s", path, e)
+ return 0
+
+
def _walk_cache_dirs(root: Path) -> list[Path]:
# Normalize first: the search recognizes a cache directory by the names of
# its own path components, which a relative path such as "." lacks.
diff --git a/marimo/_save/loaders/loader.py b/marimo/_save/loaders/loader.py
index fad69b607e..5850763d17 100644
--- a/marimo/_save/loaders/loader.py
+++ b/marimo/_save/loaders/loader.py
@@ -207,6 +207,22 @@ class Loader(ABC):
# Default implementation: no-op for loaders that don't support clearing
return
+ def storage_dirs(self) -> list[Path]:
+ """Cache directories this loader writes its block into.
+
+ Empty when the cache keeps nothing on the local filesystem, so there
+ is no disk usage to attribute to it.
+ """
+ return []
+
+ def clearable_paths(self) -> list[Path]:
+ """Files and directories on disk that `clear()` removes."""
+ return []
+
+ def clearable_bytes(self) -> int:
+ """Bytes on disk that `clear()` frees."""
+ return 0
+
class BasePersistenceLoader(Loader):
"""Abstract base for cache written to disk."""
@@ -265,22 +281,46 @@ class BasePersistenceLoader(Loader):
except FileNotFoundError as e:
raise LoaderError("Unexpected cache miss.") from e
- def clear(self) -> None:
- """Clear all cached items for this loader."""
- # Clear all files in the loader's directory
- import glob
+ def storage_dirs(self) -> list[Path]:
+ return self.store.local_dirs()
+
+ def _clearable_root(self) -> Path | None:
+ """Root directory `clear()` removes this loader's entry files under.
+ `None` unless the store itself maps every key to a path below a root
+ it owns; anything else, a wrapper or a remote store included, holds
+ entries that cannot be enumerated as paths.
+ """
from marimo._save.stores.file import FileStore
- # Only FileStore has save_path, so we need to check
- if not isinstance(self.store, FileStore):
+ if isinstance(self.store, FileStore):
+ # A file store keeps its entries in exactly one directory.
+ return self.store.local_dirs()[0]
+ return None
+
+ def _clearable_paths(self, root: Path) -> list[Path]:
+ """Paths under `root` that `clear()` removes."""
+ import glob
+
+ pattern = str(root / self.name / f"*.{self.suffix}")
+ return [Path(match) for match in glob.glob(pattern)]
+
+ def clear(self) -> None:
+ """Clear all cached items for this loader."""
+ root = self._clearable_root()
+ if root is None:
return
+ for cache_file in self._clearable_paths(root):
+ self.store.clear(str(cache_file.relative_to(root)))
+
+ def clearable_paths(self) -> list[Path]:
+ root = self._clearable_root()
+ return [] if root is None else self._clearable_paths(root)
+
+ def clearable_bytes(self) -> int:
+ from marimo._save.cache_dirs import entry_bytes
- pattern = str(Path(self.name) / f"*.{self.suffix}")
- # Get all matching cache files through the store's base path
- for cache_file in glob.glob(str(self.store.save_path / pattern)):
- key = str(Path(cache_file).relative_to(self.store.save_path))
- self.store.clear(key)
+ return sum(entry_bytes(path) for path in self.clearable_paths())
@abstractmethod
def restore_cache(self, key: HashKey, blob: bytes) -> Cache:
diff --git a/tests/_cli/test_cli_cache.py b/tests/_cli/test_cli_cache.py
index d2f85ed617..1f6d1b3956 100644
--- a/tests/_cli/test_cli_cache.py
+++ b/tests/_cli/test_cli_cache.py
@@ -5,16 +5,15 @@ from __future__ import annotations
from typing import TYPE_CHECKING
+import pytest
from click.testing import CliRunner
-from marimo._cli.cache.commands import cache
+from marimo._cli.cache.commands import cache, format_bytes
from marimo._cli.cli import main
if TYPE_CHECKING:
from pathlib import Path
- import pytest
-
NOTEBOOK = """
import marimo
@@ -163,3 +162,83 @@ def test_dir_marks_a_file_in_the_way_of_the_cache_directory(
assert result.exit_code == 0, result.output
assert result.output == f"{blocker} (not a directory)\n"
+
+
+def write_entry(cache_dir: Path, block: str, name: str, size: int) -> None:
+ entry = cache_dir / block / name
+ entry.parent.mkdir(parents=True, exist_ok=True)
+ entry.write_bytes(b"x" * size)
+
+
+ ("size", "rendered"),
+ [
+ (0, "0 B"),
+ (512, "512 B"),
+ (1024, "1.0 KB"),
+ (1536, "1.5 KB"),
+ # A byte short of the next unit rounds up to it, never to "1024.0".
+ (1024**2 - 1, "1.0 MB"),
+ (1024**3 - 1, "1.0 GB"),
+ (3 * 1024 * 1024, "3.0 MB"),
+ (1024**5, "1.0 PB"),
+ # Nothing bigger than a petabyte to fall back on.
+ (2 * 1024**6, "2048.0 PB"),
+ ],
+)
+def test_format_bytes(size: int, rendered: str) -> None:
+ assert format_bytes(size) == rendered
+
+
+def test_size_reports_each_directory_and_a_total(tmp_path: Path) -> None:
+ first = make_cache_dir(tmp_path, "a")
+ write_entry(first, "train", "C_ab12.pickle", 2048)
+ second = make_cache_dir(tmp_path, "b")
+ write_entry(second, "train", "C_77e0.pickle", 1024)
+ write_entry(second, "train", "E_9f00.pickle", 1024)
+
+ result = CliRunner().invoke(main, ["cache", "size", "-r", str(tmp_path)])
+
+ assert result.exit_code == 0, result.output
+ assert result.output.splitlines() == [
+ f"{first}\t2.0 KB\t1 entry",
+ f"{second}\t2.0 KB\t2 entries",
+ "Total\t4.0 KB\t3 entries",
+ ]
+
+
+def test_size_of_a_single_directory_omits_the_total(tmp_path: Path) -> None:
+ cache_dir = make_cache_dir(tmp_path)
+ write_entry(cache_dir, "train", "C_ab12.pickle", 10)
+
+ result = CliRunner().invoke(main, ["cache", "size", str(tmp_path)])
+
+ assert result.exit_code == 0, result.output
+ assert result.output.splitlines() == [f"{cache_dir}\t10 B\t1 entry"]
+
+
+def test_size_of_a_cache_directory_that_does_not_exist(tmp_path: Path) -> None:
+ result = CliRunner().invoke(main, ["cache", "size", str(tmp_path)])
+
+ assert result.exit_code == 0, result.output
+ expected = tmp_path / "__marimo__" / "cache"
+ assert result.output == f"{expected} (does not exist)\t0 B\t0 entries\n"
+
+
+def test_size_of_a_notebook_path(tmp_path: Path) -> None:
+ notebook = tmp_path / "nb.py"
+ notebook.write_text(NOTEBOOK)
+ cache_dir = make_cache_dir(tmp_path)
+ write_entry(cache_dir, "train", "C_ab12.pickle", 4)
+
+ result = CliRunner().invoke(main, ["cache", "size", str(notebook)])
+
+ assert result.exit_code == 0, result.output
+ assert result.output == f"{cache_dir}\t4 B\t1 entry\n"
+
+
+def test_size_recursive_reports_no_cache_directories(tmp_path: Path) -> None:
+ result = CliRunner().invoke(main, ["cache", "size", "-r", str(tmp_path)])
+
+ assert result.exit_code == 0, result.output
+ assert result.output == "No cache directories found.\n"
diff --git a/tests/_runtime/test_cache_callbacks.py b/tests/_runtime/test_cache_callbacks.py
new file mode 100644
index 0000000000..bdfa30b973
--- /dev/null
+++ b/tests/_runtime/test_cache_callbacks.py
@@ -0,0 +1,263 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Cache statistics reported to the cache panel."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Any
+
+import marimo._runtime.callbacks.cache as cache_callbacks
+from marimo._runtime.callbacks.cache import CacheCallbacks
+from marimo._runtime.commands import ClearCacheCommand, GetCacheInfoCommand
+from marimo._runtime.state import State
+from marimo._save.cache import CacheContext
+from marimo._save.loaders.lazy import LazyLoader, LazyStore
+from marimo._save.loaders.memory import MemoryLoader
+from marimo._save.loaders.pickle import PickleLoader
+from marimo._save.stores.file import FileStore
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ import pytest
+
+ from marimo._messaging.notification import CacheInfoNotification
+ from marimo._save.loaders.loader import Loader
+
+
+class FakeCache(CacheContext):
+ """A cache as the notebook's globals hold one, without a kernel."""
+
+ __slots__ = ()
+
+ def __init__(self, loader: Loader | None) -> None:
+ self._loader = None if loader is None else State(loader)
+
+ @property
+ def last_hash(self) -> str | None:
+ return None
+
+
+def make_cache(cache_dir: Path, block: str) -> FakeCache:
+ return FakeCache(PickleLoader(block, store=FileStore(str(cache_dir))))
+
+
+def write_entry(cache_dir: Path, block: str, name: str, size: int) -> None:
+ entry = cache_dir / block / name
+ entry.parent.mkdir(parents=True, exist_ok=True)
+ entry.write_bytes(b"x" * size)
+
+
+def capture_notifications(
+ monkeypatch: pytest.MonkeyPatch,
+) -> list[CacheInfoNotification]:
+ sent: list[Any] = []
+ monkeypatch.setattr(cache_callbacks, "broadcast_notification", sent.append)
+ return sent
+
+
+async def test_get_cache_info_measures_the_loaders_cache_directory(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "train", "C_ab12.pickle", 100)
+ # A block no cache in scope owns. It counts as disk used, but purging
+ # the notebook's caches does not free it.
+ write_entry(cache_dir, "stale", "C_77e0.pickle", 40)
+ scope = SimpleNamespace(globals={"train": make_cache(cache_dir, "train")})
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ assert len(sent) == 1
+ assert sent[0].disk_total == 140
+ assert sent[0].disk_to_free == 100
+
+
+async def test_get_cache_info_follows_a_custom_save_path(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # `save_path=` moves a cache off the notebook-adjacent directory, and a
+ # same-named block left behind there belongs to no cache in scope.
+ elsewhere = tmp_path / "scratch"
+ write_entry(elsewhere, "train", "C_ab12.pickle", 100)
+ write_entry(tmp_path / "__marimo__" / "cache", "train", "C_77e0.pickle", 7)
+ scope = SimpleNamespace(globals={"train": make_cache(elsewhere, "train")})
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ assert sent[0].disk_total == 100
+ assert sent[0].disk_to_free == 100
+
+
+async def test_get_cache_info_reports_no_disk_for_a_memory_cache(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # An in-memory cache holds no disk, whatever a directory of the same name
+ # beside the notebook happens to hold.
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "train", "C_ab12.pickle", 100)
+ scope = SimpleNamespace(
+ globals={"train": FakeCache(MemoryLoader("train", max_size=10))}
+ )
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ assert sent[0].disk_total == 0
+ assert sent[0].disk_to_free == 0
+
+
+async def test_get_cache_info_promises_only_what_purging_frees(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "train", "C_ab12.pickle", 100)
+ # A value too large to inline is stored as a directory of blobs, which
+ # purging leaves behind.
+ write_entry(cache_dir, "train", "ab12/return.npy", 500)
+ cache = make_cache(cache_dir, "train")
+ scope = SimpleNamespace(globals={"train": cache})
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+ await callbacks.clear_cache(ClearCacheCommand())
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ promised = sent[0].disk_to_free
+ assert promised == 100
+ assert promised == sent[0].disk_total - sent[-1].disk_total
+ assert sent[-1].disk_to_free == 0
+
+
+async def test_get_cache_info_promises_a_shared_block_once(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Two names can land in one block once made path components. The
+ files there are promised once, not once per cache."""
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "train_v2", "C_ab12.pickle", 100)
+ scope = SimpleNamespace(
+ globals={
+ "a": make_cache(cache_dir, "train/v2"),
+ "b": make_cache(cache_dir, "train:v2"),
+ }
+ )
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ assert sent[0].disk_total == 100
+ assert sent[0].disk_to_free == 100
+
+
+async def test_get_cache_info_promises_nothing_a_lazy_cache_keeps(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # A lazy cache stores through a wrapper its loader cannot clear by path,
+ # so its bytes count as used but none of them as freeable.
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "cell_cache", "P_ab12.jsonl", 100)
+ write_entry(cache_dir, "cell_cache", "ab12/return.npy", 500)
+ loader = LazyLoader(
+ "cell_cache",
+ store=LazyStore(FileStore(str(cache_dir))),
+ verification="off",
+ )
+ scope = SimpleNamespace(globals={"cell_cache": FakeCache(loader)})
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+ await callbacks.clear_cache(ClearCacheCommand())
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ assert sent[0].disk_total == 600
+ assert sent[0].disk_to_free == 0
+ assert sent[-1].disk_total == 600
+
+
+async def test_get_cache_info_ignores_a_cache_without_a_loader(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # `mo.cache(pin_modules=True)` assigned to a name is a cache waiting for
+ # a function. Reading its loader raises.
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "train", "C_ab12.pickle", 100)
+ cache = make_cache(cache_dir, "train")
+ cache.loader._hits = 3
+ scope = SimpleNamespace(
+ globals={"train": cache, "unbound": FakeCache(None)}
+ )
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ assert len(sent) == 1
+ assert sent[0].hits == 3
+ assert sent[0].disk_total == 100
+
+
+async def test_get_cache_info_when_the_cache_cannot_be_measured(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def unreadable(cache_dir: Path) -> None:
+ raise OSError(f"Cannot read {cache_dir}")
+
+ monkeypatch.setattr("marimo._save.cache_dirs.cache_dir_stats", unreadable)
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "train", "C_ab12.pickle", 100)
+ cache = make_cache(cache_dir, "train")
+ cache.loader._hits = 2
+ scope = SimpleNamespace(globals={"train": cache})
+ sent = capture_notifications(monkeypatch)
+
+ callbacks = CacheCallbacks(
+ scope, notebook_filename=str(tmp_path / "nb.py")
+ )
+ await callbacks.get_cache_info(GetCacheInfoCommand())
+
+ assert len(sent) == 1
+ assert sent[0].hits == 2
+ assert sent[0].disk_total == 0
+ assert sent[0].disk_to_free == 0
+
+
+async def test_get_cache_info_without_a_notebook_filename(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # A notebook run from a buffer has no path to derive a cache directory
+ # from. The loader still knows where it writes.
+ cache_dir = tmp_path / "__marimo__" / "cache"
+ write_entry(cache_dir, "train", "C_ab12.pickle", 100)
+ cache = make_cache(cache_dir, "train")
+ cache.loader._hits = 3
+ scope = SimpleNamespace(globals={"train": cache})
+ sent = capture_notifications(monkeypatch)
+
+ await CacheCallbacks(scope).get_cache_info(GetCacheInfoCommand())
+
+ assert sent[0].hits == 3
+ assert sent[0].disk_total == 100
+ assert sent[0].disk_to_free == 100
diff --git a/tests/_save/test_cache_dirs.py b/tests/_save/test_cache_dirs.py
index 8a82322072..502b4ce3a3 100644
--- a/tests/_save/test_cache_dirs.py
+++ b/tests/_save/test_cache_dirs.py
@@ -10,8 +10,11 @@ import pytest
from marimo import _loggers
from marimo._save.cache_dirs import (
+ CacheDirStats,
NotANotebookError,
+ cache_dir_stats,
directory_cache_dir,
+ entry_bytes,
notebook_cache_dir,
resolve_cache_dirs,
)
@@ -287,3 +290,146 @@ def test_resolve_rejects_a_file_that_cannot_be_read_as_text(
with pytest.raises(NotANotebookError, match="not a notebook file"):
resolve_cache_dirs(pipe)
+
+
+def populate_cache_dir(cache_dir: Path) -> None:
+ """Write a cache directory in the shape the loaders leave behind."""
+ block = cache_dir / "train"
+ block.mkdir(parents=True)
+ (block / "C_ab12.pickle").write_bytes(b"x" * 10)
+ (block / "E_9f00.pickle").write_bytes(b"y" * 20)
+ # A lazily loaded value is split over a directory of blobs, named after
+ # the hash its entry file also names.
+ blob = block / "ab12"
+ blob.mkdir()
+ (blob / "return.npy").write_bytes(b"z" * 30)
+ (blob / "meta.json").write_bytes(b"w" * 40)
+ (cache_dir / ".nb-export.json").write_bytes(b"m" * 5)
+
+
+def test_cache_dir_stats_counts_an_entry_and_its_blobs_once(
+ tmp_path: Path,
+) -> None:
+ cache_dir = make_cache_dir(tmp_path)
+ populate_cache_dir(cache_dir)
+
+ assert cache_dir_stats(cache_dir) == CacheDirStats(
+ total_bytes=105, entries=2
+ )
+
+
+def test_cache_dir_stats_counts_an_orphaned_blob_directory(
+ tmp_path: Path,
+) -> None:
+ # Nothing names the hash, so the blobs are all that is left of the value.
+ cache_dir = make_cache_dir(tmp_path)
+ populate_cache_dir(cache_dir)
+ (cache_dir / "train" / "C_ab12.pickle").unlink()
+
+ assert cache_dir_stats(cache_dir) == CacheDirStats(
+ total_bytes=95, entries=2
+ )
+
+
+def test_cache_dir_stats_counts_several_blocks(tmp_path: Path) -> None:
+ cache_dir = make_cache_dir(tmp_path)
+ populate_cache_dir(cache_dir)
+ other = cache_dir / "evaluate"
+ other.mkdir()
+ (other / "C_77e0.pickle").write_bytes(b"x" * 7)
+
+ assert cache_dir_stats(cache_dir) == CacheDirStats(
+ total_bytes=112, entries=3
+ )
+
+
+ sys.platform == "win32", reason="chmod does not restrict reads on Windows"
+)
+def test_cache_dir_stats_skips_an_unreadable_block(
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cache_dir = make_cache_dir(tmp_path)
+ populate_cache_dir(cache_dir)
+ unreadable = cache_dir / "locked"
+ unreadable.mkdir()
+ (unreadable / "C_77e0.pickle").write_bytes(b"x" * 7)
+ unreadable.chmod(0o000)
+ if os.access(unreadable, os.R_OK): # running as root
+ unreadable.chmod(0o755)
+ pytest.skip("chmod does not restrict reads for this user")
+ monkeypatch.setattr(_loggers.marimo_logger(), "propagate", True)
+
+ try:
+ with caplog.at_level(logging.WARNING):
+ stats = cache_dir_stats(cache_dir)
+ finally:
+ unreadable.chmod(0o755)
+
+ assert stats == CacheDirStats(total_bytes=105, entries=2)
+ assert str(unreadable) in caplog.text
+
+
+ sys.platform == "win32", reason="symlinks need a privileged user"
+)
+def test_cache_dir_stats_counts_a_symlink_as_itself(tmp_path: Path) -> None:
+ # Deleting the cache would not free what the link points at.
+ outside = tmp_path / "checkpoint.bin"
+ outside.write_bytes(b"x" * 4096)
+ cache_dir = make_cache_dir(tmp_path)
+ block = cache_dir / "train"
+ block.mkdir()
+ link = block / "C_ab12.pickle"
+ link.symlink_to(outside)
+
+ stats = cache_dir_stats(cache_dir)
+
+ assert stats.entries == 1
+ assert stats.total_bytes == link.lstat().st_size
+ assert stats.total_bytes < 4096
+
+
+ sys.platform == "win32", reason="symlinks need a privileged user"
+)
+def test_cache_dir_stats_does_not_walk_a_linked_block(tmp_path: Path) -> None:
+ outside = tmp_path / "elsewhere"
+ outside.mkdir()
+ (outside / "C_ab12.pickle").write_bytes(b"x" * 4096)
+ cache_dir = make_cache_dir(tmp_path)
+ (cache_dir / "train").symlink_to(outside, target_is_directory=True)
+
+ stats = cache_dir_stats(cache_dir)
+
+ assert stats.entries == 0
+ assert stats.total_bytes < 4096
+
+
+def test_cache_dir_stats_of_an_empty_directory(tmp_path: Path) -> None:
+ assert cache_dir_stats(make_cache_dir(tmp_path)) == CacheDirStats()
+
+
+def test_cache_dir_stats_of_a_directory_that_does_not_exist(
+ tmp_path: Path,
+) -> None:
+ assert cache_dir_stats(tmp_path / "absent") == CacheDirStats()
+
+
+def test_cache_dir_stats_add() -> None:
+ left = CacheDirStats(total_bytes=10, entries=1)
+ right = CacheDirStats(total_bytes=5, entries=2)
+
+ assert left + right == CacheDirStats(total_bytes=15, entries=3)
+
+
+def test_entry_bytes(tmp_path: Path) -> None:
+ cache_dir = make_cache_dir(tmp_path)
+ populate_cache_dir(cache_dir)
+
+ assert entry_bytes(cache_dir / "train") == 100
+ assert entry_bytes(cache_dir / "train" / "ab12") == 70
+ assert entry_bytes(cache_dir / "train" / "C_ab12.pickle") == 10
+ assert entry_bytes(cache_dir / "never-written") == 0