Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Keep current results on the notebook and run history on the host (#11235)

Trevor Manz802eec04d666
diff --git a/marimo/_core/execution_state.py b/marimo/_core/execution_state.py
deleted file mode 100644
index de861ac455..0000000000
--- a/marimo/_core/execution_state.py
+++ /dev/null
@@ -1,420 +0,0 @@
-# Copyright 2026 Marimo. All rights reserved.
-"""Execution state for connecting and reconnecting clients.
-
-Reconnecting clients replay past events, so events are retained.
-Retaining all of them is impractical. `ExecutionState` applies the
-host's retention policy and emits removal events so connected clients
-can update their snapshots.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from math import isfinite
-from typing import TYPE_CHECKING
-
-from msgspec.structs import replace
-
-from marimo._core import backend as be, protocol
-from marimo._core.errors import NoRuntime
-from marimo._core.stream import Event, Stream, kind_of, message, utcnow
-
-if TYPE_CHECKING:
- from collections.abc import Callable
-
-Item = protocol.Console | protocol.Display
-
-
-@dataclass
-class _RetainedExecution:
- """An execution with the outputs it produced."""
-
- execution: protocol.Execution
- outputs: dict[protocol.ItemId, Item] = field(default_factory=dict)
-
-
-@dataclass(frozen=True)
-class RetentionPolicy:
- """How long the host retains ephemeral results after a run completes."""
-
- ephemeral_seconds: float = 60.0
-
- def __post_init__(self) -> None:
- if not isfinite(self.ephemeral_seconds) or self.ephemeral_seconds < 0:
- raise ValueError("invalid result retention")
-
-
-class ExecutionState:
- """Retain current results and remove completed runs with no executions."""
-
- def __init__(
- self,
- notebook_id: protocol.NotebookId,
- stream: Stream,
- retention: RetentionPolicy,
- clock: Callable[[], float],
- forget: Callable[[protocol.RunId], None],
- ) -> None:
- self._id = notebook_id
- self._stream = stream
- self._retention = retention
- self._clock = clock
- self._forget = forget
- self._runs: dict[str, protocol.Run] = {}
- self._run_executions: dict[str, set[protocol.ExecutionId]] = {}
- self._run_requests: dict[str, str | None] = {}
- self._executions: dict[protocol.ExecutionId, _RetainedExecution] = {}
- self._cell_executions: dict[protocol.CellId, protocol.ExecutionId] = {}
- self._retired_executions: dict[
- protocol.ExecutionId, protocol.RunId
- ] = {}
- self._unsuccessful: set[protocol.RunId] = set()
- self._completed_at: dict[str, float] = {}
-
- @property
- def id(self) -> protocol.NotebookId:
- return self._id
-
- @property
- def active_run(self) -> protocol.Run | None:
- return next(
- (r for r in self._runs.values() if r.status != "completed"), None
- )
-
- def run(self, run_id: str) -> protocol.Run | None:
- return self._runs.get(run_id)
-
- def add_run(self, run: protocol.Run, request_id: str) -> None:
- self._run_requests[run.id] = request_id
- self._set_run(run)
-
- def expire(self) -> None:
- now = self._clock()
- for retained in list(self._executions.values()):
- execution = retained.execution
- ended = self._completed_at.get(execution.run_id or "")
- if (
- execution.cell_id is None
- and ended is not None
- and now >= ended + self._retention.ephemeral_seconds
- ):
- self._retire_execution(retained)
-
- def _prune_run(self, run_id: str | None) -> None:
- run = self._runs.get(run_id or "")
- if run is None or run.status != "completed":
- return
- if self._run_executions[run.id]:
- return
- del self._runs[run.id]
- del self._run_executions[run.id]
- self._remove("run", run.id, run.id)
- self._run_requests.pop(run.id, None)
- self._completed_at.pop(run.id, None)
- self._forget(run.id)
-
- def snapshot(self) -> list[Event]:
- bodies: list[protocol.Run | protocol.Execution | Item] = [
- *self._runs.values(),
- *(retained.execution for retained in self._executions.values()),
- *(
- item
- for retained in self._executions.values()
- for item in retained.outputs.values()
- ),
- ]
- return [
- Event(
- kind_of(body),
- message(body, request_id=self._pending_request(body)),
- notebook_id=self.id,
- run_id=self._run_id(body),
- )
- for body in bodies
- ]
-
- def _pending_request(
- self, body: protocol.Run | protocol.Execution | Item
- ) -> str | None:
- run_id = self._run_id(body)
- run = self._runs.get(run_id or "")
- return (
- self._run_requests.get(run.id)
- if run is not None and run.status != "completed"
- else None
- )
-
- def observe(
- self, msg: be.Message, runtime: protocol.Runtime | None
- ) -> None:
- match msg:
- case be.Scheduled(run_id=run_id, caused_by=caused_by):
- if run_id not in self._runs:
- if runtime is None:
- raise NoRuntime(f"notebook {self.id} has no runtime")
- self._run_requests[run_id] = None
- self._set_run(
- protocol.Run(
- id=run_id,
- notebook_id=self.id,
- caused_by=caused_by,
- status="queued",
- outcome=None,
- started_at=None,
- completed_at=None,
- )
- )
- case be.Planned(run_id=run_id, executions=planned):
- self._observe_planned(run_id, planned, runtime)
- case be.Status(execution_id=execution_id, status=status):
- self._observe_status(execution_id, status)
- case be.Produced(item=item):
- if (
- retained := self._executions.get(item.execution_id)
- ) is not None:
- retained.outputs[item.id] = item
- self._publish(item)
- case be.Concluded(execution_id=execution_id, outcome=outcome):
- if (
- retained := self._executions.get(execution_id)
- ) is not None and retained.execution.status != "completed":
- self._set_execution(
- replace(retained.execution, outcome=outcome)
- )
- case be.Done(run_id=run_id, outcome=outcome):
- if (
- run := self._runs.get(run_id)
- ) is not None and run.status != "completed":
- members = [
- self._executions[eid].execution
- for eid in self._run_executions[run_id]
- ]
- if any(
- execution.status != "completed"
- for execution in members
- ):
- raise ValueError(
- "a run cannot complete with unfinished executions"
- )
- if outcome == "succeeded" and (
- run_id in self._unsuccessful
- or any(
- not isinstance(
- execution.outcome, protocol.ValueOutcome
- )
- for execution in members
- )
- ):
- raise ValueError(
- "a successful run cannot contain failed or interrupted executions"
- )
- self._set_run(
- replace(
- run,
- status="completed",
- outcome=outcome,
- completed_at=utcnow(),
- )
- )
- self._retired_executions = {
- eid: rid
- for eid, rid in self._retired_executions.items()
- if rid != run_id
- }
- self._unsuccessful.discard(run_id)
-
- self._prune_run(run_id)
-
- def clear(self) -> None:
- self._retired_executions.clear()
- self._unsuccessful.clear()
- for retained in list(self._executions.values()):
- self._remove_execution(retained)
- for run in list(self._runs.values()):
- del self._runs[run.id]
- del self._run_executions[run.id]
- self._remove("run", run.id, run.id)
- self._run_requests.pop(run.id, None)
- self._forget(run.id)
- self._completed_at.clear()
-
- def _set_run(self, run: protocol.Run) -> None:
- self._runs[run.id] = run
- self._run_executions.setdefault(run.id, set())
- if run.status == "completed":
- self._completed_at.setdefault(run.id, self._clock())
- self._publish(run)
-
- def _set_execution(self, execution: protocol.Execution) -> None:
- if (retained := self._executions.get(execution.id)) is None:
- self._executions[execution.id] = _RetainedExecution(execution)
- else:
- retained.execution = execution
- if execution.cell_id is not None:
- self._cell_executions[execution.cell_id] = execution.id
- if execution.run_id is not None:
- self._run_executions[execution.run_id].add(execution.id)
- self._publish(execution)
-
- def interrupt(self, outcome: protocol.RunOutcome = "interrupted") -> None:
- now = utcnow()
- for retained in list(self._executions.values()):
- execution = retained.execution
- if execution.status != "completed":
- self._set_execution(
- replace(
- execution,
- status="completed",
- outcome=protocol.InterruptedOutcome(),
- completed_at=now,
- )
- )
- for run in list(self._runs.values()):
- if run.status != "completed":
- self._set_run(
- replace(
- run,
- status="completed",
- outcome=outcome,
- completed_at=now,
- )
- )
- self._retired_executions.clear()
- self._unsuccessful.clear()
-
- for run_id in list(self._runs):
- self._prune_run(run_id)
-
- def _observe_planned(
- self,
- run_id: protocol.RunId,
- planned: tuple[be.PlannedExecution, ...],
- runtime: protocol.Runtime | None,
- ) -> None:
- if (
- run := self._runs.get(run_id)
- ) is None or run.status == "completed":
- return
- if runtime is None:
- raise NoRuntime(f"notebook {self.id} has no runtime")
- for entry in planned:
- if (
- entry.id in self._executions
- or entry.id in self._retired_executions
- ):
- continue
- if entry.cell_id is not None:
- previous = self._cell_executions.get(
- protocol.CellId(entry.cell_id)
- )
- if previous is not None:
- self._retire_execution(self._executions[previous])
- self._set_execution(
- protocol.Execution(
- id=entry.id,
- run_id=run_id,
- runtime_id=runtime.id,
- caused_by=entry.caused_by,
- cell_id=(
- protocol.CellId(entry.cell_id)
- if entry.cell_id is not None
- else None
- ),
- code=entry.code,
- code_hash=entry.code_hash,
- bindings=(),
- status="queued",
- outcome=None,
- stale=None,
- started_at=None,
- completed_at=None,
- )
- )
-
- def _retire_execution(self, retained: _RetainedExecution) -> None:
- execution = retained.execution
- if execution.status != "completed":
- raise ValueError("cannot replace an unfinished cell execution")
- run_id = execution.run_id
- run = self._runs.get(run_id) if run_id is not None else None
- if run is not None and run.status != "completed":
- self._retired_executions[execution.id] = run.id
- if not isinstance(execution.outcome, protocol.ValueOutcome):
- self._unsuccessful.add(run.id)
- self._remove_execution(retained)
- self._prune_run(run_id)
-
- def _remove_execution(self, retained: _RetainedExecution) -> None:
- execution = retained.execution
- run_id = execution.run_id
- for item in list(retained.outputs.values()):
- del retained.outputs[item.id]
- self._remove(kind_of(item), item.id, run_id)
- del self._executions[execution.id]
- if execution.cell_id is not None:
- del self._cell_executions[execution.cell_id]
- if run_id is not None:
- self._run_executions[run_id].remove(execution.id)
- self._remove("execution", execution.id, run_id)
-
- def _observe_status(
- self, execution_id: protocol.ExecutionId, status: protocol.Lifecycle
- ) -> None:
- retained = self._executions.get(execution_id)
- if retained is None:
- return
- execution = retained.execution
- if execution.status in (status, "completed"):
- return
- now = utcnow()
- if status == "running":
- execution = replace(execution, status=status, started_at=now)
- assert execution.run_id is not None
- run = self._runs[execution.run_id]
- if run.status == "queued":
- self._set_run(replace(run, status="running", started_at=now))
- elif status == "completed":
- if execution.outcome is None:
- raise ValueError("a completed execution must have an outcome")
- execution = replace(execution, status=status, completed_at=now)
- else:
- execution = replace(execution, status=status)
- self._set_execution(execution)
-
- def _publish(
- self,
- body: protocol.Run | protocol.Execution | Item,
- ) -> None:
- run_id = self._run_id(body)
- self._stream.publish(
- body,
- notebook_id=self.id,
- request_id=self._run_requests.get(run_id or ""),
- run_id=run_id,
- )
-
- def _run_id(
- self,
- body: protocol.Run | protocol.Execution | Item,
- ) -> protocol.RunId | None:
- if isinstance(body, protocol.Run):
- return body.id
- if isinstance(body, protocol.Execution):
- return body.run_id
- if isinstance(body, (protocol.Console, protocol.Display)):
- return self._executions[body.execution_id].execution.run_id
- return None
-
- def _remove(
- self,
- kind: str,
- object_id: str,
- run_id: protocol.RunId | None,
- ) -> None:
- self._stream.remove(
- kind,
- object_id,
- notebook_id=self.id,
- request_id=self._run_requests.get(run_id or ""),
- run_id=run_id,
- )
diff --git a/marimo/_core/executions.py b/marimo/_core/executions.py
new file mode 100644
index 0000000000..547c2cdab2
--- /dev/null
+++ b/marimo/_core/executions.py
@@ -0,0 +1,445 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Runs, cell results, and their outputs."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Literal
+
+from msgspec.structs import replace
+
+from marimo._core import backend as be, protocol
+from marimo._core.errors import NoRuntime
+from marimo._core.stream import utcnow
+
+
+@dataclass(frozen=True)
+class CellExecution:
+ """A completed cell execution with its console and display outputs.
+
+ Its source may be older than the cell's current source.
+ """
+
+ execution: protocol.Execution
+ outputs: tuple[protocol.Console | protocol.Display, ...] = ()
+
+
+@dataclass
+class _Run:
+ """A run in progress or retained by a cell result."""
+
+ run: protocol.Run
+ request_id: str | None = None
+ executions: set[protocol.ExecutionId] = field(default_factory=set)
+ # Cell executions replaced while the run was in progress, and whether
+ # each succeeded. A successful run cannot have replaced a failure.
+ retired: dict[protocol.ExecutionId, bool] = field(default_factory=dict)
+
+
+@dataclass
+class _Result:
+ """A current execution and its outputs."""
+
+ execution: protocol.Execution
+ outputs: dict[protocol.ItemId, protocol.Console | protocol.Display] = (
+ field(default_factory=dict)
+ )
+
+
+@dataclass(frozen=True)
+class Removed:
+ """A current record that a transition removed."""
+
+ kind: Literal["execution", "console", "display"]
+ id: str
+ run_id: protocol.RunId | None
+
+
+@dataclass(frozen=True)
+class Released:
+ """A run to release after applying its preceding changes.
+
+ Releasing emits no event. Earlier changes still use the run's request ID.
+ """
+
+ run_id: protocol.RunId
+
+
+Record = (
+ protocol.Run | protocol.Execution | protocol.Console | protocol.Display
+)
+Change = Record | Removed | Released
+
+
+class Executions:
+ """Runs in progress and each cell's current result.
+
+ Cell results remain until replaced or removed. Scratch executions
+ leave when their run completes.
+ """
+
+ def __init__(self) -> None:
+ self._runs: dict[protocol.RunId, _Run] = {}
+ self._results: dict[protocol.ExecutionId, _Result] = {}
+ self._cells: dict[protocol.CellId, protocol.ExecutionId] = {}
+
+ # ---- reads ---------------------------------------------------------
+
+ def current(self, cell_id: protocol.CellId) -> CellExecution | None:
+ """A copy of the cell's completed result, or `None` while it runs."""
+ execution_id = self._cells.get(cell_id)
+ current = (
+ self._results.get(execution_id)
+ if execution_id is not None
+ else None
+ )
+ if current is None or current.execution.status != "completed":
+ return None
+ return deepcopy(
+ CellExecution(current.execution, tuple(current.outputs.values()))
+ )
+
+ def run(self, run_id: protocol.RunId) -> protocol.Run | None:
+ tracked = self._runs.get(run_id)
+ return tracked.run if tracked is not None else None
+
+ def active(self) -> protocol.Run | None:
+ """The run in progress, if any."""
+ return next(
+ (
+ tracked.run
+ for tracked in self._runs.values()
+ if tracked.run.status != "completed"
+ ),
+ None,
+ )
+
+ def _records(self) -> list[Record]:
+ """Return detached current records, including unfinished executions."""
+ return deepcopy(
+ [
+ *(tracked.run for tracked in self._runs.values()),
+ *(result.execution for result in self._results.values()),
+ *(
+ item
+ for result in self._results.values()
+ for item in result.outputs.values()
+ ),
+ ]
+ )
+
+ def run_id_of(self, change: Record | Removed) -> protocol.RunId | None:
+ """Resolve a record or removal to the run it belongs to."""
+ if isinstance(change, protocol.Run):
+ return change.id
+ if isinstance(change, (protocol.Execution, Removed)):
+ return change.run_id
+ return self._results[change.execution_id].execution.run_id
+
+ def request_id(self, run_id: protocol.RunId | None) -> str | None:
+ """The request behind a run still tracked."""
+ tracked = self._runs.get(run_id) if run_id is not None else None
+ return tracked.request_id if tracked is not None else None
+
+ def snapshot_request(self, run_id: protocol.RunId | None) -> str | None:
+ """The request behind a run still in progress."""
+ tracked = self._runs.get(run_id) if run_id is not None else None
+ if tracked is not None and tracked.run.status != "completed":
+ return tracked.request_id
+ return None
+
+ # ---- transitions ---------------------------------------------------
+
+ def begin(self, run: protocol.Run, request_id: str) -> list[Change]:
+ """Track a run the notebook requested."""
+ self._runs[run.id] = _Run(run, request_id)
+ return [run]
+
+ def release(self, run_id: protocol.RunId) -> None:
+ """Stop tracking a released run."""
+ del self._runs[run_id]
+
+ def observe(
+ self, msg: be.Message, runtime: protocol.Runtime | None
+ ) -> list[Change]:
+ """Apply a backend update and return its changes in publication order."""
+ changes: list[Change] = []
+ match msg:
+ case be.Scheduled(run_id=run_id, caused_by=caused_by):
+ if run_id not in self._runs:
+ runtime = self._require(runtime)
+ changes.append(
+ self._set_run(
+ protocol.Run(
+ id=run_id,
+ notebook_id=runtime.notebook_id,
+ caused_by=caused_by,
+ status="queued",
+ outcome=None,
+ started_at=None,
+ completed_at=None,
+ )
+ )
+ )
+ case be.Planned(run_id=run_id, executions=planned):
+ changes.extend(self._observe_planned(run_id, planned, runtime))
+ case be.Status(execution_id=execution_id, status=status):
+ changes.extend(self._observe_status(execution_id, status))
+ case be.Produced(item=item):
+ if (
+ result := self._results.get(item.execution_id)
+ ) is not None:
+ result.outputs[item.id] = item
+ changes.append(item)
+ case be.Concluded(execution_id=execution_id, outcome=outcome):
+ if (
+ result := self._results.get(execution_id)
+ ) is not None and result.execution.status != "completed":
+ changes.append(
+ self._set_execution(
+ replace(result.execution, outcome=outcome)
+ )
+ )
+ case be.Done(run_id=run_id, outcome=outcome):
+ if (
+ tracked := self._runs.get(run_id)
+ ) is not None and tracked.run.status != "completed":
+ members = [
+ self._results[eid].execution
+ for eid in tracked.executions
+ ]
+ if any(
+ execution.status != "completed"
+ for execution in members
+ ):
+ raise ValueError(
+ "a run cannot complete with unfinished executions"
+ )
+ if outcome == "succeeded" and (
+ not all(tracked.retired.values())
+ or any(
+ not isinstance(
+ execution.outcome, protocol.ValueOutcome
+ )
+ for execution in members
+ )
+ ):
+ raise ValueError(
+ "a successful run cannot contain failed or interrupted executions"
+ )
+ changes.append(
+ self._set_run(
+ replace(
+ tracked.run,
+ status="completed",
+ outcome=outcome,
+ completed_at=utcnow(),
+ )
+ )
+ )
+ tracked.retired.clear()
+ changes.extend(self._finish_run(tracked))
+ return changes
+
+ def interrupt(
+ self, outcome: protocol.RunOutcome = "interrupted"
+ ) -> list[Change]:
+ """Complete every unfinished execution and run."""
+ changes: list[Change] = []
+ now = utcnow()
+ for result in list(self._results.values()):
+ if result.execution.status != "completed":
+ changes.append(
+ self._set_execution(
+ replace(
+ result.execution,
+ status="completed",
+ outcome=protocol.InterruptedOutcome(),
+ completed_at=now,
+ )
+ )
+ )
+ for tracked in list(self._runs.values()):
+ if tracked.run.status != "completed":
+ changes.append(
+ self._set_run(
+ replace(
+ tracked.run,
+ status="completed",
+ outcome=outcome,
+ completed_at=now,
+ )
+ )
+ )
+ tracked.retired.clear()
+ changes.extend(self._finish_run(tracked))
+ return changes
+
+ def clear(self) -> list[Change]:
+ """Remove every current execution and release every run."""
+ changes: list[Change] = []
+ for execution_id in list(self._results):
+ changes.extend(self._remove_execution(execution_id))
+ changes.extend(Released(run_id) for run_id in self._runs)
+ return changes
+
+ # ---- internals -----------------------------------------------------
+
+ @staticmethod
+ def _require(runtime: protocol.Runtime | None) -> protocol.Runtime:
+ if runtime is None:
+ raise NoRuntime("the notebook has no runtime; start one first.")
+ return runtime
+
+ def _observe_planned(
+ self,
+ run_id: protocol.RunId,
+ planned: tuple[be.PlannedExecution, ...],
+ runtime: protocol.Runtime | None,
+ ) -> list[Change]:
+ changes: list[Change] = []
+ if (
+ tracked := self._runs.get(run_id)
+ ) is None or tracked.run.status == "completed":
+ return []
+ runtime = self._require(runtime)
+ for entry in planned:
+ if entry.id in self._results or entry.id in tracked.retired:
+ continue
+ if entry.cell_id is not None:
+ previous = self._cells.get(protocol.CellId(entry.cell_id))
+ if previous is not None:
+ changes.extend(self._retire_execution(previous))
+ changes.append(
+ self._set_execution(
+ protocol.Execution(
+ id=entry.id,
+ run_id=run_id,
+ runtime_id=runtime.id,
+ caused_by=entry.caused_by,
+ cell_id=protocol.CellId(entry.cell_id)
+ if entry.cell_id is not None
+ else None,
+ code=entry.code,
+ code_hash=entry.code_hash,
+ bindings=(),
+ status="queued",
+ outcome=None,
+ stale=None,
+ started_at=None,
+ completed_at=None,
+ )
+ )
+ )
+ return changes
+
+ def _observe_status(
+ self, execution_id: protocol.ExecutionId, status: protocol.Lifecycle
+ ) -> list[Change]:
+ changes: list[Change] = []
+ result = self._results.get(execution_id)
+ if result is None:
+ return []
+ execution = result.execution
+ if execution.status in (status, "completed"):
+ return []
+ now = utcnow()
+ if status == "running":
+ execution = replace(execution, status=status, started_at=now)
+ assert execution.run_id is not None
+ run = self._runs[execution.run_id].run
+ if run.status == "queued":
+ changes.append(
+ self._set_run(
+ replace(run, status="running", started_at=now)
+ )
+ )
+ elif status == "completed":
+ if execution.outcome is None:
+ raise ValueError("a completed execution must have an outcome")
+ execution = replace(execution, status=status, completed_at=now)
+ else:
+ execution = replace(execution, status=status)
+ changes.append(self._set_execution(execution))
+ return changes
+
+ def _set_run(self, run: protocol.Run) -> protocol.Run:
+ self._runs.setdefault(run.id, _Run(run)).run = run
+ return run
+
+ def _set_execution(
+ self, execution: protocol.Execution
+ ) -> protocol.Execution:
+ result = self._results.get(execution.id)
+ if result is None:
+ self._results[execution.id] = _Result(execution)
+ else:
+ result.execution = execution
+ if execution.cell_id is not None:
+ self._cells[execution.cell_id] = execution.id
+ if execution.run_id is not None:
+ if (tracked := self._runs.get(execution.run_id)) is not None:
+ tracked.executions.add(execution.id)
+ return execution
+
+ def _finish_run(self, tracked: _Run) -> list[Released]:
+ """Release completed work that no cell retains."""
+ for execution_id in list(tracked.executions):
+ if self._results[execution_id].execution.cell_id is None:
+ self._forget_result(execution_id)
+ return self._prune_run(tracked)
+
+ def _prune_run(self, tracked: _Run) -> list[Released]:
+ if tracked.run.status != "completed" or tracked.executions:
+ return []
+ return [Released(tracked.run.id)]
+
+ def _retire_execution(
+ self, execution_id: protocol.ExecutionId
+ ) -> list[Change]:
+ """Replace a cell's current execution with a newer one."""
+ execution = self._results[execution_id].execution
+ if execution.status != "completed":
+ raise ValueError("cannot replace an unfinished cell execution")
+ tracked = (
+ self._runs.get(execution.run_id)
+ if execution.run_id is not None
+ else None
+ )
+ if tracked is not None and tracked.run.status != "completed":
+ tracked.retired[execution.id] = isinstance(
+ execution.outcome, protocol.ValueOutcome
+ )
+ changes: list[Change] = list(self._remove_execution(execution_id))
+ if tracked is not None:
+ changes.extend(self._prune_run(tracked))
+ return changes
+
+ def _remove_execution(
+ self, execution_id: protocol.ExecutionId
+ ) -> list[Removed]:
+ """Remove an execution and its outputs, outputs first."""
+ result = self._results[execution_id]
+ run_id = result.execution.run_id
+ changes: list[Removed] = [
+ Removed(
+ "console" if isinstance(item, protocol.Console) else "display",
+ item.id,
+ run_id,
+ )
+ for item in result.outputs.values()
+ ]
+ changes.append(Removed("execution", execution_id, run_id))
+ self._forget_result(execution_id)
+ return changes
+
+ def _forget_result(self, execution_id: protocol.ExecutionId) -> None:
+ result = self._results.pop(execution_id)
+ if result.execution.cell_id is not None:
+ del self._cells[result.execution.cell_id]
+ if result.execution.run_id is not None:
+ if (
+ tracked := self._runs.get(result.execution.run_id)
+ ) is not None:
+ tracked.executions.discard(execution_id)
diff --git a/marimo/_core/host.py b/marimo/_core/host.py
index 9c76b9973d..f58209acd0 100644
--- a/marimo/_core/host.py
+++ b/marimo/_core/host.py
@@ -5,22 +5,32 @@ from __future__ import annotations
from collections.abc import Mapping
from copy import deepcopy
+from dataclasses import dataclass
from time import monotonic
from typing import TYPE_CHECKING
from marimo._core import protocol
from marimo._core.backend import Backend
from marimo._core.errors import NotFound
-from marimo._core.execution_state import RetentionPolicy
from marimo._core.notebook import Notebook
+from marimo._core.run_log import RetentionPolicy, RunLog
from marimo._core.stream import Event, Listener, Stream, message
if TYPE_CHECKING:
from collections.abc import Callable, Collection, Iterator, Sequence
+@dataclass
+class _Entry:
+ """A notebook, the run history kept for it, and its subscription."""
+
+ notebook: Notebook
+ log: RunLog
+ unsubscribe: Callable[[], None]
+
+
class _NotebookRecords(Mapping[str, protocol.Notebook]):
- """A catalog view whose readers own their copies of mutable records."""
+ """Catalog records copied on read."""
def __init__(self, records: Mapping[str, protocol.Notebook]) -> None:
self._records = records
@@ -35,6 +45,11 @@ class _NotebookRecords(Mapping[str, protocol.Notebook]):
return len(self._records)
+def _identity(event: Event) -> tuple[str, str]:
+ assert event.kind is not None
+ return event.name, getattr(event.data, event.kind).id
+
+
class Host:
"""A host's project and notebook catalog, with events for changes."""
@@ -53,7 +68,7 @@ class Host:
# Lookups accept wire strings; stored objects carry branded IDs.
self._projects: dict[str, protocol.Project] = {}
self._records: dict[str, protocol.Notebook] = {}
- self._notebooks: dict[str, Notebook] = {}
+ self._entries: dict[str, _Entry] = {}
@property
def notebooks(self) -> Mapping[str, protocol.Notebook]:
@@ -80,18 +95,28 @@ class Host:
*,
request_id: str | None = None,
) -> Notebook:
- """Register a notebook and the backend that runs it."""
+ """Register a notebook and its backend."""
if record.id in self._records:
raise ValueError(f"notebook {record.id} is already in the catalog")
if record.project_id not in self._projects:
raise ValueError(
f"project {record.project_id} is not in the catalog"
)
- notebook = Notebook(
- record.id, backend, self._stream, self.retention, self.clock
- )
+ notebook = Notebook(record.id, backend)
+ log = RunLog(lambda: self.clock())
+
+ def publish(event: Event) -> None:
+ removals = log.observe(event)
+ self._stream.append(event)
+ for removal in removals:
+ self._stream.append(removal)
+
self._records[record.id] = deepcopy(record)
- self._notebooks[record.id] = notebook
+ self._entries[record.id] = _Entry(
+ notebook,
+ log,
+ notebook._stream.subscribe(publish, downstream=self._stream),
+ )
self._stream.publish(
record, notebook_id=record.id, request_id=request_id
)
@@ -118,34 +143,44 @@ class Host:
self, notebook_id: str, *, request_id: str | None = None
) -> None:
"""Remove a notebook, its runtime, attachments, and runs."""
- notebook = self._notebooks.pop(notebook_id, None)
- if notebook is None:
+ entry = self._entries.pop(notebook_id, None)
+ if entry is None:
return
- del self._records[notebook.id]
- notebook._dispose(request_id=request_id)
- self._stream.remove(
- "notebook",
- notebook.id,
- notebook_id=notebook.id,
- request_id=request_id,
- )
+ record = self._records.pop(notebook_id)
+ entry.notebook._dispose(request_id=request_id)
+
+ def finish() -> None:
+ with self._stream.deferred():
+ for event in entry.log.clear():
+ self._stream.append(event)
+ entry.unsubscribe()
+ self._stream.remove(
+ "notebook",
+ record.id,
+ notebook_id=record.id,
+ request_id=request_id,
+ )
+
+ entry.notebook._stream.after_delivery(finish)
def notebook(self, notebook_id: str) -> Notebook:
"""Return a notebook or raise `NotFound`."""
- notebook = self._notebooks.get(notebook_id)
- if notebook is None:
+ entry = self._entries.get(notebook_id)
+ if entry is None:
raise NotFound(f"no notebook {notebook_id}")
- return notebook
+ return entry.notebook
+
+ def run(self, notebook_id: str, run_id: str) -> protocol.Run | None:
+ """Return a run the host still retains, if any."""
+ entry = self._entries.get(notebook_id)
+ if entry is None:
+ raise NotFound(f"no notebook {notebook_id}")
+ return entry.log.run(run_id)
def _snapshot(
self, notebook_id: protocol.NotebookId | None = None
) -> list[Event]:
- """Return snapshot events for the catalog or a single notebook.
-
- Catalog snapshots contain the host, projects, then notebooks with
- their runtimes and attachments. Notebook snapshots also include
- runs, executions, and outputs.
- """
+ """Return catalog or notebook state with retained run history."""
if notebook_id is not None:
records = [self._records[notebook_id]]
events: list[Event] = []
@@ -159,11 +194,16 @@ class Host:
events.append(
Event("notebook", message(record), notebook_id=record.id)
)
- events.extend(
- e
- for e in self._notebooks[record.id]._snapshot()
- if e.visible_in(notebook_id)
- )
+ entry = self._entries[record.id]
+ seen: set[tuple[str, str]] = set()
+ for event in [
+ *entry.notebook._snapshot(),
+ *entry.log.snapshot(),
+ ]:
+ identity = _identity(event)
+ if identity not in seen and event.visible_in(notebook_id):
+ seen.add(identity)
+ events.append(event)
return events
def events(
@@ -174,11 +214,10 @@ class Host:
types: Collection[str] | None = None,
run_id: str | None = None,
) -> list[Event]:
- """Return initial events followed by `ready` with the latest cursor.
+ """Return a snapshot or replay followed by `ready`.
- Replay events after `cursor`, or return a snapshot if no cursor
- is given. An unavailable cursor produces `reset` and a snapshot.
- A run filter includes only that run, its executions, and their items.
+ A cursor requests the events after it. An unavailable cursor returns
+ `reset` and a snapshot.
"""
scope = None if notebook_id is None else self.notebook(notebook_id).id
ready = Event("ready", id=self._stream.last_id)
@@ -193,9 +232,11 @@ class Host:
return [e for e in [*events, ready] if e.wanted(types, run_id)]
def expire(self) -> None:
- """Sweep results according to the host's retention policy."""
- for notebook in list(self._notebooks.values()):
- notebook.expire()
+ """Remove scratch results whose retention has ended."""
+ with self._stream.deferred():
+ for entry in list(self._entries.values()):
+ for event in entry.log.expire(self.retention):
+ self._stream.append(event)
def subscribe(
self,
@@ -205,7 +246,7 @@ class Host:
types: Collection[str] | None = None,
run_id: str | None = None,
) -> Callable[[], None]:
- """Subscribe to matching live events; return an unsubscribe function."""
+ """Subscribe to matching events and return an unsubscribe function."""
def filtered(event: Event) -> None:
if event.visible_in(notebook_id) and event.wanted(types, run_id):
diff --git a/marimo/_core/notebook.py b/marimo/_core/notebook.py
index 2523ccb15f..b8fa68e5df 100644
--- a/marimo/_core/notebook.py
+++ b/marimo/_core/notebook.py
@@ -1,13 +1,8 @@
# Copyright 2026 Marimo. All rights reserved.
-"""Notebook runtimes, client attachments, and runs.
-
-Runtime and run commands call a backend. `Notebook.observe` applies
-backend updates and publishes events.
-"""
+"""Notebook state and commands."""
from __future__ import annotations
-from time import monotonic
from typing import TYPE_CHECKING, Any
import msgspec
@@ -15,58 +10,54 @@ from msgspec.structs import replace
from marimo._core import backend as be, protocol
from marimo._core.errors import Conflict, NoRuntime, NotFound
-from marimo._core.execution_state import ExecutionState, RetentionPolicy
+from marimo._core.executions import Executions, Released, Removed
from marimo._core.ids import mint
-from marimo._core.stream import Event, kind_of, message, utcnow
+from marimo._core.stream import (
+ Event,
+ Kind,
+ Listener,
+ Stream,
+ kind_of,
+ message,
+ utcnow,
+)
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Sequence
- from marimo._core.stream import Body, Stream
+ from marimo._core.executions import Change
+ from marimo._core.stream import Body
class Notebook:
- """A document, its results, and the runtime that produces them.
-
- Clients attach to a notebook, edit it, and run it. Every change is
- published as an event so that each client sees the same notebook.
- Commands change it and call the backend, which reports back through
- `observe`.
+ """A notebook with execution results, a runtime, and client attachments.
- Subscribers hear about a command once it has reached the backend.
- Results survive runtime stops and restarts, and the host decides how
- long completed scratch results are kept. Commands on a notebook the
- host has removed raise `NotFound`.
+ Commands call the backend. Backend reports update state and reach
+ subscribers through notebook events. Change execution state through
+ the notebook so each change reaches subscribers.
"""
def __init__(
- self,
- notebook_id: protocol.NotebookId,
- backend: be.Backend,
- stream: Stream,
- retention: RetentionPolicy | None = None,
- clock: Callable[[], float] = monotonic,
+ self, notebook_id: protocol.NotebookId, backend: be.Backend
) -> None:
self._id = notebook_id
self._backend = backend
- self._stream = stream
+ self._stream = Stream(history=0)
self._closed = False
self._runtime: protocol.Runtime | None = None
self._attachments: dict[str, protocol.Attachment] = {}
- self._state = ExecutionState(
- notebook_id,
- stream,
- retention or RetentionPolicy(),
- clock,
- forget=backend.forget,
- )
+ self._executions = Executions()
self._replacement: protocol.Runtime | None = None
# Correlate runtime events with the pending start, stop, or restart.
self._runtime_request: str | None = None
- def expire(self) -> None:
- """Apply the host's result retention policy."""
- self._state.expire()
+ def subscribe(self, listener: Listener) -> Callable[[], None]:
+ """Subscribe to future notebook events."""
+ return self._stream.subscribe(listener)
+
+ @property
+ def executions(self) -> Executions:
+ return self._executions
@property
def id(self) -> protocol.NotebookId:
@@ -74,7 +65,7 @@ class Notebook:
@property
def runtime(self) -> protocol.Runtime | None:
- """The current runtime, updated by commands and backend observations."""
+ """The current runtime, if any."""
return self._runtime
def _require_open(self) -> None:
@@ -86,11 +77,7 @@ class Notebook:
def start(
self, *, request_id: str | None = None
) -> tuple[protocol.Runtime, bool]:
- """Start a runtime if none exists.
-
- Returns the runtime and whether a new runtime was created.
- An existing runtime is returned unchanged, regardless of status.
- """
+ """Return the runtime and whether this call started it."""
with self._stream.deferred():
self._require_open()
if self._runtime is not None:
@@ -102,14 +89,10 @@ class Notebook:
return runtime, True
def stop(self, *, request_id: str | None = None) -> protocol.Runtime:
- """Stop the runtime for all attached clients, retaining its results.
+ """Stop the runtime and retain its results.
- Removal waits for the backend to report that the kernel exited.
- Failed runtimes are removed immediately. Returns the runtime
- with status `terminating`.
-
- Raises:
- NoRuntime: If the notebook has no runtime.
+ The runtime stays until the backend reports its exit. A failed runtime
+ is removed immediately.
"""
with self._stream.deferred():
self._require_open()
@@ -127,14 +110,9 @@ class Notebook:
return terminating
def restart(self, *, request_id: str | None = None) -> protocol.Runtime:
- """Stop the runtime and start a replacement with a new ID.
-
- Returns the replacement in its starting state. The replacement
- appears on the stream after the old runtime is removed.
+ """Stop a running runtime and return its replacement.
- Raises:
- NoRuntime: If the notebook has no runtime.
- Conflict: If the runtime is not running.
+ The replacement starts after the old runtime exits.
"""
with self._stream.deferred():
self._require_open()
@@ -152,20 +130,13 @@ class Notebook:
return replacement
# TODO: Remove once sessions start their runtime through `start`.
- # marimo still creates kernels for browser tabs behind the notebook's
- # back, so the host learns about them afterwards.
+ # Browser sessions create kernels outside the notebook.
def adopt(
self, attach: Callable[[protocol.RuntimeId], None]
) -> protocol.Runtime:
- """Track a kernel started outside the host API.
+ """Attach the backend to an existing kernel.
- Returns a starting runtime. Calls `attach` with its ID before
- delivering events to subscribers. The callback binds the backend
- synchronously and reports status, including attachment failures,
- through `observe`.
-
- Raises:
- Conflict: If the notebook has a runtime that has not failed.
+ Calls `attach` with a new runtime ID before delivering events.
"""
with self._stream.deferred():
self._require_open()
@@ -195,7 +166,7 @@ class Notebook:
def _end_runtime(self) -> None:
assert self._runtime is not None
- self._state.interrupt()
+ self._apply(self.executions.interrupt())
runtime, self._runtime = self._runtime, None
self._remove("runtime", runtime.id, self._runtime_request)
self._runtime_request = None
@@ -203,21 +174,14 @@ class Notebook:
# ---- runs ----------------------------------------------------------
def run(self, code: str, *, request_id: str) -> protocol.Run:
- """Queue scratch code on the runtime.
-
- The host assigns the run ID. Request IDs correlate events. The
- host retains retry receipts separately from execution results.
-
- Raises:
- NoRuntime: If the notebook has no runtime.
- Conflict: If the runtime is not running, or a run is in progress.
- """
+ """Queue scratch code on a running, idle runtime."""
with self._stream.deferred():
self._require_open()
runtime = self._require_runtime()
if runtime.status != "running":
raise Conflict(f"the runtime is {runtime.status}.")
- if (active := self._state.active_run) is not None:
+ active = self.executions.active()
+ if active is not None:
raise Conflict(f"run {active.id} is in progress.")
run = protocol.Run(
id=mint(protocol.RunId, "run"),
@@ -228,19 +192,14 @@ class Notebook:
started_at=None,
completed_at=None,
)
- self._state.add_run(run, request_id)
+ self._apply(self.executions.begin(run, request_id))
self._backend.run(run.id, code)
return run
def interrupt(self, run_id: str) -> protocol.Run:
- """Request interruption of a run in progress.
-
- Raises:
- NotFound: If there is no such run.
- Conflict: If the run is already completed.
- """
+ """Request interruption of a run in progress."""
self._require_open()
- run = self._state.run(run_id)
+ run = self.executions.run(protocol.RunId(run_id))
if run is None:
raise NotFound(f"notebook {self.id} has no run {run_id}.")
if run.status == "completed":
@@ -257,15 +216,9 @@ class Notebook:
kind: str | None = None,
name: str | None = None,
) -> protocol.Attachment:
- """Attach a client to this notebook.
+ """Create a client attachment.
- Each call creates a separate attachment. `identity` is stored as
- data. It does not affect the attachment lifetime. An omitted `kind`
- is `client`. An empty `name` is stored as null.
-
- Raises:
- msgspec.ValidationError: If a value breaks the protocol's
- constraints on the attachment field of the same name.
+ Identity is metadata. Each attachment has its own lifetime.
"""
self._require_open()
fields: dict[str, Any] = {
@@ -283,10 +236,7 @@ class Notebook:
return attachment
def detach(self, attachment_id: str) -> None:
- """Remove the attachment and publish its removal.
-
- Unknown IDs are ignored.
- """
+ """Remove an attachment if it exists."""
attachment = self._attachments.pop(attachment_id, None)
if attachment is not None:
self._remove("attachment", attachment.id, None)
@@ -304,7 +254,7 @@ class Notebook:
return # A kernel this notebook no longer tracks.
self._observe_runtime(runtime, msg)
case _:
- self._state.observe(msg, self._runtime)
+ self._apply(self.executions.observe(msg, self._runtime))
def _observe_runtime(
self, runtime: protocol.Runtime, report: be.RuntimeReport
@@ -341,7 +291,7 @@ class Notebook:
self._end_runtime()
else:
# Preserve results after an unexpected exit.
- self._state.interrupt("failed")
+ self._apply(self.executions.interrupt("failed"))
self._set_runtime(
replace(runtime, status="failed", message=reason)
)
@@ -368,20 +318,11 @@ class Notebook:
*,
request_id: str | None = None,
) -> protocol.Custom:
- """Publish a custom event on the notebook stream.
-
- A custom event is a notebook occurrence with a producer-defined
- payload.
-
- Raises:
- TypeError: If `data` is not a JSON value.
- ValueError: If `type` is empty.
- """
+ """Publish a named custom event with a JSON payload."""
self._require_open()
if not type:
raise ValueError("type must be a non-empty string")
- # Fail here, in the producer, instead of inside every subscriber's
- # stream when the event is framed.
+ # Validate before delivering to subscribers.
msgspec.json.encode(data)
custom = protocol.Custom(type=type, data=data)
self._publish(custom, request_id)
@@ -390,31 +331,38 @@ class Notebook:
# ---- stream --------------------------------------------------------
def _snapshot(self) -> list[Event]:
- """Return notebook objects as snapshot events, parents first.
-
- Runs, executions, and outputs keep their insertion order.
- """
- bodies: list[protocol.Runtime | protocol.Attachment] = []
+ """Return current state as events in dependency order."""
+ bodies: list[Body] = []
if self._runtime is not None:
bodies.append(self._runtime)
bodies.extend(self._attachments.values())
- return [
- Event(
- kind_of(body),
- message(body),
- notebook_id=self.id,
- )
+ events = [
+ Event(kind_of(body), message(body), notebook_id=self.id)
for body in bodies
- ] + self._state.snapshot()
+ ]
+ executions = self.executions
+ for record in executions._records():
+ run_id = executions.run_id_of(record)
+ events.append(
+ Event(
+ kind_of(record),
+ message(
+ record, request_id=executions.snapshot_request(run_id)
+ ),
+ notebook_id=self.id,
+ run_id=run_id,
+ )
+ )
+ return events
def _dispose(self, *, request_id: str | None = None) -> None:
"""Close the notebook and remove its state, children first.
- The host must publish `notebook.removed` after disposing of its state.
+ The host publishes `notebook.removed` after disposing of this state.
"""
self._closed = True
- self._state.interrupt()
- self._state.clear()
+ self._apply(self.executions.interrupt())
+ self._apply(self.executions.clear())
for attachment_id in list(self._attachments):
self.detach(attachment_id)
if self._runtime is not None:
@@ -430,7 +378,7 @@ class Notebook:
def _remove(
self,
- kind: str,
+ kind: Kind,
object_id: str,
request_id: str | None,
) -> None:
@@ -440,3 +388,30 @@ class Notebook:
notebook_id=self.id,
request_id=request_id,
)
+
+ def _apply(self, changes: Sequence[Change]) -> None:
+ """Publish changes and release completed runs."""
+ executions = self.executions
+ with self._stream.deferred():
+ for change in changes:
+ if isinstance(change, Released):
+ executions.release(change.run_id)
+ self._backend.forget(change.run_id)
+ continue
+ run_id = executions.run_id_of(change)
+ request_id = executions.request_id(run_id)
+ if isinstance(change, Removed):
+ self._stream.remove(
+ change.kind,
+ change.id,
+ notebook_id=self.id,
+ run_id=run_id,
+ request_id=request_id,
+ )
+ else:
+ self._stream.publish(
+ change,
+ notebook_id=self.id,
+ run_id=run_id,
+ request_id=request_id,
+ )
diff --git a/marimo/_core/run_log.py b/marimo/_core/run_log.py
new file mode 100644
index 0000000000..cabf5baf7a
--- /dev/null
+++ b/marimo/_core/run_log.py
@@ -0,0 +1,210 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Retained run history for reconnecting clients."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from math import isfinite
+from typing import TYPE_CHECKING
+
+from marimo._core import protocol
+from marimo._core.stream import Event, Kind, message, removal
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+
+@dataclass(frozen=True)
+class RetentionPolicy:
+ """How long reconnecting clients can recover completed scratch runs."""
+
+ ephemeral_seconds: float = 60.0
+
+ def __post_init__(self) -> None:
+ if not isfinite(self.ephemeral_seconds) or self.ephemeral_seconds < 0:
+ raise ValueError("invalid result retention")
+
+
+@dataclass
+class _Scratch:
+ """A scratch execution's latest event and its outputs."""
+
+ execution: Event
+ outputs: dict[str, Event] = field(default_factory=dict)
+
+
+@dataclass
+class _Run:
+ """A run's latest event and what still keeps it visible."""
+
+ event: Event
+ request_id: str | None
+ live: set[str] = field(default_factory=set)
+ scratch: dict[str, _Scratch] = field(default_factory=dict)
+ completed_at: float | None = None
+ had_executions: bool = False
+
+
+class RunLog:
+ """Run history for one notebook.
+
+ Completed scratch results expire under the retention policy. A run
+ remains while any visible execution refers to it. Runs that end before
+ executing are retained for the same period as scratch results.
+
+ A notebook releases completed scratch work without a removal event.
+ The log keeps it until retention ends and publishes its removals.
+ """
+
+ def __init__(self, clock: Callable[[], float]) -> None:
+ self._clock = clock
+ self._runs: dict[str, _Run] = {}
+
+ def observe(self, event: Event) -> list[Event]:
+ """Record an event and return removals to publish after it."""
+ match event:
+ case Event(
+ data=protocol.RunMessage(run=run, request_id=request_id)
+ ):
+ entry = self._runs.get(run.id)
+ if entry is None:
+ entry = self._runs[run.id] = _Run(
+ event,
+ request_id if isinstance(request_id, str) else None,
+ )
+ entry.event = event
+ if run.status == "completed" and entry.completed_at is None:
+ entry.completed_at = self._clock()
+ return self._close(run.id)
+ case Event(data=protocol.ExecutionMessage(execution=execution)):
+ if (entry := self._entry(event)) is not None:
+ entry.had_executions = True
+ entry.live.add(execution.id)
+ if execution.cell_id is None:
+ scratch = entry.scratch.get(execution.id)
+ if scratch is None:
+ entry.scratch[execution.id] = _Scratch(event)
+ else:
+ scratch.execution = event
+ case Event(data=protocol.ConsoleMessage(console=item)):
+ self._record_output(event, item)
+ case Event(data=protocol.DisplayMessage(display=item)):
+ self._record_output(event, item)
+ case Event(
+ kind="execution", data=protocol.RemovedMessage(id=execution_id)
+ ):
+ if (entry := self._entry(event)) is not None:
+ entry.live.discard(execution_id)
+ entry.scratch.pop(execution_id, None)
+ assert event.run_id is not None
+ return self._close(event.run_id)
+ return []
+
+ def run(self, run_id: str) -> protocol.Run | None:
+ """Return a retained run without reading its executions or outputs."""
+ entry = self._runs.get(run_id)
+ if entry is None:
+ return None
+ assert isinstance(entry.event.data, protocol.RunMessage)
+ return entry.event.data.run
+
+ def snapshot(self) -> list[Event]:
+ """Return the runs and scratch the log holds, parents first."""
+ events = [
+ self._snapshot_event(e.event, e) for e in self._runs.values()
+ ]
+ for entry in self._runs.values():
+ for scratch in entry.scratch.values():
+ events.append(self._snapshot_event(scratch.execution, entry))
+ events.extend(
+ self._snapshot_event(output, entry)
+ for output in scratch.outputs.values()
+ )
+ return events
+
+ def expire(self, policy: RetentionPolicy) -> list[Event]:
+ """Return removals for scratch whose retention has ended."""
+ now = self._clock()
+ events: list[Event] = []
+ for run_id, entry in list(self._runs.items()):
+ if (
+ (entry.scratch or not entry.had_executions)
+ and entry.completed_at is not None
+ and now >= entry.completed_at + policy.ephemeral_seconds
+ ):
+ events.extend(self._release(run_id))
+ return events
+
+ def clear(self) -> list[Event]:
+ """Remove all retained history."""
+ events: list[Event] = []
+ for run_id, entry in list(self._runs.items()):
+ entry.live.clear()
+ if entry.completed_at is None:
+ entry.completed_at = self._clock()
+ events.extend(self._release(run_id))
+ return events
+
+ def _entry(self, event: Event) -> _Run | None:
+ return None if event.run_id is None else self._runs.get(event.run_id)
+
+ def _record_output(
+ self, event: Event, item: protocol.Console | protocol.Display
+ ) -> None:
+ entry = self._entry(event)
+ if entry is None:
+ return
+ scratch = entry.scratch.get(item.execution_id)
+ if scratch is not None:
+ scratch.outputs[item.id] = event
+
+ def _release(self, run_id: str) -> list[Event]:
+ """Remove a run's scratch, then the run if nothing else holds it."""
+ entry = self._runs[run_id]
+ events: list[Event] = []
+ for execution_id, scratch in entry.scratch.items():
+ for output in scratch.outputs.values():
+ assert output.kind is not None
+ item = getattr(output.data, output.kind)
+ events.append(self._removal(entry, output.kind, item.id))
+ events.append(self._removal(entry, "execution", execution_id))
+ entry.live.discard(execution_id)
+ entry.scratch.clear()
+ return [*events, *self._close(run_id, retain_empty=False)]
+
+ def _close(self, run_id: str, *, retain_empty: bool = True) -> list[Event]:
+ entry = self._runs[run_id]
+ if (
+ entry.completed_at is None
+ or entry.live
+ or (retain_empty and not entry.had_executions)
+ ):
+ return []
+ del self._runs[run_id]
+ return [self._removal(entry, "run", run_id)]
+
+ def _removal(self, entry: _Run, kind: Kind, object_id: str) -> Event:
+ return removal(
+ kind,
+ object_id,
+ notebook_id=entry.event.notebook_id,
+ request_id=entry.request_id,
+ run_id=entry.event.run_id,
+ )
+
+ def _snapshot_event(self, event: Event, entry: _Run) -> Event:
+ assert event.kind is not None
+ body = getattr(event.data, event.kind)
+ return Event(
+ event.name,
+ message(
+ body,
+ request_id=(
+ None
+ if entry.completed_at is not None
+ else entry.request_id
+ ),
+ ),
+ notebook_id=event.notebook_id,
+ run_id=event.run_id,
+ )
diff --git a/marimo/_core/stream.py b/marimo/_core/stream.py
index ddad41f112..47e15a211d 100644
--- a/marimo/_core/stream.py
+++ b/marimo/_core/stream.py
@@ -10,11 +10,11 @@ from __future__ import annotations
import logging
from collections import deque
from collections.abc import Callable, Collection, Iterator
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
from copy import deepcopy
from dataclasses import dataclass
from datetime import datetime, timezone
-from typing import TypeAlias, get_args
+from typing import Literal, TypeAlias, get_args
import msgspec
from msgspec import UNSET
@@ -39,21 +39,39 @@ Body: TypeAlias = (
derive from it; the catalog kinds are a chosen subset."""
-def _kind(body_type: type[msgspec.Struct]) -> str:
- return body_type.__name__.lower()
+Kind: TypeAlias = Literal[
+ "host",
+ "project",
+ "notebook",
+ "runtime",
+ "attachment",
+ "run",
+ "execution",
+ "console",
+ "display",
+ "custom",
+]
+"""The kinds of object a stream carries, one per `Body` type."""
+
+_KIND_BY_NAME: dict[str, Kind] = {kind: kind for kind in get_args(Kind)}
+assert set(_KIND_BY_NAME) == {
+ body_type.__name__.lower() for body_type in get_args(Body)
+}
-def kind_of(body: Body) -> str:
+def kind_of(body: Body) -> Kind:
"""Return the object's event kind, used by the `types` filter."""
- return _kind(type(body))
+ return _KIND_BY_NAME[type(body).__name__.lower()]
-_WRAPPERS: dict[str, type[msgspec.Struct]] = {
- _kind(body_type): getattr(protocol, f"{body_type.__name__}Message")
+_WRAPPERS: dict[Kind, type[msgspec.Struct]] = {
+ _KIND_BY_NAME[body_type.__name__.lower()]: getattr(
+ protocol, f"{body_type.__name__}Message"
+ )
for body_type in get_args(Body)
}
-CATALOG_KINDS = frozenset(
+CATALOG_KINDS: frozenset[Kind] = frozenset(
{"host", "project", "notebook", "runtime", "attachment"}
)
# Host and project describe the whole host, so they never belong to one
@@ -69,8 +87,9 @@ def message(
) -> msgspec.Struct:
"""Wrap an object in its protocol message."""
kind = kind_of(body)
+ fields: dict[str, Body] = {kind: deepcopy(body)}
return _WRAPPERS[kind](
- **{kind: deepcopy(body)},
+ **fields,
request_id=UNSET if request_id is None else request_id,
time=UNSET if time is None else time,
)
@@ -80,11 +99,30 @@ def utcnow() -> datetime:
return datetime.now(timezone.utc)
+def removal(
+ kind: Kind,
+ object_id: str,
+ *,
+ notebook_id: protocol.NotebookId | None = None,
+ request_id: str | None = None,
+ run_id: protocol.RunId | None = None,
+) -> Event:
+ """Return an event announcing an object's removal."""
+ data = protocol.RemovedMessage(
+ id=object_id,
+ request_id=UNSET if request_id is None else request_id,
+ time=utcnow(),
+ )
+ return Event(
+ f"{kind}.removed", data, notebook_id=notebook_id, run_id=run_id
+ )
+
+
@dataclass(frozen=True)
class Event:
"""An event with an optional replay cursor and notebook scope.
- Snapshot events have no `id`; `ready` and `reset` have no `data`.
+ Snapshot events have no `id`. Control events have no `data`.
"""
name: str
@@ -94,9 +132,12 @@ class Event:
run_id: protocol.RunId | None = None
@property
- def kind(self) -> str:
- """The object kind without the `.removed` suffix."""
- return self.name.partition(".")[0]
+ def kind(self) -> Kind | None:
+ """The object kind without the `.removed` suffix.
+
+ `ready` and `reset` carry no object and have no kind.
+ """
+ return _KIND_BY_NAME.get(self.name.partition(".")[0])
def visible_in(self, notebook_id: str | None) -> bool:
"""Whether the event belongs in the catalog or notebook stream."""
@@ -129,7 +170,10 @@ class Stream:
self._last_id = 0
self._history: deque[Event] = deque(maxlen=history)
self._listeners: list[Listener] = []
- self._pending: deque[tuple[Event, tuple[Listener, ...]]] = deque()
+ self._downstream: list[Stream] = []
+ self._pending: deque[
+ tuple[Event, tuple[Listener, ...]] | Callable[[], None]
+ ] = deque()
self._dispatching = False
@property
@@ -153,7 +197,7 @@ class Stream:
def remove(
self,
- kind: str,
+ kind: Kind,
object_id: str,
*,
notebook_id: protocol.NotebookId | None = None,
@@ -161,14 +205,13 @@ class Stream:
run_id: protocol.RunId | None = None,
) -> Event:
"""Publish an object's removal."""
- data = protocol.RemovedMessage(
- id=object_id,
- request_id=UNSET if request_id is None else request_id,
- time=utcnow(),
- )
return self._append(
- Event(
- f"{kind}.removed", data, notebook_id=notebook_id, run_id=run_id
+ removal(
+ kind,
+ object_id,
+ notebook_id=notebook_id,
+ request_id=request_id,
+ run_id=run_id,
)
)
@@ -186,17 +229,24 @@ class Stream:
[e for e in self._history if e.id is not None and e.id > cursor]
)
- def subscribe(self, listener: Listener) -> Callable[[], None]:
- """Deliver future events in order; return an unsubscribe function.
+ def subscribe(
+ self, listener: Listener, *, downstream: Stream | None = None
+ ) -> Callable[[], None]:
+ """Subscribe to future events and return an unsubscribe function.
- Subscriber exceptions are logged without interrupting publication.
- Events published by a subscriber follow the event being delivered.
+ Subscriber exceptions are logged. Events published during delivery
+ follow the event being delivered. A downstream stream delivers
+ forwarded events after this stream drains.
"""
self._listeners.append(listener)
+ if downstream is not None:
+ self._downstream.append(downstream)
def unsubscribe() -> None:
if listener in self._listeners:
self._listeners.remove(listener)
+ if downstream is not None:
+ self._downstream.remove(downstream)
return unsubscribe
@@ -217,6 +267,10 @@ class Stream:
self._dispatching = False
self._drain()
+ def append(self, event: Event) -> Event:
+ """Record an emitted event with a cursor belonging to this log."""
+ return self._append(deepcopy(event))
+
def _append(self, event: Event) -> Event:
self._last_id += 1
stamped = Event(
@@ -232,15 +286,29 @@ class Stream:
self._drain()
return deepcopy(stamped)
+ def after_delivery(self, callback: Callable[[], None]) -> None:
+ """Call after queued events reach their listeners."""
+ if self._dispatching:
+ self._pending.append(callback)
+ else:
+ callback()
+
def _drain(self) -> None:
- self._dispatching = True
- try:
- while self._pending:
- pending, listeners = self._pending.popleft()
- for listener in listeners:
- try:
- listener(deepcopy(pending))
- except Exception:
- LOGGER.exception("Error delivering host event")
- finally:
- self._dispatching = False
+ with ExitStack() as delivery:
+ for stream in self._downstream:
+ delivery.enter_context(stream.deferred())
+ self._dispatching = True
+ try:
+ while self._pending:
+ item = self._pending.popleft()
+ if callable(item):
+ item()
+ continue
+ pending, listeners = item
+ for listener in listeners:
+ try:
+ listener(deepcopy(pending))
+ except Exception:
+ LOGGER.exception("Error delivering host event")
+ finally:
+ self._dispatching = False
diff --git a/marimo/_server/host/api.py b/marimo/_server/host/api.py
index f191dfcd86..09145c94ca 100644
--- a/marimo/_server/host/api.py
+++ b/marimo/_server/host/api.py
@@ -659,5 +659,13 @@ async def _run(request: Request) -> Response:
async def _interrupt(request: Request) -> Response:
"""Request a run interruption: 202 if active, 409 if completed."""
notebook = _notebook(request)
- interrupted = notebook.interrupt(request.path_params["run_id"])
+ run_id = request.path_params["run_id"]
+ try:
+ interrupted = notebook.interrupt(run_id)
+ except NotFound:
+ # A completed scratch run is host history, not notebook state.
+ run = _context(request).bridge.host.run(notebook.id, run_id)
+ if run is not None and run.status == "completed":
+ raise Conflict(f"run {run_id} has already ended.") from None
+ raise
return _reply(202, interrupted)
diff --git a/pyproject.toml b/pyproject.toml
index 5916ad1398..2bf5900ccc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -528,6 +528,11 @@ exclude = [
]
warn_unused_ignores = false
+[[tool.mypy.overrides]]
+# Core matches on typed events; a misspelled kind must not match nothing.
+module = ["marimo._core.*"]
+warn_unreachable = true
+
[tool.pytest.ini_options]
minversion = "6.0"
addopts = "-ra -q -v --ignore tests/_cli/fixtures --ignore tests/_ast/codegen_data --ignore tests/_ast/app_data"
diff --git a/tests/_core/conftest.py b/tests/_core/conftest.py
index 29668ad550..79b686ec1d 100644
--- a/tests/_core/conftest.py
+++ b/tests/_core/conftest.py
@@ -5,9 +5,10 @@ from __future__ import annotations
import pytest
-from marimo._core import protocol
+from marimo._core import backend as be, protocol
from marimo._core.host import Host
from marimo._core.notebook import Notebook
+from marimo._core.run_log import RetentionPolicy
from marimo._core.stream import Event
OPERATIONS = ["notebook.create", "runtime.start", "runtime.run"]
@@ -95,3 +96,68 @@ def names(events: list[Event]) -> list[str]:
def request_ids(events: list[Event]) -> list[object]:
return [getattr(e.data, "request_id", None) for e in events]
+
+
+class Clock:
+ now = 0.0
+
+ def __call__(self) -> float:
+ return self.now
+
+
+def host_at(clock: Clock, history: int = 1000) -> tuple[Host, Notebook]:
+ host = Host(
+ operations=[],
+ history=history,
+ clock=clock,
+ retention=RetentionPolicy(ephemeral_seconds=10),
+ )
+ host.update_project(project())
+ notebook = host.add_notebook(record(), ScriptedBackend())
+ notebook.adopt(lambda _: None)
+ notebook.observe(be.Running(runtime_id_of(notebook)))
+ return host, notebook
+
+
+def plan(
+ notebook: Notebook, run: protocol.Run, eid: str, cell: str | None = None
+) -> protocol.ExecutionId:
+ execution = protocol.ExecutionId(eid)
+ notebook.observe(
+ be.Planned(
+ run.id,
+ (
+ be.PlannedExecution(
+ execution, cell, "hash", "code" if cell is None else None
+ ),
+ ),
+ )
+ )
+ notebook.observe(be.Status(execution, "running"))
+ return execution
+
+
+def finish(notebook: Notebook, execution: protocol.ExecutionId) -> None:
+ notebook.observe(
+ be.Concluded(
+ execution,
+ protocol.ValueOutcome(
+ protocol.DisplayData(mimetype="text/plain", data="value")
+ ),
+ )
+ )
+ notebook.observe(be.Status(execution, "completed"))
+
+
+def fold(events: list[Event], initial: dict | None = None) -> dict:
+ state = dict(initial or {})
+ for event in events:
+ if event.name == "reset":
+ state.clear()
+ elif event.kind in {"run", "execution", "console", "display"}:
+ if event.name.endswith(".removed"):
+ state.pop((event.kind, event.data.id), None)
+ else:
+ body = getattr(event.data, event.kind)
+ state[event.kind, body.id] = body
+ return state
diff --git a/tests/_core/test_execution_state.py b/tests/_core/test_execution_state.py
deleted file mode 100644
index 353ce01c34..0000000000
--- a/tests/_core/test_execution_state.py
+++ /dev/null
@@ -1,435 +0,0 @@
-# Copyright 2026 Marimo. All rights reserved.
-"""Result retention across live streams, replay, and snapshots."""
-
-from __future__ import annotations
-
-import pytest
-from msgspec.structs import replace
-
-from marimo._core import backend as be, protocol as p
-from marimo._core.execution_state import RetentionPolicy
-from marimo._core.host import Host
-from marimo._core.notebook import Notebook
-from marimo._core.stream import Event
-from tests._core.conftest import (
- ScriptedBackend,
- names,
- project,
- record,
- runtime_id_of,
-)
-
-
-class Clock:
- now = 0.0
-
- def __call__(self) -> float:
- return self.now
-
-
-def host_at(clock: Clock, history: int = 1000) -> tuple[Host, Notebook]:
- host = Host(
- operations=[],
- history=history,
- clock=clock,
- retention=RetentionPolicy(ephemeral_seconds=10),
- )
- host.update_project(project())
- notebook = host.add_notebook(record(), ScriptedBackend())
- notebook.adopt(lambda _: None)
- notebook.observe(be.Running(runtime_id_of(notebook)))
- return host, notebook
-
-
-def plan(
- notebook: Notebook, run: p.Run, eid: str, cell: str | None = None
-) -> p.ExecutionId:
- execution = p.ExecutionId(eid)
- notebook.observe(
- be.Planned(
- run.id,
- (
- be.PlannedExecution(
- execution, cell, "hash", "code" if cell is None else None
- ),
- ),
- )
- )
- notebook.observe(be.Status(execution, "running"))
- return execution
-
-
-def finish(notebook: Notebook, execution: p.ExecutionId) -> None:
- notebook.observe(
- be.Concluded(
- execution,
- p.ValueOutcome(p.DisplayData(mimetype="text/plain", data="value")),
- )
- )
- notebook.observe(be.Status(execution, "completed"))
-
-
-def fold(events: list[Event], initial: dict | None = None) -> dict:
- state = dict(initial or {})
- for event in events:
- if event.name == "reset":
- state.clear()
- elif event.kind in {"run", "execution", "console", "display"}:
- if event.name.endswith(".removed"):
- state.pop((event.kind, event.data.id), None)
- else:
- body = getattr(event.data, event.kind)
- state[event.kind, body.id] = body
- return state
-
-
[email protected]("history", [2, 1000])
-def test_expiration_converges_for_live_replay_and_snapshot(
- history: int,
-) -> None:
- clock = Clock()
- host, notebook = host_at(clock, history)
- run = notebook.run("code", request_id="request")
- execution = plan(notebook, run, "e")
- item = p.Console(
- p.ExecutionId("e"), p.ItemId("console"), "stdout", 0, "output"
- )
- notebook.observe(be.Produced(item))
- notebook.observe(
- be.Produced(
- p.Display(
- execution_id=execution,
- id=p.ItemId("display"),
- seq=1,
- mimetype="text/html",
- data="value",
- )
- )
- )
- finish(notebook, execution)
- notebook.observe(be.Done(run.id, "succeeded"))
- snapshot = host.events(notebook_id=notebook.id, run_id=run.id)
- initial = fold(snapshot)
- assert {kind for kind, _ in initial} == {
- "run",
- "execution",
- "console",
- "display",
- }
- live: list[Event] = []
- host.subscribe(live.append, notebook_id=notebook.id, run_id=run.id)
- clock.now = 9
- host.expire()
- assert live == []
- clock.now = 10
- host.expire()
- assert names(live) == [
- "console.removed",
- "display.removed",
- "execution.removed",
- "run.removed",
- ]
- assert all(event.data.request_id == "request" for event in live)
- replay = host.events(
- snapshot[-1].id, notebook_id=notebook.id, run_id=run.id
- )
- assert (replay[0].name == "reset") == (history == 2)
- assert (
- fold(live, initial)
- == fold(replay, initial)
- == fold(host.events(notebook_id=notebook.id, run_id=run.id))
- == {}
- )
- live.clear()
- notebook.observe(be.Produced(item))
- notebook.observe(be.Status(execution, "running"))
- notebook.observe(be.Done(run.id, "succeeded"))
- assert live == []
-
-
-def test_cell_results_outlive_ephemeral_members_and_old_runs_are_removed() -> (
- None
-):
- clock = Clock()
- host, notebook = host_at(clock)
- run = notebook.run("code", request_id="first")
- ephemeral = plan(notebook, run, "scratch")
- cell = plan(notebook, run, "cell", "a")
- finish(notebook, ephemeral)
- clock.now = 100
- host.expire()
- assert ("execution", ephemeral) in fold(
- host.events(notebook_id=notebook.id)
- )
- finish(notebook, cell)
- notebook.observe(be.Done(run.id, "succeeded"))
- clock.now = 110
- host.expire()
- state = fold(host.events(notebook_id=notebook.id))
- assert set(state) == {("run", run.id), ("execution", cell)}
- replacement = notebook.run("code", request_id="second")
- current = plan(notebook, replacement, "current", "a")
- finish(notebook, current)
- notebook.observe(be.Done(replacement.id, "succeeded"))
- clock.now = 1000
- state = fold(host.events(notebook_id=notebook.id))
- assert set(state) == {("run", replacement.id), ("execution", current)}
-
-
-def test_run_survives_until_its_last_cell_execution_is_replaced() -> None:
- host, notebook = host_at(Clock())
- original = notebook.run("code", request_id="original")
- first = plan(notebook, original, "first", "a")
- second = plan(notebook, original, "second", "b")
- finish(notebook, first)
- finish(notebook, second)
- notebook.observe(be.Done(original.id, "succeeded"))
- live: list[Event] = []
- host.subscribe(live.append, notebook_id=notebook.id, run_id=original.id)
-
- replacement = notebook.run("code", request_id="replacement")
- current_a = plan(notebook, replacement, "current-a", "a")
- assert set(fold(host.events(notebook_id=notebook.id))) == {
- ("run", original.id),
- ("run", replacement.id),
- ("execution", second),
- ("execution", current_a),
- }
- assert names(live) == ["execution.removed"]
-
- current_b = plan(notebook, replacement, "current-b", "b")
- assert set(fold(host.events(notebook_id=notebook.id))) == {
- ("run", replacement.id),
- ("execution", current_a),
- ("execution", current_b),
- }
- assert names(live) == [
- "execution.removed",
- "execution.removed",
- "run.removed",
- ]
-
-
-def test_stopping_starts_expiry_for_active_work_without_extending_old_results() -> (
- None
-):
- clock = Clock()
- host, notebook = host_at(clock)
- done = notebook.run("code", request_id="done")
- execution = plan(notebook, done, "done")
- finish(notebook, execution)
- notebook.observe(be.Done(done.id, "succeeded"))
- clock.now = 5
- active = notebook.run("sleep", request_id="active")
- plan(notebook, active, "active")
- notebook.stop()
- notebook.observe(be.Exited(runtime_id_of(notebook)))
- clock.now = 10
- host.expire()
- state = fold(host.events(notebook_id=notebook.id))
- assert set(state) == {("run", active.id), ("execution", "active")}
- assert state["run", active.id].outcome == "interrupted"
- clock.now = 15
- host.expire()
- assert fold(host.events(notebook_id=notebook.id)) == {}
-
-
-def test_repeated_expiration_releases_runs_and_request_correlations() -> None:
- clock = Clock()
- host, notebook = host_at(clock, history=2)
- for index in range(100):
- run = notebook.run("code", request_id=f"request-{index}")
- execution = plan(notebook, run, str(index))
- finish(notebook, execution)
- notebook.observe(be.Done(run.id, "succeeded"))
- clock.now += 10
- host.expire()
- assert fold(host.events(notebook_id=notebook.id)) == {}
- assert (
- len({call[1] for call in notebook._backend.calls if call[0] == "run"})
- == 100
- )
-
-
-def test_removal_listeners_can_read_snapshots() -> None:
- clock = Clock()
- host, notebook = host_at(clock)
- run = notebook.run("code", request_id="request")
- execution = plan(notebook, run, "execution")
- notebook.observe(
- be.Produced(
- p.Console(execution, p.ItemId("console"), "stdout", 0, "x")
- )
- )
- finish(notebook, execution)
- notebook.observe(be.Done(run.id, "succeeded"))
- snapshots: list[dict] = []
- host.subscribe(
- lambda _: snapshots.append(fold(host.events(notebook_id=notebook.id))),
- notebook_id=notebook.id,
- )
- clock.now = 10
- host.expire()
- assert [set(state) for state in snapshots] == [
- {("run", run.id), ("execution", execution)},
- {("run", run.id)},
- set(),
- ]
-
-
[email protected]("cleanup", ["expire", "replace", "clear"])
-def test_outputs_follow_their_execution_through_updates_and_removal(
- cleanup: str,
-) -> None:
- clock = Clock()
- host, notebook = host_at(clock)
- run = notebook.run("code", request_id="request")
- first = plan(notebook, run, "first", "a" if cleanup != "expire" else None)
- second = plan(
- notebook, run, "second", "b" if cleanup != "expire" else None
- )
- first_console = p.Console(
- first, p.ItemId("first-console"), "stdout", 0, "a"
- )
- first_display = p.Display(
- execution_id=first,
- id=p.ItemId("first-display"),
- seq=1,
- mimetype="text/plain",
- data="b",
- )
- second_console = p.Console(
- second, p.ItemId("second-console"), "stdout", 0, "c"
- )
- second_display = p.Display(
- execution_id=second,
- id=p.ItemId("second-display"),
- seq=1,
- mimetype="text/plain",
- data="d",
- )
- for item in (second_console, first_console, second_display, first_display):
- notebook.observe(be.Produced(item))
- first_console = replace(first_console, text="updated")
- notebook.observe(be.Produced(first_console))
- finish(notebook, first)
- finish(notebook, second)
- notebook.observe(be.Done(run.id, "succeeded"))
- initial = host.events(notebook_id=notebook.id)
- assert [
- getattr(event.data, event.kind)
- for event in initial
- if event.kind in {"console", "display"}
- ] == [first_console, first_display, second_console, second_display]
-
- live: list[Event] = []
- snapshots: list[dict] = []
-
- def updated(event: Event) -> None:
- live.append(event)
- snapshots.append(fold(notebook._snapshot()))
-
- host.subscribe(
- updated,
- notebook_id=notebook.id,
- types={"run", "execution", "console", "display"},
- )
- if cleanup == "expire":
- clock.now = 10
- host.expire()
- elif cleanup == "replace":
- replacement = notebook.run("code", request_id="replacement")
- plan(notebook, replacement, "new", "a")
- else:
- host.remove_notebook(notebook.id)
-
- assert [
- (event.name, event.data.id)
- for event in live
- if event.name.endswith(".removed")
- ] == [
- ("console.removed", first_console.id),
- ("display.removed", first_display.id),
- ("execution.removed", first),
- *(
- []
- if cleanup == "replace"
- else [
- ("console.removed", second_console.id),
- ("display.removed", second_display.id),
- ("execution.removed", second),
- ("run.removed", run.id),
- ]
- ),
- ]
- state = fold(initial)
- for event, observed in zip(live, snapshots, strict=True):
- state = fold([event], state)
- assert state == observed
-
-
-def test_expiration_does_not_scan_unrelated_outputs(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- clock = Clock()
- host, notebook = host_at(clock, history=2)
- run = notebook.run("code", request_id="request")
- for index in range(1000):
- execution = plan(notebook, run, str(index))
- notebook.observe(
- be.Produced(
- p.Console(execution, p.ItemId(str(index)), "stdout", 0, "x")
- )
- )
- finish(notebook, execution)
- notebook.observe(be.Done(run.id, "succeeded"))
- visits = 0
- getattribute = p.Console.__getattribute__
-
- def counted(item: p.Console, name: str) -> object:
- nonlocal visits
- if name == "execution_id":
- visits += 1
- return getattribute(item, name)
-
- monkeypatch.setattr(p.Console, "__getattribute__", counted)
- clock.now = 10
- host.expire()
- assert visits <= 1000
- assert fold(host.events(notebook_id=notebook.id)) == {}
-
-
-def test_planning_does_not_scan_unrelated_cells(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- host, notebook = host_at(Clock(), history=2)
- run = notebook.run("code", request_id="request")
- visits = 0
- getattribute = p.Execution.__getattribute__
-
- def counted(execution: p.Execution, name: str) -> object:
- nonlocal visits
- if name == "cell_id":
- visits += 1
- return getattribute(execution, name)
-
- monkeypatch.setattr(p.Execution, "__getattribute__", counted)
- notebook.observe(
- be.Planned(
- run.id,
- tuple(
- be.PlannedExecution(
- p.ExecutionId(str(index)), str(index), "hash", None
- )
- for index in range(1000)
- ),
- )
- )
- assert visits <= 2000
- assert (
- sum(
- event.kind == "execution"
- for event in host.events(notebook_id=notebook.id)
- )
- == 1000
- )
diff --git a/tests/_core/test_executions.py b/tests/_core/test_executions.py
new file mode 100644
index 0000000000..ab6c9a0611
--- /dev/null
+++ b/tests/_core/test_executions.py
@@ -0,0 +1,267 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Run and execution transitions, without a stream or a backend."""
+
+from __future__ import annotations
+
+import pytest
+
+from marimo._core import backend as be, protocol
+from marimo._core.errors import NoRuntime
+from marimo._core.executions import Change, Executions, Released, Removed
+from marimo._core.stream import kind_of, utcnow
+
+NOTEBOOK = protocol.NotebookId("n1")
+RUNTIME = protocol.Runtime(
+ id=protocol.RuntimeId("rt-1"),
+ notebook_id=NOTEBOOK,
+ status="running",
+ message=None,
+ marimo_version=None,
+ started_at=utcnow(),
+)
+
+
+def run(run_id: str) -> protocol.Run:
+ return protocol.Run(
+ id=protocol.RunId(run_id),
+ notebook_id=NOTEBOOK,
+ caused_by=None,
+ status="queued",
+ outcome=None,
+ started_at=None,
+ completed_at=None,
+ )
+
+
+def planned(run_id: str, *executions: tuple[str, str | None]) -> be.Planned:
+ """Plan executions as `(id, cell_id)` pairs; a `None` cell is scratch."""
+ return be.Planned(
+ protocol.RunId(run_id),
+ tuple(
+ be.PlannedExecution(
+ protocol.ExecutionId(eid),
+ cell,
+ "hash",
+ "code" if cell is None else None,
+ )
+ for eid, cell in executions
+ ),
+ )
+
+
+def complete(
+ executions: Executions,
+ eid: str,
+ outcome: protocol.Outcome | None = None,
+) -> list[Change]:
+ execution = protocol.ExecutionId(eid)
+ changes: list[Change] = []
+ for msg in (
+ be.Status(execution, "running"),
+ be.Concluded(execution, outcome or protocol.ValueOutcome(None)),
+ be.Status(execution, "completed"),
+ ):
+ changes.extend(executions.observe(msg, RUNTIME))
+ return changes
+
+
+def describe(changes: list[Change]) -> list[str]:
+ """Render changes as `kind id`, with `.removed` and `released` marked."""
+ out: list[str] = []
+ for change in changes:
+ if isinstance(change, Released):
+ out.append(f"released {change.run_id}")
+ elif isinstance(change, Removed):
+ out.append(f"{change.kind}.removed {change.id}")
+ else:
+ out.append(f"{kind_of(change)} {change.id}")
+ return out
+
+
+def test_a_cell_result_outlives_its_run_until_the_next_replaces_it() -> None:
+ executions = Executions()
+ executions.begin(run("r1"), "first")
+ executions.observe(planned("r1", ("e1", "a")), RUNTIME)
+ complete(executions, "e1")
+ done = executions.observe(
+ be.Done(protocol.RunId("r1"), "succeeded"), RUNTIME
+ )
+ assert describe(done) == ["run r1"]
+ assert executions.active() is None
+ assert describe(executions._records()) == ["run r1", "execution e1"]
+ assert executions.request_id(protocol.RunId("r1")) == "first"
+ assert executions.snapshot_request(protocol.RunId("r1")) is None
+
+ executions.begin(run("r2"), "second")
+ replaced = executions.observe(planned("r2", ("e2", "a")), RUNTIME)
+ assert describe(replaced) == [
+ "execution.removed e1",
+ "released r1",
+ "execution e2",
+ ]
+ executions.release(protocol.RunId("r1"))
+ assert executions.run(protocol.RunId("r1")) is None
+ assert describe(executions._records()) == ["run r2", "execution e2"]
+
+
+def test_scratch_work_leaves_with_its_run() -> None:
+ executions = Executions()
+ executions.begin(run("r1"), "request")
+ executions.observe(planned("r1", ("e1", None)), RUNTIME)
+ produced = executions.observe(
+ be.Produced(
+ protocol.Console(
+ protocol.ExecutionId("e1"),
+ protocol.ItemId("c1"),
+ "stdout",
+ 0,
+ "x",
+ )
+ ),
+ RUNTIME,
+ )
+ assert describe(produced) == ["console c1"]
+ assert executions.run_id_of(produced[0]) == "r1"
+ complete(executions, "e1")
+ done = executions.observe(
+ be.Done(protocol.RunId("r1"), "succeeded"), RUNTIME
+ )
+ assert describe(done) == ["run r1", "released r1"]
+ executions.release(protocol.RunId("r1"))
+ assert executions._records() == []
+
+
+def test_a_run_completes_only_after_its_executions_and_reports_failures() -> (
+ None
+):
+ executions = Executions()
+ executions.begin(run("r1"), "request")
+ executions.observe(planned("r1", ("e1", "a")), RUNTIME)
+ executions.observe(
+ be.Status(protocol.ExecutionId("e1"), "running"), RUNTIME
+ )
+ with pytest.raises(ValueError, match="unfinished"):
+ executions.observe(be.Done(protocol.RunId("r1"), "succeeded"), RUNTIME)
+ complete(executions, "e1", protocol.InterruptedOutcome())
+ with pytest.raises(ValueError, match="successful"):
+ executions.observe(be.Done(protocol.RunId("r1"), "succeeded"), RUNTIME)
+ done = executions.observe(be.Done(protocol.RunId("r1"), "failed"), RUNTIME)
+ assert [
+ change.outcome for change in done if isinstance(change, protocol.Run)
+ ] == ["failed"]
+
+
+def test_interrupting_completes_everything_in_flight() -> None:
+ executions = Executions()
+ executions.begin(run("r1"), "request")
+ executions.observe(planned("r1", ("e1", "a"), ("e2", None)), RUNTIME)
+ for eid in ("e1", "e2"):
+ executions.observe(
+ be.Status(protocol.ExecutionId(eid), "running"), RUNTIME
+ )
+ changes = executions.interrupt()
+ assert describe(changes) == ["execution e1", "execution e2", "run r1"]
+ assert all(
+ isinstance(change.outcome, protocol.InterruptedOutcome)
+ for change in changes
+ if isinstance(change, protocol.Execution)
+ )
+ # The scratch execution is forgotten silently; the cell result stays.
+ assert describe(executions._records()) == ["run r1", "execution e1"]
+
+
+def test_clearing_removes_results_and_releases_runs() -> None:
+ executions = Executions()
+ executions.begin(run("r1"), "request")
+ executions.observe(planned("r1", ("e1", "a")), RUNTIME)
+ executions.observe(
+ be.Produced(
+ protocol.Console(
+ protocol.ExecutionId("e1"),
+ protocol.ItemId("c1"),
+ "stdout",
+ 0,
+ "x",
+ )
+ ),
+ RUNTIME,
+ )
+ complete(executions, "e1")
+ executions.observe(be.Done(protocol.RunId("r1"), "succeeded"), RUNTIME)
+ assert describe(executions.clear()) == [
+ "console.removed c1",
+ "execution.removed e1",
+ "released r1",
+ ]
+
+
+def test_scheduling_needs_a_runtime_but_stray_updates_do_not() -> None:
+ executions = Executions()
+ with pytest.raises(NoRuntime):
+ executions.observe(be.Scheduled(protocol.RunId("r1")), None)
+ assert (
+ executions.observe(
+ be.Status(protocol.ExecutionId("e1"), "running"), None
+ )
+ == []
+ )
+ scheduled = executions.observe(be.Scheduled(protocol.RunId("r1")), RUNTIME)
+ assert describe(scheduled) == ["run r1"]
+ assert executions.request_id(protocol.RunId("r1")) is None
+
+
+def test_planning_does_not_scan_unrelated_cells(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ executions = Executions()
+ executions.begin(run("r1"), "request")
+ visits = 0
+ getattribute = protocol.Execution.__getattribute__
+
+ def counted(execution: protocol.Execution, name: str) -> object:
+ nonlocal visits
+ if name == "cell_id":
+ visits += 1
+ return getattribute(execution, name)
+
+ monkeypatch.setattr(protocol.Execution, "__getattribute__", counted)
+ changes = executions.observe(
+ planned("r1", *((str(index), str(index)) for index in range(1000))),
+ RUNTIME,
+ )
+ assert visits <= 3000
+ assert sum(isinstance(c, protocol.Execution) for c in changes) == 1000
+
+
+def test_current_result_keeps_its_outputs_as_the_execution_changes() -> None:
+ executions = Executions()
+ cell_id = protocol.CellId("a")
+ assert executions.current(cell_id) is None
+ executions.begin(run("r1"), "request")
+ executions.observe(planned("r1", ("e1", "a")), RUNTIME)
+ output = protocol.Display(
+ execution_id=protocol.ExecutionId("e1"),
+ id=protocol.ItemId("d1"),
+ seq=0,
+ mimetype="application/json",
+ data='{"values":[1]}',
+ )
+ executions.observe(be.Produced(output), RUNTIME)
+ assert executions.current(cell_id) is None
+ complete(executions, "e1")
+ result = executions.current(cell_id)
+ assert result is not None
+ assert result.outputs == (output,)
+
+ updated = protocol.Display(
+ execution_id=output.execution_id,
+ id=output.id,
+ seq=output.seq,
+ mimetype=output.mimetype,
+ data='{"values":[1,2]}',
+ )
+ executions.observe(be.Produced(updated), RUNTIME)
+ current = executions.current(cell_id)
+ assert current is not None
+ assert current.outputs == (updated,)
+ assert result.outputs[0].data == '{"values":[1]}'
diff --git a/tests/_core/test_host.py b/tests/_core/test_host.py
index b838eb01cb..ca469865e2 100644
--- a/tests/_core/test_host.py
+++ b/tests/_core/test_host.py
@@ -14,8 +14,13 @@ from marimo._core.errors import NotFound
from marimo._core.host import Host
from marimo._core.stream import Event
from tests._core.conftest import (
+ Clock,
ScriptedBackend,
+ finish,
+ fold,
+ host_at,
names,
+ plan,
project,
record,
runtime_id_of,
@@ -27,6 +32,79 @@ if TYPE_CHECKING:
from marimo._core.notebook import Notebook
[email protected]("trigger", ["runtime", "run"])
[email protected]("source", ["host", "notebook"])
+def test_a_subscriber_can_remove_a_notebook_after_its_children(
+ host: Host, notebook: Notebook, trigger: str, source: str
+) -> None:
+ if trigger == "run":
+ notebook.start()
+ notebook.observe(be.Running(runtime_id_of(notebook)))
+ events: list[Event] = []
+
+ def remove(event: Event) -> None:
+ events.append(event)
+ if event.name == trigger:
+ host.remove_notebook(notebook.id)
+
+ if source == "host":
+ host.subscribe(remove, notebook_id=notebook.id)
+ else:
+ host.subscribe(events.append, notebook_id=notebook.id)
+ notebook.subscribe(
+ lambda event: (
+ host.remove_notebook(notebook.id)
+ if event.name == trigger
+ else None
+ )
+ )
+ if trigger == "run":
+ notebook.run("1", request_id="request")
+ assert names(events) == [
+ "run",
+ "run",
+ "runtime.removed",
+ "run.removed",
+ "notebook.removed",
+ ]
+ else:
+ notebook.start()
+ assert names(events) == [
+ "runtime",
+ "runtime.removed",
+ "notebook.removed",
+ ]
+ assert notebook.id not in host.notebooks
+
+
+def test_replacing_a_cell_result_updates_history_before_delivery(
+ host: Host, notebook: Notebook
+) -> None:
+ notebook.start()
+ notebook.observe(be.Running(runtime_id_of(notebook)))
+ run = notebook.run("1", request_id="first")
+ execution = plan(notebook, run, "old", cell="a")
+ notebook.observe(
+ be.Produced(
+ protocol.Console(
+ execution, protocol.ItemId("out"), "stdout", 0, "old output"
+ )
+ )
+ )
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ snapshots = []
+
+ def read_snapshot(event: Event) -> None:
+ if event.name == "console.removed":
+ snapshots.append(set(fold(host.events(notebook_id=notebook.id))))
+
+ host.subscribe(read_snapshot, notebook_id=notebook.id)
+ replacement = notebook.run("2", request_id="second")
+ plan(notebook, replacement, "new", cell="a")
+ assert snapshots == [{("run", replacement.id), ("execution", "new")}]
+
+
def test_catalog_snapshot_and_replay_exclude_runs(host: Host) -> None:
host.update_project(project())
notebook = host.add_notebook(record(), ScriptedBackend())
@@ -162,9 +240,9 @@ def test_removing_a_notebook_removes_what_it_owns_first(host: Host) -> None:
assert names(log) == [
"run",
- "run.removed",
"attachment.removed",
"runtime.removed",
+ "run.removed",
"notebook.removed",
]
assert getattr(log[-1].data, "request_id", None) == "req-del"
@@ -299,3 +377,266 @@ def test_catalog_updates_do_not_share_mutable_records(host: Host) -> None:
assert host.notebooks[original.id] == replace(
record(), name="caller mutation", operations=()
)
+
+
[email protected]("history", [2, 1000])
+def test_expiration_converges_for_live_replay_and_snapshot(
+ history: int,
+) -> None:
+ clock = Clock()
+ host, notebook = host_at(clock, history)
+ run = notebook.run("code", request_id="request")
+ execution = plan(notebook, run, "e")
+ item = protocol.Console(
+ protocol.ExecutionId("e"),
+ protocol.ItemId("console"),
+ "stdout",
+ 0,
+ "output",
+ )
+ notebook.observe(be.Produced(item))
+ notebook.observe(
+ be.Produced(
+ protocol.Display(
+ execution_id=execution,
+ id=protocol.ItemId("display"),
+ seq=1,
+ mimetype="text/html",
+ data="value",
+ )
+ )
+ )
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ snapshot = host.events(notebook_id=notebook.id, run_id=run.id)
+ initial = fold(snapshot)
+ assert {kind for kind, _ in initial} == {
+ "run",
+ "execution",
+ "console",
+ "display",
+ }
+ live: list[Event] = []
+ host.subscribe(live.append, notebook_id=notebook.id, run_id=run.id)
+ clock.now = 9
+ host.expire()
+ assert live == []
+ clock.now = 10
+ host.expire()
+ assert names(live) == [
+ "console.removed",
+ "display.removed",
+ "execution.removed",
+ "run.removed",
+ ]
+ assert all(event.data.request_id == "request" for event in live)
+ replay = host.events(
+ snapshot[-1].id, notebook_id=notebook.id, run_id=run.id
+ )
+ assert (replay[0].name == "reset") == (history == 2)
+ assert (
+ fold(live, initial)
+ == fold(replay, initial)
+ == fold(host.events(notebook_id=notebook.id, run_id=run.id))
+ == {}
+ )
+ live.clear()
+ notebook.observe(be.Produced(item))
+ notebook.observe(be.Status(execution, "running"))
+ notebook.observe(be.Done(run.id, "succeeded"))
+ assert live == []
+
+
+def test_cell_results_outlive_ephemeral_members_and_old_runs_are_removed() -> (
+ None
+):
+ clock = Clock()
+ host, notebook = host_at(clock)
+ run = notebook.run("code", request_id="first")
+ ephemeral = plan(notebook, run, "scratch")
+ cell = plan(notebook, run, "cell", "a")
+ finish(notebook, ephemeral)
+ clock.now = 100
+ host.expire()
+ assert ("execution", ephemeral) in fold(
+ host.events(notebook_id=notebook.id)
+ )
+ finish(notebook, cell)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ clock.now = 110
+ host.expire()
+ state = fold(host.events(notebook_id=notebook.id))
+ assert set(state) == {("run", run.id), ("execution", cell)}
+ replacement = notebook.run("code", request_id="second")
+ current = plan(notebook, replacement, "current", "a")
+ finish(notebook, current)
+ notebook.observe(be.Done(replacement.id, "succeeded"))
+ clock.now = 1000
+ state = fold(host.events(notebook_id=notebook.id))
+ assert set(state) == {("run", replacement.id), ("execution", current)}
+
+
+def test_run_survives_until_its_last_cell_execution_is_replaced() -> None:
+ host, notebook = host_at(Clock())
+ original = notebook.run("code", request_id="original")
+ first = plan(notebook, original, "first", "a")
+ second = plan(notebook, original, "second", "b")
+ finish(notebook, first)
+ finish(notebook, second)
+ notebook.observe(be.Done(original.id, "succeeded"))
+ live: list[Event] = []
+ host.subscribe(live.append, notebook_id=notebook.id, run_id=original.id)
+
+ replacement = notebook.run("code", request_id="replacement")
+ current_a = plan(notebook, replacement, "current-a", "a")
+ assert set(fold(host.events(notebook_id=notebook.id))) == {
+ ("run", original.id),
+ ("run", replacement.id),
+ ("execution", second),
+ ("execution", current_a),
+ }
+ assert names(live) == ["execution.removed"]
+
+ current_b = plan(notebook, replacement, "current-b", "b")
+ assert set(fold(host.events(notebook_id=notebook.id))) == {
+ ("run", replacement.id),
+ ("execution", current_a),
+ ("execution", current_b),
+ }
+ assert names(live) == [
+ "execution.removed",
+ "execution.removed",
+ "run.removed",
+ ]
+
+
+def test_stopping_starts_expiry_for_active_work_without_extending_old_results() -> (
+ None
+):
+ clock = Clock()
+ host, notebook = host_at(clock)
+ done = notebook.run("code", request_id="done")
+ execution = plan(notebook, done, "done")
+ finish(notebook, execution)
+ notebook.observe(be.Done(done.id, "succeeded"))
+ clock.now = 5
+ active = notebook.run("sleep", request_id="active")
+ plan(notebook, active, "active")
+ notebook.stop()
+ notebook.observe(be.Exited(runtime_id_of(notebook)))
+ clock.now = 10
+ host.expire()
+ state = fold(host.events(notebook_id=notebook.id))
+ assert set(state) == {("run", active.id), ("execution", "active")}
+ assert state["run", active.id].outcome == "interrupted"
+ clock.now = 15
+ host.expire()
+ assert fold(host.events(notebook_id=notebook.id)) == {}
+
+
+def test_repeated_expiration_releases_runs_and_request_correlations() -> None:
+ clock = Clock()
+ host, notebook = host_at(clock, history=2)
+ for index in range(100):
+ run = notebook.run("code", request_id=f"request-{index}")
+ execution = plan(notebook, run, str(index))
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ clock.now += 10
+ host.expire()
+ assert fold(host.events(notebook_id=notebook.id)) == {}
+ assert (
+ len({call[1] for call in notebook._backend.calls if call[0] == "run"})
+ == 100
+ )
+
+
+def test_removal_listeners_can_read_snapshots() -> None:
+ clock = Clock()
+ host, notebook = host_at(clock)
+ run = notebook.run("code", request_id="request")
+ execution = plan(notebook, run, "execution")
+ notebook.observe(
+ be.Produced(
+ protocol.Console(
+ execution, protocol.ItemId("console"), "stdout", 0, "x"
+ )
+ )
+ )
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ snapshots: list[dict] = []
+ host.subscribe(
+ lambda _: snapshots.append(fold(host.events(notebook_id=notebook.id))),
+ notebook_id=notebook.id,
+ )
+ clock.now = 10
+ host.expire()
+ assert [set(state) for state in snapshots] == [set(), set(), set()]
+
+
+def test_expiration_does_not_scan_unrelated_outputs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ clock = Clock()
+ host, notebook = host_at(clock, history=2)
+ run = notebook.run("code", request_id="request")
+ for index in range(1000):
+ execution = plan(notebook, run, str(index))
+ notebook.observe(
+ be.Produced(
+ protocol.Console(
+ execution, protocol.ItemId(str(index)), "stdout", 0, "x"
+ )
+ )
+ )
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ visits = 0
+ getattribute = protocol.Console.__getattribute__
+
+ def counted(item: protocol.Console, name: str) -> object:
+ nonlocal visits
+ if name == "execution_id":
+ visits += 1
+ return getattribute(item, name)
+
+ monkeypatch.setattr(protocol.Console, "__getattribute__", counted)
+ clock.now = 10
+ host.expire()
+ assert visits <= 1000
+ assert fold(host.events(notebook_id=notebook.id)) == {}
+
+
+def test_completed_scratch_runs_belong_to_the_host() -> None:
+ clock = Clock()
+ host, notebook = host_at(clock)
+ run = notebook.run("code", request_id="request")
+ execution = plan(notebook, run, "scratch")
+ notebook.observe(
+ be.Produced(
+ protocol.Console(
+ execution, protocol.ItemId("out"), "stdout", 0, "x"
+ )
+ )
+ )
+ assert ("execution", execution) in fold(notebook._snapshot())
+ live: list[Event] = []
+ host.subscribe(live.append, notebook_id=notebook.id)
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ # The notebook lets go without a word; the host still shows the run.
+ assert fold(notebook._snapshot()) == {}
+ assert names(live) == ["execution", "execution", "run"]
+ assert set(fold(host.events(notebook_id=notebook.id))) == {
+ ("run", run.id),
+ ("execution", execution),
+ ("console", "out"),
+ }
+ assert notebook._backend.calls.count(("forget", run.id)) == 1
+ with pytest.raises(NotFound):
+ notebook.interrupt(run.id)
+ clock.now = 10
+ host.expire()
+ assert fold(host.events(notebook_id=notebook.id)) == {}
+ assert notebook._backend.calls.count(("forget", run.id)) == 1
diff --git a/tests/_core/test_notebook.py b/tests/_core/test_notebook.py
index 937b38c377..4d9ae64c85 100644
--- a/tests/_core/test_notebook.py
+++ b/tests/_core/test_notebook.py
@@ -20,6 +20,7 @@ from marimo._core.protocol import (
ConsoleMessage,
Custom,
CustomMessage,
+ Display,
DisplayData,
Error,
ErrorOutcome,
@@ -40,8 +41,13 @@ from marimo._core.protocol import (
)
from marimo._core.stream import Event
from tests._core.conftest import (
+ Clock,
ScriptedBackend,
+ finish,
+ fold,
+ host_at,
names,
+ plan,
request_ids,
runtime_id_of,
)
@@ -290,6 +296,7 @@ class TestRestart:
assert notebook.runtime.id == response.id
assert backend.calls == [
("stop", before.id),
+ ("forget", run.id),
("start", notebook.runtime.id),
]
notebook.observe(
@@ -461,31 +468,20 @@ class TestRun:
notebook.run("2", request_id="next")
notebook.attach()
- # Output can arrive after completed and while another run is active.
+ # Completed scratch outputs live in the host history; late reports
+ # cannot recreate notebook state.
notebook.observe(be.Produced(console("late", "e1", "background")))
- assert names(live) == ["console"]
+ assert live == []
assert names(
host.events(cursor, notebook_id=notebook.id, run_id=run.id)
- ) == ["console", "ready"]
+ ) == ["ready"]
assert names(host.events(notebook_id=notebook.id, run_id=run.id)) == [
"run",
"execution",
- "console",
"ready",
]
- assert names(
- host.events(
- notebook_id=notebook.id, run_id=run.id, types={"console"}
- )
- ) == ["console", "ready"]
-
host.remove_notebook(notebook.id)
- assert names(live) == [
- "console",
- "console.removed",
- "execution.removed",
- "run.removed",
- ]
+ assert names(live) == ["execution.removed", "run.removed"]
unsubscribe()
def test_needs_a_running_runtime(self, notebook: Notebook) -> None:
@@ -525,7 +521,7 @@ class TestRun:
notebook.observe(be.Done(run.id, "succeeded"))
assert notebook.runtime is not None
- assert backend.calls == [("run", run.id, "3 * 3")]
+ assert backend.calls == [("run", run.id, "3 * 3"), ("forget", run.id)]
assert names(log) == [
"run",
"execution",
@@ -696,7 +692,7 @@ class TestRunCompletion:
log.clear()
notebook.observe(be.Scheduled(RunId("later"), ExecutionId("e1")))
notebook.observe(be.Done(RunId("later"), "succeeded"))
- assert request_ids(log) == [UNSET, UNSET, UNSET]
+ assert request_ids(log) == [UNSET, UNSET]
assert all(event.run_id == "later" for event in log)
final = next(
event.data
@@ -705,9 +701,7 @@ class TestRunCompletion:
)
assert isinstance(final, RunMessage)
assert final.run.caused_by == "e1"
- retained = notebook._state.run(original.id)
- assert retained is not None
- assert retained.status == "completed"
+ assert notebook.executions.run(original.id) is None
@pytest.mark.parametrize("outcome", ["failed", "interrupted"])
def test_can_end_before_any_execution(
@@ -780,7 +774,7 @@ class TestRunCompletion:
)
notebook.observe(be.Status(ExecutionId("child"), "completed"))
# Even an empty execution queue is not an authoritative end to scheduling.
- current = notebook._state.run(run.id)
+ current = notebook.executions.run(run.id)
assert current is not None
assert current.status == "running"
assert current.outcome is None
@@ -796,7 +790,7 @@ class TestRunCompletion:
assert final.run.outcome == outcome
assert final.run.status == "completed"
log.clear()
- # Final records cannot reopen or acquire executions, but output keeps its origin.
+ # Released scratch records cannot reopen or acquire more output.
notebook.observe(be.Status(ExecutionId("e1"), "running"))
notebook.observe(be.Concluded(ExecutionId("e1"), value("changed")))
notebook.observe(be.Done(run.id, "succeeded"))
@@ -814,7 +808,7 @@ class TestRunCompletion:
notebook.observe(
be.Produced(console("late-output", "e1", "still owned"))
)
- assert names(log) == ["console"]
+ assert log == []
class TestInterrupt:
@@ -839,7 +833,7 @@ class TestInterrupt:
notebook.observe(be.Concluded(ExecutionId("e1"), value("9")))
notebook.observe(be.Status(ExecutionId("e1"), "completed"))
notebook.observe(be.Done(run.id, "succeeded"))
- with pytest.raises(Conflict):
+ with pytest.raises(NotFound):
notebook.interrupt(run.id)
@@ -914,7 +908,7 @@ class TestCellResults:
notebook.observe(
be.Done(second.id, "failed" if same_run else "succeeded")
)
- assert not notebook._state._retired_executions
+ assert not any(r.retired for r in notebook.executions._runs.values())
log.clear()
notebook.observe(be.Produced(item))
assert not log
@@ -1031,3 +1025,95 @@ def _attachment_ids(notebook: Notebook) -> set[str]:
for event in notebook._snapshot()
if isinstance(event.data, AttachmentMessage)
}
+
+
[email protected]("cleanup", ["expire", "replace", "clear"])
+def test_outputs_follow_their_execution_through_updates_and_removal(
+ cleanup: str,
+) -> None:
+ clock = Clock()
+ host, notebook = host_at(clock)
+ run = notebook.run("code", request_id="request")
+ first = plan(notebook, run, "first", "a" if cleanup != "expire" else None)
+ second = plan(
+ notebook, run, "second", "b" if cleanup != "expire" else None
+ )
+ first_console = Console(first, ItemId("first-console"), "stdout", 0, "a")
+ first_display = Display(
+ execution_id=first,
+ id=ItemId("first-display"),
+ seq=1,
+ mimetype="text/plain",
+ data="b",
+ )
+ second_console = Console(
+ second, ItemId("second-console"), "stdout", 0, "c"
+ )
+ second_display = Display(
+ execution_id=second,
+ id=ItemId("second-display"),
+ seq=1,
+ mimetype="text/plain",
+ data="d",
+ )
+ for item in (second_console, first_console, second_display, first_display):
+ notebook.observe(be.Produced(item))
+ first_console = replace(first_console, text="updated")
+ notebook.observe(be.Produced(first_console))
+ finish(notebook, first)
+ finish(notebook, second)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ initial = host.events(notebook_id=notebook.id)
+ assert [
+ getattr(event.data, event.kind)
+ for event in initial
+ if event.kind in {"console", "display"}
+ ] == [first_console, first_display, second_console, second_display]
+
+ live: list[Event] = []
+ snapshots: list[dict] = []
+
+ def updated(event: Event) -> None:
+ live.append(event)
+ snapshots.append(fold(notebook._snapshot()))
+
+ host.subscribe(
+ updated,
+ notebook_id=notebook.id,
+ types={"run", "execution", "console", "display"},
+ )
+ if cleanup == "expire":
+ clock.now = 10
+ host.expire()
+ elif cleanup == "replace":
+ replacement = notebook.run("code", request_id="replacement")
+ plan(notebook, replacement, "new", "a")
+ else:
+ host.remove_notebook(notebook.id)
+
+ assert [
+ (event.name, event.data.id)
+ for event in live
+ if event.name.endswith(".removed")
+ ] == [
+ ("console.removed", first_console.id),
+ ("display.removed", first_display.id),
+ ("execution.removed", first),
+ *(
+ []
+ if cleanup == "replace"
+ else [
+ ("console.removed", second_console.id),
+ ("display.removed", second_display.id),
+ ("execution.removed", second),
+ ("run.removed", run.id),
+ ]
+ ),
+ ]
+ assert fold(live, fold(initial)) == fold(notebook._snapshot())
+ # A transition updates state before notifying observers.
+ assert all(
+ ("execution", first) not in state
+ for event, state in zip(live, snapshots, strict=True)
+ if event.name.endswith(".removed")
+ )
diff --git a/tests/_core/test_stream.py b/tests/_core/test_stream.py
index cacd3ad324..316ecfbb48 100644
--- a/tests/_core/test_stream.py
+++ b/tests/_core/test_stream.py
@@ -98,6 +98,39 @@ def test_readers_cannot_change_other_readers_or_replay() -> None:
assert reread[0].data.project == project()
[email protected]("external", [False, True])
+def test_mutable_payloads_are_isolated_from_callers_and_listeners(
+ external: bool,
+) -> None:
+ stream = Stream()
+ payload = protocol.Custom(type="example", data={"values": [1]})
+ delivered: list[Event] = []
+
+ def mutate(event: Event) -> None:
+ assert isinstance(event.data, protocol.CustomMessage)
+ event.data.custom.data["values"].append(2)
+
+ stream.subscribe(mutate)
+ stream.subscribe(delivered.append)
+ if external:
+ event = Event(
+ "custom",
+ protocol.CustomMessage(custom=payload, time=msgspec.UNSET),
+ )
+ published = stream.append(event)
+ else:
+ published = stream.publish(payload)
+ payload.data["values"].append(3)
+ assert isinstance(published.data, protocol.CustomMessage)
+ published.data.custom.data["values"].append(4)
+ replay = stream.since(0)
+ assert replay is not None
+ assert [event.data.custom.data for event in [*delivered, *replay]] == [
+ {"values": [1]},
+ {"values": [1]},
+ ]
+
+
def test_deferred_events_join_history_before_delivery() -> None:
stream = Stream()
delivered: list[Event] = []
diff --git a/tests/_server/host/test_host_api.py b/tests/_server/host/test_host_api.py
index 1d0ea9c921..82d4a33b79 100644
--- a/tests/_server/host/test_host_api.py
+++ b/tests/_server/host/test_host_api.py
@@ -15,9 +15,9 @@ import pytest
from starlette.routing import Route
from marimo._core import backend as be, protocol
-from marimo._core.execution_state import RetentionPolicy
from marimo._core.host import Host
from marimo._core.protocol import ProjectMessage
+from marimo._core.run_log import RetentionPolicy
from marimo._server.host import api as api_module
from marimo._server.host.bridge import SessionBridge
from marimo._server.host.context import HostContext
@@ -913,6 +913,41 @@ async def test_run_filter_over_http(harness: Harness) -> None:
assert invalid.status == 400
+async def test_interrupting_an_unstarted_run_after_shutdown(
+ harness: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ nid = await harness.running()
+ runs = f"/notebooks/{nid}/runtime/runs"
+ accepted = await finished(
+ harness.app, "POST", runs, headers=key("queued"), body=b'{"code":"1"}'
+ )
+ assert accepted.status == 202
+ run_id = accepted.json()["id"]
+ notebook = harness.host.notebook(nid)
+ runtime_id = runtime_id_of(notebook)
+ notebook.stop()
+ notebook.observe(be.Exited(runtime_id))
+
+ def snapshot_unavailable(*_args: object, **_kwargs: object) -> None:
+ raise AssertionError("run lookup must not build a snapshot")
+
+ monkeypatch.setattr(harness.host, "events", snapshot_unavailable)
+ completed = await finished(
+ harness.app, "DELETE", f"{runs}/{run_id}", headers=key("completed")
+ )
+ assert completed.status == 409
+ unknown = await finished(
+ harness.app, "DELETE", f"{runs}/unknown", headers=key("unknown")
+ )
+ assert unknown.status == 404
+ harness.now = 61
+ harness.host.expire()
+ expired = await finished(
+ harness.app, "DELETE", f"{runs}/{run_id}", headers=key("expired")
+ )
+ assert expired.status == 404
+
+
async def test_scratch_runs_over_http(harness: Harness) -> None:
app, nid = harness.app, harness.notebook_id
runs = f"/notebooks/{nid}/runtime/runs"
diff --git a/tests/_server/host/test_host_bridge.py b/tests/_server/host/test_host_bridge.py
index 0b6ab67b94..ded6f6051a 100644
--- a/tests/_server/host/test_host_bridge.py
+++ b/tests/_server/host/test_host_bridge.py
@@ -219,7 +219,8 @@ async def test_removed_runs_release_backend_translation_state(
session.broadcast(
CompletedRunNotification(run_id="run", outcome="succeeded")
)
- assert set(backend._translators) == {"run"}
+ retained = {"run"} if cell_id != SCRATCH_CELL_ID else set()
+ assert set(backend._translators) == retained
now = 60
bridge.host.expire()
# Cell results are retained, so their translation state stays too.
diff --git a/tests/_server/host/test_host_execution.py b/tests/_server/host/test_host_execution.py
index 38a5a9cd2f..9d415096b2 100644
--- a/tests/_server/host/test_host_execution.py
+++ b/tests/_server/host/test_host_execution.py
@@ -221,18 +221,25 @@ async def test_scratch_outcomes_from_the_kernel(
assert outcome["error"]["message"]
-async def test_background_output_stays_with_its_execution(
+async def test_background_output_after_a_scratch_run_is_dropped(
live_host: tuple[Starlette, str, Response],
) -> None:
+ """A completed scratch run is history; the backend forgets it.
+
+ Output its code produces later, from a background thread, has no
+ current execution to join and never reaches clients.
+ """
app, notebook_id, events = live_host
path = f"/notebooks/{notebook_id}/runtime/runs"
code = """import builtins
import threading
import marimo as mo
builtins._host_test_gate = threading.Event()
+builtins._host_test_printed = threading.Event()
def worker():
builtins._host_test_gate.wait(5)
print("late output")
+ builtins._host_test_printed.set()
mo.Thread(target=worker).start()
"""
first = await finished(
@@ -258,7 +265,7 @@ mo.Thread(target=worker).start()
headers=key("second"),
body=json.dumps(
{
- "code": "import builtins\nimport time\nbuiltins._host_test_gate.set()\ntime.sleep(0.2)\ndel builtins._host_test_gate"
+ "code": "import builtins\nbuiltins._host_test_gate.set()\nassert builtins._host_test_printed.wait(5)\ndel builtins._host_test_gate, builtins._host_test_printed"
}
).encode(),
)
@@ -271,21 +278,21 @@ mo.Thread(target=worker).start()
for r in response.records
),
)
+ assert (
+ next(
+ r["data"]["run"]["outcome"]
+ for r in reversed(events.records)
+ if r.get("data", {}).get("run", {}).get("id")
+ == second.json()["id"]
+ )
+ == "succeeded"
+ )
console = [
r["data"]["console"]
for r in events.records
if r.get("event") == "console"
]
- assert any("late output" in item["text"] for item in console)
- first_execution = next(
- record["data"]["execution"]
- for record in events.records
- if record.get("event") == "execution"
- and record["data"]["execution"]["run_id"] == first.json()["id"]
- )
- assert {item["execution_id"] for item in console} == {
- first_execution["id"]
- }
+ assert not any("late output" in item["text"] for item in console)
async def test_queued_cancellation_never_executes_the_code(