Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Retain host results by execution lifetime (#11188)

Trevor Manz3a5d285e2b36
diff --git a/marimo/_core/backend.py b/marimo/_core/backend.py
index 1205887903..bb335a9d98 100644
--- a/marimo/_core/backend.py
+++ b/marimo/_core/backend.py
@@ -47,6 +47,12 @@ class Backend(Protocol):
def interrupt(self, run_id: protocol.RunId) -> None:
"""Interrupt a run and report an `interrupted` execution outcome."""
+ def forget(self, run_id: protocol.RunId) -> None:
+ """Release a run the host no longer retains.
+
+ Later kernel messages for the run are dropped.
+ """
+
# Runtime lifecycle
diff --git a/marimo/_core/execution_state.py b/marimo/_core/execution_state.py
new file mode 100644
index 0000000000..1996868bfb
--- /dev/null
+++ b/marimo/_core/execution_state.py
@@ -0,0 +1,430 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Execution state for connecting and reconnecting clients.
+
+Reconnecting clients replay past events, so events are retained.
+Retaining all of them is impractical. `ExecutionState` applies the
+host's retention policy and emits removal events so connected clients
+can update their snapshots.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from math import isfinite
+from typing import TYPE_CHECKING
+
+from msgspec.structs import replace
+
+from marimo._core import backend as be, protocol
+from marimo._core.errors import NoRuntime
+from marimo._core.stream import Event, Stream, kind_of, message, utcnow
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+Item = protocol.Console | protocol.Display
+
+
+@dataclass
+class _RetainedExecution:
+ """An execution with the outputs it produced."""
+
+ execution: protocol.Execution
+ outputs: dict[protocol.ItemId, Item] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class RetentionPolicy:
+ """How long the host retains ephemeral results after a run completes."""
+
+ ephemeral_seconds: float = 60.0
+
+ def __post_init__(self) -> None:
+ if not isfinite(self.ephemeral_seconds) or self.ephemeral_seconds < 0:
+ raise ValueError("invalid result retention")
+
+
+class ExecutionState:
+ """Retain current results and remove completed runs with no executions."""
+
+ def __init__(
+ self,
+ notebook_id: protocol.NotebookId,
+ stream: Stream,
+ retention: RetentionPolicy,
+ clock: Callable[[], float],
+ forget: Callable[[protocol.RunId], None],
+ ) -> None:
+ self._id = notebook_id
+ self._stream = stream
+ self._retention = retention
+ self._clock = clock
+ self._forget = forget
+ self._runs: dict[str, protocol.Run] = {}
+ self._run_executions: dict[str, set[protocol.ExecutionId]] = {}
+ self._run_requests: dict[str, str | None] = {}
+ self._executions: dict[protocol.ExecutionId, _RetainedExecution] = {}
+ self._cell_executions: dict[protocol.CellId, protocol.ExecutionId] = {}
+ self._retired_executions: dict[
+ protocol.ExecutionId, protocol.RunId
+ ] = {}
+ self._unsuccessful: set[protocol.RunId] = set()
+ self._completed_at: dict[str, float] = {}
+
+ @property
+ def id(self) -> protocol.NotebookId:
+ return self._id
+
+ @property
+ def active_run(self) -> protocol.Run | None:
+ return next(
+ (r for r in self._runs.values() if r.status != "completed"), None
+ )
+
+ def run(self, run_id: str) -> protocol.Run | None:
+ return self._runs.get(run_id)
+
+ def request_run(self, request_id: str) -> protocol.Run | None:
+ return next(
+ (
+ self._runs[rid]
+ for rid, key in self._run_requests.items()
+ if key == request_id
+ ),
+ None,
+ )
+
+ def add_run(self, run: protocol.Run, request_id: str) -> None:
+ self._run_requests[run.id] = request_id
+ self._set_run(run)
+
+ def expire(self) -> None:
+ now = self._clock()
+ for retained in list(self._executions.values()):
+ execution = retained.execution
+ ended = self._completed_at.get(execution.run_id or "")
+ if (
+ execution.cell_id is None
+ and ended is not None
+ and now >= ended + self._retention.ephemeral_seconds
+ ):
+ self._retire_execution(retained)
+
+ def _prune_run(self, run_id: str | None) -> None:
+ run = self._runs.get(run_id or "")
+ if run is None or run.status != "completed":
+ return
+ if self._run_executions[run.id]:
+ return
+ del self._runs[run.id]
+ del self._run_executions[run.id]
+ self._remove("run", run.id, run.id)
+ self._run_requests.pop(run.id, None)
+ self._completed_at.pop(run.id, None)
+ self._forget(run.id)
+
+ def snapshot(self) -> list[Event]:
+ bodies: list[protocol.Run | protocol.Execution | Item] = [
+ *self._runs.values(),
+ *(retained.execution for retained in self._executions.values()),
+ *(
+ item
+ for retained in self._executions.values()
+ for item in retained.outputs.values()
+ ),
+ ]
+ return [
+ Event(
+ kind_of(body),
+ message(body, request_id=self._pending_request(body)),
+ notebook_id=self.id,
+ run_id=self._run_id(body),
+ )
+ for body in bodies
+ ]
+
+ def _pending_request(
+ self, body: protocol.Run | protocol.Execution | Item
+ ) -> str | None:
+ run_id = self._run_id(body)
+ run = self._runs.get(run_id or "")
+ return (
+ self._run_requests.get(run.id)
+ if run is not None and run.status != "completed"
+ else None
+ )
+
+ def observe(
+ self, msg: be.Message, runtime: protocol.Runtime | None
+ ) -> None:
+ match msg:
+ case be.Scheduled(run_id=run_id, caused_by=caused_by):
+ if run_id not in self._runs:
+ if runtime is None:
+ raise NoRuntime(f"notebook {self.id} has no runtime")
+ self._run_requests[run_id] = None
+ self._set_run(
+ protocol.Run(
+ id=run_id,
+ notebook_id=self.id,
+ caused_by=caused_by,
+ status="queued",
+ outcome=None,
+ started_at=None,
+ completed_at=None,
+ )
+ )
+ case be.Planned(run_id=run_id, executions=planned):
+ self._observe_planned(run_id, planned, runtime)
+ case be.Status(execution_id=execution_id, status=status):
+ self._observe_status(execution_id, status)
+ case be.Produced(item=item):
+ if (
+ retained := self._executions.get(item.execution_id)
+ ) is not None:
+ retained.outputs[item.id] = item
+ self._publish(item)
+ case be.Concluded(execution_id=execution_id, outcome=outcome):
+ if (
+ retained := self._executions.get(execution_id)
+ ) is not None and retained.execution.status != "completed":
+ self._set_execution(
+ replace(retained.execution, outcome=outcome)
+ )
+ case be.Done(run_id=run_id, outcome=outcome):
+ if (
+ run := self._runs.get(run_id)
+ ) is not None and run.status != "completed":
+ members = [
+ self._executions[eid].execution
+ for eid in self._run_executions[run_id]
+ ]
+ if any(
+ execution.status != "completed"
+ for execution in members
+ ):
+ raise ValueError(
+ "a run cannot complete with unfinished executions"
+ )
+ if outcome == "succeeded" and (
+ run_id in self._unsuccessful
+ or any(
+ not isinstance(
+ execution.outcome, protocol.ValueOutcome
+ )
+ for execution in members
+ )
+ ):
+ raise ValueError(
+ "a successful run cannot contain failed or interrupted executions"
+ )
+ self._set_run(
+ replace(
+ run,
+ status="completed",
+ outcome=outcome,
+ completed_at=utcnow(),
+ )
+ )
+ self._retired_executions = {
+ eid: rid
+ for eid, rid in self._retired_executions.items()
+ if rid != run_id
+ }
+ self._unsuccessful.discard(run_id)
+
+ self._prune_run(run_id)
+
+ def clear(self) -> None:
+ self._retired_executions.clear()
+ self._unsuccessful.clear()
+ for retained in list(self._executions.values()):
+ self._remove_execution(retained)
+ for run in list(self._runs.values()):
+ del self._runs[run.id]
+ del self._run_executions[run.id]
+ self._remove("run", run.id, run.id)
+ self._run_requests.pop(run.id, None)
+ self._forget(run.id)
+ self._completed_at.clear()
+
+ def _set_run(self, run: protocol.Run) -> None:
+ self._runs[run.id] = run
+ self._run_executions.setdefault(run.id, set())
+ if run.status == "completed":
+ self._completed_at.setdefault(run.id, self._clock())
+ self._publish(run)
+
+ def _set_execution(self, execution: protocol.Execution) -> None:
+ if (retained := self._executions.get(execution.id)) is None:
+ self._executions[execution.id] = _RetainedExecution(execution)
+ else:
+ retained.execution = execution
+ if execution.cell_id is not None:
+ self._cell_executions[execution.cell_id] = execution.id
+ if execution.run_id is not None:
+ self._run_executions[execution.run_id].add(execution.id)
+ self._publish(execution)
+
+ def interrupt(self, outcome: protocol.RunOutcome = "interrupted") -> None:
+ now = utcnow()
+ for retained in list(self._executions.values()):
+ execution = retained.execution
+ if execution.status != "completed":
+ self._set_execution(
+ replace(
+ execution,
+ status="completed",
+ outcome=protocol.InterruptedOutcome(),
+ completed_at=now,
+ )
+ )
+ for run in list(self._runs.values()):
+ if run.status != "completed":
+ self._set_run(
+ replace(
+ run,
+ status="completed",
+ outcome=outcome,
+ completed_at=now,
+ )
+ )
+ self._retired_executions.clear()
+ self._unsuccessful.clear()
+
+ for run_id in list(self._runs):
+ self._prune_run(run_id)
+
+ def _observe_planned(
+ self,
+ run_id: protocol.RunId,
+ planned: tuple[be.PlannedExecution, ...],
+ runtime: protocol.Runtime | None,
+ ) -> None:
+ if (
+ run := self._runs.get(run_id)
+ ) is None or run.status == "completed":
+ return
+ if runtime is None:
+ raise NoRuntime(f"notebook {self.id} has no runtime")
+ for entry in planned:
+ if (
+ entry.id in self._executions
+ or entry.id in self._retired_executions
+ ):
+ continue
+ if entry.cell_id is not None:
+ previous = self._cell_executions.get(
+ protocol.CellId(entry.cell_id)
+ )
+ if previous is not None:
+ self._retire_execution(self._executions[previous])
+ self._set_execution(
+ protocol.Execution(
+ id=entry.id,
+ run_id=run_id,
+ runtime_id=runtime.id,
+ caused_by=entry.caused_by,
+ cell_id=(
+ protocol.CellId(entry.cell_id)
+ if entry.cell_id is not None
+ else None
+ ),
+ code=entry.code,
+ code_hash=entry.code_hash,
+ bindings=(),
+ status="queued",
+ outcome=None,
+ stale=None,
+ started_at=None,
+ completed_at=None,
+ )
+ )
+
+ def _retire_execution(self, retained: _RetainedExecution) -> None:
+ execution = retained.execution
+ if execution.status != "completed":
+ raise ValueError("cannot replace an unfinished cell execution")
+ run_id = execution.run_id
+ run = self._runs.get(run_id) if run_id is not None else None
+ if run is not None and run.status != "completed":
+ self._retired_executions[execution.id] = run.id
+ if not isinstance(execution.outcome, protocol.ValueOutcome):
+ self._unsuccessful.add(run.id)
+ self._remove_execution(retained)
+ self._prune_run(run_id)
+
+ def _remove_execution(self, retained: _RetainedExecution) -> None:
+ execution = retained.execution
+ run_id = execution.run_id
+ for item in list(retained.outputs.values()):
+ del retained.outputs[item.id]
+ self._remove(kind_of(item), item.id, run_id)
+ del self._executions[execution.id]
+ if execution.cell_id is not None:
+ del self._cell_executions[execution.cell_id]
+ if run_id is not None:
+ self._run_executions[run_id].remove(execution.id)
+ self._remove("execution", execution.id, run_id)
+
+ def _observe_status(
+ self, execution_id: protocol.ExecutionId, status: protocol.Lifecycle
+ ) -> None:
+ retained = self._executions.get(execution_id)
+ if retained is None:
+ return
+ execution = retained.execution
+ if execution.status in (status, "completed"):
+ return
+ now = utcnow()
+ if status == "running":
+ execution = replace(execution, status=status, started_at=now)
+ assert execution.run_id is not None
+ run = self._runs[execution.run_id]
+ if run.status == "queued":
+ self._set_run(replace(run, status="running", started_at=now))
+ elif status == "completed":
+ if execution.outcome is None:
+ raise ValueError("a completed execution must have an outcome")
+ execution = replace(execution, status=status, completed_at=now)
+ else:
+ execution = replace(execution, status=status)
+ self._set_execution(execution)
+
+ def _publish(
+ self,
+ body: protocol.Run | protocol.Execution | Item,
+ ) -> None:
+ run_id = self._run_id(body)
+ self._stream.publish(
+ body,
+ notebook_id=self.id,
+ request_id=self._run_requests.get(run_id or ""),
+ run_id=run_id,
+ )
+
+ def _run_id(
+ self,
+ body: protocol.Run | protocol.Execution | Item,
+ ) -> protocol.RunId | None:
+ if isinstance(body, protocol.Run):
+ return body.id
+ if isinstance(body, protocol.Execution):
+ return body.run_id
+ if isinstance(body, (protocol.Console, protocol.Display)):
+ return self._executions[body.execution_id].execution.run_id
+ return None
+
+ def _remove(
+ self,
+ kind: str,
+ object_id: str,
+ run_id: protocol.RunId | None,
+ ) -> None:
+ self._stream.remove(
+ kind,
+ object_id,
+ notebook_id=self.id,
+ request_id=self._run_requests.get(run_id or ""),
+ run_id=run_id,
+ )
diff --git a/marimo/_core/host.py b/marimo/_core/host.py
index 52a217fe5f..9c76b9973d 100644
--- a/marimo/_core/host.py
+++ b/marimo/_core/host.py
@@ -5,11 +5,13 @@ from __future__ import annotations
from collections.abc import Mapping
from copy import deepcopy
+from time import monotonic
from typing import TYPE_CHECKING
from marimo._core import protocol
from marimo._core.backend import Backend
from marimo._core.errors import NotFound
+from marimo._core.execution_state import RetentionPolicy
from marimo._core.notebook import Notebook
from marimo._core.stream import Event, Listener, Stream, message
@@ -41,8 +43,12 @@ class Host:
*,
operations: Sequence[str],
history: int = 1000,
+ retention: RetentionPolicy | None = None,
+ clock: Callable[[], float] = monotonic,
) -> None:
- self._host = protocol.Host(operations=list(operations))
+ self.retention = retention or RetentionPolicy()
+ self.clock = clock
+ self._host = protocol.Host(operations=tuple(operations))
self._stream = Stream(history=history)
# Lookups accept wire strings; stored objects carry branded IDs.
self._projects: dict[str, protocol.Project] = {}
@@ -81,7 +87,9 @@ class Host:
raise ValueError(
f"project {record.project_id} is not in the catalog"
)
- notebook = Notebook(record.id, backend, self._stream)
+ notebook = Notebook(
+ record.id, backend, self._stream, self.retention, self.clock
+ )
self._records[record.id] = deepcopy(record)
self._notebooks[record.id] = notebook
self._stream.publish(
@@ -184,6 +192,11 @@ class Host:
events = [e for e in missed if e.visible_in(notebook_id)]
return [e for e in [*events, ready] if e.wanted(types, run_id)]
+ def expire(self) -> None:
+ """Sweep results according to the host's retention policy."""
+ for notebook in list(self._notebooks.values()):
+ notebook.expire()
+
def subscribe(
self,
listener: Listener,
diff --git a/marimo/_core/notebook.py b/marimo/_core/notebook.py
index 63582a1703..f6dee8a761 100644
--- a/marimo/_core/notebook.py
+++ b/marimo/_core/notebook.py
@@ -7,26 +7,30 @@ backend updates and publishes events.
from __future__ import annotations
+from time import monotonic
from typing import TYPE_CHECKING
from msgspec.structs import replace
from marimo._core import backend as be, protocol
from marimo._core.errors import Conflict, NoRuntime, NotFound
+from marimo._core.execution_state import ExecutionState, RetentionPolicy
from marimo._core.ids import mint
from marimo._core.stream import Event, kind_of, message, utcnow
if TYPE_CHECKING:
- from marimo._core.stream import Stream
+ from collections.abc import Callable
-Item = protocol.Console | protocol.Display
+ from marimo._core.stream import Stream
class Notebook:
"""A notebook's state and commands.
- Snapshots preserve insertion order for runs, executions, and items.
- Completed results remain available across runtime stops and restarts.
+ Snapshots preserve insertion order for runs and executions, and for
+ outputs within each execution.
+ Current cell results survive runtime stops and restarts. The host
+ chooses how long to retain completed ephemeral results.
Removing the notebook closes it; further commands raise `NotFound`.
"""
@@ -35,29 +39,39 @@ class Notebook:
notebook_id: protocol.NotebookId,
backend: be.Backend,
stream: Stream,
+ retention: RetentionPolicy | None = None,
+ clock: Callable[[], float] = monotonic,
) -> None:
self._id = notebook_id
self._backend = backend
self._stream = stream
self._closed = False
- self.runtime: protocol.Runtime | None = None
+ self._runtime: protocol.Runtime | None = None
self._attachments: dict[str, protocol.Attachment] = {}
- self._runs: dict[str, protocol.Run] = {}
- self._run_requests: dict[str, str | None] = {}
- self._executions: dict[protocol.ExecutionId, protocol.Execution] = {}
- # Superseded results still contribute to their unfinished run.
- self._retired_executions: dict[
- protocol.ExecutionId, protocol.Execution
- ] = {}
- self._items: dict[protocol.ItemId, Item] = {}
+ self._state = ExecutionState(
+ notebook_id,
+ stream,
+ retention or RetentionPolicy(),
+ clock,
+ forget=backend.forget,
+ )
self._replacement: protocol.Runtime | None = None
# Associate runtime events with the pending start, stop, or restart.
self._lifecycle_request: str | None = None
+ def expire(self) -> None:
+ """Apply the host's result retention policy."""
+ self._state.expire()
+
@property
def id(self) -> protocol.NotebookId:
return self._id
+ @property
+ def runtime(self) -> protocol.Runtime | None:
+ """The current runtime, updated by commands and backend observations."""
+ return self._runtime
+
def _require_open(self) -> None:
if self._closed:
raise NotFound(f"no notebook {self.id}")
@@ -73,8 +87,8 @@ class Notebook:
An existing runtime is returned unchanged, regardless of status.
"""
self._require_open()
- if self.runtime is not None:
- return self.runtime, False
+ if self._runtime is not None:
+ return self._runtime, False
runtime = self._new_runtime("starting", None, None)
self._lifecycle_request = request_id
self._set_runtime(runtime)
@@ -130,22 +144,22 @@ class Notebook:
return replacement
def _require_runtime(self) -> protocol.Runtime:
- if self.runtime is None:
+ if self._runtime is None:
raise NoRuntime(
f"notebook {self.id} has no runtime; start one first."
)
- return self.runtime
+ return self._runtime
def _set_runtime(self, runtime: protocol.Runtime) -> None:
- self.runtime = runtime
+ self._runtime = runtime
self._publish(runtime, self._lifecycle_request)
if runtime.status in ("running", "failed"):
self._lifecycle_request = None
def _end_runtime(self) -> None:
- assert self.runtime is not None
- self._interrupt_in_flight()
- runtime, self.runtime = self.runtime, None
+ assert self._runtime is not None
+ self._state.interrupt()
+ runtime, self._runtime = self._runtime, None
self._remove("runtime", runtime.id, self._lifecycle_request)
self._lifecycle_request = None
@@ -162,15 +176,14 @@ class Notebook:
Conflict: If the runtime is not running, or a run is in progress.
"""
self._require_open()
- run = self._runs.get(request_id)
+ run = self._state.request_run(request_id)
if run is not None:
return run
runtime = self._require_runtime()
if runtime.status != "running":
raise Conflict(f"the runtime is {runtime.status}.")
- in_flight = [r for r in self._runs.values() if r.status != "completed"]
- if in_flight:
- raise Conflict(f"run {in_flight[0].id} is in progress.")
+ if (active := self._state.active_run) is not None:
+ raise Conflict(f"run {active.id} is in progress.")
run = protocol.Run(
id=protocol.RunId(request_id),
notebook_id=self.id,
@@ -180,8 +193,7 @@ class Notebook:
started_at=None,
completed_at=None,
)
- self._run_requests[run.id] = request_id
- self._set_run(run)
+ self._state.add_run(run, request_id)
self._backend.run(run.id, code)
return run
@@ -193,7 +205,7 @@ class Notebook:
Conflict: If the run is already completed.
"""
self._require_open()
- run = self._runs.get(run_id)
+ run = self._state.run(run_id)
if run is None:
raise NotFound(f"notebook {self.id} has no run {run_id}.")
if run.status == "completed":
@@ -201,59 +213,6 @@ class Notebook:
self._backend.interrupt(run.id)
return run
- def _clear_runs(self) -> None:
- self._retired_executions.clear()
- for item in list(self._items.values()):
- del self._items[item.id]
- run_id = self._executions[item.execution_id].run_id
- self._remove(kind_of(item), item.id, run_id, run_id=run_id)
- for execution in list(self._executions.values()):
- del self._executions[execution.id]
- self._remove(
- "execution",
- execution.id,
- execution.run_id,
- run_id=execution.run_id,
- )
- for run in list(self._runs.values()):
- del self._runs[run.id]
- self._remove("run", run.id, run.id, run_id=run.id)
- self._run_requests.pop(run.id, None)
-
- def _set_run(self, run: protocol.Run) -> None:
- self._runs[run.id] = run
- self._publish(run, run.id)
-
- def _set_execution(self, execution: protocol.Execution) -> None:
- self._executions[execution.id] = execution
- self._publish(execution, execution.run_id)
-
- def _interrupt_in_flight(
- self, outcome: protocol.RunOutcome = "interrupted"
- ) -> None:
- now = utcnow()
- for execution in list(self._executions.values()):
- if execution.status != "completed":
- self._set_execution(
- replace(
- execution,
- status="completed",
- outcome=protocol.InterruptedOutcome(),
- completed_at=now,
- )
- )
- for run in list(self._runs.values()):
- if run.status != "completed":
- self._set_run(
- replace(
- run,
- status="completed",
- outcome=outcome,
- completed_at=now,
- )
- )
- self._retired_executions.clear()
-
# ---- attachments ---------------------------------------------------
def attach(self) -> protocol.Attachment:
@@ -288,94 +247,28 @@ class Notebook:
self._observe_running(version)
case be.Exited(reason=reason):
self._observe_exited(reason)
- case be.Scheduled(run_id=run_id, caused_by=caused_by):
- if run_id not in self._runs:
- self._require_runtime()
- self._run_requests[run_id] = None
- self._set_run(
- protocol.Run(
- id=run_id,
- notebook_id=self.id,
- caused_by=caused_by,
- status="queued",
- outcome=None,
- started_at=None,
- completed_at=None,
- )
- )
- case be.Planned(run_id=run_id, executions=planned):
- self._observe_planned(run_id, planned)
- case be.Status(execution_id=execution_id, status=status):
- self._observe_status(execution_id, status)
- case be.Produced(item=item):
- if item.execution_id in self._executions:
- self._items[item.id] = item
- self._publish(
- item, self._executions[item.execution_id].run_id
- )
- case be.Concluded(execution_id=execution_id, outcome=outcome):
- if (
- execution := self._executions.get(execution_id)
- ) is not None and execution.status != "completed":
- self._set_execution(replace(execution, outcome=outcome))
- case be.Done(run_id=run_id, outcome=outcome):
- if (
- run := self._runs.get(run_id)
- ) is not None and run.status != "completed":
- members = [
- execution
- for execution in (
- *self._executions.values(),
- *self._retired_executions.values(),
- )
- if execution.run_id == run_id
- ]
- if any(
- execution.status != "completed"
- for execution in members
- ):
- raise ValueError(
- "a run cannot complete with unfinished executions"
- )
- if outcome == "succeeded" and any(
- not isinstance(
- execution.outcome, protocol.ValueOutcome
- )
- for execution in members
- ):
- raise ValueError(
- "a successful run cannot contain failed or interrupted executions"
- )
- self._set_run(
- replace(
- run,
- status="completed",
- outcome=outcome,
- completed_at=utcnow(),
- )
- )
- for member in members:
- self._retired_executions.pop(member.id, None)
+ case _:
+ self._state.observe(msg, self._runtime)
def _observe_starting(self, text: str | None) -> None:
- if self.runtime is None:
+ if self._runtime is None:
# Track kernels started outside the host API.
self._set_runtime(self._new_runtime("starting", text, None))
- elif self.runtime.status == "failed":
+ elif self._runtime.status == "failed":
self._replace_runtime(text)
- elif self.runtime.status == "starting":
- self._set_runtime(replace(self.runtime, message=text))
+ elif self._runtime.status == "starting":
+ self._set_runtime(replace(self._runtime, message=text))
def _observe_running(self, version: str | None) -> None:
- if self.runtime is None:
+ if self._runtime is None:
self._set_runtime(self._new_runtime("running", None, version))
return
- if self.runtime.status == "failed":
+ if self._runtime.status == "failed":
self._replace_runtime()
- if self.runtime.status in ("starting", "running"):
+ if self._runtime.status in ("starting", "running"):
self._set_runtime(
replace(
- self.runtime,
+ self._runtime,
status="running",
message=None,
marimo_version=version,
@@ -383,7 +276,7 @@ class Notebook:
)
def _observe_exited(self, reason: str | None) -> None:
- runtime = self.runtime
+ runtime = self._runtime
if runtime is None:
return
if runtime.status == "terminating" and self._replacement is not None:
@@ -392,13 +285,13 @@ class Notebook:
self._end_runtime()
self._lifecycle_request = request_id
self._set_runtime(replacement)
- assert self.runtime is not None
- self._backend.start(self.runtime.id)
+ assert self._runtime is not None
+ self._backend.start(self._runtime.id)
elif runtime.status == "terminating":
self._end_runtime()
else:
# Preserve results after an unexpected exit.
- self._interrupt_in_flight("failed")
+ self._state.interrupt("failed")
self._set_runtime(
replace(runtime, status="failed", message=reason)
)
@@ -424,105 +317,22 @@ class Notebook:
started_at=utcnow(),
)
- def _observe_planned(
- self,
- run_id: protocol.RunId,
- planned: tuple[be.PlannedExecution, ...],
- ) -> None:
- if (
- run := self._runs.get(run_id)
- ) is None or run.status == "completed":
- return
- runtime = self._require_runtime()
- for entry in planned:
- if (
- entry.id in self._executions
- or entry.id in self._retired_executions
- ):
- continue
- if entry.cell_id is not None:
- for previous in list(self._executions.values()):
- if previous.cell_id == entry.cell_id:
- self._retire_execution(previous)
- self._set_execution(
- protocol.Execution(
- id=entry.id,
- run_id=run_id,
- runtime_id=runtime.id,
- caused_by=entry.caused_by,
- cell_id=entry.cell_id,
- code=entry.code,
- code_hash=entry.code_hash,
- bindings=[],
- status="queued",
- outcome=None,
- stale=None,
- started_at=None,
- completed_at=None,
- )
- )
-
- def _retire_execution(self, execution: protocol.Execution) -> None:
- if execution.status != "completed":
- raise ValueError("cannot replace an unfinished cell execution")
- run_id = execution.run_id
- run = self._runs.get(run_id) if run_id is not None else None
- if run is not None and run.status != "completed":
- self._retired_executions[execution.id] = execution
- for item in list(self._items.values()):
- if item.execution_id == execution.id:
- del self._items[item.id]
- self._remove(kind_of(item), item.id, run_id, run_id=run_id)
- del self._executions[execution.id]
- self._remove("execution", execution.id, run_id, run_id=run_id)
-
- def _observe_status(
- self, execution_id: protocol.ExecutionId, status: protocol.Lifecycle
- ) -> None:
- execution = self._executions.get(execution_id)
- if execution is None or execution.status in (status, "completed"):
- return
- now = utcnow()
- if status == "running":
- execution = replace(execution, status=status, started_at=now)
- assert execution.run_id is not None
- run = self._runs[execution.run_id]
- if run.status == "queued":
- self._set_run(replace(run, status="running", started_at=now))
- elif status == "completed":
- if execution.outcome is None:
- raise ValueError("a completed execution must have an outcome")
- execution = replace(execution, status=status, completed_at=now)
- else:
- execution = replace(execution, status=status)
- self._set_execution(execution)
-
# ---- stream --------------------------------------------------------
def _snapshot(self) -> list[Event]:
"""Return notebook objects as snapshot events, parents first."""
- bodies: list[
- protocol.Runtime
- | protocol.Attachment
- | protocol.Run
- | protocol.Execution
- | Item
- ] = []
- if self.runtime is not None:
- bodies.append(self.runtime)
+ bodies: list[protocol.Runtime | protocol.Attachment] = []
+ if self._runtime is not None:
+ bodies.append(self._runtime)
bodies.extend(self._attachments.values())
- bodies.extend(self._runs.values())
- bodies.extend(self._executions.values())
- bodies.extend(self._items.values())
return [
Event(
kind_of(body),
message(body),
notebook_id=self.id,
- run_id=self._run_id(body),
)
for body in bodies
- ]
+ ] + self._state.snapshot()
def _dispose(self, *, request_id: str | None = None) -> None:
"""Close the notebook and remove its state, children first.
@@ -530,61 +340,34 @@ class Notebook:
The host must publish `notebook.removed` after disposing of its state.
"""
self._closed = True
- self._clear_runs()
+ self._state.interrupt()
+ self._state.clear()
for attachment_id in list(self._attachments):
self.detach(attachment_id)
- if self.runtime is not None:
- runtime, self.runtime = self.runtime, None
+ if self._runtime is not None:
+ runtime, self._runtime = self._runtime, None
self._remove("runtime", runtime.id, request_id)
def _publish(
self,
- body: protocol.Runtime
- | protocol.Attachment
- | protocol.Run
- | protocol.Execution
- | Item,
+ body: protocol.Runtime | protocol.Attachment,
request_id: str | None,
) -> None:
- if (run_id := self._run_id(body)) is not None:
- request_id = self._run_requests.get(run_id)
self._stream.publish(
body,
notebook_id=self.id,
request_id=request_id,
- run_id=self._run_id(body),
)
- def _run_id(
- self,
- body: protocol.Runtime
- | protocol.Attachment
- | protocol.Run
- | protocol.Execution
- | Item,
- ) -> protocol.RunId | None:
- if isinstance(body, protocol.Run):
- return body.id
- if isinstance(body, protocol.Execution):
- return body.run_id
- if isinstance(body, (protocol.Console, protocol.Display)):
- return self._executions[body.execution_id].run_id
- return None
-
def _remove(
self,
kind: str,
object_id: str,
request_id: str | None,
- *,
- run_id: protocol.RunId | None = None,
) -> None:
- if run_id is not None:
- request_id = self._run_requests.get(run_id)
self._stream.remove(
kind,
object_id,
notebook_id=self.id,
request_id=request_id,
- run_id=run_id,
)
diff --git a/marimo/_core/protocol.py b/marimo/_core/protocol.py
index 14e050d6c3..fa84a56752 100644
--- a/marimo/_core/protocol.py
+++ b/marimo/_core/protocol.py
@@ -3,6 +3,7 @@
# Generated from host-protocol/openapi.yaml in marimo-team/pair.
# Regenerate with `cargo xtask generate-host-protocol`.
# ruff: noqa: TC003 - msgspec resolves annotations at runtime
+# ruff: noqa: RUF036 - the generator orders unions; the wire shape is the same
from __future__ import annotations
@@ -26,7 +27,7 @@ else:
)
-class Binding(Struct):
+class Binding(Struct, frozen=True):
name: str
preview: (
Annotated[
@@ -48,7 +49,7 @@ class Binding(Struct):
)
-class CellConfig(Struct):
+class CellConfig(Struct, frozen=True):
column: (
Annotated[
int,
@@ -72,17 +73,32 @@ class CellConfig(Struct):
]
-class CellConfigInput(Struct):
- column: int | UnsetType | None = UNSET
+class CellConfigInput(Struct, frozen=True):
+ column: int | None | UnsetType = UNSET
disabled: bool | UnsetType = False
expand_output: bool | UnsetType = False
hide_code: bool | UnsetType = False
-class CellInput(Struct):
+if TYPE_CHECKING:
+ CellId = NewType("CellId", str)
+else:
+ CellId = NewType(
+ "CellId",
+ Annotated[
+ str,
+ Meta(
+ description="Identifies a cell within its notebook. Stable while the host serves the notebook; whether it survives a reload of the store depends on the format. `setup` is the setup cell.",
+ min_length=1,
+ ),
+ ],
+ )
+
+
+class CellInput(Struct, frozen=True):
code: Annotated[str, Meta(description="The cell's source code.")]
id: Annotated[
- str,
+ CellId,
Meta(
description="The cell's id. A client chooses the id of a cell it creates."
),
@@ -97,19 +113,19 @@ class CellInput(Struct):
) = "_"
-class DecisionOption(Struct):
+class DecisionOption(Struct, frozen=True):
id: Annotated[str, Meta(description="Identifies this answer.")]
label: Annotated[str, Meta(description="Text to show a person.")]
-class DeleteCell(Struct, tag_field="kind", tag="delete_cell"):
+class DeleteCell(Struct, frozen=True, tag_field="kind", tag="delete_cell"):
base_etag: Annotated[
str, Meta(description="The cell's etag as last read, or `*`.")
]
- id: Annotated[str, Meta(description="The cell to remove.")]
+ id: Annotated[CellId, Meta(description="The cell to remove.")]
-class DisplayData(Struct):
+class DisplayData(Struct, frozen=True):
data: Annotated[
str,
Meta(
@@ -147,22 +163,22 @@ else:
)
-class Host(Struct):
+class Host(Struct, frozen=True):
operations: Annotated[
- list[str],
+ tuple[str, ...],
Meta(
description="The operations this host implements, such as `runtime.start`. Whether a client may call one on a given project or notebook is that object's `operations`. Clients accept names they do not recognize."
),
]
-class HostMessage(Struct):
+class HostMessage(Struct, frozen=True):
host: Host
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -178,7 +194,9 @@ class HostMessage(Struct):
) = UNSET
-class InterruptedOutcome(Struct, tag_field="kind", tag="interrupted"):
+class InterruptedOutcome(
+ Struct, frozen=True, tag_field="kind", tag="interrupted"
+):
pass
@@ -240,7 +258,7 @@ ObjectType: TypeAlias = Annotated[
]
-class OpenNotebookResponse(Struct):
+class OpenNotebookResponse(Struct, frozen=True):
url: Annotated[
str,
Meta(
@@ -249,7 +267,7 @@ class OpenNotebookResponse(Struct):
]
-class Position(Struct):
+class Position(Struct, frozen=True):
character: Annotated[
int,
Meta(
@@ -262,7 +280,7 @@ class Position(Struct):
]
-class Process(Struct):
+class Process(Struct, frozen=True):
pid: Annotated[int, Meta(description="The process id.", ge=1)]
started_at: Annotated[
datetime, Meta(description="When the process started.")
@@ -292,7 +310,7 @@ ProjectStatus: TypeAlias = Annotated[
]
-class PutCell(Struct, tag_field="kind", tag="put_cell"):
+class PutCell(Struct, frozen=True, tag_field="kind", tag="put_cell"):
base_etag: (
Annotated[
str,
@@ -305,7 +323,7 @@ class PutCell(Struct, tag_field="kind", tag="put_cell"):
cell: CellInput
after: (
Annotated[
- str,
+ CellId,
Meta(
description="The cell a created cell goes after. Ignored for an existing cell."
),
@@ -314,7 +332,7 @@ class PutCell(Struct, tag_field="kind", tag="put_cell"):
) = UNSET
before: (
Annotated[
- str,
+ CellId,
Meta(
description="The cell a created cell goes before. Ignored for an existing cell. With neither `after` nor `before`, a created cell goes at the end."
),
@@ -323,7 +341,7 @@ class PutCell(Struct, tag_field="kind", tag="put_cell"):
) = UNSET
-class Range(Struct):
+class Range(Struct, frozen=True):
end: Annotated[
Position,
Meta(
@@ -333,8 +351,8 @@ class Range(Struct):
start: Position
-class Related(Struct):
- cell_id: str
+class Related(Struct, frozen=True):
+ cell_id: CellId
message: Annotated[
str,
Meta(
@@ -347,7 +365,7 @@ class Related(Struct):
)
-class RemovedMessage(Struct):
+class RemovedMessage(Struct, frozen=True):
id: Annotated[
str, Meta(description="The id of the object that was removed.")
]
@@ -355,7 +373,7 @@ class RemovedMessage(Struct):
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -371,6 +389,19 @@ class RemovedMessage(Struct):
) = UNSET
+class RequestedCell(Struct, frozen=True):
+ id: CellId
+ code_hash: (
+ Annotated[
+ Etag,
+ Meta(
+ description="The cell's `code_hash` as last read. A cell whose code differs is a `conflict`. Absent, the cell runs as it is."
+ ),
+ ]
+ | UnsetType
+ ) = UNSET
+
+
if TYPE_CHECKING:
RunId = NewType("RunId", str)
else:
@@ -379,7 +410,7 @@ else:
Annotated[
str,
Meta(
- description="Identifies a run. The request's `Idempotency-Key` for a client request; minted by the host when the runtime schedules a run.",
+ description="Identifies a run independently of the request's `Idempotency-Key`. Events carry the key in `request_id`.",
min_length=1,
),
],
@@ -394,10 +425,21 @@ RunOutcome: TypeAlias = Annotated[
]
-class RunRequest(Struct):
- code: Annotated[
- str, Meta(description="The Python code to run as scratch code.")
- ]
+class RunRequest(Struct, frozen=True):
+ cells: (
+ Annotated[
+ tuple[RequestedCell, ...],
+ Meta(
+ description="Cells of the document to run, in no particular order. What depends on them runs too, under the document's policy.",
+ min_length=1,
+ ),
+ ]
+ | UnsetType
+ ) = UNSET
+ code: (
+ Annotated[str, Meta(description="Python code to run as scratch code.")]
+ | UnsetType
+ ) = UNSET
if TYPE_CHECKING:
@@ -423,7 +465,7 @@ RuntimeStatus: TypeAlias = Annotated[
]
-class SaveRequest(Struct):
+class SaveRequest(Struct, frozen=True):
decisions: (
Annotated[
dict[str, str],
@@ -435,7 +477,9 @@ class SaveRequest(Struct):
) = UNSET
-class SetAppConfig(Struct, tag_field="kind", tag="set_app_config"):
+class SetAppConfig(
+ Struct, frozen=True, tag_field="kind", tag="set_app_config"
+):
app_config: Annotated[
dict[str, Any], Meta(description="The new app config, whole.")
]
@@ -447,20 +491,20 @@ class SetAppConfig(Struct, tag_field="kind", tag="set_app_config"):
]
-class SetOrder(Struct, tag_field="kind", tag="set_order"):
+class SetOrder(Struct, frozen=True, tag_field="kind", tag="set_order"):
base_etag: Annotated[
str,
Meta(description="The document's `order_etag` as last read, or `*`."),
]
order: Annotated[
- list[str],
+ tuple[CellId, ...],
Meta(
description="Every cell id exactly once, in the new order, counting cells this edit creates and leaving out cells it deletes."
),
]
-class StartRuntimeRequest(Struct):
+class StartRuntimeRequest(Struct, frozen=True):
decisions: (
Annotated[
dict[str, str],
@@ -472,7 +516,7 @@ class StartRuntimeRequest(Struct):
) = UNSET
-class UpdateNotebookRequest(Struct):
+class UpdateNotebookRequest(Struct, frozen=True):
name: (
Annotated[str, Meta(description="The new label to show people.")]
| UnsetType
@@ -488,7 +532,7 @@ class UpdateNotebookRequest(Struct):
) = UNSET
-class ValueOutcome(Struct, tag_field="kind", tag="value"):
+class ValueOutcome(Struct, frozen=True, tag_field="kind", tag="value"):
display: (
Annotated[
DisplayData,
@@ -500,7 +544,7 @@ class ValueOutcome(Struct, tag_field="kind", tag="value"):
)
-class Attachment(Struct):
+class Attachment(Struct, frozen=True):
id: AttachmentId
kind: Annotated[
str,
@@ -521,13 +565,13 @@ class Attachment(Struct):
since: Annotated[datetime, Meta(description="When it attached.")]
-class AttachmentMessage(Struct):
+class AttachmentMessage(Struct, frozen=True):
attachment: Attachment
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -543,7 +587,7 @@ class AttachmentMessage(Struct):
) = UNSET
-class CellEtags(Struct):
+class CellEtags(Struct, frozen=True):
code_hash: Etag
etag: Etag
@@ -554,7 +598,7 @@ Change: TypeAlias = Annotated[
]
-class Console(Struct):
+class Console(Struct, frozen=True):
execution_id: ExecutionId
id: ItemId
name: Annotated[
@@ -571,13 +615,13 @@ class Console(Struct):
text: Annotated[str, Meta(description="The text, exactly as written.")]
-class ConsoleMessage(Struct):
+class ConsoleMessage(Struct, frozen=True):
console: Console
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -593,7 +637,7 @@ class ConsoleMessage(Struct):
) = UNSET
-class CreateNotebookRequest(Struct):
+class CreateNotebookRequest(Struct, frozen=True):
project_id: ProjectId
name: (
Annotated[
@@ -615,7 +659,7 @@ class CreateNotebookRequest(Struct):
) = UNSET
-class Decision(Struct):
+class Decision(Struct, frozen=True):
id: Annotated[
str,
Meta(
@@ -623,14 +667,14 @@ class Decision(Struct):
),
]
options: Annotated[
- list[DecisionOption], Meta(description="The possible answers.")
+ tuple[DecisionOption, ...], Meta(description="The possible answers.")
]
prompt: Annotated[
str, Meta(description="The question, worded for a person.")
]
-class Display(Struct):
+class Display(Struct, frozen=True):
data: Annotated[
str,
Meta(
@@ -651,13 +695,13 @@ class Display(Struct):
]
-class DisplayMessage(Struct):
+class DisplayMessage(Struct, frozen=True):
display: Display
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -673,7 +717,7 @@ class DisplayMessage(Struct):
) = UNSET
-class Document(Struct):
+class Document(Struct, frozen=True):
app_config: Annotated[
dict[str, Any],
Meta(
@@ -696,7 +740,7 @@ class Document(Struct):
NotebookId, Meta(description="The notebook this document belongs to.")
]
order: Annotated[
- list[str],
+ tuple[CellId, ...],
Meta(
description="Cell ids in display order. A permutation of the notebook's cells, each appearing exactly once."
),
@@ -724,13 +768,13 @@ class Document(Struct):
)
-class DocumentMessage(Struct):
+class DocumentMessage(Struct, frozen=True):
document: Document
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -746,14 +790,14 @@ class DocumentMessage(Struct):
) = UNSET
-class EditRequest(Struct):
+class EditRequest(Struct, frozen=True):
changes: Annotated[
- list[Change],
+ tuple[Change, ...],
Meta(description="The changes, applied in order as one unit."),
]
-class EditResponse(Struct):
+class EditResponse(Struct, frozen=True):
cells: Annotated[
dict[str, CellEtags | None],
Meta(
@@ -763,7 +807,7 @@ class EditResponse(Struct):
document: Document
-class Error(Struct):
+class Error(Struct, frozen=True):
kind: Annotated[
str,
Meta(
@@ -787,7 +831,7 @@ class Error(Struct):
| None
)
related: Annotated[
- list[Related],
+ tuple[Related, ...],
Meta(
description="Every involved cell, including this one, in canonical order. Each copy of a cross-cell error carries the same list; clients deduplicate on kind and related cell ids. Empty when the error involves this code alone."
),
@@ -818,11 +862,11 @@ class Error(Struct):
)
-class ErrorOutcome(Struct, tag_field="kind", tag="error"):
+class ErrorOutcome(Struct, frozen=True, tag_field="kind", tag="error"):
error: Error
-class HostRecord(Struct):
+class HostRecord(Struct, frozen=True):
name: Annotated[
str,
Meta(
@@ -851,7 +895,7 @@ class HostRecord(Struct):
]
-class Notebook(Struct):
+class Notebook(Struct, frozen=True):
id: NotebookId
name: Annotated[
str,
@@ -860,7 +904,7 @@ class Notebook(Struct):
),
]
operations: Annotated[
- list[str],
+ tuple[str, ...],
Meta(
description="Operations this client may call on the notebook, such as `notebook.edit` or `runtime.start`. A subset of `host.operations`, narrowed by permission and never by state; whether a runtime is attached shows in the `runtime` object, and preconditions still apply when an operation is called. The host sends the notebook again when it changes."
),
@@ -886,13 +930,13 @@ class Notebook(Struct):
)
-class NotebookMessage(Struct):
+class NotebookMessage(Struct, frozen=True):
notebook: Notebook
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -916,7 +960,7 @@ Outcome: TypeAlias = Annotated[
]
-class Project(Struct):
+class Project(Struct, frozen=True):
id: ProjectId
message: (
Annotated[
@@ -934,7 +978,7 @@ class Project(Struct):
),
]
operations: Annotated[
- list[str],
+ tuple[str, ...],
Meta(
description="Operations this client may call on the project, today `notebook.create`. A subset of `host.operations`, narrowed by permission and never by state. The host sends the project again when it changes."
),
@@ -951,13 +995,13 @@ class Project(Struct):
status: ProjectStatus
-class ProjectMessage(Struct):
+class ProjectMessage(Struct, frozen=True):
project: Project
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -973,7 +1017,7 @@ class ProjectMessage(Struct):
) = UNSET
-class Run(Struct):
+class Run(Struct, frozen=True):
caused_by: (
Annotated[
ExecutionId,
@@ -1020,13 +1064,13 @@ class Run(Struct):
]
-class RunMessage(Struct):
+class RunMessage(Struct, frozen=True):
run: Run
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -1042,7 +1086,7 @@ class RunMessage(Struct):
) = UNSET
-class Runtime(Struct):
+class Runtime(Struct, frozen=True):
id: RuntimeId
marimo_version: (
Annotated[
@@ -1069,13 +1113,13 @@ class Runtime(Struct):
status: RuntimeStatus
-class RuntimeMessage(Struct):
+class RuntimeMessage(Struct, frozen=True):
runtime: Runtime
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -1091,20 +1135,23 @@ class RuntimeMessage(Struct):
) = UNSET
-class Analysis(Struct):
- defs: Annotated[list[str], Meta(description="Names this cell defines.")]
+class Analysis(Struct, frozen=True):
+ defs: Annotated[
+ tuple[str, ...], Meta(description="Names this cell defines.")
+ ]
errors: Annotated[
- list[Error],
+ tuple[Error, ...],
Meta(
description="Diagnostics about the code. Only an `error` affects scheduling. `multiple-definitions` lists the definers in `related`, each `range` at its definition; `cycle-dependencies` lists the cycle. Both are reported on every cell involved, differing only in `range`. Every other kind belongs to the analyzer named in `source`."
),
]
refs: Annotated[
- list[str], Meta(description="Names this cell reads from other cells.")
+ tuple[str, ...],
+ Meta(description="Names this cell reads from other cells."),
]
-class Cell(Struct):
+class Cell(Struct, frozen=True):
analysis: (
Annotated[
Analysis,
@@ -1133,12 +1180,7 @@ class Cell(Struct):
description="Identifies code, name, and config together, and is the precondition for editing the cell. Its text is `code_hash`, `name`, and the config values `column`, `disabled`, `hide_code`, and `expand_output` in that order, each written as a JSON scalar such as `null`, `0`, or `true`."
),
]
- id: Annotated[
- str,
- Meta(
- description="Identifies the cell within its notebook. Stable while the host serves the notebook; whether it survives a reload of the store depends on the format. `setup` is the setup cell."
- ),
- ]
+ id: CellId
name: Annotated[
str,
Meta(
@@ -1148,13 +1190,13 @@ class Cell(Struct):
notebook_id: NotebookId
-class CellMessage(Struct):
+class CellMessage(Struct, frozen=True):
cell: Cell
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -1170,12 +1212,12 @@ class CellMessage(Struct):
) = UNSET
-class EditConflict(Struct):
+class EditConflict(Struct, frozen=True):
cell: (
Annotated[
Cell,
Meta(
- description="The cell the change named, as it now is. Null when it no longer exists, or when the change named the order or the app config."
+ description="The cell the change named, as it now is. Null when the document does not have it, or when the change named the order or the app config."
),
]
| None
@@ -1183,16 +1225,16 @@ class EditConflict(Struct):
change: Annotated[
int,
Meta(
- description="The index of the stale change in the request's `changes`. `0` for a save with a stale `If-Match`.",
+ description="The index of the stale change in the request's `changes`, or of the stale cell in a run's `cells`. `0` for a save with a stale `If-Match`.",
ge=0,
),
]
document: Document
-class Execution(Struct):
+class Execution(Struct, frozen=True):
bindings: Annotated[
- list[Binding],
+ tuple[Binding, ...],
Meta(
description="Names bound in the namespace, with a type and preview, stamped when the execution ends. Empty while running and for scratch code, whose bindings die with the execution."
),
@@ -1208,7 +1250,7 @@ class Execution(Struct):
)
cell_id: (
Annotated[
- str,
+ CellId,
Meta(
description="The cell this execution belongs to. Null for scratch code, a cell outside the document."
),
@@ -1288,13 +1330,13 @@ class Execution(Struct):
status: Lifecycle
-class ExecutionMessage(Struct):
+class ExecutionMessage(Struct, frozen=True):
execution: Execution
request_id: (
Annotated[
str,
Meta(
- description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did."
+ description="The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress."
),
]
| UnsetType
@@ -1310,7 +1352,7 @@ class ExecutionMessage(Struct):
) = UNSET
-class Problem(Struct):
+class Problem(Struct, frozen=True):
status: Annotated[
int,
Meta(
@@ -1331,7 +1373,7 @@ class Problem(Struct):
]
decisions: (
Annotated[
- list[Decision],
+ tuple[Decision, ...],
Meta(
description="The choices the client has to make before the host can carry out the request. Only present for `decision-required`."
),
@@ -1351,7 +1393,7 @@ class Problem(Struct):
Annotated[
EditConflict,
Meta(
- description="Which change was stale and what the host holds instead. Only present for a `conflict` answering an edit or a save."
+ description="Which change was stale and what the host holds instead. Only present for a `conflict` answering an edit, a save, or a run of cells."
),
]
| UnsetType
diff --git a/marimo/_server/host/api.py b/marimo/_server/host/api.py
index 7773a34baa..be73dfd2dd 100644
--- a/marimo/_server/host/api.py
+++ b/marimo/_server/host/api.py
@@ -485,6 +485,10 @@ async def _run(request: Request) -> Response:
key = request.headers["Idempotency-Key"]
notebook = _notebook(request)
body = await _decode(request, protocol.RunRequest)
+ if body.code is msgspec.UNSET or body.cells is not msgspec.UNSET:
+ return problem(
+ 400, "invalid-request", "This host requires scratch code"
+ )
started = notebook.run(body.code, request_id=key)
return _reply(202, started)
diff --git a/marimo/_server/host/backend.py b/marimo/_server/host/backend.py
index 6b1e247648..0443a475f7 100644
--- a/marimo/_server/host/backend.py
+++ b/marimo/_server/host/backend.py
@@ -177,7 +177,7 @@ class SessionBackend(SessionConsumer):
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
# Detachment already ended this session's runs in the core model.
if self.session is run.session:
@@ -202,6 +202,9 @@ class SessionBackend(SessionConsumer):
elif run.task is not None:
run.task.cancel()
+ def forget(self, run_id: protocol.RunId) -> None:
+ self._translators.pop(run_id, None)
+
# Existing sessions
def adopt(self, session: Session) -> None:
diff --git a/marimo/_server/host/bridge.py b/marimo/_server/host/bridge.py
index 6a352d875d..2fef9178fe 100644
--- a/marimo/_server/host/bridge.py
+++ b/marimo/_server/host/bridge.py
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
from pathlib import Path
from marimo._core.backend import Message
+ from marimo._core.stream import Event
from marimo._server.session_manager import SessionManager
from marimo._session.session import Session
@@ -38,6 +39,9 @@ class SessionBridge(SessionEventListener):
self.index = WorkspaceIndex(
self.host, manager.workspace, self._make_backend
)
+ self._unsubscribe = self.host.subscribe(
+ self._forget_backend, types={"notebook"}
+ )
def open(self) -> None:
"""Index notebooks, adopt sessions, and subscribe to session events."""
@@ -48,6 +52,7 @@ class SessionBridge(SessionEventListener):
def close(self) -> None:
self._manager.event_bus.unsubscribe(self)
+ self._unsubscribe()
async def refresh(self) -> None:
"""Refresh the catalog without observing a half-finished file move."""
@@ -194,6 +199,10 @@ class SessionBridge(SessionEventListener):
self._backends[notebook_id] = backend
return backend
+ def _forget_backend(self, event: Event) -> None:
+ if event.name == "notebook.removed" and event.notebook_id is not None:
+ self._backends.pop(event.notebook_id, None)
+
def _adopt(self, session: Session) -> None:
path = session.app_file_manager.path
notebook = self.index.notebook_for(path)
diff --git a/marimo/_server/host/index.py b/marimo/_server/host/index.py
index 4970253596..e6e5b0bed7 100644
--- a/marimo/_server/host/index.py
+++ b/marimo/_server/host/index.py
@@ -83,7 +83,7 @@ class WorkspaceIndex:
root=None if self._root is None else str(self._root),
status="partial" if truncated else "indexed",
message=_TRUNCATED if truncated else None,
- operations=list(OPERATIONS),
+ operations=tuple(OPERATIONS),
)
)
seen: set[protocol.NotebookId] = set()
@@ -205,7 +205,9 @@ class WorkspaceIndex:
project_id=self.project_id,
path=str(relative),
name=self._names.get(notebook_id, relative.stem),
- operations=[op for op in OPERATIONS if op != "notebook.create"],
+ operations=tuple(
+ op for op in OPERATIONS if op != "notebook.create"
+ ),
updated_at=None
if modified is None
else datetime.fromtimestamp(modified, tz=timezone.utc),
diff --git a/marimo/_server/host/lifespan.py b/marimo/_server/host/lifespan.py
index 84e5e29b2c..77a73930b2 100644
--- a/marimo/_server/host/lifespan.py
+++ b/marimo/_server/host/lifespan.py
@@ -3,6 +3,7 @@
from __future__ import annotations
+import asyncio
import contextlib
import secrets
from typing import TYPE_CHECKING
@@ -35,9 +36,15 @@ async def host(app: Starlette) -> AsyncIterator[None]:
token = secrets.token_urlsafe(32)
server = state.server.config
url = host_url(server.host, server.port, state.base_url)
- app.state.host_context = HostContext(
- bridge=bridge, token=token, port=server.port
- )
+ context = HostContext(bridge=bridge, token=token, port=server.port)
+ app.state.host_context = context
+
+ async def maintain() -> None:
+ while True:
+ await asyncio.sleep(1)
+ bridge.host.expire()
+
+ maintenance = asyncio.create_task(maintain(), name="host.retention")
record = HostRecordFile(url=url, token=token)
try:
record.write()
@@ -46,6 +53,9 @@ async def host(app: Starlette) -> AsyncIterator[None]:
try:
yield
finally:
+ maintenance.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await maintenance
record.remove()
bridge.close()
app.state.host_context = None
diff --git a/marimo/_server/host/translate.py b/marimo/_server/host/translate.py
index 4e3e4af8f4..f878736999 100644
--- a/marimo/_server/host/translate.py
+++ b/marimo/_server/host/translate.py
@@ -167,6 +167,14 @@ class RunTranslator:
if not execution.reported or execution.outcome != previous:
out.append(be.Concluded(execution.id, execution.outcome))
out.append(be.Status(execution.id, "completed"))
+ self._failed |= isinstance(execution.outcome, protocol.ErrorOutcome)
+ self.interrupted |= isinstance(
+ execution.outcome, protocol.InterruptedOutcome
+ )
+ # The core keeps the reported outcome; release the payload here
+ # while the translator stays alive for late output.
+ execution.outcome = protocol.ValueOutcome(None)
+ execution.errors.clear()
return out
def finish(
@@ -196,20 +204,12 @@ class RunTranslator:
execution, error=error, interrupted=self.interrupted
)
)
+ self._code = None
+ self._error = None
outcome: protocol.RunOutcome = "succeeded"
- if (
- error is not None
- or self._failed
- or any(
- isinstance(e.outcome, protocol.ErrorOutcome)
- for e in self._executions.values()
- )
- ):
+ if error is not None or self._failed:
outcome = "failed"
- if self.interrupted or any(
- isinstance(e.outcome, protocol.InterruptedOutcome)
- for e in self._executions.values()
- ):
+ if self.interrupted:
outcome = "interrupted"
out.append(be.Done(self.run_id, outcome))
return out
@@ -274,7 +274,7 @@ def _describe(error: Error) -> protocol.Error:
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
if isinstance(error, MarimoSyntaxError):
return protocol.Error(
@@ -285,7 +285,7 @@ def _describe(error: Error) -> protocol.Error:
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
if isinstance(error, UnknownError):
return protocol.Error(
@@ -296,7 +296,7 @@ def _describe(error: Error) -> protocol.Error:
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
return protocol.Error(
kind="internal",
@@ -306,5 +306,5 @@ def _describe(error: Error) -> protocol.Error:
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
diff --git a/tests/_core/conftest.py b/tests/_core/conftest.py
index 92e592ac8b..40c9d3f9e4 100644
--- a/tests/_core/conftest.py
+++ b/tests/_core/conftest.py
@@ -31,6 +31,9 @@ class ScriptedBackend:
def interrupt(self, run_id: str) -> None:
self.calls.append(("interrupt", run_id))
+ def forget(self, run_id: str) -> None:
+ self.calls.append(("forget", run_id))
+
def project(project_id: str = "p1") -> protocol.Project:
return protocol.Project(
@@ -39,7 +42,7 @@ def project(project_id: str = "p1") -> protocol.Project:
root="/Users/me/forecasting",
status="indexed",
message=None,
- operations=["notebook.create"],
+ operations=("notebook.create",),
)
@@ -51,7 +54,7 @@ def record(
project_id=protocol.ProjectId("p1"),
path=path,
name=path.removesuffix(".py"),
- operations=["runtime.start", "runtime.run"],
+ operations=("runtime.start", "runtime.run"),
updated_at=None,
)
diff --git a/tests/_core/test_execution_state.py b/tests/_core/test_execution_state.py
new file mode 100644
index 0000000000..9ff60d12d7
--- /dev/null
+++ b/tests/_core/test_execution_state.py
@@ -0,0 +1,428 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Result retention across live streams, replay, and snapshots."""
+
+from __future__ import annotations
+
+import pytest
+from msgspec.structs import replace
+
+from marimo._core import backend as be, protocol as p
+from marimo._core.execution_state import RetentionPolicy
+from marimo._core.host import Host
+from marimo._core.notebook import Notebook
+from marimo._core.stream import Event
+from tests._core.conftest import ScriptedBackend, names, project, record
+
+
+class Clock:
+ now = 0.0
+
+ def __call__(self) -> float:
+ return self.now
+
+
+def host_at(clock: Clock, history: int = 1000) -> tuple[Host, Notebook]:
+ host = Host(
+ operations=[],
+ history=history,
+ clock=clock,
+ retention=RetentionPolicy(ephemeral_seconds=10),
+ )
+ host.update_project(project())
+ notebook = host.add_notebook(record(), ScriptedBackend())
+ notebook.observe(be.Running())
+ return host, notebook
+
+
+def plan(
+ notebook: Notebook, run: p.Run, eid: str, cell: str | None = None
+) -> p.ExecutionId:
+ execution = p.ExecutionId(eid)
+ notebook.observe(
+ be.Planned(
+ run.id,
+ (
+ be.PlannedExecution(
+ execution, cell, "hash", "code" if cell is None else None
+ ),
+ ),
+ )
+ )
+ notebook.observe(be.Status(execution, "running"))
+ return execution
+
+
+def finish(notebook: Notebook, execution: p.ExecutionId) -> None:
+ notebook.observe(
+ be.Concluded(
+ execution,
+ p.ValueOutcome(p.DisplayData(mimetype="text/plain", data="value")),
+ )
+ )
+ notebook.observe(be.Status(execution, "completed"))
+
+
+def fold(events: list[Event], initial: dict | None = None) -> dict:
+ state = dict(initial or {})
+ for event in events:
+ if event.name == "reset":
+ state.clear()
+ elif event.kind in {"run", "execution", "console", "display"}:
+ if event.name.endswith(".removed"):
+ state.pop((event.kind, event.data.id), None)
+ else:
+ body = getattr(event.data, event.kind)
+ state[event.kind, body.id] = body
+ return state
+
+
[email protected]("history", [2, 1000])
+def test_expiration_converges_for_live_replay_and_snapshot(
+ history: int,
+) -> None:
+ clock = Clock()
+ host, notebook = host_at(clock, history)
+ run = notebook.run("code", request_id="request")
+ execution = plan(notebook, run, "e")
+ item = p.Console(
+ p.ExecutionId("e"), p.ItemId("console"), "stdout", 0, "output"
+ )
+ notebook.observe(be.Produced(item))
+ notebook.observe(
+ be.Produced(
+ p.Display(
+ execution_id=execution,
+ id=p.ItemId("display"),
+ seq=1,
+ mimetype="text/html",
+ data="value",
+ )
+ )
+ )
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ snapshot = host.events(notebook_id=notebook.id, run_id=run.id)
+ initial = fold(snapshot)
+ assert {kind for kind, _ in initial} == {
+ "run",
+ "execution",
+ "console",
+ "display",
+ }
+ live: list[Event] = []
+ host.subscribe(live.append, notebook_id=notebook.id, run_id=run.id)
+ clock.now = 9
+ host.expire()
+ assert live == []
+ clock.now = 10
+ host.expire()
+ assert names(live) == [
+ "console.removed",
+ "display.removed",
+ "execution.removed",
+ "run.removed",
+ ]
+ assert all(event.data.request_id == "request" for event in live)
+ replay = host.events(
+ snapshot[-1].id, notebook_id=notebook.id, run_id=run.id
+ )
+ assert (replay[0].name == "reset") == (history == 2)
+ assert (
+ fold(live, initial)
+ == fold(replay, initial)
+ == fold(host.events(notebook_id=notebook.id, run_id=run.id))
+ == {}
+ )
+ live.clear()
+ notebook.observe(be.Produced(item))
+ notebook.observe(be.Status(execution, "running"))
+ notebook.observe(be.Done(run.id, "succeeded"))
+ assert live == []
+
+
+def test_cell_results_outlive_ephemeral_members_and_old_runs_are_removed() -> (
+ None
+):
+ clock = Clock()
+ host, notebook = host_at(clock)
+ run = notebook.run("code", request_id="first")
+ ephemeral = plan(notebook, run, "scratch")
+ cell = plan(notebook, run, "cell", "a")
+ finish(notebook, ephemeral)
+ clock.now = 100
+ host.expire()
+ assert ("execution", ephemeral) in fold(
+ host.events(notebook_id=notebook.id)
+ )
+ finish(notebook, cell)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ clock.now = 110
+ host.expire()
+ state = fold(host.events(notebook_id=notebook.id))
+ assert set(state) == {("run", run.id), ("execution", cell)}
+ replacement = notebook.run("code", request_id="second")
+ current = plan(notebook, replacement, "current", "a")
+ finish(notebook, current)
+ notebook.observe(be.Done(replacement.id, "succeeded"))
+ clock.now = 1000
+ state = fold(host.events(notebook_id=notebook.id))
+ assert set(state) == {("run", replacement.id), ("execution", current)}
+
+
+def test_run_survives_until_its_last_cell_execution_is_replaced() -> None:
+ host, notebook = host_at(Clock())
+ original = notebook.run("code", request_id="original")
+ first = plan(notebook, original, "first", "a")
+ second = plan(notebook, original, "second", "b")
+ finish(notebook, first)
+ finish(notebook, second)
+ notebook.observe(be.Done(original.id, "succeeded"))
+ live: list[Event] = []
+ host.subscribe(live.append, notebook_id=notebook.id, run_id=original.id)
+
+ replacement = notebook.run("code", request_id="replacement")
+ current_a = plan(notebook, replacement, "current-a", "a")
+ assert set(fold(host.events(notebook_id=notebook.id))) == {
+ ("run", original.id),
+ ("run", replacement.id),
+ ("execution", second),
+ ("execution", current_a),
+ }
+ assert names(live) == ["execution.removed"]
+
+ current_b = plan(notebook, replacement, "current-b", "b")
+ assert set(fold(host.events(notebook_id=notebook.id))) == {
+ ("run", replacement.id),
+ ("execution", current_a),
+ ("execution", current_b),
+ }
+ assert names(live) == [
+ "execution.removed",
+ "execution.removed",
+ "run.removed",
+ ]
+
+
+def test_stopping_starts_expiry_for_active_work_without_extending_old_results() -> (
+ None
+):
+ clock = Clock()
+ host, notebook = host_at(clock)
+ done = notebook.run("code", request_id="done")
+ execution = plan(notebook, done, "done")
+ finish(notebook, execution)
+ notebook.observe(be.Done(done.id, "succeeded"))
+ clock.now = 5
+ active = notebook.run("sleep", request_id="active")
+ plan(notebook, active, "active")
+ notebook.stop()
+ notebook.observe(be.Exited())
+ clock.now = 10
+ host.expire()
+ state = fold(host.events(notebook_id=notebook.id))
+ assert set(state) == {("run", active.id), ("execution", "active")}
+ assert state["run", active.id].outcome == "interrupted"
+ clock.now = 15
+ host.expire()
+ assert fold(host.events(notebook_id=notebook.id)) == {}
+
+
+def test_repeated_expiration_releases_runs_and_request_correlations() -> None:
+ clock = Clock()
+ host, notebook = host_at(clock, history=2)
+ for index in range(100):
+ run = notebook.run("code", request_id=f"request-{index}")
+ execution = plan(notebook, run, str(index))
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ clock.now += 10
+ host.expire()
+ assert fold(host.events(notebook_id=notebook.id)) == {}
+ assert (
+ len({call[1] for call in notebook._backend.calls if call[0] == "run"})
+ == 100
+ )
+
+
+def test_removal_listeners_can_read_snapshots() -> None:
+ clock = Clock()
+ host, notebook = host_at(clock)
+ run = notebook.run("code", request_id="request")
+ execution = plan(notebook, run, "execution")
+ notebook.observe(
+ be.Produced(
+ p.Console(execution, p.ItemId("console"), "stdout", 0, "x")
+ )
+ )
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ snapshots: list[dict] = []
+ host.subscribe(
+ lambda _: snapshots.append(fold(host.events(notebook_id=notebook.id))),
+ notebook_id=notebook.id,
+ )
+ clock.now = 10
+ host.expire()
+ assert [set(state) for state in snapshots] == [
+ {("run", run.id), ("execution", execution)},
+ {("run", run.id)},
+ set(),
+ ]
+
+
[email protected]("cleanup", ["expire", "replace", "clear"])
+def test_outputs_follow_their_execution_through_updates_and_removal(
+ cleanup: str,
+) -> None:
+ clock = Clock()
+ host, notebook = host_at(clock)
+ run = notebook.run("code", request_id="request")
+ first = plan(notebook, run, "first", "a" if cleanup != "expire" else None)
+ second = plan(
+ notebook, run, "second", "b" if cleanup != "expire" else None
+ )
+ first_console = p.Console(
+ first, p.ItemId("first-console"), "stdout", 0, "a"
+ )
+ first_display = p.Display(
+ execution_id=first,
+ id=p.ItemId("first-display"),
+ seq=1,
+ mimetype="text/plain",
+ data="b",
+ )
+ second_console = p.Console(
+ second, p.ItemId("second-console"), "stdout", 0, "c"
+ )
+ second_display = p.Display(
+ execution_id=second,
+ id=p.ItemId("second-display"),
+ seq=1,
+ mimetype="text/plain",
+ data="d",
+ )
+ for item in (second_console, first_console, second_display, first_display):
+ notebook.observe(be.Produced(item))
+ first_console = replace(first_console, text="updated")
+ notebook.observe(be.Produced(first_console))
+ finish(notebook, first)
+ finish(notebook, second)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ initial = host.events(notebook_id=notebook.id)
+ assert [
+ getattr(event.data, event.kind)
+ for event in initial
+ if event.kind in {"console", "display"}
+ ] == [first_console, first_display, second_console, second_display]
+
+ live: list[Event] = []
+ snapshots: list[dict] = []
+
+ def updated(event: Event) -> None:
+ live.append(event)
+ snapshots.append(fold(notebook._snapshot()))
+
+ host.subscribe(
+ updated,
+ notebook_id=notebook.id,
+ types={"run", "execution", "console", "display"},
+ )
+ if cleanup == "expire":
+ clock.now = 10
+ host.expire()
+ elif cleanup == "replace":
+ replacement = notebook.run("code", request_id="replacement")
+ plan(notebook, replacement, "new", "a")
+ else:
+ host.remove_notebook(notebook.id)
+
+ assert [
+ (event.name, event.data.id)
+ for event in live
+ if event.name.endswith(".removed")
+ ] == [
+ ("console.removed", first_console.id),
+ ("display.removed", first_display.id),
+ ("execution.removed", first),
+ *(
+ []
+ if cleanup == "replace"
+ else [
+ ("console.removed", second_console.id),
+ ("display.removed", second_display.id),
+ ("execution.removed", second),
+ ("run.removed", run.id),
+ ]
+ ),
+ ]
+ state = fold(initial)
+ for event, observed in zip(live, snapshots, strict=True):
+ state = fold([event], state)
+ assert state == observed
+
+
+def test_expiration_does_not_scan_unrelated_outputs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ clock = Clock()
+ host, notebook = host_at(clock, history=2)
+ run = notebook.run("code", request_id="request")
+ for index in range(1000):
+ execution = plan(notebook, run, str(index))
+ notebook.observe(
+ be.Produced(
+ p.Console(execution, p.ItemId(str(index)), "stdout", 0, "x")
+ )
+ )
+ finish(notebook, execution)
+ notebook.observe(be.Done(run.id, "succeeded"))
+ visits = 0
+ getattribute = p.Console.__getattribute__
+
+ def counted(item: p.Console, name: str) -> object:
+ nonlocal visits
+ if name == "execution_id":
+ visits += 1
+ return getattribute(item, name)
+
+ monkeypatch.setattr(p.Console, "__getattribute__", counted)
+ clock.now = 10
+ host.expire()
+ assert visits <= 1000
+ assert fold(host.events(notebook_id=notebook.id)) == {}
+
+
+def test_planning_does_not_scan_unrelated_cells(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ host, notebook = host_at(Clock(), history=2)
+ run = notebook.run("code", request_id="request")
+ visits = 0
+ getattribute = p.Execution.__getattribute__
+
+ def counted(execution: p.Execution, name: str) -> object:
+ nonlocal visits
+ if name == "cell_id":
+ visits += 1
+ return getattribute(execution, name)
+
+ monkeypatch.setattr(p.Execution, "__getattribute__", counted)
+ notebook.observe(
+ be.Planned(
+ run.id,
+ tuple(
+ be.PlannedExecution(
+ p.ExecutionId(str(index)), str(index), "hash", None
+ )
+ for index in range(1000)
+ ),
+ )
+ )
+ assert visits <= 2000
+ assert (
+ sum(
+ event.kind == "execution"
+ for event in host.events(notebook_id=notebook.id)
+ )
+ == 1000
+ )
diff --git a/tests/_core/test_host.py b/tests/_core/test_host.py
index 3d1512b127..667b3df2f7 100644
--- a/tests/_core/test_host.py
+++ b/tests/_core/test_host.py
@@ -160,6 +160,7 @@ def test_removing_a_notebook_removes_what_it_owns_first(host: Host) -> None:
host.remove_notebook("n1", request_id="req-del")
assert names(log) == [
+ "run",
"run.removed",
"attachment.removed",
"runtime.removed",
@@ -262,8 +263,9 @@ def test_catalog_requires_the_project(host: Host) -> None:
def test_catalog_updates_do_not_share_mutable_records(host: Host) -> None:
source = project()
host.update_project(source)
- source.name = "changed"
- source.operations.clear()
+ with pytest.raises(AttributeError):
+ source.name = "changed"
+ source = replace(source, name="changed", operations=())
host.update_project(source)
replay = host.events(0, types={"project"})
@@ -274,20 +276,22 @@ def test_catalog_updates_do_not_share_mutable_records(host: Host) -> None:
]
assert projects == [
project(),
- replace(project(), name="changed", operations=[]),
+ replace(project(), name="changed", operations=()),
]
original = record()
host.add_notebook(original, ScriptedBackend())
- original.name = "caller mutation"
- original.operations.clear()
+ with pytest.raises(AttributeError):
+ original.name = "caller mutation"
+ original = replace(original, name="caller mutation", operations=())
view = host.notebooks[original.id]
- view.name = "reader mutation"
- view.operations.clear()
+ with pytest.raises(AttributeError):
+ view.name = "reader mutation"
assert host.notebooks[original.id] == record()
host.update_notebook(original)
- original.name = "another mutation"
+ with pytest.raises(AttributeError):
+ original.name = "another mutation"
assert host.notebooks[original.id] == replace(
- record(), name="caller mutation", operations=[]
+ record(), name="caller mutation", operations=()
)
diff --git a/tests/_core/test_notebook.py b/tests/_core/test_notebook.py
index d51dcb95f1..28a8d03512 100644
--- a/tests/_core/test_notebook.py
+++ b/tests/_core/test_notebook.py
@@ -7,6 +7,7 @@ import pytest
from dirty_equals import IsDatetime
from inline_snapshot import snapshot
from msgspec import UNSET
+from msgspec.structs import replace
from marimo._core import backend as be
from marimo._core.errors import Conflict, NoRuntime, NotFound
@@ -29,6 +30,7 @@ from marimo._core.protocol import (
RunMessage,
RunOutcome,
Runtime,
+ RuntimeMessage,
ValueOutcome,
)
from marimo._core.stream import Event
@@ -56,6 +58,28 @@ def scratch(notebook: Notebook, run_id: str = "run-1") -> Run:
class TestStart:
+ def test_runtime_cannot_change_without_an_event(
+ self, notebook: Notebook, host: Host, backend: ScriptedBackend
+ ) -> None:
+ runtime, _ = notebook.start()
+ with pytest.raises(AttributeError):
+ runtime.status = "running"
+ with pytest.raises(AttributeError):
+ notebook.runtime = replace(runtime, status="running")
+ with pytest.raises(Conflict, match="starting"):
+ notebook.run("1", request_id="too-early")
+
+ for events in (
+ host.events(notebook_id=notebook.id),
+ host.events(0, notebook_id=notebook.id),
+ ):
+ assert [
+ event.data.runtime
+ for event in events
+ if isinstance(event.data, RuntimeMessage)
+ ] == [runtime]
+ assert backend.calls == [("start", runtime.id)]
+
def test_starts_a_runtime_and_asks_the_backend(
self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]
) -> None:
@@ -501,7 +525,7 @@ class TestRun:
code="3 * 3",
code_hash="9f",
completed_at=IsDatetime(),
- bindings=[],
+ bindings=(),
id=ExecutionId("e7"),
outcome=ValueOutcome(
display=DisplayData(
@@ -536,11 +560,15 @@ class TestRun:
self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]
) -> None:
running(notebook, backend)
- notebook.run("1", request_id="first")
- with pytest.raises(Conflict, match="first"):
+ first = notebook.run("1", request_id="first")
+ with pytest.raises(Conflict, match=first.id):
notebook.run("2", request_id="second")
- notebook.observe(be.Done(RunId("first"), "succeeded"))
- final = log[-1].data
+ notebook.observe(be.Done(first.id, "succeeded"))
+ final = next(
+ event.data
+ for event in reversed(log)
+ if isinstance(event.data, RunMessage)
+ )
assert isinstance(final, RunMessage)
assert final.run.status == "completed"
assert final.run.started_at is None
@@ -592,14 +620,16 @@ class TestRunCompletion:
log.clear()
notebook.observe(be.Scheduled(RunId("later"), ExecutionId("e1")))
notebook.observe(be.Done(RunId("later"), "succeeded"))
- assert request_ids(log) == [UNSET, UNSET]
+ assert request_ids(log) == [UNSET, UNSET, UNSET]
assert all(event.run_id == "later" for event in log)
- final = log[-1].data
+ final = next(
+ event.data
+ for event in reversed(log)
+ if isinstance(event.data, RunMessage)
+ )
assert isinstance(final, RunMessage)
assert final.run.caused_by == "e1"
- assert (
- notebook.run("3 * 3", request_id=original.id).status == "completed"
- )
+ assert notebook.run("3 * 3", request_id="run-1").status == "completed"
@pytest.mark.parametrize("outcome", ["failed", "interrupted"])
def test_can_end_before_any_execution(
@@ -613,7 +643,11 @@ class TestRunCompletion:
run = notebook.run("pass", request_id="never-started")
assert run.outcome is None
notebook.observe(be.Done(run.id, outcome))
- final = log[-1].data
+ final = next(
+ event.data
+ for event in reversed(log)
+ if isinstance(event.data, RunMessage)
+ )
assert isinstance(final, RunMessage)
assert final.run.status == "completed"
assert final.run.outcome == outcome
@@ -661,7 +695,7 @@ class TestRunCompletion:
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
),
)
@@ -753,7 +787,7 @@ class TestCellResults:
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
)
notebook.observe(be.Concluded(old, failed))
@@ -780,6 +814,7 @@ class TestCellResults:
assert names(log) == [
"console.removed",
"execution.removed",
+ *([] if same_run else ["run.removed"]),
"execution",
]
visible = [
@@ -796,7 +831,7 @@ class TestCellResults:
notebook.observe(
be.Done(second.id, "failed" if same_run else "succeeded")
)
- assert not notebook._retired_executions
+ assert not notebook._state._retired_executions
log.clear()
notebook.observe(be.Produced(item))
assert not log
diff --git a/tests/_core/test_stream.py b/tests/_core/test_stream.py
index 38be3228a9..6cd53055cc 100644
--- a/tests/_core/test_stream.py
+++ b/tests/_core/test_stream.py
@@ -3,6 +3,8 @@
from __future__ import annotations
+import pytest
+
from marimo._core import protocol
from marimo._core.host import Host
from marimo._core.notebook import Notebook
@@ -51,28 +53,34 @@ def test_readers_cannot_change_other_readers_or_replay() -> None:
def mutate(event: Event) -> None:
assert isinstance(event.data, protocol.ProjectMessage)
- event.data.project.name = "subscriber mutation"
- event.data.project.operations.clear()
+ with pytest.raises(AttributeError):
+ event.data.project.name = "subscriber mutation"
+ with pytest.raises(AttributeError):
+ event.data.project.operations.clear()
stream.subscribe(mutate)
delivered: list[Event] = []
stream.subscribe(delivered.append)
original = project()
published = stream.publish(original)
- original.name = "caller mutation"
+ with pytest.raises(AttributeError):
+ original.name = "caller mutation"
assert isinstance(published.data, protocol.ProjectMessage)
assert published.data.project == project()
- published.data.project.name = "return value mutation"
+ with pytest.raises(AttributeError):
+ published.data.project.name = "return value mutation"
assert isinstance(delivered[0].data, protocol.ProjectMessage)
assert delivered[0].data.project == project()
- delivered[0].data.project.name = "listener mutation"
+ with pytest.raises(AttributeError):
+ delivered[0].data.project.name = "listener mutation"
replay = stream.since(0)
assert replay is not None
assert isinstance(replay[0].data, protocol.ProjectMessage)
assert replay[0].data.project == project()
- replay[0].data.project.name = "replay mutation"
+ with pytest.raises(AttributeError):
+ replay[0].data.project.name = "replay mutation"
reread = stream.since(0)
assert reread is not None
assert isinstance(reread[0].data, protocol.ProjectMessage)
diff --git a/tests/_server/host/test_host_api.py b/tests/_server/host/test_host_api.py
index bf657e0dd9..814202ef49 100644
--- a/tests/_server/host/test_host_api.py
+++ b/tests/_server/host/test_host_api.py
@@ -324,20 +324,30 @@ async def test_runtime_lifecycle_over_http(harness: Harness) -> None:
async def test_run_filter_over_http(harness: Harness) -> None:
nid = await harness.running()
notebook = harness.host.notebook(nid)
- notebook.run("1", request_id="first")
- notebook.observe(be.Done(protocol.RunId("first"), "succeeded"))
+ first = notebook.run("1", request_id="first")
+ notebook.observe(
+ be.Planned(
+ first.id,
+ (be.PlannedExecution(protocol.ExecutionId("e1"), None, "h", "1"),),
+ )
+ )
+ notebook.observe(
+ be.Concluded(protocol.ExecutionId("e1"), protocol.ValueOutcome(None))
+ )
+ notebook.observe(be.Status(protocol.ExecutionId("e1"), "completed"))
+ notebook.observe(be.Done(first.id, "succeeded"))
notebook.run("2", request_id="second")
response, task, disconnect = request(
harness.app,
"GET",
f"/notebooks/{nid}/events",
headers=AUTH,
- query="run_id=first",
+ query=f"run_id={first.id}&types=run",
)
try:
await wait_for(response, until_ready)
assert response.names() == ["run", "ready"]
- assert response.records[0]["data"]["run"]["id"] == "first"
+ assert response.records[0]["data"]["run"]["id"] == first.id
finally:
disconnect()
await asyncio.wait_for(task, 5)
diff --git a/tests/_server/host/test_host_backend.py b/tests/_server/host/test_host_backend.py
index 53afe32fb8..51802b96f8 100644
--- a/tests/_server/host/test_host_backend.py
+++ b/tests/_server/host/test_host_backend.py
@@ -339,6 +339,6 @@ async def test_submission_failure_is_an_error(
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
)
)
diff --git a/tests/_server/host/test_host_bridge.py b/tests/_server/host/test_host_bridge.py
index ef678198b8..085ae97dfb 100644
--- a/tests/_server/host/test_host_bridge.py
+++ b/tests/_server/host/test_host_bridge.py
@@ -8,7 +8,12 @@ from typing import TYPE_CHECKING
import pytest
from marimo._core.errors import Conflict
-from marimo._messaging.notification import StartupProgressNotification
+from marimo._messaging.notification import (
+ CompletedRunNotification,
+ ExecutionNotification,
+ StartupProgressNotification,
+)
+from marimo._runtime.scratch import SCRATCH_CELL_ID
from marimo._server.host.bridge import SessionBridge
from marimo._server.models.home import MarimoFile
from marimo._server.workspace import FixedFilesWorkspace
@@ -105,6 +110,45 @@ async def test_session_updates_after_catalog_removal_are_ignored(
bridge.close()
[email protected]("cell_id", [SCRATCH_CELL_ID, "a"])
+async def test_removed_runs_release_backend_translation_state(
+ manager: FakeManager, tmp_path: Path, cell_id: str
+) -> None:
+ prepare(manager, tmp_path, "a.py")
+ session = FakeSession(str(tmp_path / "a.py"))
+ manager.sessions["browser"] = session
+ bridge = SessionBridge(manager) # type: ignore[arg-type]
+ now = 0.0
+ bridge.host.clock = lambda: now
+ bridge.open()
+ record = bridge.index.notebook_for(str(tmp_path / "a.py"))
+ assert record is not None
+ backend = bridge._backends[record.id]
+ for phase in ("planned", "completed"):
+ session.broadcast(
+ ExecutionNotification(
+ run_id="run",
+ execution_id="execution",
+ cell_id=cell_id,
+ phase=phase,
+ code="42",
+ )
+ )
+ session.broadcast(
+ CompletedRunNotification(run_id="run", outcome="succeeded")
+ )
+ assert set(backend._translators) == {"run"}
+ now = 60
+ bridge.host.expire()
+ # Cell results are retained, so their translation state stays too.
+ retained = {"run"} if cell_id != SCRATCH_CELL_ID else set()
+ assert set(backend._translators) == retained
+ bridge.index.forget(record.id)
+ assert backend._translators == {}
+ assert bridge._backends == {}
+ bridge.close()
+
+
async def test_failed_session_move_preserves_the_notebook(
manager: FakeManager,
tmp_path: Path,
diff --git a/tests/_server/host/test_host_record.py b/tests/_server/host/test_host_record.py
index 62960f75d4..16175f621c 100644
--- a/tests/_server/host/test_host_record.py
+++ b/tests/_server/host/test_host_record.py
@@ -3,6 +3,7 @@
from __future__ import annotations
+import asyncio
import os
import stat
import sys
@@ -12,12 +13,13 @@ import msgspec
import pytest
import uvicorn
-from marimo._core import protocol
+from marimo._core import backend as be, protocol
from marimo._server.host import HOST_API_PATH, record as record_module
from marimo._server.host.lifespan import host as host_lifespan
from marimo._server.host.record import host_url
from marimo._server.main import create_starlette_app
from marimo._utils.lifespans import Lifespans
+from tests._core.conftest import ScriptedBackend, project, record
from tests._server.host.drive import finished
from tests._server.mocks import get_starlette_server_state_init
@@ -80,3 +82,58 @@ async def test_the_lifespan_serves_the_host_while_the_server_runs(
assert admitted.status == 404
assert list((tmp_path / "hosts-v1").glob("*.json")) == []
+
+
+async def test_idle_host_expires_results_and_stops_maintenance(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(record_module, "marimo_state_dir", lambda: tmp_path)
+ app = create_starlette_app(base_url="", enable_auth=True)
+ get_starlette_server_state_init().apply(app.state)
+ app.state.server = uvicorn.Server(
+ uvicorn.Config(app, host="127.0.0.1", port=1234)
+ )
+ async with host_lifespan(app):
+ context = app.state.host_context
+ host = context.bridge.host
+ now = 0.0
+ host.clock = lambda: now
+ host.update_project(project())
+ notebook = host.add_notebook(record(), ScriptedBackend())
+ notebook.observe(be.Running())
+ headers = {
+ "Authorization": f"Bearer {context.token}",
+ "Idempotency-Key": "run",
+ }
+ response = await finished(
+ app,
+ "POST",
+ f"/notebooks/{notebook.id}/runtime/runs",
+ headers=headers,
+ body=b'{"code":"42"}',
+ )
+ assert response.status == 202
+ run_id = protocol.RunId(response.json()["id"])
+ eid = protocol.ExecutionId("execution")
+ notebook.observe(
+ be.Planned(run_id, (be.PlannedExecution(eid, None, "hash", "42"),))
+ )
+ notebook.observe(be.Concluded(eid, protocol.ValueOutcome(None)))
+ notebook.observe(be.Status(eid, "completed"))
+ notebook.observe(be.Done(run_id, "succeeded"))
+ removed = asyncio.Event()
+ host.subscribe(
+ lambda event: (
+ removed.set() if event.name == "run.removed" else None
+ ),
+ notebook_id=notebook.id,
+ )
+ [maintenance] = [
+ task
+ for task in asyncio.all_tasks()
+ if task.get_name() == "host.retention"
+ ]
+ now = 300
+ await asyncio.wait_for(removed.wait(), timeout=5)
+ assert maintenance.done()
+ assert app.state.host_context is None
diff --git a/tests/_server/host/test_host_translate.py b/tests/_server/host/test_host_translate.py
index 30405d1556..d32572c8d4 100644
--- a/tests/_server/host/test_host_translate.py
+++ b/tests/_server/host/test_host_translate.py
@@ -163,7 +163,7 @@ def test_an_exception_ends_the_run_in_error() -> None:
source="marimo",
severity="error",
range=None,
- related=[],
+ related=(),
kind="exception",
message="division by zero",
name="ZeroDivisionError",