A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Expire completed host request receipts (#11189)
Trevor Manz4b50662dfc06
diff --git a/marimo/_core/execution_state.py b/marimo/_core/execution_state.pyindex 1996868bfb..de861ac455 100644--- a/marimo/_core/execution_state.py+++ b/marimo/_core/execution_state.py@@ -84,16 +84,6 @@ class ExecutionState:def run(self, run_id: str) -> protocol.Run | None:return self._runs.get(run_id)- def request_run(self, request_id: str) -> protocol.Run | None:- return next(- (- self._runs[rid]- for rid, key in self._run_requests.items()- if key == request_id- ),- None,- )-def add_run(self, run: protocol.Run, request_id: str) -> None:self._run_requests[run.id] = request_idself._set_run(run)diff --git a/marimo/_core/notebook.py b/marimo/_core/notebook.pyindex f6dee8a761..f9b13c8177 100644--- a/marimo/_core/notebook.py+++ b/marimo/_core/notebook.py@@ -168,24 +168,21 @@ class Notebook:def run(self, code: str, *, request_id: str) -> protocol.Run:"""Queue scratch code on the runtime.- The request ID identifies the run. Repeating a request returns- the existing run while it is retained.+ The host assigns the run ID. Request IDs correlate events. The+ host retains retry receipts separately from execution results.Raises:NoRuntime: If the notebook has no runtime.Conflict: If the runtime is not running, or a run is in progress."""self._require_open()- run = self._state.request_run(request_id)- if run is not None:- return runruntime = self._require_runtime()if runtime.status != "running":raise Conflict(f"the runtime is {runtime.status}.")if (active := self._state.active_run) is not None:raise Conflict(f"run {active.id} is in progress.")run = protocol.Run(- id=protocol.RunId(request_id),+ id=mint(protocol.RunId, "run"),notebook_id=self.id,caused_by=None,status="queued",diff --git a/marimo/_server/host/api.py b/marimo/_server/host/api.pyindex be73dfd2dd..74b84e866a 100644--- a/marimo/_server/host/api.py+++ b/marimo/_server/host/api.py@@ -148,7 +148,7 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:)digest = hashlib.sha256(await request.body()).hexdigest()fingerprint = f"{request.method} {request.url.path} {digest}"- remembered = context.replies.get(key)+ remembered = context.remembered(key)if remembered is not None:if remembered.fingerprint != fingerprint:return problem(@@ -165,18 +165,19 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:mutation = CachedMutation(fingerprint, task)context.replies[key] = mutationunsubscribe: Callable[[], None] | None = None+ lifecycle_accepted: Callable[[Response], None] | None = Noneif operation == "runtime.run" and request.method == "POST":- # Retain the last run state even after a restart removes it- # from the notebook. Replaying the key must never execute again.+ # The receipt outlives results without retaining their payloads.def updated(event: Event) -> None:if isinstance(event.data, protocol.RunMessage):run = event.data.run- if run.id == key:+ if event.data.request_id == key:mutation.run = runif (run.status == "completed"and unsubscribe is not None):+ context.settle(mutation)unsubscribe()unsubscribe = context.bridge.host.subscribe(@@ -184,6 +185,18 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:notebook_id=request.path_params["id"],types={"run"},)+ elif operation in {+ "runtime.start",+ "runtime.stop",+ "runtime.restart",+ }:+ lifecycle_accepted, unsubscribe = _watch_lifecycle(+ context,+ mutation,+ operation=operation,+ notebook_id=request.path_params["id"],+ request_id=key,+ )def completed(task: asyncio.Task[Response]) -> None:if (@@ -194,6 +207,12 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:context.replies.pop(key, None)if unsubscribe is not None:unsubscribe()+ elif lifecycle_accepted is not None:+ lifecycle_accepted(task.result())+ elif mutation.expires_at is None and (+ mutation.run is None or mutation.run.status == "completed"+ ):+ context.settle(mutation)task.add_done_callback(completed)# A disconnected waiter must not cancel a shared mutation.@@ -204,6 +223,76 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:return decorate+def _watch_lifecycle(+ context: HostContext,+ mutation: CachedMutation,+ *,+ operation: str,+ notebook_id: str,+ request_id: str,+) -> tuple[Callable[[Response], None], Callable[[], None]]:+ runtime_id: protocol.RuntimeId | None = None+ previous_id: protocol.RuntimeId | None = None+ early: list[Event] = []++ def settle() -> None:+ context.settle(mutation)+ unsubscribe()++ def updated(event: Event) -> None:+ nonlocal previous_id+ if mutation.expires_at is not None:+ return+ if runtime_id is None:+ # A backend can finish synchronously, before the response exists.+ early.append(event)+ return+ if event.name == "notebook.removed":+ settle()+ elif isinstance(event.data, protocol.RuntimeMessage):+ runtime = event.data.runtime+ if (+ operation == "runtime.restart"+ and event.data.request_id == request_id+ and runtime.status == "terminating"+ and runtime.id != runtime_id+ ):+ previous_id = runtime.id+ if (+ operation != "runtime.stop"+ and runtime.id == runtime_id+ and runtime.status in ("running", "failed")+ ):+ settle()+ elif event.name == "runtime.removed" and isinstance(+ event.data, protocol.RemovedMessage+ ):+ if event.data.id == runtime_id or (+ event.data.id == previous_id+ and event.data.request_id != request_id+ ):+ # A later stop can cancel a restart before its replacement starts.+ settle()++ unsubscribe = context.bridge.host.subscribe(+ updated, notebook_id=notebook_id, types={"runtime", "notebook"}+ )++ def accepted(response: Response) -> None:+ nonlocal runtime_id+ if response.status_code == 202:+ runtime_id = msgspec.json.decode(+ response.body, type=protocol.Runtime+ ).id+ for event in early:+ updated(event)+ else:+ settle()+ early.clear()++ return accepted, unsubscribe++async def _execute_mutation(endpoint: _Endpoint, request: Request) -> Response:try:return await endpoint(request)diff --git a/marimo/_server/host/context.py b/marimo/_server/host/context.pyindex 069e7b1be3..07dcf4860f 100644--- a/marimo/_server/host/context.py+++ b/marimo/_server/host/context.py@@ -4,10 +4,12 @@from __future__ import annotationsfrom dataclasses import dataclass, field+from time import monotonicfrom typing import TYPE_CHECKINGif TYPE_CHECKING:import asyncio+ from collections.abc import Callablefrom starlette.responses import Response@@ -22,6 +24,7 @@ class CachedMutation:fingerprint: strtask: asyncio.Task[Response]run: protocol.Run | None = None+ expires_at: float | None = None@dataclass@@ -32,3 +35,31 @@ class HostContext:token: strport: intreplies: dict[str, CachedMutation] = field(default_factory=dict)+ # A receipt covers a client's retry loop after a lost response, so it+ # outlasts retry backoff rather than a person's absence.+ receipt_seconds: float = 300.0+ clock: Callable[[], float] = monotonic++ def remembered(self, key: str) -> CachedMutation | None:+ """Return the receipt for a key, dropping it if it has expired."""+ reply = self.replies.get(key)+ if reply is not None and self._expired(reply):+ del self.replies[key]+ return None+ return reply++ def settle(self, mutation: CachedMutation) -> None:+ """Start the receipt's retention window once its work is complete."""+ mutation.expires_at = self.clock() + self.receipt_seconds++ def expire(self) -> None:+ """Sweep expired receipts and the host's results."""+ self.bridge.host.expire()+ for key, reply in list(self.replies.items()):+ if self._expired(reply):+ del self.replies[key]++ def _expired(self, reply: CachedMutation) -> bool:+ return (+ reply.expires_at is not None and self.clock() >= reply.expires_at+ )diff --git a/marimo/_server/host/lifespan.py b/marimo/_server/host/lifespan.pyindex 77a73930b2..b73894c5f4 100644--- a/marimo/_server/host/lifespan.py+++ b/marimo/_server/host/lifespan.py@@ -42,7 +42,7 @@ async def host(app: Starlette) -> AsyncIterator[None]:async def maintain() -> None:while True:await asyncio.sleep(1)- bridge.host.expire()+ context.expire()maintenance = asyncio.create_task(maintain(), name="host.retention")record = HostRecordFile(url=url, token=token)diff --git a/tests/_core/test_host.py b/tests/_core/test_host.pyindex 667b3df2f7..37ea2a7bfa 100644--- a/tests/_core/test_host.py+++ b/tests/_core/test_host.py@@ -31,13 +31,13 @@ def test_catalog_snapshot_and_replay_exclude_runs(host: Host) -> None:notebook = host.add_notebook(record(), ScriptedBackend())notebook.start()notebook.observe(be.Running())- notebook.run("1", request_id="r1")+ run = notebook.run("1", request_id="r1")cursor = host.events()[-1].idassert cursor is not Nonenotebook.observe(be.Planned(- protocol.RunId("r1"),+ run.id,(be.PlannedExecution(protocol.ExecutionId("e1"), None, "h", "1"),),))@@ -64,10 +64,10 @@ def test_a_notebook_stream_sees_only_its_notebook(host: Host) -> None:second = host.add_notebook(record("n2", "b.py"), ScriptedBackend())first.start()first.observe(be.Running())- first.run("1", request_id="r1")+ run = first.run("1", request_id="r1")first.observe(be.Planned(- protocol.RunId("r1"),+ run.id,(be.PlannedExecution(protocol.ExecutionId("e1"), None, "h", "1"),),))@@ -138,7 +138,7 @@ def test_subscribe_filters_the_live_stream(host: Host) -> None:notebook.start()notebook.observe(be.Running())- notebook.run("1", request_id="r1")+ run = notebook.run("1", request_id="r1")unsubscribe()host.update_project(project("p2"))@@ -153,7 +153,7 @@ def test_removing_a_notebook_removes_what_it_owns_first(host: Host) -> None:notebook.start()notebook.observe(be.Running())notebook.attach()- notebook.run("1", request_id="r1")+ run = notebook.run("1", request_id="r1")log: list[Event] = []host.subscribe(log.append, notebook_id="n1")@@ -179,7 +179,7 @@ def test_updating_a_notebook_publishes_only_changes(host: Host) -> None:notebook = host.add_notebook(record(), backend)notebook.start()notebook.observe(be.Running())- notebook.run("1", request_id="run")+ run = notebook.run("1", request_id="run")runtime = notebook.runtimebackend.calls.clear()log: list[Event] = []@@ -194,7 +194,9 @@ def test_updating_a_notebook_publishes_only_changes(host: Host) -> None:assert host.notebook("n1") is notebookassert notebook.runtime is runtimeassert backend.calls == []- assert notebook.run("1", request_id="run").id == "run"+ assert host.events(notebook_id="n1", types={"run"})[+ 0+ ].data == protocol.RunMessage(run=run, request_id="run")assert host.events(notebook_id="n1")[0].data == protocol.NotebookMessage(notebook=moved)diff --git a/tests/_core/test_notebook.py b/tests/_core/test_notebook.pyindex 28a8d03512..d91e7fe44b 100644--- a/tests/_core/test_notebook.py+++ b/tests/_core/test_notebook.py@@ -172,7 +172,7 @@ class TestStop:log: list[Event],) -> None:running(notebook, backend)- scratch(notebook)+ run = scratch(notebook)notebook.observe(be.Produced(console("o1", "e1", "hi\n")))log.clear()@@ -220,11 +220,11 @@ class TestStop:log: list[Event],) -> None:running(notebook, backend)- scratch(notebook)+ run = scratch(notebook)notebook.observe(be.Produced(console("o1", "e1", "x")))notebook.observe(be.Concluded(ExecutionId("e1"), value("")))notebook.observe(be.Status(ExecutionId("e1"), "completed"))- notebook.observe(be.Done(RunId("run-1"), "succeeded"))+ notebook.observe(be.Done(run.id, "succeeded"))log.clear()notebook.stop(request_id="stop")@@ -256,7 +256,7 @@ class TestRestart:self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]) -> None:before = running(notebook, backend)- scratch(notebook)+ run = scratch(notebook)backend.calls.clear()log.clear()@@ -303,7 +303,7 @@ class TestAdoption:reports_starting: bool,) -> None:before = running(notebook, backend)- scratch(notebook)+ run = scratch(notebook)notebook.observe(be.Exited("Kernel exited"))backend.calls.clear()@@ -322,8 +322,8 @@ class TestAdoption:"ready",]assert backend.calls == []- notebook.run("1", request_id="after-relaunch")- assert backend.calls == [("run", "after-relaunch", "1")]+ run = notebook.run("1", request_id="after-relaunch")+ assert backend.calls == [("run", run.id, "1")]def test_a_kernel_someone_else_started_is_published(self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]@@ -344,7 +344,7 @@ class TestAdoption:log: list[Event],) -> None:running(notebook, backend)- scratch(notebook)+ run = scratch(notebook)log.clear()notebook.observe(be.Exited("killed"))@@ -458,7 +458,7 @@ class TestRun:notebook.observe(be.Done(run.id, "succeeded"))assert notebook.runtime is not None- assert backend.calls == [("run", key, "3 * 3")]+ assert backend.calls == [("run", run.id, "3 * 3")]assert names(log) == ["run","execution",@@ -513,7 +513,7 @@ class TestRun:Run(caused_by=None,completed_at=IsDatetime(),- id=RunId("7c1e9a3f"),+ id=run.id,notebook_id=NotebookId("n1"),outcome="succeeded",started_at=IsDatetime(),@@ -532,7 +532,7 @@ class TestRun:data="working<pre>9</pre>", mimetype="text/html")),- run_id=RunId("7c1e9a3f"),+ run_id=run.id,runtime_id=notebook.runtime.id,stale=None,started_at=IsDatetime(),@@ -548,13 +548,22 @@ class TestRun:])- def test_repeating_the_request_returns_the_same_run(+ def test_request_keys_do_not_identify_runs(self, notebook: Notebook, backend: ScriptedBackend) -> None:running(notebook, backend)first = notebook.run("1", request_id="same")- assert notebook.run("1", request_id="same") == first- assert backend.calls == [("run", "same", "1")]+ with pytest.raises(Conflict, match=first.id):+ notebook.run("1", request_id="same")+ notebook.observe(be.Done(first.id, "succeeded"))+ second = notebook.run("2", request_id="same")+ assert second.id != first.id+ # A completed run with no executions is released at once.+ assert backend.calls == [+ ("run", first.id, "1"),+ ("forget", first.id),+ ("run", second.id, "2"),+ ]def test_one_run_at_a_time(self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]@@ -579,7 +588,7 @@ class TestRun:self, host: Host, notebook: Notebook, backend: ScriptedBackend) -> None:running(notebook, backend)- scratch(notebook)+ run = scratch(notebook)notebook.observe(be.Produced(console("o1", "e1", "0\n")))notebook.observe(be.Produced(console("o1", "e1", "0\n1\n")))items = [@@ -629,7 +638,9 @@ class TestRunCompletion:)assert isinstance(final, RunMessage)assert final.run.caused_by == "e1"- assert notebook.run("3 * 3", request_id="run-1").status == "completed"+ retained = notebook._state.run(original.id)+ assert retained is not None+ assert retained.status == "completed"@pytest.mark.parametrize("outcome", ["failed", "interrupted"])def test_can_end_before_any_execution(@@ -702,13 +713,18 @@ class TestRunCompletion:)notebook.observe(be.Status(ExecutionId("child"), "completed"))# Even an empty execution queue is not an authoritative end to scheduling.- current = notebook.run("3 * 3", request_id=run.id)+ current = notebook._state.run(run.id)+ assert current is not Noneassert current.status == "running"assert current.outcome is Nonewith pytest.raises(ValueError, match="successful"):notebook.observe(be.Done(run.id, "succeeded"))notebook.observe(be.Done(run.id, outcome))- final = log[-1].data+ final = next(+ event.data+ for event in reversed(log)+ if isinstance(event.data, RunMessage)+ )assert isinstance(final, RunMessage)assert final.run.outcome == outcomeassert final.run.status == "completed"@@ -739,12 +755,12 @@ class TestInterrupt:self, notebook: Notebook, backend: ScriptedBackend) -> None:running(notebook, backend)- scratch(notebook)+ run = scratch(notebook)backend.calls.clear()- notebook.interrupt("run-1")+ notebook.interrupt(run.id)- assert backend.calls == [("interrupt", "run-1")]+ assert backend.calls == [("interrupt", run.id)]def test_refuses_unknown_and_finished_runs(self, notebook: Notebook, backend: ScriptedBackend@@ -752,12 +768,12 @@ class TestInterrupt:running(notebook, backend)with pytest.raises(NotFound):notebook.interrupt("nope")- scratch(notebook)+ run = scratch(notebook)notebook.observe(be.Concluded(ExecutionId("e1"), value("9")))notebook.observe(be.Status(ExecutionId("e1"), "completed"))- notebook.observe(be.Done(RunId("run-1"), "succeeded"))+ notebook.observe(be.Done(run.id, "succeeded"))with pytest.raises(Conflict):- notebook.interrupt("run-1")+ notebook.interrupt(run.id)class TestCellResults:diff --git a/tests/_server/host/test_host_api.py b/tests/_server/host/test_host_api.pyindex 814202ef49..80a3526617 100644--- a/tests/_server/host/test_host_api.py+++ b/tests/_server/host/test_host_api.py@@ -13,6 +13,7 @@ from urllib.parse import parse_qs, urlsplitimport pytestfrom marimo._core import backend as be, protocol+from marimo._core.execution_state import RetentionPolicyfrom marimo._core.host import Hostfrom marimo._core.protocol import ProjectMessagefrom marimo._server.host import api as api_module@@ -53,6 +54,8 @@ class Harness:self.app = appself.bridge = bridgeself.root = root+ self.now = 0.0+ self.host.clock = lambda: self.nowself.backends: dict[str, ScriptedBackend] = {}@property@@ -84,7 +87,7 @@ class Harness:@pytest.fixture-def harness(tmp_path: Path) -> Harness:+def harness(tmp_path: Path, request: pytest.FixtureRequest) -> Harness:(tmp_path / "forecasts.py").write_text("import marimo\n")workspace = FixedFilesWorkspace([MarimoFile(name="forecasts.py", path=str(tmp_path / "forecasts.py"))],@@ -94,6 +97,7 @@ def harness(tmp_path: Path) -> Harness:manager = get_mock_session_manager()get_starlette_server_state_init(session_manager=manager).apply(app.state)bridge = SessionBridge(manager)+ bridge.host.retention = getattr(request, "param", RetentionPolicy())harness = Harness(app, bridge, tmp_path)def make(notebook_id: str) -> ScriptedBackend:@@ -107,6 +111,8 @@ def harness(tmp_path: Path) -> Harness:bridge=bridge,token=TOKEN,port=PORT,+ receipt_seconds=30,+ clock=lambda: harness.now,)return harness@@ -321,6 +327,228 @@ async def test_runtime_lifecycle_over_http(harness: Harness) -> None:assert gone.json()["type"].endswith("/no-runtime")+async def test_slow_restart_keeps_its_receipt(harness: Harness) -> None:+ nid = await harness.running()+ notebook = harness.host.notebook(nid)+ path = f"/notebooks/{nid}/runtime/restart"+ accepted = await finished(+ harness.app, "POST", path, headers=key("restart")+ )+ assert accepted.status == 202++ # The receipt's 30-second window must not begin during shutdown/startup.+ harness.now = 100+ harness.app.state.host_context.expire()+ notebook.observe(be.Exited())+ notebook.observe(be.Running("0.25.0"))+ calls = list(harness.backend().calls)++ retried = await finished(harness.app, "POST", path, headers=key("restart"))+ assert (retried.status, retried.json(), harness.backend().calls) == (+ accepted.status,+ accepted.json(),+ calls,+ )++ harness.now = 129+ retried = await finished(harness.app, "POST", path, headers=key("restart"))+ assert (retried.json(), harness.backend().calls) == (+ accepted.json(),+ calls,+ )+ harness.now = 130+ fresh = await finished(harness.app, "POST", path, headers=key("restart"))+ assert fresh.status == 202+ assert fresh.json()["id"] != accepted.json()["id"]+ assert harness.backend().calls == [*calls, ("stop",)]+++ ("operation", "failed"),+ [+ ("start", False),+ ("start", True),+ ("stop", False),+ ("restart", False),+ ("restart", True),+ ],+)+async def test_lifecycle_receipts_expire_after_terminal_events(+ harness: Harness, operation: str, failed: bool+) -> None:+ nid = harness.notebook_id+ if operation != "start":+ await harness.running()+ notebook = harness.host.notebook(nid)+ context = harness.app.state.host_context+ listeners = list(harness.host._stream._listeners)+ method = {"start": "PUT", "stop": "DELETE", "restart": "POST"}[operation]+ path = f"/notebooks/{nid}/runtime"+ if operation == "restart":+ path += "/restart"+ accepted = await finished(+ harness.app, method, path, headers=key("lifecycle")+ )+ assert accepted.status == 202+ for now in (100, 200):+ harness.now = now+ context.expire()+ calls = list(harness.backend().calls)+ replayed = await finished(+ harness.app, method, path, headers=key("lifecycle")+ )+ assert (replayed.status, replayed.json(), harness.backend().calls) == (+ 202,+ accepted.json(),+ calls,+ )+ if operation == "restart" and now == 100:+ notebook.observe(be.Exited())+ notebook.observe(+ be.Exited("failed" if failed else None)+ if failed or operation == "stop"+ else be.Running()+ )+ assert harness.host._stream._listeners == listeners+ calls = list(harness.backend().calls)+ harness.now = 229+ context.expire()+ replayed = await finished(+ harness.app, method, path, headers=key("lifecycle")+ )+ assert (replayed.status, replayed.json(), harness.backend().calls) == (+ 202,+ accepted.json(),+ calls,+ )+ harness.now = 230+ assert context.remembered("lifecycle") is None+++ "operation", ["start", "stop", "restart", "replacement"]+)+async def test_stop_settles_superseded_lifecycle_receipts(+ harness: Harness, operation: str+) -> None:+ nid = harness.notebook_id+ if operation != "start":+ await harness.running()+ notebook = harness.host.notebook(nid)+ context = harness.app.state.host_context+ listeners = list(harness.host._stream._listeners)+ path = f"/notebooks/{nid}/runtime"+ method = "PUT" if operation == "start" else "DELETE"+ pending_path = path+ if operation in {"restart", "replacement"}:+ method, pending_path = "POST", path + "/restart"+ accepted = await finished(+ harness.app, method, pending_path, headers=key("pending")+ )+ if operation == "replacement":+ notebook.observe(be.Exited())+ harness.now = 100+ stops = [+ await finished(harness.app, "DELETE", path, headers=key(request_id))+ for request_id in ("stop-1", "stop-2")+ ]+ harness.now = 200+ notebook.observe(be.Exited())+ assert harness.host._stream._listeners == listeners+ calls = list(harness.backend().calls)+ harness.now = 229+ context.expire()+ for request_id, verb, url, response in [+ ("pending", method, pending_path, accepted),+ ("stop-1", "DELETE", path, stops[0]),+ ("stop-2", "DELETE", path, stops[1]),+ ]:+ replayed = await finished(+ harness.app, verb, url, headers=key(request_id)+ )+ assert (replayed.status, replayed.json(), harness.backend().calls) == (+ 202,+ response.json(),+ calls,+ )+ harness.now = 230+ assert [+ context.remembered(request_id)+ for request_id in ("pending", "stop-1", "stop-2")+ ] == [None, None, None]+++ "operation", ["start", "stop", "restart", "failed-stop"]+)+async def test_synchronous_lifecycle_completion(+ harness: Harness, monkeypatch: pytest.MonkeyPatch, operation: str+) -> None:+ nid = harness.notebook_id+ if operation != "start":+ await harness.running()+ notebook = harness.host.notebook(nid)+ if operation == "failed-stop":+ notebook.observe(be.Exited("failed"))+ backend = harness.backend()+ start, stop = backend.start, backend.stop++ def start_immediately(runtime_id: str) -> None:+ start(runtime_id)+ notebook.observe(be.Running())++ def stop_immediately() -> None:+ stop()+ notebook.observe(be.Exited())++ monkeypatch.setattr(backend, "start", start_immediately)+ monkeypatch.setattr(backend, "stop", stop_immediately)+ listeners = list(harness.host._stream._listeners)+ method = "PUT" if operation == "start" else "DELETE"+ path = f"/notebooks/{nid}/runtime"+ if operation == "restart":+ method, path = "POST", path + "/restart"+ accepted = await finished(+ harness.app, method, path, headers=key("immediate")+ )+ assert accepted.status == 202+ assert harness.host._stream._listeners == listeners+ calls = list(backend.calls)+ harness.now = 29+ replayed = await finished(+ harness.app, method, path, headers=key("immediate")+ )+ assert (replayed.status, replayed.json(), backend.calls) == (+ 202,+ accepted.json(),+ calls,+ )+ harness.now = 30+ assert harness.app.state.host_context.remembered("immediate") is None+++async def test_noop_and_rejected_lifecycle_requests_release_their_listeners(+ harness: Harness,+) -> None:+ nid = await harness.running()+ context = harness.app.state.host_context+ listeners = list(harness.host._stream._listeners)+ path = f"/notebooks/{nid}/runtime"+ existing = await finished(+ harness.app, "PUT", path, headers=key("existing")+ )+ assert existing.status == 200+ assert harness.host._stream._listeners == listeners+ rejected = await finished(+ harness.app, "PUT", path, headers=key("invalid"), body=b"{"+ )+ assert rejected.status == 400+ assert context.remembered("invalid") is None+ assert harness.host._stream._listeners == listeners+ harness.now = 30+ assert context.remembered("existing") is None++async def test_run_filter_over_http(harness: Harness) -> None:nid = await harness.running()notebook = harness.host.notebook(nid)@@ -428,6 +656,66 @@ async def test_scratch_runs_over_http(harness: Harness) -> None:assert unkeyed.status == 400+ "harness",+ [RetentionPolicy(ephemeral_seconds=10)],+ indirect=True,+)[email protected]("cell_id", [None, "a"])+async def test_request_receipts_expire_independently_of_results(+ harness: Harness, cell_id: str | None+) -> None:+ nid = await harness.running()+ notebook = harness.host.notebook(nid)+ path = f"/notebooks/{nid}/runtime/runs"+ body = b'{"code":"42"}'+ accepted = await finished(+ harness.app, "POST", path, headers=key("run"), body=body+ )+ run_id = protocol.RunId(accepted.json()["id"])+ eid = protocol.ExecutionId("execution")+ notebook.observe(+ be.Planned(run_id, (be.PlannedExecution(eid, cell_id, "hash", "42"),))+ )+ notebook.observe(be.Status(eid, "running"))+ harness.now = 100+ active = await finished(+ harness.app, "POST", path, headers=key("run"), body=body+ )+ assert active.json()["id"] == run_id+ assert active.json()["status"] == "running"+ notebook.observe(be.Concluded(eid, protocol.ValueOutcome(None)))+ notebook.observe(be.Status(eid, "completed"))+ notebook.observe(be.Done(run_id, "succeeded"))+ harness.now = 129+ harness.host.expire()+ replayed = await finished(+ harness.app, "POST", path, headers=key("run"), body=body+ )+ assert replayed.json()["id"] == run_id+ assert replayed.json()["status"] == "completed"+ assert replayed.json()["outcome"] == "succeeded"+ assert any(+ event.kind == "execution"+ for event in harness.host.events(notebook_id=nid)+ ) == (cell_id is not None)+ conflicting = await finished(+ harness.app, "POST", path, headers=key("run"), body=b'{"code":"43"}'+ )+ assert conflicting.status == 409++ harness.now = 130+ fresh = await finished(+ harness.app, "POST", path, headers=key("run"), body=b'{"code":"43"}'+ )+ assert fresh.status == 202+ assert fresh.json()["id"] != run_id+ assert [call for call in harness.backend().calls if call[0] == "run"] == [+ ("run", run_id, "42"),+ ("run", fresh.json()["id"], "43"),+ ]++@pytest.mark.parametrize("cancel_waiter", [False, True])async def test_concurrent_retries_share_the_mutation(harness: Harness, monkeypatch: pytest.MonkeyPatch, cancel_waiter: booldiff --git a/tests/_server/host/test_host_execution.py b/tests/_server/host/test_host_execution.pyindex 46b716adf1..38a5a9cd2f 100644--- a/tests/_server/host/test_host_execution.py+++ b/tests/_server/host/test_host_execution.py@@ -246,7 +246,7 @@ mo.Thread(target=worker).start()await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "first"+ r.get("data", {}).get("run", {}).get("id") == first.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),@@ -266,7 +266,7 @@ mo.Thread(target=worker).start()await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "second"+ r.get("data", {}).get("run", {}).get("id") == second.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),@@ -281,7 +281,7 @@ mo.Thread(target=worker).start()record["data"]["execution"]for record in events.recordsif record.get("event") == "execution"- and record["data"]["execution"]["run_id"] == "first"+ and record["data"]["execution"]["run_id"] == first.json()["id"])assert {item["execution_id"] for item in console} == {first_execution["id"]@@ -309,7 +309,10 @@ async def test_queued_cancellation_never_executes_the_code()assert accepted.status == 202interrupted = await finished(- app, "DELETE", runs + "/canceled", headers=key("interrupt")+ app,+ "DELETE",+ runs + "/" + accepted.json()["id"],+ headers=key("interrupt"),)assert interrupted.status == 202await wait_for(@@ -332,7 +335,8 @@ async def test_queued_cancellation_never_executes_the_code(await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "next"+ r.get("data", {}).get("run", {}).get("id")+ == following.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),@@ -491,7 +495,7 @@ async def test_scratch_run_includes_the_full_reactive_cycle(await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "cascade"+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),@@ -514,7 +518,7 @@ async def test_scratch_run_includes_the_full_reactive_cycle(assert root["cell_id"] is Noneassert root["outcome"]["kind"] == "value"assert all(- e["run_id"] == "cascade"+ e["run_id"] == accepted.json()["id"]and e["cell_id"] is not Noneand e["caused_by"] == root["id"]for e in downstream@@ -553,7 +557,8 @@ async def test_detached_reactivity_starts_a_new_run(if commandelse "mo.Thread(target=lambda: (gate.wait(), set_x(20))).start()")- for run_id, code in [+ submitted = {}+ for request_id, code in [("origin", origin),("release", "gate.set()"),]:@@ -561,10 +566,12 @@ async def test_detached_reactivity_starts_a_new_run(app,"POST",f"/notebooks/{notebook_id}/runtime/runs",- headers=key(run_id),+ headers=key(request_id),body=json.dumps({"code": code}).encode(),)assert accepted.status == 202+ run_id = accepted.json()["id"]+ submitted[request_id] = run_idawait wait_for(events,lambda response, run_id=run_id: any(@@ -591,7 +598,7 @@ async def test_detached_reactivity_starts_a_new_run(if r.get("data", {}).get("execution", {}).get("id")== printed["execution_id"])- assert execution["run_id"] not in ("origin", "release")+ assert execution["run_id"] not in submitted.values()await wait_for(events,lambda response: any(@@ -603,7 +610,7 @@ async def test_detached_reactivity_starts_a_new_run(origin = [r["data"]["run"]for r in events.records- if r.get("data", {}).get("run", {}).get("id") == "origin"+ if r.get("data", {}).get("run", {}).get("id") == submitted["origin"]]assert [r["status"] for r in origin] == ["queued", "running", "completed"]assert origin[-1]["outcome"] == "succeeded"@@ -633,7 +640,7 @@ async def test_nested_evaluations_keep_the_scratch_result_and_skipped_child(await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "nested"+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),@@ -683,7 +690,7 @@ async def test_a_fixed_cell_replaces_its_failed_compilation(await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "retry"+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),@@ -702,7 +709,7 @@ async def test_a_fixed_cell_replaces_its_failed_compilation(runs = [r["data"]["run"]for r in events.records- if r.get("data", {}).get("run", {}).get("id") == "retry"+ if r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]]assert runs[-1]["outcome"] == "failed"@@ -735,7 +742,7 @@ async def test_nested_commands_can_run_a_cell_queued_by_the_outer_command(await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "nested-queue"+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),@@ -751,7 +758,7 @@ async def test_nested_commands_can_run_a_cell_queued_by_the_outer_command(runs = [r["data"]["run"]for r in events.records- if r.get("data", {}).get("run", {}).get("id") == "nested-queue"+ if r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]]assert runs[-1]["outcome"] == "succeeded"@@ -791,14 +798,14 @@ async def test_awaited_commands_join_the_enclosing_run(await wait_for(events,lambda response: any(- r.get("data", {}).get("run", {}).get("id") == "awaited"+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]and r["data"]["run"]["status"] == "completed"for r in response.records),)records = events.records[before:]runs = [r["data"]["run"] for r in records if r.get("event") == "run"]- assert {r["id"] for r in runs} == {"awaited"}+ assert {r["id"] for r in runs} == {accepted.json()["id"]}assert runs[-1]["outcome"] == "failed"executions = [r["data"]["execution"]@@ -817,7 +824,7 @@ async def test_awaited_commands_join_the_enclosing_run(assert (child["code_hash"] == hashlib.sha256(child_code.encode()).hexdigest())- assert all(e["run_id"] == "awaited" for e in executions)+ assert all(e["run_id"] == accepted.json()["id"] for e in executions)@pytest.mark.parametrize(diff --git a/tests/_server/host/test_host_record.py b/tests/_server/host/test_host_record.pyindex 16175f621c..a62f34d6bf 100644--- a/tests/_server/host/test_host_record.py+++ b/tests/_server/host/test_host_record.py@@ -97,7 +97,7 @@ async def test_idle_host_expires_results_and_stops_maintenance(context = app.state.host_contexthost = context.bridge.hostnow = 0.0- host.clock = lambda: now+ host.clock = context.clock = lambda: nowhost.update_project(project())notebook = host.add_notebook(record(), ScriptedBackend())notebook.observe(be.Running())@@ -135,5 +135,6 @@ async def test_idle_host_expires_results_and_stops_maintenance(]now = 300await asyncio.wait_for(removed.wait(), timeout=5)+ assert context.replies == {}assert maintenance.done()assert app.state.host_context is None