Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Expire completed host request receipts (#11189)

Trevor Manz4b50662dfc06
diff --git a/marimo/_core/execution_state.py b/marimo/_core/execution_state.py
index 1996868bfb..de861ac455 100644
--- a/marimo/_core/execution_state.py
+++ b/marimo/_core/execution_state.py
@@ -84,16 +84,6 @@ class ExecutionState:
def run(self, run_id: str) -> protocol.Run | None:
return self._runs.get(run_id)
- def request_run(self, request_id: str) -> protocol.Run | None:
- return next(
- (
- self._runs[rid]
- for rid, key in self._run_requests.items()
- if key == request_id
- ),
- None,
- )
-
def add_run(self, run: protocol.Run, request_id: str) -> None:
self._run_requests[run.id] = request_id
self._set_run(run)
diff --git a/marimo/_core/notebook.py b/marimo/_core/notebook.py
index f6dee8a761..f9b13c8177 100644
--- a/marimo/_core/notebook.py
+++ b/marimo/_core/notebook.py
@@ -168,24 +168,21 @@ class Notebook:
def run(self, code: str, *, request_id: str) -> protocol.Run:
"""Queue scratch code on the runtime.
- The request ID identifies the run. Repeating a request returns
- the existing run while it is retained.
+ The host assigns the run ID. Request IDs correlate events. The
+ host retains retry receipts separately from execution results.
Raises:
NoRuntime: If the notebook has no runtime.
Conflict: If the runtime is not running, or a run is in progress.
"""
self._require_open()
- run = self._state.request_run(request_id)
- if run is not None:
- return run
runtime = self._require_runtime()
if runtime.status != "running":
raise Conflict(f"the runtime is {runtime.status}.")
if (active := self._state.active_run) is not None:
raise Conflict(f"run {active.id} is in progress.")
run = protocol.Run(
- id=protocol.RunId(request_id),
+ id=mint(protocol.RunId, "run"),
notebook_id=self.id,
caused_by=None,
status="queued",
diff --git a/marimo/_server/host/api.py b/marimo/_server/host/api.py
index be73dfd2dd..74b84e866a 100644
--- a/marimo/_server/host/api.py
+++ b/marimo/_server/host/api.py
@@ -148,7 +148,7 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:
)
digest = hashlib.sha256(await request.body()).hexdigest()
fingerprint = f"{request.method} {request.url.path} {digest}"
- remembered = context.replies.get(key)
+ remembered = context.remembered(key)
if remembered is not None:
if remembered.fingerprint != fingerprint:
return problem(
@@ -165,18 +165,19 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:
mutation = CachedMutation(fingerprint, task)
context.replies[key] = mutation
unsubscribe: Callable[[], None] | None = None
+ lifecycle_accepted: Callable[[Response], None] | None = None
if operation == "runtime.run" and request.method == "POST":
- # Retain the last run state even after a restart removes it
- # from the notebook. Replaying the key must never execute again.
+ # The receipt outlives results without retaining their payloads.
def updated(event: Event) -> None:
if isinstance(event.data, protocol.RunMessage):
run = event.data.run
- if run.id == key:
+ if event.data.request_id == key:
mutation.run = run
if (
run.status == "completed"
and unsubscribe is not None
):
+ context.settle(mutation)
unsubscribe()
unsubscribe = context.bridge.host.subscribe(
@@ -184,6 +185,18 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:
notebook_id=request.path_params["id"],
types={"run"},
)
+ elif operation in {
+ "runtime.start",
+ "runtime.stop",
+ "runtime.restart",
+ }:
+ lifecycle_accepted, unsubscribe = _watch_lifecycle(
+ context,
+ mutation,
+ operation=operation,
+ notebook_id=request.path_params["id"],
+ request_id=key,
+ )
def completed(task: asyncio.Task[Response]) -> None:
if (
@@ -194,6 +207,12 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:
context.replies.pop(key, None)
if unsubscribe is not None:
unsubscribe()
+ elif lifecycle_accepted is not None:
+ lifecycle_accepted(task.result())
+ elif mutation.expires_at is None and (
+ mutation.run is None or mutation.run.status == "completed"
+ ):
+ context.settle(mutation)
task.add_done_callback(completed)
# A disconnected waiter must not cancel a shared mutation.
@@ -204,6 +223,76 @@ def _mutation(operation: str) -> Callable[[_Endpoint], _Endpoint]:
return decorate
+def _watch_lifecycle(
+ context: HostContext,
+ mutation: CachedMutation,
+ *,
+ operation: str,
+ notebook_id: str,
+ request_id: str,
+) -> tuple[Callable[[Response], None], Callable[[], None]]:
+ runtime_id: protocol.RuntimeId | None = None
+ previous_id: protocol.RuntimeId | None = None
+ early: list[Event] = []
+
+ def settle() -> None:
+ context.settle(mutation)
+ unsubscribe()
+
+ def updated(event: Event) -> None:
+ nonlocal previous_id
+ if mutation.expires_at is not None:
+ return
+ if runtime_id is None:
+ # A backend can finish synchronously, before the response exists.
+ early.append(event)
+ return
+ if event.name == "notebook.removed":
+ settle()
+ elif isinstance(event.data, protocol.RuntimeMessage):
+ runtime = event.data.runtime
+ if (
+ operation == "runtime.restart"
+ and event.data.request_id == request_id
+ and runtime.status == "terminating"
+ and runtime.id != runtime_id
+ ):
+ previous_id = runtime.id
+ if (
+ operation != "runtime.stop"
+ and runtime.id == runtime_id
+ and runtime.status in ("running", "failed")
+ ):
+ settle()
+ elif event.name == "runtime.removed" and isinstance(
+ event.data, protocol.RemovedMessage
+ ):
+ if event.data.id == runtime_id or (
+ event.data.id == previous_id
+ and event.data.request_id != request_id
+ ):
+ # A later stop can cancel a restart before its replacement starts.
+ settle()
+
+ unsubscribe = context.bridge.host.subscribe(
+ updated, notebook_id=notebook_id, types={"runtime", "notebook"}
+ )
+
+ def accepted(response: Response) -> None:
+ nonlocal runtime_id
+ if response.status_code == 202:
+ runtime_id = msgspec.json.decode(
+ response.body, type=protocol.Runtime
+ ).id
+ for event in early:
+ updated(event)
+ else:
+ settle()
+ early.clear()
+
+ return accepted, unsubscribe
+
+
async def _execute_mutation(endpoint: _Endpoint, request: Request) -> Response:
try:
return await endpoint(request)
diff --git a/marimo/_server/host/context.py b/marimo/_server/host/context.py
index 069e7b1be3..07dcf4860f 100644
--- a/marimo/_server/host/context.py
+++ b/marimo/_server/host/context.py
@@ -4,10 +4,12 @@
from __future__ import annotations
from dataclasses import dataclass, field
+from time import monotonic
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import asyncio
+ from collections.abc import Callable
from starlette.responses import Response
@@ -22,6 +24,7 @@ class CachedMutation:
fingerprint: str
task: asyncio.Task[Response]
run: protocol.Run | None = None
+ expires_at: float | None = None
@dataclass
@@ -32,3 +35,31 @@ class HostContext:
token: str
port: int
replies: dict[str, CachedMutation] = field(default_factory=dict)
+ # A receipt covers a client's retry loop after a lost response, so it
+ # outlasts retry backoff rather than a person's absence.
+ receipt_seconds: float = 300.0
+ clock: Callable[[], float] = monotonic
+
+ def remembered(self, key: str) -> CachedMutation | None:
+ """Return the receipt for a key, dropping it if it has expired."""
+ reply = self.replies.get(key)
+ if reply is not None and self._expired(reply):
+ del self.replies[key]
+ return None
+ return reply
+
+ def settle(self, mutation: CachedMutation) -> None:
+ """Start the receipt's retention window once its work is complete."""
+ mutation.expires_at = self.clock() + self.receipt_seconds
+
+ def expire(self) -> None:
+ """Sweep expired receipts and the host's results."""
+ self.bridge.host.expire()
+ for key, reply in list(self.replies.items()):
+ if self._expired(reply):
+ del self.replies[key]
+
+ def _expired(self, reply: CachedMutation) -> bool:
+ return (
+ reply.expires_at is not None and self.clock() >= reply.expires_at
+ )
diff --git a/marimo/_server/host/lifespan.py b/marimo/_server/host/lifespan.py
index 77a73930b2..b73894c5f4 100644
--- a/marimo/_server/host/lifespan.py
+++ b/marimo/_server/host/lifespan.py
@@ -42,7 +42,7 @@ async def host(app: Starlette) -> AsyncIterator[None]:
async def maintain() -> None:
while True:
await asyncio.sleep(1)
- bridge.host.expire()
+ context.expire()
maintenance = asyncio.create_task(maintain(), name="host.retention")
record = HostRecordFile(url=url, token=token)
diff --git a/tests/_core/test_host.py b/tests/_core/test_host.py
index 667b3df2f7..37ea2a7bfa 100644
--- a/tests/_core/test_host.py
+++ b/tests/_core/test_host.py
@@ -31,13 +31,13 @@ def test_catalog_snapshot_and_replay_exclude_runs(host: Host) -> None:
notebook = host.add_notebook(record(), ScriptedBackend())
notebook.start()
notebook.observe(be.Running())
- notebook.run("1", request_id="r1")
+ run = notebook.run("1", request_id="r1")
cursor = host.events()[-1].id
assert cursor is not None
notebook.observe(
be.Planned(
- protocol.RunId("r1"),
+ run.id,
(be.PlannedExecution(protocol.ExecutionId("e1"), None, "h", "1"),),
)
)
@@ -64,10 +64,10 @@ def test_a_notebook_stream_sees_only_its_notebook(host: Host) -> None:
second = host.add_notebook(record("n2", "b.py"), ScriptedBackend())
first.start()
first.observe(be.Running())
- first.run("1", request_id="r1")
+ run = first.run("1", request_id="r1")
first.observe(
be.Planned(
- protocol.RunId("r1"),
+ run.id,
(be.PlannedExecution(protocol.ExecutionId("e1"), None, "h", "1"),),
)
)
@@ -138,7 +138,7 @@ def test_subscribe_filters_the_live_stream(host: Host) -> None:
notebook.start()
notebook.observe(be.Running())
- notebook.run("1", request_id="r1")
+ run = notebook.run("1", request_id="r1")
unsubscribe()
host.update_project(project("p2"))
@@ -153,7 +153,7 @@ def test_removing_a_notebook_removes_what_it_owns_first(host: Host) -> None:
notebook.start()
notebook.observe(be.Running())
notebook.attach()
- notebook.run("1", request_id="r1")
+ run = notebook.run("1", request_id="r1")
log: list[Event] = []
host.subscribe(log.append, notebook_id="n1")
@@ -179,7 +179,7 @@ def test_updating_a_notebook_publishes_only_changes(host: Host) -> None:
notebook = host.add_notebook(record(), backend)
notebook.start()
notebook.observe(be.Running())
- notebook.run("1", request_id="run")
+ run = notebook.run("1", request_id="run")
runtime = notebook.runtime
backend.calls.clear()
log: list[Event] = []
@@ -194,7 +194,9 @@ def test_updating_a_notebook_publishes_only_changes(host: Host) -> None:
assert host.notebook("n1") is notebook
assert notebook.runtime is runtime
assert backend.calls == []
- assert notebook.run("1", request_id="run").id == "run"
+ assert host.events(notebook_id="n1", types={"run"})[
+ 0
+ ].data == protocol.RunMessage(run=run, request_id="run")
assert host.events(notebook_id="n1")[0].data == protocol.NotebookMessage(
notebook=moved
)
diff --git a/tests/_core/test_notebook.py b/tests/_core/test_notebook.py
index 28a8d03512..d91e7fe44b 100644
--- a/tests/_core/test_notebook.py
+++ b/tests/_core/test_notebook.py
@@ -172,7 +172,7 @@ class TestStop:
log: list[Event],
) -> None:
running(notebook, backend)
- scratch(notebook)
+ run = scratch(notebook)
notebook.observe(be.Produced(console("o1", "e1", "hi\n")))
log.clear()
@@ -220,11 +220,11 @@ class TestStop:
log: list[Event],
) -> None:
running(notebook, backend)
- scratch(notebook)
+ run = scratch(notebook)
notebook.observe(be.Produced(console("o1", "e1", "x")))
notebook.observe(be.Concluded(ExecutionId("e1"), value("")))
notebook.observe(be.Status(ExecutionId("e1"), "completed"))
- notebook.observe(be.Done(RunId("run-1"), "succeeded"))
+ notebook.observe(be.Done(run.id, "succeeded"))
log.clear()
notebook.stop(request_id="stop")
@@ -256,7 +256,7 @@ class TestRestart:
self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]
) -> None:
before = running(notebook, backend)
- scratch(notebook)
+ run = scratch(notebook)
backend.calls.clear()
log.clear()
@@ -303,7 +303,7 @@ class TestAdoption:
reports_starting: bool,
) -> None:
before = running(notebook, backend)
- scratch(notebook)
+ run = scratch(notebook)
notebook.observe(be.Exited("Kernel exited"))
backend.calls.clear()
@@ -322,8 +322,8 @@ class TestAdoption:
"ready",
]
assert backend.calls == []
- notebook.run("1", request_id="after-relaunch")
- assert backend.calls == [("run", "after-relaunch", "1")]
+ run = notebook.run("1", request_id="after-relaunch")
+ assert backend.calls == [("run", run.id, "1")]
def test_a_kernel_someone_else_started_is_published(
self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]
@@ -344,7 +344,7 @@ class TestAdoption:
log: list[Event],
) -> None:
running(notebook, backend)
- scratch(notebook)
+ run = scratch(notebook)
log.clear()
notebook.observe(be.Exited("killed"))
@@ -458,7 +458,7 @@ class TestRun:
notebook.observe(be.Done(run.id, "succeeded"))
assert notebook.runtime is not None
- assert backend.calls == [("run", key, "3 * 3")]
+ assert backend.calls == [("run", run.id, "3 * 3")]
assert names(log) == [
"run",
"execution",
@@ -513,7 +513,7 @@ class TestRun:
Run(
caused_by=None,
completed_at=IsDatetime(),
- id=RunId("7c1e9a3f"),
+ id=run.id,
notebook_id=NotebookId("n1"),
outcome="succeeded",
started_at=IsDatetime(),
@@ -532,7 +532,7 @@ class TestRun:
data="working<pre>9</pre>", mimetype="text/html"
)
),
- run_id=RunId("7c1e9a3f"),
+ run_id=run.id,
runtime_id=notebook.runtime.id,
stale=None,
started_at=IsDatetime(),
@@ -548,13 +548,22 @@ class TestRun:
]
)
- def test_repeating_the_request_returns_the_same_run(
+ def test_request_keys_do_not_identify_runs(
self, notebook: Notebook, backend: ScriptedBackend
) -> None:
running(notebook, backend)
first = notebook.run("1", request_id="same")
- assert notebook.run("1", request_id="same") == first
- assert backend.calls == [("run", "same", "1")]
+ with pytest.raises(Conflict, match=first.id):
+ notebook.run("1", request_id="same")
+ notebook.observe(be.Done(first.id, "succeeded"))
+ second = notebook.run("2", request_id="same")
+ assert second.id != first.id
+ # A completed run with no executions is released at once.
+ assert backend.calls == [
+ ("run", first.id, "1"),
+ ("forget", first.id),
+ ("run", second.id, "2"),
+ ]
def test_one_run_at_a_time(
self, notebook: Notebook, backend: ScriptedBackend, log: list[Event]
@@ -579,7 +588,7 @@ class TestRun:
self, host: Host, notebook: Notebook, backend: ScriptedBackend
) -> None:
running(notebook, backend)
- scratch(notebook)
+ run = scratch(notebook)
notebook.observe(be.Produced(console("o1", "e1", "0\n")))
notebook.observe(be.Produced(console("o1", "e1", "0\n1\n")))
items = [
@@ -629,7 +638,9 @@ class TestRunCompletion:
)
assert isinstance(final, RunMessage)
assert final.run.caused_by == "e1"
- assert notebook.run("3 * 3", request_id="run-1").status == "completed"
+ retained = notebook._state.run(original.id)
+ assert retained is not None
+ assert retained.status == "completed"
@pytest.mark.parametrize("outcome", ["failed", "interrupted"])
def test_can_end_before_any_execution(
@@ -702,13 +713,18 @@ class TestRunCompletion:
)
notebook.observe(be.Status(ExecutionId("child"), "completed"))
# Even an empty execution queue is not an authoritative end to scheduling.
- current = notebook.run("3 * 3", request_id=run.id)
+ current = notebook._state.run(run.id)
+ assert current is not None
assert current.status == "running"
assert current.outcome is None
with pytest.raises(ValueError, match="successful"):
notebook.observe(be.Done(run.id, "succeeded"))
notebook.observe(be.Done(run.id, outcome))
- final = log[-1].data
+ final = next(
+ event.data
+ for event in reversed(log)
+ if isinstance(event.data, RunMessage)
+ )
assert isinstance(final, RunMessage)
assert final.run.outcome == outcome
assert final.run.status == "completed"
@@ -739,12 +755,12 @@ class TestInterrupt:
self, notebook: Notebook, backend: ScriptedBackend
) -> None:
running(notebook, backend)
- scratch(notebook)
+ run = scratch(notebook)
backend.calls.clear()
- notebook.interrupt("run-1")
+ notebook.interrupt(run.id)
- assert backend.calls == [("interrupt", "run-1")]
+ assert backend.calls == [("interrupt", run.id)]
def test_refuses_unknown_and_finished_runs(
self, notebook: Notebook, backend: ScriptedBackend
@@ -752,12 +768,12 @@ class TestInterrupt:
running(notebook, backend)
with pytest.raises(NotFound):
notebook.interrupt("nope")
- scratch(notebook)
+ run = scratch(notebook)
notebook.observe(be.Concluded(ExecutionId("e1"), value("9")))
notebook.observe(be.Status(ExecutionId("e1"), "completed"))
- notebook.observe(be.Done(RunId("run-1"), "succeeded"))
+ notebook.observe(be.Done(run.id, "succeeded"))
with pytest.raises(Conflict):
- notebook.interrupt("run-1")
+ notebook.interrupt(run.id)
class TestCellResults:
diff --git a/tests/_server/host/test_host_api.py b/tests/_server/host/test_host_api.py
index 814202ef49..80a3526617 100644
--- a/tests/_server/host/test_host_api.py
+++ b/tests/_server/host/test_host_api.py
@@ -13,6 +13,7 @@ from urllib.parse import parse_qs, urlsplit
import pytest
from marimo._core import backend as be, protocol
+from marimo._core.execution_state import RetentionPolicy
from marimo._core.host import Host
from marimo._core.protocol import ProjectMessage
from marimo._server.host import api as api_module
@@ -53,6 +54,8 @@ class Harness:
self.app = app
self.bridge = bridge
self.root = root
+ self.now = 0.0
+ self.host.clock = lambda: self.now
self.backends: dict[str, ScriptedBackend] = {}
@property
@@ -84,7 +87,7 @@ class Harness:
@pytest.fixture
-def harness(tmp_path: Path) -> Harness:
+def harness(tmp_path: Path, request: pytest.FixtureRequest) -> Harness:
(tmp_path / "forecasts.py").write_text("import marimo\n")
workspace = FixedFilesWorkspace(
[MarimoFile(name="forecasts.py", path=str(tmp_path / "forecasts.py"))],
@@ -94,6 +97,7 @@ def harness(tmp_path: Path) -> Harness:
manager = get_mock_session_manager()
get_starlette_server_state_init(session_manager=manager).apply(app.state)
bridge = SessionBridge(manager)
+ bridge.host.retention = getattr(request, "param", RetentionPolicy())
harness = Harness(app, bridge, tmp_path)
def make(notebook_id: str) -> ScriptedBackend:
@@ -107,6 +111,8 @@ def harness(tmp_path: Path) -> Harness:
bridge=bridge,
token=TOKEN,
port=PORT,
+ receipt_seconds=30,
+ clock=lambda: harness.now,
)
return harness
@@ -321,6 +327,228 @@ async def test_runtime_lifecycle_over_http(harness: Harness) -> None:
assert gone.json()["type"].endswith("/no-runtime")
+async def test_slow_restart_keeps_its_receipt(harness: Harness) -> None:
+ nid = await harness.running()
+ notebook = harness.host.notebook(nid)
+ path = f"/notebooks/{nid}/runtime/restart"
+ accepted = await finished(
+ harness.app, "POST", path, headers=key("restart")
+ )
+ assert accepted.status == 202
+
+ # The receipt's 30-second window must not begin during shutdown/startup.
+ harness.now = 100
+ harness.app.state.host_context.expire()
+ notebook.observe(be.Exited())
+ notebook.observe(be.Running("0.25.0"))
+ calls = list(harness.backend().calls)
+
+ retried = await finished(harness.app, "POST", path, headers=key("restart"))
+ assert (retried.status, retried.json(), harness.backend().calls) == (
+ accepted.status,
+ accepted.json(),
+ calls,
+ )
+
+ harness.now = 129
+ retried = await finished(harness.app, "POST", path, headers=key("restart"))
+ assert (retried.json(), harness.backend().calls) == (
+ accepted.json(),
+ calls,
+ )
+ harness.now = 130
+ fresh = await finished(harness.app, "POST", path, headers=key("restart"))
+ assert fresh.status == 202
+ assert fresh.json()["id"] != accepted.json()["id"]
+ assert harness.backend().calls == [*calls, ("stop",)]
+
+
+ ("operation", "failed"),
+ [
+ ("start", False),
+ ("start", True),
+ ("stop", False),
+ ("restart", False),
+ ("restart", True),
+ ],
+)
+async def test_lifecycle_receipts_expire_after_terminal_events(
+ harness: Harness, operation: str, failed: bool
+) -> None:
+ nid = harness.notebook_id
+ if operation != "start":
+ await harness.running()
+ notebook = harness.host.notebook(nid)
+ context = harness.app.state.host_context
+ listeners = list(harness.host._stream._listeners)
+ method = {"start": "PUT", "stop": "DELETE", "restart": "POST"}[operation]
+ path = f"/notebooks/{nid}/runtime"
+ if operation == "restart":
+ path += "/restart"
+ accepted = await finished(
+ harness.app, method, path, headers=key("lifecycle")
+ )
+ assert accepted.status == 202
+ for now in (100, 200):
+ harness.now = now
+ context.expire()
+ calls = list(harness.backend().calls)
+ replayed = await finished(
+ harness.app, method, path, headers=key("lifecycle")
+ )
+ assert (replayed.status, replayed.json(), harness.backend().calls) == (
+ 202,
+ accepted.json(),
+ calls,
+ )
+ if operation == "restart" and now == 100:
+ notebook.observe(be.Exited())
+ notebook.observe(
+ be.Exited("failed" if failed else None)
+ if failed or operation == "stop"
+ else be.Running()
+ )
+ assert harness.host._stream._listeners == listeners
+ calls = list(harness.backend().calls)
+ harness.now = 229
+ context.expire()
+ replayed = await finished(
+ harness.app, method, path, headers=key("lifecycle")
+ )
+ assert (replayed.status, replayed.json(), harness.backend().calls) == (
+ 202,
+ accepted.json(),
+ calls,
+ )
+ harness.now = 230
+ assert context.remembered("lifecycle") is None
+
+
+ "operation", ["start", "stop", "restart", "replacement"]
+)
+async def test_stop_settles_superseded_lifecycle_receipts(
+ harness: Harness, operation: str
+) -> None:
+ nid = harness.notebook_id
+ if operation != "start":
+ await harness.running()
+ notebook = harness.host.notebook(nid)
+ context = harness.app.state.host_context
+ listeners = list(harness.host._stream._listeners)
+ path = f"/notebooks/{nid}/runtime"
+ method = "PUT" if operation == "start" else "DELETE"
+ pending_path = path
+ if operation in {"restart", "replacement"}:
+ method, pending_path = "POST", path + "/restart"
+ accepted = await finished(
+ harness.app, method, pending_path, headers=key("pending")
+ )
+ if operation == "replacement":
+ notebook.observe(be.Exited())
+ harness.now = 100
+ stops = [
+ await finished(harness.app, "DELETE", path, headers=key(request_id))
+ for request_id in ("stop-1", "stop-2")
+ ]
+ harness.now = 200
+ notebook.observe(be.Exited())
+ assert harness.host._stream._listeners == listeners
+ calls = list(harness.backend().calls)
+ harness.now = 229
+ context.expire()
+ for request_id, verb, url, response in [
+ ("pending", method, pending_path, accepted),
+ ("stop-1", "DELETE", path, stops[0]),
+ ("stop-2", "DELETE", path, stops[1]),
+ ]:
+ replayed = await finished(
+ harness.app, verb, url, headers=key(request_id)
+ )
+ assert (replayed.status, replayed.json(), harness.backend().calls) == (
+ 202,
+ response.json(),
+ calls,
+ )
+ harness.now = 230
+ assert [
+ context.remembered(request_id)
+ for request_id in ("pending", "stop-1", "stop-2")
+ ] == [None, None, None]
+
+
+ "operation", ["start", "stop", "restart", "failed-stop"]
+)
+async def test_synchronous_lifecycle_completion(
+ harness: Harness, monkeypatch: pytest.MonkeyPatch, operation: str
+) -> None:
+ nid = harness.notebook_id
+ if operation != "start":
+ await harness.running()
+ notebook = harness.host.notebook(nid)
+ if operation == "failed-stop":
+ notebook.observe(be.Exited("failed"))
+ backend = harness.backend()
+ start, stop = backend.start, backend.stop
+
+ def start_immediately(runtime_id: str) -> None:
+ start(runtime_id)
+ notebook.observe(be.Running())
+
+ def stop_immediately() -> None:
+ stop()
+ notebook.observe(be.Exited())
+
+ monkeypatch.setattr(backend, "start", start_immediately)
+ monkeypatch.setattr(backend, "stop", stop_immediately)
+ listeners = list(harness.host._stream._listeners)
+ method = "PUT" if operation == "start" else "DELETE"
+ path = f"/notebooks/{nid}/runtime"
+ if operation == "restart":
+ method, path = "POST", path + "/restart"
+ accepted = await finished(
+ harness.app, method, path, headers=key("immediate")
+ )
+ assert accepted.status == 202
+ assert harness.host._stream._listeners == listeners
+ calls = list(backend.calls)
+ harness.now = 29
+ replayed = await finished(
+ harness.app, method, path, headers=key("immediate")
+ )
+ assert (replayed.status, replayed.json(), backend.calls) == (
+ 202,
+ accepted.json(),
+ calls,
+ )
+ harness.now = 30
+ assert harness.app.state.host_context.remembered("immediate") is None
+
+
+async def test_noop_and_rejected_lifecycle_requests_release_their_listeners(
+ harness: Harness,
+) -> None:
+ nid = await harness.running()
+ context = harness.app.state.host_context
+ listeners = list(harness.host._stream._listeners)
+ path = f"/notebooks/{nid}/runtime"
+ existing = await finished(
+ harness.app, "PUT", path, headers=key("existing")
+ )
+ assert existing.status == 200
+ assert harness.host._stream._listeners == listeners
+ rejected = await finished(
+ harness.app, "PUT", path, headers=key("invalid"), body=b"{"
+ )
+ assert rejected.status == 400
+ assert context.remembered("invalid") is None
+ assert harness.host._stream._listeners == listeners
+ harness.now = 30
+ assert context.remembered("existing") is None
+
+
async def test_run_filter_over_http(harness: Harness) -> None:
nid = await harness.running()
notebook = harness.host.notebook(nid)
@@ -428,6 +656,66 @@ async def test_scratch_runs_over_http(harness: Harness) -> None:
assert unkeyed.status == 400
+ "harness",
+ [RetentionPolicy(ephemeral_seconds=10)],
+ indirect=True,
+)
[email protected]("cell_id", [None, "a"])
+async def test_request_receipts_expire_independently_of_results(
+ harness: Harness, cell_id: str | None
+) -> None:
+ nid = await harness.running()
+ notebook = harness.host.notebook(nid)
+ path = f"/notebooks/{nid}/runtime/runs"
+ body = b'{"code":"42"}'
+ accepted = await finished(
+ harness.app, "POST", path, headers=key("run"), body=body
+ )
+ run_id = protocol.RunId(accepted.json()["id"])
+ eid = protocol.ExecutionId("execution")
+ notebook.observe(
+ be.Planned(run_id, (be.PlannedExecution(eid, cell_id, "hash", "42"),))
+ )
+ notebook.observe(be.Status(eid, "running"))
+ harness.now = 100
+ active = await finished(
+ harness.app, "POST", path, headers=key("run"), body=body
+ )
+ assert active.json()["id"] == run_id
+ assert active.json()["status"] == "running"
+ notebook.observe(be.Concluded(eid, protocol.ValueOutcome(None)))
+ notebook.observe(be.Status(eid, "completed"))
+ notebook.observe(be.Done(run_id, "succeeded"))
+ harness.now = 129
+ harness.host.expire()
+ replayed = await finished(
+ harness.app, "POST", path, headers=key("run"), body=body
+ )
+ assert replayed.json()["id"] == run_id
+ assert replayed.json()["status"] == "completed"
+ assert replayed.json()["outcome"] == "succeeded"
+ assert any(
+ event.kind == "execution"
+ for event in harness.host.events(notebook_id=nid)
+ ) == (cell_id is not None)
+ conflicting = await finished(
+ harness.app, "POST", path, headers=key("run"), body=b'{"code":"43"}'
+ )
+ assert conflicting.status == 409
+
+ harness.now = 130
+ fresh = await finished(
+ harness.app, "POST", path, headers=key("run"), body=b'{"code":"43"}'
+ )
+ assert fresh.status == 202
+ assert fresh.json()["id"] != run_id
+ assert [call for call in harness.backend().calls if call[0] == "run"] == [
+ ("run", run_id, "42"),
+ ("run", fresh.json()["id"], "43"),
+ ]
+
+
@pytest.mark.parametrize("cancel_waiter", [False, True])
async def test_concurrent_retries_share_the_mutation(
harness: Harness, monkeypatch: pytest.MonkeyPatch, cancel_waiter: bool
diff --git a/tests/_server/host/test_host_execution.py b/tests/_server/host/test_host_execution.py
index 46b716adf1..38a5a9cd2f 100644
--- a/tests/_server/host/test_host_execution.py
+++ b/tests/_server/host/test_host_execution.py
@@ -246,7 +246,7 @@ mo.Thread(target=worker).start()
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "first"
+ r.get("data", {}).get("run", {}).get("id") == first.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
@@ -266,7 +266,7 @@ mo.Thread(target=worker).start()
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "second"
+ r.get("data", {}).get("run", {}).get("id") == second.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
@@ -281,7 +281,7 @@ mo.Thread(target=worker).start()
record["data"]["execution"]
for record in events.records
if record.get("event") == "execution"
- and record["data"]["execution"]["run_id"] == "first"
+ and record["data"]["execution"]["run_id"] == first.json()["id"]
)
assert {item["execution_id"] for item in console} == {
first_execution["id"]
@@ -309,7 +309,10 @@ async def test_queued_cancellation_never_executes_the_code(
)
assert accepted.status == 202
interrupted = await finished(
- app, "DELETE", runs + "/canceled", headers=key("interrupt")
+ app,
+ "DELETE",
+ runs + "/" + accepted.json()["id"],
+ headers=key("interrupt"),
)
assert interrupted.status == 202
await wait_for(
@@ -332,7 +335,8 @@ async def test_queued_cancellation_never_executes_the_code(
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "next"
+ r.get("data", {}).get("run", {}).get("id")
+ == following.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
@@ -491,7 +495,7 @@ async def test_scratch_run_includes_the_full_reactive_cycle(
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "cascade"
+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
@@ -514,7 +518,7 @@ async def test_scratch_run_includes_the_full_reactive_cycle(
assert root["cell_id"] is None
assert root["outcome"]["kind"] == "value"
assert all(
- e["run_id"] == "cascade"
+ e["run_id"] == accepted.json()["id"]
and e["cell_id"] is not None
and e["caused_by"] == root["id"]
for e in downstream
@@ -553,7 +557,8 @@ async def test_detached_reactivity_starts_a_new_run(
if command
else "mo.Thread(target=lambda: (gate.wait(), set_x(20))).start()"
)
- for run_id, code in [
+ submitted = {}
+ for request_id, code in [
("origin", origin),
("release", "gate.set()"),
]:
@@ -561,10 +566,12 @@ async def test_detached_reactivity_starts_a_new_run(
app,
"POST",
f"/notebooks/{notebook_id}/runtime/runs",
- headers=key(run_id),
+ headers=key(request_id),
body=json.dumps({"code": code}).encode(),
)
assert accepted.status == 202
+ run_id = accepted.json()["id"]
+ submitted[request_id] = run_id
await wait_for(
events,
lambda response, run_id=run_id: any(
@@ -591,7 +598,7 @@ async def test_detached_reactivity_starts_a_new_run(
if r.get("data", {}).get("execution", {}).get("id")
== printed["execution_id"]
)
- assert execution["run_id"] not in ("origin", "release")
+ assert execution["run_id"] not in submitted.values()
await wait_for(
events,
lambda response: any(
@@ -603,7 +610,7 @@ async def test_detached_reactivity_starts_a_new_run(
origin = [
r["data"]["run"]
for r in events.records
- if r.get("data", {}).get("run", {}).get("id") == "origin"
+ if r.get("data", {}).get("run", {}).get("id") == submitted["origin"]
]
assert [r["status"] for r in origin] == ["queued", "running", "completed"]
assert origin[-1]["outcome"] == "succeeded"
@@ -633,7 +640,7 @@ async def test_nested_evaluations_keep_the_scratch_result_and_skipped_child(
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "nested"
+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
@@ -683,7 +690,7 @@ async def test_a_fixed_cell_replaces_its_failed_compilation(
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "retry"
+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
@@ -702,7 +709,7 @@ async def test_a_fixed_cell_replaces_its_failed_compilation(
runs = [
r["data"]["run"]
for r in events.records
- if r.get("data", {}).get("run", {}).get("id") == "retry"
+ if r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]
]
assert runs[-1]["outcome"] == "failed"
@@ -735,7 +742,7 @@ async def test_nested_commands_can_run_a_cell_queued_by_the_outer_command(
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "nested-queue"
+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
@@ -751,7 +758,7 @@ async def test_nested_commands_can_run_a_cell_queued_by_the_outer_command(
runs = [
r["data"]["run"]
for r in events.records
- if r.get("data", {}).get("run", {}).get("id") == "nested-queue"
+ if r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]
]
assert runs[-1]["outcome"] == "succeeded"
@@ -791,14 +798,14 @@ async def test_awaited_commands_join_the_enclosing_run(
await wait_for(
events,
lambda response: any(
- r.get("data", {}).get("run", {}).get("id") == "awaited"
+ r.get("data", {}).get("run", {}).get("id") == accepted.json()["id"]
and r["data"]["run"]["status"] == "completed"
for r in response.records
),
)
records = events.records[before:]
runs = [r["data"]["run"] for r in records if r.get("event") == "run"]
- assert {r["id"] for r in runs} == {"awaited"}
+ assert {r["id"] for r in runs} == {accepted.json()["id"]}
assert runs[-1]["outcome"] == "failed"
executions = [
r["data"]["execution"]
@@ -817,7 +824,7 @@ async def test_awaited_commands_join_the_enclosing_run(
assert (
child["code_hash"] == hashlib.sha256(child_code.encode()).hexdigest()
)
- assert all(e["run_id"] == "awaited" for e in executions)
+ assert all(e["run_id"] == accepted.json()["id"] for e in executions)
@pytest.mark.parametrize(
diff --git a/tests/_server/host/test_host_record.py b/tests/_server/host/test_host_record.py
index 16175f621c..a62f34d6bf 100644
--- a/tests/_server/host/test_host_record.py
+++ b/tests/_server/host/test_host_record.py
@@ -97,7 +97,7 @@ async def test_idle_host_expires_results_and_stops_maintenance(
context = app.state.host_context
host = context.bridge.host
now = 0.0
- host.clock = lambda: now
+ host.clock = context.clock = lambda: now
host.update_project(project())
notebook = host.add_notebook(record(), ScriptedBackend())
notebook.observe(be.Running())
@@ -135,5 +135,6 @@ async def test_idle_host_expires_results_and_stops_maintenance(
]
now = 300
await asyncio.wait_for(removed.wait(), timeout=5)
+ assert context.replies == {}
assert maintenance.done()
assert app.state.host_context is None