Python
Persist error/interrupt markers in history; add Retry on failed turns
Rohit C Prasad878b858ece77
diff --git a/coworker/engine.py b/coworker/engine.pyindex 3c14b179..bbfd75ce 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -177,6 +177,31 @@ class TurnEngine:async for event in self._loop():yield event+ def _append_notice(self, kind: str, text: Optional[str] = None) -> None:+ """Persist a turn-ending marker (error/interrupted) as a display-only `notice`+ message: it survives reload like the transcript does, but `_outbound_messages`+ drops the role so no provider ever sees it."""+ notice: dict[str, Any] = {"role": "notice", "kind": kind, "ts": time.time()}+ if text:+ notice["text"] = text+ self.messages.append(notice)++ async def retry(self) -> AsyncIterator[Event]:+ """Re-run the model loop after a provider error — no new user message; the failed+ turn's input is already the tail of history. Guarded on the tail being an error+ notice so a stray retry frame can't re-answer a completed turn."""+ last = self.messages[-1] if self.messages else None+ if not (+ isinstance(last, dict)+ and last.get("role") == "notice"+ and last.get("kind") == "error"+ ):+ return+ self._cancel.clear()+ yield Event(EventType.TURN_START, {"input": ""})+ async for event in self._loop():+ yield event+async def resume(self) -> AsyncIterator[Event]:"""Continue a turn that was suspended at a prompt and persisted — durable resume after arestart (or engine eviction). Re-process the trailing assistant message's UNANSWERED@@ -257,6 +282,7 @@ class TurnEngine:}if friendly:payload["raw"] = str(exc)+ self._append_notice("error", friendly or str(exc))yield Event(EventType.ERROR, payload)returnif self._cancel.is_set() and turn is None:@@ -267,6 +293,7 @@ class TurnEngine:self.messages.append(_assistant_message(AssistantTurn(text="".join(streamed))))+ self._append_notice("interrupted")yield Event(EventType.INTERRUPTED, {"iterations": iterations})returnif turn is None:@@ -294,6 +321,7 @@ class TurnEngine:yield Event(EventType.ITERATION_END, {"iteration": iterations})if self._cancel.is_set():+ self._append_notice("interrupted")yield Event(EventType.INTERRUPTED, {"iterations": iterations})returnif self._steering:@@ -802,7 +830,8 @@ class TurnEngine:"""# Strip the display-only sidecars — `source` (connector cards), `_display`# (e.g. filter-hidden counts), and `ts` (append-time timestamps) — copying only- # messages that carry one.+ # messages that carry one. Whole `notice` messages (error/interrupted markers)+ # are display-only too: dropped entirely._SIDECARS = ("source", "_display", "ts")out = [(@@ -811,6 +840,7 @@ class TurnEngine:else msg)for msg in self.messages+ if msg.get("role") != "notice"]# PDF attachments (stored as `file` parts) are adapted to the ACTIVE model right# here — never in the persisted history — so a mid-session model switch alwaysdiff --git a/coworker/server/app.py b/coworker/server/app.pyindex 5c50bcb3..3908ccc3 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -1576,12 +1576,13 @@ def create_app(manager: SessionManager) -> FastAPI:"iteration_end",}- async def run_turn(content) -> None:+ async def run_turn(content, *, retry: bool = False) -> None:manager.mark_running(session_id) # busy → self-wakes steer instead of collidingtry:- async for event in engine.run(content):+ events = engine.retry() if retry else engine.run(content)+ async for event in events:# Broadcast to every socket viewing this session (this socket included — it's a# registered client), so a second view of the same session stays in sync too.await manager.broadcast_session(@@ -1629,6 +1630,11 @@ def create_app(manager: SessionManager) -> FastAPI:_resolve_pending(str(message.get("answer", "")))elif kind == "interrupt":engine.request_interrupt()+ elif kind == "retry":+ # Re-run after a provider error (engine guards on the error-notice+ # tail, so a stray frame is a no-op that still ends with turn_done).+ if not manager.is_running(session_id):+ asyncio.create_task(run_turn(None, retry=True))elif kind == "set_mode":try:engine.permissions.mode = Mode(message.get("mode"))diff --git a/surfaces/gui/e2e/error-retry.spec.ts b/surfaces/gui/e2e/error-retry.spec.tsnew file mode 100644index 00000000..12fd3395--- /dev/null+++ b/surfaces/gui/e2e/error-retry.spec.ts@@ -0,0 +1,27 @@+// Model-layer roadmap item 1 (2026-07-22): a turn that dies on a provider error leaves a+// visible, persistent marker with a Retry affordance. Retry re-runs the failed turn with NO+// new user bubble; once the turn recovers, the button disappears (the notice is history).+import { expect } from "@playwright/test";+import { test } from "./fixtures";++test("provider error shows a retriable notice; Retry re-runs without a new user message", async ({+ page,+}) => {+ await page.goto("/");+ await page.getByText("Draft the launch note").first().click();+ const box = page.getByPlaceholder(/Ask the coworker/);+ await box.fill("please fail the turn");+ await box.press("Enter");++ await expect(page.getByText("Error: model unreachable").first()).toBeVisible({ timeout: 10_000 });+ const retry = page.getByTestId("notice-retry");+ await expect(retry).toBeVisible();++ await retry.click();+ await expect(page.getByText("Recovered after retry.").first()).toBeVisible({ timeout: 10_000 });++ // No fake user bubble from the retry turn, exactly one real one…+ await expect(page.locator(".bubble-user")).toHaveCount(1);+ // …and the button is gone now that the error notice is no longer the transcript tail.+ await expect(page.getByTestId("notice-retry")).toHaveCount(0);+});diff --git a/surfaces/gui/e2e/fixtures.ts b/surfaces/gui/e2e/fixtures.tsindex 3bf66025..5d11d97e 100644--- a/surfaces/gui/e2e/fixtures.ts+++ b/surfaces/gui/e2e/fixtures.ts@@ -646,6 +646,12 @@ export async function mockApi(page: import("@playwright/test").Page) {});return;}+ // A turn that dies on a provider error; the follow-up {type:"retry"} recovers.+ if (/fail the turn/i.test(msg.text)) {+ send("error", { error: "model unreachable" });+ send("turn_done");+ return;+ }// A deliberately SLOW multi-second stream (~40 ticks × 120ms) so specs can// interact mid-turn — the follow/pin scroll contract (FB-004) is untestable// against the instant echo below.@@ -697,6 +703,11 @@ export async function mockApi(page: import("@playwright/test").Page) {}send("interrupted", {});send("turn_done");+ } else if (msg.type === "retry") {+ // Like the real engine: re-runs with NO new user message (turn_start input is empty).+ send("turn_start", { input: "" });+ send("assistant_message", { text: "Recovered after retry." });+ send("turn_done");}});});diff --git a/surfaces/gui/src/App.tsx b/surfaces/gui/src/App.tsxindex 64223c89..62ca9fff 100644--- a/surfaces/gui/src/App.tsx+++ b/surfaces/gui/src/App.tsx@@ -644,7 +644,10 @@ export function App() {break;case "error":flushPartialStream();- setItems((p) => [...p, { kind: "notice", tone: "warn", text: "Error: " + (d.error || "unknown") }]);+ setItems((p) => [+ ...p,+ { kind: "notice", tone: "warn", text: "Error: " + (d.error || "unknown"), retriable: true },+ ]);break;case "turn_done":setRunning(false);@@ -799,6 +802,11 @@ export function App() {const prefillComposer = (text: string, attachments?: Attachment[]) =>setComposerPrefill((p) => ({ text, attachments, nonce: (p?.nonce ?? 0) + 1 }));const interrupt = () => sessionRef.current?.interrupt();+ const retry = () => {+ // Optimistic running: turn_start confirms; a rejected retry still ends in turn_done.+ setRunning(true);+ sessionRef.current?.retry();+ };const changeMode = (m: string) => {setMode(m);sessionRef.current?.setMode(m);@@ -1400,6 +1408,7 @@ export function App() {items={items}onApprove={approve}running={running}+ onRetry={retry}// §33 ref #3: sub-threshold streamed text renders INSIDE the live turn// group (header when collapsed, quiet line when expanded) — never as a// floating paragraph.diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.tsindex 10592365..b1589719 100644--- a/surfaces/gui/src/api.ts+++ b/surfaces/gui/src/api.ts@@ -1746,6 +1746,12 @@ export class Session {this.send({ type: "interrupt" });}+ // Re-run a turn that ended in a provider error — no new user message; the server+ // guards on the history tail so a stray frame is a no-op.+ retry() {+ this.send({ type: "retry" });+ }+setMode(mode: string) {this.send({ type: "set_mode", mode });}diff --git a/surfaces/gui/src/components/Transcript.tsx b/surfaces/gui/src/components/Transcript.tsxindex dcb1342b..3551d661 100644--- a/surfaces/gui/src/components/Transcript.tsx+++ b/surfaces/gui/src/components/Transcript.tsx@@ -280,9 +280,12 @@ interface Props {running?: boolean;// Sub-threshold streamed text (streamGate mode "quiet") — handed to the live turn group.streamingText?: string;+ // Re-run the failed turn (no new user message). Offered only on a retriable notice that+ // is the transcript tail of an idle session — anywhere else the error is history.+ onRetry?: () => void;}-export function Transcript({ items, running, streamingText }: Props) {+export function Transcript({ items, running, streamingText, onRetry }: Props) {// §33 grouping: a turn = the maximal run of assistant/tool/resolved-approval items between// breakers (user, connector, notices, plan/dir requests…). Trailing assistant texts are the// ANSWER and render as bubbles after the group; interior assistant texts are narration and@@ -396,6 +399,11 @@ export function Transcript({ items, running, streamingText }: Props) {return (<div className={"notice " + (item.tone === "warn" ? "warn" : "")} key={bi}>{item.text}+ {item.retriable && !running && onRetry && block.i === items.length - 1 && (+ <button className="btn ml-2" data-testid="notice-retry" onClick={onRetry}>+ Retry+ </button>+ )}</div>);default:diff --git a/surfaces/gui/src/itemsFromMessages.test.ts b/surfaces/gui/src/itemsFromMessages.test.tsindex 3ae23e86..f8b9cf95 100644--- a/surfaces/gui/src/itemsFromMessages.test.ts+++ b/surfaces/gui/src/itemsFromMessages.test.ts@@ -48,3 +48,23 @@ describe("itemsFromMessages timestamps", () => {]);});});++describe("itemsFromMessages notices", () => {+ it("replays persisted error/interrupted markers; only errors are retriable", () => {+ const items = itemsFromMessages([+ { role: "user", content: "hi" },+ { role: "assistant", content: "partial ans" },+ { role: "notice", kind: "interrupted", ts: 1752969720 },+ { role: "user", content: "again" },+ { role: "notice", kind: "error", text: "model down", ts: 1752969724 },+ ] as any);++ expect(items).toEqual([+ { kind: "user", text: "hi" },+ { kind: "assistant", text: "partial ans" },+ { kind: "notice", tone: "warn", text: "Interrupted." },+ { kind: "user", text: "again" },+ { kind: "notice", tone: "warn", text: "Error: model down", retriable: true },+ ]);+ });+});diff --git a/surfaces/gui/src/itemsFromMessages.ts b/surfaces/gui/src/itemsFromMessages.tsindex e7c67f8b..1ac8b53c 100644--- a/surfaces/gui/src/itemsFromMessages.ts+++ b/surfaces/gui/src/itemsFromMessages.ts@@ -58,6 +58,15 @@ export function itemsFromMessages(messages: ConversationMessage[]): Item[] {...(hidden ? { hidden } : {}),});}+ } else if (m.role === "notice") {+ // Persisted turn-ending marker (engine `_append_notice`): error/interrupted survive+ // reload exactly like the live view rendered them. An error notice is retriable —+ // the Transcript only offers the button when it's the transcript tail.+ items.push(+ m.kind === "interrupted"+ ? { kind: "notice", tone: "warn", text: "Interrupted." }+ : { kind: "notice", tone: "warn", text: "Error: " + (m.text || "unknown"), retriable: true },+ );}// system messages are omitted; tool-result messages are folded into the tool row above}diff --git a/surfaces/gui/src/types.ts b/surfaces/gui/src/types.tsindex b135bed7..b835eeee 100644--- a/surfaces/gui/src/types.ts+++ b/surfaces/gui/src/types.ts@@ -114,4 +114,4 @@ export type Item =multi?: boolean;resolved?: string;}- | { kind: "notice"; tone: "info" | "warn"; text: string };+ | { kind: "notice"; tone: "info" | "warn"; text: string; retriable?: boolean };diff --git a/tests/test_engine_stop.py b/tests/test_engine_stop.pyindex 790fc08b..38da2d68 100644--- a/tests/test_engine_stop.py+++ b/tests/test_engine_stop.py@@ -86,10 +86,18 @@ def test_stop_mid_stream_keeps_partial_text(tmp_path):assert events[-1].type == EventType.INTERRUPTED# Far fewer than the full 200 chunks were consumed…assert provider.chunks_produced < 100- # …and the partial text the user watched is persisted, with no tool calls.- last = engine.messages[-1]- assert last["role"] == "assistant" and last["content"].startswith("w0 ")- assert "tool_calls" not in last+ # …and the partial text the user watched is persisted, with no tool calls,+ # capped by the interrupted marker (display-only notice role).+ assert engine.messages[-1] == {+ "role": "notice",+ "kind": "interrupted",+ "ts": engine.messages[-1]["ts"],+ }+ partial = engine.messages[-2]+ assert partial["role"] == "assistant" and partial["content"].startswith("w0 ")+ assert "tool_calls" not in partial+ # The notice never reaches a provider.+ assert all(m.get("role") != "notice" for m in engine._outbound_messages())class FailingStreamProvider(ProviderClient):@@ -120,9 +128,72 @@ def test_provider_error_mid_stream_keeps_partial_text(tmp_path):events = asyncio.run(run())assert events[-1].type == EventType.ERROR- last = engine.messages[-1]- assert last["role"] == "assistant" and last["content"] == "partial answer"- assert "tool_calls" not in last+ notice = engine.messages[-1]+ assert notice["role"] == "notice" and notice["kind"] == "error"+ assert "provider went away" in notice["text"]+ partial = engine.messages[-2]+ assert partial["role"] == "assistant" and partial["content"] == "partial answer"+ assert "tool_calls" not in partial+++class FlakyProvider(ProviderClient):+ """Fails the first N stream calls, then answers — a provider outage that recovers."""++ def __init__(self, failures=1):+ self._failures = failures+ self.calls = 0++ def complete(self, **kwargs): # pragma: no cover+ raise NotImplementedError++ def capabilities(self, model):+ return ModelCapabilities()++ def stream(self, *, model, messages, tools=None, **settings):+ self.calls += 1+ if self.calls <= self._failures:+ raise RuntimeError("outage")+ yield StreamChunk(turn=AssistantTurn(text="recovered", finish_reason="stop"))+++def test_retry_reruns_failed_turn_without_new_user_message(tmp_path):+ provider = FlakyProvider(failures=1)+ engine = TurnEngine(+ provider=provider,+ registry=ToolRegistry(),+ permissions=PermissionEngine(workspace_root=tmp_path),+ model="gpt-5.5",+ )++ async def scenario():+ first = [ev async for ev in engine.run("hello")]+ second = [ev async for ev in engine.retry()]+ return first, second++ first, second = asyncio.run(scenario())+ assert first[-1].type == EventType.ERROR+ assert second[-1].type == EventType.TURN_END+ # Exactly one user message — retry re-runs, it doesn't re-ask.+ assert sum(1 for m in engine.messages if m.get("role") == "user") == 1+ assert engine.messages[-1]["content"] == "recovered"+++def test_retry_is_noop_unless_tail_is_error_notice(tmp_path):+ engine = TurnEngine(+ provider=OneTurnProvider(AssistantTurn(text="done", finish_reason="stop")),+ registry=ToolRegistry(),+ permissions=PermissionEngine(workspace_root=tmp_path),+ model="gpt-5.5",+ )++ async def scenario():+ async for _ in engine.run("hello"):+ pass+ return [ev async for ev in engine.retry()]++ # A completed session must not grow a second answer from a stray retry frame.+ assert asyncio.run(scenario()) == []+ assert engine.messages[-1]["content"] == "done"def test_stop_while_awaiting_approval(tmp_path):diff --git a/tests/test_server.py b/tests/test_server.pyindex b6eded07..40614c64 100644--- a/tests/test_server.py+++ b/tests/test_server.py@@ -323,6 +323,38 @@ def test_ws_simple_turn(tmp_path):assert "turn_end" in types+def test_ws_error_persists_notice_and_retry_reruns(tmp_path):+ class FlakyProvider(ProviderClient):+ def __init__(self):+ self.calls = 0++ def complete(self, *, model, messages, tools=None, **settings):+ self.calls += 1+ if self.calls == 1:+ raise RuntimeError("outage")+ return _text("recovered")++ def capabilities(self, model):+ return ModelCapabilities()++ manager = SessionManager(workspace=tmp_path, provider=FlakyProvider())+ client = TestClient(create_app(manager))+ with client.websocket_connect("/ws/session/flaky") as ws:+ assert ws.receive_json()["type"] == "ready"+ ws.send_json({"type": "user_message", "text": "hello"})+ assert "error" in _drain(ws)+ # The error survives as a persisted notice (reload shows what happened)…+ messages = client.get("/v1/sessions/flaky/messages").json()["messages"]+ assert messages[-1]["role"] == "notice" and messages[-1]["kind"] == "error"+ # …and retry re-runs the turn without a new user message.+ ws.send_json({"type": "retry"})+ types = _drain(ws)+ assert "turn_start" in types and "assistant_message" in types+ messages = client.get("/v1/sessions/flaky/messages").json()["messages"]+ assert messages[-1]["role"] == "assistant" and messages[-1]["content"] == "recovered"+ assert sum(1 for m in messages if m["role"] == "user") == 1++# -- origin gate (local-API hardening): a browser page on a foreign origin must not be able to# read the API cross-origin or open the driving WebSocket. -------------------------------------