Multiplayer agent harness for work.
fix: make failed web sends retryable (#693)
Joshua France5bbff464d9b5
diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.tsindex 4c15b1c6f..3bc374989 100644--- a/plugins/web-ui/server/index.ts+++ b/plugins/web-ui/server/index.ts@@ -1715,10 +1715,16 @@ const apiRoutes: readonly WebRoute[] = [const attachments: CoreAttachment[] = [];let approval: { requestId: string; approved: boolean; scope?: string } | undefined;let proactiveOpener = false;+ let clientTurnId: string | undefined;try {const p = JSON.parse(await readBody(req));text = String(p.text ?? "");if (p.proactiveOpener === true) proactiveOpener = true;+ if (+ typeof p.clientTurnId === "string" &&+ /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(p.clientTurnId)+ )+ clientTurnId = p.clientTurnId;if (p.approval && typeof p.approval.requestId === "string" && typeof p.approval.approved === "boolean") {approval = {requestId: p.approval.requestId,@@ -1786,6 +1792,7 @@ const apiRoutes: readonly WebRoute[] = [...(attachments.length ? { attachments } : {}),...(approval ? { approval } : {}),...(proactiveOpener ? { proactiveOpener: true } : {}),+ ...(clientTurnId ? { idempotencyKey: `web:${user}:${clientTurnId}` } : {}),};return postTurnAndMint(res, turn, user, threadRef);},diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.tsindex 5e5289c9b..1b1d5efce 100644--- a/plugins/web-ui/src/chat.ts+++ b/plugins/web-ui/src/chat.ts@@ -1200,6 +1200,20 @@ export function createChatSurface(`;}+ async function retryFailedSend(message: AgentMessage, index: number): Promise<void> {+ const agent = chatState.agent;+ if (!agent || agent.state.isStreaming || agent.state.messages[index] !== message) return;+ if (!(message as AssistantWork).retryableSend) return;+ agent.state.messages = agent.state.messages.filter((_, current) => current !== index);+ ctx.composer.state.error = "";+ drawActiveChat(agent);+ try {+ await agent.continue();+ } catch (err) {+ if (agent === chatState.agent) ctx.composer.state.error = errMessage(err, "Could not retry the message.");+ }+ }+function visibleMessages(agent: Agent): AgentMessage[] {const out = [...agent.state.messages];if (agent.state.streamingMessage) out.push(agent.state.streamingMessage);@@ -1279,12 +1293,22 @@ export function createChatSurface(Boolean(deliveredFiles?.length) ||msg.content.some((chunk) => chunk.type === "thinking" && chunk.thinking.trim());if (!hasVisibleContent && msg.stopReason !== "error" && msg.stopReason !== "aborted") return nothing;+ let errorTpl: TemplateResult | typeof nothing = nothing;+ if (msg.stopReason === "error" && msg.errorMessage) {+ errorTpl = (msg as AssistantWork).retryableSend+ ? html`<div class="send-failure">+ <span>${msg.errorMessage}</span>+ <button class="btn compact" type="button" @click=${() => void retryFailedSend(message, index)}>+ ${icon(RefreshCw, 12)} Retry+ </button>+ </div>`+ : html`<div class="composer-error inline">${msg.errorMessage}</div>`;+ }return html`<article class="message-row assistant-row ${isStreaming ? "streaming" : ""}" data-index=${index}><div class="assistant-body">${showWork ? workBlock(work, isStreaming) : nothing} ${assistantContent(msg, isStreaming, showWork)}- ${assistantFileList(deliveredFiles)}- ${msg.stopReason === "error" && msg.errorMessage ? html`<div class="composer-error inline">${msg.errorMessage}</div>` : nothing}+ ${assistantFileList(deliveredFiles)} ${errorTpl}${msg.stopReason === "aborted" ? html`<div class="stopped-note">${icon(Ban, 13)}<span>Stopped</span></div>` : nothing}${isStreaming ? nothing : messageMeta(msg, index)}</div>diff --git a/plugins/web-ui/src/core-bridge.ts b/plugins/web-ui/src/core-bridge.tsindex adedab9d6..68effdcfc 100644--- a/plugins/web-ui/src/core-bridge.ts+++ b/plugins/web-ui/src/core-bridge.ts@@ -320,7 +320,11 @@ export interface ApprovalDecision {approved: boolean;scope?: "once" | "session" | "always";}-export type AssistantWork = AssistantMessage & { work?: WorkBlock; deliveredFiles?: DeliveredFile[] };+export type AssistantWork = AssistantMessage & {+ work?: WorkBlock;+ deliveredFiles?: DeliveredFile[];+ retryableSend?: boolean;+};export interface RunPoll {status: "pending" | "running" | "done" | "failed";@@ -388,7 +392,9 @@ function baseAssistant(model: Model<Api>): AssistantMessage {};}-async function latestUserTurn(agent: Agent): Promise<{ text: string; attachments: CoreAttachment[] }> {+async function latestUserTurn(+ agent: Agent,+): Promise<{ text: string; attachments: CoreAttachment[]; clientTurnId: string }> {const messages = agent.state.messages as Array<AgentMessage & { attachments?: PiAttachment[] }>;for (let i = messages.length - 1; i >= 0; i--) {const m = messages[i];@@ -400,12 +406,14 @@ async function latestUserTurn(agent: Agent): Promise<{ text: string; attachments.filter((c) => c.type === "text").map((c) => c.text ?? "").join("\n");+ const identified = m as AgentMessage & { attachments?: PiAttachment[]; clientTurnId?: string };+ identified.clientTurnId ??= crypto.randomUUID();const attachments = await Promise.all((m.attachments ?? []).filter((a) => typeof a.content === "string" && a.content.length > 0).map(toCoreAttachment),);- return { text, attachments };+ return { text, attachments, clientTurnId: identified.clientTurnId };}- return { text: "", attachments: [] };+ return { text: "", attachments: [], clientTurnId: crypto.randomUUID() };}function attachmentBytes(a: PiAttachment): Uint8Array {@@ -723,14 +731,15 @@ async function drive(stream.push({ type: "start", partial });stream.push({ type: "text_start", contentIndex: 0, partial });- const { text, attachments } = opener- ? { text: "", attachments: [] as CoreAttachment[] }+ const { text, attachments, clientTurnId } = opener+ ? { text: "", attachments: [] as CoreAttachment[], clientTurnId: undefined }: await latestUserTurn(agent);const submit = await api<{ status?: string; runId?: string; reply?: string }>("/api/turn", {method: "POST",body: JSON.stringify({...turnRequestBody(threadRef, text, model, agent, getTurnOptions, attachments),+ ...(clientTurnId ? { clientTurnId } : {}),...(approval ? { approval } : {}),...(opener ? { proactiveOpener: true } : {}),}),@@ -749,6 +758,10 @@ async function drive(work.status = "failed";work.finishedAt = Date.now();notify();+ if (e instanceof TypeError) {+ fail(stream, partial, "Message wasn’t sent. Check your connection and try again.", true);+ return;+ }fail(stream, partial, e instanceof Error ? e.message : String(e));}}@@ -1095,7 +1108,12 @@ function streamRunViaSse(});}-function fail(stream: AssistantMessageEventStream, partial: AssistantMessage, errorMessage: string): void {+function fail(+ stream: AssistantMessageEventStream,+ partial: AssistantMessage,+ errorMessage: string,+ retryableSend = false,+): void {const block = partial.content[0];const soFar = block?.type === "text" ? block.text : "";const error: AssistantMessage = {@@ -1103,6 +1121,7 @@ function fail(stream: AssistantMessageEventStream, partial: AssistantMessage, ercontent: [{ type: "text", text: soFar }],stopReason: "error",errorMessage,+ ...(retryableSend ? { retryableSend: true } : {}),};stream.push({ type: "error", reason: "error", error });stream.end(error);diff --git a/plugins/web-ui/src/shell.css b/plugins/web-ui/src/shell.cssindex 16b1b53f0..6ecee76c5 100644--- a/plugins/web-ui/src/shell.css+++ b/plugins/web-ui/src/shell.css@@ -2835,6 +2835,18 @@ a.chat-row-open {.composer-error.inline {margin: 4px 0 0;}+.send-failure {+ display: flex;+ align-items: center;+ flex-wrap: wrap;+ gap: 8px;+ margin-top: 6px;+ color: var(--destructive, #b42318);+ font-size: 12px;+}+.send-failure .btn {+ color: var(--foreground);+}.stopped-note {display: inline-flex;align-items: center;diff --git a/plugins/web-ui/test/core-bridge-retry.test.ts b/plugins/web-ui/test/core-bridge-retry.test.tsindex 67c90d1ad..2f51b4f26 100644--- a/plugins/web-ui/test/core-bridge-retry.test.ts+++ b/plugins/web-ui/test/core-bridge-retry.test.ts@@ -175,3 +175,29 @@ test("poll requests never put bearer credentials in the URL", async () => {assert.ok(urls[0]!.endsWith("/api/runs/run-r5"), urls[0]);assert.doesNotMatch(urls[0]!, /[?&](?:rt|token)=/);});++test("a failed initial submission is retryable with the same client turn id", async () => {+ const { makeCoreStreamFn } = await import("../src/core-bridge.ts");+ const submitted: Record<string, unknown>[] = [];+ globalThis.fetch = (async (_input, init) => {+ submitted.push(JSON.parse(String(init?.body)) as Record<string, unknown>);+ if (submitted.length === 1) throw new TypeError("Failed to fetch");+ return Response.json({ status: "ok", reply: "delivered" });+ }) as typeof fetch;+ const agent = {+ state: { model: MODEL, messages: [{ role: "user", content: "hello" }], tools: [] },+ } as unknown as import("@earendil-works/pi-agent-core").Agent;+ const streamFn = makeCoreStreamFn("web:alice:one", agent);++ const failedStream = await streamFn(MODEL, {} as import("@earendil-works/pi-ai").Context);+ const failed = await failedStream.result();+ assert.equal(failed.stopReason, "error");+ assert.equal(failed.errorMessage, "Message wasn’t sent. Check your connection and try again.");+ assert.equal((failed as { retryableSend?: boolean }).retryableSend, true);++ const retriedStream = await streamFn(MODEL, {} as import("@earendil-works/pi-ai").Context);+ const retried = await retriedStream.result();+ assert.equal(retried.stopReason, "stop");+ assert.equal(submitted[0]?.clientTurnId, submitted[1]?.clientTurnId);+ assert.match(String(submitted[0]?.clientTurnId), /^[0-9a-f-]{36}$/);+});diff --git a/plugins/web-ui/test/send-retry-source.test.ts b/plugins/web-ui/test/send-retry-source.test.tsnew file mode 100644index 000000000..3b0192902--- /dev/null+++ b/plugins/web-ui/test/send-retry-source.test.ts@@ -0,0 +1,21 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";++const source = readFileSync(new URL("../src/chat.ts", import.meta.url), "utf8");++test("retryable send errors render an inline retry action", () => {+ assert.match(source, /\(message as AssistantWork\)\.retryableSend/);+ assert.match(source, /retryFailedSend\(message, index\)/);+ assert.match(source, /Retry\s*<\/button>/);+});++test("retry removes only the failed assistant row and continues the original user turn", () => {+ const retry = source.slice(+ source.indexOf("async function retryFailedSend"),+ source.indexOf("function visibleMessages"),+ );+ assert.match(retry, /agent\.state\.messages\.filter/);+ assert.match(retry, /await agent\.continue\(\)/);+ assert.doesNotMatch(retry, /agent\.prompt/);+});diff --git a/plugins/web-ui/test/turn-idempotency-route.test.ts b/plugins/web-ui/test/turn-idempotency-route.test.tsnew file mode 100644index 000000000..ff3752daa--- /dev/null+++ b/plugins/web-ui/test/turn-idempotency-route.test.ts@@ -0,0 +1,60 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createServer, type IncomingMessage } from "node:http";+import type { AddressInfo } from "node:net";+import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";++const turns: Record<string, unknown>[] = [];+const core = createServer((req: IncomingMessage, res) => {+ let raw = "";+ req.on("data", (chunk) => (raw += chunk));+ req.on("end", () => {+ if (req.method === "POST") turns.push(JSON.parse(raw) as Record<string, unknown>);+ res.writeHead(202, { "content-type": "application/json" });+ res.end(JSON.stringify({ status: "queued", runId: "run-1" }));+ });+});+await new Promise<void>((resolve) => core.listen(0, resolve));++process.env.CORE_API_URL = `http://localhost:${(core.address() as AddressInfo).port}`;+process.env.CORE_SIGNING_SECRET = "turn-idempotency-test";+process.env.WEB_UI_PRINCIPALS = "alice";++const { handler } = await import("../server/index.ts");+const surface = createServer((req, res) => void handler(req, res));+await new Promise<void>((resolve) => surface.listen(0, resolve));+const base = `http://localhost:${(surface.address() as AddressInfo).port}`;+const headers = {+ [PORTAL_IDENTITY_HEADER]: mintPortalIdentity({ p: "alice", exp: Date.now() + 60_000 }, "turn-idempotency-test"),+ "content-type": "application/json",+};++test.after(() => {+ surface.close();+ core.close();+});++test("web retries forward one user-scoped idempotency key", async () => {+ const clientTurnId = "123e4567-e89b-42d3-a456-426614174000";+ for (let i = 0; i < 2; i++) {+ const response = await fetch(`${base}/api/turn`, {+ method: "POST",+ headers,+ body: JSON.stringify({ text: "hello", threadRef: "web:alice:one", clientTurnId }),+ });+ assert.equal(response.status, 202);+ }+ assert.equal(turns.length, 2);+ assert.equal(turns[0]?.idempotencyKey, `web:alice:${clientTurnId}`);+ assert.equal(turns[1]?.idempotencyKey, turns[0]?.idempotencyKey);+});++test("malformed client turn ids are not forwarded", async () => {+ const response = await fetch(`${base}/api/turn`, {+ method: "POST",+ headers,+ body: JSON.stringify({ text: "hello", threadRef: "web:alice:two", clientTurnId: "shared-key" }),+ });+ assert.equal(response.status, 202);+ assert.equal(turns.at(-1)?.idempotencyKey, undefined);+});