Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

fix: make failed web sends retryable (#693)

Joshua France5bbff464d9b5
diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.ts
index 4c15b1c6f..3bc374989 100644
--- a/plugins/web-ui/server/index.ts
+++ b/plugins/web-ui/server/index.ts
@@ -1715,10 +1715,16 @@ const apiRoutes: readonly WebRoute[] = [
const attachments: CoreAttachment[] = [];
let approval: { requestId: string; approved: boolean; scope?: string } | undefined;
let proactiveOpener = false;
+ let clientTurnId: string | undefined;
try {
const p = JSON.parse(await readBody(req));
text = String(p.text ?? "");
if (p.proactiveOpener === true) proactiveOpener = true;
+ if (
+ typeof p.clientTurnId === "string" &&
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(p.clientTurnId)
+ )
+ clientTurnId = p.clientTurnId;
if (p.approval && typeof p.approval.requestId === "string" && typeof p.approval.approved === "boolean") {
approval = {
requestId: p.approval.requestId,
@@ -1786,6 +1792,7 @@ const apiRoutes: readonly WebRoute[] = [
...(attachments.length ? { attachments } : {}),
...(approval ? { approval } : {}),
...(proactiveOpener ? { proactiveOpener: true } : {}),
+ ...(clientTurnId ? { idempotencyKey: `web:${user}:${clientTurnId}` } : {}),
};
return postTurnAndMint(res, turn, user, threadRef);
},
diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.ts
index 5e5289c9b..1b1d5efce 100644
--- a/plugins/web-ui/src/chat.ts
+++ b/plugins/web-ui/src/chat.ts
@@ -1200,6 +1200,20 @@ export function createChatSurface(
`;
}
+ async function retryFailedSend(message: AgentMessage, index: number): Promise<void> {
+ const agent = chatState.agent;
+ if (!agent || agent.state.isStreaming || agent.state.messages[index] !== message) return;
+ if (!(message as AssistantWork).retryableSend) return;
+ agent.state.messages = agent.state.messages.filter((_, current) => current !== index);
+ ctx.composer.state.error = "";
+ drawActiveChat(agent);
+ try {
+ await agent.continue();
+ } catch (err) {
+ if (agent === chatState.agent) ctx.composer.state.error = errMessage(err, "Could not retry the message.");
+ }
+ }
+
function visibleMessages(agent: Agent): AgentMessage[] {
const out = [...agent.state.messages];
if (agent.state.streamingMessage) out.push(agent.state.streamingMessage);
@@ -1279,12 +1293,22 @@ export function createChatSurface(
Boolean(deliveredFiles?.length) ||
msg.content.some((chunk) => chunk.type === "thinking" && chunk.thinking.trim());
if (!hasVisibleContent && msg.stopReason !== "error" && msg.stopReason !== "aborted") return nothing;
+ let errorTpl: TemplateResult | typeof nothing = nothing;
+ if (msg.stopReason === "error" && msg.errorMessage) {
+ errorTpl = (msg as AssistantWork).retryableSend
+ ? html`<div class="send-failure">
+ <span>${msg.errorMessage}</span>
+ <button class="btn compact" type="button" @click=${() => void retryFailedSend(message, index)}>
+ ${icon(RefreshCw, 12)} Retry
+ </button>
+ </div>`
+ : html`<div class="composer-error inline">${msg.errorMessage}</div>`;
+ }
return html`
<article class="message-row assistant-row ${isStreaming ? "streaming" : ""}" data-index=${index}>
<div class="assistant-body">
${showWork ? workBlock(work, isStreaming) : nothing} ${assistantContent(msg, isStreaming, showWork)}
- ${assistantFileList(deliveredFiles)}
- ${msg.stopReason === "error" && msg.errorMessage ? html`<div class="composer-error inline">${msg.errorMessage}</div>` : nothing}
+ ${assistantFileList(deliveredFiles)} ${errorTpl}
${msg.stopReason === "aborted" ? html`<div class="stopped-note">${icon(Ban, 13)}<span>Stopped</span></div>` : nothing}
${isStreaming ? nothing : messageMeta(msg, index)}
</div>
diff --git a/plugins/web-ui/src/core-bridge.ts b/plugins/web-ui/src/core-bridge.ts
index adedab9d6..68effdcfc 100644
--- a/plugins/web-ui/src/core-bridge.ts
+++ b/plugins/web-ui/src/core-bridge.ts
@@ -320,7 +320,11 @@ export interface ApprovalDecision {
approved: boolean;
scope?: "once" | "session" | "always";
}
-export type AssistantWork = AssistantMessage & { work?: WorkBlock; deliveredFiles?: DeliveredFile[] };
+export type AssistantWork = AssistantMessage & {
+ work?: WorkBlock;
+ deliveredFiles?: DeliveredFile[];
+ retryableSend?: boolean;
+};
export interface RunPoll {
status: "pending" | "running" | "done" | "failed";
@@ -388,7 +392,9 @@ function baseAssistant(model: Model<Api>): AssistantMessage {
};
}
-async function latestUserTurn(agent: Agent): Promise<{ text: string; attachments: CoreAttachment[] }> {
+async function latestUserTurn(
+ agent: Agent,
+): Promise<{ text: string; attachments: CoreAttachment[]; clientTurnId: string }> {
const messages = agent.state.messages as Array<AgentMessage & { attachments?: PiAttachment[] }>;
for (let i = messages.length - 1; i >= 0; i--) {
const m = messages[i];
@@ -400,12 +406,14 @@ async function latestUserTurn(agent: Agent): Promise<{ text: string; attachments
.filter((c) => c.type === "text")
.map((c) => c.text ?? "")
.join("\n");
+ const identified = m as AgentMessage & { attachments?: PiAttachment[]; clientTurnId?: string };
+ identified.clientTurnId ??= crypto.randomUUID();
const attachments = await Promise.all(
(m.attachments ?? []).filter((a) => typeof a.content === "string" && a.content.length > 0).map(toCoreAttachment),
);
- return { text, attachments };
+ return { text, attachments, clientTurnId: identified.clientTurnId };
}
- return { text: "", attachments: [] };
+ return { text: "", attachments: [], clientTurnId: crypto.randomUUID() };
}
function attachmentBytes(a: PiAttachment): Uint8Array {
@@ -723,14 +731,15 @@ async function drive(
stream.push({ type: "start", partial });
stream.push({ type: "text_start", contentIndex: 0, partial });
- const { text, attachments } = opener
- ? { text: "", attachments: [] as CoreAttachment[] }
+ const { text, attachments, clientTurnId } = opener
+ ? { text: "", attachments: [] as CoreAttachment[], clientTurnId: undefined }
: await latestUserTurn(agent);
const submit = await api<{ status?: string; runId?: string; reply?: string }>("/api/turn", {
method: "POST",
body: JSON.stringify({
...turnRequestBody(threadRef, text, model, agent, getTurnOptions, attachments),
+ ...(clientTurnId ? { clientTurnId } : {}),
...(approval ? { approval } : {}),
...(opener ? { proactiveOpener: true } : {}),
}),
@@ -749,6 +758,10 @@ async function drive(
work.status = "failed";
work.finishedAt = Date.now();
notify();
+ if (e instanceof TypeError) {
+ fail(stream, partial, "Message wasn’t sent. Check your connection and try again.", true);
+ return;
+ }
fail(stream, partial, e instanceof Error ? e.message : String(e));
}
}
@@ -1095,7 +1108,12 @@ function streamRunViaSse(
});
}
-function fail(stream: AssistantMessageEventStream, partial: AssistantMessage, errorMessage: string): void {
+function fail(
+ stream: AssistantMessageEventStream,
+ partial: AssistantMessage,
+ errorMessage: string,
+ retryableSend = false,
+): void {
const block = partial.content[0];
const soFar = block?.type === "text" ? block.text : "";
const error: AssistantMessage = {
@@ -1103,6 +1121,7 @@ function fail(stream: AssistantMessageEventStream, partial: AssistantMessage, er
content: [{ type: "text", text: soFar }],
stopReason: "error",
errorMessage,
+ ...(retryableSend ? { retryableSend: true } : {}),
};
stream.push({ type: "error", reason: "error", error });
stream.end(error);
diff --git a/plugins/web-ui/src/shell.css b/plugins/web-ui/src/shell.css
index 16b1b53f0..6ecee76c5 100644
--- a/plugins/web-ui/src/shell.css
+++ b/plugins/web-ui/src/shell.css
@@ -2835,6 +2835,18 @@ a.chat-row-open {
.composer-error.inline {
margin: 4px 0 0;
}
+.send-failure {
+ display: flex;
+ align-items: center;
+ flex-wrap: wrap;
+ gap: 8px;
+ margin-top: 6px;
+ color: var(--destructive, #b42318);
+ font-size: 12px;
+}
+.send-failure .btn {
+ color: var(--foreground);
+}
.stopped-note {
display: inline-flex;
align-items: center;
diff --git a/plugins/web-ui/test/core-bridge-retry.test.ts b/plugins/web-ui/test/core-bridge-retry.test.ts
index 67c90d1ad..2f51b4f26 100644
--- a/plugins/web-ui/test/core-bridge-retry.test.ts
+++ b/plugins/web-ui/test/core-bridge-retry.test.ts
@@ -175,3 +175,29 @@ test("poll requests never put bearer credentials in the URL", async () => {
assert.ok(urls[0]!.endsWith("/api/runs/run-r5"), urls[0]);
assert.doesNotMatch(urls[0]!, /[?&](?:rt|token)=/);
});
+
+test("a failed initial submission is retryable with the same client turn id", async () => {
+ const { makeCoreStreamFn } = await import("../src/core-bridge.ts");
+ const submitted: Record<string, unknown>[] = [];
+ globalThis.fetch = (async (_input, init) => {
+ submitted.push(JSON.parse(String(init?.body)) as Record<string, unknown>);
+ if (submitted.length === 1) throw new TypeError("Failed to fetch");
+ return Response.json({ status: "ok", reply: "delivered" });
+ }) as typeof fetch;
+ const agent = {
+ state: { model: MODEL, messages: [{ role: "user", content: "hello" }], tools: [] },
+ } as unknown as import("@earendil-works/pi-agent-core").Agent;
+ const streamFn = makeCoreStreamFn("web:alice:one", agent);
+
+ const failedStream = await streamFn(MODEL, {} as import("@earendil-works/pi-ai").Context);
+ const failed = await failedStream.result();
+ assert.equal(failed.stopReason, "error");
+ assert.equal(failed.errorMessage, "Message wasn’t sent. Check your connection and try again.");
+ assert.equal((failed as { retryableSend?: boolean }).retryableSend, true);
+
+ const retriedStream = await streamFn(MODEL, {} as import("@earendil-works/pi-ai").Context);
+ const retried = await retriedStream.result();
+ assert.equal(retried.stopReason, "stop");
+ assert.equal(submitted[0]?.clientTurnId, submitted[1]?.clientTurnId);
+ assert.match(String(submitted[0]?.clientTurnId), /^[0-9a-f-]{36}$/);
+});
diff --git a/plugins/web-ui/test/send-retry-source.test.ts b/plugins/web-ui/test/send-retry-source.test.ts
new file mode 100644
index 000000000..3b0192902
--- /dev/null
+++ b/plugins/web-ui/test/send-retry-source.test.ts
@@ -0,0 +1,21 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+
+const source = readFileSync(new URL("../src/chat.ts", import.meta.url), "utf8");
+
+test("retryable send errors render an inline retry action", () => {
+ assert.match(source, /\(message as AssistantWork\)\.retryableSend/);
+ assert.match(source, /retryFailedSend\(message, index\)/);
+ assert.match(source, /Retry\s*<\/button>/);
+});
+
+test("retry removes only the failed assistant row and continues the original user turn", () => {
+ const retry = source.slice(
+ source.indexOf("async function retryFailedSend"),
+ source.indexOf("function visibleMessages"),
+ );
+ assert.match(retry, /agent\.state\.messages\.filter/);
+ assert.match(retry, /await agent\.continue\(\)/);
+ assert.doesNotMatch(retry, /agent\.prompt/);
+});
diff --git a/plugins/web-ui/test/turn-idempotency-route.test.ts b/plugins/web-ui/test/turn-idempotency-route.test.ts
new file mode 100644
index 000000000..ff3752daa
--- /dev/null
+++ b/plugins/web-ui/test/turn-idempotency-route.test.ts
@@ -0,0 +1,60 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer, type IncomingMessage } from "node:http";
+import type { AddressInfo } from "node:net";
+import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";
+
+const turns: Record<string, unknown>[] = [];
+const core = createServer((req: IncomingMessage, res) => {
+ let raw = "";
+ req.on("data", (chunk) => (raw += chunk));
+ req.on("end", () => {
+ if (req.method === "POST") turns.push(JSON.parse(raw) as Record<string, unknown>);
+ res.writeHead(202, { "content-type": "application/json" });
+ res.end(JSON.stringify({ status: "queued", runId: "run-1" }));
+ });
+});
+await new Promise<void>((resolve) => core.listen(0, resolve));
+
+process.env.CORE_API_URL = `http://localhost:${(core.address() as AddressInfo).port}`;
+process.env.CORE_SIGNING_SECRET = "turn-idempotency-test";
+process.env.WEB_UI_PRINCIPALS = "alice";
+
+const { handler } = await import("../server/index.ts");
+const surface = createServer((req, res) => void handler(req, res));
+await new Promise<void>((resolve) => surface.listen(0, resolve));
+const base = `http://localhost:${(surface.address() as AddressInfo).port}`;
+const headers = {
+ [PORTAL_IDENTITY_HEADER]: mintPortalIdentity({ p: "alice", exp: Date.now() + 60_000 }, "turn-idempotency-test"),
+ "content-type": "application/json",
+};
+
+test.after(() => {
+ surface.close();
+ core.close();
+});
+
+test("web retries forward one user-scoped idempotency key", async () => {
+ const clientTurnId = "123e4567-e89b-42d3-a456-426614174000";
+ for (let i = 0; i < 2; i++) {
+ const response = await fetch(`${base}/api/turn`, {
+ method: "POST",
+ headers,
+ body: JSON.stringify({ text: "hello", threadRef: "web:alice:one", clientTurnId }),
+ });
+ assert.equal(response.status, 202);
+ }
+ assert.equal(turns.length, 2);
+ assert.equal(turns[0]?.idempotencyKey, `web:alice:${clientTurnId}`);
+ assert.equal(turns[1]?.idempotencyKey, turns[0]?.idempotencyKey);
+});
+
+test("malformed client turn ids are not forwarded", async () => {
+ const response = await fetch(`${base}/api/turn`, {
+ method: "POST",
+ headers,
+ body: JSON.stringify({ text: "hello", threadRef: "web:alice:two", clientTurnId: "shared-key" }),
+ });
+ assert.equal(response.status, 202);
+ assert.equal(turns.at(-1)?.idempotencyKey, undefined);
+});