Multiplayer agent harness for work.
Retry interrupted session reads without replaying unsafe requests (#1429)
Regan Bellacc57dd5ea9b
diff --git a/plugins/chassis/src/core-client.ts b/plugins/chassis/src/core-client.tsindex 05a265ee0..3bf411f68 100644--- a/plugins/chassis/src/core-client.ts+++ b/plugins/chassis/src/core-client.ts@@ -22,3 +22,60 @@ export function withSourceAuthNonce(pathWithQuery: string, secret: string | undeurl.searchParams.set("_sourceAuthNonce", `${Date.now()}-${Math.random().toString(16).slice(2)}`);return `${url.pathname}${url.search}`;}++export async function fetchCoreText(input: {+ origin: string;+ secret?: string;+ method: HttpMethod;+ path: string;+ body?: string;+ headers?: Record<string, string>;+ signal?: AbortSignal;+ retrySafeRead?: boolean;+}): Promise<{ status: number; text: string }> {+ if (input.retrySafeRead && (input.method !== "GET" || input.body)) {+ throw new Error("Core read retries require a bodyless GET");+ }+ const signal = input.signal ?? (input.retrySafeRead ? AbortSignal.timeout(30_000) : undefined);+ for (let attempt = 0; ; attempt++) {+ signal?.throwIfAborted();+ const path = withSourceAuthNonce(input.path, input.secret);+ let phase = "headers";+ let requestId: string | null = null;+ let status: number | undefined;+ try {+ const response = await fetch(`${input.origin}${path}`, {+ method: input.method,+ headers: { ...input.headers, ...signedHeaders(input.secret, input.method, path, input.body) },+ ...(input.body ? { body: input.body } : {}),+ signal,+ redirect: "manual",+ });+ status = response.status;+ phase = "body";+ requestId = response.headers.get("x-request-id");+ return { status: response.status, text: await response.text() };+ } catch (error) {+ const cause = error instanceof Error ? error.cause : undefined;+ const code = cause && typeof cause === "object" && "code" in cause ? cause.code : undefined;+ const retry =+ input.retrySafeRead === true &&+ attempt === 0 &&+ !signal?.aborted &&+ (status === undefined || (status >= 200 && status < 300)) &&+ typeof code === "string" &&+ ["UND_ERR_SOCKET", "ECONNRESET", "ECONNREFUSED", "EPIPE"].includes(code);+ if (input.retrySafeRead)+ console.warn("core_read_interrupted", {+ status,+ phase,+ attempt: attempt + 1,+ code: typeof code === "string" ? code : "unknown",+ requestId: requestId?.slice(0, 128),+ retry,+ aborted: signal?.aborted,+ });+ if (!retry) throw error;+ }+ }+}diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.tsindex 96ae13596..9fb679df1 100644--- a/plugins/web-ui/server/index.ts+++ b/plugins/web-ui/server/index.ts@@ -13,6 +13,7 @@ import { dirname, extname, join, normalize } from "node:path";import { randomBytes } from "node:crypto";import { LRUCache } from "lru-cache";import {+ fetchCoreText,signedHeaders,withSourceAuthNonce,CAPABILITY_HEADER,@@ -594,19 +595,16 @@ async function coreFetch(rawBody = "",timeoutMs?: number,): Promise<{ status: number; text: string }> {- const signedPath = withSourceAuthNonce(pathWithQuery, CORE_SIGNING_SECRET);const portalTok = portalTokenStore.getStore();- const r = await fetch(`${CORE}${signedPath}`, {+ return fetchCoreText({+ origin: CORE,+ secret: CORE_SIGNING_SECRET,method,- headers: {- ...signedHeaders(CORE_SIGNING_SECRET, method, signedPath, rawBody),- ...(portalTok ? { [PORTAL_IDENTITY_HEADER]: portalTok } : {}),- },- ...(rawBody ? { body: rawBody } : {}),- ...(timeoutMs ? { signal: AbortSignal.timeout(timeoutMs) } : {}),- redirect: "manual",+ path: pathWithQuery,+ body: rawBody,+ headers: portalTok ? { [PORTAL_IDENTITY_HEADER]: portalTok } : undefined,+ signal: timeoutMs ? AbortSignal.timeout(timeoutMs) : undefined,});- return { status: r.status, text: await r.text() };}async function coreFetchCap(@@ -1881,7 +1879,28 @@ const apiRoutes: readonly WebRoute[] = [const v = url.searchParams.get(p);if (v !== null) qs.set(p, v);}- return relayCore(res, "GET", `/v1/sessions/${encodeURIComponent(id)}?${qs.toString()}`);+ const cancel = new AbortController();+ const onClose = () => cancel.abort();+ res.once("close", onClose);+ try {+ const portalTok = portalTokenStore.getStore();+ return relay(+ res,+ await fetchCoreText({+ origin: CORE,+ secret: CORE_SIGNING_SECRET,+ method: "GET",+ path: `/v1/sessions/${encodeURIComponent(id)}?${qs.toString()}`,+ headers: portalTok ? { [PORTAL_IDENTITY_HEADER]: portalTok } : undefined,+ signal: AbortSignal.any([cancel.signal, AbortSignal.timeout(30_000)]),+ retrySafeRead: true,+ }),+ );+ } catch (error) {+ if (!cancel.signal.aborted) throw error;+ } finally {+ res.off("close", onClose);+ }},},{diff --git a/plugins/web-ui/test/session-read-retry.test.ts b/plugins/web-ui/test/session-read-retry.test.tsnew file mode 100644index 000000000..c332afd0c--- /dev/null+++ b/plugins/web-ui/test/session-read-retry.test.ts@@ -0,0 +1,86 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createServer, type IncomingHttpHeaders } from "node:http";+import { once } from "node:events";+import type { AddressInfo } from "node:net";+import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";+import { canonicalPayload, signRequest } from "../../chassis/src/source-auth-sign.ts";++const secret = "session-read-test";+let bodyStarted: (() => void) | undefined;+let bodyClosed: (() => void) | undefined;+let cancelAttempts = 0;+const requests: { url: string; headers: IncomingHttpHeaders }[] = [];+const core = createServer((req, res) => {+ if (req.url?.startsWith("/v1/sessions/cancel?")) {+ cancelAttempts++;+ res.writeHead(200, { "content-length": "100" });+ res.write("partial");+ res.once("close", () => bodyClosed?.());+ bodyStarted?.();+ return;+ }+ if (!req.url?.startsWith("/v1/sessions/test?")) return void res.end("{}");+ requests.push({ url: req.url, headers: req.headers });+ if (requests.length === 1) {+ res.writeHead(200, { "content-length": "100" });+ res.write("partial");+ setTimeout(() => res.destroy(), 10);+ } else res.end(JSON.stringify({ session: { id: "test" }, entries: [] }));+}).listen(0, "127.0.0.1");+await once(core, "listening");+process.env.CORE_API_URL = `http://127.0.0.1:${(core.address() as AddressInfo).port}`;+process.env.CORE_SIGNING_SECRET = secret;+process.env.WEB_UI_PRINCIPALS = "alice";+const { handler } = await import("../server/index.ts");+const web = createServer((req, res) => void handler(req, res)).listen(0, "127.0.0.1");+await once(web, "listening");+test.after(() => {+ web.closeAllConnections();+ web.close();+ core.closeAllConnections();+ core.close();+});++test("session route retries interrupted body with the same viewer and fresh valid signatures", async () => {+ const token = mintPortalIdentity({ p: "alice", exp: Date.now() + 60_000 }, secret);+ const response = await fetch(+ `http://127.0.0.1:${(web.address() as AddressInfo).port}/api/sessions/test?sinceSeq=825`,+ { headers: { [PORTAL_IDENTITY_HEADER]: token } },+ );+ assert.equal(response.status, 200);+ assert.deepEqual(await response.json(), { session: { id: "test" }, entries: [] });+ assert.equal(requests.length, 2);+ assert.notEqual(requests[0]!.url, requests[1]!.url);+ for (const request of requests) {+ const url = new URL(request.url, "http://core");+ assert.equal(url.searchParams.get("viewer"), "alice");+ assert.equal(url.searchParams.get("sinceSeq"), "825");+ assert.equal(request.headers[PORTAL_IDENTITY_HEADER], token);+ assert.equal(+ request.headers["x-signature"],+ signRequest(secret, Number(request.headers["x-timestamp"]), canonicalPayload("GET", request.url, "")),+ );+ }+});++test("closing the browser request cancels core body collection without retry", async () => {+ const started = new Promise<void>((resolve) => {+ bodyStarted = resolve;+ });+ const closed = new Promise<void>((resolve) => {+ bodyClosed = resolve;+ });+ const controller = new AbortController();+ const token = mintPortalIdentity({ p: "alice", exp: Date.now() + 60_000 }, secret);+ const pending = fetch(`http://127.0.0.1:${(web.address() as AddressInfo).port}/api/sessions/cancel`, {+ headers: { [PORTAL_IDENTITY_HEADER]: token },+ signal: controller.signal,+ });+ const rejected = assert.rejects(pending, /abort/i);+ await started;+ controller.abort();+ await rejected;+ await closed;+ assert.equal(cancelAttempts, 1);+});diff --git a/test/core-client-read.test.ts b/test/core-client-read.test.tsnew file mode 100644index 000000000..3790b29cf--- /dev/null+++ b/test/core-client-read.test.ts@@ -0,0 +1,132 @@+import assert from "node:assert/strict";+import { test } from "node:test";+import { createServer } from "node:http";+import { once } from "node:events";+import { fetchCoreText } from "../plugins/chassis/src/core-client.ts";++for (const retrySafeRead of [true, false]) {+ test(`body interruption retries only opted-in reads: ${retrySafeRead}`, async (t) => {+ const paths: string[] = [];+ const server = createServer((req, res) => {+ paths.push(req.url!);+ if (paths.length === 1) {+ res.writeHead(200, { "content-type": "application/json", "content-length": "100" });+ res.write("partial");+ setTimeout(() => res.destroy(), 10);+ } else res.end("complete");+ }).listen(0, "127.0.0.1");+ await once(server, "listening");+ t.after(() => {+ server.closeAllConnections();+ server.close();+ });+ const address = server.address();+ assert.ok(address && typeof address !== "string");+ const result = fetchCoreText({+ origin: `http://127.0.0.1:${address.port}`,+ secret: "test-secret",+ method: "GET",+ path: "/v1/sessions/test?sinceSeq=825",+ retrySafeRead,+ });+ if (retrySafeRead) {+ assert.deepEqual(await result, { status: 200, text: "complete" });+ assert.equal(paths.length, 2);+ assert.notEqual(+ new URL(paths[0]!, "http://core").searchParams.get("_sourceAuthNonce"),+ new URL(paths[1]!, "http://core").searchParams.get("_sourceAuthNonce"),+ );+ } else {+ await assert.rejects(result);+ assert.equal(paths.length, 1);+ }+ });+}++test("safe read budget stops after two transport failures", async (t) => {+ let attempts = 0;+ t.mock.method(globalThis, "fetch", async () => {+ attempts++;+ throw new TypeError("fetch failed", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) });+ });+ await assert.rejects(fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true }));+ assert.equal(attempts, 2);+});++test("cancellation prevents replay", async (t) => {+ const cancel = new AbortController();+ let attempts = 0;+ t.mock.method(globalThis, "fetch", async () => {+ attempts++;+ cancel.abort();+ throw new TypeError("fetch failed", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) });+ });+ await assert.rejects(+ fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true, signal: cancel.signal }),+ );+ assert.equal(attempts, 1);+});++test("application errors do not replay and writes cannot opt in", async (t) => {+ let attempts = 0;+ t.mock.method(globalThis, "fetch", async () => {+ attempts++;+ return new Response("denied", { status: 403 });+ });+ assert.equal(+ (await fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true })).status,+ 403,+ );+ await assert.rejects(+ fetchCoreText({ origin: "http://core", method: "POST", path: "/test", retrySafeRead: true }),+ /bodyless GET/,+ );+ assert.equal(attempts, 1);+});++test("an interrupted error response does not replay", async (t) => {+ let attempts = 0;+ t.mock.method(globalThis, "fetch", async () => {+ attempts++;+ return new Response(+ new ReadableStream({+ start(controller) {+ controller.error(+ new TypeError("terminated", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) }),+ );+ },+ }),+ { status: 403 },+ );+ });+ await assert.rejects(fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true }));+ assert.equal(attempts, 1);+});++test("the original deadline aborts body collection on the second attempt", async (t) => {+ let attempts = 0;+ const signal = AbortSignal.timeout(50);+ t.mock.method(globalThis, "fetch", async (_url: unknown, options: RequestInit) => {+ attempts++;+ assert.equal(options.signal, signal);+ if (attempts === 1)+ throw new TypeError("fetch failed", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) });+ return new Response(+ new ReadableStream({+ start(controller) {+ signal.addEventListener("abort", () => controller.error(signal.reason), { once: true });+ },+ }),+ );+ });+ const keepAlive = setTimeout(() => {}, 1_000);+ try {+ await assert.rejects(+ fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true, signal }),+ /timeout/i,+ );+ assert.equal(attempts, 2);+ } finally {+ clearTimeout(keepAlive);+ }+});diff --git a/test/web-ui-sse.test.ts b/test/web-ui-sse.test.tsindex 065bdf3d8..f07a738a7 100644--- a/test/web-ui-sse.test.ts+++ b/test/web-ui-sse.test.ts@@ -121,7 +121,14 @@ test("SSE relays tool activity frames and folds them into the done frame", asyncassert.equal(typeof d.finishedAt, "number", "done frame carries finishedAt");});-test("active run lookup returns the latest tracked run for the caller's web thread", async () => {+test("active run lookup returns the latest tracked run for the caller's web thread", async (t) => {+ const completion = Promise.withResolvers<void>();+ const complete = built.runs.complete.bind(built.runs);+ t.mock.method(built.runs, "complete", async (...args: Parameters<typeof complete>) => {+ await completion.promise;+ return complete(...args);+ });+ t.after(() => completion.resolve());const threadRef = `web:alice:${randomUUID()}`;const submit = (await (await fetch(@@ -138,10 +145,7 @@ test("active run lookup returns the latest tracked run for the caller's web threassert.equal(activeRes.status, 200);const active = (await activeRes.json()) as { runId?: string | null; run?: { status?: string } | null };assert.equal(active.runId, submit.runId);- assert.ok(- ["pending", "running", "done", "failed"].includes(active.run?.status ?? ""),- "active lookup returns a run snapshot",- );+ assert.ok(["pending", "running"].includes(active.run?.status ?? ""), "active lookup returns a run snapshot");const otherUser = await fetch(`${webBase}/api/runs/active?threadRef=${encodeURIComponent(threadRef)}`,