Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Retry interrupted session reads without replaying unsafe requests (#1429)

Regan Bellacc57dd5ea9b
diff --git a/plugins/chassis/src/core-client.ts b/plugins/chassis/src/core-client.ts
index 05a265ee0..3bf411f68 100644
--- a/plugins/chassis/src/core-client.ts
+++ b/plugins/chassis/src/core-client.ts
@@ -22,3 +22,60 @@ export function withSourceAuthNonce(pathWithQuery: string, secret: string | unde
url.searchParams.set("_sourceAuthNonce", `${Date.now()}-${Math.random().toString(16).slice(2)}`);
return `${url.pathname}${url.search}`;
}
+
+export async function fetchCoreText(input: {
+ origin: string;
+ secret?: string;
+ method: HttpMethod;
+ path: string;
+ body?: string;
+ headers?: Record<string, string>;
+ signal?: AbortSignal;
+ retrySafeRead?: boolean;
+}): Promise<{ status: number; text: string }> {
+ if (input.retrySafeRead && (input.method !== "GET" || input.body)) {
+ throw new Error("Core read retries require a bodyless GET");
+ }
+ const signal = input.signal ?? (input.retrySafeRead ? AbortSignal.timeout(30_000) : undefined);
+ for (let attempt = 0; ; attempt++) {
+ signal?.throwIfAborted();
+ const path = withSourceAuthNonce(input.path, input.secret);
+ let phase = "headers";
+ let requestId: string | null = null;
+ let status: number | undefined;
+ try {
+ const response = await fetch(`${input.origin}${path}`, {
+ method: input.method,
+ headers: { ...input.headers, ...signedHeaders(input.secret, input.method, path, input.body) },
+ ...(input.body ? { body: input.body } : {}),
+ signal,
+ redirect: "manual",
+ });
+ status = response.status;
+ phase = "body";
+ requestId = response.headers.get("x-request-id");
+ return { status: response.status, text: await response.text() };
+ } catch (error) {
+ const cause = error instanceof Error ? error.cause : undefined;
+ const code = cause && typeof cause === "object" && "code" in cause ? cause.code : undefined;
+ const retry =
+ input.retrySafeRead === true &&
+ attempt === 0 &&
+ !signal?.aborted &&
+ (status === undefined || (status >= 200 && status < 300)) &&
+ typeof code === "string" &&
+ ["UND_ERR_SOCKET", "ECONNRESET", "ECONNREFUSED", "EPIPE"].includes(code);
+ if (input.retrySafeRead)
+ console.warn("core_read_interrupted", {
+ status,
+ phase,
+ attempt: attempt + 1,
+ code: typeof code === "string" ? code : "unknown",
+ requestId: requestId?.slice(0, 128),
+ retry,
+ aborted: signal?.aborted,
+ });
+ if (!retry) throw error;
+ }
+ }
+}
diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.ts
index 96ae13596..9fb679df1 100644
--- a/plugins/web-ui/server/index.ts
+++ b/plugins/web-ui/server/index.ts
@@ -13,6 +13,7 @@ import { dirname, extname, join, normalize } from "node:path";
import { randomBytes } from "node:crypto";
import { LRUCache } from "lru-cache";
import {
+ fetchCoreText,
signedHeaders,
withSourceAuthNonce,
CAPABILITY_HEADER,
@@ -594,19 +595,16 @@ async function coreFetch(
rawBody = "",
timeoutMs?: number,
): Promise<{ status: number; text: string }> {
- const signedPath = withSourceAuthNonce(pathWithQuery, CORE_SIGNING_SECRET);
const portalTok = portalTokenStore.getStore();
- const r = await fetch(`${CORE}${signedPath}`, {
+ return fetchCoreText({
+ origin: CORE,
+ secret: CORE_SIGNING_SECRET,
method,
- headers: {
- ...signedHeaders(CORE_SIGNING_SECRET, method, signedPath, rawBody),
- ...(portalTok ? { [PORTAL_IDENTITY_HEADER]: portalTok } : {}),
- },
- ...(rawBody ? { body: rawBody } : {}),
- ...(timeoutMs ? { signal: AbortSignal.timeout(timeoutMs) } : {}),
- redirect: "manual",
+ path: pathWithQuery,
+ body: rawBody,
+ headers: portalTok ? { [PORTAL_IDENTITY_HEADER]: portalTok } : undefined,
+ signal: timeoutMs ? AbortSignal.timeout(timeoutMs) : undefined,
});
- return { status: r.status, text: await r.text() };
}
async function coreFetchCap(
@@ -1881,7 +1879,28 @@ const apiRoutes: readonly WebRoute[] = [
const v = url.searchParams.get(p);
if (v !== null) qs.set(p, v);
}
- return relayCore(res, "GET", `/v1/sessions/${encodeURIComponent(id)}?${qs.toString()}`);
+ const cancel = new AbortController();
+ const onClose = () => cancel.abort();
+ res.once("close", onClose);
+ try {
+ const portalTok = portalTokenStore.getStore();
+ return relay(
+ res,
+ await fetchCoreText({
+ origin: CORE,
+ secret: CORE_SIGNING_SECRET,
+ method: "GET",
+ path: `/v1/sessions/${encodeURIComponent(id)}?${qs.toString()}`,
+ headers: portalTok ? { [PORTAL_IDENTITY_HEADER]: portalTok } : undefined,
+ signal: AbortSignal.any([cancel.signal, AbortSignal.timeout(30_000)]),
+ retrySafeRead: true,
+ }),
+ );
+ } catch (error) {
+ if (!cancel.signal.aborted) throw error;
+ } finally {
+ res.off("close", onClose);
+ }
},
},
{
diff --git a/plugins/web-ui/test/session-read-retry.test.ts b/plugins/web-ui/test/session-read-retry.test.ts
new file mode 100644
index 000000000..c332afd0c
--- /dev/null
+++ b/plugins/web-ui/test/session-read-retry.test.ts
@@ -0,0 +1,86 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer, type IncomingHttpHeaders } from "node:http";
+import { once } from "node:events";
+import type { AddressInfo } from "node:net";
+import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";
+import { canonicalPayload, signRequest } from "../../chassis/src/source-auth-sign.ts";
+
+const secret = "session-read-test";
+let bodyStarted: (() => void) | undefined;
+let bodyClosed: (() => void) | undefined;
+let cancelAttempts = 0;
+const requests: { url: string; headers: IncomingHttpHeaders }[] = [];
+const core = createServer((req, res) => {
+ if (req.url?.startsWith("/v1/sessions/cancel?")) {
+ cancelAttempts++;
+ res.writeHead(200, { "content-length": "100" });
+ res.write("partial");
+ res.once("close", () => bodyClosed?.());
+ bodyStarted?.();
+ return;
+ }
+ if (!req.url?.startsWith("/v1/sessions/test?")) return void res.end("{}");
+ requests.push({ url: req.url, headers: req.headers });
+ if (requests.length === 1) {
+ res.writeHead(200, { "content-length": "100" });
+ res.write("partial");
+ setTimeout(() => res.destroy(), 10);
+ } else res.end(JSON.stringify({ session: { id: "test" }, entries: [] }));
+}).listen(0, "127.0.0.1");
+await once(core, "listening");
+process.env.CORE_API_URL = `http://127.0.0.1:${(core.address() as AddressInfo).port}`;
+process.env.CORE_SIGNING_SECRET = secret;
+process.env.WEB_UI_PRINCIPALS = "alice";
+const { handler } = await import("../server/index.ts");
+const web = createServer((req, res) => void handler(req, res)).listen(0, "127.0.0.1");
+await once(web, "listening");
+test.after(() => {
+ web.closeAllConnections();
+ web.close();
+ core.closeAllConnections();
+ core.close();
+});
+
+test("session route retries interrupted body with the same viewer and fresh valid signatures", async () => {
+ const token = mintPortalIdentity({ p: "alice", exp: Date.now() + 60_000 }, secret);
+ const response = await fetch(
+ `http://127.0.0.1:${(web.address() as AddressInfo).port}/api/sessions/test?sinceSeq=825`,
+ { headers: { [PORTAL_IDENTITY_HEADER]: token } },
+ );
+ assert.equal(response.status, 200);
+ assert.deepEqual(await response.json(), { session: { id: "test" }, entries: [] });
+ assert.equal(requests.length, 2);
+ assert.notEqual(requests[0]!.url, requests[1]!.url);
+ for (const request of requests) {
+ const url = new URL(request.url, "http://core");
+ assert.equal(url.searchParams.get("viewer"), "alice");
+ assert.equal(url.searchParams.get("sinceSeq"), "825");
+ assert.equal(request.headers[PORTAL_IDENTITY_HEADER], token);
+ assert.equal(
+ request.headers["x-signature"],
+ signRequest(secret, Number(request.headers["x-timestamp"]), canonicalPayload("GET", request.url, "")),
+ );
+ }
+});
+
+test("closing the browser request cancels core body collection without retry", async () => {
+ const started = new Promise<void>((resolve) => {
+ bodyStarted = resolve;
+ });
+ const closed = new Promise<void>((resolve) => {
+ bodyClosed = resolve;
+ });
+ const controller = new AbortController();
+ const token = mintPortalIdentity({ p: "alice", exp: Date.now() + 60_000 }, secret);
+ const pending = fetch(`http://127.0.0.1:${(web.address() as AddressInfo).port}/api/sessions/cancel`, {
+ headers: { [PORTAL_IDENTITY_HEADER]: token },
+ signal: controller.signal,
+ });
+ const rejected = assert.rejects(pending, /abort/i);
+ await started;
+ controller.abort();
+ await rejected;
+ await closed;
+ assert.equal(cancelAttempts, 1);
+});
diff --git a/test/core-client-read.test.ts b/test/core-client-read.test.ts
new file mode 100644
index 000000000..3790b29cf
--- /dev/null
+++ b/test/core-client-read.test.ts
@@ -0,0 +1,132 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { createServer } from "node:http";
+import { once } from "node:events";
+import { fetchCoreText } from "../plugins/chassis/src/core-client.ts";
+
+for (const retrySafeRead of [true, false]) {
+ test(`body interruption retries only opted-in reads: ${retrySafeRead}`, async (t) => {
+ const paths: string[] = [];
+ const server = createServer((req, res) => {
+ paths.push(req.url!);
+ if (paths.length === 1) {
+ res.writeHead(200, { "content-type": "application/json", "content-length": "100" });
+ res.write("partial");
+ setTimeout(() => res.destroy(), 10);
+ } else res.end("complete");
+ }).listen(0, "127.0.0.1");
+ await once(server, "listening");
+ t.after(() => {
+ server.closeAllConnections();
+ server.close();
+ });
+ const address = server.address();
+ assert.ok(address && typeof address !== "string");
+ const result = fetchCoreText({
+ origin: `http://127.0.0.1:${address.port}`,
+ secret: "test-secret",
+ method: "GET",
+ path: "/v1/sessions/test?sinceSeq=825",
+ retrySafeRead,
+ });
+ if (retrySafeRead) {
+ assert.deepEqual(await result, { status: 200, text: "complete" });
+ assert.equal(paths.length, 2);
+ assert.notEqual(
+ new URL(paths[0]!, "http://core").searchParams.get("_sourceAuthNonce"),
+ new URL(paths[1]!, "http://core").searchParams.get("_sourceAuthNonce"),
+ );
+ } else {
+ await assert.rejects(result);
+ assert.equal(paths.length, 1);
+ }
+ });
+}
+
+test("safe read budget stops after two transport failures", async (t) => {
+ let attempts = 0;
+ t.mock.method(globalThis, "fetch", async () => {
+ attempts++;
+ throw new TypeError("fetch failed", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) });
+ });
+ await assert.rejects(fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true }));
+ assert.equal(attempts, 2);
+});
+
+test("cancellation prevents replay", async (t) => {
+ const cancel = new AbortController();
+ let attempts = 0;
+ t.mock.method(globalThis, "fetch", async () => {
+ attempts++;
+ cancel.abort();
+ throw new TypeError("fetch failed", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) });
+ });
+ await assert.rejects(
+ fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true, signal: cancel.signal }),
+ );
+ assert.equal(attempts, 1);
+});
+
+test("application errors do not replay and writes cannot opt in", async (t) => {
+ let attempts = 0;
+ t.mock.method(globalThis, "fetch", async () => {
+ attempts++;
+ return new Response("denied", { status: 403 });
+ });
+ assert.equal(
+ (await fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true })).status,
+ 403,
+ );
+ await assert.rejects(
+ fetchCoreText({ origin: "http://core", method: "POST", path: "/test", retrySafeRead: true }),
+ /bodyless GET/,
+ );
+ assert.equal(attempts, 1);
+});
+
+test("an interrupted error response does not replay", async (t) => {
+ let attempts = 0;
+ t.mock.method(globalThis, "fetch", async () => {
+ attempts++;
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.error(
+ new TypeError("terminated", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) }),
+ );
+ },
+ }),
+ { status: 403 },
+ );
+ });
+ await assert.rejects(fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true }));
+ assert.equal(attempts, 1);
+});
+
+test("the original deadline aborts body collection on the second attempt", async (t) => {
+ let attempts = 0;
+ const signal = AbortSignal.timeout(50);
+ t.mock.method(globalThis, "fetch", async (_url: unknown, options: RequestInit) => {
+ attempts++;
+ assert.equal(options.signal, signal);
+ if (attempts === 1)
+ throw new TypeError("fetch failed", { cause: Object.assign(new Error("reset"), { code: "ECONNRESET" }) });
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ signal.addEventListener("abort", () => controller.error(signal.reason), { once: true });
+ },
+ }),
+ );
+ });
+ const keepAlive = setTimeout(() => {}, 1_000);
+ try {
+ await assert.rejects(
+ fetchCoreText({ origin: "http://core", method: "GET", path: "/test", retrySafeRead: true, signal }),
+ /timeout/i,
+ );
+ assert.equal(attempts, 2);
+ } finally {
+ clearTimeout(keepAlive);
+ }
+});
diff --git a/test/web-ui-sse.test.ts b/test/web-ui-sse.test.ts
index 065bdf3d8..f07a738a7 100644
--- a/test/web-ui-sse.test.ts
+++ b/test/web-ui-sse.test.ts
@@ -121,7 +121,14 @@ test("SSE relays tool activity frames and folds them into the done frame", async
assert.equal(typeof d.finishedAt, "number", "done frame carries finishedAt");
});
-test("active run lookup returns the latest tracked run for the caller's web thread", async () => {
+test("active run lookup returns the latest tracked run for the caller's web thread", async (t) => {
+ const completion = Promise.withResolvers<void>();
+ const complete = built.runs.complete.bind(built.runs);
+ t.mock.method(built.runs, "complete", async (...args: Parameters<typeof complete>) => {
+ await completion.promise;
+ return complete(...args);
+ });
+ t.after(() => completion.resolve());
const threadRef = `web:alice:${randomUUID()}`;
const submit = (await (
await fetch(
@@ -138,10 +145,7 @@ test("active run lookup returns the latest tracked run for the caller's web thre
assert.equal(activeRes.status, 200);
const active = (await activeRes.json()) as { runId?: string | null; run?: { status?: string } | null };
assert.equal(active.runId, submit.runId);
- assert.ok(
- ["pending", "running", "done", "failed"].includes(active.run?.status ?? ""),
- "active lookup returns a run snapshot",
- );
+ assert.ok(["pending", "running"].includes(active.run?.status ?? ""), "active lookup returns a run snapshot");
const otherUser = await fetch(
`${webBase}/api/runs/active?threadRef=${encodeURIComponent(threadRef)}`,