Multiplayer agent harness for work.
fix: retry transient admin probes (#531)
Joshua France99275b0fcf41
diff --git a/plugins/portal/src/index.ts b/plugins/portal/src/index.tsindex f361018aa..a0abdee9d 100644--- a/plugins/portal/src/index.ts+++ b/plugins/portal/src/index.ts@@ -202,11 +202,10 @@ export function consumeState(state: string): boolean {const ADMIN_TTL_MS = 60_000;const ADMIN_PROBE_TIMEOUT_MS = 1500;+const ADMIN_PROBE_ATTEMPTS = 2;const adminCache = new LRUCache<string, boolean>({ max: 10_000, ttl: ADMIN_TTL_MS });-async function adminProbe(sub: string): Promise<{ isAdmin: boolean; failed: boolean }> {- const hit = adminCache.get(sub);- if (hit !== undefined) return { isAdmin: hit, failed: false };+async function adminProbeAttempt(sub: string): Promise<boolean | null> {const ctrl = new AbortController();const timer = setTimeout(() => ctrl.abort(), ADMIN_PROBE_TIMEOUT_MS);try {@@ -218,18 +217,28 @@ async function adminProbe(sub: string): Promise<{ isAdmin: boolean; failed: bool);}const r = await fetch(`${UPSTREAMS.admin}/api/whoami`, { headers, signal: ctrl.signal });- if (!r.ok) return { isAdmin: false, failed: true };+ if (!r.ok) return null;const j = (await r.json()) as { isAdmin?: boolean };- const v = j.isAdmin === true;- adminCache.set(sub, v);- return { isAdmin: v, failed: false };+ return j.isAdmin === true;} catch {- return { isAdmin: false, failed: true };+ return null;} finally {clearTimeout(timer);}}+async function adminProbe(sub: string): Promise<{ isAdmin: boolean; failed: boolean }> {+ const hit = adminCache.get(sub);+ if (hit !== undefined) return { isAdmin: hit, failed: false };+ for (let attempt = 0; attempt < ADMIN_PROBE_ATTEMPTS; attempt++) {+ const isAdmin = await adminProbeAttempt(sub);+ if (isAdmin === null) continue;+ adminCache.set(sub, isAdmin);+ return { isAdmin, failed: false };+ }+ return { isAdmin: false, failed: true };+}+async function isAdmin(sub: string): Promise<boolean> {return (await adminProbe(sub)).isAdmin;}diff --git a/plugins/portal/test/proxy-errors.test.ts b/plugins/portal/test/proxy-errors.test.tsindex 9c58887e7..255b8d744 100644--- a/plugins/portal/test/proxy-errors.test.ts+++ b/plugins/portal/test/proxy-errors.test.ts@@ -5,11 +5,13 @@ import { connect } from "node:net";import type { AddressInfo } from "node:net";import { verifyPortalIdentity } from "../../chassis/src/portal-identity.ts";-let whoamiMode: "ok" | "down" = "ok";+let whoamiMode: "ok" | "down" | "fail-once" = "ok";+let whoamiRequests = 0;const upstream = createServer((req: IncomingMessage, res) => {if (req.url === "/api/whoami") {- if (whoamiMode === "down") {+ whoamiRequests++;+ if (whoamiMode === "down" || (whoamiMode === "fail-once" && whoamiRequests === 1)) {res.writeHead(502, { "content-type": "application/json" });return void res.end(JSON.stringify({ error: "core_unreachable" }));}@@ -128,6 +130,15 @@ test("a prototype-chain segment like /constructor/ never matches a keyed surface}});+test("a transient admin-probe failure is retried before denying access", async () => {+ whoamiRequests = 0;+ whoamiMode = "fail-once";+ const ok = await fetch(`${base}/admin/api/me`, { headers: { cookie: sessionCookie("U-admin-transient") } });+ assert.equal(ok.status, 200);+ assert.equal(whoamiRequests, 2);+ whoamiMode = "ok";+});+test("an admin-probe outage is reported as unavailable and is NOT negative-cached", async () => {whoamiMode = "down";const denied = await fetch(`${base}/admin/`, {