Multiplayer agent harness for work.
Harden admin authorization readiness (#564)
Joshua France7733a1cb38b9
diff --git a/plugins/admin/src/index.ts b/plugins/admin/src/index.tsindex d0415cd81..fb0386175 100644--- a/plugins/admin/src/index.ts+++ b/plugins/admin/src/index.ts@@ -7,6 +7,7 @@ import { signedRequestHeaders, withSourceAuthNonce } from "../../chassis/src/corimport { json, readBody, cookie } from "../../chassis/src/http.ts";import { createBrandingCache, injectBranding, type OrgBranding } from "../../chassis/src/branding.ts";import { verifyPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";+import { errMessage } from "../../chassis/src/errors.ts";import {CORE_API_URL as CORE,CORE_ORG_ID as ORG,@@ -20,6 +21,9 @@ import { dirname, join } from "node:path";const PORT = portFromEnv(8090);const ADMIN_BASE_PATH = (process.env.ADMIN_BASE_PATH ?? "").replace(/\/$/, "");+const CORE_WHOAMI_ATTEMPTS = 2;+const CORE_WHOAMI_TIMEOUT_MS = 2_500;+const CORE_WHOAMI_RETRY_DELAY_MS = 250;function signedHeaders(method: string, corePath: string, rawBody: string): Record<string, string> {return signedRequestHeaders(CORE_SIGNING_SECRET, method, corePath, rawBody, { "content-type": "application/json" });}@@ -243,19 +247,37 @@ async function uploadFileFromRequest(async function coreWhoami(principal: string): Promise<{ isAdmin: boolean; role?: string; scopeId?: string } | null> {const corePath = "/v1/admin/whoami";- try {- const r = await fetch(`${CORE}${corePath}`, {- headers: {- ...signedHeaders("GET", corePath, ""),- ...portalIdentityHeader(),- "x-admin-actor": `${principal}@${ORG}`,- },- });- if (!r.ok) return null;- return (await r.json()) as { isAdmin: boolean; role?: string; scopeId?: string };- } catch {- return null;+ const started = Date.now();+ let failure = "unknown failure";+ let attempts = 0;+ for (; attempts < CORE_WHOAMI_ATTEMPTS; attempts++) {+ try {+ const r = await fetch(`${CORE}${corePath}`, {+ headers: {+ ...signedHeaders("GET", corePath, ""),+ ...portalIdentityHeader(),+ "x-admin-actor": `${principal}@${ORG}`,+ },+ signal: AbortSignal.timeout(CORE_WHOAMI_TIMEOUT_MS),+ });+ if (r.ok) {+ const body = (await r.json()) as { isAdmin?: unknown; role?: string; scopeId?: string };+ if (typeof body.isAdmin !== "boolean") throw new Error("core returned an invalid admin status");+ return { ...body, isAdmin: body.isAdmin };+ }+ failure = `HTTP ${r.status}`;+ if (r.status < 500 && r.status !== 429) break;+ } catch (error) {+ failure = errMessage(error);+ }+ if (attempts + 1 < CORE_WHOAMI_ATTEMPTS) {+ await new Promise((resolve) => setTimeout(resolve, CORE_WHOAMI_RETRY_DELAY_MS));+ }}+ console.warn(+ `[admin] core whoami failed after ${Math.min(attempts + 1, CORE_WHOAMI_ATTEMPTS)} attempt(s) in ${Date.now() - started}ms: ${failure}`,+ );+ return null;}const WRITES = new Map<string, string[]>([diff --git a/plugins/admin/test/whoami.test.ts b/plugins/admin/test/whoami.test.tsindex 27233e453..db7762a25 100644--- a/plugins/admin/test/whoami.test.ts+++ b/plugins/admin/test/whoami.test.ts@@ -7,8 +7,16 @@ import { mintPortalIdentity } from "../../chassis/src/portal-identity.ts";let lastActor: string | null = null;let lastSigned = false;let lastPortalIdentity: string | null = null;+let transientWhoamiFailures = 0;+let whoamiRequests = 0;const core = createServer((req: IncomingMessage, res) => {if (req.method === "GET" && (req.url ?? "").startsWith("/v1/admin/whoami")) {+ whoamiRequests++;+ if (transientWhoamiFailures > 0) {+ transientWhoamiFailures--;+ res.writeHead(502, { "content-type": "application/json" });+ return void res.end(JSON.stringify({ error: "temporarily_unavailable" }));+ }lastActor = (req.headers["x-admin-actor"] as string) ?? null;lastSigned = Boolean(req.headers["x-timestamp"] && req.headers["x-signature"]);lastPortalIdentity = (req.headers["x-portal-identity"] as string) ?? null;@@ -134,6 +142,15 @@ test("an unsigned or wrongly-signed portal identity is not accepted as an admin}});+test("a transient core failure is retried before admin bootstrap fails", async () => {+ transientWhoamiFailures = 1;+ whoamiRequests = 0;+ const r = await api("/api/me", "admin=U-admin");+ assert.equal(r.status, 200);+ assert.equal(whoamiRequests, 2);+ assert.equal(((await r.json()) as { isAdmin?: boolean }).isAdmin, true);+});+test("core unreachable → /api/whoami returns 502 core_unreachable (outage, not a not-admin verdict)", async () => {await new Promise<void>((r) => core.close(() => r()));const r = await api("/api/whoami", "admin=U-admin");diff --git a/plugins/portal/src/index.ts b/plugins/portal/src/index.tsindex a0abdee9d..fcb7ff941 100644--- a/plugins/portal/src/index.ts+++ b/plugins/portal/src/index.ts@@ -201,8 +201,9 @@ export function consumeState(state: string): boolean {}const ADMIN_TTL_MS = 60_000;-const ADMIN_PROBE_TIMEOUT_MS = 1500;+const ADMIN_PROBE_TIMEOUT_MS = 6_500;const ADMIN_PROBE_ATTEMPTS = 2;+const ADMIN_PROBE_RETRY_DELAY_MS = 250;const adminCache = new LRUCache<string, boolean>({ max: 10_000, ttl: ADMIN_TTL_MS });async function adminProbeAttempt(sub: string): Promise<boolean | null> {@@ -217,10 +218,18 @@ async function adminProbeAttempt(sub: string): Promise<boolean | null> {);}const r = await fetch(`${UPSTREAMS.admin}/api/whoami`, { headers, signal: ctrl.signal });- if (!r.ok) return null;- const j = (await r.json()) as { isAdmin?: boolean };- return j.isAdmin === true;- } catch {+ if (!r.ok) {+ console.warn(`[portal] admin probe returned HTTP ${r.status}`);+ return null;+ }+ const j = (await r.json()) as { isAdmin?: unknown };+ if (typeof j.isAdmin !== "boolean") {+ console.warn("[portal] admin probe returned an invalid admin status");+ return null;+ }+ return j.isAdmin;+ } catch (error) {+ console.warn(`[portal] admin probe failed: ${errMessage(error)}`);return null;} finally {clearTimeout(timer);@@ -232,7 +241,11 @@ async function adminProbe(sub: string): Promise<{ isAdmin: boolean; failed: boolif (hit !== undefined) return { isAdmin: hit, failed: false };for (let attempt = 0; attempt < ADMIN_PROBE_ATTEMPTS; attempt++) {const isAdmin = await adminProbeAttempt(sub);- if (isAdmin === null) continue;+ if (isAdmin === null) {+ if (attempt + 1 < ADMIN_PROBE_ATTEMPTS)+ await new Promise((resolve) => setTimeout(resolve, ADMIN_PROBE_RETRY_DELAY_MS));+ continue;+ }adminCache.set(sub, isAdmin);return { isAdmin, failed: false };}diff --git a/plugins/portal/test/proxy-errors.test.ts b/plugins/portal/test/proxy-errors.test.tsindex 255b8d744..413e71390 100644--- a/plugins/portal/test/proxy-errors.test.ts+++ b/plugins/portal/test/proxy-errors.test.ts@@ -5,7 +5,7 @@ import { connect } from "node:net";import type { AddressInfo } from "node:net";import { verifyPortalIdentity } from "../../chassis/src/portal-identity.ts";-let whoamiMode: "ok" | "down" | "fail-once" = "ok";+let whoamiMode: "ok" | "down" | "fail-once" | "malformed" = "ok";let whoamiRequests = 0;const upstream = createServer((req: IncomingMessage, res) => {@@ -15,6 +15,10 @@ const upstream = createServer((req: IncomingMessage, res) => {res.writeHead(502, { "content-type": "application/json" });return void res.end(JSON.stringify({ error: "core_unreachable" }));}+ if (whoamiMode === "malformed") {+ res.writeHead(200, { "content-type": "application/json" });+ return void res.end("{}");+ }const m = (req.headers.cookie ?? "").match(/admin=([^;]+)/);const sub = m ? decodeURIComponent(m[1] ?? "") : "";res.writeHead(200, { "content-type": "application/json" });@@ -152,6 +156,16 @@ test("an admin-probe outage is reported as unavailable and is NOT negative-cacheassert.equal(ok.status, 200);});+test("a malformed admin verdict fails readiness instead of being cached as non-admin", async () => {+ whoamiMode = "malformed";+ const denied = await fetch(`${base}/admin/`, {+ headers: { cookie: sessionCookie("U-admin-malformed"), accept: "text/html" },+ });+ assert.equal(denied.status, 403);+ assert.match(await denied.text(), /temporarily unavailable/i);+ whoamiMode = "ok";+});+test("consumeState: single-use, TTL-bounded, never wholesale-wiped", () => {assert.equal(consumeState("state-a"), true);assert.equal(consumeState("state-a"), false, "a consumed state cannot be replayed");