Multiplayer agent harness for work.
Run active deployment canaries through authenticated core (#1293)
Regan Bellbc406dafafc8
diff --git a/cli/README.md b/cli/README.mdindex 15df9d960..4308beced 100644--- a/cli/README.md+++ b/cli/README.md@@ -100,6 +100,16 @@ return its recorded boot flag and optional deployment identity. It returnsboot flags or identities. Deployment wrappers can preserve the boot environmentwhile changing durable ownership independently.+Live checks use the active ownership cohort's authenticated canary endpoint to+verify a real model reply, session persistence, generated title, error records,+session cleanup, and database catalog health. The CLI proves the exact healthy+task cohort and ownership generation before and after the check. It requires a+final success bound to the request and responding task; heartbeats alone do not+count. An uncertain result never triggers an automatic replay or fallback.+Legacy deployments retain the Fargate canary. A controlled deployment uses that+same path only when a successful ownership read proves bootstrap is disabled or+another cohort owns background work.+Once bootstrapped, background mode changes use generation-checked ownershiprequests without restarting ECS tasks. Activation waits for prior owners to stopclaiming and every expected task to finish activation. Pausing stops new claims;diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex c9d78eb2d..9723c2174 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -73,6 +73,7 @@ import {streamLabeled,} from "../util.ts";import { doctorCommon } from "./doctor.ts";+import { checkControlledLiveSession } from "../live-session.ts";import {assertTerraformScaffoldSupportsConfig,awsObjectStoreBucket,@@ -102,13 +103,34 @@ export async function awsCoreRequest(config: QmConfig,url: URL,init: RequestInit,+ maxResponseBytes?: number,): Promise<{ status: number; body: string }> {const target = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");if (awsPublicOrigin(config).protocol === "http:") {assertCloudFrontLayerTarget(config, url, target);const response = await fetch(url, init);- return { status: response.status, body: await response.text() };+ if (maxResponseBytes === undefined) return { status: response.status, body: await response.text() };+ const chunks: Uint8Array[] = [];+ let size = 0;+ const reader = response.body?.getReader();+ if (reader) {+ try {+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ size += value.byteLength;+ if (size > maxResponseBytes) throw new CliError("AWS core response exceeded its size limit");+ chunks.push(value);+ }+ } catch (error) {+ await reader.cancel().catch(() => {});+ throw error;+ } finally {+ reader.releaseLock();+ }+ }+ return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };}return new Promise((resolve, reject) => {const request = https.request(@@ -123,7 +145,17 @@ export async function awsCoreRequest(},(response) => {const chunks: Buffer[] = [];- response.on("data", (chunk: Buffer) => chunks.push(chunk));+ let size = 0;+ response.on("data", (chunk: Buffer) => {+ size += chunk.length;+ if (maxResponseBytes !== undefined && size > maxResponseBytes) {+ const error = new CliError("AWS core response exceeded its size limit");+ reject(error);+ response.destroy(error);+ return;+ }+ chunks.push(chunk);+ });response.on("error", reject);response.on("end", () =>resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }),@@ -2839,7 +2871,12 @@ export function taskDefinitionForBackgroundWork(};}-function awsBackgroundWorkTransport(config: QmConfig): BackgroundWorkTransport {+function awsBackgroundWorkTransport(+ config: QmConfig,+ path = "/v1/background-work",+ timeoutMs = 30_000,+ maxResponseBytes?: number,+): BackgroundWorkTransport {const aws = requireAws(config);const secret = (name: string): string =>awsText(aws, [@@ -2855,24 +2892,30 @@ function awsBackgroundWorkTransport(config: QmConfig): BackgroundWorkTransport {if (isInvalidSecret("CORE_SIGNING_SECRET", signing) || control.trim().length < 32 || control === signing)throw new CliError("deployment control requires a distinct secret of at least 32 characters");const url = new URL(config.publicUrl);- url.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/background-work`;+ url.pathname = `${url.pathname.replace(/\/+$/, "")}${path}`;url.search = "";return (method, body = "") => {const timestamp = Math.floor(Date.now() / 1000);const signature = createHmac("sha256", signing).update(`v0:${timestamp}:${method}\n${url.pathname}\n${body}`).digest("hex");- return awsCoreRequest(config, url, {- method,- headers: {- "content-type": "application/json",- "x-timestamp": String(timestamp),- "x-signature": `v0=${signature}`,- authorization: `Bearer ${control}`,+ return awsCoreRequest(+ config,+ url,+ {+ method,+ headers: {+ "content-type": "application/json",+ "x-timestamp": String(timestamp),+ "x-signature": `v0=${signature}`,+ authorization: `Bearer ${control}`,+ },+ ...(method === "POST" ? { body } : {}),+ redirect: "error",+ signal: AbortSignal.timeout(timeoutMs),},- ...(method === "POST" ? { body } : {}),- signal: AbortSignal.timeout(30_000),- });+ maxResponseBytes,+ );};}@@ -4719,7 +4762,17 @@ async function checkLive(}if (!failures.length) {try {- awsLiveSession(config, states.get("core")!);+ const cohort = aws.backgroundWorkControl ? await awsBackgroundWorkStatus(config, configDir) : undefined;+ if (cohort?.status.enabled && cohort.status.desiredDeploymentId === cohort.deploymentId) {+ const transport = awsBackgroundWorkTransport(config, "/v1/deployment/live-session", 600_000, 65_536);+ await checkControlledLiveSession({+ before: cohort,+ read: () => awsBackgroundWorkStatus(config, configDir),+ request: (body) => transport("POST", body),+ });+ } else {+ awsLiveSession(config, states.get("core")!);+ }if (opts.report ?? true) step("core: private live session smoke passed");} catch (error) {failures.push(`core: private live session smoke failed: ${errMessage(error)}`);diff --git a/cli/src/live-session.ts b/cli/src/live-session.tsnew file mode 100644index 000000000..c224e5e6f--- /dev/null+++ b/cli/src/live-session.ts@@ -0,0 +1,115 @@+import { randomUUID } from "node:crypto";+import type { BackgroundWorkStatus } from "./background-work.ts";+import { CliError } from "./log.ts";++export interface LiveSessionCohort {+ deploymentId: string;+ taskArns: string[];+ status: BackgroundWorkStatus;+}++function assertReady(cohort: LiveSessionCohort): void {+ const { deploymentId, taskArns, status } = cohort;+ if (+ !status.enabled ||+ status.deploymentId !== deploymentId ||+ status.desiredDeploymentId !== deploymentId ||+ taskArns.length === 0 ||+ new Set(taskArns).size !== taskArns.length ||+ taskArns.some(+ (taskArn) =>+ !status.members.some(+ (member) =>+ member.taskArn === taskArn &&+ member.deploymentId === deploymentId &&+ member.generation === status.generation &&+ member.state === "admitted" &&+ member.ready &&+ !member.retired,+ ),+ ) ||+ status.members.some(+ (member) =>+ !member.retired &&+ member.state === "admitted" &&+ (member.deploymentId !== deploymentId ||+ member.generation !== status.generation ||+ member.taskArn === null ||+ !taskArns.includes(member.taskArn)),+ )+ )+ throw new CliError("live session requires the exact ready deployment cohort to own background work");+}++export async function checkControlledLiveSession(options: {+ before: LiveSessionCohort;+ read: () => Promise<LiveSessionCohort>;+ request: (body: string) => Promise<{ status: number; body: string }>;+}): Promise<void> {+ const { before } = options;+ assertReady(before);+ const requestId = randomUUID();+ let response: { status: number; body: string };+ try {+ response = await options.request(+ JSON.stringify({+ requestId,+ expectedDeploymentId: before.deploymentId,+ expectedGeneration: before.status.generation,+ expectedTaskArns: before.taskArns,+ }),+ );+ } catch {+ throw new CliError(+ `live session result is unconfirmed for request ${requestId}; no automatic replay or fallback was attempted`,+ );+ }+ if (response.status !== 200) throw new CliError(`live session rejected with HTTP ${response.status}`);+ let result: unknown;+ try {+ result = JSON.parse(response.body);+ } catch {+ throw new CliError(`live session returned no valid final result for request ${requestId}`);+ }+ if (!result || typeof result !== "object" || Array.isArray(result))+ throw new CliError("live session returned an invalid final result");+ const value = result as Record<string, unknown>;+ if (+ value.ok !== true ||+ value.requestId !== requestId ||+ value.deploymentId !== before.deploymentId ||+ value.generation !== before.status.generation ||+ !before.status.members.some(+ (member) =>+ member.instanceId === value.instanceId &&+ member.taskArn === value.taskArn &&+ before.taskArns.includes(member.taskArn ?? "") &&+ member.deploymentId === before.deploymentId &&+ member.generation === before.status.generation &&+ member.state === "admitted" &&+ member.ready &&+ !member.retired,+ )+ )+ throw new CliError(+ `live session did not confirm success for the expected request and deployment cohort (${requestId})`,+ );+ const after = await options.read();+ assertReady(after);+ if (+ after.deploymentId !== before.deploymentId ||+ after.status.generation !== before.status.generation ||+ after.taskArns.length !== before.taskArns.length ||+ after.taskArns.some((arn) => !before.taskArns.includes(arn)) ||+ !after.status.members.some(+ (member) =>+ member.instanceId === value.instanceId &&+ member.taskArn === value.taskArn &&+ member.state === "admitted" &&+ member.ready &&+ !member.retired &&+ member.generation === before.status.generation,+ )+ )+ throw new CliError("live session deployment cohort changed during qualification");+}diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 8ee229e7c..222a8f020 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -14,6 +14,7 @@ import {assertAwsPublicRouting,assertGithubDeployTrust,awsCheckLive,+ awsCoreRequest,awsBackgroundWorkStatus,awsBackgroundWorkBootState,awsBootstrapBackgroundWork,@@ -5060,6 +5061,38 @@ test("AWS layer transport uses the HTTPS front door when an HTTP ALB origin is c.digest("hex")}`,);}+ const fetchBeforeLimit = globalThis.fetch;+ let cancelled = false;+ globalThis.fetch = async () =>+ new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(new TextEncoder().encode(" ".repeat(33)));+ },+ cancel() {+ cancelled = true;+ },+ }),+ );+ try {+ await assert.rejects(+ awsCoreRequest(+ configured,+ new URL(configured.publicUrl),+ {+ method: "POST",+ body: "{}",+ redirect: "error",+ signal: AbortSignal.timeout(1000),+ },+ 32,+ ),+ /size limit/,+ );+ assert.equal(cancelled, true);+ } finally {+ globalThis.fetch = fetchBeforeLimit;+ }for (const invalid of [{ ...distribution, DomainName: "another.cloudfront.net" },{ ...distribution, Enabled: false },@@ -5104,6 +5137,49 @@ test("AWS layer transport uses the HTTPS front door when an HTTP ALB origin is c}});+test("AWS core transport bounds streamed TLS bodies and destroys oversized responses", async (t) => {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-core-body-limit-"));+ const fake = fakeAws(dir, "console.log('')");+ let destroyed = false;+ t.mock.method(https, "request", (_url: URL, _options: https.RequestOptions, callback: (value: unknown) => void) => {+ const request = new EventEmitter() as EventEmitter & { end(): void };+ request.end = () => {+ const response = Object.assign(new EventEmitter(), {+ statusCode: 200,+ destroy(error: Error) {+ destroyed = true;+ response.emit("error", error);+ },+ });+ callback(response);+ response.emit("data", Buffer.from(" ".repeat(16)));+ response.emit("data", Buffer.from(" ".repeat(17)));+ response.emit("end");+ };+ return request;+ });+ try {+ await assert.rejects(+ awsCoreRequest(+ config,+ new URL(config.publicUrl),+ {+ method: "POST",+ body: "{}",+ signal: AbortSignal.timeout(1000),+ },+ 32,+ ),+ /size limit/,+ );+ assert.equal(destroyed, true);+ } finally {+ t.mock.restoreAll();+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});+test("AWS layer transport rejects invalid targets and propagates TLS and body failures without fallback", async (t) => {const dir = mkdtempSync(join(tmpdir(), "qm-aws-layer-failure-"));const fake = fakeAws(dir, "console.log('')");@@ -5558,7 +5634,37 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECS};const fetchLayer = globalThis.fetch;const mutations: unknown[] = [];+ let smokeCalls = 0;+ let smokeFailure = false;globalThis.fetch = async (url, init) => {+ if (String(url).includes("/v1/deployment/live-session")) {+ smokeCalls++;+ const body = String(init?.body ?? "");+ const request = JSON.parse(body);+ const headers = new Headers(init?.headers);+ assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");+ assert.equal(+ headers.get("x-signature"),+ `v0=${createHmac("sha256", TEST_SECRET_VALUE)+ .update(`v0:${headers.get("x-timestamp")}:POST\n/v1/deployment/live-session\n${body}`)+ .digest("hex")}`,+ );+ assert.equal(request.expectedDeploymentId, manifest.backgroundDeploymentId);+ assert.equal(request.expectedGeneration, ownership.generation);+ assert.deepEqual(request.expectedTaskArns, [taskArn]);+ assert.ok(init?.signal instanceof AbortSignal);+ if (smokeFailure) throw new Error("disconnected with a sensitive credential");+ return new Response(+ ` \n\n${JSON.stringify({+ ok: true,+ requestId: request.requestId,+ deploymentId: ownership.deploymentId,+ generation: ownership.generation,+ instanceId: "instance-one",+ taskArn,+ })}\n`,+ );+ }if (!String(url).includes("/v1/background-work")) return fetchLayer(url, init);const headers = new Headers(init?.headers);assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");@@ -5591,6 +5697,10 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECSreturn new Response(JSON.stringify(ownership), { status: 200 });};assert.deepEqual((await awsBackgroundWorkStatus(single, dir)).taskArns, [taskArn]);+ writeFileSync(fake.log, "");+ await awsCheckLive(single, { configDir: dir, report: false });+ assert.equal(smokeCalls, 0);+ assert.match(readFileSync(fake.log, "utf8"), /ecs run-task/);await assert.rejects(awsSetBackgroundWork(single, dir, true), /explicitly bootstrap/);const stoppedLegacyArn = "arn:aws:ecs:us-west-2:123456789012:task/stopped-legacy-core";ownership.members.push({@@ -5625,6 +5735,25 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECS);assert.equal(bootstrapped.generation, 1);writeFileSync(fake.log, "");+ await awsCheckLive(single, { configDir: dir, report: false });+ assert.equal(smokeCalls, 1);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs run-task/);+ smokeFailure = true;+ await assert.rejects(awsCheckLive(single, { configDir: dir, report: false }), /unconfirmed/);+ assert.equal(smokeCalls, 2);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs run-task/);+ smokeFailure = false;+ const activeOwnership = structuredClone(ownership);+ ownership.generation++;+ ownership.desiredDeploymentId = "other-owner";+ ownership.members[0]!.state = "relinquished";+ ownership.members[0]!.ready = false;+ writeFileSync(fake.log, "");+ await awsCheckLive(single, { configDir: dir, report: false });+ assert.equal(smokeCalls, 2);+ assert.match(readFileSync(fake.log, "utf8"), /ecs run-task/);+ Object.assign(ownership, activeOwnership);+ writeFileSync(fake.log, "");await assert.rejects(awsUp(single, dir, { yes: true }), /pause or hand over/);assert.doesNotMatch(readFileSync(fake.log, "utf8"),diff --git a/cli/test/live-session.test.ts b/cli/test/live-session.test.tsnew file mode 100644index 000000000..a26159631--- /dev/null+++ b/cli/test/live-session.test.ts@@ -0,0 +1,182 @@+import test from "node:test";+import assert from "node:assert/strict";+import { checkControlledLiveSession, type LiveSessionCohort } from "../src/live-session.ts";++function cohort(): LiveSessionCohort {+ return {+ deploymentId: "core:release",+ taskArns: ["task-a", "task-b"],+ status: {+ protocol: 1,+ enabled: true,+ deploymentId: "core:release",+ instanceId: "instance-a",+ generation: 4,+ desiredDeploymentId: "core:release",+ lastRequestId: "handover",+ members: ["a", "b"].map((suffix) => ({+ instanceId: `instance-${suffix}`,+ taskArn: `task-${suffix}`,+ deploymentId: "core:release",+ generation: 4,+ state: "admitted",+ retired: false,+ ready: true,+ })),+ },+ };+}++function success(body: string): Record<string, unknown> {+ const request = JSON.parse(body);+ assert.equal(request.expectedDeploymentId, "core:release");+ assert.equal(request.expectedGeneration, 4);+ assert.deepEqual(request.expectedTaskArns, ["task-a", "task-b"]);+ return {+ ok: true,+ requestId: request.requestId,+ deploymentId: "core:release",+ generation: 4,+ instanceId: "instance-a",+ taskArn: "task-a",+ };+}++test("live session accepts whitespace heartbeats and checks the exact cohort afterward", async () => {+ let reads = 0;+ await checkControlledLiveSession({+ before: cohort(),+ request: async (body) => ({ status: 200, body: ` \n\t\n${JSON.stringify(success(body))}\n` }),+ read: async () => {+ reads++;+ return cohort();+ },+ });+ assert.equal(reads, 1);+});++test("live session refuses incomplete or competing ownership before requesting a canary", async () => {+ const mutations: Array<(value: LiveSessionCohort) => void> = [+ (value) => {+ value.status.enabled = false;+ },+ (value) => {+ value.status.desiredDeploymentId = "other";+ },+ (value) => {+ value.status.members[0]!.ready = false;+ },+ (value) => {+ value.status.members[0]!.generation--;+ },+ (value) => {+ value.status.members[0]!.retired = true;+ },+ (value) => {+ value.status.members[0]!.state = "relinquished";+ },+ (value) => {+ value.status.members.push({ ...value.status.members[0]!, instanceId: "other", taskArn: "unexpected" });+ },+ ];+ for (const mutate of mutations) {+ const before = cohort();+ mutate(before);+ await assert.rejects(+ checkControlledLiveSession({+ before,+ request: async () => {+ assert.fail("no canary before ownership proof");+ },+ read: async () => cohort(),+ }),+ /exact ready deployment cohort/,+ );+ }+});++test("live session requires an explicit final success bound to request and instance", async () => {+ for (const patch of [+ { ok: false, error: "sensitive server error" },+ { ok: undefined },+ { requestId: "another-request" },+ { deploymentId: "stale" },+ { generation: 3 },+ { instanceId: "old-instance" },+ { taskArn: "old-task" },+ ]) {+ let calls = 0;+ await assert.rejects(+ checkControlledLiveSession({+ before: cohort(),+ request: async (body) => {+ calls++;+ return { status: 200, body: JSON.stringify({ ...success(body), ...patch }) };+ },+ read: async () => cohort(),+ }),+ (error: Error) => {+ assert.match(error.message, /did not confirm success/);+ assert.doesNotMatch(error.message, /sensitive server error/);+ return true;+ },+ );+ assert.equal(calls, 1);+ }+});++test("live session never retries an ambiguous or truncated response", async () => {+ for (const kind of ["disconnect", "heartbeat-only", "truncated", "http-error"]) {+ let calls = 0;+ await assert.rejects(+ checkControlledLiveSession({+ before: cohort(),+ request: async () => {+ calls++;+ if (kind === "disconnect") throw new Error("Bearer secret must never escape");+ return {+ status: kind === "http-error" ? 503 : 200,+ body: kind === "heartbeat-only" ? " \n\n" : '{"ok":true',+ };+ },+ read: async () => cohort(),+ }),+ (error: Error) => {+ assert.doesNotMatch(error.message, /Bearer secret/);+ return true;+ },+ );+ assert.equal(calls, 1);+ }+});++test("live session rejects ownership or task replacement during a successful model check", async () => {+ const mutations: Array<(value: LiveSessionCohort) => void> = [+ (value) => {+ value.status.generation++;+ value.status.members.forEach((member) => member.generation++);+ },+ (value) => {+ value.taskArns[1] = "replacement";+ value.status.members[1]!.taskArn = "replacement";+ },+ (value) => {+ value.status.members[0]!.instanceId = "replacement-instance";+ },+ (value) => {+ value.status.desiredDeploymentId = null;+ },+ ];+ for (const mutate of mutations) {+ const after = cohort();+ mutate(after);+ await assert.rejects(+ checkControlledLiveSession({+ before: cohort(),+ request: async (body) => ({ status: 200, body: JSON.stringify(success(body)) }),+ read: async () => after,+ }),+ /cohort/,+ );+ }+});