Multiplayer agent harness for work.
Prove a stopped AWS core cohort is not the background owner through the production route
Regan Bell8576e5cce622
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 10a9bbe51..4d40fa6c6 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -92,11 +92,41 @@ import {import {awaitBackgroundWork,readBackgroundWork,+ readProductionOwner,setBackgroundOwner,type BackgroundWorkStatus,type BackgroundWorkTransport,} from "../background-work.ts";+async function fetchCore(+ url: URL,+ init: RequestInit,+ maxResponseBytes?: number,+): Promise<{ status: number; body: string }> {+ const response = await fetch(url, init);+ if (maxResponseBytes === undefined) return { status: response.status, body: await response.text() };+ const chunks: Uint8Array[] = [];+ let size = 0;+ const reader = response.body?.getReader();+ if (reader) {+ try {+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ size += value.byteLength;+ if (size > maxResponseBytes) throw new CliError("AWS core response exceeded its size limit");+ chunks.push(value);+ }+ } catch (error) {+ await reader.cancel().catch(() => {});+ throw error;+ } finally {+ reader.releaseLock();+ }+ }+ return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };+}+export async function awsCoreRequest(config: QmConfig,url: URL,@@ -107,28 +137,7 @@ export async function awsCoreRequest(if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");if (awsPublicOrigin(config).protocol === "http:") {assertCloudFrontLayerTarget(config, url, target);- const response = await fetch(url, init);- if (maxResponseBytes === undefined) return { status: response.status, body: await response.text() };- const chunks: Uint8Array[] = [];- let size = 0;- const reader = response.body?.getReader();- if (reader) {- try {- for (;;) {- const { done, value } = await reader.read();- if (done) break;- size += value.byteLength;- if (size > maxResponseBytes) throw new CliError("AWS core response exceeded its size limit");- chunks.push(value);- }- } catch (error) {- await reader.cancel().catch(() => {});- throw error;- } finally {- reader.releaseLock();- }- }- return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };+ return fetchCore(url, init, maxResponseBytes);}return new Promise((resolve, reject) => {const request = https.request(@@ -2942,6 +2951,7 @@ function awsBackgroundWorkTransport(path = "/v1/background-work",timeoutMs = 30_000,maxResponseBytes?: number,+ productionRoute = false,): BackgroundWorkTransport {const aws = requireAws(config);const secret = (name: string): string =>@@ -2957,31 +2967,32 @@ function awsBackgroundWorkTransport(const control = secret("DEPLOYMENT_CONTROL_SECRET");if (isInvalidSecret("CORE_SIGNING_SECRET", signing) || control.trim().length < 32 || control === signing)throw new CliError("deployment control requires a distinct secret of at least 32 characters");- const url = new URL(config.publicUrl);+ const url = new URL(productionRoute ? (config.apiUrl ?? config.publicUrl) : config.publicUrl);+ if (productionRoute && (url.protocol !== "https:" || url.username || url.password))+ throw new CliError("the production background ownership route must be HTTPS without credentials");url.pathname = `${url.pathname.replace(/\/+$/, "")}${path}`;url.search = "";+ url.hash = "";return (method, body = "") => {const timestamp = Math.floor(Date.now() / 1000);const signature = createHmac("sha256", signing).update(`v0:${timestamp}:${method}\n${url.pathname}\n${body}`).digest("hex");- return awsCoreRequest(- config,- url,- {- method,- headers: {- "content-type": "application/json",- "x-timestamp": String(timestamp),- "x-signature": `v0=${signature}`,- authorization: `Bearer ${control}`,- },- ...(method === "POST" ? { body } : {}),- redirect: "error",- signal: AbortSignal.timeout(timeoutMs),+ const init: RequestInit = {+ method,+ headers: {+ "content-type": "application/json",+ "x-timestamp": String(timestamp),+ "x-signature": `v0=${signature}`,+ authorization: `Bearer ${control}`,},- maxResponseBytes,- );+ ...(method === "POST" ? { body } : {}),+ redirect: "error",+ signal: AbortSignal.timeout(timeoutMs),+ };+ return productionRoute+ ? fetchCore(url, init, maxResponseBytes ?? 65_536)+ : awsCoreRequest(config, url, init, maxResponseBytes);};}@@ -3277,8 +3288,19 @@ export async function awsBackgroundWorkCapacity(return proof;}+export async function awsStoppedCoreOwner(+ config: QmConfig,+ deploymentId: string,+): Promise<BackgroundWorkStatus | undefined> {+ const state = describedServices(config, ["core"]).get("core");+ if (!state || state.desiredCount !== 0) return undefined;+ stableWorkloadCapacity(config, "core", state, state.taskDefinition ?? "", 0);+ return readProductionOwner(awsBackgroundWorkTransport(config, undefined, undefined, undefined, true), deploymentId);+}+async function assertNotBackgroundOwner(config: QmConfig, current: DeploymentManifest | undefined): Promise<void> {if (!current?.backgroundDeploymentId) return;+ if (await awsStoppedCoreOwner(config, current.backgroundDeploymentId)) return;const status = await readBackgroundWork(awsBackgroundWorkTransport(config), current.backgroundDeploymentId);if (status.ownerDeploymentId === current.backgroundDeploymentId)throw new CliError("hand background ownership to another deployment before replacing the current core tasks");diff --git a/cli/src/background-work.ts b/cli/src/background-work.tsindex f1bc7e6e5..8c1e25338 100644--- a/cli/src/background-work.ts+++ b/cli/src/background-work.ts@@ -93,6 +93,30 @@ export async function readBackgroundWork(return parseBackgroundWorkStatus(response.body, deploymentId);}+export async function readProductionOwner(+ transport: BackgroundWorkTransport,+ stoppedDeploymentId: string,+): Promise<BackgroundWorkStatus> {+ const response = await transport("GET");+ if (response.status !== 200)+ throw new CliError(`production background ownership read failed with HTTP ${response.status}`);+ let value: unknown;+ try {+ value = JSON.parse(response.body);+ } catch {+ throw new CliError("background ownership returned invalid JSON");+ }+ if (!record(value) || value.protocol !== 2)+ throw new CliError("production background ownership must use the single-owner protocol");+ const responder = value.deploymentId;+ if (!text(responder) || responder === stoppedDeploymentId)+ throw new CliError("production background ownership was not reported by another deployment");+ const status = parseBackgroundWorkStatus(response.body, responder);+ if (status.ownerDeploymentId !== responder || !status.active)+ throw new CliError("production does not have an active background owner");+ return status;+}+export async function setBackgroundOwner(transport: BackgroundWorkTransport,deploymentId: string,diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex c91da392d..2cd638f42 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -25,6 +25,7 @@ import {awsRollback,awsSecretsPush,awsSetBackgroundWork,+ awsStoppedCoreOwner,taskDefinitionForBackgroundWork,awsStatus,awsUp,@@ -6124,6 +6125,124 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECS}});+test("a stopped AWS core cohort is replaceable only while the production route reports another active owner", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-stopped-background-"));+ const dockerBin = join(dir, "docker");+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);+ chmodSync(dockerBin, 0o755);+ const base = oneServiceConfig();+ const single: QmConfig = { ...base, aws: { ...base.aws!, backgroundWorkControl: true } };+ const fake = statefulAws(dir, single);+ const priorPath = process.env.PATH;+ process.env.PATH = `${dir}:${priorPath}`;+ try {+ await awsUp(single, dir, { yes: true });+ const deployed = JSON.parse(readFileSync(fake.state, "utf8"));+ const manifest = JSON.parse(+ deployed.dynamo[`deployment/manifest/${deployed.dynamo["deployment/current"].manifestId.S}`].manifest.S,+ );+ const taskDefinition = deployed.services["acme-core"].taskDefinition;+ const stopped = (lastStatus = "STOPPED", runningCount = 0) => ({+ ...deployed,+ serviceOverrides: {+ "acme-core": {+ desiredCount: 0,+ runningCount,+ pendingCount: 0,+ deployments: [+ {+ id: "scaled-down",+ status: "PRIMARY",+ taskDefinition,+ rolloutState: "COMPLETED",+ runningCount: 0,+ pendingCount: 0,+ },+ ],+ },+ },+ taskInventory: {+ "acme-core": [+ {+ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/old-core",+ taskDefinitionArn: taskDefinition,+ desiredStatus: "STOPPED",+ lastStatus,+ },+ ],+ },+ });+ const production = {+ protocol: 2,+ deploymentId: "other-core:release",+ instanceId: "other-instance",+ ownerDeploymentId: "other-core:release" as string | null,+ setAt: "2026-10-19T00:00:00.000Z",+ setBy: "other-core:release",+ active: true,+ };+ let status = 200;+ const reads: string[] = [];+ const fetchLayer = globalThis.fetch;+ globalThis.fetch = async (url, init) => {+ if (!String(url).includes("/v1/background-work")) return fetchLayer(url, init);+ reads.push(String(url));+ const headers = new Headers(init?.headers);+ assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");+ assert.equal(init?.method, "GET");+ return new Response(JSON.stringify(production), { status });+ };+ writeFileSync(fake.state, JSON.stringify(stopped()));+ assert.deepEqual(await awsStoppedCoreOwner(single, manifest.backgroundDeploymentId), {+ ...production,+ ownerDeploymentId: "other-core:release",+ });+ assert.deepEqual(reads, ["https://agent.acme.example/v1/background-work"]);+ reads.length = 0;+ await awsStoppedCoreOwner({ ...single, apiUrl: "https://api.acme.example" }, manifest.backgroundDeploymentId);+ assert.deepEqual(reads, ["https://api.acme.example/v1/background-work"]);+ await assert.rejects(+ awsStoppedCoreOwner({ ...single, apiUrl: "http://api.acme.example" }, manifest.backgroundDeploymentId),+ /must be HTTPS/,+ );+ const refusals: Array<[() => void, RegExp]> = [+ [() => Object.assign(production, { deploymentId: manifest.backgroundDeploymentId }), /another deployment/],+ [+ () => Object.assign(production, { ownerDeploymentId: manifest.backgroundDeploymentId }),+ /active background owner/,+ ],+ [() => Object.assign(production, { ownerDeploymentId: null }), /active background owner/],+ [() => Object.assign(production, { active: false }), /active background owner/],+ [() => (status = 503), /HTTP 503/],+ [() => writeFileSync(fake.state, JSON.stringify(stopped("DEACTIVATING"))), /retiring task/],+ [() => writeFileSync(fake.state, JSON.stringify(stopped("STOPPED", 1))), /stable deployment capacity/],+ ];+ const healthy = structuredClone(production);+ for (const [arrange, error] of refusals) {+ Object.assign(production, healthy);+ status = 200;+ writeFileSync(fake.state, JSON.stringify(stopped()));+ arrange();+ writeFileSync(fake.log, "");+ await assert.rejects(awsUp(single, dir, { yes: true }), error);+ await assert.rejects(awsRollback(single, manifest.id), error);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service|register-task-definition/);+ }+ Object.assign(production, healthy);+ status = 200;+ writeFileSync(fake.state, JSON.stringify(stopped()));+ reads.length = 0;+ writeFileSync(fake.log, "");+ await assert.rejects(awsUp(single, dir, { yes: true }), /0 desired, expected 1/);+ assert.deepEqual(reads, ["https://agent.acme.example/v1/background-work"]);+ assert.match(readFileSync(fake.log, "utf8"), /ecs update-service/);+ } finally {+ process.env.PATH = priorPath;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});+for (const mode of ["unchanged", "migration", "missing", "unstable", "failure"] as const) {test(`AWS web routing transition gate: ${mode}`, async () => {const dir = mkdtempSync(join(tmpdir(), "qm-aws-web-routing-"));diff --git a/cli/test/background-work.test.ts b/cli/test/background-work.test.tsindex 3f2446afd..7bcd1cab3 100644--- a/cli/test/background-work.test.ts+++ b/cli/test/background-work.test.ts@@ -3,6 +3,7 @@ import assert from "node:assert/strict";import {awaitBackgroundWork,parseBackgroundWorkStatus,+ readProductionOwner,setBackgroundOwner,type BackgroundWorkStatus,type BackgroundWorkTransport,@@ -176,3 +177,29 @@ test("activation waits for every expected process to report activity and refuses/timed out awaiting 1 new-cohort process\(es\) to stop background work; 0 confirmed/,);});++test("production ownership for a stopped cohort must come from another active single-owner deployment", async () => {+ const read = (status: number, value: unknown) =>+ readProductionOwner(+ async () => ({ status, body: typeof value === "string" ? value : JSON.stringify(value) }),+ "old-cohort",+ );+ assert.deepEqual(await read(200, state()), state());+ await assert.rejects(read(503, state()), /HTTP 503/);+ await assert.rejects(read(200, "not json"), /invalid JSON/);+ await assert.rejects(read(200, null), /single-owner protocol/);+ await assert.rejects(+ read(200, { ...state(), protocol: 1, enabled: true, desiredDeploymentId: "new-cohort", members: [] }),+ /single-owner protocol/,+ );+ await assert.rejects(+ read(200, state({ deploymentId: "old-cohort", ownerDeploymentId: "old-cohort" })),+ /another deployment/,+ );+ await assert.rejects(read(200, { ...state(), deploymentId: "" }), /another deployment/);+ await assert.rejects(read(200, state({ ownerDeploymentId: "old-cohort" })), /active background owner/);+ await assert.rejects(read(200, state({ ownerDeploymentId: "third-cohort" })), /active background owner/);+ await assert.rejects(read(200, state({ ownerDeploymentId: null })), /active background owner/);+ await assert.rejects(read(200, state({ active: false })), /active background owner/);+ await assert.rejects(read(200, { ...state(), active: "yes" }), /does not match/);+});