Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Prove a stopped AWS core cohort is not the background owner through the production route

Regan Bell8576e5cce622
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index 10a9bbe51..4d40fa6c6 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -92,11 +92,41 @@ import {
import {
awaitBackgroundWork,
readBackgroundWork,
+ readProductionOwner,
setBackgroundOwner,
type BackgroundWorkStatus,
type BackgroundWorkTransport,
} from "../background-work.ts";
+async function fetchCore(
+ url: URL,
+ init: RequestInit,
+ maxResponseBytes?: number,
+): Promise<{ status: number; body: string }> {
+ const response = await fetch(url, init);
+ if (maxResponseBytes === undefined) return { status: response.status, body: await response.text() };
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ const reader = response.body?.getReader();
+ if (reader) {
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ size += value.byteLength;
+ if (size > maxResponseBytes) throw new CliError("AWS core response exceeded its size limit");
+ chunks.push(value);
+ }
+ } catch (error) {
+ await reader.cancel().catch(() => {});
+ throw error;
+ } finally {
+ reader.releaseLock();
+ }
+ }
+ return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
+}
+
export async function awsCoreRequest(
config: QmConfig,
url: URL,
@@ -107,28 +137,7 @@ export async function awsCoreRequest(
if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");
if (awsPublicOrigin(config).protocol === "http:") {
assertCloudFrontLayerTarget(config, url, target);
- const response = await fetch(url, init);
- if (maxResponseBytes === undefined) return { status: response.status, body: await response.text() };
- const chunks: Uint8Array[] = [];
- let size = 0;
- const reader = response.body?.getReader();
- if (reader) {
- try {
- for (;;) {
- const { done, value } = await reader.read();
- if (done) break;
- size += value.byteLength;
- if (size > maxResponseBytes) throw new CliError("AWS core response exceeded its size limit");
- chunks.push(value);
- }
- } catch (error) {
- await reader.cancel().catch(() => {});
- throw error;
- } finally {
- reader.releaseLock();
- }
- }
- return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
+ return fetchCore(url, init, maxResponseBytes);
}
return new Promise((resolve, reject) => {
const request = https.request(
@@ -2942,6 +2951,7 @@ function awsBackgroundWorkTransport(
path = "/v1/background-work",
timeoutMs = 30_000,
maxResponseBytes?: number,
+ productionRoute = false,
): BackgroundWorkTransport {
const aws = requireAws(config);
const secret = (name: string): string =>
@@ -2957,31 +2967,32 @@ function awsBackgroundWorkTransport(
const control = secret("DEPLOYMENT_CONTROL_SECRET");
if (isInvalidSecret("CORE_SIGNING_SECRET", signing) || control.trim().length < 32 || control === signing)
throw new CliError("deployment control requires a distinct secret of at least 32 characters");
- const url = new URL(config.publicUrl);
+ const url = new URL(productionRoute ? (config.apiUrl ?? config.publicUrl) : config.publicUrl);
+ if (productionRoute && (url.protocol !== "https:" || url.username || url.password))
+ throw new CliError("the production background ownership route must be HTTPS without credentials");
url.pathname = `${url.pathname.replace(/\/+$/, "")}${path}`;
url.search = "";
+ url.hash = "";
return (method, body = "") => {
const timestamp = Math.floor(Date.now() / 1000);
const signature = createHmac("sha256", signing)
.update(`v0:${timestamp}:${method}\n${url.pathname}\n${body}`)
.digest("hex");
- return awsCoreRequest(
- config,
- url,
- {
- method,
- headers: {
- "content-type": "application/json",
- "x-timestamp": String(timestamp),
- "x-signature": `v0=${signature}`,
- authorization: `Bearer ${control}`,
- },
- ...(method === "POST" ? { body } : {}),
- redirect: "error",
- signal: AbortSignal.timeout(timeoutMs),
+ const init: RequestInit = {
+ method,
+ headers: {
+ "content-type": "application/json",
+ "x-timestamp": String(timestamp),
+ "x-signature": `v0=${signature}`,
+ authorization: `Bearer ${control}`,
},
- maxResponseBytes,
- );
+ ...(method === "POST" ? { body } : {}),
+ redirect: "error",
+ signal: AbortSignal.timeout(timeoutMs),
+ };
+ return productionRoute
+ ? fetchCore(url, init, maxResponseBytes ?? 65_536)
+ : awsCoreRequest(config, url, init, maxResponseBytes);
};
}
@@ -3277,8 +3288,19 @@ export async function awsBackgroundWorkCapacity(
return proof;
}
+export async function awsStoppedCoreOwner(
+ config: QmConfig,
+ deploymentId: string,
+): Promise<BackgroundWorkStatus | undefined> {
+ const state = describedServices(config, ["core"]).get("core");
+ if (!state || state.desiredCount !== 0) return undefined;
+ stableWorkloadCapacity(config, "core", state, state.taskDefinition ?? "", 0);
+ return readProductionOwner(awsBackgroundWorkTransport(config, undefined, undefined, undefined, true), deploymentId);
+}
+
async function assertNotBackgroundOwner(config: QmConfig, current: DeploymentManifest | undefined): Promise<void> {
if (!current?.backgroundDeploymentId) return;
+ if (await awsStoppedCoreOwner(config, current.backgroundDeploymentId)) return;
const status = await readBackgroundWork(awsBackgroundWorkTransport(config), current.backgroundDeploymentId);
if (status.ownerDeploymentId === current.backgroundDeploymentId)
throw new CliError("hand background ownership to another deployment before replacing the current core tasks");
diff --git a/cli/src/background-work.ts b/cli/src/background-work.ts
index f1bc7e6e5..8c1e25338 100644
--- a/cli/src/background-work.ts
+++ b/cli/src/background-work.ts
@@ -93,6 +93,30 @@ export async function readBackgroundWork(
return parseBackgroundWorkStatus(response.body, deploymentId);
}
+export async function readProductionOwner(
+ transport: BackgroundWorkTransport,
+ stoppedDeploymentId: string,
+): Promise<BackgroundWorkStatus> {
+ const response = await transport("GET");
+ if (response.status !== 200)
+ throw new CliError(`production background ownership read failed with HTTP ${response.status}`);
+ let value: unknown;
+ try {
+ value = JSON.parse(response.body);
+ } catch {
+ throw new CliError("background ownership returned invalid JSON");
+ }
+ if (!record(value) || value.protocol !== 2)
+ throw new CliError("production background ownership must use the single-owner protocol");
+ const responder = value.deploymentId;
+ if (!text(responder) || responder === stoppedDeploymentId)
+ throw new CliError("production background ownership was not reported by another deployment");
+ const status = parseBackgroundWorkStatus(response.body, responder);
+ if (status.ownerDeploymentId !== responder || !status.active)
+ throw new CliError("production does not have an active background owner");
+ return status;
+}
+
export async function setBackgroundOwner(
transport: BackgroundWorkTransport,
deploymentId: string,
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index c91da392d..2cd638f42 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -25,6 +25,7 @@ import {
awsRollback,
awsSecretsPush,
awsSetBackgroundWork,
+ awsStoppedCoreOwner,
taskDefinitionForBackgroundWork,
awsStatus,
awsUp,
@@ -6124,6 +6125,124 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECS
}
});
+test("a stopped AWS core cohort is replaceable only while the production route reports another active owner", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-stopped-background-"));
+ const dockerBin = join(dir, "docker");
+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);
+ chmodSync(dockerBin, 0o755);
+ const base = oneServiceConfig();
+ const single: QmConfig = { ...base, aws: { ...base.aws!, backgroundWorkControl: true } };
+ const fake = statefulAws(dir, single);
+ const priorPath = process.env.PATH;
+ process.env.PATH = `${dir}:${priorPath}`;
+ try {
+ await awsUp(single, dir, { yes: true });
+ const deployed = JSON.parse(readFileSync(fake.state, "utf8"));
+ const manifest = JSON.parse(
+ deployed.dynamo[`deployment/manifest/${deployed.dynamo["deployment/current"].manifestId.S}`].manifest.S,
+ );
+ const taskDefinition = deployed.services["acme-core"].taskDefinition;
+ const stopped = (lastStatus = "STOPPED", runningCount = 0) => ({
+ ...deployed,
+ serviceOverrides: {
+ "acme-core": {
+ desiredCount: 0,
+ runningCount,
+ pendingCount: 0,
+ deployments: [
+ {
+ id: "scaled-down",
+ status: "PRIMARY",
+ taskDefinition,
+ rolloutState: "COMPLETED",
+ runningCount: 0,
+ pendingCount: 0,
+ },
+ ],
+ },
+ },
+ taskInventory: {
+ "acme-core": [
+ {
+ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/old-core",
+ taskDefinitionArn: taskDefinition,
+ desiredStatus: "STOPPED",
+ lastStatus,
+ },
+ ],
+ },
+ });
+ const production = {
+ protocol: 2,
+ deploymentId: "other-core:release",
+ instanceId: "other-instance",
+ ownerDeploymentId: "other-core:release" as string | null,
+ setAt: "2026-10-19T00:00:00.000Z",
+ setBy: "other-core:release",
+ active: true,
+ };
+ let status = 200;
+ const reads: string[] = [];
+ const fetchLayer = globalThis.fetch;
+ globalThis.fetch = async (url, init) => {
+ if (!String(url).includes("/v1/background-work")) return fetchLayer(url, init);
+ reads.push(String(url));
+ const headers = new Headers(init?.headers);
+ assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");
+ assert.equal(init?.method, "GET");
+ return new Response(JSON.stringify(production), { status });
+ };
+ writeFileSync(fake.state, JSON.stringify(stopped()));
+ assert.deepEqual(await awsStoppedCoreOwner(single, manifest.backgroundDeploymentId), {
+ ...production,
+ ownerDeploymentId: "other-core:release",
+ });
+ assert.deepEqual(reads, ["https://agent.acme.example/v1/background-work"]);
+ reads.length = 0;
+ await awsStoppedCoreOwner({ ...single, apiUrl: "https://api.acme.example" }, manifest.backgroundDeploymentId);
+ assert.deepEqual(reads, ["https://api.acme.example/v1/background-work"]);
+ await assert.rejects(
+ awsStoppedCoreOwner({ ...single, apiUrl: "http://api.acme.example" }, manifest.backgroundDeploymentId),
+ /must be HTTPS/,
+ );
+ const refusals: Array<[() => void, RegExp]> = [
+ [() => Object.assign(production, { deploymentId: manifest.backgroundDeploymentId }), /another deployment/],
+ [
+ () => Object.assign(production, { ownerDeploymentId: manifest.backgroundDeploymentId }),
+ /active background owner/,
+ ],
+ [() => Object.assign(production, { ownerDeploymentId: null }), /active background owner/],
+ [() => Object.assign(production, { active: false }), /active background owner/],
+ [() => (status = 503), /HTTP 503/],
+ [() => writeFileSync(fake.state, JSON.stringify(stopped("DEACTIVATING"))), /retiring task/],
+ [() => writeFileSync(fake.state, JSON.stringify(stopped("STOPPED", 1))), /stable deployment capacity/],
+ ];
+ const healthy = structuredClone(production);
+ for (const [arrange, error] of refusals) {
+ Object.assign(production, healthy);
+ status = 200;
+ writeFileSync(fake.state, JSON.stringify(stopped()));
+ arrange();
+ writeFileSync(fake.log, "");
+ await assert.rejects(awsUp(single, dir, { yes: true }), error);
+ await assert.rejects(awsRollback(single, manifest.id), error);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service|register-task-definition/);
+ }
+ Object.assign(production, healthy);
+ status = 200;
+ writeFileSync(fake.state, JSON.stringify(stopped()));
+ reads.length = 0;
+ writeFileSync(fake.log, "");
+ await assert.rejects(awsUp(single, dir, { yes: true }), /0 desired, expected 1/);
+ assert.deepEqual(reads, ["https://agent.acme.example/v1/background-work"]);
+ assert.match(readFileSync(fake.log, "utf8"), /ecs update-service/);
+ } finally {
+ process.env.PATH = priorPath;
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
for (const mode of ["unchanged", "migration", "missing", "unstable", "failure"] as const) {
test(`AWS web routing transition gate: ${mode}`, async () => {
const dir = mkdtempSync(join(tmpdir(), "qm-aws-web-routing-"));
diff --git a/cli/test/background-work.test.ts b/cli/test/background-work.test.ts
index 3f2446afd..7bcd1cab3 100644
--- a/cli/test/background-work.test.ts
+++ b/cli/test/background-work.test.ts
@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
import {
awaitBackgroundWork,
parseBackgroundWorkStatus,
+ readProductionOwner,
setBackgroundOwner,
type BackgroundWorkStatus,
type BackgroundWorkTransport,
@@ -176,3 +177,29 @@ test("activation waits for every expected process to report activity and refuses
/timed out awaiting 1 new-cohort process\(es\) to stop background work; 0 confirmed/,
);
});
+
+test("production ownership for a stopped cohort must come from another active single-owner deployment", async () => {
+ const read = (status: number, value: unknown) =>
+ readProductionOwner(
+ async () => ({ status, body: typeof value === "string" ? value : JSON.stringify(value) }),
+ "old-cohort",
+ );
+ assert.deepEqual(await read(200, state()), state());
+ await assert.rejects(read(503, state()), /HTTP 503/);
+ await assert.rejects(read(200, "not json"), /invalid JSON/);
+ await assert.rejects(read(200, null), /single-owner protocol/);
+ await assert.rejects(
+ read(200, { ...state(), protocol: 1, enabled: true, desiredDeploymentId: "new-cohort", members: [] }),
+ /single-owner protocol/,
+ );
+ await assert.rejects(
+ read(200, state({ deploymentId: "old-cohort", ownerDeploymentId: "old-cohort" })),
+ /another deployment/,
+ );
+ await assert.rejects(read(200, { ...state(), deploymentId: "" }), /another deployment/);
+ await assert.rejects(read(200, state({ ownerDeploymentId: "old-cohort" })), /active background owner/);
+ await assert.rejects(read(200, state({ ownerDeploymentId: "third-cohort" })), /active background owner/);
+ await assert.rejects(read(200, state({ ownerDeploymentId: null })), /active background owner/);
+ await assert.rejects(read(200, state({ active: false })), /active background owner/);
+ await assert.rejects(read(200, { ...state(), active: "yes" }), /does not match/);
+});