Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Run active deployment canaries through authenticated core (#1293)

Regan Bellbc406dafafc8
diff --git a/cli/README.md b/cli/README.md
index 15df9d960..4308beced 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -100,6 +100,16 @@ return its recorded boot flag and optional deployment identity. It returns
boot flags or identities. Deployment wrappers can preserve the boot environment
while changing durable ownership independently.
+Live checks use the active ownership cohort's authenticated canary endpoint to
+verify a real model reply, session persistence, generated title, error records,
+session cleanup, and database catalog health. The CLI proves the exact healthy
+task cohort and ownership generation before and after the check. It requires a
+final success bound to the request and responding task; heartbeats alone do not
+count. An uncertain result never triggers an automatic replay or fallback.
+Legacy deployments retain the Fargate canary. A controlled deployment uses that
+same path only when a successful ownership read proves bootstrap is disabled or
+another cohort owns background work.
+
Once bootstrapped, background mode changes use generation-checked ownership
requests without restarting ECS tasks. Activation waits for prior owners to stop
claiming and every expected task to finish activation. Pausing stops new claims;
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index c9d78eb2d..9723c2174 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -73,6 +73,7 @@ import {
streamLabeled,
} from "../util.ts";
import { doctorCommon } from "./doctor.ts";
+import { checkControlledLiveSession } from "../live-session.ts";
import {
assertTerraformScaffoldSupportsConfig,
awsObjectStoreBucket,
@@ -102,13 +103,34 @@ export async function awsCoreRequest(
config: QmConfig,
url: URL,
init: RequestInit,
+ maxResponseBytes?: number,
): Promise<{ status: number; body: string }> {
const target = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");
if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");
if (awsPublicOrigin(config).protocol === "http:") {
assertCloudFrontLayerTarget(config, url, target);
const response = await fetch(url, init);
- return { status: response.status, body: await response.text() };
+ if (maxResponseBytes === undefined) return { status: response.status, body: await response.text() };
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ const reader = response.body?.getReader();
+ if (reader) {
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ size += value.byteLength;
+ if (size > maxResponseBytes) throw new CliError("AWS core response exceeded its size limit");
+ chunks.push(value);
+ }
+ } catch (error) {
+ await reader.cancel().catch(() => {});
+ throw error;
+ } finally {
+ reader.releaseLock();
+ }
+ }
+ return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
}
return new Promise((resolve, reject) => {
const request = https.request(
@@ -123,7 +145,17 @@ export async function awsCoreRequest(
},
(response) => {
const chunks: Buffer[] = [];
- response.on("data", (chunk: Buffer) => chunks.push(chunk));
+ let size = 0;
+ response.on("data", (chunk: Buffer) => {
+ size += chunk.length;
+ if (maxResponseBytes !== undefined && size > maxResponseBytes) {
+ const error = new CliError("AWS core response exceeded its size limit");
+ reject(error);
+ response.destroy(error);
+ return;
+ }
+ chunks.push(chunk);
+ });
response.on("error", reject);
response.on("end", () =>
resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }),
@@ -2839,7 +2871,12 @@ export function taskDefinitionForBackgroundWork(
};
}
-function awsBackgroundWorkTransport(config: QmConfig): BackgroundWorkTransport {
+function awsBackgroundWorkTransport(
+ config: QmConfig,
+ path = "/v1/background-work",
+ timeoutMs = 30_000,
+ maxResponseBytes?: number,
+): BackgroundWorkTransport {
const aws = requireAws(config);
const secret = (name: string): string =>
awsText(aws, [
@@ -2855,24 +2892,30 @@ function awsBackgroundWorkTransport(config: QmConfig): BackgroundWorkTransport {
if (isInvalidSecret("CORE_SIGNING_SECRET", signing) || control.trim().length < 32 || control === signing)
throw new CliError("deployment control requires a distinct secret of at least 32 characters");
const url = new URL(config.publicUrl);
- url.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/background-work`;
+ url.pathname = `${url.pathname.replace(/\/+$/, "")}${path}`;
url.search = "";
return (method, body = "") => {
const timestamp = Math.floor(Date.now() / 1000);
const signature = createHmac("sha256", signing)
.update(`v0:${timestamp}:${method}\n${url.pathname}\n${body}`)
.digest("hex");
- return awsCoreRequest(config, url, {
- method,
- headers: {
- "content-type": "application/json",
- "x-timestamp": String(timestamp),
- "x-signature": `v0=${signature}`,
- authorization: `Bearer ${control}`,
+ return awsCoreRequest(
+ config,
+ url,
+ {
+ method,
+ headers: {
+ "content-type": "application/json",
+ "x-timestamp": String(timestamp),
+ "x-signature": `v0=${signature}`,
+ authorization: `Bearer ${control}`,
+ },
+ ...(method === "POST" ? { body } : {}),
+ redirect: "error",
+ signal: AbortSignal.timeout(timeoutMs),
},
- ...(method === "POST" ? { body } : {}),
- signal: AbortSignal.timeout(30_000),
- });
+ maxResponseBytes,
+ );
};
}
@@ -4719,7 +4762,17 @@ async function checkLive(
}
if (!failures.length) {
try {
- awsLiveSession(config, states.get("core")!);
+ const cohort = aws.backgroundWorkControl ? await awsBackgroundWorkStatus(config, configDir) : undefined;
+ if (cohort?.status.enabled && cohort.status.desiredDeploymentId === cohort.deploymentId) {
+ const transport = awsBackgroundWorkTransport(config, "/v1/deployment/live-session", 600_000, 65_536);
+ await checkControlledLiveSession({
+ before: cohort,
+ read: () => awsBackgroundWorkStatus(config, configDir),
+ request: (body) => transport("POST", body),
+ });
+ } else {
+ awsLiveSession(config, states.get("core")!);
+ }
if (opts.report ?? true) step("core: private live session smoke passed");
} catch (error) {
failures.push(`core: private live session smoke failed: ${errMessage(error)}`);
diff --git a/cli/src/live-session.ts b/cli/src/live-session.ts
new file mode 100644
index 000000000..c224e5e6f
--- /dev/null
+++ b/cli/src/live-session.ts
@@ -0,0 +1,115 @@
+import { randomUUID } from "node:crypto";
+import type { BackgroundWorkStatus } from "./background-work.ts";
+import { CliError } from "./log.ts";
+
+export interface LiveSessionCohort {
+ deploymentId: string;
+ taskArns: string[];
+ status: BackgroundWorkStatus;
+}
+
+function assertReady(cohort: LiveSessionCohort): void {
+ const { deploymentId, taskArns, status } = cohort;
+ if (
+ !status.enabled ||
+ status.deploymentId !== deploymentId ||
+ status.desiredDeploymentId !== deploymentId ||
+ taskArns.length === 0 ||
+ new Set(taskArns).size !== taskArns.length ||
+ taskArns.some(
+ (taskArn) =>
+ !status.members.some(
+ (member) =>
+ member.taskArn === taskArn &&
+ member.deploymentId === deploymentId &&
+ member.generation === status.generation &&
+ member.state === "admitted" &&
+ member.ready &&
+ !member.retired,
+ ),
+ ) ||
+ status.members.some(
+ (member) =>
+ !member.retired &&
+ member.state === "admitted" &&
+ (member.deploymentId !== deploymentId ||
+ member.generation !== status.generation ||
+ member.taskArn === null ||
+ !taskArns.includes(member.taskArn)),
+ )
+ )
+ throw new CliError("live session requires the exact ready deployment cohort to own background work");
+}
+
+export async function checkControlledLiveSession(options: {
+ before: LiveSessionCohort;
+ read: () => Promise<LiveSessionCohort>;
+ request: (body: string) => Promise<{ status: number; body: string }>;
+}): Promise<void> {
+ const { before } = options;
+ assertReady(before);
+ const requestId = randomUUID();
+ let response: { status: number; body: string };
+ try {
+ response = await options.request(
+ JSON.stringify({
+ requestId,
+ expectedDeploymentId: before.deploymentId,
+ expectedGeneration: before.status.generation,
+ expectedTaskArns: before.taskArns,
+ }),
+ );
+ } catch {
+ throw new CliError(
+ `live session result is unconfirmed for request ${requestId}; no automatic replay or fallback was attempted`,
+ );
+ }
+ if (response.status !== 200) throw new CliError(`live session rejected with HTTP ${response.status}`);
+ let result: unknown;
+ try {
+ result = JSON.parse(response.body);
+ } catch {
+ throw new CliError(`live session returned no valid final result for request ${requestId}`);
+ }
+ if (!result || typeof result !== "object" || Array.isArray(result))
+ throw new CliError("live session returned an invalid final result");
+ const value = result as Record<string, unknown>;
+ if (
+ value.ok !== true ||
+ value.requestId !== requestId ||
+ value.deploymentId !== before.deploymentId ||
+ value.generation !== before.status.generation ||
+ !before.status.members.some(
+ (member) =>
+ member.instanceId === value.instanceId &&
+ member.taskArn === value.taskArn &&
+ before.taskArns.includes(member.taskArn ?? "") &&
+ member.deploymentId === before.deploymentId &&
+ member.generation === before.status.generation &&
+ member.state === "admitted" &&
+ member.ready &&
+ !member.retired,
+ )
+ )
+ throw new CliError(
+ `live session did not confirm success for the expected request and deployment cohort (${requestId})`,
+ );
+ const after = await options.read();
+ assertReady(after);
+ if (
+ after.deploymentId !== before.deploymentId ||
+ after.status.generation !== before.status.generation ||
+ after.taskArns.length !== before.taskArns.length ||
+ after.taskArns.some((arn) => !before.taskArns.includes(arn)) ||
+ !after.status.members.some(
+ (member) =>
+ member.instanceId === value.instanceId &&
+ member.taskArn === value.taskArn &&
+ member.state === "admitted" &&
+ member.ready &&
+ !member.retired &&
+ member.generation === before.status.generation,
+ )
+ )
+ throw new CliError("live session deployment cohort changed during qualification");
+}
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index 8ee229e7c..222a8f020 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -14,6 +14,7 @@ import {
assertAwsPublicRouting,
assertGithubDeployTrust,
awsCheckLive,
+ awsCoreRequest,
awsBackgroundWorkStatus,
awsBackgroundWorkBootState,
awsBootstrapBackgroundWork,
@@ -5060,6 +5061,38 @@ test("AWS layer transport uses the HTTPS front door when an HTTP ALB origin is c
.digest("hex")}`,
);
}
+ const fetchBeforeLimit = globalThis.fetch;
+ let cancelled = false;
+ globalThis.fetch = async () =>
+ new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(new TextEncoder().encode(" ".repeat(33)));
+ },
+ cancel() {
+ cancelled = true;
+ },
+ }),
+ );
+ try {
+ await assert.rejects(
+ awsCoreRequest(
+ configured,
+ new URL(configured.publicUrl),
+ {
+ method: "POST",
+ body: "{}",
+ redirect: "error",
+ signal: AbortSignal.timeout(1000),
+ },
+ 32,
+ ),
+ /size limit/,
+ );
+ assert.equal(cancelled, true);
+ } finally {
+ globalThis.fetch = fetchBeforeLimit;
+ }
for (const invalid of [
{ ...distribution, DomainName: "another.cloudfront.net" },
{ ...distribution, Enabled: false },
@@ -5104,6 +5137,49 @@ test("AWS layer transport uses the HTTPS front door when an HTTP ALB origin is c
}
});
+test("AWS core transport bounds streamed TLS bodies and destroys oversized responses", async (t) => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-core-body-limit-"));
+ const fake = fakeAws(dir, "console.log('')");
+ let destroyed = false;
+ t.mock.method(https, "request", (_url: URL, _options: https.RequestOptions, callback: (value: unknown) => void) => {
+ const request = new EventEmitter() as EventEmitter & { end(): void };
+ request.end = () => {
+ const response = Object.assign(new EventEmitter(), {
+ statusCode: 200,
+ destroy(error: Error) {
+ destroyed = true;
+ response.emit("error", error);
+ },
+ });
+ callback(response);
+ response.emit("data", Buffer.from(" ".repeat(16)));
+ response.emit("data", Buffer.from(" ".repeat(17)));
+ response.emit("end");
+ };
+ return request;
+ });
+ try {
+ await assert.rejects(
+ awsCoreRequest(
+ config,
+ new URL(config.publicUrl),
+ {
+ method: "POST",
+ body: "{}",
+ signal: AbortSignal.timeout(1000),
+ },
+ 32,
+ ),
+ /size limit/,
+ );
+ assert.equal(destroyed, true);
+ } finally {
+ t.mock.restoreAll();
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
test("AWS layer transport rejects invalid targets and propagates TLS and body failures without fallback", async (t) => {
const dir = mkdtempSync(join(tmpdir(), "qm-aws-layer-failure-"));
const fake = fakeAws(dir, "console.log('')");
@@ -5558,7 +5634,37 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECS
};
const fetchLayer = globalThis.fetch;
const mutations: unknown[] = [];
+ let smokeCalls = 0;
+ let smokeFailure = false;
globalThis.fetch = async (url, init) => {
+ if (String(url).includes("/v1/deployment/live-session")) {
+ smokeCalls++;
+ const body = String(init?.body ?? "");
+ const request = JSON.parse(body);
+ const headers = new Headers(init?.headers);
+ assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");
+ assert.equal(
+ headers.get("x-signature"),
+ `v0=${createHmac("sha256", TEST_SECRET_VALUE)
+ .update(`v0:${headers.get("x-timestamp")}:POST\n/v1/deployment/live-session\n${body}`)
+ .digest("hex")}`,
+ );
+ assert.equal(request.expectedDeploymentId, manifest.backgroundDeploymentId);
+ assert.equal(request.expectedGeneration, ownership.generation);
+ assert.deepEqual(request.expectedTaskArns, [taskArn]);
+ assert.ok(init?.signal instanceof AbortSignal);
+ if (smokeFailure) throw new Error("disconnected with a sensitive credential");
+ return new Response(
+ ` \n\n${JSON.stringify({
+ ok: true,
+ requestId: request.requestId,
+ deploymentId: ownership.deploymentId,
+ generation: ownership.generation,
+ instanceId: "instance-one",
+ taskArn,
+ })}\n`,
+ );
+ }
if (!String(url).includes("/v1/background-work")) return fetchLayer(url, init);
const headers = new Headers(init?.headers);
assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");
@@ -5591,6 +5697,10 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECS
return new Response(JSON.stringify(ownership), { status: 200 });
};
assert.deepEqual((await awsBackgroundWorkStatus(single, dir)).taskArns, [taskArn]);
+ writeFileSync(fake.log, "");
+ await awsCheckLive(single, { configDir: dir, report: false });
+ assert.equal(smokeCalls, 0);
+ assert.match(readFileSync(fake.log, "utf8"), /ecs run-task/);
await assert.rejects(awsSetBackgroundWork(single, dir, true), /explicitly bootstrap/);
const stoppedLegacyArn = "arn:aws:ecs:us-west-2:123456789012:task/stopped-legacy-core";
ownership.members.push({
@@ -5625,6 +5735,25 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECS
);
assert.equal(bootstrapped.generation, 1);
writeFileSync(fake.log, "");
+ await awsCheckLive(single, { configDir: dir, report: false });
+ assert.equal(smokeCalls, 1);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs run-task/);
+ smokeFailure = true;
+ await assert.rejects(awsCheckLive(single, { configDir: dir, report: false }), /unconfirmed/);
+ assert.equal(smokeCalls, 2);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs run-task/);
+ smokeFailure = false;
+ const activeOwnership = structuredClone(ownership);
+ ownership.generation++;
+ ownership.desiredDeploymentId = "other-owner";
+ ownership.members[0]!.state = "relinquished";
+ ownership.members[0]!.ready = false;
+ writeFileSync(fake.log, "");
+ await awsCheckLive(single, { configDir: dir, report: false });
+ assert.equal(smokeCalls, 2);
+ assert.match(readFileSync(fake.log, "utf8"), /ecs run-task/);
+ Object.assign(ownership, activeOwnership);
+ writeFileSync(fake.log, "");
await assert.rejects(awsUp(single, dir, { yes: true }), /pause or hand over/);
assert.doesNotMatch(
readFileSync(fake.log, "utf8"),
diff --git a/cli/test/live-session.test.ts b/cli/test/live-session.test.ts
new file mode 100644
index 000000000..a26159631
--- /dev/null
+++ b/cli/test/live-session.test.ts
@@ -0,0 +1,182 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { checkControlledLiveSession, type LiveSessionCohort } from "../src/live-session.ts";
+
+function cohort(): LiveSessionCohort {
+ return {
+ deploymentId: "core:release",
+ taskArns: ["task-a", "task-b"],
+ status: {
+ protocol: 1,
+ enabled: true,
+ deploymentId: "core:release",
+ instanceId: "instance-a",
+ generation: 4,
+ desiredDeploymentId: "core:release",
+ lastRequestId: "handover",
+ members: ["a", "b"].map((suffix) => ({
+ instanceId: `instance-${suffix}`,
+ taskArn: `task-${suffix}`,
+ deploymentId: "core:release",
+ generation: 4,
+ state: "admitted",
+ retired: false,
+ ready: true,
+ })),
+ },
+ };
+}
+
+function success(body: string): Record<string, unknown> {
+ const request = JSON.parse(body);
+ assert.equal(request.expectedDeploymentId, "core:release");
+ assert.equal(request.expectedGeneration, 4);
+ assert.deepEqual(request.expectedTaskArns, ["task-a", "task-b"]);
+ return {
+ ok: true,
+ requestId: request.requestId,
+ deploymentId: "core:release",
+ generation: 4,
+ instanceId: "instance-a",
+ taskArn: "task-a",
+ };
+}
+
+test("live session accepts whitespace heartbeats and checks the exact cohort afterward", async () => {
+ let reads = 0;
+ await checkControlledLiveSession({
+ before: cohort(),
+ request: async (body) => ({ status: 200, body: ` \n\t\n${JSON.stringify(success(body))}\n` }),
+ read: async () => {
+ reads++;
+ return cohort();
+ },
+ });
+ assert.equal(reads, 1);
+});
+
+test("live session refuses incomplete or competing ownership before requesting a canary", async () => {
+ const mutations: Array<(value: LiveSessionCohort) => void> = [
+ (value) => {
+ value.status.enabled = false;
+ },
+ (value) => {
+ value.status.desiredDeploymentId = "other";
+ },
+ (value) => {
+ value.status.members[0]!.ready = false;
+ },
+ (value) => {
+ value.status.members[0]!.generation--;
+ },
+ (value) => {
+ value.status.members[0]!.retired = true;
+ },
+ (value) => {
+ value.status.members[0]!.state = "relinquished";
+ },
+ (value) => {
+ value.status.members.push({ ...value.status.members[0]!, instanceId: "other", taskArn: "unexpected" });
+ },
+ ];
+ for (const mutate of mutations) {
+ const before = cohort();
+ mutate(before);
+ await assert.rejects(
+ checkControlledLiveSession({
+ before,
+ request: async () => {
+ assert.fail("no canary before ownership proof");
+ },
+ read: async () => cohort(),
+ }),
+ /exact ready deployment cohort/,
+ );
+ }
+});
+
+test("live session requires an explicit final success bound to request and instance", async () => {
+ for (const patch of [
+ { ok: false, error: "sensitive server error" },
+ { ok: undefined },
+ { requestId: "another-request" },
+ { deploymentId: "stale" },
+ { generation: 3 },
+ { instanceId: "old-instance" },
+ { taskArn: "old-task" },
+ ]) {
+ let calls = 0;
+ await assert.rejects(
+ checkControlledLiveSession({
+ before: cohort(),
+ request: async (body) => {
+ calls++;
+ return { status: 200, body: JSON.stringify({ ...success(body), ...patch }) };
+ },
+ read: async () => cohort(),
+ }),
+ (error: Error) => {
+ assert.match(error.message, /did not confirm success/);
+ assert.doesNotMatch(error.message, /sensitive server error/);
+ return true;
+ },
+ );
+ assert.equal(calls, 1);
+ }
+});
+
+test("live session never retries an ambiguous or truncated response", async () => {
+ for (const kind of ["disconnect", "heartbeat-only", "truncated", "http-error"]) {
+ let calls = 0;
+ await assert.rejects(
+ checkControlledLiveSession({
+ before: cohort(),
+ request: async () => {
+ calls++;
+ if (kind === "disconnect") throw new Error("Bearer secret must never escape");
+ return {
+ status: kind === "http-error" ? 503 : 200,
+ body: kind === "heartbeat-only" ? " \n\n" : '{"ok":true',
+ };
+ },
+ read: async () => cohort(),
+ }),
+ (error: Error) => {
+ assert.doesNotMatch(error.message, /Bearer secret/);
+ return true;
+ },
+ );
+ assert.equal(calls, 1);
+ }
+});
+
+test("live session rejects ownership or task replacement during a successful model check", async () => {
+ const mutations: Array<(value: LiveSessionCohort) => void> = [
+ (value) => {
+ value.status.generation++;
+ value.status.members.forEach((member) => member.generation++);
+ },
+ (value) => {
+ value.taskArns[1] = "replacement";
+ value.status.members[1]!.taskArn = "replacement";
+ },
+ (value) => {
+ value.status.members[0]!.instanceId = "replacement-instance";
+ },
+ (value) => {
+ value.status.desiredDeploymentId = null;
+ },
+ ];
+ for (const mutate of mutations) {
+ const after = cohort();
+ mutate(after);
+ await assert.rejects(
+ checkControlledLiveSession({
+ before: cohort(),
+ request: async (body) => ({ status: 200, body: JSON.stringify(success(body)) }),
+ read: async () => after,
+ }),
+ /cohort/,
+ );
+ }
+});