Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Prove inactive AWS capacity before release admission (#1296)

Regan Bell3068269a880d
diff --git a/cli/README.md b/cli/README.md
index 4308beced..eb47506bc 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -100,6 +100,20 @@ return its recorded boot flag and optional deployment identity. It returns
boot flags or identities. Deployment wrappers can preserve the boot environment
while changing durable ownership independently.
+The exported `awsBackgroundWorkCapacity(config, configDir, candidatePath?)` proves
+that an inactive controlled stack is currently reusable. It requires another
+owner, fully drained current membership, stable native deployments and exact task
+inventories for every workload, resolved deployment preparation, and explicitly
+disabled protection on every current core task. It never changes task protection
+or deployment state. The deploy role needs `ecs:GetTaskProtection` on its tasks.
+The result binds the manifest and deployment identities, ownership generation,
+workload task definitions, native deployment IDs, task ARNs, and protection proof.
+`awsBackgroundWorkStatus` also returns the current manifest ID for an active-owner
+proof. A release coordinator can combine both snapshots with immutable candidate
+provenance and compare them again under its production lock before any mutation.
+This is a point-in-time check, not a reservation: intervening maintenance or task
+replacement invalidates the proof and must block promotion.
+
Live checks use the active ownership cohort's authenticated canary endpoint to
verify a real model reply, session persistence, generated title, error records,
session cleanup, and database catalog health. The CLI proves the exact healthy
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index 9723c2174..df6e3ba36 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -1111,6 +1111,7 @@ interface EcsDeploymentState {
taskDefinition?: string;
rolloutState?: string;
runningCount?: number;
+ pendingCount?: number;
failedTasks?: number;
}
@@ -1119,6 +1120,7 @@ interface EcsServiceState {
status?: string;
desiredCount?: number;
runningCount?: number;
+ pendingCount?: number;
taskDefinition?: string;
deploymentConfiguration?: { strategy?: string };
networkConfiguration?: {
@@ -1284,11 +1286,11 @@ function clearBackgroundPreparation(aws: AwsConfig): void {
]);
}
-function reconcileBackgroundPreparation(
+function assertBackgroundPreparationResolved(
aws: AwsConfig,
current: DeploymentManifest | undefined,
before: BackgroundDeploymentPreparation["before"],
-): void {
+): boolean {
const response = awsJson<{ Item?: Record<string, { S?: string }> }>(aws, [
"dynamodb",
"get-item",
@@ -1298,7 +1300,7 @@ function reconcileBackgroundPreparation(
JSON.stringify({ lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } }),
"--consistent-read",
]);
- if (!response.Item) return;
+ if (!response.Item) return false;
let preparation: BackgroundDeploymentPreparation;
try {
preparation = JSON.parse(response.Item.preparation?.S ?? "");
@@ -1323,7 +1325,15 @@ function reconcileBackgroundPreparation(
throw new CliError(
`background deployment ${preparation.id} remains unresolved; reconcile its recorded tasks before retrying instead of allocating another cohort`,
);
- clearBackgroundPreparation(aws);
+ return true;
+}
+
+function reconcileBackgroundPreparation(
+ aws: AwsConfig,
+ current: DeploymentManifest | undefined,
+ before: BackgroundDeploymentPreparation["before"],
+): void {
+ if (assertBackgroundPreparationResolved(aws, current, before)) clearBackgroundPreparation(aws);
}
function prepareBackgroundDeployment(aws: AwsConfig, preparation: BackgroundDeploymentPreparation): void {
@@ -2964,6 +2974,7 @@ async function awsBackgroundCohort(
config: QmConfig,
configDir: string,
candidatePath?: string,
+ waitForReady = true,
): Promise<AwsBackgroundCohort> {
const { aws, workloads } = awsTopology(config, configDir);
if (!aws.backgroundWorkControl)
@@ -3007,9 +3018,10 @@ async function awsBackgroundCohort(
}
const count = before.counts.core;
if (!count) throw new CliError("background ownership requires running core capacity");
- await awaitServiceTargets(config, {
- core: { taskDefinition: before.tasks.core!, desiredCount: count, waitForDrain: false },
- });
+ if (waitForReady)
+ await awaitServiceTargets(config, {
+ core: { taskDefinition: before.tasks.core!, desiredCount: count, waitForDrain: false },
+ });
const taskArns =
awsJson<{ taskArns?: string[] }>(aws, [
"ecs",
@@ -3050,6 +3062,7 @@ export async function awsBackgroundWorkStatus(
status: BackgroundWorkStatus;
deploymentId: string;
taskArns: string[];
+ manifestId: string;
}> {
assertAwsCallerAccount(requireAws(config));
const cohort = await awsBackgroundCohort(config, configDir, candidatePath);
@@ -3062,7 +3075,190 @@ export async function awsBackgroundWorkStatus(
)
throw new CliError("a running core task has not enrolled in the ownership protocol");
}
- return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns };
+ return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns, manifestId: cohort.manifest.id };
+}
+
+export interface AwsBackgroundWorkCapacity {
+ manifestId: string;
+ deploymentId: string;
+ generation: number;
+ desiredDeploymentId: string;
+ coreTaskProtection: Record<string, false>;
+ workloads: Record<string, { taskDefinition: string; deploymentId: string; desiredCount: number; taskArns: string[] }>;
+}
+
+export async function awsBackgroundWorkCapacity(
+ config: QmConfig,
+ configDir: string,
+ candidatePath?: string,
+): Promise<AwsBackgroundWorkCapacity> {
+ const { aws, workloads } = awsTopology(config, configDir);
+ assertAwsCallerAccount(aws);
+ const cohort = await awsBackgroundCohort(config, configDir, candidatePath, false);
+ const transport = awsBackgroundWorkTransport(config);
+ const ownership = await readBackgroundWork(transport, cohort.deploymentId);
+ if (!ownership.enabled || !ownership.desiredDeploymentId || ownership.desiredDeploymentId === cohort.deploymentId)
+ throw new CliError("capacity requires enabled background ownership held by another deployment");
+ assertBackgroundMembersDrained(ownership, cohort.deploymentId);
+ if (
+ cohort.taskArns.some(
+ (taskArn) =>
+ !ownership.members.some(
+ (member) =>
+ member.taskArn === taskArn &&
+ member.deploymentId === cohort.deploymentId &&
+ !member.retired &&
+ member.state === "drained",
+ ),
+ )
+ )
+ throw new CliError("capacity requires every current core task to be enrolled and drained");
+ const states = describedServices(config, workloads);
+ assertOwnedServices(config, states, workloads);
+ const snapshot = serviceSnapshotFromStates(states, workloads);
+ assertBackgroundPreparationResolved(aws, cohort.manifest, snapshot);
+ const proof: AwsBackgroundWorkCapacity = {
+ manifestId: cohort.manifest.id,
+ deploymentId: cohort.deploymentId,
+ generation: ownership.generation,
+ desiredDeploymentId: ownership.desiredDeploymentId,
+ coreTaskProtection: {},
+ workloads: {},
+ };
+ for (const workload of [...workloads].sort()) {
+ const state = states.get(workload)!;
+ const taskDefinition = cohort.manifest.tasks[workload]!;
+ const desiredCount = cohort.manifest.counts?.[workload];
+ const primary = state.deployments?.filter((deployment) => deployment.status === "PRIMARY") ?? [];
+ const deployment = primary[0];
+ if (
+ typeof desiredCount !== "number" ||
+ !Number.isSafeInteger(desiredCount) ||
+ desiredCount < 0 ||
+ state.status !== "ACTIVE" ||
+ state.taskDefinition !== taskDefinition ||
+ state.desiredCount !== desiredCount ||
+ state.runningCount !== desiredCount ||
+ state.pendingCount !== 0 ||
+ primary.length !== 1 ||
+ !deployment?.id ||
+ deployment.taskDefinition !== taskDefinition ||
+ deployment.runningCount !== desiredCount ||
+ state.deployments?.some((item) => item.pendingCount !== 0 || (item !== deployment && item.runningCount !== 0)) ||
+ (state.deploymentConfiguration?.strategy === "BLUE_GREEN"
+ ? nativeBlueGreenStatus(config, workload, taskDefinition) !== "SUCCESSFUL"
+ : deployment.rolloutState !== "COMPLETED")
+ )
+ throw new CliError(`${workload} has not reached stable deployment capacity`);
+ const listed = new Set<string>();
+ for (const desiredStatus of ["RUNNING", "STOPPED"]) {
+ const response = awsJson<{ taskArns?: string[] }>(aws, [
+ "ecs",
+ "list-tasks",
+ "--cluster",
+ aws.cluster,
+ "--service-name",
+ aws.services[workload]!.ecsService,
+ "--desired-status",
+ desiredStatus,
+ ]);
+ if (!Array.isArray(response.taskArns) || response.taskArns.some((arn) => typeof arn !== "string" || !arn))
+ throw new CliError(`${workload} task inventory is unavailable`);
+ for (const arn of response.taskArns) listed.add(arn);
+ }
+ const live: string[] = [];
+ for (const batch of chunks([...listed], 100)) {
+ const response = awsJson<{
+ tasks?: Array<{
+ taskArn?: string;
+ taskDefinitionArn?: string;
+ lastStatus?: string;
+ desiredStatus?: string;
+ healthStatus?: string;
+ }>;
+ failures?: unknown[];
+ }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);
+ if (
+ response.failures?.length ||
+ !Array.isArray(response.tasks) ||
+ response.tasks.length !== batch.length ||
+ new Set(response.tasks.map((task) => task.taskArn)).size !== batch.length ||
+ response.tasks.some((task) => !task.taskArn || !batch.includes(task.taskArn))
+ )
+ throw new CliError(`${workload} capacity requires complete task inventory`);
+ for (const task of response.tasks) {
+ if (task.lastStatus === "STOPPED") continue;
+ if (
+ task.lastStatus !== "RUNNING" ||
+ task.desiredStatus !== "RUNNING" ||
+ task.taskDefinitionArn !== taskDefinition ||
+ (task.healthStatus !== "HEALTHY" && (isServiceName(workload) || task.healthStatus !== "UNKNOWN"))
+ )
+ throw new CliError(`${workload} still has a pending, unhealthy, or retiring task`);
+ live.push(task.taskArn!);
+ }
+ }
+ if (live.length !== desiredCount) throw new CliError(`${workload} does not have its exact stable task cohort`);
+ proof.workloads[workload] = { taskDefinition, deploymentId: deployment.id, desiredCount, taskArns: live.sort() };
+ }
+ if (canonicalJson(proof.workloads.core?.taskArns) !== canonicalJson([...cohort.taskArns].sort()))
+ throw new CliError("core task cohort changed while checking capacity");
+ for (const batch of chunks(proof.workloads.core!.taskArns, 100)) {
+ const response = awsJson<{
+ protectedTasks?: Array<{ taskArn?: string; protectionEnabled?: boolean }>;
+ failures?: unknown[];
+ }>(aws, ["ecs", "get-task-protection", "--cluster", aws.cluster, "--tasks", ...batch]);
+ const taskId = (arn: string | undefined): string | undefined => {
+ const prefix = `arn:aws:ecs:${aws.region}:${aws.accountId}:task/`;
+ if (!arn?.startsWith(prefix)) return undefined;
+ const parts = arn.slice(prefix.length).split("/");
+ if (parts.length === 1 && parts[0]) return parts[0];
+ if (parts.length === 2 && parts[0] === aws.cluster && parts[1]) return parts[1];
+ return undefined;
+ };
+ const requested = new Map(batch.map((arn) => [taskId(arn), arn]));
+ const received = response.protectedTasks?.map((task) => taskId(task.taskArn)) ?? [];
+ if (
+ requested.has(undefined) ||
+ requested.size !== batch.length ||
+ response.failures?.length ||
+ !Array.isArray(response.protectedTasks) ||
+ received.length !== batch.length ||
+ new Set(received).size !== batch.length ||
+ received.some((id) => !id || !requested.has(id)) ||
+ response.protectedTasks.some((task) => task.protectionEnabled !== false)
+ )
+ throw new CliError(
+ "capacity requires explicit unprotected status for every current core task; protection was not changed",
+ );
+ for (const taskArn of batch) proof.coreTaskProtection[taskArn] = false;
+ }
+ const finalManifest = currentDeploymentManifest(aws);
+ if (canonicalJson(finalManifest) !== canonicalJson(cohort.manifest))
+ throw new CliError("deployment manifest changed while checking capacity");
+ assertBackgroundPreparationResolved(aws, finalManifest, snapshot);
+ const finalOwnership = await readBackgroundWork(transport, cohort.deploymentId);
+ if (
+ !finalOwnership.enabled ||
+ finalOwnership.generation !== proof.generation ||
+ finalOwnership.desiredDeploymentId !== proof.desiredDeploymentId
+ )
+ throw new CliError("background ownership changed while checking capacity");
+ assertBackgroundMembersDrained(finalOwnership, cohort.deploymentId);
+ if (
+ cohort.taskArns.some(
+ (taskArn) =>
+ !finalOwnership.members.some(
+ (member) =>
+ member.taskArn === taskArn &&
+ member.deploymentId === cohort.deploymentId &&
+ !member.retired &&
+ member.state === "drained",
+ ),
+ )
+ )
+ throw new CliError("current core enrollment changed while checking capacity");
+ return proof;
}
export interface AwsBackgroundWorkPeer {
@@ -3202,13 +3398,15 @@ async function assertBackgroundCohortReplaceable(
): Promise<void> {
if (!current?.backgroundDeploymentId) return;
const status = await readBackgroundWork(awsBackgroundWorkTransport(config), current.backgroundDeploymentId);
+ if (status.enabled) assertBackgroundMembersDrained(status, current.backgroundDeploymentId);
+}
+
+function assertBackgroundMembersDrained(status: BackgroundWorkStatus, deploymentId: string): void {
if (
- status.enabled &&
- (status.desiredDeploymentId === current.backgroundDeploymentId ||
- status.members.some(
- (member) =>
- !member.retired && member.deploymentId === current.backgroundDeploymentId && member.state !== "drained",
- ))
+ status.desiredDeploymentId === deploymentId ||
+ status.members.some(
+ (member) => !member.retired && member.deploymentId === deploymentId && member.state !== "drained",
+ )
)
throw new CliError(
"pause or hand over background ownership and wait for every member to drain before replacing the current core cohort",
diff --git a/cli/templates/aws/main.tf b/cli/templates/aws/main.tf
index 5f966f1ac..7ff85f3bd 100644
--- a/cli/templates/aws/main.tf
+++ b/cli/templates/aws/main.tf
@@ -443,7 +443,7 @@ resource "aws_iam_role_policy" "github_deploy" {
{
Sid = "InspectDeploymentCanaries"
Effect = "Allow"
- Action = ["ecs:DescribeTasks"]
+ Action = ["ecs:DescribeTasks", "ecs:GetTaskProtection"]
Resource = ["arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task/${var.cluster_name}/*"]
},
{
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index 222a8f020..3c778e2de 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -16,6 +16,7 @@ import {
awsCheckLive,
awsCoreRequest,
awsBackgroundWorkStatus,
+ awsBackgroundWorkCapacity,
awsBackgroundWorkBootState,
awsBootstrapBackgroundWork,
awsRetireBackgroundWorkMembers,
@@ -375,7 +376,7 @@ else if (a.includes("ecs describe-services")) {
const service = s.services[name];
if (!service) return [];
if (name === staleName && service.previousTaskDefinition) {
- return [{ serviceName: name, status: "ACTIVE", desiredCount: service.desiredCount, runningCount: service.desiredCount, taskDefinition: service.previousTaskDefinition, deployments: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.previousTaskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }], tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];
+ return [{ serviceName: name, status: "ACTIVE", desiredCount: service.desiredCount, runningCount: service.desiredCount, taskDefinition: service.previousTaskDefinition, deployments: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.previousTaskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }], tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }], ...(s.serviceOverrides?.[name] || {}) }];
}
const deployments = draining
? [
@@ -389,7 +390,7 @@ else if (a.includes("ecs describe-services")) {
: blueGreenBakePolls
? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }]
: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: transientFailedTaskPolls && s.updated ? 1 : 0 }];
- return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: draining ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];
+ return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: draining ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments: deployments.map(deployment => ({ pendingCount: 0, ...deployment })), pendingCount: 0, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }], ...(s.serviceOverrides?.[name] || {}) }];
}), failures: names.filter((name) => !s.services[name]).map((name) => ({ arn: name, reason: "MISSING" })) }));
}
else if (a.includes("ecs list-service-deployments") && ${JSON.stringify(opts.failNativeStatusOnceAfterUpdate ?? false)} && s.updated && !s.nativeStatusFailedOnce) {
@@ -402,7 +403,7 @@ else if (a.includes("ecs list-service-deployments")) {
const name = after("--service");
const service = s.services[name];
const revision = "service-revision-for-" + String(service?.deploymentId || "").replace(/^ecs-svc\\//, "");
- const status = (s.blueGreenPolls || 0) === 1 ? "PENDING" : (s.blueGreenPolls || 0) <= ${JSON.stringify(opts.blueGreenBakePolls ?? 0)} ? "IN_PROGRESS" : "SUCCESSFUL";
+ const status = s.nativeStatus || ((s.blueGreenPolls || 0) === 1 ? "PENDING" : (s.blueGreenPolls || 0) <= ${JSON.stringify(opts.blueGreenBakePolls ?? 0)} ? "IN_PROGRESS" : "SUCCESSFUL");
const prior = service?.previousTaskDefinition ? [{ targetServiceRevisionArn: "arn:aws:ecs:us-west-2:123456789012:service-revision/acme-qm/" + name + "/previous", status: "SUCCESSFUL", createdAt: 1700000000000 }] : [];
const current = service && (s.blueGreenPolls || 0) !== 1 ? [{ targetServiceRevisionArn: "arn:aws:ecs:us-west-2:123456789012:service-revision/acme-qm/" + name + "/" + revision, status, createdAt: 1700000000000 + s.revision }] : [];
console.log(JSON.stringify({ serviceDeployments: [...prior, ...current] }));
@@ -417,6 +418,15 @@ else if (a.includes("ecs describe-service-revisions")) {
return service ? [{ serviceRevisionArn: arn, taskDefinition: arn.endsWith("/previous") ? service.previousTaskDefinition : service.taskDefinition }] : [];
}) }));
}
+else if (a.includes("ecs list-tasks") && s.taskInventory) console.log(JSON.stringify({ taskArns: (s.taskInventory[after("--service-name")] || []).filter(task => task.desiredStatus === after("--desired-status")).map(task => task.taskArn) }));
+else if (a.includes("ecs describe-tasks") && s.taskInventory) {
+ const requested = args.slice(args.indexOf("--tasks") + 1).filter(arg => arg.startsWith("arn:"));
+ console.log(JSON.stringify({tasks: Object.values(s.taskInventory).flat().filter(task => requested.includes(task.taskArn) && task.taskArn !== s.omitTask), failures: s.inventoryFailures || []}));
+}
+else if (a.includes("ecs get-task-protection")) {
+ const requested = args.slice(args.indexOf("--tasks") + 1).filter(arg => arg.startsWith("arn:"));
+ console.log(JSON.stringify(s.protectionResponse || {protectedTasks: requested.map(taskArn => ({taskArn, protectionEnabled:false})), failures:[]}));
+}
else if (a.includes("ecs list-tasks")) console.log(JSON.stringify({ taskArns: process.env.AWS_FAKE_NO_RUNNING_TASK ? [] : [...(process.env.AWS_FAKE_LARGE_ROLLOUT ? Array.from({ length: 100 }, (_, i) => "arn:aws:ecs:us-west-2:123456789012:task/old-core-" + i) : []), "arn:aws:ecs:us-west-2:123456789012:task/live-core"] }));
else if (a.includes("ecs describe-tasks") && s.stoppedTasks?.[after("--tasks")]) console.log(JSON.stringify({tasks: [s.stoppedTasks[after("--tasks")]], failures: []}));
else if (a.includes("ecs describe-tasks") && a.includes("task/old-core-")) console.log(JSON.stringify({ tasks: [] }));
@@ -5590,6 +5600,221 @@ test("AWS deployment progress rejects invalid options before AWS calls", async (
}
});
+test("inactive capacity proves exact drained unprotected cohorts without mutating deployment state", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-capacity-"));
+ const dockerBin = join(dir, "docker");
+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);
+ chmodSync(dockerBin, 0o755);
+ const base = twoServiceConfig();
+ const configured: QmConfig = { ...base, aws: { ...base.aws!, backgroundWorkControl: true } };
+ const fake = statefulAws(dir, configured);
+ const priorPath = process.env.PATH;
+ process.env.PATH = `${dir}:${priorPath}`;
+ try {
+ await awsUp(configured, dir, { yes: true });
+ const baseline = JSON.parse(readFileSync(fake.state, "utf8"));
+ const manifest = JSON.parse(
+ baseline.dynamo[`deployment/manifest/${baseline.dynamo["deployment/current"].manifestId.S}`].manifest.S,
+ );
+ baseline.taskInventory = Object.fromEntries(
+ Object.entries(baseline.services).map(([service, value]) => {
+ const state = value as { workload: string; taskDefinition: string };
+ return [
+ service,
+ [
+ {
+ taskArn: `arn:aws:ecs:us-west-2:123456789012:task/${configured.aws!.cluster}/live-${state.workload}`,
+ taskDefinitionArn: state.taskDefinition,
+ lastStatus: "RUNNING",
+ desiredStatus: "RUNNING",
+ healthStatus: "HEALTHY",
+ },
+ ],
+ ];
+ }),
+ );
+ const coreArn = baseline.taskInventory[configured.aws!.services.core!.ecsService][0].taskArn;
+ const ownership = {
+ protocol: 1,
+ enabled: true,
+ deploymentId: manifest.backgroundDeploymentId,
+ instanceId: "inactive-instance",
+ generation: 2,
+ desiredDeploymentId: "peer:active",
+ lastRequestId: "handover",
+ members: [
+ {
+ instanceId: "inactive-instance",
+ deploymentId: manifest.backgroundDeploymentId,
+ taskArn: coreArn,
+ generation: 1,
+ state: "drained",
+ ready: false,
+ retired: false,
+ },
+ ],
+ };
+ let reads = 0;
+ let changeOwnership = false;
+ globalThis.fetch = async () => {
+ reads++;
+ return new Response(
+ JSON.stringify({ ...ownership, generation: ownership.generation + (changeOwnership && reads > 1 ? 1 : 0) }),
+ );
+ };
+ const reset = () => {
+ reads = 0;
+ writeFileSync(fake.state, JSON.stringify(baseline));
+ writeFileSync(fake.log, "");
+ };
+ reset();
+ const proof = await awsBackgroundWorkCapacity(configured, dir);
+ assert.equal(proof.manifestId, manifest.id);
+ assert.equal(proof.deploymentId, manifest.backgroundDeploymentId);
+ assert.equal(proof.generation, 2);
+ assert.equal(proof.desiredDeploymentId, "peer:active");
+ assert.deepEqual(Object.keys(proof.workloads), ["core", "web-ui"]);
+ assert.deepEqual(proof.workloads.core!.taskArns, [coreArn]);
+ assert.deepEqual(proof.coreTaskProtection, { [coreArn]: false });
+ assert.equal((await awsBackgroundWorkStatus(configured, dir)).manifestId, manifest.id);
+ assert.equal(readFileSync(fake.state, "utf8"), JSON.stringify(baseline));
+ const stableLog = readFileSync(fake.log, "utf8");
+ assert.doesNotMatch(
+ stableLog,
+ /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|update-task-protection|stop-task/,
+ );
+
+ for (const state of ["admitted", "relinquished"]) {
+ ownership.members[0]!.state = state;
+ reset();
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /every member to drain/);
+ }
+ ownership.members[0]!.state = "drained";
+ ownership.desiredDeploymentId = manifest.backgroundDeploymentId;
+ reset();
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /held by another deployment/);
+ ownership.desiredDeploymentId = "peer:active";
+
+ const shortArn = coreArn.replace(`task/${configured.aws!.cluster}/`, "task/");
+ reset();
+ writeFileSync(
+ fake.state,
+ JSON.stringify({
+ ...baseline,
+ protectionResponse: { protectedTasks: [{ taskArn: shortArn, protectionEnabled: false }] },
+ }),
+ );
+ assert.deepEqual((await awsBackgroundWorkCapacity(configured, dir)).coreTaskProtection, { [coreArn]: false });
+ for (const response of [
+ ...[
+ shortArn.replace("123456789012", "999999999999"),
+ shortArn.replace("us-west-2", "us-east-1"),
+ coreArn.replace(`task/${configured.aws!.cluster}/`, "task/other-cluster/"),
+ shortArn + "unknown",
+ ].map((taskArn) => ({ protectedTasks: [{ taskArn, protectionEnabled: false }] })),
+ {
+ protectedTasks: [
+ { taskArn: coreArn, protectionEnabled: false },
+ { taskArn: shortArn, protectionEnabled: false },
+ ],
+ },
+ { protectedTasks: [{ taskArn: coreArn, protectionEnabled: true }] },
+ { protectedTasks: [{ taskArn: coreArn }] },
+ { protectedTasks: [] },
+ { protectedTasks: [{ taskArn: coreArn, protectionEnabled: false }], failures: [{ arn: "unknown" }] },
+ ]) {
+ reset();
+ writeFileSync(fake.state, JSON.stringify({ ...baseline, protectionResponse: response }));
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /explicit unprotected status/);
+ }
+ reset();
+ const unresolved = structuredClone(baseline);
+ unresolved.dynamo["deployment/background-preparation"] = {
+ preparation: {
+ S: JSON.stringify({
+ id: "unresolved",
+ previousManifestId: "other",
+ backgroundDeploymentId: "unresolved-cohort",
+ before: { tasks: {}, counts: {} },
+ }),
+ },
+ };
+ writeFileSync(fake.state, JSON.stringify(unresolved));
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /remains unresolved/);
+ unresolved.dynamo["deployment/background-preparation"].preparation.S = JSON.stringify({
+ id: manifest.id,
+ backgroundDeploymentId: manifest.backgroundDeploymentId,
+ before: { tasks: {}, counts: {} },
+ });
+ writeFileSync(fake.state, JSON.stringify(unresolved));
+ await awsBackgroundWorkCapacity(configured, dir);
+ assert.equal(readFileSync(fake.state, "utf8"), JSON.stringify(unresolved));
+
+ reset();
+ const retiringCurrent = structuredClone(baseline);
+ retiringCurrent.taskInventory[configured.aws!.services["web-ui"]!.ecsService][0].desiredStatus = "STOPPED";
+ writeFileSync(fake.state, JSON.stringify(retiringCurrent));
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /retiring task/);
+
+ for (const workload of ["core", "web-ui"]) {
+ reset();
+ const lingering = structuredClone(baseline);
+ const service = configured.aws!.services[workload]!.ecsService;
+ lingering.taskInventory[service].push({
+ ...lingering.taskInventory[service][0],
+ taskArn: `arn:aws:ecs:us-west-2:123456789012:task/old-${workload}`,
+ desiredStatus: "STOPPED",
+ lastStatus: "STOPPING",
+ });
+ writeFileSync(fake.state, JSON.stringify(lingering));
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /retiring task/);
+ }
+ reset();
+ writeFileSync(
+ fake.state,
+ JSON.stringify({
+ ...baseline,
+ omitTask: baseline.taskInventory[configured.aws!.services["web-ui"]!.ecsService][0].taskArn,
+ }),
+ );
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /complete task inventory/);
+ reset();
+ writeFileSync(
+ fake.state,
+ JSON.stringify({
+ ...baseline,
+ serviceOverrides: {
+ [configured.aws!.services["web-ui"]!.ecsService]: { deploymentConfiguration: { strategy: "BLUE_GREEN" } },
+ },
+ nativeStatus: "IN_PROGRESS",
+ }),
+ );
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /stable deployment capacity/);
+ for (const pendingCount of [null, 1]) {
+ reset();
+ writeFileSync(
+ fake.state,
+ JSON.stringify({
+ ...baseline,
+ serviceOverrides: { [configured.aws!.services.core!.ecsService]: { pendingCount } },
+ }),
+ );
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /stable deployment capacity/);
+ }
+ reset();
+ changeOwnership = true;
+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /ownership changed/);
+ assert.doesNotMatch(
+ readFileSync(fake.log, "utf8"),
+ /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|update-task-protection|stop-task/,
+ );
+ } finally {
+ process.env.PATH = priorPath;
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
test("controlled AWS cohorts bind immutable identities and hand over without ECS replacement", async () => {
const dir = mkdtempSync(join(tmpdir(), "qm-controlled-background-"));
const dockerBin = join(dir, "docker");
diff --git a/cli/test/terraform.test.ts b/cli/test/terraform.test.ts
index 08192df6e..902f2db8a 100644
--- a/cli/test/terraform.test.ts
+++ b/cli/test/terraform.test.ts
@@ -350,6 +350,8 @@ test("the deploy role can run and inspect only stack-scoped deployment canaries"
const inspect = policy.match(/Sid\s*= "InspectDeploymentCanaries"([\s\S]*?)\n\s*\},/)?.[1] ?? "";
assert.match(inspect, /ecs:DescribeTasks/);
+ assert.match(inspect, /ecs:GetTaskProtection/);
+ assert.doesNotMatch(policy, /ecs:UpdateTaskProtection/);
assert.match(inspect, /task\/\$\{var\.cluster_name\}\/\*/);
assert.doesNotMatch(inspect, /Resource\s*= "\*"/);
});