Multiplayer agent harness for work.
Prove inactive AWS capacity before release admission (#1296)
Regan Bell3068269a880d
diff --git a/cli/README.md b/cli/README.mdindex 4308beced..eb47506bc 100644--- a/cli/README.md+++ b/cli/README.md@@ -100,6 +100,20 @@ return its recorded boot flag and optional deployment identity. It returnsboot flags or identities. Deployment wrappers can preserve the boot environmentwhile changing durable ownership independently.+The exported `awsBackgroundWorkCapacity(config, configDir, candidatePath?)` proves+that an inactive controlled stack is currently reusable. It requires another+owner, fully drained current membership, stable native deployments and exact task+inventories for every workload, resolved deployment preparation, and explicitly+disabled protection on every current core task. It never changes task protection+or deployment state. The deploy role needs `ecs:GetTaskProtection` on its tasks.+The result binds the manifest and deployment identities, ownership generation,+workload task definitions, native deployment IDs, task ARNs, and protection proof.+`awsBackgroundWorkStatus` also returns the current manifest ID for an active-owner+proof. A release coordinator can combine both snapshots with immutable candidate+provenance and compare them again under its production lock before any mutation.+This is a point-in-time check, not a reservation: intervening maintenance or task+replacement invalidates the proof and must block promotion.+Live checks use the active ownership cohort's authenticated canary endpoint toverify a real model reply, session persistence, generated title, error records,session cleanup, and database catalog health. The CLI proves the exact healthydiff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 9723c2174..df6e3ba36 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -1111,6 +1111,7 @@ interface EcsDeploymentState {taskDefinition?: string;rolloutState?: string;runningCount?: number;+ pendingCount?: number;failedTasks?: number;}@@ -1119,6 +1120,7 @@ interface EcsServiceState {status?: string;desiredCount?: number;runningCount?: number;+ pendingCount?: number;taskDefinition?: string;deploymentConfiguration?: { strategy?: string };networkConfiguration?: {@@ -1284,11 +1286,11 @@ function clearBackgroundPreparation(aws: AwsConfig): void {]);}-function reconcileBackgroundPreparation(+function assertBackgroundPreparationResolved(aws: AwsConfig,current: DeploymentManifest | undefined,before: BackgroundDeploymentPreparation["before"],-): void {+): boolean {const response = awsJson<{ Item?: Record<string, { S?: string }> }>(aws, ["dynamodb","get-item",@@ -1298,7 +1300,7 @@ function reconcileBackgroundPreparation(JSON.stringify({ lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } }),"--consistent-read",]);- if (!response.Item) return;+ if (!response.Item) return false;let preparation: BackgroundDeploymentPreparation;try {preparation = JSON.parse(response.Item.preparation?.S ?? "");@@ -1323,7 +1325,15 @@ function reconcileBackgroundPreparation(throw new CliError(`background deployment ${preparation.id} remains unresolved; reconcile its recorded tasks before retrying instead of allocating another cohort`,);- clearBackgroundPreparation(aws);+ return true;+}++function reconcileBackgroundPreparation(+ aws: AwsConfig,+ current: DeploymentManifest | undefined,+ before: BackgroundDeploymentPreparation["before"],+): void {+ if (assertBackgroundPreparationResolved(aws, current, before)) clearBackgroundPreparation(aws);}function prepareBackgroundDeployment(aws: AwsConfig, preparation: BackgroundDeploymentPreparation): void {@@ -2964,6 +2974,7 @@ async function awsBackgroundCohort(config: QmConfig,configDir: string,candidatePath?: string,+ waitForReady = true,): Promise<AwsBackgroundCohort> {const { aws, workloads } = awsTopology(config, configDir);if (!aws.backgroundWorkControl)@@ -3007,9 +3018,10 @@ async function awsBackgroundCohort(}const count = before.counts.core;if (!count) throw new CliError("background ownership requires running core capacity");- await awaitServiceTargets(config, {- core: { taskDefinition: before.tasks.core!, desiredCount: count, waitForDrain: false },- });+ if (waitForReady)+ await awaitServiceTargets(config, {+ core: { taskDefinition: before.tasks.core!, desiredCount: count, waitForDrain: false },+ });const taskArns =awsJson<{ taskArns?: string[] }>(aws, ["ecs",@@ -3050,6 +3062,7 @@ export async function awsBackgroundWorkStatus(status: BackgroundWorkStatus;deploymentId: string;taskArns: string[];+ manifestId: string;}> {assertAwsCallerAccount(requireAws(config));const cohort = await awsBackgroundCohort(config, configDir, candidatePath);@@ -3062,7 +3075,190 @@ export async function awsBackgroundWorkStatus()throw new CliError("a running core task has not enrolled in the ownership protocol");}- return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns };+ return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns, manifestId: cohort.manifest.id };+}++export interface AwsBackgroundWorkCapacity {+ manifestId: string;+ deploymentId: string;+ generation: number;+ desiredDeploymentId: string;+ coreTaskProtection: Record<string, false>;+ workloads: Record<string, { taskDefinition: string; deploymentId: string; desiredCount: number; taskArns: string[] }>;+}++export async function awsBackgroundWorkCapacity(+ config: QmConfig,+ configDir: string,+ candidatePath?: string,+): Promise<AwsBackgroundWorkCapacity> {+ const { aws, workloads } = awsTopology(config, configDir);+ assertAwsCallerAccount(aws);+ const cohort = await awsBackgroundCohort(config, configDir, candidatePath, false);+ const transport = awsBackgroundWorkTransport(config);+ const ownership = await readBackgroundWork(transport, cohort.deploymentId);+ if (!ownership.enabled || !ownership.desiredDeploymentId || ownership.desiredDeploymentId === cohort.deploymentId)+ throw new CliError("capacity requires enabled background ownership held by another deployment");+ assertBackgroundMembersDrained(ownership, cohort.deploymentId);+ if (+ cohort.taskArns.some(+ (taskArn) =>+ !ownership.members.some(+ (member) =>+ member.taskArn === taskArn &&+ member.deploymentId === cohort.deploymentId &&+ !member.retired &&+ member.state === "drained",+ ),+ )+ )+ throw new CliError("capacity requires every current core task to be enrolled and drained");+ const states = describedServices(config, workloads);+ assertOwnedServices(config, states, workloads);+ const snapshot = serviceSnapshotFromStates(states, workloads);+ assertBackgroundPreparationResolved(aws, cohort.manifest, snapshot);+ const proof: AwsBackgroundWorkCapacity = {+ manifestId: cohort.manifest.id,+ deploymentId: cohort.deploymentId,+ generation: ownership.generation,+ desiredDeploymentId: ownership.desiredDeploymentId,+ coreTaskProtection: {},+ workloads: {},+ };+ for (const workload of [...workloads].sort()) {+ const state = states.get(workload)!;+ const taskDefinition = cohort.manifest.tasks[workload]!;+ const desiredCount = cohort.manifest.counts?.[workload];+ const primary = state.deployments?.filter((deployment) => deployment.status === "PRIMARY") ?? [];+ const deployment = primary[0];+ if (+ typeof desiredCount !== "number" ||+ !Number.isSafeInteger(desiredCount) ||+ desiredCount < 0 ||+ state.status !== "ACTIVE" ||+ state.taskDefinition !== taskDefinition ||+ state.desiredCount !== desiredCount ||+ state.runningCount !== desiredCount ||+ state.pendingCount !== 0 ||+ primary.length !== 1 ||+ !deployment?.id ||+ deployment.taskDefinition !== taskDefinition ||+ deployment.runningCount !== desiredCount ||+ state.deployments?.some((item) => item.pendingCount !== 0 || (item !== deployment && item.runningCount !== 0)) ||+ (state.deploymentConfiguration?.strategy === "BLUE_GREEN"+ ? nativeBlueGreenStatus(config, workload, taskDefinition) !== "SUCCESSFUL"+ : deployment.rolloutState !== "COMPLETED")+ )+ throw new CliError(`${workload} has not reached stable deployment capacity`);+ const listed = new Set<string>();+ for (const desiredStatus of ["RUNNING", "STOPPED"]) {+ const response = awsJson<{ taskArns?: string[] }>(aws, [+ "ecs",+ "list-tasks",+ "--cluster",+ aws.cluster,+ "--service-name",+ aws.services[workload]!.ecsService,+ "--desired-status",+ desiredStatus,+ ]);+ if (!Array.isArray(response.taskArns) || response.taskArns.some((arn) => typeof arn !== "string" || !arn))+ throw new CliError(`${workload} task inventory is unavailable`);+ for (const arn of response.taskArns) listed.add(arn);+ }+ const live: string[] = [];+ for (const batch of chunks([...listed], 100)) {+ const response = awsJson<{+ tasks?: Array<{+ taskArn?: string;+ taskDefinitionArn?: string;+ lastStatus?: string;+ desiredStatus?: string;+ healthStatus?: string;+ }>;+ failures?: unknown[];+ }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);+ if (+ response.failures?.length ||+ !Array.isArray(response.tasks) ||+ response.tasks.length !== batch.length ||+ new Set(response.tasks.map((task) => task.taskArn)).size !== batch.length ||+ response.tasks.some((task) => !task.taskArn || !batch.includes(task.taskArn))+ )+ throw new CliError(`${workload} capacity requires complete task inventory`);+ for (const task of response.tasks) {+ if (task.lastStatus === "STOPPED") continue;+ if (+ task.lastStatus !== "RUNNING" ||+ task.desiredStatus !== "RUNNING" ||+ task.taskDefinitionArn !== taskDefinition ||+ (task.healthStatus !== "HEALTHY" && (isServiceName(workload) || task.healthStatus !== "UNKNOWN"))+ )+ throw new CliError(`${workload} still has a pending, unhealthy, or retiring task`);+ live.push(task.taskArn!);+ }+ }+ if (live.length !== desiredCount) throw new CliError(`${workload} does not have its exact stable task cohort`);+ proof.workloads[workload] = { taskDefinition, deploymentId: deployment.id, desiredCount, taskArns: live.sort() };+ }+ if (canonicalJson(proof.workloads.core?.taskArns) !== canonicalJson([...cohort.taskArns].sort()))+ throw new CliError("core task cohort changed while checking capacity");+ for (const batch of chunks(proof.workloads.core!.taskArns, 100)) {+ const response = awsJson<{+ protectedTasks?: Array<{ taskArn?: string; protectionEnabled?: boolean }>;+ failures?: unknown[];+ }>(aws, ["ecs", "get-task-protection", "--cluster", aws.cluster, "--tasks", ...batch]);+ const taskId = (arn: string | undefined): string | undefined => {+ const prefix = `arn:aws:ecs:${aws.region}:${aws.accountId}:task/`;+ if (!arn?.startsWith(prefix)) return undefined;+ const parts = arn.slice(prefix.length).split("/");+ if (parts.length === 1 && parts[0]) return parts[0];+ if (parts.length === 2 && parts[0] === aws.cluster && parts[1]) return parts[1];+ return undefined;+ };+ const requested = new Map(batch.map((arn) => [taskId(arn), arn]));+ const received = response.protectedTasks?.map((task) => taskId(task.taskArn)) ?? [];+ if (+ requested.has(undefined) ||+ requested.size !== batch.length ||+ response.failures?.length ||+ !Array.isArray(response.protectedTasks) ||+ received.length !== batch.length ||+ new Set(received).size !== batch.length ||+ received.some((id) => !id || !requested.has(id)) ||+ response.protectedTasks.some((task) => task.protectionEnabled !== false)+ )+ throw new CliError(+ "capacity requires explicit unprotected status for every current core task; protection was not changed",+ );+ for (const taskArn of batch) proof.coreTaskProtection[taskArn] = false;+ }+ const finalManifest = currentDeploymentManifest(aws);+ if (canonicalJson(finalManifest) !== canonicalJson(cohort.manifest))+ throw new CliError("deployment manifest changed while checking capacity");+ assertBackgroundPreparationResolved(aws, finalManifest, snapshot);+ const finalOwnership = await readBackgroundWork(transport, cohort.deploymentId);+ if (+ !finalOwnership.enabled ||+ finalOwnership.generation !== proof.generation ||+ finalOwnership.desiredDeploymentId !== proof.desiredDeploymentId+ )+ throw new CliError("background ownership changed while checking capacity");+ assertBackgroundMembersDrained(finalOwnership, cohort.deploymentId);+ if (+ cohort.taskArns.some(+ (taskArn) =>+ !finalOwnership.members.some(+ (member) =>+ member.taskArn === taskArn &&+ member.deploymentId === cohort.deploymentId &&+ !member.retired &&+ member.state === "drained",+ ),+ )+ )+ throw new CliError("current core enrollment changed while checking capacity");+ return proof;}export interface AwsBackgroundWorkPeer {@@ -3202,13 +3398,15 @@ async function assertBackgroundCohortReplaceable(): Promise<void> {if (!current?.backgroundDeploymentId) return;const status = await readBackgroundWork(awsBackgroundWorkTransport(config), current.backgroundDeploymentId);+ if (status.enabled) assertBackgroundMembersDrained(status, current.backgroundDeploymentId);+}++function assertBackgroundMembersDrained(status: BackgroundWorkStatus, deploymentId: string): void {if (- status.enabled &&- (status.desiredDeploymentId === current.backgroundDeploymentId ||- status.members.some(- (member) =>- !member.retired && member.deploymentId === current.backgroundDeploymentId && member.state !== "drained",- ))+ status.desiredDeploymentId === deploymentId ||+ status.members.some(+ (member) => !member.retired && member.deploymentId === deploymentId && member.state !== "drained",+ ))throw new CliError("pause or hand over background ownership and wait for every member to drain before replacing the current core cohort",diff --git a/cli/templates/aws/main.tf b/cli/templates/aws/main.tfindex 5f966f1ac..7ff85f3bd 100644--- a/cli/templates/aws/main.tf+++ b/cli/templates/aws/main.tf@@ -443,7 +443,7 @@ resource "aws_iam_role_policy" "github_deploy" {{Sid = "InspectDeploymentCanaries"Effect = "Allow"- Action = ["ecs:DescribeTasks"]+ Action = ["ecs:DescribeTasks", "ecs:GetTaskProtection"]Resource = ["arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task/${var.cluster_name}/*"]},{diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 222a8f020..3c778e2de 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -16,6 +16,7 @@ import {awsCheckLive,awsCoreRequest,awsBackgroundWorkStatus,+ awsBackgroundWorkCapacity,awsBackgroundWorkBootState,awsBootstrapBackgroundWork,awsRetireBackgroundWorkMembers,@@ -375,7 +376,7 @@ else if (a.includes("ecs describe-services")) {const service = s.services[name];if (!service) return [];if (name === staleName && service.previousTaskDefinition) {- return [{ serviceName: name, status: "ACTIVE", desiredCount: service.desiredCount, runningCount: service.desiredCount, taskDefinition: service.previousTaskDefinition, deployments: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.previousTaskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }], tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];+ return [{ serviceName: name, status: "ACTIVE", desiredCount: service.desiredCount, runningCount: service.desiredCount, taskDefinition: service.previousTaskDefinition, deployments: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.previousTaskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }], tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }], ...(s.serviceOverrides?.[name] || {}) }];}const deployments = draining? [@@ -389,7 +390,7 @@ else if (a.includes("ecs describe-services")) {: blueGreenBakePolls? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }]: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: transientFailedTaskPolls && s.updated ? 1 : 0 }];- return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: draining ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];+ return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: draining ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments: deployments.map(deployment => ({ pendingCount: 0, ...deployment })), pendingCount: 0, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }], ...(s.serviceOverrides?.[name] || {}) }];}), failures: names.filter((name) => !s.services[name]).map((name) => ({ arn: name, reason: "MISSING" })) }));}else if (a.includes("ecs list-service-deployments") && ${JSON.stringify(opts.failNativeStatusOnceAfterUpdate ?? false)} && s.updated && !s.nativeStatusFailedOnce) {@@ -402,7 +403,7 @@ else if (a.includes("ecs list-service-deployments")) {const name = after("--service");const service = s.services[name];const revision = "service-revision-for-" + String(service?.deploymentId || "").replace(/^ecs-svc\\//, "");- const status = (s.blueGreenPolls || 0) === 1 ? "PENDING" : (s.blueGreenPolls || 0) <= ${JSON.stringify(opts.blueGreenBakePolls ?? 0)} ? "IN_PROGRESS" : "SUCCESSFUL";+ const status = s.nativeStatus || ((s.blueGreenPolls || 0) === 1 ? "PENDING" : (s.blueGreenPolls || 0) <= ${JSON.stringify(opts.blueGreenBakePolls ?? 0)} ? "IN_PROGRESS" : "SUCCESSFUL");const prior = service?.previousTaskDefinition ? [{ targetServiceRevisionArn: "arn:aws:ecs:us-west-2:123456789012:service-revision/acme-qm/" + name + "/previous", status: "SUCCESSFUL", createdAt: 1700000000000 }] : [];const current = service && (s.blueGreenPolls || 0) !== 1 ? [{ targetServiceRevisionArn: "arn:aws:ecs:us-west-2:123456789012:service-revision/acme-qm/" + name + "/" + revision, status, createdAt: 1700000000000 + s.revision }] : [];console.log(JSON.stringify({ serviceDeployments: [...prior, ...current] }));@@ -417,6 +418,15 @@ else if (a.includes("ecs describe-service-revisions")) {return service ? [{ serviceRevisionArn: arn, taskDefinition: arn.endsWith("/previous") ? service.previousTaskDefinition : service.taskDefinition }] : [];}) }));}+else if (a.includes("ecs list-tasks") && s.taskInventory) console.log(JSON.stringify({ taskArns: (s.taskInventory[after("--service-name")] || []).filter(task => task.desiredStatus === after("--desired-status")).map(task => task.taskArn) }));+else if (a.includes("ecs describe-tasks") && s.taskInventory) {+ const requested = args.slice(args.indexOf("--tasks") + 1).filter(arg => arg.startsWith("arn:"));+ console.log(JSON.stringify({tasks: Object.values(s.taskInventory).flat().filter(task => requested.includes(task.taskArn) && task.taskArn !== s.omitTask), failures: s.inventoryFailures || []}));+}+else if (a.includes("ecs get-task-protection")) {+ const requested = args.slice(args.indexOf("--tasks") + 1).filter(arg => arg.startsWith("arn:"));+ console.log(JSON.stringify(s.protectionResponse || {protectedTasks: requested.map(taskArn => ({taskArn, protectionEnabled:false})), failures:[]}));+}else if (a.includes("ecs list-tasks")) console.log(JSON.stringify({ taskArns: process.env.AWS_FAKE_NO_RUNNING_TASK ? [] : [...(process.env.AWS_FAKE_LARGE_ROLLOUT ? Array.from({ length: 100 }, (_, i) => "arn:aws:ecs:us-west-2:123456789012:task/old-core-" + i) : []), "arn:aws:ecs:us-west-2:123456789012:task/live-core"] }));else if (a.includes("ecs describe-tasks") && s.stoppedTasks?.[after("--tasks")]) console.log(JSON.stringify({tasks: [s.stoppedTasks[after("--tasks")]], failures: []}));else if (a.includes("ecs describe-tasks") && a.includes("task/old-core-")) console.log(JSON.stringify({ tasks: [] }));@@ -5590,6 +5600,221 @@ test("AWS deployment progress rejects invalid options before AWS calls", async (}});+test("inactive capacity proves exact drained unprotected cohorts without mutating deployment state", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-capacity-"));+ const dockerBin = join(dir, "docker");+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);+ chmodSync(dockerBin, 0o755);+ const base = twoServiceConfig();+ const configured: QmConfig = { ...base, aws: { ...base.aws!, backgroundWorkControl: true } };+ const fake = statefulAws(dir, configured);+ const priorPath = process.env.PATH;+ process.env.PATH = `${dir}:${priorPath}`;+ try {+ await awsUp(configured, dir, { yes: true });+ const baseline = JSON.parse(readFileSync(fake.state, "utf8"));+ const manifest = JSON.parse(+ baseline.dynamo[`deployment/manifest/${baseline.dynamo["deployment/current"].manifestId.S}`].manifest.S,+ );+ baseline.taskInventory = Object.fromEntries(+ Object.entries(baseline.services).map(([service, value]) => {+ const state = value as { workload: string; taskDefinition: string };+ return [+ service,+ [+ {+ taskArn: `arn:aws:ecs:us-west-2:123456789012:task/${configured.aws!.cluster}/live-${state.workload}`,+ taskDefinitionArn: state.taskDefinition,+ lastStatus: "RUNNING",+ desiredStatus: "RUNNING",+ healthStatus: "HEALTHY",+ },+ ],+ ];+ }),+ );+ const coreArn = baseline.taskInventory[configured.aws!.services.core!.ecsService][0].taskArn;+ const ownership = {+ protocol: 1,+ enabled: true,+ deploymentId: manifest.backgroundDeploymentId,+ instanceId: "inactive-instance",+ generation: 2,+ desiredDeploymentId: "peer:active",+ lastRequestId: "handover",+ members: [+ {+ instanceId: "inactive-instance",+ deploymentId: manifest.backgroundDeploymentId,+ taskArn: coreArn,+ generation: 1,+ state: "drained",+ ready: false,+ retired: false,+ },+ ],+ };+ let reads = 0;+ let changeOwnership = false;+ globalThis.fetch = async () => {+ reads++;+ return new Response(+ JSON.stringify({ ...ownership, generation: ownership.generation + (changeOwnership && reads > 1 ? 1 : 0) }),+ );+ };+ const reset = () => {+ reads = 0;+ writeFileSync(fake.state, JSON.stringify(baseline));+ writeFileSync(fake.log, "");+ };+ reset();+ const proof = await awsBackgroundWorkCapacity(configured, dir);+ assert.equal(proof.manifestId, manifest.id);+ assert.equal(proof.deploymentId, manifest.backgroundDeploymentId);+ assert.equal(proof.generation, 2);+ assert.equal(proof.desiredDeploymentId, "peer:active");+ assert.deepEqual(Object.keys(proof.workloads), ["core", "web-ui"]);+ assert.deepEqual(proof.workloads.core!.taskArns, [coreArn]);+ assert.deepEqual(proof.coreTaskProtection, { [coreArn]: false });+ assert.equal((await awsBackgroundWorkStatus(configured, dir)).manifestId, manifest.id);+ assert.equal(readFileSync(fake.state, "utf8"), JSON.stringify(baseline));+ const stableLog = readFileSync(fake.log, "utf8");+ assert.doesNotMatch(+ stableLog,+ /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|update-task-protection|stop-task/,+ );++ for (const state of ["admitted", "relinquished"]) {+ ownership.members[0]!.state = state;+ reset();+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /every member to drain/);+ }+ ownership.members[0]!.state = "drained";+ ownership.desiredDeploymentId = manifest.backgroundDeploymentId;+ reset();+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /held by another deployment/);+ ownership.desiredDeploymentId = "peer:active";++ const shortArn = coreArn.replace(`task/${configured.aws!.cluster}/`, "task/");+ reset();+ writeFileSync(+ fake.state,+ JSON.stringify({+ ...baseline,+ protectionResponse: { protectedTasks: [{ taskArn: shortArn, protectionEnabled: false }] },+ }),+ );+ assert.deepEqual((await awsBackgroundWorkCapacity(configured, dir)).coreTaskProtection, { [coreArn]: false });+ for (const response of [+ ...[+ shortArn.replace("123456789012", "999999999999"),+ shortArn.replace("us-west-2", "us-east-1"),+ coreArn.replace(`task/${configured.aws!.cluster}/`, "task/other-cluster/"),+ shortArn + "unknown",+ ].map((taskArn) => ({ protectedTasks: [{ taskArn, protectionEnabled: false }] })),+ {+ protectedTasks: [+ { taskArn: coreArn, protectionEnabled: false },+ { taskArn: shortArn, protectionEnabled: false },+ ],+ },+ { protectedTasks: [{ taskArn: coreArn, protectionEnabled: true }] },+ { protectedTasks: [{ taskArn: coreArn }] },+ { protectedTasks: [] },+ { protectedTasks: [{ taskArn: coreArn, protectionEnabled: false }], failures: [{ arn: "unknown" }] },+ ]) {+ reset();+ writeFileSync(fake.state, JSON.stringify({ ...baseline, protectionResponse: response }));+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /explicit unprotected status/);+ }+ reset();+ const unresolved = structuredClone(baseline);+ unresolved.dynamo["deployment/background-preparation"] = {+ preparation: {+ S: JSON.stringify({+ id: "unresolved",+ previousManifestId: "other",+ backgroundDeploymentId: "unresolved-cohort",+ before: { tasks: {}, counts: {} },+ }),+ },+ };+ writeFileSync(fake.state, JSON.stringify(unresolved));+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /remains unresolved/);+ unresolved.dynamo["deployment/background-preparation"].preparation.S = JSON.stringify({+ id: manifest.id,+ backgroundDeploymentId: manifest.backgroundDeploymentId,+ before: { tasks: {}, counts: {} },+ });+ writeFileSync(fake.state, JSON.stringify(unresolved));+ await awsBackgroundWorkCapacity(configured, dir);+ assert.equal(readFileSync(fake.state, "utf8"), JSON.stringify(unresolved));++ reset();+ const retiringCurrent = structuredClone(baseline);+ retiringCurrent.taskInventory[configured.aws!.services["web-ui"]!.ecsService][0].desiredStatus = "STOPPED";+ writeFileSync(fake.state, JSON.stringify(retiringCurrent));+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /retiring task/);++ for (const workload of ["core", "web-ui"]) {+ reset();+ const lingering = structuredClone(baseline);+ const service = configured.aws!.services[workload]!.ecsService;+ lingering.taskInventory[service].push({+ ...lingering.taskInventory[service][0],+ taskArn: `arn:aws:ecs:us-west-2:123456789012:task/old-${workload}`,+ desiredStatus: "STOPPED",+ lastStatus: "STOPPING",+ });+ writeFileSync(fake.state, JSON.stringify(lingering));+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /retiring task/);+ }+ reset();+ writeFileSync(+ fake.state,+ JSON.stringify({+ ...baseline,+ omitTask: baseline.taskInventory[configured.aws!.services["web-ui"]!.ecsService][0].taskArn,+ }),+ );+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /complete task inventory/);+ reset();+ writeFileSync(+ fake.state,+ JSON.stringify({+ ...baseline,+ serviceOverrides: {+ [configured.aws!.services["web-ui"]!.ecsService]: { deploymentConfiguration: { strategy: "BLUE_GREEN" } },+ },+ nativeStatus: "IN_PROGRESS",+ }),+ );+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /stable deployment capacity/);+ for (const pendingCount of [null, 1]) {+ reset();+ writeFileSync(+ fake.state,+ JSON.stringify({+ ...baseline,+ serviceOverrides: { [configured.aws!.services.core!.ecsService]: { pendingCount } },+ }),+ );+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /stable deployment capacity/);+ }+ reset();+ changeOwnership = true;+ await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /ownership changed/);+ assert.doesNotMatch(+ readFileSync(fake.log, "utf8"),+ /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|update-task-protection|stop-task/,+ );+ } finally {+ process.env.PATH = priorPath;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});+test("controlled AWS cohorts bind immutable identities and hand over without ECS replacement", async () => {const dir = mkdtempSync(join(tmpdir(), "qm-controlled-background-"));const dockerBin = join(dir, "docker");diff --git a/cli/test/terraform.test.ts b/cli/test/terraform.test.tsindex 08192df6e..902f2db8a 100644--- a/cli/test/terraform.test.ts+++ b/cli/test/terraform.test.ts@@ -350,6 +350,8 @@ test("the deploy role can run and inspect only stack-scoped deployment canaries"const inspect = policy.match(/Sid\s*= "InspectDeploymentCanaries"([\s\S]*?)\n\s*\},/)?.[1] ?? "";assert.match(inspect, /ecs:DescribeTasks/);+ assert.match(inspect, /ecs:GetTaskProtection/);+ assert.doesNotMatch(policy, /ecs:UpdateTaskProtection/);assert.match(inspect, /task\/\$\{var\.cluster_name\}\/\*/);assert.doesNotMatch(inspect, /Resource\s*= "\*"/);});