Multiplayer agent harness for work.
Drop ECS task protection and the linger-until-drained deploy wait (#2052)
Regan Bell6ca4d2c995b0
diff --git a/cli/README.md b/cli/README.mdindex 1fd2489db..f72b9229f 100644--- a/cli/README.md+++ b/cli/README.md@@ -85,6 +85,16 @@ configuration only, so it prints that timestamp as the matching data restorepoint (`aws rds restore-db-instance-to-point-in-time`);`aws.predeployDbSnapshot: false` opts out.+On AWS, `up` waits for every rolling service to reach `COMPLETED` with a single+deployment, polling until `QM_AWS_ROLLOUT_DEADLINE_MS` (default 20 minutes); a+timeout rolls the changed services back. ECS stops the old tasks once they exit+or their `stopTimeout` elapses (core drains for `SHUTDOWN_DRAIN_MS`, 10 s by+default), and a turn cut off there resumes on the new tasks. Core images older+than this release protect their task while a turn runs, so the first `qm up`+that replaces them cannot finish before their longest in-flight turn ends+(protection expires after 60 minutes at most); run that one deploy with a+longer deadline or when no long turn is in flight.+AWS deployments can opt into durable background ownership with`aws.backgroundWorkControl: true`. Deploy protocol-capable core images to everyparticipating stack before bootstrapping ownership. The CLI allocates a unique@@ -111,11 +121,10 @@ while changing durable ownership independently.The exported `awsBackgroundWorkCapacity(config, configDir, candidatePath?)` provesthat an inactive controlled stack is currently reusable. It requires anotherowner, a current membership that has relinquished, stable native deployments and-exact task inventories for every workload, resolved deployment preparation, and-explicitly disabled protection on every current core task. It never changes task protection-or deployment state. The deploy role needs `ecs:GetTaskProtection` on its tasks.-The result binds the manifest and deployment identities, ownership generation,-workload task definitions, native deployment IDs, task ARNs, and protection proof.+exact task inventories for every workload, and resolved deployment preparation. It+never changes deployment state. The result binds the manifest and deployment+identities, ownership generation, workload task definitions, native deployment IDs,+and task ARNs.`awsBackgroundWorkStatus` also returns the current manifest ID for an active-ownerproof. A release coordinator can combine both snapshots with immutable candidateprovenance and compare them again under its production lock before any mutation.@@ -137,8 +146,8 @@ requests without restarting ECS tasks. Activation waits for prior owners to stopclaiming and every expected task to finish activation. Pausing stops new claims;in-flight turns can continue draining. Replacing or rolling back the active corecohort requires an explicit pause or handover and proof that every member has-relinquished first; members still finishing admitted work keep running under-task protection beside the replacement tasks until they drain. A demotion+relinquished first; ECS then stops the old tasks, and any turn still running+there resumes on the replacement tasks. A demotionrefuses to pause a different current owner. Unresponsive members are neverassumed dead: `awsRetireBackgroundWorkMembers` requires exact instance, task ARN,and generation identities plus ECS evidence that each task stopped. This recoverydiff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 6e7a304c5..57d253547 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -1478,7 +1478,6 @@ interface RolloutTarget {taskDefinition: string;desiredCount: number;deploymentId?: string;- waitForDrain?: boolean;}async function awaitServiceTargets(@@ -1488,7 +1487,6 @@ async function awaitServiceTargets(): Promise<void> {const pollMs = envNum("QM_AWS_ROLLOUT_POLL_MS", 15_000);const deadline = Date.now() + (timeoutMs ?? envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 20 * 60_000));- let healthyStreak = 0;let failurePolls = new Map<string, number>();let describeFailures = 0;const failing = new Map<string, { polls: number; baseRunning: number }>();@@ -1516,7 +1514,6 @@ async function awaitServiceTargets(const fail = (workload: string, kind: string, message: string): void =>void failures.push({ identity: `${workload}:${kind}`, message });const waiting: string[] = [];- let draining = 0;const failingNow = new Set<string>();for (const [workload, want] of Object.entries(expected)) {const state = states.get(workload)!;@@ -1572,12 +1569,6 @@ async function awaitServiceTargets(}} else if (running < want.desiredCount) {waiting.push(`${workload} (${running}/${want.desiredCount})`);- } else if (- want.waitForDrain &&- (deployment.rolloutState !== "COMPLETED" || state.deployments?.length !== 1) &&- state.deploymentConfiguration?.strategy !== "BLUE_GREEN"- ) {- waiting.push(`${workload} (prior tasks are still draining)`);} else if (state.deploymentConfiguration?.strategy === "BLUE_GREEN") {const nativeStatus = nativeStatuses.get(workload);if (nativeStatus === "SUCCESSFUL") continue;@@ -1586,15 +1577,14 @@ async function awaitServiceTargets(} else {waiting.push(`${workload} (blue/green traffic shift or bake is still in progress)`);}- } else if (deployment.rolloutState !== "COMPLETED") {- draining += 1;+ } else if (deployment.rolloutState !== "COMPLETED" || state.deployments?.length !== 1) {+ waiting.push(`${workload} (prior tasks are still stopping)`);}}for (const workload of failing.keys()) {if (!failingNow.has(workload)) failing.delete(workload);}if (failures.length) {- healthyStreak = 0;const next = new Map<string, number>();const confirmed: string[] = [];for (const failure of failures) {@@ -1614,23 +1604,11 @@ async function awaitServiceTargets(continue;}failurePolls = new Map();- if (waiting.length === 0) {- if (draining === 0) return;- healthyStreak += 1;- if (healthyStreak >= 3) return;- } else {- healthyStreak = 0;- }+ if (waiting.length === 0) return;if (Date.now() > deadline) {- throw new CliError(- `timed out waiting for the AWS rollout: ${waiting.length ? `still starting ${waiting.join(", ")}` : "confirming health while old tasks drain"}`,- );+ throw new CliError(`timed out waiting for the AWS rollout: ${waiting.join(", ")}`);}- note(- waiting.length- ? `waiting on ${waiting.join(", ")}`- : `new tasks healthy; old task(s) still draining protected turns (${healthyStreak}/3)`,- );+ note(`waiting on ${waiting.join(", ")}`);await sleep(pollMs);}}@@ -1979,8 +1957,6 @@ async function applyServiceTargets(targets: Record<string, string>,desiredCounts?: Record<string, number>,options: {- waitForDrain?: boolean;- waitForCompensationDrain?: boolean;timeoutMs?: number;webBeforePortal?: boolean;onSubmitted?: () => void;@@ -2008,7 +1984,6 @@ async function applyServiceTargets("web-ui": {taskDefinition: targets["web-ui"]!,desiredCount: expectedCounts["web-ui"]!,- waitForDrain: true,},},options.timeoutMs,@@ -2038,7 +2013,6 @@ async function applyServiceTargets({taskDefinition: targets[workload]!,desiredCount: expectedCounts[workload]!,- waitForDrain: options.waitForDrain,},]),),@@ -2074,7 +2048,6 @@ async function applyServiceTargets({taskDefinition: before.tasks[workload]!,desiredCount: before.counts[workload]!,- waitForDrain: options.waitForCompensationDrain,},]),),@@ -2757,7 +2730,7 @@ export async function awsDown(config: QmConfig, configDir = process.cwd()): PromreleaseLease(aws, lease);}ok(- "AWS services set to zero desired tasks; protected tasks finish in-flight turns before stopping. Infrastructure and data were retained.",+ "AWS services set to zero desired tasks; in-flight turns resume on the next deployment. Infrastructure and data were retained.",);}@@ -3096,7 +3069,7 @@ async function awsBackgroundCohort(if (!count) throw new CliError("background ownership requires running core capacity");if (waitForReady)await awaitServiceTargets(config, {- core: { taskDefinition: before.tasks.core!, desiredCount: count, waitForDrain: false },+ core: { taskDefinition: before.tasks.core!, desiredCount: count },});const taskArns =awsJson<{ taskArns?: string[] }>(aws, [@@ -3159,7 +3132,6 @@ export interface AwsBackgroundWorkCapacity {deploymentId: string;generation: number;desiredDeploymentId: string;- coreTaskProtection: Record<string, false>;workloads: Record<string, { taskDefinition: string; deploymentId: string; desiredCount: number; taskArns: string[] }>;}@@ -3279,7 +3251,6 @@ export async function awsBackgroundWorkCapacity(deploymentId: cohort.deploymentId,generation: ownership.generation,desiredDeploymentId: ownership.desiredDeploymentId,- coreTaskProtection: {},workloads: {},};for (const workload of [...workloads].sort()) {@@ -3293,36 +3264,6 @@ export async function awsBackgroundWorkCapacity(}if (canonicalJson(proof.workloads.core?.taskArns) !== canonicalJson([...cohort.taskArns].sort()))throw new CliError("core task cohort changed while checking capacity");- for (const batch of chunks(proof.workloads.core!.taskArns, 100)) {- const response = awsJson<{- protectedTasks?: Array<{ taskArn?: string; protectionEnabled?: boolean }>;- failures?: unknown[];- }>(aws, ["ecs", "get-task-protection", "--cluster", aws.cluster, "--tasks", ...batch]);- const taskId = (arn: string | undefined): string | undefined => {- const prefix = `arn:aws:ecs:${aws.region}:${aws.accountId}:task/`;- if (!arn?.startsWith(prefix)) return undefined;- const parts = arn.slice(prefix.length).split("/");- if (parts.length === 1 && parts[0]) return parts[0];- if (parts.length === 2 && parts[0] === aws.cluster && parts[1]) return parts[1];- return undefined;- };- const requested = new Map(batch.map((arn) => [taskId(arn), arn]));- const received = response.protectedTasks?.map((task) => taskId(task.taskArn)) ?? [];- if (- requested.has(undefined) ||- requested.size !== batch.length ||- response.failures?.length ||- !Array.isArray(response.protectedTasks) ||- received.length !== batch.length ||- new Set(received).size !== batch.length ||- received.some((id) => !id || !requested.has(id)) ||- response.protectedTasks.some((task) => task.protectionEnabled !== false)- )- throw new CliError(- "capacity requires explicit unprotected status for every current core task; protection was not changed",- );- for (const taskArn of batch) proof.coreTaskProtection[taskArn] = false;- }const finalManifest = currentDeploymentManifest(aws);if (canonicalJson(finalManifest) !== canonicalJson(cohort.manifest))throw new CliError("deployment manifest changed while checking capacity");@@ -3599,7 +3540,7 @@ export async function awsSetBackgroundWork(await awaitServiceTargets(config,{- core: { taskDefinition: before.tasks.core!, desiredCount: before.counts.core, waitForDrain: !enabled },+ core: { taskDefinition: before.tasks.core!, desiredCount: before.counts.core },},timeoutMs,);@@ -3610,12 +3551,7 @@ export async function awsSetBackgroundWork(const file = join(dir, "core.json");writeFileSync(file, JSON.stringify(desired));const target = registerTaskDefinition(config, file);- await applyServiceTargets(- config,- { core: target },- { core: before.counts.core },- { waitForDrain: !enabled, waitForCompensationDrain: true, timeoutMs },- );+ await applyServiceTargets(config, { core: target }, { core: before.counts.core }, { timeoutMs });try {recordDeploymentManifest(aws,@@ -3631,12 +3567,7 @@ export async function awsSetBackgroundWork(} catch (error) {const failures: string[] = [];try {- await applyServiceTargets(- config,- { core: before.tasks.core! },- { core: before.counts.core },- { waitForDrain: true, waitForCompensationDrain: true, timeoutMs },- );+ await applyServiceTargets(config, { core: before.tasks.core! }, { core: before.counts.core }, { timeoutMs });manifestTransaction(aws, current, current.id);} catch (restoreError) {failures.push(`restoring background work: ${errMessage(restoreError)}`);@@ -4975,12 +4906,7 @@ async function checkLive(}const primary = (state.deployments ?? []).filter((deployment) => deployment.status === "PRIMARY");const wantCount = workloadDesiredCount(config, service);- const draining = primary[0]?.rolloutState === "IN_PROGRESS" && (primary[0]?.runningCount ?? 0) >= wantCount;- if (- state.status !== "ACTIVE" ||- state.desiredCount !== wantCount ||- (draining ? (state.runningCount ?? 0) < wantCount : state.runningCount !== wantCount)- ) {+ if (state.status !== "ACTIVE" || state.desiredCount !== wantCount || state.runningCount !== wantCount) {failures.push(`${service}: runtime is ${state.status ?? "missing"} with ${state.runningCount ?? 0}/${state.desiredCount ?? 0} running, expected ${wantCount}`,);@@ -4988,7 +4914,7 @@ async function checkLive(if (primary.length !== 1 ||primary[0]?.taskDefinition !== state.taskDefinition ||- (primary[0]?.rolloutState !== "COMPLETED" && !draining)+ primary[0]?.rolloutState !== "COMPLETED") {failures.push(`${service}: configured task is not the sole healthy PRIMARY deployment`);}diff --git a/cli/templates/aws/main.tf b/cli/templates/aws/main.tfindex ad1557434..1a312b53a 100644--- a/cli/templates/aws/main.tf+++ b/cli/templates/aws/main.tf@@ -443,7 +443,7 @@ resource "aws_iam_role_policy" "github_deploy" {{Sid = "InspectDeploymentCanaries"Effect = "Allow"- Action = ["ecs:DescribeTasks", "ecs:GetTaskProtection"]+ Action = ["ecs:DescribeTasks"]Resource = ["arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task/${var.cluster_name}/*"]},{@@ -730,7 +730,7 @@ resource "aws_iam_role_policy" "task_objects" {},{Effect = "Allow"- Action = ["ecs:GetTaskProtection", "ecs:UpdateTaskProtection", "ecs:DescribeTasks"]+ Action = ["ecs:DescribeTasks"]Resource = "arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task/${var.cluster_name}/*"},{diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 4f151e454..c83406c85 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -253,7 +253,6 @@ function statefulAws(failTransactionPuts?: number;failPromotion?: boolean;promotionAlreadyCurrent?: boolean;- drainRollout?: boolean;drainPolls?: number;primaryFailedTasks?: boolean;rolloutFailed?: boolean;@@ -349,7 +348,7 @@ else if (a.includes("ecs describe-services")) {const end = Math.min(...[args.indexOf("--output", start), args.indexOf("--region", start)].filter((index) => index >= 0));const names = args.slice(start, end);const blocked = s.blockDisabledDrain && names.some((name) => s.definitions[s.services[name]?.taskDefinition]?.containerDefinitions?.some((container) => container.name === "core" && container.environment?.some((entry) => entry.name === "BACKGROUND_WORK_ENABLED" && entry.value === "0")));- const draining = ${JSON.stringify(opts.drainRollout ?? false)} || (s.drainPolls || 0) > 0 || blocked;+ const draining = (s.drainPolls || 0) > 0 || blocked;if (s.drainPolls > 0) { s.drainPolls--; save(); }const transientFailedTaskPolls = ${JSON.stringify(opts.transientFailedTaskPolls ?? 0)};let transientlyFailing = false;@@ -381,7 +380,7 @@ else if (a.includes("ecs describe-services")) {const deployments = draining? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "IN_PROGRESS", runningCount: service.desiredCount, failedTasks: 1 },- { id: "old-protected", status: "ACTIVE", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: 1, failedTasks: 0 },+ { id: "old", status: "ACTIVE", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: 1, failedTasks: 0 },]: (${JSON.stringify(opts.rolloutFailed ?? false)} || s.failedTaskDefinition === service.taskDefinition)? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "FAILED", runningCount: 0, failedTasks: 1 }]@@ -423,10 +422,6 @@ else if (a.includes("ecs describe-tasks") && s.taskInventory) {const requested = args.slice(args.indexOf("--tasks") + 1).filter(arg => arg.startsWith("arn:"));console.log(JSON.stringify({tasks: Object.values(s.taskInventory).flat().filter(task => requested.includes(task.taskArn) && task.taskArn !== s.omitTask), failures: s.inventoryFailures || []}));}-else if (a.includes("ecs get-task-protection")) {- const requested = args.slice(args.indexOf("--tasks") + 1).filter(arg => arg.startsWith("arn:"));- console.log(JSON.stringify(s.protectionResponse || {protectedTasks: requested.map(taskArn => ({taskArn, protectionEnabled:false})), failures:[]}));-}else if (a.includes("ecs list-tasks") && s.taskInventory?.[after("--service-name")]) console.log(JSON.stringify({ taskArns: after("--desired-status") === "STOPPED" ? [] : s.taskInventory[after("--service-name")].map(task => task.taskArn) }));else if (a.includes("ecs describe-tasks") && Object.values(s.taskInventory || {}).flat().some(task => args.includes(task.taskArn))) console.log(JSON.stringify({ tasks: Object.values(s.taskInventory).flat().filter(task => args.includes(task.taskArn)), failures: [] }));else if (a.includes("ecs list-tasks")) console.log(JSON.stringify({ taskArns: process.env.AWS_FAKE_NO_RUNNING_TASK ? [] : [...(process.env.AWS_FAKE_LARGE_ROLLOUT ? Array.from({ length: 100 }, (_, i) => "arn:aws:ecs:us-west-2:123456789012:task/old-core-" + i) : []), "arn:aws:ecs:us-west-2:123456789012:task/live-core"] }));@@ -4536,22 +4531,22 @@ test("AWS up cleans staging tags when ECS deployment fails", async () => {}});-test("AWS up succeeds while a protected old task keeps the rollout from completing, even with a historical failed task", async () => {- const dir = mkdtempSync(join(tmpdir(), "qm-aws-up-drain-"));+test("AWS up looks up the front door by the configured ALB name", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-up-alb-"));const dockerBin = join(dir, "docker");writeFileSync(dockerBin, "#!/bin/sh\nexit 0\n");chmodSync(dockerBin, 0o755);- const drainConfig = (): QmConfig => {+ const albConfig = (): QmConfig => {const base = oneServiceConfig();return { ...base, aws: { ...base.aws!, alb: "legacy-alb" } };};- const fake = statefulAws(dir, drainConfig(), {}, { drainRollout: true });+ const fake = statefulAws(dir, albConfig());const priorPath = process.env.PATH;process.env.PATH = `${dir}:${priorPath}`;try {- await awsUp(drainConfig(), dir, { yes: true });+ await awsUp(albConfig(), dir, { yes: true });const state = JSON.parse(readFileSync(fake.state, "utf8"));- assert.ok(state.dynamo["deployment/current"], "deployment manifest recorded despite the draining old task");+ assert.ok(state.dynamo["deployment/current"]);const calls = readFileSync(fake.log, "utf8");assert.doesNotMatch(calls, /ecs wait services-stable/);assert.match(@@ -4819,37 +4814,6 @@ test("AWS front door tolerates exactly one extra port-80 HTTPS-redirect listener}});-test("AWS live check accepts a successful deploy mid-drain: PRIMARY at full strength (historical failed task included) while a protected old task keeps the rollout IN_PROGRESS", async () => {- const dir = mkdtempSync(join(tmpdir(), "qm-aws-check-drain-"));- const single = oneServiceConfig();- const taskArn = "arn:aws:ecs:us-west-2:123456789012:task-definition/acme-core:1";- const fake = statefulAws(- dir,- single,- manifestItems([{ id: "current", imageLabel: "release", tasks: { core: taskArn } }], "current"),- { drainRollout: true },- );- const state = JSON.parse(readFileSync(fake.state, "utf8"));- const secretArn = "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf";- const arns = Object.fromEntries(computedSecrets(single).map((secret) => [secret.name, secretArn]));- state.definitions[taskArn] = {- ...renderTaskDefinition(- single,- "core",- `123456789012.dkr.ecr.us-west-2.amazonaws.com/qm-core@sha256:${"a".repeat(64)}`,- arns,- ),- taskDefinitionArn: taskArn,- };- writeFileSync(fake.state, JSON.stringify(state));- try {- await assert.doesNotReject(() => awsCheckLive(single, { report: false, configDir: dir }));- } finally {- fake.restore();- rmSync(dir, { recursive: true, force: true });- }-});-test("AWS renders config secretEnv extras and aliases as Secrets Manager references", () => {const extras: QmConfig = {...config,@@ -5773,7 +5737,7 @@ test("AWS deployment progress rejects invalid options before AWS calls", async (}});-test("inactive capacity proves exact relinquished unprotected cohorts without mutating deployment state", async () => {+test("inactive capacity proves exact relinquished cohorts without mutating deployment state", async () => {const dir = mkdtempSync(join(tmpdir(), "qm-capacity-"));const dockerBin = join(dir, "docker");writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);@@ -5848,13 +5812,12 @@ test("inactive capacity proves exact relinquished unprotected cohorts without muassert.equal(proof.desiredDeploymentId, "peer:active");assert.deepEqual(Object.keys(proof.workloads), ["core", "web-ui"]);assert.deepEqual(proof.workloads.core!.taskArns, [coreArn]);- assert.deepEqual(proof.coreTaskProtection, { [coreArn]: false });assert.equal((await awsBackgroundWorkStatus(configured, dir)).manifestId, manifest.id);assert.equal(readFileSync(fake.state, "utf8"), JSON.stringify(baseline));const stableLog = readFileSync(fake.log, "utf8");assert.doesNotMatch(stableLog,- /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|update-task-protection|stop-task/,+ /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|stop-task/,);ownership.members[0]!.state = "admitted";@@ -5869,38 +5832,6 @@ test("inactive capacity proves exact relinquished unprotected cohorts without muawait assert.rejects(awsBackgroundWorkCapacity(configured, dir), /held by another deployment/);ownership.desiredDeploymentId = "peer:active";- const shortArn = coreArn.replace(`task/${configured.aws!.cluster}/`, "task/");- reset();- writeFileSync(- fake.state,- JSON.stringify({- ...baseline,- protectionResponse: { protectedTasks: [{ taskArn: shortArn, protectionEnabled: false }] },- }),- );- assert.deepEqual((await awsBackgroundWorkCapacity(configured, dir)).coreTaskProtection, { [coreArn]: false });- for (const response of [- ...[- shortArn.replace("123456789012", "999999999999"),- shortArn.replace("us-west-2", "us-east-1"),- coreArn.replace(`task/${configured.aws!.cluster}/`, "task/other-cluster/"),- shortArn + "unknown",- ].map((taskArn) => ({ protectedTasks: [{ taskArn, protectionEnabled: false }] })),- {- protectedTasks: [- { taskArn: coreArn, protectionEnabled: false },- { taskArn: shortArn, protectionEnabled: false },- ],- },- { protectedTasks: [{ taskArn: coreArn, protectionEnabled: true }] },- { protectedTasks: [{ taskArn: coreArn }] },- { protectedTasks: [] },- { protectedTasks: [{ taskArn: coreArn, protectionEnabled: false }], failures: [{ arn: "unknown" }] },- ]) {- reset();- writeFileSync(fake.state, JSON.stringify({ ...baseline, protectionResponse: response }));- await assert.rejects(awsBackgroundWorkCapacity(configured, dir), /explicit unprotected status/);- }reset();const unresolved = structuredClone(baseline);unresolved.dynamo["deployment/background-preparation"] = {@@ -5980,7 +5911,7 @@ test("inactive capacity proves exact relinquished unprotected cohorts without muawait assert.rejects(awsBackgroundWorkCapacity(configured, dir), /ownership changed/);assert.doesNotMatch(readFileSync(fake.log, "utf8"),- /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|update-task-protection|stop-task/,+ /transact-write|put-item|delete-item|update-item|update-service|register-task-definition|run-task|stop-task/,);} finally {process.env.PATH = priorPath;diff --git a/cli/test/terraform.test.ts b/cli/test/terraform.test.tsindex c65423831..386f6229d 100644--- a/cli/test/terraform.test.ts+++ b/cli/test/terraform.test.ts@@ -389,8 +389,7 @@ test("the deploy role can run and inspect only stack-scoped deployment canaries"const inspect = policy.match(/Sid\s*= "InspectDeploymentCanaries"([\s\S]*?)\n\s*\},/)?.[1] ?? "";assert.match(inspect, /ecs:DescribeTasks/);- assert.match(inspect, /ecs:GetTaskProtection/);- assert.doesNotMatch(policy, /ecs:UpdateTaskProtection/);+ assert.doesNotMatch(mainTf, /TaskProtection/);assert.match(inspect, /task\/\$\{var\.cluster_name\}\/\*/);assert.doesNotMatch(inspect, /Resource\s*= "\*"/);});@@ -677,7 +676,7 @@ test("AWS module reuses account OIDC, guards account and passes configured task/Sid\s*= "ManageDeploymentLayers"[\s\S]*"s3:GetObject", "s3:PutObject"[\s\S]*deployment\/layers\/\*/,);assert.match(mainTf, /Sid\s*= "InspectGithubOidcProvider"[\s\S]*iam:GetOpenIDConnectProvider/);- assert.match(mainTf, /"ecs:GetTaskProtection", "ecs:UpdateTaskProtection", "ecs:DescribeTasks"/);+ assert.match(mainTf, /Action {3}= \["ecs:DescribeTasks"\]\n\s+Resource = "arn:aws:ecs:/);assert.match(mainTf, /task\/\$\{var\.cluster_name\}\/\*/);assert.match(mainTf, /"lambda:RunMicrovm"[\s\S]*"lambda:CreateMicrovmAuthToken"/);assert.match(mainTf, /"lambda:ListMicrovmImages"/);diff --git a/docs/background-ownership.md b/docs/background-ownership.mdindex c3fd03fee..fab396345 100644--- a/docs/background-ownership.md+++ b/docs/background-ownership.md@@ -46,7 +46,7 @@ A process records `admitted` before starting background resources. The desired d`relinquished` acknowledges that the process stopped new claims and closed its Slack ingress. Already admitted turns may still run. Cron polling can resume after rollback while earlier callbacks keep their queue connection and heartbeats. Maintenance callbacks that cannot relinquish safely are joined before acknowledgment.-`drained` means the process's admitted background work has finished. Releasing ownership and making a deployment safe to replace are separate gates: `qm up` replaces a cohort once every member has relinquished, and members still finishing admitted work keep running beside the new tasks under their leases and task protection until they drain. The ECS service's `maximumPercent` must leave room for those lingering tasks. Do not terminate a relinquished but undrained process merely to meet a rollout time target.+`drained` means the process's admitted background work has finished. `qm up` replaces a cohort once every member has relinquished. Members still finishing admitted work get the shutdown drain window, after which the platform terminates them; a turn cut off that way releases its lease on exit and resumes on the replacement tasks.Database errors or an expired local validity watchdog fence new local work. They do not establish durable relinquishment or authorize another deployment to bypass an outstanding member.@@ -104,13 +104,12 @@ legacy supersession results and continue using durable ownership admission.Inactive, unready, or fenced processes do not publish this compatibility heartbeat;publishing also stops once durable ownership is enabled.-This bridge preserves the legacy worker drain behavior and its running-turn task-protection. It does not add missing lifecycle controls to older binaries: their-Slack ingress, cron callbacks, and inline HTTP execution can remain active.+This bridge preserves the legacy worker drain behavior. It does not add missing+lifecycle controls to older binaries: their Slack ingress, cron callbacks, and+inline HTTP execution can remain active.Never treat the heartbeat as a deployment relinquishment acknowledgment or asproof that a legacy task is safe to terminate. Bootstrap still requires explicitinfrastructure proof that every legacy task has retired, including pending tasks.-Do not clear task protection or terminate live turns to finish enrollment.If enrollment is abandoned, the legacy heartbeat expiry permits older workersto resume their existing claim loop.@@ -120,4 +119,3 @@ asynchronous submissions for the active workers. Accepted turns, scheduledcallbacks, and their nested work keep running with their existing leases; theircompletion is part of the deployment's drain acknowledgment. Resuming ownershiprestores synchronous admission without restarting or canceling those calls.-Task protection also counts admitted foreground work while the process drains.diff --git a/src/config.ts b/src/config.tsindex ce381c279..df35cd61d 100644--- a/src/config.ts+++ b/src/config.ts@@ -128,8 +128,6 @@ export interface Config {backgroundDeploymentId?: string;deploymentControlSecret?: string;buildSha?: string;- ecsTaskProtection: boolean;- ecsAgentUri?: string;monitorPollMs: number;skillSyncPollMs: number;monitorHeartbeatMs: number;@@ -1563,8 +1561,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {? { backgroundDeploymentId: env.BACKGROUND_DEPLOYMENT_ID, deploymentControlSecret: env.DEPLOYMENT_CONTROL_SECRET }: {}),...(env.GIT_SHA ? { buildSha: env.GIT_SHA } : {}),- ecsTaskProtection: boolEnvStrict("ECS_TASK_PROTECTION", env.ECS_TASK_PROTECTION) ?? true,- ...(env.ECS_AGENT_URI ? { ecsAgentUri: env.ECS_AGENT_URI } : {}),monitorPollMs: numEnvStrict("MONITOR_POLL_MS", env.MONITOR_POLL_MS) ?? CONFIG_DEFAULTS.monitorPollMs,skillSyncPollMs: numEnvStrict("SKILL_SYNC_POLL_MS", env.SKILL_SYNC_POLL_MS) ?? CONFIG_DEFAULTS.skillSyncPollMs,monitorHeartbeatMs:diff --git a/src/runs/drain.ts b/src/runs/drain.tsindex c3138e73c..a699e8a64 100644--- a/src/runs/drain.ts+++ b/src/runs/drain.ts@@ -1,24 +1,16 @@import type { InstanceRegistry } from "./instance-registry.ts";-import type { TaskProtection } from "./task-protection.ts";import { createSweeper, type Sweeper } from "../util/sweeper.ts";export interface DrainController {start(): void;stop(): Promise<void>;canClaim(): boolean;- noteBusy(): void;}const DRAIN_SWEEP_MS = 10_000;-export function createDrainController(opts: {- registry: InstanceRegistry;- protection: TaskProtection | null;- busy: () => boolean;- sweepMs?: number;-}): DrainController {+export function createDrainController(opts: { registry: InstanceRegistry; sweepMs?: number }): DrainController {let superseded = false;- let protectionOn = false;const sweeper: Sweeper = createSweeper(async () => {const wasSuperseded = superseded;@@ -28,33 +20,13 @@ export function createDrainController(opts: {`[drain] ${superseded ? "newer build is live — draining: no new run claims, finishing in-flight turns" : "newer build gone — resuming run claims"}`,);}- if (!opts.protection) return;- const busy = opts.busy();- if (busy) {- await opts.protection.set(true);- protectionOn = true;- } else if (protectionOn) {- await opts.protection.set(false);- protectionOn = false;- }},opts.sweepMs ?? DRAIN_SWEEP_MS,{ label: "deploy-drain", immediate: true },);return {start: () => sweeper.start(),- stop: async () => {- await sweeper.stop();- if (protectionOn && opts.protection) {- protectionOn = false;- await opts.protection.set(false);- }- },+ stop: () => sweeper.stop(),canClaim: () => !superseded,- noteBusy: () => {- if (!opts.protection || protectionOn) return;- protectionOn = true;- void opts.protection.set(true);- },};}diff --git a/src/runs/task-protection.ts b/src/runs/task-protection.tsdeleted file mode 100644index befd0f8d2..000000000--- a/src/runs/task-protection.ts+++ /dev/null@@ -1,35 +0,0 @@-import { errMessage } from "../util/errors.ts";--export interface TaskProtection {- set(enabled: boolean): Promise<void>;-}--const PROTECTION_EXPIRES_MINUTES = 60;--export function createEcsTaskProtection(agentUri: string, opts?: { fetchFn?: typeof fetch }): TaskProtection {- const fetchFn = opts?.fetchFn ?? fetch;- let lastFailure: string | null = null;- return {- async set(enabled: boolean): Promise<void> {- try {- const res = await fetchFn(`${agentUri.replace(/\/$/, "")}/task-protection/v1/state`, {- method: "PUT",- headers: { "content-type": "application/json" },- body: JSON.stringify(- enabled- ? { ProtectionEnabled: true, ExpiresInMinutes: PROTECTION_EXPIRES_MINUTES }- : { ProtectionEnabled: false },- ),- });- if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => "")}`.trim());- lastFailure = null;- } catch (e) {- const msg = errMessage(e);- if (msg !== lastFailure) {- lastFailure = msg;- console.error(`[task-protection] set(${enabled}) failed (turns fall back to drain+resume): ${msg}`);- }- }- },- };-}diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex 15a027afd..4220d7e1a 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -129,7 +129,6 @@ export interface WorkerDeps extends ProcessDeps {recoveryPollMs?: number;workerId?: string;canClaim?: () => boolean;- onClaimed?: () => void;admittedWork?: AdmittedWork;}@@ -224,7 +223,6 @@ export function createWorker(deps: WorkerDeps): Worker {console.log(`[worker] claimed worker=${workerId} run=${run.id} thread=${run.sessionId}`);claimed();claimDone = null;- deps.onClaimed?.();try {const work = () => processRun(deps, run, { background: true, shutdown: shutdown.signal });if (deps.admittedWork) await deps.admittedWork.run(work);diff --git a/src/util/admitted-work.ts b/src/util/admitted-work.tsindex a7bf91876..1c3a20e53 100644--- a/src/util/admitted-work.ts+++ b/src/util/admitted-work.ts@@ -6,9 +6,7 @@ export class WorkAdmissionClosed extends Error {}}-export function createAdmittedWork(- options: { paused?: boolean; canStart?: () => boolean; onAdmitted?: () => void | Promise<void> } = {},-) {+export function createAdmittedWork(options: { paused?: boolean; canStart?: () => boolean } = {}) {const context = new AsyncLocalStorage<{ active: boolean }>();const pending = new Set<Promise<unknown>>();let paused = options.paused ?? false;@@ -25,12 +23,7 @@ export function createAdmittedWork(run<T>(work: () => T | Promise<T>): Promise<T> {if (!canRun()) return Promise.reject(new WorkAdmissionClosed());const admission = { active: true };- const task = context.run(admission, () =>- Promise.resolve().then(async () => {- await options.onAdmitted?.();- return work();- }),- );+ const task = context.run(admission, () => Promise.resolve().then(work));pending.add(task);void task.finally(() => {diff --git a/src/wiring.ts b/src/wiring.tsindex 3f31ee3c4..8e9205521 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -342,7 +342,6 @@ import {createPostgresInstanceRegistry,type InstanceRegistry,} from "./runs/instance-registry.ts";-import { createEcsTaskProtection, type TaskProtection } from "./runs/task-protection.ts";import { createDrainController, type DrainController } from "./runs/drain.ts";import { createReaper, REAPER_LEASE_KEY, type Reaper } from "./runs/reaper.ts";import { createSweeper as createUntrackedSweeper, type Sweeper } from "./util/sweeper.ts";@@ -576,10 +575,8 @@ export function buildApp(} = {},): BuiltApp {let backgroundAdmission = () => !config.backgroundDeploymentId;- let noteAdmitted = () => {};const admittedWork = createAdmittedWork({canStart: () => !config.backgroundDeploymentId || backgroundAdmission(),- onAdmitted: () => noteAdmitted(),});let inlineShutdown = new AbortController();const createSweeper: typeof createUntrackedSweeper = (work, interval, options) =>@@ -2574,14 +2571,7 @@ export function buildApp();}): legacyRegistry;- const taskProtection: TaskProtection | null =- config.ecsTaskProtection && config.ecsAgentUri ? createEcsTaskProtection(config.ecsAgentUri) : null;- const drain: DrainController = createDrainController({- registry: instanceRegistry,- protection: taskProtection,- busy: () => admittedWork.busy() || workers.some((w) => w.busy()),- });- noteAdmitted = () => drain.noteBusy();+ const drain: DrainController = createDrainController({ registry: instanceRegistry });const workers: Worker[] = Array.from({ length: Math.max(1, config.workers) }, () =>createWorker({admittedWork,@@ -2592,7 +2582,6 @@ export function buildApp(errors,pollMs: 250,canClaim: () => backgroundAdmission() && drain.canClaim(),- onClaimed: () => drain.noteBusy(),}),);const processReaper: ProcessReaper | null = processesdiff --git a/test/admitted-work.test.ts b/test/admitted-work.test.tsindex 90a0cca73..9fd220102 100644--- a/test/admitted-work.test.ts+++ b/test/admitted-work.test.ts@@ -68,15 +68,9 @@ test("detached async contexts lose admission when their accepting callback finis});test("admission is registered before callback execution and errors cannot strand busy state", async () => {- let admitted = false;- const work = createAdmittedWork({- onAdmitted: () => {- admitted = true;- },- });+ const work = createAdmittedWork();await assert.rejects(work.run(() => {- assert.equal(admitted, true);assert.equal(work.busy(), true);throw new Error("failed work");}),diff --git a/test/deploy-drain.test.ts b/test/deploy-drain.test.tsindex 758c6625a..ed63fba4e 100644--- a/test/deploy-drain.test.ts+++ b/test/deploy-drain.test.ts@@ -1,11 +1,8 @@import { test } from "node:test";import assert from "node:assert/strict";-import { createServer } from "node:http";-import type { AddressInfo } from "node:net";import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";import { createWorker } from "../src/runs/worker.ts";import { createDrainController } from "../src/runs/drain.ts";-import { createEcsTaskProtection } from "../src/runs/task-protection.ts";import type { InstanceRegistry } from "../src/runs/instance-registry.ts";import type { Orchestrator, OrchestratorInput } from "../src/core/orchestrator.ts";import type { Principal, TurnResult } from "../src/types.ts";@@ -24,7 +21,7 @@ test("a superseded worker stops claiming; in-flight turns finish; claiming resumconst { runs } = createMemoryRunStore();let superseded = false;const registry: InstanceRegistry = { beat: async () => superseded };- const drain = createDrainController({ registry, protection: null, busy: () => false, sweepMs: 10 });+ const drain = createDrainController({ registry, sweepMs: 10 });drain.start();let release: (() => void) | null = null;@@ -63,129 +60,5 @@ test("a superseded worker stops claiming; in-flight turns finish; claiming resumassert.equal(worker.busy(), true, "claiming resumed once the newer build disappeared");release!();await worker.stop();- drain.stop();-});--test("task protection tracks busyness: asserted while a turn runs, released once idle", async () => {- const puts: Array<{ ProtectionEnabled: boolean; ExpiresInMinutes?: number }> = [];- const server = createServer((req, res) => {- let body = "";- req.on("data", (c: Buffer) => (body += String(c)));- req.on("end", () => {- puts.push(JSON.parse(body) as (typeof puts)[number]);- res.writeHead(200, { "content-type": "application/json" });- res.end("{}");- });- });- await new Promise<void>((r) => server.listen(0, r));- const uri = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;-- let busy = true;- const drain = createDrainController({- registry: { beat: async () => false },- protection: createEcsTaskProtection(uri),- busy: () => busy,- sweepMs: 10,- });- drain.start();- await sleep(60);- assert.ok(- puts.some((p) => p.ProtectionEnabled === true),- "protection asserted while busy",- );- assert.equal(puts.at(-1)?.ProtectionEnabled, true, "re-asserted every sweep (expiry refresh)");- assert.ok((puts.at(-1)?.ExpiresInMinutes ?? 0) > 0);-- busy = false;- await sleep(60);- assert.equal(puts.at(-1)?.ProtectionEnabled, false, "released once idle");- const releases = puts.filter((p) => p.ProtectionEnabled === false).length;- await sleep(40);- assert.equal(puts.filter((p) => p.ProtectionEnabled === false).length, releases, "released once, not every sweep");-- drain.stop();- server.close();-});--test("noteBusy asserts protection at the idle→busy edge without waiting for a sweep", async () => {- const puts: Array<{ ProtectionEnabled: boolean }> = [];- const server = createServer((req, res) => {- let body = "";- req.on("data", (c: Buffer) => (body += String(c)));- req.on("end", () => {- puts.push(JSON.parse(body) as (typeof puts)[number]);- res.writeHead(200, { "content-type": "application/json" });- res.end("{}");- });- });- await new Promise<void>((r) => server.listen(0, r));- const uri = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;- const drain = createDrainController({- registry: { beat: async () => false },- protection: createEcsTaskProtection(uri),- busy: () => true,- sweepMs: 60_000,- });- drain.noteBusy();- await sleep(50);- assert.deepEqual(puts.length && puts[0]?.ProtectionEnabled, true, "protection asserted immediately on claim");- drain.noteBusy();- await sleep(30);- assert.equal(puts.length, 1, "already-on is a no-op");- drain.stop();- await sleep(30);- assert.equal(puts.at(-1)?.ProtectionEnabled, false, "stop releases protection best-effort");- server.close();-});--test("a failing protection endpoint degrades silently and canClaim stays governed by supersession only", async () => {- const protection = createEcsTaskProtection("http://127.0.0.1:1");- const drain = createDrainController({- registry: { beat: async () => false },- protection,- busy: () => true,- sweepMs: 10,- });- drain.start();- await sleep(50);- assert.equal(drain.canClaim(), true);- drain.stop();-});--test("admitted foreground work keeps task protection through background pause", async () => {- const { createAdmittedWork } = await import("../src/util/admitted-work.ts");- const changes: boolean[] = [];- let noteBusy = () => {};- const work = createAdmittedWork({ onAdmitted: () => noteBusy() });- const drain = createDrainController({- registry: { beat: async () => false },- protection: {- set: async (enabled) => {- changes.push(enabled);- },- },- busy: work.busy,- sweepMs: 5,- });- noteBusy = () => drain.noteBusy();- const finish = Promise.withResolvers<void>();- drain.start();- const admitted = work.run(() => finish.promise);- try {- await sleep(15);- work.pause();- assert.equal(changes.at(-1), true);- changes.length = 0;- await sleep(15);- assert.ok(changes.length > 0 && changes.every(Boolean));- finish.resolve();- await admitted;- await work.drained();- await sleep(15);- assert.equal(changes.at(-1), false);- } finally {- finish.resolve();- await admitted;- await drain.stop();- }+ await drain.stop();});diff --git a/test/session-state-events.test.ts b/test/session-state-events.test.tsindex 28d48e394..cb85438cb 100644--- a/test/session-state-events.test.ts+++ b/test/session-state-events.test.ts@@ -284,10 +284,15 @@ test("a parent stays working while a subagent runs after its own turn ends — pconst built = freshApp();const parentThread = "web:U1:parent";const childThread = "web:U1:parent-child";+ const got = record(built.sessionStateBus);const parent = await built.app.turn(dm("hello", parentThread));const child = await built.app.turn(dm("hello", childThread));+ assert.ok(+ await waitFor(() => statesFor(got, parentThread).includes("idle") && statesFor(got, childThread).includes("idle")),+ "both greeting turns settle before the delegation starts",+ );+ got.length = 0;await built.sessions.setParentSession(child.sessionId!, parent.sessionId!);- const got = record(built.sessionStateBus);const { run: parentRun } = await built.runs.enqueue({sessionId: parentThread,request: resolvedDm("delegate", parentThread),