Multiplayer agent harness for work.
Activate background work without redeploying the candidate (#1029)
Regan Bell3bd425b96329
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex d5b4b6020..754586e62 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -1240,11 +1240,16 @@ interface RolloutTarget {taskDefinition: string;desiredCount: number;deploymentId?: string;+ waitForDrain?: boolean;}-async function awaitServiceTargets(config: QmConfig, expected: Record<string, RolloutTarget>): Promise<void> {+async function awaitServiceTargets(+ config: QmConfig,+ expected: Record<string, RolloutTarget>,+ timeoutMs?: number,+): Promise<void> {const pollMs = envNum("QM_AWS_ROLLOUT_POLL_MS", 15_000);- const deadline = Date.now() + envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 20 * 60_000);+ const deadline = Date.now() + (timeoutMs ?? envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 20 * 60_000));let healthyStreak = 0;let failurePolls = new Map<string, number>();let describeFailures = 0;@@ -1329,6 +1334,12 @@ async function awaitServiceTargets(config: QmConfig, expected: Record<string, Ro}} else if (running < want.desiredCount) {waiting.push(`${workload} (${running}/${want.desiredCount})`);+ } else if (+ want.waitForDrain &&+ (deployment.rolloutState !== "COMPLETED" || state.deployments?.length !== 1) &&+ state.deploymentConfiguration?.strategy !== "BLUE_GREEN"+ ) {+ waiting.push(`${workload} (prior tasks are still draining)`);} else if (state.deploymentConfiguration?.strategy === "BLUE_GREEN") {const nativeStatus = nativeStatuses.get(workload);if (nativeStatus === "SUCCESSFUL") continue;@@ -1676,6 +1687,7 @@ async function applyServiceTargets(config: QmConfig,targets: Record<string, string>,desiredCounts?: Record<string, number>,+ options: { waitForDrain?: boolean; waitForCompensationDrain?: boolean; timeoutMs?: number } = {},): Promise<void> {const aws = requireAws(config);const workloads = Object.keys(targets);@@ -1708,9 +1720,14 @@ async function applyServiceTargets(Object.fromEntries(workloads.map((workload) => [workload,- { taskDefinition: targets[workload]!, desiredCount: expectedCounts[workload]! },+ {+ taskDefinition: targets[workload]!,+ desiredCount: expectedCounts[workload]!,+ waitForDrain: options.waitForDrain,+ },]),),+ options.timeoutMs,);} catch (error) {const restoreFailures: string[] = [];@@ -1739,9 +1756,14 @@ async function applyServiceTargets(Object.fromEntries(changed.map((workload) => [workload,- { taskDefinition: before.tasks[workload]!, desiredCount: before.counts[workload]! },+ {+ taskDefinition: before.tasks[workload]!,+ desiredCount: before.counts[workload]!,+ waitForDrain: options.waitForCompensationDrain,+ },]),),+ options.timeoutMs,);} catch (restoreError) {restoreFailures.push(errMessage(restoreError));@@ -2471,6 +2493,157 @@ export async function awsRollback(ok(`rolled back ${config.orgId}`);}+export function taskDefinitionForBackgroundWork(+ task: Record<string, unknown>,+ enabled: boolean,+): Record<string, unknown> {+ const containers = structuredClone(task.containerDefinitions) as Array<Record<string, unknown>> | undefined;+ const core = containers?.find((container) => container.name === "core");+ if (!core) throw new CliError("core task definition has no core container");+ const secrets = core.secrets as Array<{ name: string }> | undefined;+ if (secrets?.some((entry) => entry.name === "BACKGROUND_WORK_ENABLED")) {+ throw new CliError("BACKGROUND_WORK_ENABLED cannot be changed while supplied as a secret");+ }+ const environment = (core.environment ?? []) as Array<{ name: string; value: string }>;+ core.environment = [+ ...environment.filter((entry) => entry.name !== "BACKGROUND_WORK_ENABLED"),+ { name: "BACKGROUND_WORK_ENABLED", value: enabled ? "1" : "0" },+ ];+ const fields = [+ "family",+ "taskRoleArn",+ "executionRoleArn",+ "networkMode",+ "volumes",+ "placementConstraints",+ "requiresCompatibilities",+ "cpu",+ "memory",+ "pidMode",+ "ipcMode",+ "proxyConfiguration",+ "inferenceAccelerators",+ "ephemeralStorage",+ "runtimePlatform",+ "enableFaultInjection",+ ];+ return {+ ...Object.fromEntries(fields.flatMap((field) => (task[field] == null ? [] : [[field, task[field]]]))),+ containerDefinitions: containers,+ };+}++export async function awsSetBackgroundWork(+ config: QmConfig,+ configDir: string,+ enabled: boolean,+ candidatePath?: string,+): Promise<void> {+ const { aws, workloads } = awsTopology(config, configDir);+ if (!workloads.includes("core")) throw new CliError("background work requires the core workload");+ const candidate = candidatePath ? releaseCandidate(config, candidatePath) : undefined;+ assertAwsCallerAccount(aws);+ await withAwsLease(aws, async () => {+ const current = currentDeploymentManifest(aws);+ if (!current) throw new CliError("background work requires a recorded deployment");+ const states = describedServices(config, workloads);+ assertOwnedServices(config, states, workloads);+ const before = serviceSnapshotFromStates(states, workloads);+ for (const workload of workloads) {+ if (+ current.tasks[workload] !== before.tasks[workload] ||+ current.counts?.[workload] !== before.counts[workload]+ ) {+ throw new CliError(`cannot change background work while ${workload} differs from the deployment manifest`);+ }+ }+ if (!before.counts.core) throw new CliError("cannot change background work while core is scaled down");+ let coreTask: Record<string, unknown> | undefined;+ for (const workload of candidate ? workloads : ["core"]) {+ const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [+ "ecs",+ "describe-task-definition",+ "--task-definition",+ before.tasks[workload]!,+ ]).taskDefinition;+ const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(+ (item) => item.name === workload,+ );+ if (!task || typeof container?.image !== "string" || !isPinnedWorkloadImage(config, workload, container.image)) {+ throw new CliError(`${workload} does not have a trusted digest-pinned image`);+ }+ if (+ candidate &&+ (container.image !== candidate.images[workload] ||+ !candidate.imageProvenance[workload] ||+ canonicalJson(candidate.imageProvenance[workload]) !== canonicalJson(current.imageProvenance?.[workload]))+ ) {+ throw new CliError(`${workload} does not match the expected release candidate`);+ }+ if (workload === "core") coreTask = task;+ }+ const desired = taskDefinitionForBackgroundWork(coreTask!, enabled);+ const core = (coreTask!.containerDefinitions as Array<Record<string, unknown>>).find(+ (item) => item.name === "core",+ )!;+ const environment = (core.environment ?? []) as Array<{ name: string; value: string }>;+ const currentFlag = environment.find((entry) => entry.name === "BACKGROUND_WORK_ENABLED")?.value;+ const timeoutMs = envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000);+ if (currentFlag === (enabled ? "1" : "0")) {+ await awaitServiceTargets(+ config,+ {+ core: { taskDefinition: before.tasks.core!, desiredCount: before.counts.core, waitForDrain: !enabled },+ },+ timeoutMs,+ );+ return;+ }+ const dir = mkdtempSync(join(tmpdir(), "qm-background-work-"));+ try {+ const file = join(dir, "core.json");+ writeFileSync(file, JSON.stringify(desired));+ const target = registerTaskDefinition(config, file);+ await applyServiceTargets(+ config,+ { core: target },+ { core: before.counts.core },+ { waitForDrain: !enabled, waitForCompensationDrain: true, timeoutMs },+ );+ try {+ recordDeploymentManifest(+ aws,+ { ...before.tasks, core: target },+ {+ counts: current.counts,+ imageLabel: current.imageLabel,+ dbRestorePoint: current.dbRestorePoint,+ layer: current.layer,+ imageProvenance: current.imageProvenance,+ },+ );+ } catch (error) {+ const failures: string[] = [];+ try {+ await applyServiceTargets(+ config,+ { core: before.tasks.core! },+ { core: before.counts.core },+ { waitForDrain: true, waitForCompensationDrain: true, timeoutMs },+ );+ manifestTransaction(aws, current, current.id);+ } catch (restoreError) {+ failures.push(`restoring background work: ${errMessage(restoreError)}`);+ }+ throwAfterCompensation(error, failures);+ }+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+ ok(`background work ${enabled ? "enabled" : "disabled"}`);+}+function envValues(configDir: string, path: string | undefined): Map<string, string> {const file = resolve(path ?? join(configDir, ".env"));if (!existsSync(file)) {diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 2465f240b..997fc15c9 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -18,6 +18,8 @@ import {awsMigrateCandidate,awsRollback,awsSecretsPush,+ awsSetBackgroundWork,+ taskDefinitionForBackgroundWork,awsStatus,awsUp,githubTrustSubject,@@ -213,6 +215,7 @@ function statefulAws(failPromotion?: boolean;promotionAlreadyCurrent?: boolean;drainRollout?: boolean;+ drainPolls?: number;primaryFailedTasks?: boolean;rolloutFailed?: boolean;transientFailedTaskPolls?: number;@@ -301,6 +304,9 @@ else if (a.includes("ecs describe-services")) {const start = args.indexOf("--services") + 1;const end = Math.min(...[args.indexOf("--output", start), args.indexOf("--region", start)].filter((index) => index >= 0));const names = args.slice(start, end);+ const blocked = s.blockDisabledDrain && names.some((name) => s.definitions[s.services[name]?.taskDefinition]?.containerDefinitions?.some((container) => container.name === "core" && container.environment?.some((entry) => entry.name === "BACKGROUND_WORK_ENABLED" && entry.value === "0")));+ const draining = ${JSON.stringify(opts.drainRollout ?? false)} || (s.drainPolls || 0) > 0 || blocked;+ if (s.drainPolls > 0) { s.drainPolls--; save(); }const transientFailedTaskPolls = ${JSON.stringify(opts.transientFailedTaskPolls ?? 0)};let transientlyFailing = false;if (transientFailedTaskPolls && s.updated) {@@ -328,7 +334,7 @@ else if (a.includes("ecs describe-services")) {if (name === staleName && service.previousTaskDefinition) {return [{ serviceName: name, status: "ACTIVE", desiredCount: service.desiredCount, runningCount: service.desiredCount, taskDefinition: service.previousTaskDefinition, deployments: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.previousTaskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }], tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];}- const deployments = ${JSON.stringify(opts.drainRollout ?? false)}+ const deployments = draining? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "IN_PROGRESS", runningCount: service.desiredCount, failedTasks: 1 },{ id: "old-protected", status: "ACTIVE", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: 1, failedTasks: 0 },@@ -340,7 +346,7 @@ else if (a.includes("ecs describe-services")) {: blueGreenBakePolls? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }]: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: transientFailedTaskPolls && s.updated ? 1 : 0 }];- return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: ${JSON.stringify(opts.drainRollout ?? false)} ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];+ return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: draining ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];}), failures: names.filter((name) => !s.services[name]).map((name) => ({ arn: name, reason: "MISSING" })) }));}else if (a.includes("ecs list-service-deployments") && ${JSON.stringify(opts.failNativeStatusOnceAfterUpdate ?? false)} && s.updated && !s.nativeStatusFailedOnce) {@@ -397,8 +403,10 @@ else if (a.includes("ecs update-service")) {if (!${JSON.stringify(opts.ignoreUpdate ?? false)} && args.includes("--desired-count")) service.desiredCount = Number(after("--desired-count"));if (!${JSON.stringify(opts.ignoreUpdate ?? false)}) service.deploymentId = "ecs-svc/" + name + "-" + (++s.revision);s.updated = true;+ s.drainPolls = ${JSON.stringify(opts.drainPolls ?? 0)};s.blueGreenPolls = 0;- if (${JSON.stringify(opts.failFirstUpdateAfterMutation ?? false)} && !s.failedFirstUpdate) {+ if (s.failNextUpdatesAfterMutation > 0 || (${JSON.stringify(opts.failFirstUpdateAfterMutation ?? false)} && !s.failedFirstUpdate)) {+ if (s.failNextUpdatesAfterMutation > 0) s.failNextUpdatesAfterMutation--;s.failedFirstUpdate = true;save();console.error("UpdateServiceResponseLost");@@ -416,6 +424,7 @@ else if (a.includes("dynamodb get-item")) {console.log(JSON.stringify(s.dynamo[key] ? { Item: s.dynamo[key] } : {}));}else if (a.includes("dynamodb transact-write-items")) {+ if (s.failNextTransactions > 0) { s.failNextTransactions--; save(); console.error("TransactionRejected"); process.exit(1); }if (${JSON.stringify(opts.failTransactions ?? false)}) { console.error("TransactionRejected"); process.exit(1); }const failTransactionPuts = ${JSON.stringify(opts.failTransactionPuts ?? 0)};if (failTransactionPuts) {@@ -1396,6 +1405,149 @@ test("AWS up coalesces a requested restart into one deployment even when the tas}});+test("background mode changes preserve task settings and reject secret-controlled modes", () => {+ const task = {+ family: "core",+ taskDefinitionArn: "read-only",+ status: "ACTIVE",+ cpu: "1024",+ volumes: [{ name: "scratch" }],+ containerDefinitions: [+ {+ name: "core",+ image: "pinned",+ environment: [{ name: "OTHER", value: "kept" }],+ secrets: [{ name: "TOKEN", valueFrom: "secret" }],+ },+ { name: "sidecar", image: "unchanged" },+ ],+ };+ const changed = taskDefinitionForBackgroundWork(task, false);+ assert.equal(changed.taskDefinitionArn, undefined);+ assert.equal(changed.status, undefined);+ assert.deepEqual(changed.volumes, task.volumes);+ const containers = changed.containerDefinitions as typeof task.containerDefinitions;+ assert.deepEqual(containers[1], task.containerDefinitions[1]);+ assert.deepEqual(containers[0]!.secrets, task.containerDefinitions[0]!.secrets);+ assert.deepEqual(containers[0]!.environment, [+ { name: "OTHER", value: "kept" },+ { name: "BACKGROUND_WORK_ENABLED", value: "0" },+ ]);+ assert.equal(task.containerDefinitions[0]!.environment!.length, 1);+ assert.throws(+ () =>+ taskDefinitionForBackgroundWork(+ { containerDefinitions: [{ name: "core", secrets: [{ name: "BACKGROUND_WORK_ENABLED" }] }] },+ true,+ ),+ /supplied as a secret/,+ );+});++test("background activation preserves candidate and manifest without another migration; demotion drains old tasks", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-background-"));+ const dockerBin = join(dir, "docker");+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);+ chmodSync(dockerBin, 0o755);+ const base = oneServiceConfig();+ const single = { ...base, env: { ...base.env, core: { ...base.env.core, BACKGROUND_WORK_ENABLED: "false" } } };+ const fake = statefulAws(dir, single, {}, { drainPolls: 6 });+ const priorPath = process.env.PATH;+ process.env.PATH = `${dir}:${priorPath}`;+ try {+ await awsUp(single, dir, { yes: true });+ const readState = () => JSON.parse(readFileSync(fake.state, "utf8"));+ const manifest = () => {+ const state = readState();+ return JSON.parse(+ state.dynamo[`deployment/manifest/${state.dynamo["deployment/current"].manifestId.S}`].manifest.S,+ );+ };+ const initial = manifest();+ const candidate = {+ contract: 1,+ accountId: single.aws!.accountId,+ region: single.aws!.region,+ label: "verified",+ images: { core: readState().definitions[initial.tasks.core].containerDefinitions[0].image },+ imageProvenance: initial.imageProvenance,+ };+ const candidatePath = join(dir, "candidate.json");+ writeFileSync(candidatePath, JSON.stringify(candidate));+ writeFileSync(fake.log, "");+ await awsSetBackgroundWork(single, dir, true, candidatePath);+ assert.match(readFileSync(fake.log, "utf8"), /ecs register-task-definition/);+ writeFileSync(fake.log, "");+ await awsSetBackgroundWork(single, dir, false);+ const disabled = manifest();+ assert.notEqual(disabled.tasks.core, initial.tasks.core);+ assert.deepEqual(disabled.imageProvenance, initial.imageProvenance);+ assert.deepEqual(disabled.layer, initial.layer);+ assert.equal(disabled.dbRestorePoint, initial.dbRestorePoint);+ const disableCalls = readFileSync(fake.log, "utf8");+ assert.ok(+ (disableCalls.slice(disableCalls.indexOf("ecs update-service")).match(/ecs describe-services/g)?.length ?? 0) >=+ 6,+ );+ const lostResponse = readState();+ lostResponse.failNextUpdatesAfterMutation = 1;+ writeFileSync(fake.state, JSON.stringify(lostResponse));+ writeFileSync(fake.log, "");+ await assert.rejects(() => awsSetBackgroundWork(single, dir, true, candidatePath), /UpdateServiceResponseLost/);+ assert.equal(manifest().id, disabled.id);+ const compensationCalls = readFileSync(fake.log, "utf8");+ assert.ok(+ (compensationCalls.slice(compensationCalls.lastIndexOf("ecs update-service")).match(/ecs describe-services/g)+ ?.length ?? 0) >= 6,+ );+ writeFileSync(fake.log, "");+ await awsSetBackgroundWork(single, dir, true, candidatePath);+ const active = manifest();+ assert.notEqual(active.tasks.core, disabled.tasks.core);+ assert.deepEqual(active.imageProvenance, initial.imageProvenance);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs run-task|rds |ecr /);+ const core = readState().definitions[active.tasks.core].containerDefinitions[0];+ assert.equal(core.image, candidate.images.core);+ assert.equal(+ core.environment.find((entry: { name: string }) => entry.name === "BACKGROUND_WORK_ENABLED").value,+ "1",+ );+ writeFileSync(fake.log, "");+ await awsSetBackgroundWork(single, dir, true, candidatePath);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service|ecs register-task-definition/);+ candidate.images.core = candidate.images.core.replace(/sha256:.+$/, "sha256:" + "b".repeat(64));+ writeFileSync(candidatePath, JSON.stringify(candidate));+ await assert.rejects(() => awsSetBackgroundWork(single, dir, true, candidatePath), /expected release candidate/);+ const faulted = readState();+ faulted.failNextTransactions = 3;+ writeFileSync(fake.state, JSON.stringify(faulted));+ await assert.rejects(() => awsSetBackgroundWork(single, dir, false), /manifest write failed/);+ assert.equal(manifest().id, active.id);+ assert.equal(readState().services["acme-core"].taskDefinition, active.tasks.core);+ const busy = readState();+ busy.blockDisabledDrain = true;+ writeFileSync(fake.state, JSON.stringify(busy));+ const priorDeadline = process.env.QM_AWS_ROLLOUT_DEADLINE_MS;+ process.env.QM_AWS_ROLLOUT_DEADLINE_MS = "1500";+ try {+ await assert.rejects(() => awsSetBackgroundWork(single, dir, false), /timed out/);+ assert.equal(manifest().id, active.id);+ assert.equal(readState().services["acme-core"].taskDefinition, active.tasks.core);+ } finally {+ if (priorDeadline === undefined) delete process.env.QM_AWS_ROLLOUT_DEADLINE_MS;+ else process.env.QM_AWS_ROLLOUT_DEADLINE_MS = priorDeadline;+ }+ const drifted = readState();+ drifted.services["acme-core"].desiredCount++;+ writeFileSync(fake.state, JSON.stringify(drifted));+ await assert.rejects(() => awsSetBackgroundWork(single, dir, false), /differs from the deployment manifest/);+ } finally {+ process.env.PATH = priorPath;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});+test("AWS up renews the deploy lease with a holder-conditioned update while it runs", async () => {const dir = mkdtempSync(join(tmpdir(), "qm-aws-lease-renew-"));const dockerBin = join(dir, "docker");diff --git a/test/arga-provisioning.test.ts b/test/arga-provisioning.test.tsindex 01e93817d..8f8288d59 100644--- a/test/arga-provisioning.test.ts+++ b/test/arga-provisioning.test.ts@@ -1,6 +1,6 @@import assert from "node:assert/strict";import { test } from "node:test";-import { provisionSlackTwin } from "./live-slack/arga.ts";+import { provisionSlackTwin, teardownTwin } from "./live-slack/arga.ts";test("failed Arga provisions are torn down before one fresh retry", async (t) => {const requests: Array<{ method: string; path: string }> = [];@@ -71,3 +71,39 @@ test("a retry is not started when cleanup cannot be confirmed", async (t) => {await assert.rejects(provisionSlackTwin("key", 60), /cleanup could not be confirmed/);assert.equal(provisions, 1);});++for (const terminal of ["expired", "torn_down"]) {+ test(`Arga cleanup accepts an already ${terminal} twin without another teardown`, async (t) => {+ let requests = 0;+ t.mock.method(globalThis, "fetch", async (input: string | URL | Request, init?: RequestInit) => {+ requests++;+ assert.equal(new URL(String(input)).pathname, "/validate/twins/provision/old-run/status");+ assert.equal(init?.method ?? "GET", "GET");+ return Response.json({ status: terminal });+ });+ await teardownTwin("key", "old-run");+ assert.equal(requests, 1);+ });+}++for (const postFails of [false, true]) {+ test(`Arga cleanup confirms expiry during teardown when POST fails=${postFails}`, async (t) => {+ const methods: string[] = [];+ t.mock.method(globalThis, "fetch", async (_input: string | URL | Request, init?: RequestInit) => {+ const method = init?.method ?? "GET";+ methods.push(method);+ if (method === "POST") return Response.json({}, { status: postFails ? 400 : 200 });+ return Response.json({ status: methods.length === 1 ? "ready" : "expired" });+ });+ await teardownTwin("key", "old-run");+ assert.deepEqual(methods, ["GET", "POST", "GET"]);+ });+}++test("Arga cleanup does not interpret missing or unauthorized status as successful recovery", async (t) => {+ for (const status of [401, 404, 503]) {+ const fetchMock = t.mock.method(globalThis, "fetch", async () => Response.json({}, { status }));+ await assert.rejects(teardownTwin("key", "old-run"), new RegExp(String(status)));+ fetchMock.mock.restore();+ }+});diff --git a/test/live-slack/arga.ts b/test/live-slack/arga.tsindex b1c86053b..2f6dc0bfb 100644--- a/test/live-slack/arga.ts+++ b/test/live-slack/arga.ts@@ -90,11 +90,19 @@ export async function provisionSlackTwin(apiKey: string, ttlMinutes: number): Pr}export async function teardownTwin(apiKey: string, runId: string): Promise<void> {- await argaFetch(apiKey, `/validate/twins/provision/${runId}/teardown`, { method: "POST" });+ const path = `/validate/twins/provision/${runId}`;+ const isGone = (status: string) => status === "torn_down" || status === "expired";+ if (isGone((await argaFetch(apiKey, `${path}/status`)).status)) return;+ try {+ await argaFetch(apiKey, `${path}/teardown`, { method: "POST" });+ } catch (err) {+ if (isGone((await argaFetch(apiKey, `${path}/status`)).status)) return;+ throw err;+ }const deadline = Date.now() + 60_000;for (;;) {const status = await argaFetch(apiKey, `/validate/twins/provision/${runId}/status`);- if (status.status === "torn_down") return;+ if (isGone(status.status)) return;if (Date.now() > deadline) throw new Error(`twin teardown timed out (last status: ${status.status})`);await sleep(2000);}