Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Activate background work without redeploying the candidate (#1029)

Regan Bell3bd425b96329
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index d5b4b6020..754586e62 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -1240,11 +1240,16 @@ interface RolloutTarget {
taskDefinition: string;
desiredCount: number;
deploymentId?: string;
+ waitForDrain?: boolean;
}
-async function awaitServiceTargets(config: QmConfig, expected: Record<string, RolloutTarget>): Promise<void> {
+async function awaitServiceTargets(
+ config: QmConfig,
+ expected: Record<string, RolloutTarget>,
+ timeoutMs?: number,
+): Promise<void> {
const pollMs = envNum("QM_AWS_ROLLOUT_POLL_MS", 15_000);
- const deadline = Date.now() + envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 20 * 60_000);
+ const deadline = Date.now() + (timeoutMs ?? envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 20 * 60_000));
let healthyStreak = 0;
let failurePolls = new Map<string, number>();
let describeFailures = 0;
@@ -1329,6 +1334,12 @@ async function awaitServiceTargets(config: QmConfig, expected: Record<string, Ro
}
} else if (running < want.desiredCount) {
waiting.push(`${workload} (${running}/${want.desiredCount})`);
+ } else if (
+ want.waitForDrain &&
+ (deployment.rolloutState !== "COMPLETED" || state.deployments?.length !== 1) &&
+ state.deploymentConfiguration?.strategy !== "BLUE_GREEN"
+ ) {
+ waiting.push(`${workload} (prior tasks are still draining)`);
} else if (state.deploymentConfiguration?.strategy === "BLUE_GREEN") {
const nativeStatus = nativeStatuses.get(workload);
if (nativeStatus === "SUCCESSFUL") continue;
@@ -1676,6 +1687,7 @@ async function applyServiceTargets(
config: QmConfig,
targets: Record<string, string>,
desiredCounts?: Record<string, number>,
+ options: { waitForDrain?: boolean; waitForCompensationDrain?: boolean; timeoutMs?: number } = {},
): Promise<void> {
const aws = requireAws(config);
const workloads = Object.keys(targets);
@@ -1708,9 +1720,14 @@ async function applyServiceTargets(
Object.fromEntries(
workloads.map((workload) => [
workload,
- { taskDefinition: targets[workload]!, desiredCount: expectedCounts[workload]! },
+ {
+ taskDefinition: targets[workload]!,
+ desiredCount: expectedCounts[workload]!,
+ waitForDrain: options.waitForDrain,
+ },
]),
),
+ options.timeoutMs,
);
} catch (error) {
const restoreFailures: string[] = [];
@@ -1739,9 +1756,14 @@ async function applyServiceTargets(
Object.fromEntries(
changed.map((workload) => [
workload,
- { taskDefinition: before.tasks[workload]!, desiredCount: before.counts[workload]! },
+ {
+ taskDefinition: before.tasks[workload]!,
+ desiredCount: before.counts[workload]!,
+ waitForDrain: options.waitForCompensationDrain,
+ },
]),
),
+ options.timeoutMs,
);
} catch (restoreError) {
restoreFailures.push(errMessage(restoreError));
@@ -2471,6 +2493,157 @@ export async function awsRollback(
ok(`rolled back ${config.orgId}`);
}
+export function taskDefinitionForBackgroundWork(
+ task: Record<string, unknown>,
+ enabled: boolean,
+): Record<string, unknown> {
+ const containers = structuredClone(task.containerDefinitions) as Array<Record<string, unknown>> | undefined;
+ const core = containers?.find((container) => container.name === "core");
+ if (!core) throw new CliError("core task definition has no core container");
+ const secrets = core.secrets as Array<{ name: string }> | undefined;
+ if (secrets?.some((entry) => entry.name === "BACKGROUND_WORK_ENABLED")) {
+ throw new CliError("BACKGROUND_WORK_ENABLED cannot be changed while supplied as a secret");
+ }
+ const environment = (core.environment ?? []) as Array<{ name: string; value: string }>;
+ core.environment = [
+ ...environment.filter((entry) => entry.name !== "BACKGROUND_WORK_ENABLED"),
+ { name: "BACKGROUND_WORK_ENABLED", value: enabled ? "1" : "0" },
+ ];
+ const fields = [
+ "family",
+ "taskRoleArn",
+ "executionRoleArn",
+ "networkMode",
+ "volumes",
+ "placementConstraints",
+ "requiresCompatibilities",
+ "cpu",
+ "memory",
+ "pidMode",
+ "ipcMode",
+ "proxyConfiguration",
+ "inferenceAccelerators",
+ "ephemeralStorage",
+ "runtimePlatform",
+ "enableFaultInjection",
+ ];
+ return {
+ ...Object.fromEntries(fields.flatMap((field) => (task[field] == null ? [] : [[field, task[field]]]))),
+ containerDefinitions: containers,
+ };
+}
+
+export async function awsSetBackgroundWork(
+ config: QmConfig,
+ configDir: string,
+ enabled: boolean,
+ candidatePath?: string,
+): Promise<void> {
+ const { aws, workloads } = awsTopology(config, configDir);
+ if (!workloads.includes("core")) throw new CliError("background work requires the core workload");
+ const candidate = candidatePath ? releaseCandidate(config, candidatePath) : undefined;
+ assertAwsCallerAccount(aws);
+ await withAwsLease(aws, async () => {
+ const current = currentDeploymentManifest(aws);
+ if (!current) throw new CliError("background work requires a recorded deployment");
+ const states = describedServices(config, workloads);
+ assertOwnedServices(config, states, workloads);
+ const before = serviceSnapshotFromStates(states, workloads);
+ for (const workload of workloads) {
+ if (
+ current.tasks[workload] !== before.tasks[workload] ||
+ current.counts?.[workload] !== before.counts[workload]
+ ) {
+ throw new CliError(`cannot change background work while ${workload} differs from the deployment manifest`);
+ }
+ }
+ if (!before.counts.core) throw new CliError("cannot change background work while core is scaled down");
+ let coreTask: Record<string, unknown> | undefined;
+ for (const workload of candidate ? workloads : ["core"]) {
+ const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
+ "ecs",
+ "describe-task-definition",
+ "--task-definition",
+ before.tasks[workload]!,
+ ]).taskDefinition;
+ const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
+ (item) => item.name === workload,
+ );
+ if (!task || typeof container?.image !== "string" || !isPinnedWorkloadImage(config, workload, container.image)) {
+ throw new CliError(`${workload} does not have a trusted digest-pinned image`);
+ }
+ if (
+ candidate &&
+ (container.image !== candidate.images[workload] ||
+ !candidate.imageProvenance[workload] ||
+ canonicalJson(candidate.imageProvenance[workload]) !== canonicalJson(current.imageProvenance?.[workload]))
+ ) {
+ throw new CliError(`${workload} does not match the expected release candidate`);
+ }
+ if (workload === "core") coreTask = task;
+ }
+ const desired = taskDefinitionForBackgroundWork(coreTask!, enabled);
+ const core = (coreTask!.containerDefinitions as Array<Record<string, unknown>>).find(
+ (item) => item.name === "core",
+ )!;
+ const environment = (core.environment ?? []) as Array<{ name: string; value: string }>;
+ const currentFlag = environment.find((entry) => entry.name === "BACKGROUND_WORK_ENABLED")?.value;
+ const timeoutMs = envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000);
+ if (currentFlag === (enabled ? "1" : "0")) {
+ await awaitServiceTargets(
+ config,
+ {
+ core: { taskDefinition: before.tasks.core!, desiredCount: before.counts.core, waitForDrain: !enabled },
+ },
+ timeoutMs,
+ );
+ return;
+ }
+ const dir = mkdtempSync(join(tmpdir(), "qm-background-work-"));
+ try {
+ const file = join(dir, "core.json");
+ writeFileSync(file, JSON.stringify(desired));
+ const target = registerTaskDefinition(config, file);
+ await applyServiceTargets(
+ config,
+ { core: target },
+ { core: before.counts.core },
+ { waitForDrain: !enabled, waitForCompensationDrain: true, timeoutMs },
+ );
+ try {
+ recordDeploymentManifest(
+ aws,
+ { ...before.tasks, core: target },
+ {
+ counts: current.counts,
+ imageLabel: current.imageLabel,
+ dbRestorePoint: current.dbRestorePoint,
+ layer: current.layer,
+ imageProvenance: current.imageProvenance,
+ },
+ );
+ } catch (error) {
+ const failures: string[] = [];
+ try {
+ await applyServiceTargets(
+ config,
+ { core: before.tasks.core! },
+ { core: before.counts.core },
+ { waitForDrain: true, waitForCompensationDrain: true, timeoutMs },
+ );
+ manifestTransaction(aws, current, current.id);
+ } catch (restoreError) {
+ failures.push(`restoring background work: ${errMessage(restoreError)}`);
+ }
+ throwAfterCompensation(error, failures);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+ ok(`background work ${enabled ? "enabled" : "disabled"}`);
+}
+
function envValues(configDir: string, path: string | undefined): Map<string, string> {
const file = resolve(path ?? join(configDir, ".env"));
if (!existsSync(file)) {
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index 2465f240b..997fc15c9 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -18,6 +18,8 @@ import {
awsMigrateCandidate,
awsRollback,
awsSecretsPush,
+ awsSetBackgroundWork,
+ taskDefinitionForBackgroundWork,
awsStatus,
awsUp,
githubTrustSubject,
@@ -213,6 +215,7 @@ function statefulAws(
failPromotion?: boolean;
promotionAlreadyCurrent?: boolean;
drainRollout?: boolean;
+ drainPolls?: number;
primaryFailedTasks?: boolean;
rolloutFailed?: boolean;
transientFailedTaskPolls?: number;
@@ -301,6 +304,9 @@ else if (a.includes("ecs describe-services")) {
const start = args.indexOf("--services") + 1;
const end = Math.min(...[args.indexOf("--output", start), args.indexOf("--region", start)].filter((index) => index >= 0));
const names = args.slice(start, end);
+ const blocked = s.blockDisabledDrain && names.some((name) => s.definitions[s.services[name]?.taskDefinition]?.containerDefinitions?.some((container) => container.name === "core" && container.environment?.some((entry) => entry.name === "BACKGROUND_WORK_ENABLED" && entry.value === "0")));
+ const draining = ${JSON.stringify(opts.drainRollout ?? false)} || (s.drainPolls || 0) > 0 || blocked;
+ if (s.drainPolls > 0) { s.drainPolls--; save(); }
const transientFailedTaskPolls = ${JSON.stringify(opts.transientFailedTaskPolls ?? 0)};
let transientlyFailing = false;
if (transientFailedTaskPolls && s.updated) {
@@ -328,7 +334,7 @@ else if (a.includes("ecs describe-services")) {
if (name === staleName && service.previousTaskDefinition) {
return [{ serviceName: name, status: "ACTIVE", desiredCount: service.desiredCount, runningCount: service.desiredCount, taskDefinition: service.previousTaskDefinition, deployments: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.previousTaskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }], tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];
}
- const deployments = ${JSON.stringify(opts.drainRollout ?? false)}
+ const deployments = draining
? [
{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "IN_PROGRESS", runningCount: service.desiredCount, failedTasks: 1 },
{ id: "old-protected", status: "ACTIVE", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: 1, failedTasks: 0 },
@@ -340,7 +346,7 @@ else if (a.includes("ecs describe-services")) {
: blueGreenBakePolls
? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: 0 }]
: [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: service.desiredCount, failedTasks: transientFailedTaskPolls && s.updated ? 1 : 0 }];
- return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: ${JSON.stringify(opts.drainRollout ?? false)} ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];
+ return [{ serviceName: name, status: "ACTIVE", launchType: "FARGATE", platformVersion: "1.4.0", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-a", "subnet-b"], securityGroups: ["sg-core"], assignPublicIp: "ENABLED" } }, desiredCount: service.desiredCount, runningCount: draining ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, deploymentConfiguration: { strategy: blueGreenBakePolls ? "BLUE_GREEN" : "ROLLING" }, deployments, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)}, ...(blueGreenBakePolls ? { advancedConfiguration: { alternateTargetGroupArn: ${JSON.stringify(frontAlternateArn)}, productionListenerRule: ${JSON.stringify(productionRuleArn)} } } : {}) }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }];
}), failures: names.filter((name) => !s.services[name]).map((name) => ({ arn: name, reason: "MISSING" })) }));
}
else if (a.includes("ecs list-service-deployments") && ${JSON.stringify(opts.failNativeStatusOnceAfterUpdate ?? false)} && s.updated && !s.nativeStatusFailedOnce) {
@@ -397,8 +403,10 @@ else if (a.includes("ecs update-service")) {
if (!${JSON.stringify(opts.ignoreUpdate ?? false)} && args.includes("--desired-count")) service.desiredCount = Number(after("--desired-count"));
if (!${JSON.stringify(opts.ignoreUpdate ?? false)}) service.deploymentId = "ecs-svc/" + name + "-" + (++s.revision);
s.updated = true;
+ s.drainPolls = ${JSON.stringify(opts.drainPolls ?? 0)};
s.blueGreenPolls = 0;
- if (${JSON.stringify(opts.failFirstUpdateAfterMutation ?? false)} && !s.failedFirstUpdate) {
+ if (s.failNextUpdatesAfterMutation > 0 || (${JSON.stringify(opts.failFirstUpdateAfterMutation ?? false)} && !s.failedFirstUpdate)) {
+ if (s.failNextUpdatesAfterMutation > 0) s.failNextUpdatesAfterMutation--;
s.failedFirstUpdate = true;
save();
console.error("UpdateServiceResponseLost");
@@ -416,6 +424,7 @@ else if (a.includes("dynamodb get-item")) {
console.log(JSON.stringify(s.dynamo[key] ? { Item: s.dynamo[key] } : {}));
}
else if (a.includes("dynamodb transact-write-items")) {
+ if (s.failNextTransactions > 0) { s.failNextTransactions--; save(); console.error("TransactionRejected"); process.exit(1); }
if (${JSON.stringify(opts.failTransactions ?? false)}) { console.error("TransactionRejected"); process.exit(1); }
const failTransactionPuts = ${JSON.stringify(opts.failTransactionPuts ?? 0)};
if (failTransactionPuts) {
@@ -1396,6 +1405,149 @@ test("AWS up coalesces a requested restart into one deployment even when the tas
}
});
+test("background mode changes preserve task settings and reject secret-controlled modes", () => {
+ const task = {
+ family: "core",
+ taskDefinitionArn: "read-only",
+ status: "ACTIVE",
+ cpu: "1024",
+ volumes: [{ name: "scratch" }],
+ containerDefinitions: [
+ {
+ name: "core",
+ image: "pinned",
+ environment: [{ name: "OTHER", value: "kept" }],
+ secrets: [{ name: "TOKEN", valueFrom: "secret" }],
+ },
+ { name: "sidecar", image: "unchanged" },
+ ],
+ };
+ const changed = taskDefinitionForBackgroundWork(task, false);
+ assert.equal(changed.taskDefinitionArn, undefined);
+ assert.equal(changed.status, undefined);
+ assert.deepEqual(changed.volumes, task.volumes);
+ const containers = changed.containerDefinitions as typeof task.containerDefinitions;
+ assert.deepEqual(containers[1], task.containerDefinitions[1]);
+ assert.deepEqual(containers[0]!.secrets, task.containerDefinitions[0]!.secrets);
+ assert.deepEqual(containers[0]!.environment, [
+ { name: "OTHER", value: "kept" },
+ { name: "BACKGROUND_WORK_ENABLED", value: "0" },
+ ]);
+ assert.equal(task.containerDefinitions[0]!.environment!.length, 1);
+ assert.throws(
+ () =>
+ taskDefinitionForBackgroundWork(
+ { containerDefinitions: [{ name: "core", secrets: [{ name: "BACKGROUND_WORK_ENABLED" }] }] },
+ true,
+ ),
+ /supplied as a secret/,
+ );
+});
+
+test("background activation preserves candidate and manifest without another migration; demotion drains old tasks", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-background-"));
+ const dockerBin = join(dir, "docker");
+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);
+ chmodSync(dockerBin, 0o755);
+ const base = oneServiceConfig();
+ const single = { ...base, env: { ...base.env, core: { ...base.env.core, BACKGROUND_WORK_ENABLED: "false" } } };
+ const fake = statefulAws(dir, single, {}, { drainPolls: 6 });
+ const priorPath = process.env.PATH;
+ process.env.PATH = `${dir}:${priorPath}`;
+ try {
+ await awsUp(single, dir, { yes: true });
+ const readState = () => JSON.parse(readFileSync(fake.state, "utf8"));
+ const manifest = () => {
+ const state = readState();
+ return JSON.parse(
+ state.dynamo[`deployment/manifest/${state.dynamo["deployment/current"].manifestId.S}`].manifest.S,
+ );
+ };
+ const initial = manifest();
+ const candidate = {
+ contract: 1,
+ accountId: single.aws!.accountId,
+ region: single.aws!.region,
+ label: "verified",
+ images: { core: readState().definitions[initial.tasks.core].containerDefinitions[0].image },
+ imageProvenance: initial.imageProvenance,
+ };
+ const candidatePath = join(dir, "candidate.json");
+ writeFileSync(candidatePath, JSON.stringify(candidate));
+ writeFileSync(fake.log, "");
+ await awsSetBackgroundWork(single, dir, true, candidatePath);
+ assert.match(readFileSync(fake.log, "utf8"), /ecs register-task-definition/);
+ writeFileSync(fake.log, "");
+ await awsSetBackgroundWork(single, dir, false);
+ const disabled = manifest();
+ assert.notEqual(disabled.tasks.core, initial.tasks.core);
+ assert.deepEqual(disabled.imageProvenance, initial.imageProvenance);
+ assert.deepEqual(disabled.layer, initial.layer);
+ assert.equal(disabled.dbRestorePoint, initial.dbRestorePoint);
+ const disableCalls = readFileSync(fake.log, "utf8");
+ assert.ok(
+ (disableCalls.slice(disableCalls.indexOf("ecs update-service")).match(/ecs describe-services/g)?.length ?? 0) >=
+ 6,
+ );
+ const lostResponse = readState();
+ lostResponse.failNextUpdatesAfterMutation = 1;
+ writeFileSync(fake.state, JSON.stringify(lostResponse));
+ writeFileSync(fake.log, "");
+ await assert.rejects(() => awsSetBackgroundWork(single, dir, true, candidatePath), /UpdateServiceResponseLost/);
+ assert.equal(manifest().id, disabled.id);
+ const compensationCalls = readFileSync(fake.log, "utf8");
+ assert.ok(
+ (compensationCalls.slice(compensationCalls.lastIndexOf("ecs update-service")).match(/ecs describe-services/g)
+ ?.length ?? 0) >= 6,
+ );
+ writeFileSync(fake.log, "");
+ await awsSetBackgroundWork(single, dir, true, candidatePath);
+ const active = manifest();
+ assert.notEqual(active.tasks.core, disabled.tasks.core);
+ assert.deepEqual(active.imageProvenance, initial.imageProvenance);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs run-task|rds |ecr /);
+ const core = readState().definitions[active.tasks.core].containerDefinitions[0];
+ assert.equal(core.image, candidate.images.core);
+ assert.equal(
+ core.environment.find((entry: { name: string }) => entry.name === "BACKGROUND_WORK_ENABLED").value,
+ "1",
+ );
+ writeFileSync(fake.log, "");
+ await awsSetBackgroundWork(single, dir, true, candidatePath);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service|ecs register-task-definition/);
+ candidate.images.core = candidate.images.core.replace(/sha256:.+$/, "sha256:" + "b".repeat(64));
+ writeFileSync(candidatePath, JSON.stringify(candidate));
+ await assert.rejects(() => awsSetBackgroundWork(single, dir, true, candidatePath), /expected release candidate/);
+ const faulted = readState();
+ faulted.failNextTransactions = 3;
+ writeFileSync(fake.state, JSON.stringify(faulted));
+ await assert.rejects(() => awsSetBackgroundWork(single, dir, false), /manifest write failed/);
+ assert.equal(manifest().id, active.id);
+ assert.equal(readState().services["acme-core"].taskDefinition, active.tasks.core);
+ const busy = readState();
+ busy.blockDisabledDrain = true;
+ writeFileSync(fake.state, JSON.stringify(busy));
+ const priorDeadline = process.env.QM_AWS_ROLLOUT_DEADLINE_MS;
+ process.env.QM_AWS_ROLLOUT_DEADLINE_MS = "1500";
+ try {
+ await assert.rejects(() => awsSetBackgroundWork(single, dir, false), /timed out/);
+ assert.equal(manifest().id, active.id);
+ assert.equal(readState().services["acme-core"].taskDefinition, active.tasks.core);
+ } finally {
+ if (priorDeadline === undefined) delete process.env.QM_AWS_ROLLOUT_DEADLINE_MS;
+ else process.env.QM_AWS_ROLLOUT_DEADLINE_MS = priorDeadline;
+ }
+ const drifted = readState();
+ drifted.services["acme-core"].desiredCount++;
+ writeFileSync(fake.state, JSON.stringify(drifted));
+ await assert.rejects(() => awsSetBackgroundWork(single, dir, false), /differs from the deployment manifest/);
+ } finally {
+ process.env.PATH = priorPath;
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
test("AWS up renews the deploy lease with a holder-conditioned update while it runs", async () => {
const dir = mkdtempSync(join(tmpdir(), "qm-aws-lease-renew-"));
const dockerBin = join(dir, "docker");
diff --git a/test/arga-provisioning.test.ts b/test/arga-provisioning.test.ts
index 01e93817d..8f8288d59 100644
--- a/test/arga-provisioning.test.ts
+++ b/test/arga-provisioning.test.ts
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { test } from "node:test";
-import { provisionSlackTwin } from "./live-slack/arga.ts";
+import { provisionSlackTwin, teardownTwin } from "./live-slack/arga.ts";
test("failed Arga provisions are torn down before one fresh retry", async (t) => {
const requests: Array<{ method: string; path: string }> = [];
@@ -71,3 +71,39 @@ test("a retry is not started when cleanup cannot be confirmed", async (t) => {
await assert.rejects(provisionSlackTwin("key", 60), /cleanup could not be confirmed/);
assert.equal(provisions, 1);
});
+
+for (const terminal of ["expired", "torn_down"]) {
+ test(`Arga cleanup accepts an already ${terminal} twin without another teardown`, async (t) => {
+ let requests = 0;
+ t.mock.method(globalThis, "fetch", async (input: string | URL | Request, init?: RequestInit) => {
+ requests++;
+ assert.equal(new URL(String(input)).pathname, "/validate/twins/provision/old-run/status");
+ assert.equal(init?.method ?? "GET", "GET");
+ return Response.json({ status: terminal });
+ });
+ await teardownTwin("key", "old-run");
+ assert.equal(requests, 1);
+ });
+}
+
+for (const postFails of [false, true]) {
+ test(`Arga cleanup confirms expiry during teardown when POST fails=${postFails}`, async (t) => {
+ const methods: string[] = [];
+ t.mock.method(globalThis, "fetch", async (_input: string | URL | Request, init?: RequestInit) => {
+ const method = init?.method ?? "GET";
+ methods.push(method);
+ if (method === "POST") return Response.json({}, { status: postFails ? 400 : 200 });
+ return Response.json({ status: methods.length === 1 ? "ready" : "expired" });
+ });
+ await teardownTwin("key", "old-run");
+ assert.deepEqual(methods, ["GET", "POST", "GET"]);
+ });
+}
+
+test("Arga cleanup does not interpret missing or unauthorized status as successful recovery", async (t) => {
+ for (const status of [401, 404, 503]) {
+ const fetchMock = t.mock.method(globalThis, "fetch", async () => Response.json({}, { status }));
+ await assert.rejects(teardownTwin("key", "old-run"), new RegExp(String(status)));
+ fetchMock.mock.restore();
+ }
+});
diff --git a/test/live-slack/arga.ts b/test/live-slack/arga.ts
index b1c86053b..2f6dc0bfb 100644
--- a/test/live-slack/arga.ts
+++ b/test/live-slack/arga.ts
@@ -90,11 +90,19 @@ export async function provisionSlackTwin(apiKey: string, ttlMinutes: number): Pr
}
export async function teardownTwin(apiKey: string, runId: string): Promise<void> {
- await argaFetch(apiKey, `/validate/twins/provision/${runId}/teardown`, { method: "POST" });
+ const path = `/validate/twins/provision/${runId}`;
+ const isGone = (status: string) => status === "torn_down" || status === "expired";
+ if (isGone((await argaFetch(apiKey, `${path}/status`)).status)) return;
+ try {
+ await argaFetch(apiKey, `${path}/teardown`, { method: "POST" });
+ } catch (err) {
+ if (isGone((await argaFetch(apiKey, `${path}/status`)).status)) return;
+ throw err;
+ }
const deadline = Date.now() + 60_000;
for (;;) {
const status = await argaFetch(apiKey, `/validate/twins/provision/${runId}/status`);
- if (status.status === "torn_down") return;
+ if (isGone(status.status)) return;
if (Date.now() > deadline) throw new Error(`twin teardown timed out (last status: ${status.status})`);
await sleep(2000);
}