Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Reuse core task definitions for candidate migrations (#1273)

Regan Bell70fa1342e8bb
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index 372f98ea5..a080f48d8 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -739,7 +739,12 @@ export async function awsMigrateCandidate(config: QmConfig, configDir: string, c
});
}
-async function runAwsCandidateMigration(config: QmConfig, coreImage: string, label: string): Promise<void> {
+async function runAwsCandidateMigration(
+ config: QmConfig,
+ coreImage: string,
+ label: string,
+ existingTaskDefinition?: string,
+): Promise<void> {
const aws = requireAws(config);
const spec = aws.services.core;
if (!spec) throw new CliError("AWS migrations require the core workload");
@@ -763,13 +768,17 @@ async function runAwsCandidateMigration(config: QmConfig, coreImage: string, lab
const temp = mkdtempSync(join(tmpdir(), "qm-migration-task-"));
let taskDefinition: string | undefined;
try {
- const definitionPath = join(temp, "core.json");
- const definition = renderTaskDefinition(config, "core", coreImage, secretArns(config));
- const container = definition.containerDefinitions.find((candidate) => candidate.name === "core");
- if (!container) throw new CliError("AWS migration task definition has no core container");
- delete container.healthCheck;
- writeFileSync(definitionPath, JSON.stringify(definition));
- taskDefinition = registerTaskDefinition(config, definitionPath);
+ if (existingTaskDefinition) {
+ taskDefinition = existingTaskDefinition;
+ } else {
+ const definitionPath = join(temp, "core.json");
+ const definition = renderTaskDefinition(config, "core", coreImage, secretArns(config));
+ const container = definition.containerDefinitions.find((candidate) => candidate.name === "core");
+ if (!container) throw new CliError("AWS migration task definition has no core container");
+ delete container.healthCheck;
+ writeFileSync(definitionPath, JSON.stringify(definition));
+ taskDefinition = registerTaskDefinition(config, definitionPath);
+ }
header(`qm migrate — ${config.orgId} (AWS candidate ${label})`);
const started = awsJson<{
tasks?: Array<{ taskArn?: string }>;
@@ -816,7 +825,7 @@ async function runAwsCandidateMigration(config: QmConfig, coreImage: string, lab
}
ok(`candidate ${label} migrations applied`);
} finally {
- if (taskDefinition) {
+ if (taskDefinition && !existingTaskDefinition) {
try {
awsText(aws, ["ecs", "deregister-task-definition", "--task-definition", taskDefinition]);
} catch (error) {
@@ -2154,10 +2163,6 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
if (aws.predeployDbSnapshot === false)
note("pre-deploy database restore point: disabled (aws.predeployDbSnapshot)");
else dbRestorePoint = assertPredeployDbRestorePoint(config);
- if (candidate?.images.core && services.includes("core")) {
- await runAwsCandidateMigration(config, candidate.images.core, candidate.label);
- migratedCandidate = true;
- }
if (current?.layer) {
desiredLayer = current.layer;
} else {
@@ -2228,6 +2233,10 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
const taskDefinition = registerTaskDefinition(config, file);
targets[item.service] = taskDefinition;
}
+ if (candidate?.images.core && services.includes("core")) {
+ await runAwsCandidateMigration(config, candidate.images.core, candidate.label, targets.core!);
+ migratedCandidate = true;
+ }
const rolloutTargets = Object.fromEntries(
services
.filter(
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index 883cc1b42..48cc5268b 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -2727,6 +2727,15 @@ test("AWS builds one immutable candidate manifest and deploys its exact digest w
const current = state.dynamo["deployment/current"].manifestId.S;
const deployed = JSON.parse(state.dynamo[`deployment/manifest/${current}`].manifest.S);
assert.equal(state.definitions[deployed.tasks.core].containerDefinitions[0].image, candidate.images.core);
+ assert.equal(deployCalls.match(/ecs register-task-definition/g)?.length, 1);
+ assert.doesNotMatch(deployCalls, /ecs deregister-task-definition/);
+ assert.ok(deployCalls.includes(`--task-definition ${deployed.tasks.core} --launch-type FARGATE`));
+ assert.ok(state.definitions[deployed.tasks.core].containerDefinitions[0].healthCheck);
+ writeFileSync(fake.log, "");
+ await awsUp(single, dir, { yes: true, candidate: candidatePath });
+ const repeatCalls = readFileSync(fake.log, "utf8");
+ assert.doesNotMatch(repeatCalls, /ecs (?:register-task-definition|deregister-task-definition|update-service)/);
+ assert.ok(repeatCalls.includes(`--task-definition ${deployed.tasks.core} --launch-type FARGATE`));
assert.ok(Number.isFinite(Date.parse(deployed.dbRestorePoint)));
} finally {
process.env.PATH = priorPath;
@@ -2975,6 +2984,41 @@ test("AWS migration failure deregisters the candidate task and releases the depl
}
});
+test("AWS candidate deploy migration failure preserves the runtime and releases the lease", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-candidate-migrate-failure-"));
+ const candidatePath = join(dir, "candidate.json");
+ writeFileSync(
+ candidatePath,
+ JSON.stringify({
+ contract: 1,
+ accountId: "123456789012",
+ region: "us-west-2",
+ label: "candidate-deadbeef",
+ images: {
+ core: `123456789012.dkr.ecr.us-west-2.amazonaws.com/qm-core@sha256:${"a".repeat(64)}`,
+ },
+ imageProvenance: { core: { kind: "source-build", source: "checkout" } },
+ }),
+ );
+ const fake = statefulAws(dir, oneServiceConfig(), {}, { migrationExitCode: 1 });
+ const before = JSON.parse(readFileSync(fake.state, "utf8"));
+ try {
+ await assert.rejects(
+ () => awsUp(oneServiceConfig(), dir, { yes: true, candidate: candidatePath }),
+ /migration task failed/,
+ );
+ const calls = readFileSync(fake.log, "utf8");
+ assert.doesNotMatch(calls, /ecs (?:deregister-task-definition|update-service)/);
+ assert.equal(calls.match(/ecs register-task-definition/g)?.length, 1);
+ const state = JSON.parse(readFileSync(fake.state, "utf8"));
+ assert.equal(state.services["acme-core"].taskDefinition, before.services["acme-core"].taskDefinition);
+ assert.match(calls, /dynamodb delete-item/);
+ } finally {
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
const twoServiceConfig = (): QmConfig => ({
...config,
services: ["core", "web-ui"],