Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Emit atomic progress receipts for candidate deployments (#1275)

Regan Belle0966ba8bcc9
diff --git a/cli/README.md b/cli/README.md
index 9e21ba773..ed7d16062 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -77,6 +77,16 @@ configuration only, so it prints that timestamp as the matching data restore
point (`aws rds restore-db-instance-to-point-in-time`);
`aws.predeployDbSnapshot: false` opts out.
+Batch operators can set `QM_DEPLOY_PROGRESS_FILE` to a new absolute file path and
+`QM_DEPLOY_PROGRESS_TOKEN` to a unique attempt identifier for candidate `up --yes`.
+After all forward service updates have been submitted, the CLI atomically creates
+a private JSON receipt with `phase: "monitoring"`, `token`, `orgId`, and `targets`
+(the selected workload-to-task-definition mapping). It then continues health
+checks, manifest recording, and rollback under the deployment lease. This receipt
+only permits the batch runner to release a submission slot; the CLI exit status
+still determines success. Use a fresh path and token for every attempt. Missing
+receipts must keep the submission slot occupied until the CLI exits.
+
`sandbox build` is a local validation build of the sandbox layer image. At runtime
sandboxes boot their platform's stock image; tools and skills arrive through the
deployment-layer sync, which every ordinary `up` performs.
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index a080f48d8..f828c0684 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -3,9 +3,19 @@ import https from "node:https";
import { createHash, randomUUID } from "node:crypto";
import { spawnSync } from "node:child_process";
import { lookup, resolveCname } from "node:dns/promises";
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import {
+ accessSync,
+ constants,
+ existsSync,
+ linkSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ statSync,
+ writeFileSync,
+} from "node:fs";
import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
+import { dirname, isAbsolute, join, resolve } from "node:path";
import {
acquireAwsLease as acquireLease,
awsText,
@@ -1728,6 +1738,7 @@ async function applyServiceTargets(
waitForCompensationDrain?: boolean;
timeoutMs?: number;
webBeforePortal?: boolean;
+ onSubmitted?: () => void;
} = {},
): Promise<void> {
const aws = requireAws(config);
@@ -1771,6 +1782,7 @@ async function applyServiceTargets(
changed.push(workload);
awsText(aws, args);
}
+ options.onSubmitted?.();
await awaitServiceTargets(
config,
Object.fromEntries(
@@ -1968,7 +1980,43 @@ function confirmDbRestorePointCovered(config: QmConfig, restorePoint: string): v
}
}
+function deploymentProgress(
+ config: QmConfig,
+ opts: AwsUpOpts,
+): ((targets: Record<string, string>) => void) | undefined {
+ const file = process.env.QM_DEPLOY_PROGRESS_FILE;
+ const token = process.env.QM_DEPLOY_PROGRESS_TOKEN;
+ if (file === undefined && token === undefined) return undefined;
+ if (!file || !token || !isAbsolute(file) || !opts.candidate || !opts.yes || opts.dryRun || opts.buildOnly) {
+ throw new CliError(
+ "deployment progress requires an absolute QM_DEPLOY_PROGRESS_FILE, QM_DEPLOY_PROGRESS_TOKEN, and candidate up --yes",
+ );
+ }
+ if (existsSync(file)) throw new CliError("deployment progress file must not already exist");
+ if (!statSync(dirname(file)).isDirectory()) throw new CliError("deployment progress parent must be a directory");
+ accessSync(dirname(file), constants.W_OK);
+ return (targets) => {
+ const temporary = `${file}.${randomUUID()}.tmp`;
+ try {
+ writeFileSync(temporary, JSON.stringify({ phase: "monitoring", token, orgId: config.orgId, targets }), {
+ flag: "wx",
+ mode: 0o600,
+ });
+ linkSync(temporary, file);
+ } catch (error) {
+ warn(`could not publish deployment progress: ${errMessage(error)}`);
+ } finally {
+ try {
+ rmSync(temporary, { force: true });
+ } catch (error) {
+ warn(`could not remove deployment progress temporary file: ${errMessage(error)}`);
+ }
+ }
+ };
+}
+
export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpts = {}): Promise<void> {
+ const reportProgress = deploymentProgress(config, opts);
const topology = awsTopology(config, _configDir);
const { aws } = topology;
if (new URL(config.publicUrl).protocol !== "https:") {
@@ -2253,9 +2301,11 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
Object.fromEntries(
Object.keys(rolloutTargets).map((service) => [service, workloadDesiredCount(config, service)]),
),
- { webBeforePortal: true },
+ { webBeforePortal: true, onSubmitted: reportProgress ? () => reportProgress(targets) : undefined },
);
applied = true;
+ } else {
+ reportProgress?.(targets);
}
await awaitServiceTargets(
config,
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index 48cc5268b..f4de5d0eb 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -4,7 +4,7 @@ import { EventEmitter } from "node:events";
import assert from "node:assert/strict";
import { spawn, spawnSync } from "node:child_process";
import { createHash, createHmac } from "node:crypto";
-import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import {
@@ -256,6 +256,8 @@ function statefulAws(
blueGreenBakePolls?: number;
migrationExitCode?: number;
foreignServiceTags?: boolean;
+ progressFile?: string;
+ sabotageProgress?: boolean;
} = {},
): {
log: string;
@@ -313,6 +315,9 @@ function statefulAws(
dir,
`
const args = process.argv.slice(2);
+const progressFile = ${JSON.stringify(opts.progressFile)};
+if (progressFile) fs.appendFileSync(progressFile + ".samples", JSON.stringify({ args, published: fs.existsSync(progressFile) }) + "\\n");
+if (progressFile && ${JSON.stringify(opts.sabotageProgress ?? false)} && a.includes("ecs update-service")) fs.mkdirSync(progressFile, { recursive: true });
const statePath = ${JSON.stringify(state)};
const s = JSON.parse(fs.readFileSync(statePath, "utf8"));
const save = () => fs.writeFileSync(statePath, JSON.stringify(s));
@@ -5234,3 +5239,122 @@ else console.log("");`,
}
});
}
+
+for (const mode of ["success", "migration-failure", "update-failure", "write-failure"] as const) {
+ test(`AWS candidate deployment progress receipt: ${mode}`, async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-progress-"));
+ const progressFile = join(dir, "progress.json");
+ const candidatePath = join(dir, "candidate.json");
+ const single = mode === "success" ? twoServiceConfig() : oneServiceConfig();
+ writeFileSync(
+ candidatePath,
+ JSON.stringify({
+ contract: 1,
+ accountId: "123456789012",
+ region: "us-west-2",
+ label: "candidate-progress",
+ images: Object.fromEntries(
+ single.services.map((name) => [
+ name,
+ `123456789012.dkr.ecr.us-west-2.amazonaws.com/${single.aws!.services[name]!.ecrRepository}@sha256:${"a".repeat(64)}`,
+ ]),
+ ),
+ imageProvenance: Object.fromEntries(
+ single.services.map((name) => [name, { kind: "source-build", source: "checkout" }]),
+ ),
+ }),
+ );
+ const fake = statefulAws(
+ dir,
+ single,
+ {},
+ {
+ progressFile,
+ migrationExitCode: mode === "migration-failure" ? 1 : 0,
+ failFirstUpdateAfterMutation: mode === "update-failure",
+ sabotageProgress: mode === "write-failure",
+ },
+ );
+ const priorFile = process.env.QM_DEPLOY_PROGRESS_FILE;
+ const priorToken = process.env.QM_DEPLOY_PROGRESS_TOKEN;
+ process.env.QM_DEPLOY_PROGRESS_FILE = progressFile;
+ process.env.QM_DEPLOY_PROGRESS_TOKEN = "attempt-unique-token";
+ try {
+ const deploy = () => awsUp(single, dir, { yes: true, candidate: candidatePath });
+ if (mode === "migration-failure" || mode === "update-failure") {
+ await assert.rejects(deploy);
+ assert.equal(existsSync(progressFile), false);
+ } else {
+ await deploy();
+ const samples = readFileSync(progressFile + ".samples", "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ const update = samples.findLastIndex((sample) => sample.args.includes("update-service"));
+ assert.ok(update >= 0);
+ assert.ok(samples.slice(0, update + 1).every((sample) => !sample.published));
+ if (mode === "success") {
+ assert.equal(samples[update + 1].published, true);
+ const state = JSON.parse(readFileSync(fake.state, "utf8"));
+ assert.deepEqual(JSON.parse(readFileSync(progressFile, "utf8")), {
+ phase: "monitoring",
+ token: "attempt-unique-token",
+ orgId: single.orgId,
+ targets: Object.fromEntries(
+ single.services.map((name) => [name, state.services[`acme-${name}`].taskDefinition]),
+ ),
+ });
+ assert.equal(statSync(progressFile).mode & 0o777, 0o600);
+ writeFileSync(fake.log, "");
+ await assert.rejects(deploy, /must not already exist/);
+ assert.equal(readFileSync(fake.log, "utf8"), "");
+ rmSync(progressFile);
+ await deploy();
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service/);
+ assert.equal(JSON.parse(readFileSync(progressFile, "utf8")).token, "attempt-unique-token");
+ } else {
+ assert.ok(statSync(progressFile).isDirectory());
+ }
+ }
+ } finally {
+ if (priorFile === undefined) delete process.env.QM_DEPLOY_PROGRESS_FILE;
+ else process.env.QM_DEPLOY_PROGRESS_FILE = priorFile;
+ if (priorToken === undefined) delete process.env.QM_DEPLOY_PROGRESS_TOKEN;
+ else process.env.QM_DEPLOY_PROGRESS_TOKEN = priorToken;
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+}
+
+test("AWS deployment progress rejects invalid options before AWS calls", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-progress-validation-"));
+ const fake = fakeAws(dir, "");
+ const priorFile = process.env.QM_DEPLOY_PROGRESS_FILE;
+ const priorToken = process.env.QM_DEPLOY_PROGRESS_TOKEN;
+ try {
+ for (const [file, token] of [
+ ["relative.json", "attempt"],
+ [join(dir, "receipt.json"), ""],
+ ["", "attempt"],
+ ]) {
+ process.env.QM_DEPLOY_PROGRESS_FILE = file;
+ process.env.QM_DEPLOY_PROGRESS_TOKEN = token;
+ await assert.rejects(
+ () => awsUp(oneServiceConfig(), dir, { yes: true, candidate: "candidate.json" }),
+ /deployment progress requires/,
+ );
+ }
+ process.env.QM_DEPLOY_PROGRESS_FILE = join(dir, "receipt.json");
+ process.env.QM_DEPLOY_PROGRESS_TOKEN = "attempt";
+ await assert.rejects(() => awsUp(oneServiceConfig(), dir, { yes: true }), /deployment progress requires/);
+ assert.equal(readFileSync(fake.log, "utf8"), "");
+ } finally {
+ if (priorFile === undefined) delete process.env.QM_DEPLOY_PROGRESS_FILE;
+ else process.env.QM_DEPLOY_PROGRESS_FILE = priorFile;
+ if (priorToken === undefined) delete process.env.QM_DEPLOY_PROGRESS_TOKEN;
+ else process.env.QM_DEPLOY_PROGRESS_TOKEN = priorToken;
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});