Multiplayer agent harness for work.
Emit atomic progress receipts for candidate deployments (#1275)
Regan Belle0966ba8bcc9
diff --git a/cli/README.md b/cli/README.mdindex 9e21ba773..ed7d16062 100644--- a/cli/README.md+++ b/cli/README.md@@ -77,6 +77,16 @@ configuration only, so it prints that timestamp as the matching data restorepoint (`aws rds restore-db-instance-to-point-in-time`);`aws.predeployDbSnapshot: false` opts out.+Batch operators can set `QM_DEPLOY_PROGRESS_FILE` to a new absolute file path and+`QM_DEPLOY_PROGRESS_TOKEN` to a unique attempt identifier for candidate `up --yes`.+After all forward service updates have been submitted, the CLI atomically creates+a private JSON receipt with `phase: "monitoring"`, `token`, `orgId`, and `targets`+(the selected workload-to-task-definition mapping). It then continues health+checks, manifest recording, and rollback under the deployment lease. This receipt+only permits the batch runner to release a submission slot; the CLI exit status+still determines success. Use a fresh path and token for every attempt. Missing+receipts must keep the submission slot occupied until the CLI exits.+`sandbox build` is a local validation build of the sandbox layer image. At runtimesandboxes boot their platform's stock image; tools and skills arrive through thedeployment-layer sync, which every ordinary `up` performs.diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex a080f48d8..f828c0684 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -3,9 +3,19 @@ import https from "node:https";import { createHash, randomUUID } from "node:crypto";import { spawnSync } from "node:child_process";import { lookup, resolveCname } from "node:dns/promises";-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import {+ accessSync,+ constants,+ existsSync,+ linkSync,+ mkdtempSync,+ readFileSync,+ rmSync,+ statSync,+ writeFileSync,+} from "node:fs";import { tmpdir } from "node:os";-import { join, resolve } from "node:path";+import { dirname, isAbsolute, join, resolve } from "node:path";import {acquireAwsLease as acquireLease,awsText,@@ -1728,6 +1738,7 @@ async function applyServiceTargets(waitForCompensationDrain?: boolean;timeoutMs?: number;webBeforePortal?: boolean;+ onSubmitted?: () => void;} = {},): Promise<void> {const aws = requireAws(config);@@ -1771,6 +1782,7 @@ async function applyServiceTargets(changed.push(workload);awsText(aws, args);}+ options.onSubmitted?.();await awaitServiceTargets(config,Object.fromEntries(@@ -1968,7 +1980,43 @@ function confirmDbRestorePointCovered(config: QmConfig, restorePoint: string): v}}+function deploymentProgress(+ config: QmConfig,+ opts: AwsUpOpts,+): ((targets: Record<string, string>) => void) | undefined {+ const file = process.env.QM_DEPLOY_PROGRESS_FILE;+ const token = process.env.QM_DEPLOY_PROGRESS_TOKEN;+ if (file === undefined && token === undefined) return undefined;+ if (!file || !token || !isAbsolute(file) || !opts.candidate || !opts.yes || opts.dryRun || opts.buildOnly) {+ throw new CliError(+ "deployment progress requires an absolute QM_DEPLOY_PROGRESS_FILE, QM_DEPLOY_PROGRESS_TOKEN, and candidate up --yes",+ );+ }+ if (existsSync(file)) throw new CliError("deployment progress file must not already exist");+ if (!statSync(dirname(file)).isDirectory()) throw new CliError("deployment progress parent must be a directory");+ accessSync(dirname(file), constants.W_OK);+ return (targets) => {+ const temporary = `${file}.${randomUUID()}.tmp`;+ try {+ writeFileSync(temporary, JSON.stringify({ phase: "monitoring", token, orgId: config.orgId, targets }), {+ flag: "wx",+ mode: 0o600,+ });+ linkSync(temporary, file);+ } catch (error) {+ warn(`could not publish deployment progress: ${errMessage(error)}`);+ } finally {+ try {+ rmSync(temporary, { force: true });+ } catch (error) {+ warn(`could not remove deployment progress temporary file: ${errMessage(error)}`);+ }+ }+ };+}+export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpts = {}): Promise<void> {+ const reportProgress = deploymentProgress(config, opts);const topology = awsTopology(config, _configDir);const { aws } = topology;if (new URL(config.publicUrl).protocol !== "https:") {@@ -2253,9 +2301,11 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOptObject.fromEntries(Object.keys(rolloutTargets).map((service) => [service, workloadDesiredCount(config, service)]),),- { webBeforePortal: true },+ { webBeforePortal: true, onSubmitted: reportProgress ? () => reportProgress(targets) : undefined },);applied = true;+ } else {+ reportProgress?.(targets);}await awaitServiceTargets(config,diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 48cc5268b..f4de5d0eb 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -4,7 +4,7 @@ import { EventEmitter } from "node:events";import assert from "node:assert/strict";import { spawn, spawnSync } from "node:child_process";import { createHash, createHmac } from "node:crypto";-import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { dirname, join } from "node:path";import {@@ -256,6 +256,8 @@ function statefulAws(blueGreenBakePolls?: number;migrationExitCode?: number;foreignServiceTags?: boolean;+ progressFile?: string;+ sabotageProgress?: boolean;} = {},): {log: string;@@ -313,6 +315,9 @@ function statefulAws(dir,`const args = process.argv.slice(2);+const progressFile = ${JSON.stringify(opts.progressFile)};+if (progressFile) fs.appendFileSync(progressFile + ".samples", JSON.stringify({ args, published: fs.existsSync(progressFile) }) + "\\n");+if (progressFile && ${JSON.stringify(opts.sabotageProgress ?? false)} && a.includes("ecs update-service")) fs.mkdirSync(progressFile, { recursive: true });const statePath = ${JSON.stringify(state)};const s = JSON.parse(fs.readFileSync(statePath, "utf8"));const save = () => fs.writeFileSync(statePath, JSON.stringify(s));@@ -5234,3 +5239,122 @@ else console.log("");`,}});}++for (const mode of ["success", "migration-failure", "update-failure", "write-failure"] as const) {+ test(`AWS candidate deployment progress receipt: ${mode}`, async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-progress-"));+ const progressFile = join(dir, "progress.json");+ const candidatePath = join(dir, "candidate.json");+ const single = mode === "success" ? twoServiceConfig() : oneServiceConfig();+ writeFileSync(+ candidatePath,+ JSON.stringify({+ contract: 1,+ accountId: "123456789012",+ region: "us-west-2",+ label: "candidate-progress",+ images: Object.fromEntries(+ single.services.map((name) => [+ name,+ `123456789012.dkr.ecr.us-west-2.amazonaws.com/${single.aws!.services[name]!.ecrRepository}@sha256:${"a".repeat(64)}`,+ ]),+ ),+ imageProvenance: Object.fromEntries(+ single.services.map((name) => [name, { kind: "source-build", source: "checkout" }]),+ ),+ }),+ );+ const fake = statefulAws(+ dir,+ single,+ {},+ {+ progressFile,+ migrationExitCode: mode === "migration-failure" ? 1 : 0,+ failFirstUpdateAfterMutation: mode === "update-failure",+ sabotageProgress: mode === "write-failure",+ },+ );+ const priorFile = process.env.QM_DEPLOY_PROGRESS_FILE;+ const priorToken = process.env.QM_DEPLOY_PROGRESS_TOKEN;+ process.env.QM_DEPLOY_PROGRESS_FILE = progressFile;+ process.env.QM_DEPLOY_PROGRESS_TOKEN = "attempt-unique-token";+ try {+ const deploy = () => awsUp(single, dir, { yes: true, candidate: candidatePath });+ if (mode === "migration-failure" || mode === "update-failure") {+ await assert.rejects(deploy);+ assert.equal(existsSync(progressFile), false);+ } else {+ await deploy();+ const samples = readFileSync(progressFile + ".samples", "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line));+ const update = samples.findLastIndex((sample) => sample.args.includes("update-service"));+ assert.ok(update >= 0);+ assert.ok(samples.slice(0, update + 1).every((sample) => !sample.published));+ if (mode === "success") {+ assert.equal(samples[update + 1].published, true);+ const state = JSON.parse(readFileSync(fake.state, "utf8"));+ assert.deepEqual(JSON.parse(readFileSync(progressFile, "utf8")), {+ phase: "monitoring",+ token: "attempt-unique-token",+ orgId: single.orgId,+ targets: Object.fromEntries(+ single.services.map((name) => [name, state.services[`acme-${name}`].taskDefinition]),+ ),+ });+ assert.equal(statSync(progressFile).mode & 0o777, 0o600);+ writeFileSync(fake.log, "");+ await assert.rejects(deploy, /must not already exist/);+ assert.equal(readFileSync(fake.log, "utf8"), "");+ rmSync(progressFile);+ await deploy();+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service/);+ assert.equal(JSON.parse(readFileSync(progressFile, "utf8")).token, "attempt-unique-token");+ } else {+ assert.ok(statSync(progressFile).isDirectory());+ }+ }+ } finally {+ if (priorFile === undefined) delete process.env.QM_DEPLOY_PROGRESS_FILE;+ else process.env.QM_DEPLOY_PROGRESS_FILE = priorFile;+ if (priorToken === undefined) delete process.env.QM_DEPLOY_PROGRESS_TOKEN;+ else process.env.QM_DEPLOY_PROGRESS_TOKEN = priorToken;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+ });+}++test("AWS deployment progress rejects invalid options before AWS calls", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-progress-validation-"));+ const fake = fakeAws(dir, "");+ const priorFile = process.env.QM_DEPLOY_PROGRESS_FILE;+ const priorToken = process.env.QM_DEPLOY_PROGRESS_TOKEN;+ try {+ for (const [file, token] of [+ ["relative.json", "attempt"],+ [join(dir, "receipt.json"), ""],+ ["", "attempt"],+ ]) {+ process.env.QM_DEPLOY_PROGRESS_FILE = file;+ process.env.QM_DEPLOY_PROGRESS_TOKEN = token;+ await assert.rejects(+ () => awsUp(oneServiceConfig(), dir, { yes: true, candidate: "candidate.json" }),+ /deployment progress requires/,+ );+ }+ process.env.QM_DEPLOY_PROGRESS_FILE = join(dir, "receipt.json");+ process.env.QM_DEPLOY_PROGRESS_TOKEN = "attempt";+ await assert.rejects(() => awsUp(oneServiceConfig(), dir, { yes: true }), /deployment progress requires/);+ assert.equal(readFileSync(fake.log, "utf8"), "");+ } finally {+ if (priorFile === undefined) delete process.env.QM_DEPLOY_PROGRESS_FILE;+ else process.env.QM_DEPLOY_PROGRESS_FILE = priorFile;+ if (priorToken === undefined) delete process.env.QM_DEPLOY_PROGRESS_TOKEN;+ else process.env.QM_DEPLOY_PROGRESS_TOKEN = priorToken;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});