Multiplayer agent harness for work.
Build AWS candidate images with bounded concurrency (#1031)
Regan Bellaca1bbfeb6d4
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 754586e62..5cd921f4e 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -47,6 +47,8 @@ import {readEnvFile,resolveBuildRepoRoot,runInherit,+ runInheritAsync,+ settleAll,sleep,streamLabeled,} from "../util.ts";@@ -129,6 +131,7 @@ export interface AwsUpOpts {sandboxDir?: string;envFile?: string;buildOnly?: boolean;+ buildConcurrency?: number;candidate?: string;candidateOut?: string;inactive?: boolean;@@ -579,13 +582,14 @@ export function imageTransferArgs(source: string, tagged: string): string[] {return ["buildx", "imagetools", "create", "--prefer-index=false", "--tag", tagged, source];}-function publishWorkloadImage(+async function publishWorkloadImage(config: QmConfig,workload: string,plugin: ResolvedPlugin | undefined,label: string,opts: AwsUpOpts,-): string {+ signal?: AbortSignal,+): Promise<string> {const aws = requireAws(config);const spec = aws.services[workload]!;const tagged = `${ecrHost(aws)}/${spec.ecrRepository}:${label}`;@@ -606,7 +610,8 @@ function publishWorkloadImage(for (const [name, value] of Object.entries(workloadBuildArgs(config, workload)))args.push("--build-arg", `${name}=${value}`);args.push(plugin.sourceDir!);- runInherit("docker", args);+ if (signal) await runInheritAsync("docker", args, { signal });+ else runInherit("docker", args);} else if (opts.buildFrom && isServiceName(workload)) {const root = resolveBuildRepoRoot(opts.buildFromPath, [workload]);const dockerfile = join(root, spec.dockerfile ?? join("deploy", workload, "Dockerfile"));@@ -630,11 +635,13 @@ function publishWorkloadImage(for (const [name, value] of Object.entries(workloadBuildArgs(config, workload)))args.push("--build-arg", `${name}=${value}`);args.push(root);- runInherit("docker", args);+ if (signal) await runInheritAsync("docker", args, { signal });+ else runInherit("docker", args);} else {const source = workloadSourceImage(config, workload, plugin);if (!source) throw new CliError(`AWS workload ${workload} has no source image`);- runInherit("docker", imageTransferArgs(source, tagged));+ if (signal) await runInheritAsync("docker", imageTransferArgs(source, tagged), { signal });+ else runInherit("docker", imageTransferArgs(source, tagged));}const response = awsJson<{ imageDetails?: Array<{ imageDigest?: string }> }>(aws, ["ecr",@@ -1922,6 +1929,16 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt}const plugins = new Map(topology.plugins.map((plugin) => [plugin.name, plugin]));const services = opts.only ?? topology.workloads;+ const buildConcurrency = opts.buildConcurrency ?? 1;+ if (!Number.isSafeInteger(buildConcurrency) || buildConcurrency < 1)+ throw new CliError("--build-concurrency requires a positive integer");+ if (opts.buildConcurrency !== undefined && !opts.buildOnly)+ throw new CliError("--build-concurrency requires --build-only");+ if (+ buildConcurrency > 1 &&+ new Set(services.map((service) => aws.services[service]?.ecrRepository)).size !== services.length+ )+ throw new CliError("--build-concurrency requires distinct ECR repositories for selected workloads");const restart = new Set(opts.restart ?? []);if (restart.size && opts.buildOnly) throw new CliError("--restart cannot be used with --build-only");for (const service of restart) {@@ -1952,7 +1969,29 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOptconst imageProvenance: Record<string, DeploymentImageProvenance> = {};for (const service of services) {imageProvenance[service] = workloadImageProvenance(config, service, plugins.get(service), opts);- images[service] = publishWorkloadImage(config, service, plugins.get(service), opts.imageLabel!, opts);+ }+ const controller = new AbortController();+ const cancelBuilds = () => controller.abort();+ process.on("SIGINT", cancelBuilds);+ process.on("SIGTERM", cancelBuilds);+ try {+ for (let offset = 0; offset < services.length; offset += buildConcurrency) {+ await settleAll(+ services.slice(offset, offset + buildConcurrency).map(async (service) => {+ images[service] = await publishWorkloadImage(+ config,+ service,+ plugins.get(service),+ opts.imageLabel!,+ opts,+ controller.signal,+ );+ }),+ );+ }+ } finally {+ process.off("SIGINT", cancelBuilds);+ process.off("SIGTERM", cancelBuilds);}const release: AwsReleaseCandidate = {contract: 1,@@ -2134,7 +2173,7 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt} else {staged.add(service);selectedImageProvenance[service] = workloadImageProvenance(config, service, plugins.get(service), opts);- images[service] = publishWorkloadImage(config, service, plugins.get(service), stagingLabel, opts);+ images[service] = await publishWorkloadImage(config, service, plugins.get(service), stagingLabel, opts);}}const desired = reportTaskChanges(config, services, images, arns, restart);diff --git a/cli/src/backends/registry.ts b/cli/src/backends/registry.tsindex ef2e21d36..d76d2818b 100644--- a/cli/src/backends/registry.ts+++ b/cli/src/backends/registry.ts@@ -249,6 +249,7 @@ const aws: HostingProvider = {"candidate-out","inactive","restart",+ "build-concurrency",],upOptions: (ctx, flags, dryRun) => {const only = workloadOptions(flags);@@ -257,6 +258,7 @@ const aws: HostingProvider = {const imageLabel = stringFlag(flags, "image-label");const candidate = stringFlag(flags, "candidate");const candidateOut = stringFlag(flags, "candidate-out");+ const buildConcurrency = stringFlag(flags, "build-concurrency");const restart = workloadOptions(flags, "restart");return {dryRun,@@ -267,6 +269,7 @@ const aws: HostingProvider = {...(imageLabel ? { imageLabel } : {}),...(candidate ? { candidate } : {}),...(candidateOut ? { candidateOut } : {}),+ ...(buildConcurrency !== undefined ? { buildConcurrency: Number(buildConcurrency) } : {}),...(restart ? { restart } : {}),...(only ? { only } : {}),};@@ -282,6 +285,7 @@ const aws: HostingProvider = {...(opts.buildOnly ? { buildOnly: true } : {}),...(opts.candidate ? { candidate: opts.candidate } : {}),...(opts.candidateOut ? { candidateOut: opts.candidateOut } : {}),+ ...(opts.buildConcurrency !== undefined ? { buildConcurrency: opts.buildConcurrency } : {}),...(opts.restart ? { restart: opts.restart } : {}),...(opts.inactive ? { inactive: true } : {}),...(opts.only ? { only: opts.only } : {}),diff --git a/cli/src/backends/types.ts b/cli/src/backends/types.tsindex 6e626e39c..e01c894d6 100644--- a/cli/src/backends/types.ts+++ b/cli/src/backends/types.ts@@ -11,6 +11,7 @@ export interface BackendUpOptions {imageFrom?: string;imageRepoPrefix?: string;buildOnly?: boolean;+ buildConcurrency?: number;candidate?: string;candidateOut?: string;inactive?: boolean;diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex eb237a24a..28b90d17c 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -126,6 +126,7 @@ ${bold("DEPLOY (operator)")} ${dim("— runs in the deployment directory")}--candidate-out <file> build immutable AWS images and write their manifest--candidate <file> snapshot, migrate, and deploy exact AWS candidate images--inactive deploy that candidate before this stack owns public DNS (AWS only)+ --build-concurrency <n> concurrent AWS candidate image builds (default: 1)--restart <workloads> replace selected AWS tasks even when their definition is unchanged--dry-run resolve the config + report the plan, change nothingplan an alias for up --dry-rundiff --git a/cli/src/util.ts b/cli/src/util.tsindex c3a738c3a..0d527ad5a 100644--- a/cli/src/util.ts+++ b/cli/src/util.ts@@ -64,6 +64,73 @@ export function runInherit(cmd: string, args: string[], opts: { cwd?: string; en});}+export function runInheritAsync(+ cmd: string,+ args: string[],+ opts: { cwd?: string; env?: NodeJS.ProcessEnv; signal?: AbortSignal } = {},+): Promise<void> {+ return new Promise((resolve, reject) => {+ if (opts.signal?.aborted) return reject(new CliError(`${cmd} cancelled`));+ const grouped = process.platform !== "win32";+ const child = spawn(cmd, args, { stdio: "inherit", detached: grouped, ...procOpts(opts) });+ let cancellation: Promise<void> | undefined;+ const groupAlive = () => {+ if (!child.pid) return false;+ try {+ process.kill(-child.pid, 0);+ return true;+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === "ESRCH") return false;+ throw error;+ }+ };+ const killGroup = (signal: NodeJS.Signals) => {+ if (!child.pid) return;+ try {+ process.kill(-child.pid, signal);+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;+ }+ };+ const cancel = () => {+ cancellation = (async () => {+ if (!child.pid) return;+ if (!grouped) {+ const killed = spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" });+ if (killed.status !== 0 && child.exitCode === null)+ throw new CliError(`${cmd} process tree could not be cancelled`);+ return;+ }+ killGroup("SIGTERM");+ const deadline = Date.now() + 5000;+ while (groupAlive() && Date.now() < deadline) await sleep(25);+ if (groupAlive()) killGroup("SIGKILL");+ const killDeadline = Date.now() + 5000;+ while (groupAlive() && Date.now() < killDeadline) await sleep(25);+ if (groupAlive()) throw new CliError(`${cmd} process group did not exit after cancellation`);+ })();+ cancellation.catch(reject);+ };+ const cleanup = () => opts.signal?.removeEventListener("abort", cancel);+ opts.signal?.addEventListener("abort", cancel, { once: true });+ child.once("error", (error) => {+ cleanup();+ reject(error);+ });+ child.once("close", async (code, signal) => {+ cleanup();+ try {+ await cancellation;+ if (opts.signal?.aborted) reject(new CliError(`${cmd} cancelled`));+ else if (code === 0) resolve();+ else reject(new CliError(`${cmd} exited with ${signal ?? code}`));+ } catch (error) {+ reject(error);+ }+ });+ });+}+export function spawnBackground(cmd: string,args: string[],diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 997fc15c9..743b8015a 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -2,7 +2,7 @@ import test from "node:test";import https from "node:https";import { EventEmitter } from "node:events";import assert from "node:assert/strict";-import { spawnSync } from "node:child_process";+import { spawn, spawnSync } from "node:child_process";import { createHash, createHmac } from "node:crypto";import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";@@ -37,6 +37,7 @@ import { computedSecrets } from "../src/secrets.ts";import { awsObjectStoreBucket } from "../src/terraform.ts";import { withAwsLease } from "../src/aws-lease.ts";import { manifestRef } from "../src/manifest.ts";+import { hostingProvider } from "../src/backends/registry.ts";process.env.QM_AWS_ROLLOUT_POLL_MS = "5";process.env.QM_AWS_LIVE_PROBE_POLL_MS = "5";@@ -2679,6 +2680,176 @@ test("AWS builds one immutable candidate manifest and deploys its exact digest w}});+test("AWS candidate builds honor their concurrency bound and join failures before publishing", async () => {+ for (const mode of ["serial", "parallel", "failure"]) {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-build-parallel-"));+ const candidatePath = join(dir, "candidate.json");+ const events = join(dir, "events");+ writeFileSync(events, "");+ const fake = statefulAws(dir, config);+ const docker = join(dir, "docker");+ writeFileSync(+ docker,+ `#!${process.execPath}+const fs = require("node:fs");+const path = require("node:path");+const args = process.argv.slice(2);+if (args[0] === "login") process.exit(0);+const name = args[args.indexOf("-t") + 1].split("/").at(-1).split(":")[0].slice(3);+const dir = ${JSON.stringify(dir)};+const events = ${JSON.stringify(events)};+const mode = ${JSON.stringify(mode)};+const log = (event) => fs.appendFileSync(events, event + ":" + name + "\\n");+log("start");+fs.writeFileSync(path.join(dir, "started-" + name), "");+(async () => {+ if (mode !== "serial" && ["core", "web-ui"].includes(name)) {+ const deadline = Date.now() + 10000;+ while (!["core", "web-ui"].every((peer) => fs.existsSync(path.join(dir, "started-" + peer)))) {+ if (Date.now() > deadline) process.exit(9);+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ }+ if (mode === "failure" && name === "core") { log("fail"); process.exit(7); }+ await new Promise((resolve) => setTimeout(resolve, 50));+ log("end");+})();+`,+ );+ chmodSync(docker, 0o755);+ const priorPath = process.env.PATH;+ process.env.PATH = `${dir}:${priorPath}`;+ try {+ const provider = hostingProvider("aws");+ const context = {+ config,+ configDir: dir,+ configPath: join(dir, "qm.config.json"),+ sandboxDir: dir,+ target: "aws" as const,+ };+ const options = provider.upOptions(+ context,+ {+ "build-only": true,+ "build-from": true,+ ...(mode === "serial" ? {} : { "build-concurrency": "2" }),+ "image-label": "parallel-candidate",+ "candidate-out": candidatePath,+ },+ false,+ );+ const result = provider.createBackend(context).up(options);+ if (mode === "failure") {+ await assert.rejects(result, /docker exited with 7/);+ assert.equal(existsSync(candidatePath), false);+ } else {+ await result;+ const candidate = JSON.parse(readFileSync(candidatePath, "utf8"));+ assert.deepEqual(Object.keys(candidate.images).sort(), Object.keys(config.aws!.services).sort());+ assert.deepEqual(Object.keys(candidate.imageProvenance).sort(), Object.keys(config.aws!.services).sort());+ assert.ok(Object.values(candidate.images).every((image) => /@sha256:[a-f0-9]{64}$/.test(String(image))));+ }+ const lines = readFileSync(events, "utf8").trim().split("\n");+ let active = 0;+ let peak = 0;+ for (const line of lines) {+ active += line.startsWith("start:") ? 1 : -1;+ peak = Math.max(peak, active);+ }+ assert.equal(active, 0);+ assert.equal(peak, mode === "serial" ? 1 : 2);+ if (mode === "failure") {+ assert.ok(lines.includes("end:web-ui"));+ assert.equal(lines.filter((line) => line.startsWith("start:")).length, 2);+ }+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /dynamodb|ecs |secretsmanager/);+ } finally {+ process.env.PATH = priorPath;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+ }+});++test("cancelling candidate builds terminates and joins every Docker child without a manifest", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-build-cancel-"));+ const candidatePath = join(dir, "candidate.json");+ const fake = statefulAws(dir, config);+ const docker = join(dir, "docker");+ writeFileSync(+ docker,+ `#!${process.execPath}+const fs = require("node:fs");+const path = require("node:path");+const args = process.argv.slice(2);+if (args[0] === "login") process.exit(0);+const name = args[args.indexOf("-t") + 1].split("/").at(-1).split(":")[0].slice(3);+const dir = ${JSON.stringify(dir)};+if (name === "core") process.on("SIGTERM", () => {});+if (name === "web-ui") {+ const plugin = 'process.on("SIGTERM", () => {}); require("node:fs").writeFileSync(' + JSON.stringify(path.join(dir, "pid-web-ui-plugin")) + ', String(process.pid)); setTimeout(() => {}, 20000);';+ require("node:child_process").spawn(process.execPath, ["-e", plugin], { stdio: "inherit" });+}+fs.writeFileSync(path.join(dir, "pid-" + name), String(process.pid));+setTimeout(() => fs.writeFileSync(path.join(dir, "completed-" + name), ""), 20000);+`,+ );+ chmodSync(docker, 0o755);+ const moduleUrl = new URL("../src/backends/aws.ts", import.meta.url).href;+ const code = `const { awsUp } = await import(${JSON.stringify(moduleUrl)}); await awsUp(${JSON.stringify(config)}, ${JSON.stringify(dir)}, ${JSON.stringify({ buildOnly: true, buildFrom: true, buildConcurrency: 2, imageLabel: "cancelled-candidate", candidateOut: candidatePath })});`;+ const parent = spawn(process.execPath, ["--input-type=module", "-e", code], {+ env: { ...process.env, PATH: `${dir}:${process.env.PATH}` },+ stdio: "ignore",+ });+ const exited = new Promise<number | null>((resolve, reject) => {+ parent.once("error", reject);+ parent.once("exit", resolve);+ });+ try {+ const deadline = Date.now() + 10000;+ while (!["core", "web-ui", "web-ui-plugin"].every((name) => existsSync(join(dir, "pid-" + name)))) {+ assert.ok(Date.now() < deadline, "Docker children and nested buildx plugin must start");+ assert.equal(parent.exitCode, null);+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ parent.kill("SIGTERM");+ assert.notEqual(await exited, 0);+ assert.equal(existsSync(candidatePath), false);+ for (const name of ["core", "web-ui", "web-ui-plugin"]) {+ const pid = Number(readFileSync(join(dir, "pid-" + name), "utf8"));+ assert.throws(() => process.kill(pid, 0), /ESRCH/);+ assert.equal(existsSync(join(dir, "completed-" + name)), false);+ }+ assert.equal(existsSync(join(dir, "pid-admin")), false);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /dynamodb|ecs |secretsmanager/);+ } finally {+ parent.kill("SIGKILL");+ for (const name of ["core", "web-ui", "web-ui-plugin"]) {+ const path = join(dir, "pid-" + name);+ if (existsSync(path)) {+ try {+ process.kill(Number(readFileSync(path, "utf8")), "SIGKILL");+ } catch {+ void 0;+ }+ }+ }+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});++test("AWS rejects invalid or misplaced candidate build concurrency before mutation", async () => {+ for (const value of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1]) {+ await assert.rejects(awsUp(config, ".", { buildOnly: true, buildConcurrency: value }), /positive integer/);+ }+ await assert.rejects(awsUp(config, ".", { buildConcurrency: 2 }), /requires --build-only/);+ const shared = structuredClone(config);+ shared.aws!.services["web-ui"]!.ecrRepository = shared.aws!.services.core!.ecrRepository;+ await assert.rejects(awsUp(shared, ".", { buildOnly: true, buildConcurrency: 2 }), /distinct ECR repositories/);+});+test("AWS candidate deploy fails closed on account, repository, or missing-workload drift", async () => {const dir = mkdtempSync(join(tmpdir(), "qm-aws-candidate-invalid-"));const candidatePath = join(dir, "candidate.json");diff --git a/cli/test/providers.test.ts b/cli/test/providers.test.tsindex 8484a6544..2478147e2 100644--- a/cli/test/providers.test.ts+++ b/cli/test/providers.test.ts@@ -29,6 +29,7 @@ test("the hosting provider registry owns target discovery and lifecycle capabili"candidate-out","inactive","restart",+ "build-concurrency",]);});diff --git a/cli/test/util.test.ts b/cli/test/util.test.tsindex 617b071fd..05847120b 100644--- a/cli/test/util.test.ts+++ b/cli/test/util.test.ts@@ -3,7 +3,7 @@ import assert from "node:assert/strict";import { chmodSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import { canonicalJson, flyBin, isInvalidSecret, readEnvFile, writeEnvValue } from "../src/util.ts";+import { canonicalJson, runInheritAsync, flyBin, isInvalidSecret, readEnvFile, writeEnvValue } from "../src/util.ts";test("managed credential encryption keys require strong material", () => {assert.equal(isInvalidSecret("CONNECTOR_SECRET_KEY", "short"), true);@@ -185,3 +185,8 @@ test("promptHidden treats Ctrl-D as enter on a non-empty buffer and as cancel onawait assert.rejects(() => pending, /secret entry cancelled/);});});++test("async inherited processes reject spawn errors and signal termination", async () => {+ await assert.rejects(runInheritAsync("/definitely-missing-qm-command", []), /ENOENT/);+ await assert.rejects(runInheritAsync(process.execPath, ["-e", 'process.kill(process.pid, "SIGTERM")']), /SIGTERM/);+});