Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Build AWS candidate images with bounded concurrency (#1031)

Regan Bellaca1bbfeb6d4
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index 754586e62..5cd921f4e 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -47,6 +47,8 @@ import {
readEnvFile,
resolveBuildRepoRoot,
runInherit,
+ runInheritAsync,
+ settleAll,
sleep,
streamLabeled,
} from "../util.ts";
@@ -129,6 +131,7 @@ export interface AwsUpOpts {
sandboxDir?: string;
envFile?: string;
buildOnly?: boolean;
+ buildConcurrency?: number;
candidate?: string;
candidateOut?: string;
inactive?: boolean;
@@ -579,13 +582,14 @@ export function imageTransferArgs(source: string, tagged: string): string[] {
return ["buildx", "imagetools", "create", "--prefer-index=false", "--tag", tagged, source];
}
-function publishWorkloadImage(
+async function publishWorkloadImage(
config: QmConfig,
workload: string,
plugin: ResolvedPlugin | undefined,
label: string,
opts: AwsUpOpts,
-): string {
+ signal?: AbortSignal,
+): Promise<string> {
const aws = requireAws(config);
const spec = aws.services[workload]!;
const tagged = `${ecrHost(aws)}/${spec.ecrRepository}:${label}`;
@@ -606,7 +610,8 @@ function publishWorkloadImage(
for (const [name, value] of Object.entries(workloadBuildArgs(config, workload)))
args.push("--build-arg", `${name}=${value}`);
args.push(plugin.sourceDir!);
- runInherit("docker", args);
+ if (signal) await runInheritAsync("docker", args, { signal });
+ else runInherit("docker", args);
} else if (opts.buildFrom && isServiceName(workload)) {
const root = resolveBuildRepoRoot(opts.buildFromPath, [workload]);
const dockerfile = join(root, spec.dockerfile ?? join("deploy", workload, "Dockerfile"));
@@ -630,11 +635,13 @@ function publishWorkloadImage(
for (const [name, value] of Object.entries(workloadBuildArgs(config, workload)))
args.push("--build-arg", `${name}=${value}`);
args.push(root);
- runInherit("docker", args);
+ if (signal) await runInheritAsync("docker", args, { signal });
+ else runInherit("docker", args);
} else {
const source = workloadSourceImage(config, workload, plugin);
if (!source) throw new CliError(`AWS workload ${workload} has no source image`);
- runInherit("docker", imageTransferArgs(source, tagged));
+ if (signal) await runInheritAsync("docker", imageTransferArgs(source, tagged), { signal });
+ else runInherit("docker", imageTransferArgs(source, tagged));
}
const response = awsJson<{ imageDetails?: Array<{ imageDigest?: string }> }>(aws, [
"ecr",
@@ -1922,6 +1929,16 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
}
const plugins = new Map(topology.plugins.map((plugin) => [plugin.name, plugin]));
const services = opts.only ?? topology.workloads;
+ const buildConcurrency = opts.buildConcurrency ?? 1;
+ if (!Number.isSafeInteger(buildConcurrency) || buildConcurrency < 1)
+ throw new CliError("--build-concurrency requires a positive integer");
+ if (opts.buildConcurrency !== undefined && !opts.buildOnly)
+ throw new CliError("--build-concurrency requires --build-only");
+ if (
+ buildConcurrency > 1 &&
+ new Set(services.map((service) => aws.services[service]?.ecrRepository)).size !== services.length
+ )
+ throw new CliError("--build-concurrency requires distinct ECR repositories for selected workloads");
const restart = new Set(opts.restart ?? []);
if (restart.size && opts.buildOnly) throw new CliError("--restart cannot be used with --build-only");
for (const service of restart) {
@@ -1952,7 +1969,29 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
const imageProvenance: Record<string, DeploymentImageProvenance> = {};
for (const service of services) {
imageProvenance[service] = workloadImageProvenance(config, service, plugins.get(service), opts);
- images[service] = publishWorkloadImage(config, service, plugins.get(service), opts.imageLabel!, opts);
+ }
+ const controller = new AbortController();
+ const cancelBuilds = () => controller.abort();
+ process.on("SIGINT", cancelBuilds);
+ process.on("SIGTERM", cancelBuilds);
+ try {
+ for (let offset = 0; offset < services.length; offset += buildConcurrency) {
+ await settleAll(
+ services.slice(offset, offset + buildConcurrency).map(async (service) => {
+ images[service] = await publishWorkloadImage(
+ config,
+ service,
+ plugins.get(service),
+ opts.imageLabel!,
+ opts,
+ controller.signal,
+ );
+ }),
+ );
+ }
+ } finally {
+ process.off("SIGINT", cancelBuilds);
+ process.off("SIGTERM", cancelBuilds);
}
const release: AwsReleaseCandidate = {
contract: 1,
@@ -2134,7 +2173,7 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
} else {
staged.add(service);
selectedImageProvenance[service] = workloadImageProvenance(config, service, plugins.get(service), opts);
- images[service] = publishWorkloadImage(config, service, plugins.get(service), stagingLabel, opts);
+ images[service] = await publishWorkloadImage(config, service, plugins.get(service), stagingLabel, opts);
}
}
const desired = reportTaskChanges(config, services, images, arns, restart);
diff --git a/cli/src/backends/registry.ts b/cli/src/backends/registry.ts
index ef2e21d36..d76d2818b 100644
--- a/cli/src/backends/registry.ts
+++ b/cli/src/backends/registry.ts
@@ -249,6 +249,7 @@ const aws: HostingProvider = {
"candidate-out",
"inactive",
"restart",
+ "build-concurrency",
],
upOptions: (ctx, flags, dryRun) => {
const only = workloadOptions(flags);
@@ -257,6 +258,7 @@ const aws: HostingProvider = {
const imageLabel = stringFlag(flags, "image-label");
const candidate = stringFlag(flags, "candidate");
const candidateOut = stringFlag(flags, "candidate-out");
+ const buildConcurrency = stringFlag(flags, "build-concurrency");
const restart = workloadOptions(flags, "restart");
return {
dryRun,
@@ -267,6 +269,7 @@ const aws: HostingProvider = {
...(imageLabel ? { imageLabel } : {}),
...(candidate ? { candidate } : {}),
...(candidateOut ? { candidateOut } : {}),
+ ...(buildConcurrency !== undefined ? { buildConcurrency: Number(buildConcurrency) } : {}),
...(restart ? { restart } : {}),
...(only ? { only } : {}),
};
@@ -282,6 +285,7 @@ const aws: HostingProvider = {
...(opts.buildOnly ? { buildOnly: true } : {}),
...(opts.candidate ? { candidate: opts.candidate } : {}),
...(opts.candidateOut ? { candidateOut: opts.candidateOut } : {}),
+ ...(opts.buildConcurrency !== undefined ? { buildConcurrency: opts.buildConcurrency } : {}),
...(opts.restart ? { restart: opts.restart } : {}),
...(opts.inactive ? { inactive: true } : {}),
...(opts.only ? { only: opts.only } : {}),
diff --git a/cli/src/backends/types.ts b/cli/src/backends/types.ts
index 6e626e39c..e01c894d6 100644
--- a/cli/src/backends/types.ts
+++ b/cli/src/backends/types.ts
@@ -11,6 +11,7 @@ export interface BackendUpOptions {
imageFrom?: string;
imageRepoPrefix?: string;
buildOnly?: boolean;
+ buildConcurrency?: number;
candidate?: string;
candidateOut?: string;
inactive?: boolean;
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index eb237a24a..28b90d17c 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -126,6 +126,7 @@ ${bold("DEPLOY (operator)")} ${dim("— runs in the deployment directory")}
--candidate-out <file> build immutable AWS images and write their manifest
--candidate <file> snapshot, migrate, and deploy exact AWS candidate images
--inactive deploy that candidate before this stack owns public DNS (AWS only)
+ --build-concurrency <n> concurrent AWS candidate image builds (default: 1)
--restart <workloads> replace selected AWS tasks even when their definition is unchanged
--dry-run resolve the config + report the plan, change nothing
plan an alias for up --dry-run
diff --git a/cli/src/util.ts b/cli/src/util.ts
index c3a738c3a..0d527ad5a 100644
--- a/cli/src/util.ts
+++ b/cli/src/util.ts
@@ -64,6 +64,73 @@ export function runInherit(cmd: string, args: string[], opts: { cwd?: string; en
});
}
+export function runInheritAsync(
+ cmd: string,
+ args: string[],
+ opts: { cwd?: string; env?: NodeJS.ProcessEnv; signal?: AbortSignal } = {},
+): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (opts.signal?.aborted) return reject(new CliError(`${cmd} cancelled`));
+ const grouped = process.platform !== "win32";
+ const child = spawn(cmd, args, { stdio: "inherit", detached: grouped, ...procOpts(opts) });
+ let cancellation: Promise<void> | undefined;
+ const groupAlive = () => {
+ if (!child.pid) return false;
+ try {
+ process.kill(-child.pid, 0);
+ return true;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ESRCH") return false;
+ throw error;
+ }
+ };
+ const killGroup = (signal: NodeJS.Signals) => {
+ if (!child.pid) return;
+ try {
+ process.kill(-child.pid, signal);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+ }
+ };
+ const cancel = () => {
+ cancellation = (async () => {
+ if (!child.pid) return;
+ if (!grouped) {
+ const killed = spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" });
+ if (killed.status !== 0 && child.exitCode === null)
+ throw new CliError(`${cmd} process tree could not be cancelled`);
+ return;
+ }
+ killGroup("SIGTERM");
+ const deadline = Date.now() + 5000;
+ while (groupAlive() && Date.now() < deadline) await sleep(25);
+ if (groupAlive()) killGroup("SIGKILL");
+ const killDeadline = Date.now() + 5000;
+ while (groupAlive() && Date.now() < killDeadline) await sleep(25);
+ if (groupAlive()) throw new CliError(`${cmd} process group did not exit after cancellation`);
+ })();
+ cancellation.catch(reject);
+ };
+ const cleanup = () => opts.signal?.removeEventListener("abort", cancel);
+ opts.signal?.addEventListener("abort", cancel, { once: true });
+ child.once("error", (error) => {
+ cleanup();
+ reject(error);
+ });
+ child.once("close", async (code, signal) => {
+ cleanup();
+ try {
+ await cancellation;
+ if (opts.signal?.aborted) reject(new CliError(`${cmd} cancelled`));
+ else if (code === 0) resolve();
+ else reject(new CliError(`${cmd} exited with ${signal ?? code}`));
+ } catch (error) {
+ reject(error);
+ }
+ });
+ });
+}
+
export function spawnBackground(
cmd: string,
args: string[],
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index 997fc15c9..743b8015a 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -2,7 +2,7 @@ import test from "node:test";
import https from "node:https";
import { EventEmitter } from "node:events";
import assert from "node:assert/strict";
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
import { createHash, createHmac } from "node:crypto";
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -37,6 +37,7 @@ import { computedSecrets } from "../src/secrets.ts";
import { awsObjectStoreBucket } from "../src/terraform.ts";
import { withAwsLease } from "../src/aws-lease.ts";
import { manifestRef } from "../src/manifest.ts";
+import { hostingProvider } from "../src/backends/registry.ts";
process.env.QM_AWS_ROLLOUT_POLL_MS = "5";
process.env.QM_AWS_LIVE_PROBE_POLL_MS = "5";
@@ -2679,6 +2680,176 @@ test("AWS builds one immutable candidate manifest and deploys its exact digest w
}
});
+test("AWS candidate builds honor their concurrency bound and join failures before publishing", async () => {
+ for (const mode of ["serial", "parallel", "failure"]) {
+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-build-parallel-"));
+ const candidatePath = join(dir, "candidate.json");
+ const events = join(dir, "events");
+ writeFileSync(events, "");
+ const fake = statefulAws(dir, config);
+ const docker = join(dir, "docker");
+ writeFileSync(
+ docker,
+ `#!${process.execPath}
+const fs = require("node:fs");
+const path = require("node:path");
+const args = process.argv.slice(2);
+if (args[0] === "login") process.exit(0);
+const name = args[args.indexOf("-t") + 1].split("/").at(-1).split(":")[0].slice(3);
+const dir = ${JSON.stringify(dir)};
+const events = ${JSON.stringify(events)};
+const mode = ${JSON.stringify(mode)};
+const log = (event) => fs.appendFileSync(events, event + ":" + name + "\\n");
+log("start");
+fs.writeFileSync(path.join(dir, "started-" + name), "");
+(async () => {
+ if (mode !== "serial" && ["core", "web-ui"].includes(name)) {
+ const deadline = Date.now() + 10000;
+ while (!["core", "web-ui"].every((peer) => fs.existsSync(path.join(dir, "started-" + peer)))) {
+ if (Date.now() > deadline) process.exit(9);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ }
+ if (mode === "failure" && name === "core") { log("fail"); process.exit(7); }
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ log("end");
+})();
+`,
+ );
+ chmodSync(docker, 0o755);
+ const priorPath = process.env.PATH;
+ process.env.PATH = `${dir}:${priorPath}`;
+ try {
+ const provider = hostingProvider("aws");
+ const context = {
+ config,
+ configDir: dir,
+ configPath: join(dir, "qm.config.json"),
+ sandboxDir: dir,
+ target: "aws" as const,
+ };
+ const options = provider.upOptions(
+ context,
+ {
+ "build-only": true,
+ "build-from": true,
+ ...(mode === "serial" ? {} : { "build-concurrency": "2" }),
+ "image-label": "parallel-candidate",
+ "candidate-out": candidatePath,
+ },
+ false,
+ );
+ const result = provider.createBackend(context).up(options);
+ if (mode === "failure") {
+ await assert.rejects(result, /docker exited with 7/);
+ assert.equal(existsSync(candidatePath), false);
+ } else {
+ await result;
+ const candidate = JSON.parse(readFileSync(candidatePath, "utf8"));
+ assert.deepEqual(Object.keys(candidate.images).sort(), Object.keys(config.aws!.services).sort());
+ assert.deepEqual(Object.keys(candidate.imageProvenance).sort(), Object.keys(config.aws!.services).sort());
+ assert.ok(Object.values(candidate.images).every((image) => /@sha256:[a-f0-9]{64}$/.test(String(image))));
+ }
+ const lines = readFileSync(events, "utf8").trim().split("\n");
+ let active = 0;
+ let peak = 0;
+ for (const line of lines) {
+ active += line.startsWith("start:") ? 1 : -1;
+ peak = Math.max(peak, active);
+ }
+ assert.equal(active, 0);
+ assert.equal(peak, mode === "serial" ? 1 : 2);
+ if (mode === "failure") {
+ assert.ok(lines.includes("end:web-ui"));
+ assert.equal(lines.filter((line) => line.startsWith("start:")).length, 2);
+ }
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /dynamodb|ecs |secretsmanager/);
+ } finally {
+ process.env.PATH = priorPath;
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }
+});
+
+test("cancelling candidate builds terminates and joins every Docker child without a manifest", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-build-cancel-"));
+ const candidatePath = join(dir, "candidate.json");
+ const fake = statefulAws(dir, config);
+ const docker = join(dir, "docker");
+ writeFileSync(
+ docker,
+ `#!${process.execPath}
+const fs = require("node:fs");
+const path = require("node:path");
+const args = process.argv.slice(2);
+if (args[0] === "login") process.exit(0);
+const name = args[args.indexOf("-t") + 1].split("/").at(-1).split(":")[0].slice(3);
+const dir = ${JSON.stringify(dir)};
+if (name === "core") process.on("SIGTERM", () => {});
+if (name === "web-ui") {
+ const plugin = 'process.on("SIGTERM", () => {}); require("node:fs").writeFileSync(' + JSON.stringify(path.join(dir, "pid-web-ui-plugin")) + ', String(process.pid)); setTimeout(() => {}, 20000);';
+ require("node:child_process").spawn(process.execPath, ["-e", plugin], { stdio: "inherit" });
+}
+fs.writeFileSync(path.join(dir, "pid-" + name), String(process.pid));
+setTimeout(() => fs.writeFileSync(path.join(dir, "completed-" + name), ""), 20000);
+`,
+ );
+ chmodSync(docker, 0o755);
+ const moduleUrl = new URL("../src/backends/aws.ts", import.meta.url).href;
+ const code = `const { awsUp } = await import(${JSON.stringify(moduleUrl)}); await awsUp(${JSON.stringify(config)}, ${JSON.stringify(dir)}, ${JSON.stringify({ buildOnly: true, buildFrom: true, buildConcurrency: 2, imageLabel: "cancelled-candidate", candidateOut: candidatePath })});`;
+ const parent = spawn(process.execPath, ["--input-type=module", "-e", code], {
+ env: { ...process.env, PATH: `${dir}:${process.env.PATH}` },
+ stdio: "ignore",
+ });
+ const exited = new Promise<number | null>((resolve, reject) => {
+ parent.once("error", reject);
+ parent.once("exit", resolve);
+ });
+ try {
+ const deadline = Date.now() + 10000;
+ while (!["core", "web-ui", "web-ui-plugin"].every((name) => existsSync(join(dir, "pid-" + name)))) {
+ assert.ok(Date.now() < deadline, "Docker children and nested buildx plugin must start");
+ assert.equal(parent.exitCode, null);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ parent.kill("SIGTERM");
+ assert.notEqual(await exited, 0);
+ assert.equal(existsSync(candidatePath), false);
+ for (const name of ["core", "web-ui", "web-ui-plugin"]) {
+ const pid = Number(readFileSync(join(dir, "pid-" + name), "utf8"));
+ assert.throws(() => process.kill(pid, 0), /ESRCH/);
+ assert.equal(existsSync(join(dir, "completed-" + name)), false);
+ }
+ assert.equal(existsSync(join(dir, "pid-admin")), false);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /dynamodb|ecs |secretsmanager/);
+ } finally {
+ parent.kill("SIGKILL");
+ for (const name of ["core", "web-ui", "web-ui-plugin"]) {
+ const path = join(dir, "pid-" + name);
+ if (existsSync(path)) {
+ try {
+ process.kill(Number(readFileSync(path, "utf8")), "SIGKILL");
+ } catch {
+ void 0;
+ }
+ }
+ }
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("AWS rejects invalid or misplaced candidate build concurrency before mutation", async () => {
+ for (const value of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1]) {
+ await assert.rejects(awsUp(config, ".", { buildOnly: true, buildConcurrency: value }), /positive integer/);
+ }
+ await assert.rejects(awsUp(config, ".", { buildConcurrency: 2 }), /requires --build-only/);
+ const shared = structuredClone(config);
+ shared.aws!.services["web-ui"]!.ecrRepository = shared.aws!.services.core!.ecrRepository;
+ await assert.rejects(awsUp(shared, ".", { buildOnly: true, buildConcurrency: 2 }), /distinct ECR repositories/);
+});
+
test("AWS candidate deploy fails closed on account, repository, or missing-workload drift", async () => {
const dir = mkdtempSync(join(tmpdir(), "qm-aws-candidate-invalid-"));
const candidatePath = join(dir, "candidate.json");
diff --git a/cli/test/providers.test.ts b/cli/test/providers.test.ts
index 8484a6544..2478147e2 100644
--- a/cli/test/providers.test.ts
+++ b/cli/test/providers.test.ts
@@ -29,6 +29,7 @@ test("the hosting provider registry owns target discovery and lifecycle capabili
"candidate-out",
"inactive",
"restart",
+ "build-concurrency",
]);
});
diff --git a/cli/test/util.test.ts b/cli/test/util.test.ts
index 617b071fd..05847120b 100644
--- a/cli/test/util.test.ts
+++ b/cli/test/util.test.ts
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
import { chmodSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { canonicalJson, flyBin, isInvalidSecret, readEnvFile, writeEnvValue } from "../src/util.ts";
+import { canonicalJson, runInheritAsync, flyBin, isInvalidSecret, readEnvFile, writeEnvValue } from "../src/util.ts";
test("managed credential encryption keys require strong material", () => {
assert.equal(isInvalidSecret("CONNECTOR_SECRET_KEY", "short"), true);
@@ -185,3 +185,8 @@ test("promptHidden treats Ctrl-D as enter on a non-empty buffer and as cancel on
await assert.rejects(() => pending, /secret entry cancelled/);
});
});
+
+test("async inherited processes reject spawn errors and signal termination", async () => {
+ await assert.rejects(runInheritAsync("/definitely-missing-qm-command", []), /ENOENT/);
+ await assert.rejects(runInheritAsync(process.execPath, ["-e", 'process.kill(process.pid, "SIGTERM")']), /SIGTERM/);
+});