Multiplayer agent harness for work.
import { awsCoreHostnames, awsPortalAppsDomain, validAlbHostname } from "../aws-routing.ts";
import https from "node:https";
import { createHash, createHmac, randomUUID } from "node:crypto";
import { spawnSync } from "node:child_process";
import { lookup, resolveCname } from "node:dns/promises";
import {
accessSync,
constants,
existsSync,
linkSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, isAbsolute, join, resolve } from "node:path";
import {
acquireAwsLease as acquireLease,
awsText,
deployLocksTable,
deploymentStateKey,
releaseAwsLease as releaseLease,
withAwsLease,
} from "../aws-lease.ts";
import { CliError, dim, errMessage, header, note, ok, step, warn } from "../log.ts";
import {
awsWorkloadArchitecture,
requiresAwsMicrovmImage,
isDigestPinned,
sandboxCoreEnv,
securityScreenEnv,
type AwsConfig,
type QmConfig,
} from "../config.ts";
import { manifestRef } from "../manifest.ts";
import {
computedSecrets,
validatedSecrets,
runtimeSecretNames,
secretsForService,
type ComputedSecret,
} from "../secrets.ts";
import {
brokerWiring,
brandEnvOf,
orgEnv,
runnableServices,
hostedServiceEnv,
serviceHost,
serviceDef,
isServiceName,
virtualServiceEnv,
type LogOpts,
type ServiceName,
} from "../services.ts";
import { discoverPlugins, type ResolvedPlugin } from "../plugins.ts";
import { canonicalJson } from "../canonical-json.ts";
import {
capture,
deploymentSecretValue,
envNum,
isInvalidSecret,
isMissingOrPlaceholder,
promptHidden,
readEnvFile,
resolveBuildRepoRoot,
runInherit,
runInheritAsync,
settleAll,
sleep,
streamLabeled,
} from "../util.ts";
import { doctorCommon } from "./doctor.ts";
import { checkControlledLiveSession } from "../live-session.ts";
import {
assertTerraformScaffoldSupportsConfig,
awsObjectStoreBucket,
declaredVariables,
terraformVarsDrift,
} from "../terraform.ts";
import {
currentDeploymentLayerState,
deploymentLayerBody,
syncDeploymentLayerBody,
type DeploymentLayerSyncResult,
httpDeploymentLayerTransport,
type DeploymentLayerTransport,
} from "../deployment-layer.ts";
import {
awaitBackgroundWork,
readBackgroundWork,
readProductionOwner,
setBackgroundOwner,
type BackgroundWorkStatus,
type BackgroundWorkTransport,
} from "../background-work.ts";
async function fetchCore(
url: URL,
init: RequestInit,
maxResponseBytes?: number,
): Promise<{ status: number; body: string }> {
const response = await fetch(url, init);
if (maxResponseBytes === undefined) return { status: response.status, body: await response.text() };
const chunks: Uint8Array[] = [];
let size = 0;
const reader = response.body?.getReader();
if (reader) {
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > maxResponseBytes) throw new CliError("AWS core response exceeded its size limit");
chunks.push(value);
}
} catch (error) {
await reader.cancel().catch(() => {});
throw error;
} finally {
reader.releaseLock();
}
}
return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
}
export async function awsCoreRequest(
config: QmConfig,
url: URL,
init: RequestInit,
maxResponseBytes?: number,
): Promise<{ status: number; body: string }> {
const target = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");
if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");
if (awsPublicOrigin(config).protocol === "http:") {
assertCloudFrontLayerTarget(config, url, target);
return fetchCore(url, init, maxResponseBytes);
}
return new Promise((resolve, reject) => {
const request = https.request(
url,
{
hostname: target,
servername: url.hostname,
method: init.method,
headers: { ...(init.headers as Record<string, string>), host: url.host },
signal: init.signal ?? undefined,
agent: false,
},
(response) => {
const chunks: Buffer[] = [];
let size = 0;
response.on("data", (chunk: Buffer) => {
size += chunk.length;
if (maxResponseBytes !== undefined && size > maxResponseBytes) {
const error = new CliError("AWS core response exceeded its size limit");
reject(error);
response.destroy(error);
return;
}
chunks.push(chunk);
});
response.on("error", reject);
response.on("end", () =>
resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }),
);
},
);
request.on("error", reject);
request.end(init.body as string | undefined);
});
}
export const awsDeploymentLayerTransport: DeploymentLayerTransport = httpDeploymentLayerTransport({
urlOf: (config) => {
const url = new URL(config.apiUrl ?? config.publicUrl);
if (url.protocol !== "https:" || url.username || url.password || url.hash)
throw new CliError("AWS deployment-layer URL must be HTTPS without credentials or a fragment");
url.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/deployment-layer`;
return url;
},
request: awsCoreRequest,
secretFallback: (config) =>
config.aws
? capture(process.env.AWS_BIN ?? "aws", [
"secretsmanager",
"get-secret-value",
"--secret-id",
`${config.aws.secretsPrefix}CORE_SIGNING_SECRET`,
"--query",
"SecretString",
"--output",
"text",
"--region",
config.aws.region,
]).trim()
: undefined,
timeoutMs: 60_000,
});
export interface AwsUpOpts {
dryRun?: boolean;
yes?: boolean;
buildFrom?: boolean;
buildFromPath?: string;
imageLabel?: string;
only?: string[];
restart?: string[];
sandboxDir?: string;
envFile?: string;
buildOnly?: boolean;
buildConcurrency?: number;
candidate?: string;
candidateOut?: string;
inactive?: boolean;
}
interface AwsMigrationService {
status?: string;
launchType?: string;
platformVersion?: string;
networkConfiguration?: {
awsvpcConfiguration?: {
subnets?: string[];
securityGroups?: string[];
assignPublicIp?: string;
};
};
}
export interface EcsTaskDefinition {
family: string;
networkMode: "awsvpc";
requiresCompatibilities: ["FARGATE"];
cpu: string;
memory: string;
runtimePlatform: { cpuArchitecture: "ARM64" | "X86_64"; operatingSystemFamily: "LINUX" };
executionRoleArn: string;
taskRoleArn: string;
containerDefinitions: Array<Record<string, unknown>>;
}
function requireAws(config: QmConfig): AwsConfig {
if (!config.aws) throw new CliError('target "aws" requires an aws block');
return config.aws;
}
function rdsInstanceIdentifier(aws: AwsConfig): string {
return aws.rdsInstance ?? `${aws.cluster}-core`;
}
function deployedAwsServices(aws: AwsConfig): string[] {
return Object.keys(aws.services).filter((name) => serviceHost(name) === name);
}
function awsTopology(
config: QmConfig,
configDir: string,
): { aws: AwsConfig; workloads: string[]; plugins: ResolvedPlugin[] } {
const aws = requireAws(config);
assertTerraformScaffoldSupportsConfig(config, configDir);
const discovered = discoverPlugins(configDir, config);
if (discovered.errors.length) throw new CliError(discovered.errors.join("\n"));
const workloads = [...runnableServices(config.services), ...discovered.plugins.map((plugin) => plugin.name)];
const enabled = new Set(workloads);
const stale = Object.keys(aws.services)
.filter((workload) => !enabled.has(workload) && !enabled.has(serviceHost(workload)))
.sort();
const missing = workloads.filter((workload) => !aws.services[workload]);
if (stale.length || missing.length) {
const problems = [
...(stale.length ? [`disabled workloads: ${stale.join(", ")}`] : []),
...(missing.length ? [`missing enabled workloads: ${missing.join(", ")}`] : []),
];
throw new CliError(`aws.services topology mismatch (${problems.join("; ")})`);
}
return { aws, workloads, plugins: discovered.plugins };
}
function containerSecretNames(service: string, secret: ComputedSecret): string[] {
const names = runtimeSecretNames(service, secret);
return names.length ? names : [secret.name];
}
function awsArgs(aws: AwsConfig, args: string[]): string[] {
return [...args, "--region", aws.region];
}
function awsJson<T>(aws: AwsConfig, args: string[]): T {
const raw = capture(process.env.AWS_BIN ?? "aws", awsArgs(aws, [...args, "--output", "json"]));
if (!raw.trim()) return {} as T;
return JSON.parse(raw) as T;
}
function assertAwsCallerAccount(aws: AwsConfig): void {
const account = awsText(aws, ["sts", "get-caller-identity", "--query", "Account"]);
if (account !== aws.accountId) {
throw new CliError(`authenticated to AWS account ${account || "unknown"}, expected ${aws.accountId}`);
}
}
function registerTaskDefinition(config: QmConfig, file: string): string {
return awsText(config.aws!, [
"ecs",
"register-task-definition",
"--cli-input-json",
`file://${file}`,
"--tags",
JSON.stringify([
{ key: "ManagedBy", value: "qm-cli" },
{ key: "Deployment", value: config.orgId },
]),
"--query",
"taskDefinition.taskDefinitionArn",
]);
}
function deployImageCoordinates(config: QmConfig): { name: string; version: string } {
const name = config.env.core?.AWS_DEPLOY_IMAGE?.trim();
const version = config.env.core?.AWS_DEPLOY_IMAGE_VERSION?.trim();
if (!name || !version || isMissingOrPlaceholder(name) || isMissingOrPlaceholder(version)) {
throw new CliError("AWS requires exact env.core.AWS_DEPLOY_IMAGE and AWS_DEPLOY_IMAGE_VERSION coordinates");
}
return { name, version };
}
export function guardLambdaMicrovms(e: unknown): never {
if (/invalid choice:?\s*'?lambda-microvms/i.test(errMessage(e))) {
throw new CliError(
"this AWS CLI lacks the `lambda-microvms` commands needed to build/verify the AWS deploy MicroVM image; install an AWS CLI with Lambda MicroVMs support",
);
}
throw e instanceof Error ? e : new Error(errMessage(e));
}
function assertAwsDeployImage(config: QmConfig): void {
if (!requiresAwsMicrovmImage(config)) return;
const aws = requireAws(config);
const { name, version } = deployImageCoordinates(config);
const expectedArn = `arn:aws:lambda:${aws.region}:${aws.accountId}:microvm-image:${name}`;
let image: Record<string, unknown>;
try {
image = awsJson<Record<string, unknown>>(aws, [
"lambda-microvms",
"get-microvm-image",
"--image-identifier",
expectedArn,
]);
} catch (e) {
if (/invalid choice:?\s*'?lambda-microvms/i.test(errMessage(e))) return;
guardLambdaMicrovms(e);
}
const detail = (image.image && typeof image.image === "object" ? image.image : image) as Record<string, unknown>;
const arn = detail.imageArn ?? detail.imageARN ?? detail.arn;
if (arn !== expectedArn)
throw new CliError(`AWS deploy image ${name} resolves to ${String(arn ?? "no ARN")}, expected ${expectedArn}`);
const versions =
awsJson<{ items?: Array<{ imageVersion?: string | number; state?: string; status?: string }> }>(aws, [
"lambda-microvms",
"list-microvm-image-versions",
"--image-identifier",
expectedArn,
]).items ?? [];
const pinned = versions.find((item) => String(item.imageVersion) === version);
if (!pinned || pinned.state !== "SUCCESSFUL" || pinned.status !== "ACTIVE") {
throw new CliError(`AWS deploy image ${name} version ${version} is not SUCCESSFUL and ACTIVE`);
}
}
function secretValueFrom(config: QmConfig, name: string, arns?: Record<string, string>): string {
if (arns?.[name]) return arns[name];
const aws = requireAws(config);
return `arn:aws:secretsmanager:${aws.region}:${aws.accountId}:secret:${aws.secretsPrefix}${name}`;
}
export function serviceEnvironment(config: QmConfig, service: ServiceName): Record<string, string> {
const aws = requireAws(config);
const def = serviceDef(service);
const coreUrl = `http://core.${aws.networking.cloudMapNamespace}:8080`;
const coreEnv =
service === "core"
? {
...(config.model ? { PI_MODEL: config.model } : {}),
...(config.modelProvider ? { MODEL_PROVIDER: config.modelProvider } : {}),
...virtualServiceEnv(config.services, config.env),
}
: {};
const env: Record<string, string> = {
...orgEnv(service, config.orgId, config.publicUrl, config.services.includes("portal"), brandEnvOf(config)),
...(service === "core"
? {}
: {
CORE_API_URL:
isServiceName(service) && serviceDef(service).tcpIngress ? (config.apiUrl ?? config.publicUrl) : coreUrl,
}),
...coreEnv,
...hostedServiceEnv(config.services, config.env, service),
...(service === "core" ? securityScreenEnv(config) : {}),
};
if (service === "core") {
const sandboxBackend = config.env.core?.SANDBOX_BACKEND?.trim() || config.sandbox?.backend;
const stores = {
DEPLOY_PROVIDER: config.env.core?.DEPLOY_PROVIDER?.trim() || "aws",
AWS_DEPLOY_REGION: aws.region,
SESSION_STORE: "postgres",
RUN_STORE: "postgres",
SNAPSHOT_STORE: "s3",
TRANSFER_STORE: "s3",
S3_BUCKET: config.env.core?.S3_BUCKET?.trim() || awsObjectStoreBucket(config),
S3_REGION: aws.region,
};
if (sandboxBackend && sandboxBackend !== "aws") {
Object.assign(env, sandboxCoreEnv(config).env, {
SANDBOX_BACKEND: sandboxBackend,
...stores,
});
} else {
Object.assign(env, {
SANDBOX_BACKEND: "aws",
AWS_SANDBOX_REGION: aws.region,
AWS_SANDBOX_IMAGE: config.env.core?.AWS_DEPLOY_IMAGE ?? "",
AWS_SANDBOX_IMAGE_VERSION: config.env.core?.AWS_DEPLOY_IMAGE_VERSION ?? "",
AWS_SANDBOX_EXEC_ROLE_ARN:
config.env.core?.AWS_DEPLOY_EXEC_ROLE_ARN ?? `arn:aws:iam::${aws.accountId}:role/${aws.cluster}-microvm-exec`,
AWS_SANDBOX_S3_BUCKET: awsObjectStoreBucket(config),
...stores,
});
}
}
if (service === "portal") {
env.WEB_UI_UPSTREAM = `http://web-ui.${aws.networking.cloudMapNamespace}:8080`;
env.ADMIN_UPSTREAM = `http://web-ui.${aws.networking.cloudMapNamespace}:8080/admin`;
env.PORTAL_XFF_TRUSTED_HOPS = "1";
}
if (config.services.includes("auth")) {
Object.assign(
env,
brokerWiring(service, {
publicUrl: config.publicUrl,
authBaseUrl: `http://auth.${aws.networking.cloudMapNamespace}:8080`,
...(config.env.auth?.AUTH_ALLOWED_EMAIL_DOMAIN
? { allowedEmailDomain: config.env.auth.AUTH_ALLOWED_EMAIL_DOMAIN }
: {}),
}),
);
}
if (config.services.includes("portal") && (service === "web-ui" || service === "admin")) {
env.REQUIRE_SIGNED_PORTAL_IDENTITY = "1";
}
env[def.docker.portEnv] = String(def.docker.internalPort);
return Object.fromEntries(Object.entries(env).sort(([a], [b]) => a.localeCompare(b)));
}
function workloadEnvironment(
config: QmConfig,
workload: string,
backgroundDeploymentId?: string,
): Record<string, string> {
if (isServiceName(workload)) {
const env = serviceEnvironment(config, workload);
if (workload === "core" && requireAws(config).backgroundWorkControl) {
if (!backgroundDeploymentId)
throw new CliError("controlled background work requires a manifest-bound deployment identity");
env.BACKGROUND_DEPLOYMENT_ID = backgroundDeploymentId;
}
return env;
}
const plugin = config.plugins.find((entry) => entry.name === workload);
return Object.fromEntries(
Object.entries({
...(plugin?.coreAccess === false
? {}
: { CORE_API_URL: `http://core.${requireAws(config).networking.cloudMapNamespace}:8080` }),
...orgEnv(workload, config.orgId, config.publicUrl, config.services.includes("portal"), brandEnvOf(config)),
...plugin?.env,
PORT: "8080",
}).sort(([a], [b]) => a.localeCompare(b)),
);
}
function workloadSecrets(config: QmConfig, workload: string, available?: Record<string, string>) {
const secrets = secretsForService(config, workload).filter(
(secret) => secret.required || Boolean(available?.[secret.name]),
);
const plugin = config.plugins.find((entry) => entry.name === workload);
if (
!isServiceName(workload) &&
plugin?.coreAccess !== false &&
!secrets.some((secret) => secret.name === "CORE_SIGNING_SECRET")
) {
const signing = computedSecrets(config).find((secret) => secret.name === "CORE_SIGNING_SECRET");
if (signing) return [...secrets, signing];
}
return secrets;
}
function workloadArchitecture(config: QmConfig, workload: string): "arm64" | "amd64" {
requireAws(config);
return awsWorkloadArchitecture(config, workload);
}
export function renderTaskDefinition(
config: QmConfig,
service: string,
image: string,
secretArns?: Record<string, string>,
backgroundDeploymentId?: string,
): EcsTaskDefinition {
if (!isDigestPinned(image)) throw new CliError(`aws task image for ${service} must be pinned by digest`);
const aws = requireAws(config);
const spec = aws.services[service];
if (!spec) throw new CliError(`aws.services.${service} is missing`);
const def = isServiceName(service) ? serviceDef(service) : undefined;
const internalPort = def?.docker.internalPort ?? 8080;
const executionRoleArn = spec.executionRoleArn ?? `arn:aws:iam::${aws.accountId}:role/${aws.cluster}-task-execution`;
const managedTaskRole = spec.assumeRoleArns?.length
? `${aws.cluster}-${service}-task`
: `${aws.cluster}-${service === "core" ? "core-task" : "task"}`;
const taskRoleArn = spec.taskRoleArn ?? `arn:aws:iam::${aws.accountId}:role/${managedTaskRole}`;
const secrets = workloadSecrets(config, service, secretArns)
.flatMap((secret) =>
containerSecretNames(service, secret).map((name) => ({
name,
valueFrom: secretValueFrom(config, secret.name, secretArns),
})),
)
.sort((a, b) => a.name.localeCompare(b.name));
const healthCheck = def
? {
command: def.tcpIngress?.healthCheck ?? [
"CMD",
"node",
"-e",
`fetch('http://127.0.0.1:${internalPort}/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))`,
],
interval: 10,
timeout: 5,
retries: 3,
startPeriod: 30,
}
: undefined;
return {
family: spec.ecsService,
networkMode: "awsvpc",
requiresCompatibilities: ["FARGATE"],
cpu: String(spec.cpu),
memory: String(spec.memory),
runtimePlatform: {
cpuArchitecture: workloadArchitecture(config, service) === "arm64" ? "ARM64" : "X86_64",
operatingSystemFamily: "LINUX",
},
executionRoleArn,
taskRoleArn,
containerDefinitions: [
{
name: service,
image,
essential: true,
environment: Object.entries(workloadEnvironment(config, service, backgroundDeploymentId)).map(
([name, value]) => ({ name, value }),
),
secrets,
portMappings: [
{
name: service,
containerPort: internalPort,
hostPort: internalPort,
protocol: "tcp",
...(def?.tcpIngress ? {} : { appProtocol: "http" }),
},
],
...(healthCheck ? { healthCheck } : {}),
...(spec.stopTimeout !== undefined ? { stopTimeout: spec.stopTimeout } : {}),
logConfiguration: {
logDriver: "awslogs",
options: {
"awslogs-group": spec.logGroup ?? `/ecs/${spec.ecsService}`,
"awslogs-region": aws.region,
"awslogs-stream-prefix": service,
},
},
},
],
};
}
function ecrHost(aws: AwsConfig): string {
return `${aws.accountId}.dkr.ecr.${aws.region}.amazonaws.com`;
}
function expectedWorkloadImageRepository(config: QmConfig, workload: string): string {
const aws = requireAws(config);
const repository = aws.services[workload]?.ecrRepository;
if (!repository) throw new CliError(`aws.services.${workload} is missing`);
return `${ecrHost(aws)}/${repository}`;
}
export function isPinnedWorkloadImage(config: QmConfig, workload: string, image: string): boolean {
const repository = expectedWorkloadImageRepository(config, workload).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
return new RegExp(`^${repository}@sha256:[0-9a-f]{64}$`).test(image);
}
function dockerLogin(aws: AwsConfig): void {
const password = awsText(aws, ["ecr", "get-login-password"]);
const result = spawnSync("docker", ["login", "--username", "AWS", "--password-stdin", ecrHost(aws)], {
input: `${password}\n`,
encoding: "utf8",
stdio: ["pipe", "inherit", "inherit"],
});
if (result.status !== 0) throw new CliError("docker login to ECR failed");
}
function sourceImage(config: QmConfig, service: ServiceName): string {
return config.imageOverrides[service] ?? manifestRef(service);
}
function workloadSourceImage(
config: QmConfig,
workload: string,
plugin: ResolvedPlugin | undefined,
): string | undefined {
return (
plugin?.image ??
config.plugins.find((entry) => entry.name === workload)?.image ??
(isServiceName(workload) ? sourceImage(config, workload) : undefined)
);
}
function workloadImageProvenance(
config: QmConfig,
workload: string,
plugin: ResolvedPlugin | undefined,
opts: Pick<AwsUpOpts, "buildFrom" | "buildFromPath">,
): DeploymentImageProvenance {
if (plugin?.kind === "source") {
return { kind: "source-build", source: "plugin", ...sourceBuildInfo(plugin.sourceDir!) };
}
if (opts.buildFrom && isServiceName(workload)) {
return {
kind: "source-build",
source: "checkout",
...sourceBuildInfo(resolveBuildRepoRoot(opts.buildFromPath, [workload])),
};
}
const source = workloadSourceImage(config, workload, plugin);
if (!source) throw new CliError(`AWS workload ${workload} has no source image`);
return { kind: "configured", source };
}
const sourceBuildInfoByRoot = new Map<string, { gitCommit?: string; dirty?: boolean }>();
function sourceBuildInfo(root: string): { gitCommit?: string; dirty?: boolean } {
const cached = sourceBuildInfoByRoot.get(root);
if (cached) return cached;
const info: { gitCommit?: string; dirty?: boolean } = {};
try {
info.gitCommit = capture("git", ["-C", root, "rev-parse", "HEAD"]).trim();
info.dirty = capture("git", ["-C", root, "status", "--porcelain"]).trim().length > 0;
} catch {
void 0;
}
sourceBuildInfoByRoot.set(root, info);
return info;
}
function sourceImageDigest(source: string): string {
const pinned = source.match(/@(?<digest>sha256:[0-9a-f]{64})$/)?.groups?.digest;
if (pinned) return pinned;
const output = capture("docker", ["buildx", "imagetools", "inspect", source]);
const digest = output.match(/^Digest:\s*(sha256:[0-9a-f]{64})\s*$/m)?.[1];
if (!digest) throw new CliError(`registry did not return an immutable digest for AWS source image ${source}`);
return digest;
}
function plannedWorkloadImage(config: QmConfig, workload: string, plugin: ResolvedPlugin | undefined): string {
const source = workloadSourceImage(config, workload, plugin);
if (!source) throw new CliError(`AWS workload ${workload} has no source image`);
return `${expectedWorkloadImageRepository(config, workload)}@${sourceImageDigest(source)}`;
}
function workloadBuildArgs(config: QmConfig, workload: string): Record<string, string> {
return { ...requireAws(config).services[workload]?.buildArgs };
}
export function imageTransferArgs(source: string, tagged: string): string[] {
return ["buildx", "imagetools", "create", "--prefer-index=false", "--tag", tagged, source];
}
async function publishWorkloadImage(
config: QmConfig,
workload: string,
plugin: ResolvedPlugin | undefined,
label: string,
opts: AwsUpOpts,
signal?: AbortSignal,
): Promise<string> {
const aws = requireAws(config);
const spec = aws.services[workload]!;
const tagged = `${ecrHost(aws)}/${spec.ecrRepository}:${label}`;
const platform = `linux/${workloadArchitecture(config, workload)}`;
if (plugin?.kind === "source") {
const args = [
"buildx",
"build",
"--platform",
platform,
"--provenance=false",
"--push",
"-f",
plugin.dockerfile!,
"-t",
tagged,
];
for (const [name, value] of Object.entries(workloadBuildArgs(config, workload)))
args.push("--build-arg", `${name}=${value}`);
args.push(plugin.sourceDir!);
if (signal) await runInheritAsync("docker", args, { signal });
else runInherit("docker", args);
} else if (opts.buildFrom && isServiceName(workload)) {
const root = resolveBuildRepoRoot(opts.buildFromPath, [workload]);
const dockerfile = join(root, spec.dockerfile ?? join("deploy", workload, "Dockerfile"));
if (spec.dockerfile && !existsSync(dockerfile)) {
throw new CliError(`aws.services.${workload}.dockerfile is missing from the build checkout: ${spec.dockerfile}`);
}
const args = [
"buildx",
"build",
"--platform",
platform,
"--provenance=false",
"--push",
"-f",
dockerfile,
"-t",
tagged,
];
const info = sourceBuildInfo(root);
if (info.gitCommit) args.push("--build-arg", `GIT_SHA=${info.gitCommit}${info.dirty ? "-dirty" : ""}`);
for (const [name, value] of Object.entries(workloadBuildArgs(config, workload)))
args.push("--build-arg", `${name}=${value}`);
args.push(root);
if (signal) await runInheritAsync("docker", args, { signal });
else runInherit("docker", args);
} else {
const source = workloadSourceImage(config, workload, plugin);
if (!source) throw new CliError(`AWS workload ${workload} has no source image`);
if (signal) await runInheritAsync("docker", imageTransferArgs(source, tagged), { signal });
else runInherit("docker", imageTransferArgs(source, tagged));
}
const response = awsJson<{ imageDetails?: Array<{ imageDigest?: string }> }>(aws, [
"ecr",
"describe-images",
"--repository-name",
spec.ecrRepository,
"--image-ids",
`imageTag=${label}`,
]);
const digest = response.imageDetails?.[0]?.imageDigest;
if (!digest) throw new CliError(`ECR did not return a digest for ${tagged}`);
return `${ecrHost(aws)}/${spec.ecrRepository}@${digest}`;
}
function releaseCandidate(config: QmConfig, path: string): AwsReleaseCandidate {
let parsed: unknown;
try {
parsed = JSON.parse(readFileSync(path, "utf8"));
} catch (error) {
throw new CliError(`could not read AWS release candidate ${path}: ${errMessage(error)}`);
}
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new CliError(`AWS release candidate ${path} is not an object`);
}
const candidate = parsed as Partial<AwsReleaseCandidate>;
const aws = requireAws(config);
if (
candidate.contract !== 1 ||
candidate.accountId !== aws.accountId ||
candidate.region !== aws.region ||
typeof candidate.label !== "string" ||
!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(candidate.label) ||
!candidate.images ||
typeof candidate.images !== "object" ||
Array.isArray(candidate.images) ||
!candidate.imageProvenance ||
typeof candidate.imageProvenance !== "object" ||
Array.isArray(candidate.imageProvenance) ||
(candidate.architectures !== undefined &&
(!candidate.architectures ||
typeof candidate.architectures !== "object" ||
Array.isArray(candidate.architectures)))
) {
throw new CliError(`AWS release candidate ${path} has an invalid contract or targets another AWS account/region`);
}
for (const [workload, architecture] of Object.entries(candidate.architectures ?? {})) {
if (!candidate.images[workload] || (architecture !== "arm64" && architecture !== "amd64"))
throw new CliError(`AWS release candidate ${path} has an invalid ${workload} architecture`);
}
for (const [workload, image] of Object.entries(candidate.images)) {
const provenance = candidate.imageProvenance[workload];
const validProvenance =
provenance?.kind === "configured"
? typeof provenance.source === "string"
: provenance?.kind === "source-build" &&
(provenance.source === undefined || provenance.source === "plugin" || provenance.source === "checkout") &&
(provenance.gitCommit === undefined || typeof provenance.gitCommit === "string") &&
(provenance.dirty === undefined || typeof provenance.dirty === "boolean");
if (
!aws.services[workload] ||
typeof image !== "string" ||
!isPinnedWorkloadImage(config, workload, image) ||
!validProvenance
) {
throw new CliError(`AWS release candidate ${path} has an invalid ${workload} image or provenance`);
}
const architecture = candidate.architectures?.[workload];
if (candidate.architectures && architecture !== workloadArchitecture(config, workload))
throw new CliError(
`AWS release candidate ${path} architecture for ${workload} does not match its configuration; use a matching architecture or rebuild the candidate`,
);
if (
architecture === undefined &&
provenance?.kind === "source-build" &&
isServiceName(workload) &&
!aws.services[workload]!.architecture
)
throw new CliError(
`AWS release candidate ${path} has no architecture for source-built ${workload}; set aws.services.${workload}.architecture to the image platform or rebuild the candidate`,
);
}
return candidate as AwsReleaseCandidate;
}
export async function awsMigrateCandidate(config: QmConfig, configDir: string, candidatePath: string): Promise<void> {
const { aws, workloads } = awsTopology(config, configDir);
if (!workloads.includes("core")) throw new CliError("AWS migrations require the core workload");
const candidate = releaseCandidate(config, candidatePath);
const coreImage = candidate.images.core;
if (!coreImage) throw new CliError(`AWS release candidate ${candidatePath} does not contain core`);
assertAwsCallerAccount(aws);
await withAwsLease(aws, async () => {
if (aws.predeployDbSnapshot === false)
note("pre-migration database restore point: disabled (aws.predeployDbSnapshot)");
else assertPredeployDbRestorePoint(config);
await runAwsCandidateMigration(config, coreImage, candidate.label);
});
}
async function runAwsCandidateMigration(
config: QmConfig,
coreImage: string,
label: string,
existingTaskDefinition?: string,
): Promise<void> {
const aws = requireAws(config);
const spec = aws.services.core;
if (!spec) throw new CliError("AWS migrations require the core workload");
const service = awsJson<{ services?: AwsMigrationService[]; failures?: Array<{ arn?: string; reason?: string }> }>(
aws,
["ecs", "describe-services", "--cluster", aws.cluster, "--services", spec.ecsService],
);
if (service.failures?.length || service.services?.[0]?.status !== "ACTIVE") {
throw new CliError(`AWS core service ${spec.ecsService} is not active`);
}
const live = service.services[0]!;
const network = live.networkConfiguration?.awsvpcConfiguration;
if (
live.launchType !== "FARGATE" ||
!network?.subnets?.length ||
!network.securityGroups?.length ||
(network.assignPublicIp !== "ENABLED" && network.assignPublicIp !== "DISABLED")
) {
throw new CliError(`AWS core service ${spec.ecsService} does not expose a reusable Fargate network configuration`);
}
const temp = mkdtempSync(join(tmpdir(), "qm-migration-task-"));
let taskDefinition: string | undefined;
try {
if (existingTaskDefinition) {
taskDefinition = existingTaskDefinition;
} else {
const definitionPath = join(temp, "core.json");
const definition = renderTaskDefinition(config, "core", coreImage, secretArns(config));
const container = definition.containerDefinitions.find((candidate) => candidate.name === "core");
if (!container) throw new CliError("AWS migration task definition has no core container");
delete container.healthCheck;
writeFileSync(definitionPath, JSON.stringify(definition));
taskDefinition = registerTaskDefinition(config, definitionPath);
}
header(`qm migrate — ${config.orgId} (AWS candidate ${label})`);
const started = awsJson<{
tasks?: Array<{ taskArn?: string }>;
failures?: Array<{ arn?: string; reason?: string; detail?: string }>;
}>(aws, [
"ecs",
"run-task",
"--cluster",
aws.cluster,
"--task-definition",
taskDefinition,
"--launch-type",
"FARGATE",
...(live.platformVersion ? ["--platform-version", live.platformVersion] : []),
"--network-configuration",
JSON.stringify({ awsvpcConfiguration: network }),
"--overrides",
JSON.stringify({ containerOverrides: [{ name: "core", command: ["node", "src/migrate-main.ts"] }] }),
"--started-by",
"qm-migration",
]);
const taskArn = started.tasks?.[0]?.taskArn;
if (!taskArn || started.failures?.length) {
const failures = started.failures
?.map((failure) => `${failure.arn ?? "unknown"}: ${failure.reason ?? failure.detail ?? "unknown"}`)
.join(", ");
throw new CliError(`AWS could not start the migration task${failures ? ` (${failures})` : ""}`);
}
step(`migration task: ${taskArn}`);
await awaitEcsTaskExit(aws, taskArn, "migration");
ok(`candidate ${label} migrations applied`);
} finally {
if (taskDefinition && !existingTaskDefinition) {
try {
awsText(aws, ["ecs", "deregister-task-definition", "--task-definition", taskDefinition]);
} catch (error) {
warn(`could not deregister migration task definition ${taskDefinition}: ${errMessage(error)}`);
}
}
rmSync(temp, { recursive: true, force: true });
}
}
export function secretArns(config: QmConfig): Record<string, string> {
const aws = requireAws(config);
const controlValues = new Map<string, string>();
const pairs = computedSecrets(config).flatMap((secret) => {
const id = `${aws.secretsPrefix}${secret.name}`;
try {
if (!secret.required) {
const metadata = awsJson<{
ARN?: string;
DeletedDate?: string;
VersionIdsToStages?: Record<string, string[]>;
}>(aws, ["secretsmanager", "describe-secret", "--secret-id", id]);
const hasCurrent = Object.values(metadata.VersionIdsToStages ?? {}).some((stages) =>
stages.includes("AWSCURRENT"),
);
return metadata.ARN && !metadata.DeletedDate && hasCurrent ? [[secret.name, metadata.ARN] as const] : [];
}
const value = awsJson<{ ARN?: string; SecretString?: string }>(aws, [
"secretsmanager",
"get-secret-value",
"--secret-id",
id,
]);
if (!value.ARN || isInvalidSecret(secret.name, value.SecretString)) {
throw new CliError(`required AWS secret ${secret.name} has no usable, non-placeholder AWSCURRENT value`);
}
if (secret.name === "CORE_SIGNING_SECRET" || secret.name === "DEPLOYMENT_CONTROL_SECRET")
controlValues.set(secret.name, value.SecretString!);
return [[secret.name, value.ARN] as const];
} catch (error) {
if (!secret.required && /ResourceNotFoundException/.test(errMessage(error))) return [];
throw error;
}
});
if (
aws.backgroundWorkControl &&
controlValues.get("CORE_SIGNING_SECRET") === controlValues.get("DEPLOYMENT_CONTROL_SECRET")
)
throw new CliError("DEPLOYMENT_CONTROL_SECRET must differ from CORE_SIGNING_SECRET");
return Object.fromEntries(pairs);
}
function assertAwsPublicApiUrl(config: QmConfig): void {
if (!computedSecrets(config).some((secret) => secret.name === "PUBLIC_API_URL")) return;
const aws = requireAws(config);
const value = awsText(aws, [
"secretsmanager",
"get-secret-value",
"--secret-id",
`${aws.secretsPrefix}PUBLIC_API_URL`,
"--query",
"SecretString",
]);
const bound = config.apiUrl ? ("apiUrl" as const) : ("publicUrl" as const);
const expected = new URL(config.apiUrl ?? config.publicUrl).toString().replace(/\/$/, "");
let normalized: string;
try {
const parsed = new URL(value);
if (parsed.protocol !== "https:") throw new Error("not HTTPS");
normalized = parsed.toString().replace(/\/$/, "");
} catch {
throw new CliError(`required AWS secret PUBLIC_API_URL must be a valid HTTPS URL equal to the configured ${bound}`);
}
if (normalized !== expected) {
throw new CliError(`required AWS secret PUBLIC_API_URL must equal the configured HTTPS ${bound} (${expected})`);
}
}
function liveTask(config: QmConfig, service: string): Record<string, unknown> | null {
const aws = requireAws(config);
const spec = aws.services[service]!;
const described = awsJson<{ services?: Array<{ taskDefinition?: string }> }>(aws, [
"ecs",
"describe-services",
"--cluster",
aws.cluster,
"--services",
spec.ecsService,
]);
const arn = described.services?.[0]?.taskDefinition;
if (!arn) return null;
return (
awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
arn,
]).taskDefinition ?? null
);
}
function normalizeLiveTask(task: Record<string, unknown> | null): Record<string, unknown> | null {
if (!task) return null;
const copy = structuredClone(task);
for (const key of [
"taskDefinitionArn",
"revision",
"status",
"requiresAttributes",
"compatibilities",
"registeredAt",
"registeredBy",
"deregisteredAt",
])
delete copy[key];
for (const key of ["volumes", "placementConstraints"] as const) {
if (Array.isArray(copy[key]) && copy[key].length === 0) delete copy[key];
}
if (copy.enableFaultInjection === false) delete copy.enableFaultInjection;
const containers = copy.containerDefinitions;
if (Array.isArray(containers)) {
for (const value of containers) {
if (typeof value !== "object" || value === null || Array.isArray(value)) continue;
const container = value as Record<string, unknown>;
if (container.cpu === 0) delete container.cpu;
for (const key of ["mountPoints", "volumesFrom", "systemControls", "resourceRequirements", "ulimits"] as const) {
if (Array.isArray(container[key]) && container[key].length === 0) delete container[key];
}
for (const key of ["environment", "secrets", "portMappings"] as const) {
if (Array.isArray(container[key])) {
container[key] = [...container[key]].sort((a, b) => canonicalJson(a).localeCompare(canonicalJson(b)));
}
}
}
}
if (Array.isArray(copy.requiresCompatibilities)) copy.requiresCompatibilities.sort();
return copy;
}
function taskReviewShape(task: Record<string, unknown>): Record<string, unknown> {
const normalized = normalizeLiveTask(task)!;
const values = Array.isArray(normalized.containerDefinitions) ? normalized.containerDefinitions : [];
normalized.containerDefinitions = Object.fromEntries(
values.map((value) => {
const container = value as Record<string, unknown>;
const shaped = { ...container };
for (const field of ["environment", "secrets"] as const) {
const entries = Array.isArray(shaped[field]) ? (shaped[field] as Array<Record<string, unknown>>) : [];
shaped[field] = Object.fromEntries(
entries.map((entry) => [String(entry.name), field === "environment" ? entry.value : entry.valueFrom]),
);
}
return [String(container.name), shaped];
}),
);
return normalized;
}
export interface TaskDefinitionChange {
path: string;
live: unknown;
desired: unknown;
}
function diffTaskValues(desired: unknown, live: unknown, path: string, out: TaskDefinitionChange[]): void {
if (canonicalJson(desired) === canonicalJson(live)) return;
if (
desired &&
live &&
typeof desired === "object" &&
typeof live === "object" &&
!Array.isArray(desired) &&
!Array.isArray(live)
) {
const expected = desired as Record<string, unknown>;
const actual = live as Record<string, unknown>;
for (const key of [...new Set([...Object.keys(expected), ...Object.keys(actual)])].sort()) {
diffTaskValues(expected[key], actual[key], path ? `${path}.${key}` : key, out);
}
return;
}
out.push({
path,
live: live === undefined ? "<absent>" : live,
desired: desired === undefined ? "<absent>" : desired,
});
}
export function taskDefinitionChanges(
desired: EcsTaskDefinition,
live: Record<string, unknown> | null,
): TaskDefinitionChange[] {
const out: TaskDefinitionChange[] = [];
diffTaskValues(
taskReviewShape(desired as unknown as Record<string, unknown>),
live ? taskReviewShape(live) : null,
"taskDefinition",
out,
);
return out;
}
export function taskDefinitionDiff(desired: EcsTaskDefinition, live: Record<string, unknown> | null): string[] {
return taskDefinitionChanges(desired, live).map((change) => change.path);
}
interface EcsDeploymentState {
id?: string;
status?: string;
taskDefinition?: string;
rolloutState?: string;
runningCount?: number;
pendingCount?: number;
failedTasks?: number;
}
interface EcsServiceState {
serviceName?: string;
status?: string;
desiredCount?: number;
runningCount?: number;
pendingCount?: number;
taskDefinition?: string;
deploymentConfiguration?: { strategy?: string };
networkConfiguration?: {
awsvpcConfiguration?: {
subnets?: string[];
securityGroups?: string[];
assignPublicIp?: "ENABLED" | "DISABLED";
};
};
deployments?: EcsDeploymentState[];
tags?: Array<{ key?: string; value?: string }>;
}
interface EcsServiceDeploymentSummary {
targetServiceRevisionArn?: string;
status?: string;
createdAt?: string | number;
}
interface EcsServiceRevision {
serviceRevisionArn?: string;
taskDefinition?: string;
}
async function awsLiveSession(config: QmConfig, core: EcsServiceState): Promise<void> {
const aws = requireAws(config);
if (!core.taskDefinition) throw new Error("core service has no live task definition");
if (!core.networkConfiguration?.awsvpcConfiguration) throw new Error("core service has no VPC network configuration");
const taskArns =
awsJson<{ taskArns?: string[] }>(aws, [
"ecs",
"list-tasks",
"--cluster",
aws.cluster,
"--service-name",
aws.services.core!.ecsService,
"--desired-status",
"RUNNING",
]).taskArns ?? [];
if (!taskArns.length) throw new Error("core service has no running tasks for the private canary");
let address: string | undefined;
for (let offset = 0; offset < taskArns.length && !address; offset += 100) {
const tasks =
awsJson<{
tasks?: Array<{
taskDefinitionArn?: string;
lastStatus?: string;
healthStatus?: string;
containers?: Array<{ name?: string; networkInterfaces?: Array<{ privateIpv4Address?: string }> }>;
}>;
}>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...taskArns.slice(offset, offset + 100)])
.tasks ?? [];
address = tasks
.filter(
(task) =>
task.taskDefinitionArn === core.taskDefinition &&
task.lastStatus === "RUNNING" &&
task.healthStatus !== "UNHEALTHY",
)
.flatMap((task) => task.containers?.filter((container) => container.name === "core") ?? [])
.flatMap((container) => container.networkInterfaces ?? [])
.find((network) => network.privateIpv4Address)?.privateIpv4Address;
}
if (!address) throw new Error("core service has no current running task with a private IPv4 address");
const started = awsJson<{
tasks?: Array<{ taskArn?: string }>;
failures?: Array<{ arn?: string; reason?: string; detail?: string }>;
}>(aws, [
"ecs",
"run-task",
"--cluster",
aws.cluster,
"--task-definition",
core.taskDefinition,
"--launch-type",
"FARGATE",
"--network-configuration",
JSON.stringify(core.networkConfiguration),
"--overrides",
JSON.stringify({
containerOverrides: [
{
name: "core",
command: ["node", "src/deployment/postdeploy-smoke.ts", "session", `http://${address}:8080`],
},
],
}),
"--count",
"1",
]);
const taskArn = started.tasks?.[0]?.taskArn;
if (!taskArn) {
const failure = started.failures?.[0];
throw new Error(
`could not start canary task: ${failure?.reason ?? failure?.detail ?? failure?.arn ?? "no task returned"}`,
);
}
await awaitEcsTaskExit(aws, taskArn, "canary");
}
async function awaitEcsTaskExit(aws: AwsConfig, taskArn: string, kind: string): Promise<void> {
const pollMs = envNum("QM_AWS_ROLLOUT_POLL_MS", 5_000);
const deadline = Date.now() + envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000);
for (;;) {
const described = awsJson<{
tasks?: Array<{
lastStatus?: string;
stoppedReason?: string;
containers?: Array<{ name?: string; exitCode?: number; reason?: string }>;
}>;
failures?: Array<{ arn?: string; reason?: string }>;
}>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", taskArn]);
const task = described.tasks?.[0];
const failures = (described.failures ?? []).map((failure) => failure.reason ?? "unknown");
if (failures.some((reason) => reason !== "MISSING")) {
throw new CliError(`AWS ${kind} task ${taskArn} failed (${failures.join(", ")})`);
}
if (task?.lastStatus === "STOPPED") {
const core = task.containers?.find((container) => container.name === "core");
if (core?.exitCode === 0) return;
throw new CliError(
`AWS ${kind} task ${taskArn} exited ${core?.exitCode ?? "without a code"}: ${[core?.reason, task.stoppedReason].filter(Boolean).join("; ") || "unknown reason"}`,
);
}
if (Date.now() > deadline) {
throw new CliError(
task
? `timed out waiting for the AWS ${kind} task; ${taskArn} is still ${task.lastStatus ?? "unknown"} and was left running`
: `timed out waiting for the AWS ${kind} task; ECS still does not list ${taskArn}`,
);
}
await sleep(pollMs);
}
}
type DeploymentImageProvenance =
| { kind: "configured"; source: string }
| { kind: "source-build"; source?: "plugin" | "checkout"; gitCommit?: string; dirty?: boolean };
interface AwsReleaseCandidate {
contract: 1;
accountId: string;
region: string;
label: string;
images: Record<string, string>;
imageProvenance: Record<string, DeploymentImageProvenance>;
architectures?: Record<string, "arm64" | "amd64">;
}
interface DeploymentManifest {
id: string;
backgroundDeploymentId?: string;
previous?: string;
createdAt: string;
imageLabel?: string;
dbSnapshot?: string;
dbRestorePoint?: string;
tasks: Record<string, string>;
counts?: Record<string, number>;
imageProvenance?: Record<string, DeploymentImageProvenance>;
layer?: { key: string; sha256: string };
}
interface BackgroundDeploymentPreparation {
id: string;
previousManifestId?: string;
backgroundDeploymentId: string;
before: { tasks: Record<string, string>; counts: Record<string, number> };
}
const BACKGROUND_PREPARATION_KEY = "deployment/background-preparation";
function clearBackgroundPreparation(aws: AwsConfig): void {
awsText(aws, [
"dynamodb",
"transact-write-items",
"--transact-items",
JSON.stringify([
{
Delete: {
TableName: deployLocksTable(aws),
Key: { lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } },
},
},
]),
]);
}
function assertBackgroundPreparationResolved(
aws: AwsConfig,
current: DeploymentManifest | undefined,
before: BackgroundDeploymentPreparation["before"],
): boolean {
const response = awsJson<{ Item?: Record<string, { S?: string }> }>(aws, [
"dynamodb",
"get-item",
"--table-name",
deployLocksTable(aws),
"--key",
JSON.stringify({ lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } }),
"--consistent-read",
]);
if (!response.Item) return false;
let preparation: BackgroundDeploymentPreparation;
try {
preparation = JSON.parse(response.Item.preparation?.S ?? "");
if (
!preparation ||
typeof preparation.id !== "string" ||
typeof preparation.backgroundDeploymentId !== "string" ||
!preparation.before?.tasks ||
!preparation.before.counts
)
throw new Error("invalid preparation");
} catch {
throw new CliError("pending background deployment preparation is invalid; refusing a new cohort");
}
const committed =
current?.id === preparation.id &&
current.backgroundDeploymentId === preparation.backgroundDeploymentId &&
canonicalJson(before) === canonicalJson({ tasks: current.tasks, counts: current.counts });
const restored =
current?.id === preparation.previousManifestId && canonicalJson(before) === canonicalJson(preparation.before);
if (!committed && !restored)
throw new CliError(
`background deployment ${preparation.id} remains unresolved; reconcile its recorded tasks before retrying instead of allocating another cohort`,
);
return true;
}
function reconcileBackgroundPreparation(
aws: AwsConfig,
current: DeploymentManifest | undefined,
before: BackgroundDeploymentPreparation["before"],
): void {
if (assertBackgroundPreparationResolved(aws, current, before)) clearBackgroundPreparation(aws);
}
function prepareBackgroundDeployment(aws: AwsConfig, preparation: BackgroundDeploymentPreparation): void {
awsText(aws, [
"dynamodb",
"transact-write-items",
"--transact-items",
JSON.stringify([
{
Put: {
TableName: deployLocksTable(aws),
Item: {
lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) },
preparation: { S: JSON.stringify(preparation) },
},
ConditionExpression: "attribute_not_exists(lockKey)",
},
},
]),
]);
}
const DEPLOYMENT_POINTER_KEY = "deployment/current";
const DEPLOYMENT_MANIFEST_PREFIX = "deployment/manifest/";
const ECS_SERVICE_BATCH_SIZE = 10;
function chunks<T>(values: T[], size: number): T[][] {
const out: T[][] = [];
for (let i = 0; i < values.length; i += size) out.push(values.slice(i, i + size));
return out;
}
function describedServices(config: QmConfig, workloads: string[]): Map<string, EcsServiceState> {
const aws = requireAws(config);
const byEcsName = new Map(workloads.map((name) => [aws.services[name]!.ecsService, name]));
const out = new Map<string, EcsServiceState>();
const failures: string[] = [];
for (const batch of chunks([...byEcsName.keys()], ECS_SERVICE_BATCH_SIZE)) {
const response = awsJson<{
services?: EcsServiceState[];
failures?: Array<{ arn?: string; reason?: string; detail?: string }>;
}>(aws, ["ecs", "describe-services", "--cluster", aws.cluster, "--include", "TAGS", "--services", ...batch]);
for (const service of response.services ?? []) {
const workload = service.serviceName ? byEcsName.get(service.serviceName) : undefined;
if (workload) out.set(workload, service);
}
for (const failure of response.failures ?? []) {
failures.push(`${failure.arn ?? "unknown service"}: ${failure.reason ?? failure.detail ?? "describe failed"}`);
}
}
for (const workload of workloads) {
if (!out.has(workload))
failures.push(`${aws.services[workload]!.ecsService}: missing from DescribeServices response`);
}
if (failures.length)
throw new CliError(`could not describe AWS services:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`);
return out;
}
function nativeBlueGreenStatus(
config: QmConfig,
workload: string,
taskDefinition: string | undefined,
): string | undefined {
if (!taskDefinition) return undefined;
const aws = requireAws(config);
const listed = awsJson<{ serviceDeployments?: EcsServiceDeploymentSummary[] }>(aws, [
"ecs",
"list-service-deployments",
"--cluster",
aws.cluster,
"--service",
aws.services[workload]!.ecsService,
]);
const deployments = listed.serviceDeployments ?? [];
const revisionArns = deployments.flatMap((deployment) =>
deployment.targetServiceRevisionArn ? [deployment.targetServiceRevisionArn] : [],
);
if (!revisionArns.length) return undefined;
const described = awsJson<{ serviceRevisions?: EcsServiceRevision[] }>(aws, [
"ecs",
"describe-service-revisions",
"--service-revision-arns",
...revisionArns,
]);
const tasksByRevision = new Map(
(described.serviceRevisions ?? []).flatMap((revision) =>
revision.serviceRevisionArn && revision.taskDefinition
? [[revision.serviceRevisionArn, revision.taskDefinition] as const]
: [],
),
);
const timestamp = (deployment: EcsServiceDeploymentSummary): number =>
typeof deployment.createdAt === "number" ? deployment.createdAt : Date.parse(deployment.createdAt ?? "") || 0;
return deployments
.filter(
(deployment) =>
deployment.targetServiceRevisionArn &&
tasksByRevision.get(deployment.targetServiceRevisionArn) === taskDefinition,
)
.sort((a, b) => timestamp(b) - timestamp(a))[0]?.status;
}
function assertOwnedServices(config: QmConfig, states: Map<string, EcsServiceState>, workloads: string[]): void {
for (const workload of workloads) {
const service = states.get(workload)!;
const tags = Object.fromEntries(
(service.tags ?? []).flatMap((tag) => (tag.key && tag.value ? [[tag.key, tag.value] as const] : [])),
);
if (tags["Deployment"] !== config.orgId || tags["ManagedBy"] !== "terraform") {
throw new CliError(
`refusing to mutate ${service.serviceName ?? workload}: ownership tags do not match deployment ${config.orgId}`,
);
}
}
}
interface RolloutTarget {
taskDefinition: string;
desiredCount: number;
deploymentId?: string;
}
async function awaitServiceTargets(
config: QmConfig,
expected: Record<string, RolloutTarget>,
timeoutMs?: number,
): Promise<void> {
const pollMs = envNum("QM_AWS_ROLLOUT_POLL_MS", 15_000);
const deadline = Date.now() + (timeoutMs ?? envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 20 * 60_000));
let failurePolls = new Map<string, number>();
let describeFailures = 0;
const failing = new Map<string, { polls: number; baseRunning: number }>();
for (;;) {
let states: ReadonlyMap<string, EcsServiceState>;
let nativeStatuses: ReadonlyMap<string, string | undefined>;
try {
states = describedServices(config, Object.keys(expected));
nativeStatuses = new Map(
Object.entries(expected).flatMap(([workload, want]) =>
states.get(workload)?.deploymentConfiguration?.strategy === "BLUE_GREEN"
? [[workload, nativeBlueGreenStatus(config, workload, want.taskDefinition)] as const]
: [],
),
);
describeFailures = 0;
} catch (error) {
describeFailures += 1;
if (describeFailures > 2 || Date.now() > deadline) throw error;
note(`could not poll AWS services (${errMessage(error)}); retrying`);
await sleep(pollMs);
continue;
}
const failures: Array<{ identity: string; message: string }> = [];
const fail = (workload: string, kind: string, message: string): void =>
void failures.push({ identity: `${workload}:${kind}`, message });
const waiting: string[] = [];
const failingNow = new Set<string>();
for (const [workload, want] of Object.entries(expected)) {
const state = states.get(workload)!;
const primary = (state.deployments ?? []).filter((deployment) => deployment.status === "PRIMARY");
const deployment = primary[0];
const running = deployment?.runningCount ?? 0;
if (state.status !== "ACTIVE") {
fail(workload, "service-status", `${workload}: service is ${state.status ?? "missing"}`);
} else if (
state.taskDefinition !== want.taskDefinition ||
primary.length !== 1 ||
deployment?.taskDefinition !== want.taskDefinition
) {
fail(
workload,
"task-definition",
`${workload}: requested task definition is not the sole PRIMARY deployment (service runs ${state.taskDefinition ?? "no task"} — an ECS circuit-breaker rollback or a concurrent deploy)`,
);
} else if (want.deploymentId !== undefined && deployment.id !== want.deploymentId) {
fail(
workload,
"deployment-id",
`${workload}: PRIMARY deployment is ${deployment.id ?? "unknown"}, expected ${want.deploymentId}`,
);
} else if (deployment.rolloutState === "FAILED") {
fail(
workload,
"rollout-failed",
`${workload}: PRIMARY rollout is FAILED (failedTasks=${deployment.failedTasks ?? 0} — an ECS circuit-breaker abort)`,
);
} else if (state.desiredCount !== want.desiredCount) {
fail(
workload,
"desired-count",
`${workload}: ${state.desiredCount ?? 0} desired, expected ${want.desiredCount}`,
);
} else if ((deployment.failedTasks ?? 0) > 0 && running < want.desiredCount) {
failingNow.add(workload);
const tracked = failing.get(workload);
if (!tracked || running > tracked.baseRunning) failing.set(workload, { polls: 1, baseRunning: running });
else tracked.polls += 1;
const streak = failing.get(workload)!;
if (streak.polls >= 4) {
fail(
workload,
"failed-tasks",
`${workload}: PRIMARY deployment keeps failing tasks with no replacement starting (failedTasks=${deployment.failedTasks}, ${running}/${want.desiredCount} running across ${streak.polls} polls)`,
);
} else {
waiting.push(
`${workload} (${running}/${want.desiredCount}, ${deployment.failedTasks} failed; awaiting replacement)`,
);
}
} else if (running < want.desiredCount) {
waiting.push(`${workload} (${running}/${want.desiredCount})`);
} else if (state.deploymentConfiguration?.strategy === "BLUE_GREEN") {
const nativeStatus = nativeStatuses.get(workload);
if (nativeStatus === "SUCCESSFUL") continue;
if (nativeStatus && nativeStatus !== "PENDING" && nativeStatus !== "IN_PROGRESS") {
fail(workload, "native-blue-green", `${workload}: native blue/green deployment is ${nativeStatus}`);
} else {
waiting.push(`${workload} (blue/green traffic shift or bake is still in progress)`);
}
} else if (
deployment.rolloutState !== "COMPLETED" ||
(state.deployments ?? []).some((item) => item !== deployment && (item.runningCount ?? 0) > 0)
) {
waiting.push(`${workload} (prior tasks are still stopping)`);
}
}
for (const workload of failing.keys()) {
if (!failingNow.has(workload)) failing.delete(workload);
}
if (failures.length) {
const next = new Map<string, number>();
const confirmed: string[] = [];
for (const failure of failures) {
const polls = (failurePolls.get(failure.identity) ?? 0) + 1;
next.set(failure.identity, polls);
if (polls >= 2) confirmed.push(failure.message);
}
failurePolls = next;
if (confirmed.length || Date.now() > deadline) {
const fatal = confirmed.length ? confirmed : failures.map((failure) => failure.message);
throw new CliError(
`AWS deployment did not reach the requested state:\n${fatal.map((failure) => ` - ${failure}`).join("\n")}`,
);
}
note(`possible rollout failure; confirming: ${failures.map((failure) => failure.message).join("; ")}`);
await sleep(pollMs);
continue;
}
failurePolls = new Map();
if (waiting.length === 0) return;
if (Date.now() > deadline) {
throw new CliError(`timed out waiting for the AWS rollout: ${waiting.join(", ")}`);
}
note(`waiting on ${waiting.join(", ")}`);
await sleep(pollMs);
}
}
function dynamoString(item: Record<string, { S?: string }> | undefined, name: string): string | undefined {
return item?.[name]?.S;
}
function deploymentManifest(aws: AwsConfig, id: string): DeploymentManifest {
const response = awsJson<{ Item?: Record<string, { S?: string }> }>(aws, [
"dynamodb",
"get-item",
"--table-name",
deployLocksTable(aws),
"--key",
JSON.stringify({ lockKey: { S: deploymentStateKey(aws, `${DEPLOYMENT_MANIFEST_PREFIX}${id}`) } }),
"--consistent-read",
]);
const raw = dynamoString(response.Item, "manifest");
if (!raw) throw new CliError(`AWS deployment manifest ${id} is missing`);
try {
return JSON.parse(raw) as DeploymentManifest;
} catch {
throw new CliError(`AWS deployment manifest ${id} is invalid`);
}
}
function currentDeploymentManifest(aws: AwsConfig): DeploymentManifest | undefined {
return deploymentManifestAtPointer(aws, DEPLOYMENT_POINTER_KEY);
}
function deploymentManifestAtPointer(aws: AwsConfig, key: string): DeploymentManifest | undefined {
const response = awsJson<{ Item?: Record<string, { S?: string }> }>(aws, [
"dynamodb",
"get-item",
"--table-name",
deployLocksTable(aws),
"--key",
JSON.stringify({ lockKey: { S: deploymentStateKey(aws, key) } }),
"--consistent-read",
]);
const id = dynamoString(response.Item, "manifestId");
return id ? deploymentManifest(aws, id) : undefined;
}
function manifestTransaction(aws: AwsConfig, manifest: DeploymentManifest | undefined, pointerId: string): void {
const table = deployLocksTable(aws);
const writes: unknown[] = [];
if (manifest) {
writes.push({
Put: {
TableName: table,
Item: {
lockKey: { S: deploymentStateKey(aws, `${DEPLOYMENT_MANIFEST_PREFIX}${manifest.id}`) },
manifest: { S: JSON.stringify(manifest) },
},
},
});
if (manifest.imageLabel)
writes.push({
Put: {
TableName: table,
Item: {
lockKey: { S: deploymentStateKey(aws, `deployment/label/${manifest.imageLabel}`) },
manifestId: { S: manifest.id },
},
},
});
}
writes.push({
Put: {
TableName: table,
Item: {
lockKey: { S: deploymentStateKey(aws, DEPLOYMENT_POINTER_KEY) },
manifestId: { S: pointerId },
},
},
});
let lastError: unknown;
for (let attempt = 0; attempt < 3; attempt++) {
try {
awsText(aws, ["dynamodb", "transact-write-items", "--transact-items", JSON.stringify(writes)]);
return;
} catch (error) {
lastError = error;
}
}
throw new CliError(
`AWS deployment manifest write failed: ${errMessage(lastError)}; run \`qm check --live\` to establish the current manifest before deploying again`,
);
}
function deploymentManifestForTarget(aws: AwsConfig, target: string): DeploymentManifest {
const labeled = deploymentManifestAtPointer(aws, `deployment/label/${target}`);
return labeled ?? deploymentManifest(aws, target);
}
function recordDeploymentManifest(
aws: AwsConfig,
tasks: Record<string, string>,
release: {
id?: string;
imageLabel?: string;
dbRestorePoint?: string;
layer?: DeploymentManifest["layer"];
imageProvenance?: DeploymentManifest["imageProvenance"];
counts?: DeploymentManifest["counts"];
backgroundDeploymentId?: string;
},
): DeploymentManifest {
const current = currentDeploymentManifest(aws);
const counts = release.counts ?? current?.counts;
const manifest: DeploymentManifest = {
id: release.id ?? randomUUID(),
...((release.backgroundDeploymentId ?? current?.backgroundDeploymentId)
? { backgroundDeploymentId: release.backgroundDeploymentId ?? current?.backgroundDeploymentId }
: {}),
...(current ? { previous: current.id } : {}),
createdAt: new Date().toISOString(),
...(release.imageLabel ? { imageLabel: release.imageLabel } : {}),
...(release.dbRestorePoint ? { dbRestorePoint: release.dbRestorePoint } : {}),
tasks,
...(counts ? { counts } : {}),
...(release.imageProvenance ? { imageProvenance: release.imageProvenance } : {}),
...(release.layer ? { layer: release.layer } : {}),
};
manifestTransaction(aws, manifest, manifest.id);
return manifest;
}
function putDeploymentLayerArtifact(config: QmConfig, body: string): NonNullable<DeploymentManifest["layer"]> {
if (Buffer.byteLength(body) > 1_000_000)
throw new CliError("deployment layer exceeds the core API's 1 MB request limit");
const aws = requireAws(config);
const sha256 = createHash("sha256").update(body).digest("hex");
const key = `deployment/layers/${sha256}.json`;
const dir = mkdtempSync(join(tmpdir(), "qm-layer-"));
const file = join(dir, "layer.json");
try {
writeFileSync(file, body);
awsText(aws, ["s3api", "put-object", "--bucket", awsObjectStoreBucket(config), "--key", key, "--body", file]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
return { key, sha256 };
}
function getDeploymentLayerArtifact(config: QmConfig, layer: DeploymentManifest["layer"]): string {
if (!layer) throw new CliError("target AWS deployment manifest has no restorable deployment layer");
const aws = requireAws(config);
const dir = mkdtempSync(join(tmpdir(), "qm-layer-"));
const file = join(dir, "layer.json");
try {
awsText(aws, ["s3api", "get-object", "--bucket", awsObjectStoreBucket(config), "--key", layer.key, file]);
const body = readFileSync(file, "utf8");
if (Buffer.byteLength(body) > 1_000_000 || createHash("sha256").update(body).digest("hex") !== layer.sha256) {
throw new CliError("AWS deployment-layer artifact is invalid or does not match its manifest");
}
return body;
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
function assertAwsLayerApplied(
result: DeploymentLayerSyncResult | undefined,
expected: NonNullable<DeploymentManifest["layer"]> | string,
): void {
const expectedHash = typeof expected === "string" ? expected : expected.sha256;
if (!result || result.status === "degraded" || result.durable !== true || result.contentHash !== expectedHash) {
throw new CliError("AWS deployment layer was not durably applied with the expected content hash");
}
}
async function retryLiveProbe<T>(probe: () => Promise<T>): Promise<T> {
const deadline = Date.now() + envNum("QM_AWS_LIVE_PROBE_DEADLINE_MS", 2 * 60_000);
for (;;) {
try {
return await probe();
} catch (error) {
if (Date.now() >= deadline) throw error;
note(`live probe failed (${errMessage(error)}); retrying while old tasks drain`);
await sleep(envNum("QM_AWS_LIVE_PROBE_POLL_MS", 5_000));
}
}
}
async function syncAwsLayerAfterRoll(
args: Parameters<typeof syncDeploymentLayerBody>[0],
body: string,
expected: NonNullable<DeploymentManifest["layer"]> | string,
): Promise<void> {
await retryLiveProbe(async () => {
assertAwsLayerApplied(await syncDeploymentLayerBody(args, body), expected);
});
}
function throwAfterCompensation(error: unknown, failures: string[]): never {
if (failures.length) throw new CliError(`${errMessage(error)}; compensation also failed (${failures.join("; ")})`);
throw error;
}
function workloadDesiredCount(config: QmConfig, workload: string): number {
return requireAws(config).services[workload]?.desiredCount ?? 1;
}
function serviceSnapshot(
config: QmConfig,
workloads: string[],
): {
tasks: Record<string, string>;
counts: Record<string, number>;
} {
return serviceSnapshotFromStates(describedServices(config, workloads), workloads);
}
function serviceSnapshotFromStates(
states: ReadonlyMap<string, EcsServiceState>,
workloads: string[],
): ReturnType<typeof serviceSnapshot> {
const tasks: Record<string, string> = {};
const counts: Record<string, number> = {};
for (const workload of workloads) {
const state = states.get(workload)!;
if (!state.taskDefinition) throw new CliError(`${workload}: ECS service has no task definition`);
tasks[workload] = state.taskDefinition;
counts[workload] = state.desiredCount ?? 0;
}
return { tasks, counts };
}
function trustedDeploymentBaseline(
config: QmConfig,
snapshot: ReturnType<typeof serviceSnapshot>,
workloads: string[],
action = "deploy --only",
): DeploymentManifest {
const aws = requireAws(config);
const current = currentDeploymentManifest(aws);
if (!current || workloads.some((workload) => !current.tasks[workload])) {
throw new CliError(
"the first AWS deployment must include every workload; omit --only until a complete deployment manifest exists",
);
}
for (const workload of workloads) {
if (
snapshot.tasks[workload] !== current.tasks[workload] ||
snapshot.counts[workload] !== workloadDesiredCount(config, workload)
) {
throw new CliError(
`cannot ${action} while untouched workload ${workload} differs from the current deployment manifest`,
);
}
const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
current.tasks[workload]!,
]).taskDefinition;
const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
(item) => item.name === workload,
);
if (
!container ||
typeof container.image !== "string" ||
!isPinnedWorkloadImage(config, workload, container.image)
) {
throw new CliError(`cannot ${action} while untouched workload ${workload} lacks a trusted digest-pinned image`);
}
}
return current;
}
function adoptMissingWorkloads(
manifest: DeploymentManifest,
snapshot: ReturnType<typeof serviceSnapshot>,
workloads: string[],
): boolean {
const missing = workloads.filter((workload) => !manifest.tasks[workload]);
const active = missing.filter((workload) => snapshot.counts[workload] !== 0);
if (active.length) {
throw new CliError(
`selected workload ${active.join(", ")} is missing from the current deployment manifest and must be scaled to zero before adoption`,
);
}
if (!missing.length) return false;
manifest.counts ??= {};
for (const workload of missing) {
manifest.tasks[workload] = snapshot.tasks[workload]!;
manifest.counts[workload] = snapshot.counts[workload]!;
}
return true;
}
function unchangedWebRouting(
config: QmConfig,
manifest: DeploymentManifest | undefined,
desired: Array<{ service: string; task: EcsTaskDefinition }>,
): boolean {
if (!manifest) return false;
const workloads = ["web-ui", "portal"];
if (workloads.some((workload) => !desired.some((item) => item.service === workload))) return false;
try {
const states = describedServices(config, workloads);
for (const workload of workloads) {
const count = manifest.counts?.[workload];
const definition = manifest.tasks[workload];
if (!definition || !count || count !== workloadDesiredCount(config, workload)) return false;
stableWorkloadCapacity(config, workload, states.get(workload)!, definition, count);
const prior = awsJson<{ taskDefinition?: Record<string, unknown> }>(requireAws(config), [
"ecs",
"describe-task-definition",
"--task-definition",
definition,
]).taskDefinition;
const next = desired.find((item) => item.service === workload)!.task;
const keys = workload === "portal" ? ["WEB_UI_UPSTREAM", "ADMIN_UPSTREAM"] : ["ADMIN_ENABLED"];
const settings = (task: Record<string, unknown> | undefined): string[] | undefined => {
const containers = task?.containerDefinitions as Array<Record<string, unknown>> | undefined;
const matches = containers?.filter((container) => container.name === workload);
if (matches?.length !== 1) return undefined;
const container = matches[0]!;
const secrets = (container.secrets ?? []) as Array<{ name: string }>;
const environment = (container.environment ?? []) as Array<{ name: string; value: string }>;
const values: string[] = [];
for (const key of keys) {
const entries = environment.filter((entry) => entry.name === key);
if (secrets.some((entry) => entry.name === key) || entries.length !== 1 || !entries[0]!.value)
return undefined;
values.push(entries[0]!.value);
}
return values;
};
const previous = settings(prior);
const upcoming = settings(next as unknown as Record<string, unknown>);
if (!previous || !upcoming || canonicalJson(previous) !== canonicalJson(upcoming)) return false;
}
return true;
} catch {
return false;
}
}
async function applyServiceTargets(
config: QmConfig,
targets: Record<string, string>,
desiredCounts?: Record<string, number>,
options: {
timeoutMs?: number;
webBeforePortal?: boolean;
onSubmitted?: () => void;
} = {},
): Promise<void> {
const aws = requireAws(config);
const workloads = Object.keys(targets);
const states = describedServices(config, workloads);
assertOwnedServices(config, states, workloads);
const before = serviceSnapshotFromStates(states, workloads);
const expectedCounts = Object.fromEntries(
workloads.map((workload) => [workload, desiredCounts?.[workload] ?? before.counts[workload]!]),
);
const changed: string[] = [];
try {
for (const workload of options.webBeforePortal
? [...workloads].sort(
(a, b) => Number(a === "portal") - Number(b === "portal") || Number(b === "web-ui") - Number(a === "web-ui"),
)
: workloads) {
if (options.webBeforePortal && workload === "portal" && targets["web-ui"]) {
await awaitServiceTargets(
config,
{
"web-ui": {
taskDefinition: targets["web-ui"]!,
desiredCount: expectedCounts["web-ui"]!,
},
},
options.timeoutMs,
);
}
const args = [
"ecs",
"update-service",
"--cluster",
aws.cluster,
"--service",
aws.services[workload]!.ecsService,
"--task-definition",
targets[workload]!,
"--desired-count",
String(expectedCounts[workload]),
];
changed.push(workload);
awsText(aws, args);
}
options.onSubmitted?.();
await awaitServiceTargets(
config,
Object.fromEntries(
workloads.map((workload) => [
workload,
{
taskDefinition: targets[workload]!,
desiredCount: expectedCounts[workload]!,
},
]),
),
options.timeoutMs,
);
} catch (error) {
const restoreFailures: string[] = [];
for (const workload of [...changed].reverse()) {
try {
awsText(aws, [
"ecs",
"update-service",
"--cluster",
aws.cluster,
"--service",
aws.services[workload]!.ecsService,
"--task-definition",
before.tasks[workload]!,
"--desired-count",
String(before.counts[workload]),
]);
} catch (restoreError) {
restoreFailures.push(`${workload}: ${errMessage(restoreError)}`);
}
}
if (changed.length && restoreFailures.length === 0) {
try {
await awaitServiceTargets(
config,
Object.fromEntries(
changed.map((workload) => [
workload,
{
taskDefinition: before.tasks[workload]!,
desiredCount: before.counts[workload]!,
},
]),
),
options.timeoutMs,
);
} catch (restoreError) {
restoreFailures.push(errMessage(restoreError));
}
}
const detail = restoreFailures.length
? `; restoring the prior service set also failed (${restoreFailures.join("; ")})`
: "";
throw new CliError(`${errMessage(error)}${detail}`);
}
}
function reportTaskChanges(
config: QmConfig,
services: string[],
images: Record<string, string>,
arns: Record<string, string>,
restart: ReadonlySet<string> = new Set(),
backgroundDeploymentId?: string,
): Array<{ service: string; task: EcsTaskDefinition; changed: boolean }> {
const desired = services.map((service) => ({
service,
task: renderTaskDefinition(config, service, images[service]!, arns, backgroundDeploymentId),
live: liveTask(config, service),
}));
return desired.map((item) => {
const changes = taskDefinitionChanges(item.task, item.live);
step(
`${item.service}: ${changes.length ? `${changes.length} task-definition change${changes.length === 1 ? "" : "s"}` : "no task-definition change"}`,
);
if (changes.length) note(JSON.stringify({ service: item.service, changes }, null, 2));
if (restart.has(item.service)) step(`${item.service}: restart requested with the deployment`);
return { service: item.service, task: item.task, changed: changes.length > 0 || restart.has(item.service) };
});
}
function promoteStagedImage(config: QmConfig, service: string, image: string, label: string): void {
const aws = requireAws(config);
const repository = aws.services[service]!.ecrRepository;
const digest = image.split("@")[1];
if (!digest) throw new CliError(`staged image for ${service} is not digest-pinned`);
const staged = awsJson<{ images?: Array<{ imageManifest?: string; imageManifestMediaType?: string }> }>(aws, [
"ecr",
"batch-get-image",
"--repository-name",
repository,
"--image-ids",
`imageDigest=${digest}`,
]).images?.[0];
if (!staged?.imageManifest) throw new CliError(`ECR did not return the staged manifest for ${service}@${digest}`);
try {
awsText(aws, [
"ecr",
"put-image",
"--repository-name",
repository,
"--image-tag",
label,
"--image-manifest",
staged.imageManifest,
...(staged.imageManifestMediaType ? ["--image-manifest-media-type", staged.imageManifestMediaType] : []),
]);
} catch (error) {
if (!/ImageAlreadyExistsException/.test(errMessage(error))) throw error;
}
}
function dbSnapshotRestoreHint(aws: AwsConfig, snapshot: string): string {
return `restore: aws rds restore-db-instance-from-db-snapshot --db-snapshot-identifier ${snapshot} --db-instance-identifier ${rdsInstanceIdentifier(aws)}-restored --region ${aws.region}, then repoint the stack at the restored instance`;
}
function dbRestorePointHint(aws: AwsConfig, restorePoint: string): string {
const database = rdsInstanceIdentifier(aws);
return `restore: aws rds restore-db-instance-to-point-in-time --source-db-instance-identifier ${database} --target-db-instance-identifier ${database}-restored --restore-time ${restorePoint} --region ${aws.region}, then repoint the stack at the restored instance; valid only while ${restorePoint} is inside the instance's automated-backup retention window`;
}
function manifestDbRestoreHint(aws: AwsConfig, manifest: DeploymentManifest): string | undefined {
if (manifest.dbRestorePoint) return dbRestorePointHint(aws, manifest.dbRestorePoint);
if (manifest.dbSnapshot) return dbSnapshotRestoreHint(aws, manifest.dbSnapshot);
return undefined;
}
function assertPredeployDbRestorePoint(config: QmConfig): string {
const aws = requireAws(config);
const database = rdsInstanceIdentifier(aws);
const instance = awsJson<{
DBInstances?: Array<{ DBInstanceStatus?: string; BackupRetentionPeriod?: number; LatestRestorableTime?: string }>;
}>(aws, ["rds", "describe-db-instances", "--db-instance-identifier", database]).DBInstances?.[0];
if (instance?.DBInstanceStatus !== "available" && instance?.DBInstanceStatus !== "storage-optimization") {
throw new CliError(
`database ${database} is ${instance?.DBInstanceStatus ?? "missing"}; refusing to deploy without an available source for the pre-deploy restore point`,
);
}
const retention = instance.BackupRetentionPeriod ?? 0;
const minimumRetention = aws.dbRetentionMinDays ?? 1;
if (retention < minimumRetention) {
throw new CliError(
`database ${database} keeps ${retention} day(s) of automated backups, below the required ${minimumRetention}; raise its backup retention (db_backup_retention_days in the reference module) or lower aws.dbRetentionMinDays before deploying`,
);
}
const latestRestorable = Date.parse(instance.LatestRestorableTime ?? "");
if (!Number.isFinite(latestRestorable)) {
throw new CliError(
`database ${database} reports no LatestRestorableTime; point-in-time recovery is not active, refusing to deploy without a restore point`,
);
}
const lagMs = Date.now() - latestRestorable;
const maxLagMs = envNum("QM_AWS_DB_MAX_RESTORE_LAG_MS", 10 * 60_000);
if (lagMs > maxLagMs) {
throw new CliError(
`database ${database} is restorable only up to ${instance.LatestRestorableTime} (${Math.round(lagMs / 1000)}s ago, above the allowed ${Math.round(maxLagMs / 1000)}s); its continuous backup stream is behind, refusing to deploy without a fresh restore point`,
);
}
const restorePoint = new Date().toISOString();
ok(
`pre-deploy database restore point: ${restorePoint} (continuous backups current to ${Math.round(lagMs / 1000)}s ago; ${dbRestorePointHint(aws, restorePoint)})`,
);
return restorePoint;
}
function confirmDbRestorePointCovered(config: QmConfig, restorePoint: string): void {
const aws = requireAws(config);
const database = rdsInstanceIdentifier(aws);
let latestRestorable: string | undefined;
try {
latestRestorable = awsJson<{ DBInstances?: Array<{ LatestRestorableTime?: string }> }>(aws, [
"rds",
"describe-db-instances",
"--db-instance-identifier",
database,
]).DBInstances?.[0]?.LatestRestorableTime;
} catch (error) {
warn(`could not confirm the pre-deploy restore point ${restorePoint} is restorable yet: ${errMessage(error)}`);
return;
}
const latest = Date.parse(latestRestorable ?? "");
if (Number.isFinite(latest) && latest >= Date.parse(restorePoint)) {
ok(`pre-deploy database restore point ${restorePoint} is covered by continuous backups`);
} else if (Date.now() - Date.parse(restorePoint) <= envNum("QM_AWS_DB_MAX_RESTORE_LAG_MS", 10 * 60_000)) {
note(
`pre-deploy database restore point ${restorePoint} is not covered by continuous backups yet (database ${database} is restorable up to ${latestRestorable ?? "an unknown time"}); the stream catches up within minutes — leave the instance intact until it does`,
);
} else {
warn(
`pre-deploy database restore point ${restorePoint} is still not restorable: database ${database} is restorable only up to ${latestRestorable ?? "an unknown time"}; if the instance is lost before its backup stream passes the restore point, that point is unrecoverable — leave the instance intact until this catches up`,
);
}
}
function deploymentProgress(
config: QmConfig,
opts: AwsUpOpts,
): ((targets: Record<string, string>) => void) | undefined {
const file = process.env.QM_DEPLOY_PROGRESS_FILE;
const token = process.env.QM_DEPLOY_PROGRESS_TOKEN;
if (file === undefined && token === undefined) return undefined;
if (!file || !token || !isAbsolute(file) || !opts.candidate || !opts.yes || opts.dryRun || opts.buildOnly) {
throw new CliError(
"deployment progress requires an absolute QM_DEPLOY_PROGRESS_FILE, QM_DEPLOY_PROGRESS_TOKEN, and candidate up --yes",
);
}
if (existsSync(file)) throw new CliError("deployment progress file must not already exist");
if (!statSync(dirname(file)).isDirectory()) throw new CliError("deployment progress parent must be a directory");
accessSync(dirname(file), constants.W_OK);
return (targets) => {
const temporary = `${file}.${randomUUID()}.tmp`;
try {
writeFileSync(temporary, JSON.stringify({ phase: "monitoring", token, orgId: config.orgId, targets }), {
flag: "wx",
mode: 0o600,
});
linkSync(temporary, file);
} catch (error) {
warn(`could not publish deployment progress: ${errMessage(error)}`);
} finally {
try {
rmSync(temporary, { force: true });
} catch (error) {
warn(`could not remove deployment progress temporary file: ${errMessage(error)}`);
}
}
};
}
export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpts = {}): Promise<void> {
const reportProgress = deploymentProgress(config, opts);
const topology = awsTopology(config, _configDir);
const { aws } = topology;
if (new URL(config.publicUrl).protocol !== "https:") {
throw new CliError(
"AWS deploy requires an HTTPS publicUrl; configure an ACM certificate, update publicUrl, and rerender/apply Terraform before running `qm up`",
);
}
const plugins = new Map(topology.plugins.map((plugin) => [plugin.name, plugin]));
const services = opts.only ? [...new Set(opts.only.map(serviceHost))] : topology.workloads;
const buildConcurrency = opts.buildConcurrency ?? 1;
if (!Number.isSafeInteger(buildConcurrency) || buildConcurrency < 1)
throw new CliError("--build-concurrency requires a positive integer");
if (opts.buildConcurrency !== undefined && !opts.buildOnly)
throw new CliError("--build-concurrency requires --build-only");
if (
buildConcurrency > 1 &&
new Set(services.map((service) => aws.services[service]?.ecrRepository)).size !== services.length
)
throw new CliError("--build-concurrency requires distinct ECR repositories for selected workloads");
const restart = new Set((opts.restart ?? []).map(serviceHost));
if (restart.size && opts.buildOnly) throw new CliError("--restart cannot be used with --build-only");
for (const service of restart) {
if (!services.includes(service)) throw new CliError(`--restart workload ${service} is not selected for deployment`);
}
for (const service of services) workloadArchitecture(config, service);
if (opts.buildOnly && opts.candidate) throw new CliError("--build-only and --candidate are mutually exclusive");
if (opts.candidate && opts.buildFrom) throw new CliError("--candidate and --build-from are mutually exclusive");
if (opts.inactive && (opts.dryRun || !opts.candidate)) {
throw new CliError("--inactive is only valid for an exact candidate deployment");
}
if (opts.buildOnly && !opts.imageLabel) throw new CliError("--build-only requires --image-label <label>");
if (opts.buildOnly && !opts.candidateOut) throw new CliError("--build-only requires --candidate-out <path>");
if (opts.imageLabel && !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(opts.imageLabel)) {
throw new CliError("--image-label must be a valid ECR tag");
}
if (opts.buildFrom) resolveBuildRepoRoot(opts.buildFromPath, services.filter(isServiceName));
if (opts.buildOnly) {
assertAwsCallerAccount(aws);
header(`qm ${opts.dryRun ? "plan" : "build"} — ${config.orgId} (aws candidate ${opts.imageLabel})`);
if (opts.dryRun) {
for (const service of services) step(`${service}: build immutable candidate image`);
note(`Plan only. Candidate manifest would be written to ${opts.candidateOut}.`);
return;
}
dockerLogin(aws);
const images: Record<string, string> = {};
const imageProvenance: Record<string, DeploymentImageProvenance> = {};
for (const service of services) {
imageProvenance[service] = workloadImageProvenance(config, service, plugins.get(service), opts);
}
const controller = new AbortController();
const cancelBuilds = () => controller.abort();
process.on("SIGINT", cancelBuilds);
process.on("SIGTERM", cancelBuilds);
try {
for (let offset = 0; offset < services.length; offset += buildConcurrency) {
await settleAll(
services.slice(offset, offset + buildConcurrency).map(async (service) => {
images[service] = await publishWorkloadImage(
config,
service,
plugins.get(service),
opts.imageLabel!,
opts,
controller.signal,
);
}),
);
}
} finally {
process.off("SIGINT", cancelBuilds);
process.off("SIGTERM", cancelBuilds);
}
const release: AwsReleaseCandidate = {
contract: 1,
accountId: aws.accountId,
region: aws.region,
label: opts.imageLabel!,
images,
imageProvenance,
architectures: Object.fromEntries(services.map((service) => [service, workloadArchitecture(config, service)])),
};
writeFileSync(opts.candidateOut!, `${JSON.stringify(release, null, 2)}\n`, { mode: 0o600 });
ok(`AWS release candidate written to ${opts.candidateOut}`);
return;
}
const candidate = opts.candidate ? releaseCandidate(config, opts.candidate) : undefined;
for (const service of services) {
if (candidate && (!candidate.images[service] || !candidate.imageProvenance[service])) {
throw new CliError(`AWS release candidate ${opts.candidate} does not contain ${service}`);
}
}
if (opts.imageLabel && opts.imageLabel !== aws.imageLabel) {
throw new CliError(
`--image-label ${opts.imageLabel} differs from durable aws.imageLabel ${aws.imageLabel}; update and commit the deployment directory first`,
);
}
const label = aws.imageLabel;
if (!label) throw new CliError("aws.imageLabel is required");
if (!opts.dryRun && !opts.yes) throw new CliError("AWS deploy requires --yes after reviewing `qm plan`");
assertAwsCallerAccount(aws);
assertAwsPublicFrontDoor(config);
if (!opts.dryRun && !opts.inactive) await assertAwsPublicNetwork(config);
assertAwsPublicApiUrl(config);
assertAwsDeployImage(config);
header(`qm ${opts.dryRun ? "plan" : "up"} — ${config.orgId} (aws)`);
const allServices = topology.workloads;
assertOwnedServices(config, describedServices(config, allServices), allServices);
const arns = secretArns(config);
if (opts.dryRun) {
step(
aws.predeployDbSnapshot === false
? "pre-deploy database restore point: disabled (aws.predeployDbSnapshot)"
: `pre-deploy database restore point: verify ${rdsInstanceIdentifier(aws)} point-in-time recovery is current before the first mutation`,
);
const before = serviceSnapshot(config, allServices);
const selected = new Set(services);
if (allServices.some((service) => !selected.has(service))) {
const baseline = trustedDeploymentBaseline(
config,
before,
allServices.filter((name) => !selected.has(name)),
);
adoptMissingWorkloads(baseline, before, services);
}
const images: Record<string, string> = {};
for (const service of services) {
if (candidate) {
images[service] = candidate.images[service]!;
step(`${service}: immutable candidate ${candidate.label}`);
continue;
}
const spec = aws.services[service]!;
const sourceBuild = (opts.buildFrom && isServiceName(service)) || plugins.get(service)?.kind === "source";
if (sourceBuild) {
images[service] = `${ecrHost(aws)}/${spec.ecrRepository}@sha256:${"0".repeat(64)}`;
step(`${service}: source build planned; image digest is unresolved until build`);
continue;
}
images[service] = plannedWorkloadImage(config, service, plugins.get(service));
}
const backgroundDeploymentId = aws.backgroundWorkControl
? (currentDeploymentManifest(aws)?.backgroundDeploymentId ??
`${aws.services.core!.ecsService}:00000000-0000-0000-0000-000000000000`)
: undefined;
reportTaskChanges(config, services, images, arns, restart, backgroundDeploymentId);
for (const service of services) {
step(`${service}: desired count ${before.counts[service] ?? 0} → ${workloadDesiredCount(config, service)}`);
}
if (opts.sandboxDir && existsSync(opts.sandboxDir) && (!opts.only || opts.only.includes("core"))) {
const hash = createHash("sha256").update(deploymentLayerBody(opts.sandboxDir)).digest("hex");
try {
const current = currentDeploymentManifest(aws);
step(`deployment layer: ${current?.layer?.sha256 === hash ? "unchanged" : "changed"} (${hash.slice(0, 12)})`);
} catch {
step(`deployment layer: desired ${hash.slice(0, 12)} (live state unavailable)`);
}
} else {
step("deployment layer: preserved (no sandbox directory selected for core)");
}
note(
`Plan only. Re-run \`qm up${opts.candidate ? ` --candidate ${opts.candidate}` : ""}${restart.size ? ` --restart ${[...restart].join(",")}` : ""} --yes\` to deploy.`,
);
return;
}
const lease = acquireLease(aws);
const releaseId = randomUUID();
const stagingLabel = "qm-staging";
const staged = new Set<string>();
const promotedServices = new Set<string>();
let before: ReturnType<typeof serviceSnapshot> | undefined;
let applied = false;
let current: DeploymentManifest | undefined;
let recorded: DeploymentManifest | undefined;
let releaseSucceeded = false;
let previousLayerBody: string | undefined;
let previousLayerBootstrapped = false;
let desiredLayerBody: string | undefined;
let desiredLayer: DeploymentManifest["layer"];
let layerChanged = false;
let layerAttempted = false;
let dbRestorePoint: string | undefined;
let migratedCandidate = false;
try {
assertAwsDeployImage(config);
before = serviceSnapshot(config, allServices);
current = currentDeploymentManifest(aws);
if (current?.backgroundDeploymentId && !aws.backgroundWorkControl)
throw new CliError("cannot disable background ownership control while a controlled deployment is recorded");
if (aws.backgroundWorkControl) {
if (!current?.backgroundDeploymentId && !services.includes("core"))
throw new CliError("enabling background ownership requires deploying core to allocate its identity");
reconcileBackgroundPreparation(aws, current, before);
if (services.includes("core")) await assertNotBackgroundOwner(config, current);
}
const selected = new Set(services);
if (allServices.some((service) => !selected.has(service))) {
current = trustedDeploymentBaseline(
config,
before,
allServices.filter((name) => !selected.has(name)),
);
if (adoptMissingWorkloads(current, before, services)) manifestTransaction(aws, current, current.id);
}
if (aws.predeployDbSnapshot === false)
note("pre-deploy database restore point: disabled (aws.predeployDbSnapshot)");
else dbRestorePoint = assertPredeployDbRestorePoint(config);
if (current?.layer) {
desiredLayer = current.layer;
} else {
if (current || before.counts.core !== 0) {
const previousState = await currentDeploymentLayerState({
config,
transport: awsDeploymentLayerTransport,
configDir: _configDir,
...(opts.envFile ? { envFile: opts.envFile } : {}),
});
previousLayerBody = previousState.body;
previousLayerBootstrapped = previousState.bootstrapped;
} else {
previousLayerBody = JSON.stringify({ contract: 1, tools: [], skills: [] });
previousLayerBootstrapped = true;
}
if (!previousLayerBootstrapped) {
desiredLayer = putDeploymentLayerArtifact(config, previousLayerBody);
if (current) {
current.layer = desiredLayer;
manifestTransaction(aws, current, current.id);
}
}
}
if (opts.sandboxDir && existsSync(opts.sandboxDir) && (!opts.only || opts.only.includes("core"))) {
previousLayerBody ??= getDeploymentLayerArtifact(config, desiredLayer);
desiredLayerBody = deploymentLayerBody(opts.sandboxDir);
desiredLayer = putDeploymentLayerArtifact(config, desiredLayerBody);
layerChanged = desiredLayer.sha256 !== current?.layer?.sha256;
if (!layerChanged && before.counts.core !== 0) {
const state = await currentDeploymentLayerState({
config,
transport: awsDeploymentLayerTransport,
configDir: _configDir,
...(opts.envFile ? { envFile: opts.envFile } : {}),
});
if (
state.status === "applied" &&
state.contentHash === desiredLayer.sha256 &&
state.runtimeContentHash === desiredLayer.sha256
) {
desiredLayerBody = undefined;
}
}
}
if (!candidate) dockerLogin(aws);
const images: Record<string, string> = {};
const selectedImageProvenance: Record<string, DeploymentImageProvenance> = {};
for (const service of services) {
if (candidate) {
selectedImageProvenance[service] = candidate.imageProvenance[service]!;
images[service] = candidate.images[service]!;
} else {
staged.add(service);
selectedImageProvenance[service] = workloadImageProvenance(config, service, plugins.get(service), opts);
images[service] = await publishWorkloadImage(config, service, plugins.get(service), stagingLabel, opts);
}
}
let backgroundDeploymentId = current?.backgroundDeploymentId;
if (aws.backgroundWorkControl && !backgroundDeploymentId)
backgroundDeploymentId = `${aws.services.core!.ecsService}:${releaseId}`;
const desired = reportTaskChanges(config, services, images, arns, restart, backgroundDeploymentId);
const changedCore = desired.find((item) => item.service === "core" && item.changed);
if (aws.backgroundWorkControl && changedCore) {
backgroundDeploymentId = `${aws.services.core!.ecsService}:${releaseId}`;
changedCore.task = renderTaskDefinition(config, "core", images.core!, arns, backgroundDeploymentId);
prepareBackgroundDeployment(aws, {
id: releaseId,
previousManifestId: current?.id,
backgroundDeploymentId,
before,
});
}
const targets: Record<string, string> = {};
for (const item of desired) {
if (!item.changed) {
targets[item.service] = before.tasks[item.service]!;
continue;
}
const file = join(mkdtempSync(join(tmpdir(), "qm-task-")), `${item.service}.json`);
writeFileSync(file, JSON.stringify(item.task));
const taskDefinition = registerTaskDefinition(config, file);
targets[item.service] = taskDefinition;
}
if (candidate?.images.core && services.includes("core")) {
await runAwsCandidateMigration(config, candidate.images.core, candidate.label, targets.core!);
migratedCandidate = true;
}
const rolloutTargets = Object.fromEntries(
services
.filter(
(service) =>
desired.find((item) => item.service === service)!.changed ||
before!.counts[service] !== workloadDesiredCount(config, service),
)
.map((service) => [service, targets[service]!]),
);
if (Object.keys(rolloutTargets).length) {
await applyServiceTargets(
config,
rolloutTargets,
Object.fromEntries(
Object.keys(rolloutTargets).map((service) => [service, workloadDesiredCount(config, service)]),
),
{
webBeforePortal: !unchangedWebRouting(config, current, desired),
onSubmitted: reportProgress ? () => reportProgress(targets) : undefined,
},
);
applied = true;
} else {
reportProgress?.(targets);
}
await awaitServiceTargets(
config,
Object.fromEntries(
services.map((service) => [
service,
{ taskDefinition: targets[service]!, desiredCount: workloadDesiredCount(config, service) },
]),
),
);
if (desiredLayerBody) {
layerAttempted = true;
await syncAwsLayerAfterRoll(
{
config,
transport: awsDeploymentLayerTransport,
configDir: _configDir,
...(opts.envFile ? { envFile: opts.envFile } : {}),
},
desiredLayerBody,
desiredLayer!,
);
}
const releaseTasks = { ...before.tasks, ...targets };
const releaseCounts = Object.fromEntries(
allServices.map((service) => [service, workloadDesiredCount(config, service)]),
);
const releaseImageProvenance = { ...current?.imageProvenance, ...selectedImageProvenance };
const sameTasks = current && allServices.every((service) => current!.tasks[service] === releaseTasks[service]);
const sameCounts = current && canonicalJson(current.counts ?? {}) === canonicalJson(releaseCounts);
const sameImageProvenance =
current && canonicalJson(current.imageProvenance ?? {}) === canonicalJson(releaseImageProvenance);
releaseSucceeded = true;
if (
!current ||
!sameTasks ||
!sameCounts ||
!sameImageProvenance ||
current.imageLabel !== label ||
layerChanged ||
migratedCandidate
) {
recorded = recordDeploymentManifest(aws, releaseTasks, {
id: releaseId,
...(backgroundDeploymentId ? { backgroundDeploymentId } : {}),
counts: releaseCounts,
imageLabel: label,
...(dbRestorePoint ? { dbRestorePoint } : {}),
...(desiredLayer ? { layer: desiredLayer } : {}),
imageProvenance: releaseImageProvenance,
});
}
if (aws.backgroundWorkControl && changedCore) clearBackgroundPreparation(aws);
if (recorded && dbRestorePoint) confirmDbRestorePointCovered(config, dbRestorePoint);
for (const service of candidate ? [] : services) {
try {
promoteStagedImage(config, service, images[service]!, label);
promotedServices.add(service);
} catch (error) {
warn(
`AWS deployment succeeded, but could not promote ${service}:${label}: ${errMessage(error)}; preserving staging tag ${stagingLabel} so the deployed digest remains pullable; rerun \`qm up --yes\` to retry promotion`,
);
}
}
} catch (error) {
if (releaseSucceeded) throw error;
const compensationFailures: string[] = [];
if (layerAttempted && previousLayerBody && !previousLayerBootstrapped) {
try {
await syncAwsLayerAfterRoll(
{
config,
transport: awsDeploymentLayerTransport,
configDir: _configDir,
...(opts.envFile ? { envFile: opts.envFile } : {}),
},
previousLayerBody,
createHash("sha256").update(previousLayerBody).digest("hex"),
);
} catch (restoreError) {
compensationFailures.push(`restoring the pre-deploy deployment layer: ${errMessage(restoreError)}`);
}
} else if (layerAttempted && previousLayerBootstrapped) {
warn(
`the pre-deploy layer snapshot was synthesized (the core had no durable layer record) — leaving the core's layer as the failed sync left it rather than durably installing the empty bundle`,
);
}
if (applied && before) {
try {
await applyServiceTargets(config, before.tasks, before.counts);
} catch (restoreError) {
compensationFailures.push(`restoring the pre-deploy manifest: ${errMessage(restoreError)}`);
}
}
throwAfterCompensation(error, compensationFailures);
} finally {
for (const service of staged) {
if (releaseSucceeded && !promotedServices.has(service)) continue;
try {
awsText(aws, [
"ecr",
"batch-delete-image",
"--repository-name",
aws.services[service]!.ecrRepository,
"--image-ids",
`imageTag=${stagingLabel}`,
]);
} catch (error) {
if (!/ImageNotFoundException/.test(errMessage(error)))
warn(`could not clean staging image ${service}:${stagingLabel}: ${errMessage(error)}`);
}
}
releaseLease(aws, lease);
}
ok(`AWS services stable on ${label} (deployment ${recorded?.id ?? current?.id})`);
}
export function awsStatus(config: QmConfig, configDir = process.cwd()): void {
const { aws, workloads } = awsTopology(config, configDir);
assertAwsCallerAccount(aws);
const result = describedServices(config, workloads);
header(`qm status — ${config.orgId} (aws)`);
for (const workload of workloads) {
const service = result.get(workload)!;
note(`${service.serviceName}: ${service.runningCount}/${service.desiredCount} running · ${service.taskDefinition}`);
}
if (config.services.includes("slack")) note("slack: virtual service running in the core task");
}
export function awsLogs(
config: QmConfig,
service: string | undefined,
opts: LogOpts,
configDir = process.cwd(),
): void | Promise<void> {
const { aws, workloads } = awsTopology(config, configDir);
assertAwsCallerAccount(aws);
if (opts.tail !== undefined)
note(`(--tail is a docker-only line count; aws logs tail has none, so it's ignored on the aws target)`);
const logArgs = (name: string): string[] => {
const spec = aws.services[name];
if (!spec) throw new CliError(`unknown AWS workload ${name}`);
return awsArgs(aws, [
"logs",
"tail",
spec.logGroup ?? `/ecs/${spec.ecsService}`,
...(opts.follow ? ["--follow"] : []),
]);
};
if (service) {
const resolved = serviceHost(service);
if (resolved !== service) note(`${service} runs in ${resolved}; showing ${resolved} logs`);
runInherit(process.env.AWS_BIN ?? "aws", logArgs(resolved));
return;
}
return streamLabeled(
workloads.map((name) => ({ label: name, command: process.env.AWS_BIN ?? "aws", args: logArgs(name) })),
(label, line) => note(`${dim(label)} | ${line}`),
);
}
export async function awsDown(config: QmConfig, configDir = process.cwd()): Promise<void> {
const { aws, workloads } = awsTopology(config, configDir);
assertAwsCallerAccount(aws);
assertOwnedServices(config, describedServices(config, workloads), workloads);
const lease = acquireLease(aws);
try {
const before = serviceSnapshot(config, workloads);
await applyServiceTargets(config, before.tasks, Object.fromEntries(workloads.map((workload) => [workload, 0])));
} finally {
releaseLease(aws, lease);
}
ok(
"AWS services set to zero desired tasks; in-flight turns resume on the next deployment. Infrastructure and data were retained.",
);
}
function firstRolledBackDbRestore(
aws: AwsConfig,
current: DeploymentManifest,
targetId: string,
): { kind: "restore point" | "snapshot"; hint: string } | undefined {
const seen = new Set<string>();
let oldest: { kind: "restore point" | "snapshot"; hint: string } | undefined;
let manifest: DeploymentManifest | undefined = current;
while (manifest && !seen.has(manifest.id)) {
if (manifest.id === targetId) return oldest;
if (manifest.dbRestorePoint) {
oldest = { kind: "restore point", hint: `${manifest.dbRestorePoint} (${manifestDbRestoreHint(aws, manifest)})` };
} else if (manifest.dbSnapshot) {
oldest = { kind: "snapshot", hint: `${manifest.dbSnapshot} (${manifestDbRestoreHint(aws, manifest)})` };
}
seen.add(manifest.id);
if (!manifest.previous) return undefined;
try {
manifest = deploymentManifest(aws, manifest.previous);
} catch {
return undefined;
}
}
return undefined;
}
export async function awsRollback(
config: QmConfig,
to?: string,
layerOpts?: { configDir: string; envFile?: string },
): Promise<void> {
const { aws, workloads: services } = awsTopology(config, layerOpts?.configDir ?? process.cwd());
assertAwsCallerAccount(aws);
const lease = acquireLease(aws);
let before: ReturnType<typeof serviceSnapshot> | undefined;
let applied = false;
let currentLayerBody: string | undefined;
let layerAttempted = false;
let rolledBack = false;
try {
before = serviceSnapshot(config, services);
const currentManifest = currentDeploymentManifest(aws);
let targetManifest: DeploymentManifest | undefined;
if (!to) {
if (!currentManifest?.previous) throw new CliError("no previous recorded AWS deployment to roll back to");
targetManifest = deploymentManifest(aws, currentManifest.previous);
} else {
targetManifest = deploymentManifestForTarget(aws, to);
}
if (currentManifest?.backgroundDeploymentId || targetManifest.backgroundDeploymentId) {
if (!aws.backgroundWorkControl || !targetManifest.backgroundDeploymentId)
throw new CliError("controlled rollback requires a protocol-capable target deployment");
await assertNotBackgroundOwner(config, currentManifest);
}
const missing = services.filter((service) => !targetManifest.tasks[service]);
if (missing.length)
throw new CliError(`target AWS deployment manifest is missing workloads: ${missing.join(", ")}`);
let targetLayerBody: string | undefined;
let layerNeedsSync = false;
if (layerOpts) {
targetLayerBody = getDeploymentLayerArtifact(config, targetManifest.layer);
if (currentManifest?.layer) currentLayerBody = getDeploymentLayerArtifact(config, currentManifest.layer);
else if (currentManifest)
throw new CliError(`current AWS deployment manifest ${currentManifest.id} has no restorable deployment layer`);
layerNeedsSync = currentManifest?.layer?.sha256 !== targetManifest.layer!.sha256;
}
const targets = Object.fromEntries(services.map((service) => [service, targetManifest!.tasks[service]!]));
const targetCounts = Object.fromEntries(
services.map((service) => [service, targetManifest.counts?.[service] ?? before!.counts[service]!]),
);
const changedTargets = Object.fromEntries(
services
.filter(
(service) => before!.tasks[service] !== targets[service] || before!.counts[service] !== targetCounts[service],
)
.map((service) => [service, targets[service]!]),
);
if (Object.keys(changedTargets).length) {
await applyServiceTargets(
config,
changedTargets,
Object.fromEntries(Object.keys(changedTargets).map((service) => [service, targetCounts[service]!])),
);
applied = true;
}
await awaitServiceTargets(
config,
Object.fromEntries(
services.map((service) => [
service,
{ taskDefinition: targets[service]!, desiredCount: targetCounts[service]! },
]),
),
);
if (targetLayerBody && layerOpts && layerNeedsSync) {
if (targetCounts.core === 0) {
note(
"deployment layer sync deferred: core is scaled to zero, so `check --live` will flag the layer until the next `qm up` applies it",
);
} else {
layerAttempted = true;
await syncAwsLayerAfterRoll(
{
config,
transport: awsDeploymentLayerTransport,
configDir: layerOpts.configDir,
...(layerOpts.envFile ? { envFile: layerOpts.envFile } : {}),
},
targetLayerBody,
targetManifest.layer!,
);
}
}
rolledBack = true;
if (currentManifest?.id !== targetManifest.id) {
manifestTransaction(aws, undefined, targetManifest.id);
const dbRestore = currentManifest && firstRolledBackDbRestore(aws, currentManifest, targetManifest.id);
if (dbRestore) {
note(
`rollback restores code and configuration, not data; the database ${dbRestore.kind} taken before the first rolled-back deployment is ${dbRestore.hint}`,
);
}
}
} catch (error) {
if (rolledBack) throw error;
const compensationFailures: string[] = [];
if (layerAttempted && currentLayerBody && layerOpts) {
try {
await syncAwsLayerAfterRoll(
{
config,
transport: awsDeploymentLayerTransport,
configDir: layerOpts.configDir,
...(layerOpts.envFile ? { envFile: layerOpts.envFile } : {}),
},
currentLayerBody,
createHash("sha256").update(currentLayerBody).digest("hex"),
);
} catch (restoreError) {
compensationFailures.push(`restoring the pre-rollback deployment layer: ${errMessage(restoreError)}`);
}
}
if (applied && before) {
try {
await applyServiceTargets(config, before.tasks, before.counts);
} catch (restoreError) {
compensationFailures.push(`restoring the pre-rollback manifest: ${errMessage(restoreError)}`);
}
}
throwAfterCompensation(error, compensationFailures);
} finally {
releaseLease(aws, lease);
}
ok(`rolled back ${config.orgId}`);
}
export function taskDefinitionForBackgroundWork(
task: Record<string, unknown>,
enabled: boolean,
): Record<string, unknown> {
const containers = structuredClone(task.containerDefinitions) as Array<Record<string, unknown>> | undefined;
const core = containers?.find((container) => container.name === "core");
if (!core) throw new CliError("core task definition has no core container");
const secrets = core.secrets as Array<{ name: string }> | undefined;
if (secrets?.some((entry) => entry.name === "BACKGROUND_WORK_ENABLED")) {
throw new CliError("BACKGROUND_WORK_ENABLED cannot be changed while supplied as a secret");
}
const environment = (core.environment ?? []) as Array<{ name: string; value: string }>;
core.environment = [
...environment.filter((entry) => entry.name !== "BACKGROUND_WORK_ENABLED"),
{ name: "BACKGROUND_WORK_ENABLED", value: enabled ? "1" : "0" },
];
const fields = [
"family",
"taskRoleArn",
"executionRoleArn",
"networkMode",
"volumes",
"placementConstraints",
"requiresCompatibilities",
"cpu",
"memory",
"pidMode",
"ipcMode",
"proxyConfiguration",
"inferenceAccelerators",
"ephemeralStorage",
"runtimePlatform",
"enableFaultInjection",
];
return {
...Object.fromEntries(fields.flatMap((field) => (task[field] == null ? [] : [[field, task[field]]]))),
containerDefinitions: containers,
};
}
function awsBackgroundWorkTransport(
config: QmConfig,
path = "/v1/background-work",
timeoutMs = 30_000,
maxResponseBytes?: number,
productionRoute = false,
): BackgroundWorkTransport {
const aws = requireAws(config);
const secret = (name: string): string =>
awsText(aws, [
"secretsmanager",
"get-secret-value",
"--secret-id",
`${aws.secretsPrefix}${name}`,
"--query",
"SecretString",
]);
const signing = secret("CORE_SIGNING_SECRET");
const control = secret("DEPLOYMENT_CONTROL_SECRET");
if (isInvalidSecret("CORE_SIGNING_SECRET", signing) || control.trim().length < 32 || control === signing)
throw new CliError("deployment control requires a distinct secret of at least 32 characters");
const url = new URL(productionRoute ? (config.apiUrl ?? config.publicUrl) : config.publicUrl);
if (productionRoute && (url.protocol !== "https:" || url.username || url.password))
throw new CliError("the production background ownership route must be HTTPS without credentials");
url.pathname = `${url.pathname.replace(/\/+$/, "")}${path}`;
url.search = "";
url.hash = "";
return (method, body = "") => {
const timestamp = Math.floor(Date.now() / 1000);
const signature = createHmac("sha256", signing)
.update(`v0:${timestamp}:${method}\n${url.pathname}\n${body}`)
.digest("hex");
const init: RequestInit = {
method,
headers: {
"content-type": "application/json",
"x-timestamp": String(timestamp),
"x-signature": `v0=${signature}`,
authorization: `Bearer ${control}`,
},
...(method === "POST" ? { body } : {}),
redirect: "error",
signal: AbortSignal.timeout(timeoutMs),
};
return productionRoute
? fetchCore(url, init, maxResponseBytes ?? 65_536)
: awsCoreRequest(config, url, init, maxResponseBytes);
};
}
interface AwsBackgroundCohort {
deploymentId: string;
taskArns: string[];
manifest: DeploymentManifest;
}
export function awsBackgroundWorkBootState(config: QmConfig): { enabled: boolean; deploymentId?: string } | undefined {
const aws = requireAws(config);
assertAwsCallerAccount(aws);
const manifest = currentDeploymentManifest(aws);
const taskDefinition = manifest?.tasks.core;
if (!taskDefinition) return undefined;
const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
taskDefinition,
]).taskDefinition;
const containers = task?.containerDefinitions as Array<Record<string, unknown>> | undefined;
const core = containers?.filter((container) => container.name === "core");
if (core?.length !== 1) throw new CliError("recorded task definition must contain exactly one core container");
const secrets = (core[0]!.secrets ?? []) as Array<{ name: string }>;
if (secrets.some((secret) => secret.name === "BACKGROUND_WORK_ENABLED" || secret.name === "BACKGROUND_DEPLOYMENT_ID"))
throw new CliError("recorded background boot settings cannot be supplied as secrets");
const environment = (core[0]!.environment ?? []) as Array<{ name: string; value: string }>;
const identities = environment.filter((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID");
if (
manifest.backgroundDeploymentId
? identities.length !== 1 || identities[0]!.value !== manifest.backgroundDeploymentId
: identities.length !== 0
)
throw new CliError("recorded core task does not match the manifest background deployment identity");
const flags = environment.filter((entry) => entry.name === "BACKGROUND_WORK_ENABLED");
if (flags.length !== 1 || !["0", "1"].includes(flags[0]!.value))
throw new CliError("recorded core task must contain one explicit BACKGROUND_WORK_ENABLED value of 0 or 1");
return {
enabled: flags[0]!.value === "1",
...(manifest.backgroundDeploymentId ? { deploymentId: manifest.backgroundDeploymentId } : {}),
};
}
async function awsBackgroundCohort(
config: QmConfig,
configDir: string,
candidatePath?: string,
waitForReady = true,
): Promise<AwsBackgroundCohort> {
const { aws, workloads } = awsTopology(config, configDir);
if (!aws.backgroundWorkControl)
throw new CliError("background ownership control is not enabled in the AWS configuration");
const manifest = currentDeploymentManifest(aws);
if (!manifest?.backgroundDeploymentId)
throw new CliError("background ownership requires a manifest-bound deployment identity");
const candidate = candidatePath ? releaseCandidate(config, candidatePath) : undefined;
const states = describedServices(config, workloads);
assertOwnedServices(config, states, workloads);
const before = serviceSnapshotFromStates(states, workloads);
for (const workload of workloads) {
if (manifest.tasks[workload] !== before.tasks[workload] || manifest.counts?.[workload] !== before.counts[workload])
throw new CliError(`cannot control background work while ${workload} differs from the deployment manifest`);
const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
before.tasks[workload]!,
]).taskDefinition;
const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
(item) => item.name === workload,
);
if (typeof container?.image !== "string" || !isPinnedWorkloadImage(config, workload, container.image))
throw new CliError(`${workload} does not have a trusted digest-pinned image`);
if (
candidate &&
(container.image !== candidate.images[workload] ||
!candidate.imageProvenance[workload] ||
canonicalJson(candidate.imageProvenance[workload]) !== canonicalJson(manifest.imageProvenance?.[workload]))
)
throw new CliError(`${workload} does not match the expected release candidate`);
if (workload === "core") {
const environment = (container.environment ?? []) as Array<{ name: string; value: string }>;
if (
environment.find((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID")?.value !==
manifest.backgroundDeploymentId
)
throw new CliError("core task definition does not match the recorded background deployment identity");
}
}
const count = before.counts.core;
if (!count) throw new CliError("background ownership requires running core capacity");
if (waitForReady)
await awaitServiceTargets(config, {
core: { taskDefinition: before.tasks.core!, desiredCount: count },
});
const taskArns =
awsJson<{ taskArns?: string[] }>(aws, [
"ecs",
"list-tasks",
"--cluster",
aws.cluster,
"--service-name",
aws.services.core!.ecsService,
"--desired-status",
"RUNNING",
]).taskArns ?? [];
const tasks = chunks(taskArns, 100).flatMap((batch) => {
const response = awsJson<{
tasks?: Array<{ taskArn?: string; taskDefinitionArn?: string; lastStatus?: string; healthStatus?: string }>;
failures?: unknown[];
}>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);
if (response.failures?.length) throw new CliError("cannot prove every running core task identity");
return response.tasks ?? [];
});
const current = tasks.filter(
(task) =>
task.taskDefinitionArn === before.tasks.core && task.lastStatus === "RUNNING" && task.healthStatus === "HEALTHY",
);
if (
current.length !== count ||
current.some((task) => !task.taskArn) ||
new Set(current.map((task) => task.taskArn)).size !== count
)
throw new CliError("core does not have the exact healthy task cohort recorded in its deployment manifest");
return { deploymentId: manifest.backgroundDeploymentId, taskArns: current.map((task) => task.taskArn!), manifest };
}
export async function awsBackgroundWorkStatus(
config: QmConfig,
configDir: string,
candidatePath?: string,
): Promise<{
status: BackgroundWorkStatus;
deploymentId: string;
taskArns: string[];
manifestId: string;
}> {
assertAwsCallerAccount(requireAws(config));
const cohort = await awsBackgroundCohort(config, configDir, candidatePath);
const status = await readBackgroundWork(awsBackgroundWorkTransport(config), cohort.deploymentId);
return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns, manifestId: cohort.manifest.id };
}
export interface AwsBackgroundWorkCapacity {
manifestId: string;
deploymentId: string;
ownerDeploymentId: string;
workloads: Record<string, { taskDefinition: string; deploymentId: string; desiredCount: number; taskArns: string[] }>;
}
function stableWorkloadCapacity(
config: QmConfig,
workload: string,
state: EcsServiceState,
taskDefinition: string,
desiredCount: number | undefined,
): AwsBackgroundWorkCapacity["workloads"][string] {
const aws = requireAws(config);
const primary = state.deployments?.filter((deployment) => deployment.status === "PRIMARY") ?? [];
const deployment = primary[0];
if (
typeof desiredCount !== "number" ||
!Number.isSafeInteger(desiredCount) ||
desiredCount < 0 ||
state.status !== "ACTIVE" ||
state.taskDefinition !== taskDefinition ||
state.desiredCount !== desiredCount ||
state.runningCount !== desiredCount ||
state.pendingCount !== 0 ||
primary.length !== 1 ||
!deployment?.id ||
deployment.taskDefinition !== taskDefinition ||
deployment.runningCount !== desiredCount ||
state.deployments?.some((item) => item.pendingCount !== 0 || (item !== deployment && item.runningCount !== 0)) ||
(state.deploymentConfiguration?.strategy === "BLUE_GREEN"
? nativeBlueGreenStatus(config, workload, taskDefinition) !== "SUCCESSFUL"
: deployment.rolloutState !== "COMPLETED")
)
throw new CliError(`${workload} has not reached stable deployment capacity`);
const listed = new Set<string>();
for (const desiredStatus of ["RUNNING", "STOPPED"]) {
const response = awsJson<{ taskArns?: string[] }>(aws, [
"ecs",
"list-tasks",
"--cluster",
aws.cluster,
"--service-name",
aws.services[workload]!.ecsService,
"--desired-status",
desiredStatus,
]);
if (!Array.isArray(response.taskArns) || response.taskArns.some((arn) => typeof arn !== "string" || !arn))
throw new CliError(`${workload} task inventory is unavailable`);
for (const arn of response.taskArns) listed.add(arn);
}
const live: string[] = [];
for (const batch of chunks([...listed], 100)) {
const response = awsJson<{
tasks?: Array<{
taskArn?: string;
taskDefinitionArn?: string;
lastStatus?: string;
desiredStatus?: string;
healthStatus?: string;
}>;
failures?: unknown[];
}>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);
if (
response.failures?.length ||
!Array.isArray(response.tasks) ||
response.tasks.length !== batch.length ||
new Set(response.tasks.map((task) => task.taskArn)).size !== batch.length ||
response.tasks.some((task) => !task.taskArn || !batch.includes(task.taskArn))
)
throw new CliError(`${workload} capacity requires complete task inventory`);
for (const task of response.tasks) {
if (task.lastStatus === "STOPPED") continue;
if (
task.lastStatus !== "RUNNING" ||
task.desiredStatus !== "RUNNING" ||
task.taskDefinitionArn !== taskDefinition ||
(task.healthStatus !== "HEALTHY" && (isServiceName(workload) || task.healthStatus !== "UNKNOWN"))
)
throw new CliError(`${workload} still has a pending, unhealthy, or retiring task`);
live.push(task.taskArn!);
}
}
if (live.length !== desiredCount) throw new CliError(`${workload} does not have its exact stable task cohort`);
return { taskDefinition, deploymentId: deployment.id, desiredCount, taskArns: live.sort() };
}
export async function awsBackgroundWorkCapacity(
config: QmConfig,
configDir: string,
candidatePath?: string,
): Promise<AwsBackgroundWorkCapacity> {
const { aws, workloads } = awsTopology(config, configDir);
assertAwsCallerAccount(aws);
const cohort = await awsBackgroundCohort(config, configDir, candidatePath, false);
const transport = awsBackgroundWorkTransport(config);
const ownership = await readBackgroundWork(transport, cohort.deploymentId);
if (!ownership.ownerDeploymentId || ownership.ownerDeploymentId === cohort.deploymentId)
throw new CliError("capacity requires background ownership held by another deployment");
const states = describedServices(config, workloads);
assertOwnedServices(config, states, workloads);
const snapshot = serviceSnapshotFromStates(states, workloads);
assertBackgroundPreparationResolved(aws, cohort.manifest, snapshot);
const proof: AwsBackgroundWorkCapacity = {
manifestId: cohort.manifest.id,
deploymentId: cohort.deploymentId,
ownerDeploymentId: ownership.ownerDeploymentId,
workloads: {},
};
for (const workload of [...workloads].sort()) {
proof.workloads[workload] = stableWorkloadCapacity(
config,
workload,
states.get(workload)!,
cohort.manifest.tasks[workload]!,
cohort.manifest.counts?.[workload],
);
}
if (canonicalJson(proof.workloads.core?.taskArns) !== canonicalJson([...cohort.taskArns].sort()))
throw new CliError("core task cohort changed while checking capacity");
const finalManifest = currentDeploymentManifest(aws);
if (canonicalJson(finalManifest) !== canonicalJson(cohort.manifest))
throw new CliError("deployment manifest changed while checking capacity");
assertBackgroundPreparationResolved(aws, finalManifest, snapshot);
const finalOwnership = await readBackgroundWork(transport, cohort.deploymentId);
if (finalOwnership.ownerDeploymentId !== proof.ownerDeploymentId)
throw new CliError("background ownership changed while checking capacity");
return proof;
}
export async function awsStoppedCoreOwner(
config: QmConfig,
deploymentId: string,
): Promise<BackgroundWorkStatus | undefined> {
const state = describedServices(config, ["core"]).get("core");
if (!state || state.desiredCount !== 0) return undefined;
stableWorkloadCapacity(config, "core", state, state.taskDefinition ?? "", 0);
return readProductionOwner(awsBackgroundWorkTransport(config, undefined, undefined, undefined, true), deploymentId);
}
async function assertNotBackgroundOwner(config: QmConfig, current: DeploymentManifest | undefined): Promise<void> {
if (!current?.backgroundDeploymentId) return;
if (await awsStoppedCoreOwner(config, current.backgroundDeploymentId)) return;
const status = await readBackgroundWork(awsBackgroundWorkTransport(config), current.backgroundDeploymentId);
if (status.ownerDeploymentId === current.backgroundDeploymentId)
throw new CliError("hand background ownership to another deployment before replacing the current core tasks");
}
export async function awsSetBackgroundWork(
config: QmConfig,
configDir: string,
enabled: boolean,
candidatePath?: string,
expectedOwnership?: Pick<BackgroundWorkStatus, "ownerDeploymentId">,
): Promise<BackgroundWorkStatus | undefined> {
const { aws, workloads } = awsTopology(config, configDir);
if (!workloads.includes("core")) throw new CliError("background work requires the core workload");
const candidate = candidatePath ? releaseCandidate(config, candidatePath) : undefined;
assertAwsCallerAccount(aws);
const confirmed = await withAwsLease(aws, async () => {
const current = currentDeploymentManifest(aws);
if (!current) throw new CliError("background work requires a recorded deployment");
const states = describedServices(config, workloads);
assertOwnedServices(config, states, workloads);
const before = serviceSnapshotFromStates(states, workloads);
for (const workload of workloads) {
if (
current.tasks[workload] !== before.tasks[workload] ||
current.counts?.[workload] !== before.counts[workload]
) {
throw new CliError(`cannot change background work while ${workload} differs from the deployment manifest`);
}
}
if (!before.counts.core) throw new CliError("cannot change background work while core is scaled down");
let coreTask: Record<string, unknown> | undefined;
for (const workload of candidate ? workloads : ["core"]) {
const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
before.tasks[workload]!,
]).taskDefinition;
const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
(item) => item.name === workload,
);
if (!task || typeof container?.image !== "string" || !isPinnedWorkloadImage(config, workload, container.image)) {
throw new CliError(`${workload} does not have a trusted digest-pinned image`);
}
if (
candidate &&
(container.image !== candidate.images[workload] ||
!candidate.imageProvenance[workload] ||
canonicalJson(candidate.imageProvenance[workload]) !== canonicalJson(current.imageProvenance?.[workload]))
) {
throw new CliError(`${workload} does not match the expected release candidate`);
}
if (workload === "core") coreTask = task;
}
if (aws.backgroundWorkControl) {
const cohort = await awsBackgroundCohort(config, configDir, candidatePath);
const transport = awsBackgroundWorkTransport(config);
const status = await readBackgroundWork(transport, cohort.deploymentId);
if (expectedOwnership && status.ownerDeploymentId !== expectedOwnership.ownerDeploymentId)
throw new CliError("background ownership changed since promotion; refusing automatic compensation");
if (!enabled && status.ownerDeploymentId !== null && status.ownerDeploymentId !== cohort.deploymentId)
throw new CliError("cannot disable background work on a different deployment's current owner");
const ownerDeploymentId = enabled ? cohort.deploymentId : null;
if (status.ownerDeploymentId !== ownerDeploymentId)
await setBackgroundOwner(transport, cohort.deploymentId, {
ownerDeploymentId,
expectedOwnerDeploymentId: status.ownerDeploymentId,
});
return awaitBackgroundWork(
transport,
cohort.deploymentId,
{ ownerDeploymentId, active: enabled, instances: cohort.taskArns.length },
{ timeoutMs: envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000), pollMs: 1000 },
);
}
if (expectedOwnership) throw new CliError("ownership preconditions require controlled background work");
if (current.backgroundDeploymentId)
throw new CliError("controlled background work cannot fall back to task replacement");
const desired = taskDefinitionForBackgroundWork(coreTask!, enabled);
const core = (coreTask!.containerDefinitions as Array<Record<string, unknown>>).find(
(item) => item.name === "core",
)!;
const environment = (core.environment ?? []) as Array<{ name: string; value: string }>;
const currentFlag = environment.find((entry) => entry.name === "BACKGROUND_WORK_ENABLED")?.value;
const timeoutMs = envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000);
if (currentFlag === (enabled ? "1" : "0")) {
await awaitServiceTargets(
config,
{
core: { taskDefinition: before.tasks.core!, desiredCount: before.counts.core },
},
timeoutMs,
);
return;
}
const dir = mkdtempSync(join(tmpdir(), "qm-background-work-"));
try {
const file = join(dir, "core.json");
writeFileSync(file, JSON.stringify(desired));
const target = registerTaskDefinition(config, file);
await applyServiceTargets(config, { core: target }, { core: before.counts.core }, { timeoutMs });
try {
recordDeploymentManifest(
aws,
{ ...before.tasks, core: target },
{
counts: current.counts,
imageLabel: current.imageLabel,
dbRestorePoint: current.dbRestorePoint,
layer: current.layer,
imageProvenance: current.imageProvenance,
},
);
} catch (error) {
const failures: string[] = [];
try {
await applyServiceTargets(config, { core: before.tasks.core! }, { core: before.counts.core }, { timeoutMs });
manifestTransaction(aws, current, current.id);
} catch (restoreError) {
failures.push(`restoring background work: ${errMessage(restoreError)}`);
}
throwAfterCompensation(error, failures);
}
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
ok(`background work ${enabled ? "enabled" : "disabled"}`);
return confirmed;
}
function envValues(configDir: string, path: string | undefined): Map<string, string> {
const file = resolve(path ?? join(configDir, ".env"));
if (!existsSync(file)) {
if (path) throw new CliError(`secret source not found: ${file}`);
return new Map();
}
return readEnvFile(file);
}
export async function awsSecretsPush(config: QmConfig, configDir: string, envFile?: string): Promise<void> {
validatedSecrets(config);
const { aws, workloads } = awsTopology(config, configDir);
assertAwsCallerAccount(aws);
const values = envValues(configDir, envFile);
const staged: Array<{ name: string; id: string; dir: string; file: string }> = [];
try {
for (const secret of computedSecrets(config).filter((item) => item.managedBy === "operator")) {
const supplied = deploymentSecretValue(secret.name, values.get(secret.name));
if (!secret.required && !supplied) {
step(`${secret.name}: optional, not supplied`);
continue;
}
const value = supplied ?? (await promptHidden(secret.name));
if (isInvalidSecret(secret.name, value)) {
throw new CliError(
`${secret.name} must have a non-empty, non-placeholder value; signing keys must be at least 32 characters`,
);
}
const dir = mkdtempSync(join(tmpdir(), "qm-secret-"));
const file = join(dir, "value");
writeFileSync(file, value, { mode: 0o600 });
staged.push({ name: secret.name, id: `${aws.secretsPrefix}${secret.name}`, dir, file });
}
await withAwsLease(aws, async () => {
const baseline = currentDeploymentManifest(aws);
if (baseline?.backgroundDeploymentId && !aws.backgroundWorkControl)
throw new CliError("controlled secret activation cannot fall back to a legacy task definition");
const states = describedServices(config, workloads);
assertOwnedServices(config, states, workloads);
const before = baseline ? serviceSnapshotFromStates(states, workloads) : undefined;
if (baseline) {
const drifted = workloads.filter(
(workload) => !baseline.tasks[workload] || before!.tasks[workload] !== baseline.tasks[workload],
);
if (drifted.length)
throw new CliError(
`cannot rotate secrets while workloads differ from the current deployment manifest: ${drifted.join(", ")}`,
);
} else {
const active = workloads.filter((workload) => {
const state = states.get(workload)!;
return state.desiredCount !== 0 || state.runningCount !== 0;
});
if (active.length) {
throw new CliError(
`cannot defer secret activation without a current deployment manifest while workloads are active: ${active.join(", ")}`,
);
}
}
const uploaded = Object.fromEntries(staged.map((secret) => [secret.name, secret.id]));
const affected = workloads.filter((workload) =>
workloadSecrets(config, workload, uploaded).some((secret) => uploaded[secret.name]),
);
if (baseline && before) {
for (const workload of affected) {
if (aws.backgroundWorkControl && workload === "core") continue;
const live = awsJson<{ taskDefinition?: { runtimePlatform?: { cpuArchitecture?: string } } }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
before.tasks[workload]!,
]).taskDefinition;
const architecture = live?.runtimePlatform?.cpuArchitecture ?? "X86_64";
const expected = workloadArchitecture(config, workload) === "arm64" ? "ARM64" : "X86_64";
if (architecture !== expected)
throw new CliError(
`cannot rotate secrets while ${workload} uses ${architecture} but its configuration selects ${expected}; set aws.services.${workload}.architecture to match the deployed image or rebuild it with qm up --build-from`,
);
}
}
if (baseline?.backgroundDeploymentId) {
for (const secret of staged.filter((item) =>
["DEPLOYMENT_CONTROL_SECRET", "CORE_SIGNING_SECRET"].includes(item.name),
)) {
const existing = awsText(aws, [
"secretsmanager",
"get-secret-value",
"--secret-id",
secret.id,
"--query",
"SecretString",
]);
if (existing !== readFileSync(secret.file, "utf8"))
throw new CliError(
`${secret.name} cannot be rotated while controlled task cohorts are deployed; coordinated control credential rotation is required`,
);
}
}
for (const secret of staged) {
try {
awsText(aws, [
"secretsmanager",
"put-secret-value",
"--secret-id",
secret.id,
"--secret-string",
`file://${secret.file}`,
]);
} catch (error) {
if (/ResourceNotFoundException/.test(errMessage(error))) {
throw new CliError(
`AWS secret container ${secret.id} is missing; apply the rendered Terraform before pushing secrets`,
);
}
throw error;
}
step(`${secret.name}: uploaded`);
}
if (!baseline || !before) {
if (affected.length) step("secret activation deferred to the first complete AWS deployment");
return;
}
if (aws.backgroundWorkControl && affected.includes("core")) {
note(
"core secret activation deferred: pause or hand over background ownership, then run qm up --restart core to allocate a new deployment identity",
);
affected.splice(affected.indexOf("core"), 1);
}
for (const [component, host, marker, expected] of [
["admin", "web-ui", "ADMIN_ENABLED", "1"],
["auth", "portal", "AUTH_EMBEDDED", "1"],
["admin", "portal", "ADMIN_UPSTREAM", serviceEnvironment(config, "portal").ADMIN_UPSTREAM],
] as const) {
if (!config.services.includes(component) || !workloads.includes(host)) continue;
const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
before.tasks[host]!,
]).taskDefinition;
const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
(item) => item.name === host,
);
const environment = (container?.environment ?? []) as Array<{ name: string; value: string }>;
if (!environment.some((entry) => entry.name === marker && entry.value === expected)) {
step("secret activation deferred until combined web-ui and portal images are deployed with qm up");
return;
}
}
const arns = secretArns(config);
const targets = { ...before.tasks };
const changed: Record<string, string> = {};
for (const workload of affected) {
const live =
awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
before.tasks[workload]!,
]).taskDefinition ?? null;
const container = (live?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
(item) => item.name === workload,
);
if (
!container ||
typeof container.image !== "string" ||
!isPinnedWorkloadImage(config, workload, container.image)
) {
throw new CliError(`cannot rotate secrets while ${workload} lacks a trusted digest-pinned image`);
}
const desired = renderTaskDefinition(config, workload, container.image, arns, baseline.backgroundDeploymentId);
if (!taskDefinitionChanges(desired, live).length) continue;
const dir = mkdtempSync(join(tmpdir(), "qm-task-"));
try {
const file = join(dir, `${workload}.json`);
writeFileSync(file, JSON.stringify(desired));
targets[workload] = registerTaskDefinition(config, file);
changed[workload] = targets[workload]!;
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
let changedApplied = false;
let rotated = false;
try {
if (Object.keys(changed).length) {
await applyServiceTargets(
config,
changed,
Object.fromEntries(Object.keys(changed).map((workload) => [workload, before.counts[workload]!])),
);
changedApplied = true;
}
const unchanged = affected.filter((workload) => !changed[workload]);
const forcedDeployments: Record<string, string> = {};
for (const workload of unchanged) {
const service = awsJson<{ service?: { deployments?: EcsDeploymentState[] } }>(aws, [
"ecs",
"update-service",
"--cluster",
aws.cluster,
"--service",
aws.services[workload]!.ecsService,
"--force-new-deployment",
]).service;
const primary = (service?.deployments ?? []).filter((deployment) => deployment.status === "PRIMARY");
if (primary.length !== 1 || !primary[0]?.id)
throw new CliError(`AWS did not identify the replacement deployment for ${workload}`);
forcedDeployments[workload] = primary[0].id;
}
if (unchanged.length) {
await awaitServiceTargets(
config,
Object.fromEntries(
unchanged.map((workload) => [
workload,
{
taskDefinition: before.tasks[workload]!,
desiredCount: before.counts[workload]!,
deploymentId: forcedDeployments[workload]!,
},
]),
),
);
}
rotated = true;
if (Object.keys(changed).length) {
recordDeploymentManifest(aws, targets, {
...(baseline.counts ? { counts: baseline.counts } : {}),
imageLabel: baseline.imageLabel ?? aws.imageLabel,
...(baseline.layer ? { layer: baseline.layer } : {}),
...(baseline.imageProvenance ? { imageProvenance: baseline.imageProvenance } : {}),
});
}
} catch (error) {
if (rotated || !changedApplied) throw error;
const compensationFailures: string[] = [];
try {
await applyServiceTargets(
config,
Object.fromEntries(Object.keys(changed).map((workload) => [workload, before.tasks[workload]!])),
Object.fromEntries(Object.keys(changed).map((workload) => [workload, before.counts[workload]!])),
);
} catch (restoreError) {
compensationFailures.push(`restoring pre-rotation task definitions: ${errMessage(restoreError)}`);
}
throwAfterCompensation(error, compensationFailures);
}
for (const workload of affected) step(`${workload}: restarted with rotated secrets`);
});
} finally {
for (const secret of staged) rmSync(secret.dir, { recursive: true, force: true });
}
ok("operator secrets uploaded to AWS Secrets Manager");
}
export function githubTrustSubject(configDir: string, deployBranch?: string, deployEnvironment?: string): string {
const tfvars = join(configDir, "infra", "terraform.tfvars");
const source = existsSync(tfvars) ? readFileSync(tfvars, "utf8") : undefined;
const tfValue = (name: string): string | undefined => {
const raw = source?.match(new RegExp(`^${name}\\s*=\\s*("(?:[^"\\\\]|\\\\.)*")\\s*$`, "m"))?.[1];
return raw ? (JSON.parse(raw) as string) : undefined;
};
let repo = process.env.GITHUB_REPOSITORY;
if (!repo && source !== undefined) {
repo = tfValue("github_repository");
if (!repo || repo === "replace-me/repository" || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
throw new Error("infra/terraform.tfvars must set an explicit github_repository owner/name");
}
}
if (!repo) {
const remote = capture("git", ["-C", configDir, "config", "--get", "remote.origin.url"]).trim();
repo = remote.match(/github\.com[/:]([^/]+\/[^/.]+)(?:\.git)?$/)?.[1];
if (!repo) throw new Error("cannot derive the GitHub repository from this deployment checkout");
}
const configuredPrefix = tfValue("github_subject_prefix");
if (
configuredPrefix !== undefined &&
configuredPrefix !== "" &&
!/^repo:[A-Za-z0-9_.@-]+\/[A-Za-z0-9_.@-]+$/.test(configuredPrefix)
) {
throw new Error("infra/terraform.tfvars github_subject_prefix must be an exact repo:owner/repository prefix");
}
const subjectPrefix = configuredPrefix || `repo:${repo}`;
if (deployEnvironment) return `${subjectPrefix}:environment:${deployEnvironment}`;
const tfRef = tfValue("github_ref");
if (tfRef !== undefined && !/^refs\/heads\/\S+$/.test(tfRef)) {
throw new Error("infra/terraform.tfvars github_ref must be a refs/heads/* branch ref");
}
const ref = deployBranch ? `refs/heads/${deployBranch}` : (tfRef ?? "refs/heads/main");
return `${subjectPrefix}:ref:${ref}`;
}
export function assertGithubDeployTrust(statements: unknown, accountId: string, expectedSubject: string): void {
let list: unknown[] = [];
if (Array.isArray(statements)) list = statements;
else if (statements) list = [statements];
if (list.length !== 1 || typeof list[0] !== "object" || list[0] === null || Array.isArray(list[0])) {
throw new Error("deploy role must have exactly one trust statement");
}
const statement = list[0] as Record<string, unknown>;
const actions = Array.isArray(statement.Action) ? statement.Action : [statement.Action];
if (statement.Effect !== "Allow" || actions.length !== 1 || actions[0] !== "sts:AssumeRoleWithWebIdentity") {
throw new Error("deploy role trust must only allow GitHub OIDC assumption");
}
const principal = statement.Principal as Record<string, unknown> | undefined;
if (
principal?.Federated !== `arn:aws:iam::${accountId}:oidc-provider/token.actions.githubusercontent.com` ||
Object.keys(principal).length !== 1
) {
throw new Error("deploy role trust must name only the account's GitHub OIDC provider");
}
const condition = statement.Condition as Record<string, unknown> | undefined;
const equals = condition?.StringEquals as Record<string, unknown> | undefined;
const live = equals?.["token.actions.githubusercontent.com:sub"];
const rawSubjects: unknown[] = Array.isArray(live) ? live : [live].filter((s) => s !== undefined);
const withoutIdPins = (subject: string): string =>
subject.replace(/^repo:([^/:@]+)(?:@\d+)?\/([^/:@]+)(?:@\d+)?:/, "repo:$1/$2:");
const expectedPinsIds = withoutIdPins(expectedSubject) !== expectedSubject;
const subjects = expectedPinsIds
? rawSubjects
: rawSubjects.map((subject) => (typeof subject === "string" ? withoutIdPins(subject) : subject));
const repoPrefix = `${expectedSubject.split(":").slice(0, 2).join(":")}:`;
if (
!condition ||
Object.keys(condition).length !== 1 ||
!equals ||
Object.keys(equals).length !== 2 ||
equals["token.actions.githubusercontent.com:aud"] !== "sts.amazonaws.com" ||
!subjects.includes(expectedSubject) ||
!subjects.every(
(subject) => typeof subject === "string" && subject.startsWith(repoPrefix) && !subject.includes("*"),
)
) {
const rendered = rawSubjects.length > 0 ? rawSubjects.join(", ") : "missing";
throw new Error(
`OIDC trust must pin audience and only ${repoPrefix}* subjects including ${expectedSubject} (live subject: ${rendered})`,
);
}
}
export function assertAwsPublicListener(
publicUrl: string,
listener: { Protocol?: string; Port?: number; Certificates?: Array<{ CertificateArn?: string }> },
): void {
const protocol = new URL(publicUrl).protocol;
if (protocol === "https:") {
if (listener.Protocol !== "HTTPS" || listener.Port !== 443) {
throw new Error(
`publicUrl is HTTPS but the ALB listener is ${listener.Protocol ?? "missing"}:${listener.Port ?? "missing"}; configure certificate_arn and apply Terraform`,
);
}
if (!listener.Certificates?.some((certificate) => Boolean(certificate.CertificateArn))) {
throw new Error(
"publicUrl is HTTPS but the ALB listener has no certificate; configure certificate_arn and apply Terraform",
);
}
return;
}
if (protocol === "http:") {
if (listener.Protocol !== "HTTP" || listener.Port !== 80) {
throw new Error(
`publicUrl is HTTP but the ALB listener is ${listener.Protocol ?? "missing"}:${listener.Port ?? "missing"}`,
);
}
return;
}
throw new Error(`publicUrl must use http or https (got ${protocol})`);
}
interface AwsPublicListener {
ListenerArn?: string;
Protocol?: string;
Port?: number;
Certificates?: Array<{ CertificateArn?: string }>;
DefaultActions?: Array<{
Type?: string;
TargetGroupArn?: string;
FixedResponseConfig?: { StatusCode?: string };
RedirectConfig?: { Protocol?: string; Port?: string; StatusCode?: string };
}>;
}
interface AwsPublicFrontDoor {
loadBalancerArn: string;
dnsName: string;
listener: AwsPublicListener;
}
function awsPublicOrigin(config: QmConfig): URL {
const value = config.env.core?.AWS_PUBLIC_ORIGIN_URL?.trim() || config.publicUrl;
try {
const url = new URL(value);
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("protocol");
return url;
} catch {
throw new Error("env.core.AWS_PUBLIC_ORIGIN_URL must be an HTTP(S) URL when set");
}
}
function isHttpsRedirectListener(listener: AwsPublicListener): boolean {
if (listener.Protocol !== "HTTP" || listener.Port !== 80) return false;
const actions = listener.DefaultActions ?? [];
return actions.length === 1 && actions[0]?.Type === "redirect" && actions[0].RedirectConfig?.Protocol === "HTTPS";
}
interface AwsServiceConnectConfiguration {
enabled?: boolean;
namespace?: string;
services?: Array<{
portName?: string;
discoveryName?: string;
clientAliases?: Array<{ dnsName?: string; port?: number }>;
}>;
}
interface AwsEcsRoutingService {
serviceName?: string;
status?: string;
serviceConnectConfiguration?: AwsServiceConnectConfiguration;
deployments?: Array<{
status?: string;
serviceConnectConfiguration?: AwsServiceConnectConfiguration;
}>;
loadBalancers?: Array<{
targetGroupArn?: string;
advancedConfiguration?: {
alternateTargetGroupArn?: string;
productionListenerRule?: string;
};
}>;
}
function awsServiceConnectConfiguration(
service: Pick<AwsEcsRoutingService, "serviceConnectConfiguration" | "deployments">,
): AwsServiceConnectConfiguration | undefined {
return (
service.serviceConnectConfiguration ??
service.deployments?.find((deployment) => deployment.status === "PRIMARY")?.serviceConnectConfiguration
);
}
export function assertAwsServiceDiscovery(
config: QmConfig,
ecsServices: ReadonlyMap<string, AwsEcsRoutingService>,
): void {
const aws = requireAws(config);
const namespaces =
awsJson<{ Namespaces?: Array<{ Id?: string; Name?: string; Arn?: string }> }>(aws, [
"servicediscovery",
"list-namespaces",
]).Namespaces ?? [];
let namespaceId: string | undefined;
let registrations: Array<{ Arn?: string; Name?: string }> = [];
for (const name of deployedAwsServices(aws)) {
const ecsService = ecsServices.get(name);
const connect = ecsService ? awsServiceConnectConfiguration(ecsService) : undefined;
if (!connect?.enabled) throw new Error(`ECS service ${name} does not have Service Connect enabled`);
const namespace = namespaces.find((item) => item.Arn === connect.namespace || item.Name === connect.namespace);
if (!connect.namespace || !namespace?.Id)
throw new Error(`ECS service ${name} references a missing Cloud Map namespace`);
if (namespaceId && namespaceId !== namespace.Id)
throw new Error("ECS services must share their Service Connect namespace");
if (!namespaceId) {
namespaceId = namespace.Id;
registrations =
awsJson<{ Services?: Array<{ Arn?: string; Name?: string }> }>(aws, [
"servicediscovery",
"list-services",
"--filters",
`Name=NAMESPACE_ID,Values=${namespaceId},Condition=EQ`,
]).Services ?? [];
}
const endpoints = connect.services?.filter((service) => service.portName === name) ?? [];
if (endpoints.length !== 1)
throw new Error(`ECS service ${name} does not publish its named ${name} port through Service Connect`);
const endpoint = endpoints[0]!;
const discoveryName = endpoint.discoveryName ?? endpoint.portName;
if (!registrations.some((service) => service.Name === discoveryName && service.Arn)) {
throw new Error(`service ${discoveryName} is missing from ${namespace.Name}`);
}
const expectedDns = `${name}.${aws.networking.cloudMapNamespace}`;
const expectedPort = isServiceName(name) ? serviceDef(name).docker.internalPort : 8080;
if (!endpoint.clientAliases?.some((alias) => alias.dnsName === expectedDns && alias.port === expectedPort)) {
throw new Error(`ECS service ${name} does not publish the ${expectedDns}:${expectedPort} client alias`);
}
}
}
function awsEcsRoutingServices(config: QmConfig): ReadonlyMap<string, AwsEcsRoutingService> {
const aws = requireAws(config);
const entries = deployedAwsServices(aws).map((name) => [name, aws.services[name]!] as const);
const services: AwsEcsRoutingService[] = [];
const failures: Array<{ arn?: string; reason?: string }> = [];
for (const batch of chunks(
entries.map(([, spec]) => spec.ecsService),
ECS_SERVICE_BATCH_SIZE,
)) {
const described = awsJson<{
services?: AwsEcsRoutingService[];
failures?: Array<{ arn?: string; reason?: string }>;
}>(aws, ["ecs", "describe-services", "--cluster", aws.cluster, "--services", ...batch]);
services.push(...(described.services ?? []));
failures.push(...(described.failures ?? []));
}
if (failures.length) {
throw new Error(
`could not describe ECS routing attachments: ${failures.map((failure) => `${failure.arn ?? "unknown"} (${failure.reason ?? "unknown"})`).join(", ")}`,
);
}
return new Map(
entries.map(([name, spec]) => {
const service = services.find((candidate) => candidate.serviceName === spec.ecsService);
if (!service) throw new Error(`ECS service ${spec.ecsService} is missing`);
return [name, service] as const;
}),
);
}
function awsPublicFrontDoor(config: QmConfig): AwsPublicFrontDoor {
const aws = requireAws(config);
const albName =
aws.alb ?? `${aws.cluster.slice(0, 23)}-${createHash("sha1").update(aws.cluster).digest("hex").slice(0, 8)}`;
const loadBalancer = awsJson<{
LoadBalancers?: Array<{ LoadBalancerArn?: string; DNSName?: string; State?: { Code?: string } }>;
}>(aws, ["elbv2", "describe-load-balancers", "--names", albName]).LoadBalancers?.[0];
if (!loadBalancer?.LoadBalancerArn || !loadBalancer.DNSName || loadBalancer.State?.Code !== "active") {
throw new Error(`load balancer is ${loadBalancer?.State?.Code ?? "missing"}`);
}
const listeners =
awsJson<{ Listeners?: AwsPublicListener[] }>(aws, [
"elbv2",
"describe-listeners",
"--load-balancer-arn",
loadBalancer.LoadBalancerArn,
]).Listeners ?? [];
const origin = awsPublicOrigin(config);
const httpsFrontDoor = origin.protocol === "https:";
const redirects = httpsFrontDoor ? listeners.filter(isHttpsRedirectListener) : [];
const candidates = listeners.filter((listener) => !redirects.includes(listener));
if (candidates.length !== 1 || redirects.length > 1) {
const found = listeners
.map(
(listener) =>
`${listener.Protocol ?? "unknown"}:${listener.Port ?? "?"} (default ${listener.DefaultActions?.map((action) => action.Type ?? "unknown").join("+") || "none"})`,
)
.join(", ");
throw new Error(
`expected exactly one public listener${httpsFrontDoor ? " plus at most one port-80 HTTPS-redirect listener" : ""}, found ${found || "none"}`,
);
}
const listener = candidates[0];
if (!listener?.ListenerArn) throw new Error("public listener is missing");
assertAwsPublicListener(origin.toString(), listener);
return { loadBalancerArn: loadBalancer.LoadBalancerArn, dnsName: loadBalancer.DNSName, listener };
}
function assertCloudFrontLayerTarget(config: QmConfig, url: URL, target: string): void {
interface Distribution {
DomainName?: string;
Status?: string;
Enabled?: boolean;
ContinuousDeploymentPolicyId?: string;
CacheBehaviors?: { Quantity?: number };
DefaultCacheBehavior?: {
TargetOriginId?: string;
MaxTTL?: number;
CachePolicyId?: string;
FunctionAssociations?: { Quantity?: number };
LambdaFunctionAssociations?: { Quantity?: number };
};
Origins?: {
Items?: Array<{
Id?: string;
DomainName?: string;
OriginPath?: string;
CustomOriginConfig?: { OriginProtocolPolicy?: string; HTTPPort?: number };
}>;
};
}
const distributions =
awsJson<{ DistributionList?: { Items?: Distribution[] } }>(requireAws(config), ["cloudfront", "list-distributions"])
.DistributionList?.Items ?? [];
const matches = distributions.filter((distribution) => distribution.DomainName === url.hostname);
const distribution = matches.length === 1 ? matches[0] : undefined;
const behavior = distribution?.DefaultCacheBehavior;
const origin = distribution?.Origins?.Items?.find((item) => item.Id === behavior?.TargetOriginId);
if (
!distribution?.Enabled ||
distribution.Status !== "Deployed" ||
distribution.ContinuousDeploymentPolicyId ||
behavior?.MaxTTL !== 0 ||
behavior.CachePolicyId ||
distribution.CacheBehaviors?.Quantity !== 0 ||
(behavior?.FunctionAssociations?.Quantity ?? 0) !== 0 ||
(behavior?.LambdaFunctionAssociations?.Quantity ?? 0) !== 0 ||
origin?.DomainName?.toLowerCase().replace(/\.$/, "") !== target ||
origin.OriginPath ||
origin.CustomOriginConfig?.OriginProtocolPolicy !== "http-only" ||
origin.CustomOriginConfig.HTTPPort !== 80 ||
(url.port && url.port !== "443")
) {
throw new CliError(
"AWS deployment-layer HTTPS proxy must be a deployed CloudFront distribution routing directly to the selected ALB without alternate behaviors, origin paths, or edge functions",
);
}
}
export function assertAwsPublicRouting(
config: QmConfig,
ecsServices?: ReadonlyMap<string, AwsEcsRoutingService>,
): ReadonlyMap<string, string> {
const aws = requireAws(config);
const hash = (value: string, length: number): string =>
createHash("sha1").update(value).digest("hex").slice(0, length);
const targetName = (name: string): string =>
aws.services[name]?.targetGroup ??
`${aws.cluster.slice(0, 20)}-${name.replaceAll("-", "").slice(0, 4)}-${hash(`${aws.cluster}:${name}`, 6)}`;
const hasPortal = Boolean(aws.services.portal);
const coreHosts = hasPortal ? awsCoreHostnames(config) : [];
let ingress = ["core"];
if (hasPortal) ingress = coreHosts.length ? ["portal", "core"] : ["portal"];
const publicPaths = new Map(
Object.entries(aws.services).flatMap(([name, service]) =>
service?.publicPaths?.length ? [[name, service.publicPaths] as const] : [],
),
);
ingress.push(...publicPaths.keys());
const { loadBalancerArn, listener } = awsPublicFrontDoor(config);
const targetGroups =
awsJson<{ TargetGroups?: Array<{ TargetGroupArn?: string; TargetGroupName?: string }> }>(aws, [
"elbv2",
"describe-target-groups",
"--load-balancer-arn",
loadBalancerArn,
]).TargetGroups ?? [];
const routingServices = ecsServices ?? awsEcsRoutingServices(config);
const targets = new Map<string, string>();
const alternateTargets = new Map<string, string>();
const productionRules = new Map<string, string>();
const expectedTargetArns = new Set<string>();
for (const name of ingress) {
const expectedName = targetName(name);
const target = targetGroups.find((group) => group.TargetGroupName === expectedName);
if (!target?.TargetGroupArn) throw new Error(`target group for ${name} is missing`);
targets.set(name, target.TargetGroupArn);
expectedTargetArns.add(target.TargetGroupArn);
const attachment = (routingServices.get(name)?.loadBalancers ?? []).find(
(item) => item.targetGroupArn === target.TargetGroupArn,
);
if (!attachment) {
throw new Error(`ECS service ${name} is not attached to its target group`);
}
const alternate = attachment.advancedConfiguration?.alternateTargetGroupArn;
const productionRule = attachment.advancedConfiguration?.productionListenerRule;
if (Boolean(alternate) !== Boolean(productionRule)) {
throw new Error(`ECS service ${name} has an incomplete blue/green load-balancer configuration`);
}
if (alternate && productionRule) {
if (!targetGroups.some((group) => group.TargetGroupArn === alternate)) {
throw new Error(`alternate target group for ${name} is missing`);
}
alternateTargets.set(name, alternate);
productionRules.set(name, productionRule);
expectedTargetArns.add(alternate);
}
}
for (const name of deployedAwsServices(aws)) {
if (
!ingress.includes(name) &&
!(isServiceName(name) && serviceDef(name).tcpIngress) &&
(routingServices.get(name)?.loadBalancers ?? []).length
) {
throw new Error(`private ECS service ${name} is attached to a load balancer`);
}
}
if (
!aws.sharedAlb &&
(targetGroups.length !== expectedTargetArns.size ||
targetGroups.some((group) => !group.TargetGroupArn || !expectedTargetArns.has(group.TargetGroupArn)))
) {
throw new Error("ALB has target groups for private or unknown services");
}
if (alternateTargets.size && alternateTargets.size !== ingress.length) {
throw new Error("ALB cannot mix rolling and blue/green ingress services");
}
if (aws.sharedAlb && (!hasPortal || coreHosts.length || publicPaths.size || alternateTargets.size !== 1)) {
throw new Error("shared ALB requires one blue/green portal ingress without additional public routes");
}
const defaults = listener.DefaultActions ?? [];
if (alternateTargets.size) {
if (
defaults.length !== 1 ||
defaults[0]?.Type !== "fixed-response" ||
defaults[0].FixedResponseConfig?.StatusCode !== "404" ||
defaults[0].TargetGroupArn
) {
throw new Error("blue/green listener default must return a fixed 404 response");
}
} else if (hasPortal) {
if (
defaults.length !== 1 ||
defaults[0]?.Type !== "forward" ||
defaults[0].TargetGroupArn !== targets.get("portal")
) {
throw new Error("portal listener default does not route only to portal");
}
} else if (
defaults.length !== 1 ||
defaults[0]?.Type !== "fixed-response" ||
defaults[0].FixedResponseConfig?.StatusCode !== "404" ||
defaults[0].TargetGroupArn
) {
throw new Error("non-portal listener default must return a fixed 404 response");
}
const rules =
awsJson<{
Rules?: Array<{
RuleArn?: string;
Priority?: string;
IsDefault?: boolean;
Actions?: Array<{
Type?: string;
TargetGroupArn?: string;
ForwardConfig?: { TargetGroups?: Array<{ TargetGroupArn?: string; Weight?: number }> };
}>;
Conditions?: Array<{
Field?: string;
Values?: string[];
PathPatternConfig?: { Values?: string[] };
HostHeaderConfig?: { Values?: string[] };
}>;
}>;
}>(aws, ["elbv2", "describe-rules", "--listener-arn", listener.ListenerArn!]).Rules ?? [];
let nonDefault = rules.filter((rule) => !rule.IsDefault);
if (aws.sharedAlb) {
const hostname = new URL(config.publicUrl).hostname.toLowerCase();
const portalAppsDomain = awsPortalAppsDomain(config);
const expectedHosts = [hostname, ...(portalAppsDomain ? [`*.${portalAppsDomain}`] : [])];
const overlaps = (a: string, b: string): boolean => {
if (a === b) return true;
if (a.startsWith("*.")) return b.endsWith(a.slice(1));
if (b.startsWith("*.")) return a.endsWith(b.slice(1));
return false;
};
const ownRule = productionRules.get("portal");
for (const rule of nonDefault) {
const hosts = rule.Conditions?.filter((condition) => condition.Field === "host-header");
const values = (hosts?.[0]?.HostHeaderConfig?.Values ?? hosts?.[0]?.Values ?? []).map((value) =>
value.toLowerCase(),
);
const companyHosts = values.filter((value) => validAlbHostname(value));
if (
hosts?.length !== 1 ||
companyHosts.length !== 1 ||
values.some(
(value) =>
!validAlbHostname(value) &&
!(
value.startsWith("*.") &&
validAlbHostname(value.slice(2)) &&
value.slice(2).endsWith(`.${companyHosts[0]}`)
),
)
) {
throw new Error("shared ALB rules require one company hostname and company-contained app wildcards");
}
const matchesHost = values.some((value) =>
expectedHosts.some((expected) => overlaps(value, expected) || overlaps(expected, value)),
);
if (rule.RuleArn === ownRule) {
if (values.length !== expectedHosts.length || expectedHosts.some((value) => !values.includes(value)))
throw new Error("shared ALB portal rule must match only this company and its configured app hostname");
} else {
const referencesOwnTarget = rule.Actions?.some(
(action) =>
expectedTargetArns.has(action.TargetGroupArn ?? "") ||
action.ForwardConfig?.TargetGroups?.some((target) => expectedTargetArns.has(target.TargetGroupArn ?? "")),
);
if (matchesHost || referencesOwnTarget) throw new Error("shared ALB has an overlapping company route");
}
}
nonDefault = nonDefault
.filter((rule) => rule.RuleArn === ownRule)
.map((rule) => ({
...rule,
Conditions: rule.Conditions?.filter((condition) => condition.Field !== "host-header"),
}));
}
const pluginRules = new Set<(typeof nonDefault)[number]>();
for (const [name, paths] of publicPaths) {
const matches = nonDefault.filter((rule) =>
alternateTargets.size
? rule.RuleArn === productionRules.get(name)
: rule.Actions?.length === 1 &&
rule.Actions[0]?.Type === "forward" &&
rule.Actions[0]?.TargetGroupArn === targets.get(name),
);
const rule = matches[0];
const condition = rule?.Conditions?.length === 1 ? rule.Conditions[0] : undefined;
const actual =
condition?.Field === "path-pattern" ? (condition.PathPatternConfig?.Values ?? condition.Values ?? []) : [];
if (
matches.length !== 1 ||
!rule ||
actual.length !== paths.length ||
paths.some((path) => !actual.includes(path))
) {
throw new Error(`public plugin ${name} must route exactly its declared publicPaths`);
}
pluginRules.add(rule);
}
for (const rule of pluginRules) {
const priority = Number(rule.Priority);
if (
!Number.isInteger(priority) ||
priority < 1 ||
nonDefault.some(
(other) =>
!pluginRules.has(other) && (!Number.isInteger(Number(other.Priority)) || priority >= Number(other.Priority)),
)
) {
throw new Error("public plugin routes must precede core and portal routes");
}
}
if (alternateTargets.size) {
if (nonDefault.length !== ingress.length) {
throw new Error("blue/green ALB has unexpected non-default rules");
}
for (const name of ingress) {
const rule = nonDefault.find((candidate) => candidate.RuleArn === productionRules.get(name));
const action = rule?.Actions?.length === 1 ? rule.Actions[0] : undefined;
const weightedTargets = action?.ForwardConfig?.TargetGroups ?? [];
const primary = targets.get(name)!;
const alternate = alternateTargets.get(name)!;
const expected = new Set([primary, alternate]);
const active = weightedTargets.filter((target) => (target.Weight ?? 0) > 0);
if (
action?.Type !== "forward" ||
weightedTargets.length !== 2 ||
weightedTargets.some((target) => !target.TargetGroupArn || !expected.has(target.TargetGroupArn)) ||
active.length !== 1 ||
!active[0]?.TargetGroupArn
) {
throw new Error(`blue/green ALB rule for ${name} must route to exactly one of its two target groups`);
}
const condition = rule?.Conditions?.length === 1 ? rule.Conditions[0] : undefined;
const pathValues =
condition?.Field === "path-pattern" ? (condition.PathPatternConfig?.Values ?? condition.Values ?? []) : [];
const hostValues =
condition?.Field === "host-header" ? (condition.HostHeaderConfig?.Values ?? condition.Values ?? []) : [];
let expectedPaths: string[];
if (publicPaths.has(name)) expectedPaths = publicPaths.get(name)!;
else if (name === "portal") expectedPaths = ["/*"];
else if (hasPortal) expectedPaths = [];
else expectedPaths = ["/v1/*"];
const expectedHosts = name === "core" && hasPortal ? coreHosts : [];
if (
pathValues.length !== expectedPaths.length ||
expectedPaths.some((value) => !pathValues.includes(value)) ||
hostValues.length !== expectedHosts.length ||
expectedHosts.some((value) => !hostValues.map((host) => host.toLowerCase()).includes(value))
) {
throw new Error(`blue/green ALB rule for ${name} does not match its production route`);
}
targets.set(name, active[0].TargetGroupArn);
}
return targets;
}
nonDefault = nonDefault.filter((rule) => !pluginRules.has(rule));
if (hasPortal && !coreHosts.length && nonDefault.length)
throw new Error("portal mode must not expose non-default ALB rules");
if (hasPortal && coreHosts.length) {
const found: string[] = [];
for (const rule of nonDefault) {
const action = rule.Actions?.length === 1 ? rule.Actions[0] : undefined;
const condition = rule.Conditions?.length === 1 ? rule.Conditions[0] : undefined;
const values =
condition?.Field === "host-header" ? (condition.HostHeaderConfig?.Values ?? condition.Values ?? []) : [];
if (action?.Type !== "forward" || action.TargetGroupArn !== targets.get("core") || !values.length) {
throw new Error(
`portal ALB has a non-default rule that is not a single host-header forward to core (expected only ${coreHosts.join(", ")})`,
);
}
found.push(...values.map((value) => value.toLowerCase()));
}
if (found.length !== coreHosts.length || coreHosts.some((host) => !found.includes(host))) {
throw new Error(
`portal ALB host rules must route exactly ${coreHosts.join(", ")} to core (live: ${found.join(", ") || "none"})`,
);
}
}
if (!hasPortal) {
const coreRule = nonDefault.filter(
(rule) =>
rule.Actions?.length === 1 &&
rule.Actions[0]?.Type === "forward" &&
rule.Actions[0].TargetGroupArn === targets.get("core") &&
rule.Conditions?.length === 1 &&
rule.Conditions[0]?.Field === "path-pattern" &&
rule.Conditions[0].PathPatternConfig?.Values?.length === 1 &&
rule.Conditions[0].PathPatternConfig.Values[0] === "/v1/*",
);
if (coreRule.length !== 1) {
throw new Error("non-portal ALB must route only /v1/* directly to core");
}
if (nonDefault.length !== 1) throw new Error("non-portal ALB has unexpected non-default rules");
}
return targets;
}
function assertAwsPublicFrontDoor(config: QmConfig): void {
try {
assertAwsPublicRouting(config);
} catch (error) {
throw new CliError(`AWS public front door is not ready for deployment: ${errMessage(error)}`);
}
}
async function assertAwsPublicNetwork(config: QmConfig, rejectServerErrors = false): Promise<void> {
const originHostname = awsPublicOrigin(config).hostname.toLowerCase().replace(/\.$/, "");
const albHostname = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");
if (originHostname !== albHostname) {
const [cnames, publicAddresses, albAddresses] = await Promise.all([
resolveCname(originHostname).catch(() => []),
lookup(originHostname, { all: true })
.then((values) => values.map((value) => value.address))
.catch(() => []),
lookup(albHostname, { all: true })
.then((values) => values.map((value) => value.address))
.catch(() => []),
]);
const cnameMatch = cnames.some((name) => name.toLowerCase().replace(/\.$/, "") === albHostname);
const albSet = new Set(albAddresses);
const addressMatch = publicAddresses.some((address) => albSet.has(address));
if (!cnameMatch && !addressMatch) {
throw new CliError(`AWS public origin ${originHostname} does not resolve to this stack's ALB ${albHostname}`);
}
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 10_000);
try {
const response = await fetch(config.publicUrl, { redirect: "manual", signal: controller.signal });
await response.body?.cancel();
if (rejectServerErrors && response.status >= 500) throw new Error(`HTTP ${response.status}`);
} catch (error) {
throw new CliError(`AWS public URL is not reachable with trusted TLS and matching DNS: ${errMessage(error)}`);
} finally {
clearTimeout(timeout);
}
}
function assertAwsHealthyIngress(config: QmConfig, targets: ReadonlyMap<string, string>): void {
const aws = requireAws(config);
for (const [name, targetGroupArn] of targets) {
const health =
awsJson<{ TargetHealthDescriptions?: Array<{ TargetHealth?: { State?: string; Reason?: string } }> }>(aws, [
"elbv2",
"describe-target-health",
"--target-group-arn",
targetGroupArn,
]).TargetHealthDescriptions ?? [];
if (!health.some((target) => target.TargetHealth?.State === "healthy")) {
const states = health.map(
(target) =>
`${target.TargetHealth?.State ?? "unknown"}${target.TargetHealth?.Reason ? ` (${target.TargetHealth.Reason})` : ""}`,
);
throw new Error(`${name} target group has no healthy targets${states.length ? `: ${states.join(", ")}` : ""}`);
}
}
}
export function assertAwsDeploymentStorage(config: QmConfig): void {
const aws = requireAws(config);
const tableName = deployLocksTable(aws);
const table = awsJson<{ Table?: { TableStatus?: string } }>(aws, [
"dynamodb",
"describe-table",
"--table-name",
tableName,
]).Table;
if (table?.TableStatus !== "ACTIVE")
throw new Error(`deploy-lock table ${tableName} is ${table?.TableStatus ?? "missing"}`);
awsText(aws, [
"s3api",
"list-objects-v2",
"--bucket",
awsObjectStoreBucket(config),
"--prefix",
"deployment/",
"--max-keys",
"1",
]);
}
export function probeAwsSecretStore(
secrets: ComputedSecret[],
read: (name: string) => string,
checkPublicApiUrl: () => void,
): { values: Map<string, string>; pending: string[]; failures: string[] } {
const values = new Map<string, string>();
const pending: string[] = [];
const failures: string[] = [];
for (const secret of secrets) {
const label = secret.required ? `secret ${secret.name}` : `optional secret ${secret.name}`;
try {
const value = read(secret.name);
if (isInvalidSecret(secret.name, value)) throw new Error("missing, placeholder, or insecure value");
values.set(secret.name, value);
if (secret.required && secret.name === "PUBLIC_API_URL") checkPublicApiUrl();
step(secret.required ? `${label}: ok` : `${label}: configured`);
} catch (error) {
if (/ResourceNotFoundException/.test(errMessage(error))) {
if (secret.required) {
pending.push(secret.name);
step(`${label}: not pushed yet — run \`qm secrets push\` before the first deploy`);
} else {
warn(`${label}: not configured`);
}
} else {
failures.push(`${label}: ${errMessage(error)}`);
warn(`${label}: failed`);
}
}
}
return { values, pending, failures };
}
export async function awsDoctor(config: QmConfig, configDir: string): Promise<void> {
const { aws } = awsTopology(config, configDir);
header(`qm doctor — ${config.orgId} (aws)`);
assertAwsCallerAccount(aws);
step("AWS caller account: ok");
const failures: string[] = [];
const check = (label: string, fn: () => void): void => {
try {
fn();
step(`${label}: ok`);
} catch (error) {
failures.push(`${label}: ${(error as Error).message}`);
warn(`${label}: failed`);
}
};
const checkAsync = async (label: string, fn: () => Promise<void>): Promise<void> => {
try {
await fn();
step(`${label}: ok`);
} catch (error) {
failures.push(`${label}: ${(error as Error).message}`);
warn(`${label}: failed`);
}
};
const tfvarsPath = join(configDir, "infra", "terraform.tfvars");
if (existsSync(tfvarsPath))
check("Terraform coordinates", () => {
const path = tfvarsPath;
const existing = readFileSync(path, "utf8");
const variablesPath = join(configDir, "infra", "variables.tf");
const declared = existsSync(variablesPath) ? declaredVariables(readFileSync(variablesPath, "utf8")) : [];
const drift = terraformVarsDrift(config, existing, declared);
if (drift.length)
throw new Error(`terraform.tfvars drift (${drift.join(", ")}); run \`qm infra render\` and commit the result`);
});
else step("Terraform coordinates: external infrastructure (no vendored infra/terraform.tfvars)");
check("ECS cluster", () => {
const cluster = awsJson<{ clusters?: Array<{ status?: string }> }>(aws, [
"ecs",
"describe-clusters",
"--clusters",
aws.cluster,
]).clusters?.[0];
if (cluster?.status !== "ACTIVE") throw new Error(`cluster is ${cluster?.status ?? "missing"}`);
});
check("deployment state stores", () => assertAwsDeploymentStorage(config));
const snapshotBucket = config.env.core?.S3_BUCKET?.trim();
if (snapshotBucket)
check(`core S3 bucket ${snapshotBucket}`, () => {
try {
awsText(aws, ["s3api", "head-bucket", "--bucket", snapshotBucket]);
} catch (error) {
throw new Error(`the env.core.S3_BUCKET override is missing or not readable: ${errMessage(error)}`, {
cause: error,
});
}
});
check("Lambda MicroVM deploy image", () => assertAwsDeployImage(config));
check("deploy role", () => {
const expectedSubject = githubTrustSubject(configDir, aws.deployBranch, aws.deployEnvironment);
const role = awsJson<{ Role?: { Arn?: string; AssumeRolePolicyDocument?: { Statement?: unknown } } }>(aws, [
"iam",
"get-role",
"--role-name",
aws.deployRoleArn.split("/").pop()!,
]);
if (role.Role?.Arn !== aws.deployRoleArn) throw new Error("role ARN differs from config");
assertGithubDeployTrust(role.Role.AssumeRolePolicyDocument?.Statement, aws.accountId, expectedSubject);
if (process.env.GITHUB_ACTIONS === "true") {
const caller = awsJson<{ Arn?: string }>(aws, ["sts", "get-caller-identity"]);
const roleName = aws.deployRoleArn.split("/").pop()!;
if (!caller.Arn?.includes(`:assumed-role/${roleName}/`))
throw new Error("workflow is not actually running as the configured deploy role");
}
});
check("GitHub OIDC provider", () => {
const arn = `arn:aws:iam::${aws.accountId}:oidc-provider/token.actions.githubusercontent.com`;
const provider = awsJson<{ Url?: string; ClientIDList?: string[] }>(aws, [
"iam",
"get-open-id-connect-provider",
"--open-id-connect-provider-arn",
arn,
]);
if (provider.Url !== "token.actions.githubusercontent.com")
throw new Error("GitHub OIDC provider URL is incorrect");
if (!provider.ClientIDList?.includes("sts.amazonaws.com"))
throw new Error("GitHub OIDC provider does not trust sts.amazonaws.com");
});
check("RDS", () => {
const database = awsJson<{
DBInstances?: Array<{
DBInstanceStatus?: string;
Endpoint?: { Address?: string; Port?: number };
VpcSecurityGroups?: Array<{ VpcSecurityGroupId?: string }>;
}>;
}>(aws, ["rds", "describe-db-instances", "--db-instance-identifier", rdsInstanceIdentifier(aws)]).DBInstances?.[0];
if (database?.DBInstanceStatus !== "available" && database?.DBInstanceStatus !== "storage-optimization")
throw new Error(`database is ${database?.DBInstanceStatus || "missing"}`);
const coreService = aws.services.core;
if (!coreService) throw new Error("aws.services.core is missing");
const coreGroups =
awsJson<{ services?: Array<{ networkConfiguration?: { awsvpcConfiguration?: { securityGroups?: string[] } } }> }>(
aws,
["ecs", "describe-services", "--cluster", aws.cluster, "--services", coreService.ecsService],
).services?.[0]?.networkConfiguration?.awsvpcConfiguration?.securityGroups ?? [];
const databaseGroups = (database.VpcSecurityGroups ?? []).flatMap((group) =>
group.VpcSecurityGroupId ? [group.VpcSecurityGroupId] : [],
);
const permissions = databaseGroups.length
? (awsJson<{
SecurityGroups?: Array<{
IpPermissions?: Array<{
IpProtocol?: string;
FromPort?: number;
ToPort?: number;
UserIdGroupPairs?: Array<{ GroupId?: string }>;
}>;
}>;
}>(aws, ["ec2", "describe-security-groups", "--group-ids", ...databaseGroups]).SecurityGroups?.flatMap(
(group) => group.IpPermissions ?? [],
) ?? [])
: [];
const reachable = permissions.some(
(permission) =>
permission.IpProtocol === "tcp" &&
(permission.FromPort ?? Infinity) <= 5432 &&
(permission.ToPort ?? -Infinity) >= 5432 &&
(permission.UserIdGroupPairs ?? []).some((pair) => pair.GroupId && coreGroups.includes(pair.GroupId)),
);
if (!reachable) throw new Error("database security groups do not allow the core ECS service on port 5432");
const databaseUrl = awsText(aws, [
"secretsmanager",
"get-secret-value",
"--secret-id",
`${aws.secretsPrefix}DATABASE_URL`,
"--query",
"SecretString",
]);
if (!database.Endpoint?.Address || new URL(databaseUrl).hostname !== database.Endpoint.Address)
throw new Error("DATABASE_URL does not point at the configured RDS endpoint");
});
const ecsServices = new Map<string, AwsEcsRoutingService>();
for (const service of deployedAwsServices(aws)) {
const spec = aws.services[service]!;
check(`ECS service ${spec.ecsService}`, () => {
const found = awsJson<{ services?: AwsEcsRoutingService[] }>(aws, [
"ecs",
"describe-services",
"--cluster",
aws.cluster,
"--services",
spec.ecsService,
]).services?.[0];
if (found?.status !== "ACTIVE") throw new Error(`service is ${found?.status ?? "missing"}`);
ecsServices.set(service, found);
});
check(`ECR ${spec.ecrRepository}`, () => {
const repository = awsJson<{ repositories?: Array<{ repositoryArn?: string }> }>(aws, [
"ecr",
"describe-repositories",
"--repository-names",
spec.ecrRepository,
]).repositories?.[0];
if (!repository?.repositoryArn) throw new Error("repository is missing");
});
}
const logGroups = new Map<string, string[]>();
for (const [name, spec] of Object.entries(aws.services)) {
const group = spec.logGroup ?? `/ecs/${spec.ecsService}`;
logGroups.set(group, [...(logGroups.get(group) ?? []), name]);
}
for (const [group, names] of logGroups) {
check(`CloudWatch log group ${group}`, () => {
const found =
awsJson<{ logGroups?: Array<{ logGroupName?: string }> }>(aws, [
"logs",
"describe-log-groups",
"--log-group-name-prefix",
group,
]).logGroups ?? [];
if (!found.some((item) => item.logGroupName === group)) {
throw new Error(
`log group is missing — the awslogs driver refuses to start ${names.join(", ")} without it; create it or fix aws.services.${names.join("/")}.logGroup`,
);
}
});
}
check("Cloud Map routing", () => assertAwsServiceDiscovery(config, ecsServices));
check("ALB routing", () => assertAwsPublicRouting(config, ecsServices));
await checkAsync("public URL DNS and TLS", () => assertAwsPublicNetwork(config));
const runtimeSecrets = new Map<string, string>();
for (const secret of computedSecrets(config)) {
const inspect = (): void => {
const id = `${aws.secretsPrefix}${secret.name}`;
const value = awsText(aws, ["secretsmanager", "get-secret-value", "--secret-id", id, "--query", "SecretString"]);
if (isInvalidSecret(secret.name, value)) throw new Error("missing, placeholder, or insecure value");
runtimeSecrets.set(secret.name, value);
};
if (secret.required)
check(`secret ${secret.name}`, () => {
inspect();
if (secret.name === "PUBLIC_API_URL") assertAwsPublicApiUrl(config);
});
else {
try {
inspect();
step(`optional secret ${secret.name}: configured`);
} catch (error) {
if (/ResourceNotFoundException|AccessDeniedException/.test(errMessage(error)))
warn(`optional secret ${secret.name}: not configured or not readable`);
else {
failures.push(`optional secret ${secret.name}: ${errMessage(error)}`);
warn(`optional secret ${secret.name}: failed`);
}
}
}
}
if (failures.length) throw new CliError(`doctor failed:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`);
const runtimeNames = ["SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"] as const;
const priorRuntime = new Map(runtimeNames.map((name) => [name, process.env[name]]));
for (const name of runtimeNames) {
const stored = runtimeSecrets.get(name);
if (stored !== undefined) process.env[name] = stored;
else delete process.env[name];
}
try {
await doctorCommon(config, runtimeSecrets, { configDir, requiredSecretValues: true });
} finally {
for (const name of runtimeNames) {
const prior = priorRuntime.get(name);
if (prior === undefined) delete process.env[name];
else process.env[name] = prior;
}
}
ok("all AWS deployment prerequisites are ready");
}
async function checkLive(
config: QmConfig,
opts: { report?: boolean; configDir?: string; envFile?: string; sandboxDir?: string } = {},
): Promise<void> {
const configDir = opts.configDir ?? process.cwd();
const { aws, workloads: services, plugins: resolvedPlugins } = awsTopology(config, configDir);
const plugins = new Map(resolvedPlugins.map((plugin) => [plugin.name, plugin]));
assertAwsCallerAccount(aws);
const failures: string[] = [];
try {
assertAwsDeployImage(config);
} catch (error) {
failures.push(`deploy image drift: ${errMessage(error)}`);
}
const arns = secretArns(config);
const states = describedServices(config, services);
const manifest = currentDeploymentManifest(aws);
if (!manifest)
throw new CliError("live drift detected: no current AWS deployment manifest", { clause: "aws.live-drift" });
const desiredImages: Record<string, string> = {};
for (const service of services) {
const provenance = manifest.imageProvenance?.[service];
if (!provenance) continue;
const plugin = plugins.get(service);
if (provenance.kind === "source-build") {
if (plugin?.kind === "image" || (isServiceName(service) && config.imageOverrides[service])) {
failures.push(
`${service}: image build provenance drift (deployed from source, current workload uses a configured image)`,
);
}
continue;
}
if (plugin?.kind === "source") {
failures.push(
`${service}: image build provenance drift (deployed from a configured image, current workload builds from source)`,
);
continue;
}
const source = workloadSourceImage(config, service, plugin);
if (!source) {
failures.push(`${service}: configured image source is missing for prebuilt deployment`);
continue;
}
try {
desiredImages[service] = plannedWorkloadImage(config, service, plugin);
} catch (error) {
failures.push(`${service}: could not resolve desired image: ${errMessage(error)}`);
}
}
if (manifest.imageLabel !== aws.imageLabel)
failures.push(
`deployment manifest label ${manifest.imageLabel ?? "missing"} does not match configured release ${aws.imageLabel ?? "missing"}`,
);
for (const service of services) {
const state = states.get(service)!;
const expectedTask = manifest.tasks[service];
if (!expectedTask) {
failures.push(`${service}: missing from current deployment manifest ${manifest.id}`);
continue;
}
const primary = (state.deployments ?? []).filter((deployment) => deployment.status === "PRIMARY");
const wantCount = workloadDesiredCount(config, service);
if (state.status !== "ACTIVE" || state.desiredCount !== wantCount || state.runningCount !== wantCount) {
failures.push(
`${service}: runtime is ${state.status ?? "missing"} with ${state.runningCount ?? 0}/${state.desiredCount ?? 0} running, expected ${wantCount}`,
);
}
if (
primary.length !== 1 ||
primary[0]?.taskDefinition !== state.taskDefinition ||
primary[0]?.rolloutState !== "COMPLETED"
) {
failures.push(`${service}: configured task is not the sole healthy PRIMARY deployment`);
}
if (state.taskDefinition !== expectedTask)
failures.push(`${service}: service task does not match deployment manifest ${manifest.id}`);
const live =
awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
"ecs",
"describe-task-definition",
"--task-definition",
expectedTask,
]).taskDefinition ?? null;
if (!live) {
failures.push(`${service}: no live task definition`);
continue;
}
const container = (live.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
(item) => item.name === service,
);
const liveEnv = Object.fromEntries(
((container?.environment as Array<{ name: string; value: string }> | undefined) ?? []).map((item) => [
item.name,
item.value,
]),
);
const expectedEnv = workloadEnvironment(config, service, manifest.backgroundDeploymentId);
if (canonicalJson(liveEnv) !== canonicalJson(expectedEnv))
failures.push(`${service}: environment drift (including live-only keys)`);
const expectedSecrets = workloadSecrets(config, service, arns)
.flatMap((secret) => containerSecretNames(service, secret))
.sort();
const liveSecretEntries = (container?.secrets as Array<{ name: string; valueFrom?: string }> | undefined) ?? [];
const liveSecrets = liveSecretEntries.map((secret) => secret.name).sort();
if (JSON.stringify(liveSecrets) !== JSON.stringify(expectedSecrets)) failures.push(`${service}: secret-name drift`);
const repository = expectedWorkloadImageRepository(config, service);
if (typeof container?.image !== "string" || !isPinnedWorkloadImage(config, service, container.image)) {
failures.push(`${service}: live image is not a digest from ${repository}`);
} else {
const desiredImage = desiredImages[service];
if (desiredImage && container.image !== desiredImage) {
failures.push(`${service}: image drift (live ${container.image}, desired ${desiredImage})`);
}
const comparisonImage = desiredImage ?? `${repository}@sha256:${"0".repeat(64)}`;
const fields = taskDefinitionDiff(
renderTaskDefinition(config, service, comparisonImage, arns, manifest.backgroundDeploymentId),
live,
).filter((field) => desiredImage || field !== `taskDefinition.containerDefinitions.${service}.image`);
if (fields.length) failures.push(`${service}: task-definition drift (${fields.join(", ")})`);
}
}
try {
const targets = assertAwsPublicRouting(config);
assertAwsHealthyIngress(config, targets);
} catch (error) {
failures.push(`public front-door drift: ${errMessage(error)}`);
}
try {
await retryLiveProbe(() => assertAwsPublicNetwork(config, true));
} catch (error) {
failures.push(`public network drift: ${errMessage(error)}`);
}
if (!manifest.layer) {
failures.push(`deployment manifest ${manifest.id} has no deployment-layer artifact`);
} else {
try {
getDeploymentLayerArtifact(config, manifest.layer);
if (opts.sandboxDir && existsSync(opts.sandboxDir)) {
const directoryHash = createHash("sha256").update(deploymentLayerBody(opts.sandboxDir)).digest("hex");
if (directoryHash !== manifest.layer.sha256)
failures.push("deployment layer does not match the deployment directory");
}
await retryLiveProbe(async () => {
const state = await currentDeploymentLayerState({
config,
transport: awsDeploymentLayerTransport,
configDir,
...(opts.envFile ? { envFile: opts.envFile } : {}),
});
if (state.contentHash !== manifest.layer!.sha256)
throw new Error("deployment layer content does not match the current manifest");
if (state.status !== "applied" || state.runtimeContentHash !== manifest.layer!.sha256)
throw new Error("deployment layer is not applied by the live core");
});
} catch (error) {
failures.push(`deployment layer drift: ${errMessage(error)}`);
}
}
if (!failures.length) {
try {
const cohort = aws.backgroundWorkControl ? await awsBackgroundWorkStatus(config, configDir) : undefined;
if (cohort && cohort.status.ownerDeploymentId === cohort.deploymentId) {
const transport = awsBackgroundWorkTransport(config, "/v1/deployment/live-session", 600_000, 65_536);
await checkControlledLiveSession({
before: cohort,
read: () => awsBackgroundWorkStatus(config, configDir),
request: (body) => transport("POST", body),
});
} else {
await awsLiveSession(config, states.get("core")!);
}
if (opts.report ?? true) step("core: private live session smoke passed");
} catch (error) {
failures.push(`core: private live session smoke failed: ${errMessage(error)}`);
}
}
if (failures.length)
throw new CliError(`live drift detected:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`, {
clause: "aws.live-drift",
});
if (opts.report ?? true) ok("live AWS deployment matches the directory in both directions");
}
export async function awsCheckLive(
config: QmConfig,
opts: { report?: boolean; configDir?: string; envFile?: string; sandboxDir?: string } = {},
): Promise<void> {
try {
await checkLive(config, opts);
} catch (error) {
if (error instanceof CliError && error.clause === "aws.live-drift") throw error;
throw new CliError(errMessage(error), { clause: "aws.live-drift", cause: error });
}
}