Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Transfer AWS background ownership without restarting tasks (#1292)

Regan Bell33e3bf2dab59
diff --git a/cli/README.md b/cli/README.md
index ed7d16062..15df9d960 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -77,6 +77,45 @@ configuration only, so it prints that timestamp as the matching data restore
point (`aws rds restore-db-instance-to-point-in-time`);
`aws.predeployDbSnapshot: false` opts out.
+AWS deployments can opt into durable background ownership with
+`aws.backgroundWorkControl: true`. Deploy protocol-capable core images to every
+participating stack before bootstrapping ownership. The CLI allocates a unique
+`BACKGROUND_DEPLOYMENT_ID` for each replacement core cohort and records it in the
+deployment manifest. A no-op deployment and automatic ECS task replacement keep
+that identity; an explicit core restart allocates a new one. Pending preparation
+is persisted before ECS changes, and an ambiguous previous deployment must be
+reconciled before another identity can be allocated.
+
+Control requests require both `CORE_SIGNING_SECRET` and a distinct
+`DEPLOYMENT_CONTROL_SECRET` of at least 32 characters. The control secret is
+restricted to core. Configuring an identity preserves the legacy background flag
+until explicit bootstrap verifies every participating task. The exported
+`awsBootstrapBackgroundWork` adapter accepts all peer configurations and an exact
+desired deployment identity, or `null` to start paused. It refuses missing cohorts
+and inconsistent durable membership.
+
+The exported `awsBackgroundWorkBootState` reads the exact manifest core task to
+return its recorded boot flag and optional deployment identity. It returns
+`undefined` only without a recorded core task, and rejects missing or ambiguous
+boot flags or identities. Deployment wrappers can preserve the boot environment
+while changing durable ownership independently.
+
+Once bootstrapped, background mode changes use generation-checked ownership
+requests without restarting ECS tasks. Activation waits for prior owners to stop
+claiming and every expected task to finish activation. Pausing stops new claims;
+in-flight turns can continue draining. Replacing or rolling back the active core
+cohort requires an explicit pause or handover and proof that every member has
+drained first. A demotion refuses to pause a different current owner. Unresponsive members are never
+assumed dead: `awsRetireBackgroundWorkMembers` requires exact instance, task ARN,
+and generation identities plus ECS evidence that each task stopped. This recovery
+is also available before bootstrap for stopped legacy members.
+
+Core secret uploads defer activation to a subsequent staged `up --restart core`.
+The generic upload path refuses changes to either control credential while a
+controlled cohort is recorded, because replacing credentials before coordinating
+all running processes would break ownership control. Legacy AWS deployments keep
+the task replacement path when ownership control is not configured.
+
Batch operators can set `QM_DEPLOY_PROGRESS_FILE` to a new absolute file path and
`QM_DEPLOY_PROGRESS_TOKEN` to a unique attempt identifier for candidate `up --yes`.
After all forward service updates have been submitted, the CLI atomically creates
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index 76cb5f2fa..c9d78eb2d 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -1,6 +1,6 @@
import { awsCoreHostnames, awsPortalAppsDomain, validAlbHostname } from "../aws-routing.ts";
import https from "node:https";
-import { createHash, randomUUID } from "node:crypto";
+import { createHash, createHmac, randomUUID } from "node:crypto";
import { spawnSync } from "node:child_process";
import { lookup, resolveCname } from "node:dns/promises";
import {
@@ -88,6 +88,53 @@ import {
type DeploymentLayerTransport,
} from "../deployment-layer.ts";
+import {
+ awaitBackgroundWork,
+ backgroundWorkMutation,
+ mutateBackgroundWork,
+ readBackgroundWork,
+ type BackgroundWorkMember,
+ type BackgroundWorkStatus,
+ type BackgroundWorkTransport,
+} from "../background-work.ts";
+
+export async function awsCoreRequest(
+ config: QmConfig,
+ url: URL,
+ init: RequestInit,
+): Promise<{ status: number; body: string }> {
+ const target = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");
+ if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");
+ if (awsPublicOrigin(config).protocol === "http:") {
+ assertCloudFrontLayerTarget(config, url, target);
+ const response = await fetch(url, init);
+ return { status: response.status, body: await response.text() };
+ }
+ return new Promise((resolve, reject) => {
+ const request = https.request(
+ url,
+ {
+ hostname: target,
+ servername: url.hostname,
+ method: init.method,
+ headers: { ...(init.headers as Record<string, string>), host: url.host },
+ signal: init.signal ?? undefined,
+ agent: false,
+ },
+ (response) => {
+ const chunks: Buffer[] = [];
+ response.on("data", (chunk: Buffer) => chunks.push(chunk));
+ response.on("error", reject);
+ response.on("end", () =>
+ resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }),
+ );
+ },
+ );
+ request.on("error", reject);
+ request.end(init.body as string | undefined);
+ });
+}
+
export const awsDeploymentLayerTransport: DeploymentLayerTransport = httpDeploymentLayerTransport({
urlOf: (config) => {
const url = new URL(config.apiUrl ?? config.publicUrl);
@@ -96,38 +143,7 @@ export const awsDeploymentLayerTransport: DeploymentLayerTransport = httpDeploym
url.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/deployment-layer`;
return url;
},
- request: async (config, url, init) => {
- const target = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");
- if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");
- if (awsPublicOrigin(config).protocol === "http:") {
- assertCloudFrontLayerTarget(config, url, target);
- const response = await fetch(url, init);
- return { status: response.status, body: await response.text() };
- }
- return new Promise((resolve, reject) => {
- const request = https.request(
- url,
- {
- hostname: target,
- servername: url.hostname,
- method: init.method,
- headers: { ...(init.headers as Record<string, string>), host: url.host },
- signal: init.signal ?? undefined,
- agent: false,
- },
- (response) => {
- const chunks: Buffer[] = [];
- response.on("data", (chunk: Buffer) => chunks.push(chunk));
- response.on("error", reject);
- response.on("end", () =>
- resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }),
- );
- },
- );
- request.on("error", reject);
- request.end(init.body as string | undefined);
- });
- },
+ request: awsCoreRequest,
secretFallback: (config) =>
config.aws
? capture(process.env.AWS_BIN ?? "aws", [
@@ -393,8 +409,20 @@ export function serviceEnvironment(config: QmConfig, service: ServiceName): Reco
return Object.fromEntries(Object.entries(env).sort(([a], [b]) => a.localeCompare(b)));
}
-function workloadEnvironment(config: QmConfig, workload: string): Record<string, string> {
- if (isServiceName(workload)) return serviceEnvironment(config, workload);
+function workloadEnvironment(
+ config: QmConfig,
+ workload: string,
+ backgroundDeploymentId?: string,
+): Record<string, string> {
+ if (isServiceName(workload)) {
+ const env = serviceEnvironment(config, workload);
+ if (workload === "core" && requireAws(config).backgroundWorkControl) {
+ if (!backgroundDeploymentId)
+ throw new CliError("controlled background work requires a manifest-bound deployment identity");
+ env.BACKGROUND_DEPLOYMENT_ID = backgroundDeploymentId;
+ }
+ return env;
+ }
const plugin = config.plugins.find((entry) => entry.name === workload);
return Object.fromEntries(
Object.entries({
@@ -434,6 +462,7 @@ export function renderTaskDefinition(
service: string,
image: string,
secretArns?: Record<string, string>,
+ backgroundDeploymentId?: string,
): EcsTaskDefinition {
if (!isDigestPinned(image)) throw new CliError(`aws task image for ${service} must be pinned by digest`);
const aws = requireAws(config);
@@ -484,7 +513,9 @@ export function renderTaskDefinition(
name: service,
image,
essential: true,
- environment: Object.entries(workloadEnvironment(config, service)).map(([name, value]) => ({ name, value })),
+ environment: Object.entries(workloadEnvironment(config, service, backgroundDeploymentId)).map(
+ ([name, value]) => ({ name, value }),
+ ),
secrets,
portMappings: [
{
@@ -848,6 +879,7 @@ async function runAwsCandidateMigration(
export function secretArns(config: QmConfig): Record<string, string> {
const aws = requireAws(config);
+ const controlValues = new Map<string, string>();
const pairs = computedSecrets(config).flatMap((secret) => {
const id = `${aws.secretsPrefix}${secret.name}`;
try {
@@ -871,12 +903,19 @@ export function secretArns(config: QmConfig): Record<string, string> {
if (!value.ARN || isInvalidSecret(secret.name, value.SecretString)) {
throw new CliError(`required AWS secret ${secret.name} has no usable, non-placeholder AWSCURRENT value`);
}
+ if (secret.name === "CORE_SIGNING_SECRET" || secret.name === "DEPLOYMENT_CONTROL_SECRET")
+ controlValues.set(secret.name, value.SecretString!);
return [[secret.name, value.ARN] as const];
} catch (error) {
if (!secret.required && /ResourceNotFoundException/.test(errMessage(error))) return [];
throw error;
}
});
+ if (
+ aws.backgroundWorkControl &&
+ controlValues.get("CORE_SIGNING_SECRET") === controlValues.get("DEPLOYMENT_CONTROL_SECRET")
+ )
+ throw new CliError("DEPLOYMENT_CONTROL_SECRET must differ from CORE_SIGNING_SECRET");
return Object.fromEntries(pairs);
}
@@ -1176,6 +1215,7 @@ interface AwsReleaseCandidate {
interface DeploymentManifest {
id: string;
+ backgroundDeploymentId?: string;
previous?: string;
createdAt: string;
imageLabel?: string;
@@ -1187,6 +1227,93 @@ interface DeploymentManifest {
layer?: { key: string; sha256: string };
}
+interface BackgroundDeploymentPreparation {
+ id: string;
+ previousManifestId?: string;
+ backgroundDeploymentId: string;
+ before: { tasks: Record<string, string>; counts: Record<string, number> };
+}
+
+const BACKGROUND_PREPARATION_KEY = "deployment/background-preparation";
+
+function clearBackgroundPreparation(aws: AwsConfig): void {
+ awsText(aws, [
+ "dynamodb",
+ "transact-write-items",
+ "--transact-items",
+ JSON.stringify([
+ {
+ Delete: {
+ TableName: deployLocksTable(aws),
+ Key: { lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } },
+ },
+ },
+ ]),
+ ]);
+}
+
+function reconcileBackgroundPreparation(
+ aws: AwsConfig,
+ current: DeploymentManifest | undefined,
+ before: BackgroundDeploymentPreparation["before"],
+): void {
+ const response = awsJson<{ Item?: Record<string, { S?: string }> }>(aws, [
+ "dynamodb",
+ "get-item",
+ "--table-name",
+ deployLocksTable(aws),
+ "--key",
+ JSON.stringify({ lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } }),
+ "--consistent-read",
+ ]);
+ if (!response.Item) return;
+ let preparation: BackgroundDeploymentPreparation;
+ try {
+ preparation = JSON.parse(response.Item.preparation?.S ?? "");
+ if (
+ !preparation ||
+ typeof preparation.id !== "string" ||
+ typeof preparation.backgroundDeploymentId !== "string" ||
+ !preparation.before?.tasks ||
+ !preparation.before.counts
+ )
+ throw new Error("invalid preparation");
+ } catch {
+ throw new CliError("pending background deployment preparation is invalid; refusing a new cohort");
+ }
+ const committed =
+ current?.id === preparation.id &&
+ current.backgroundDeploymentId === preparation.backgroundDeploymentId &&
+ canonicalJson(before) === canonicalJson({ tasks: current.tasks, counts: current.counts });
+ const restored =
+ current?.id === preparation.previousManifestId && canonicalJson(before) === canonicalJson(preparation.before);
+ if (!committed && !restored)
+ throw new CliError(
+ `background deployment ${preparation.id} remains unresolved; reconcile its recorded tasks before retrying instead of allocating another cohort`,
+ );
+ clearBackgroundPreparation(aws);
+}
+
+function prepareBackgroundDeployment(aws: AwsConfig, preparation: BackgroundDeploymentPreparation): void {
+ awsText(aws, [
+ "dynamodb",
+ "transact-write-items",
+ "--transact-items",
+ JSON.stringify([
+ {
+ Put: {
+ TableName: deployLocksTable(aws),
+ Item: {
+ lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) },
+ preparation: { S: JSON.stringify(preparation) },
+ },
+ ConditionExpression: "attribute_not_exists(lockKey)",
+ },
+ },
+ ]),
+ ]);
+}
+
const DEPLOYMENT_POINTER_KEY = "deployment/current";
const DEPLOYMENT_MANIFEST_PREFIX = "deployment/manifest/";
const ECS_SERVICE_BATCH_SIZE = 10;
@@ -1546,12 +1673,16 @@ function recordDeploymentManifest(
layer?: DeploymentManifest["layer"];
imageProvenance?: DeploymentManifest["imageProvenance"];
counts?: DeploymentManifest["counts"];
+ backgroundDeploymentId?: string;
},
): DeploymentManifest {
const current = currentDeploymentManifest(aws);
const counts = release.counts ?? current?.counts;
const manifest: DeploymentManifest = {
id: release.id ?? randomUUID(),
+ ...((release.backgroundDeploymentId ?? current?.backgroundDeploymentId)
+ ? { backgroundDeploymentId: release.backgroundDeploymentId ?? current?.backgroundDeploymentId }
+ : {}),
...(current ? { previous: current.id } : {}),
createdAt: new Date().toISOString(),
...(release.imageLabel ? { imageLabel: release.imageLabel } : {}),
@@ -1852,10 +1983,11 @@ function reportTaskChanges(
images: Record<string, string>,
arns: Record<string, string>,
restart: ReadonlySet<string> = new Set(),
+ backgroundDeploymentId?: string,
): Array<{ service: string; task: EcsTaskDefinition; changed: boolean }> {
const desired = services.map((service) => ({
service,
- task: renderTaskDefinition(config, service, images[service]!, arns),
+ task: renderTaskDefinition(config, service, images[service]!, arns, backgroundDeploymentId),
live: liveTask(config, service),
}));
return desired.map((item) => {
@@ -2159,7 +2291,11 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
}
images[service] = plannedWorkloadImage(config, service, plugins.get(service));
}
- reportTaskChanges(config, services, images, arns, restart);
+ const backgroundDeploymentId = aws.backgroundWorkControl
+ ? (currentDeploymentManifest(aws)?.backgroundDeploymentId ??
+ `${aws.services.core!.ecsService}:00000000-0000-0000-0000-000000000000`)
+ : undefined;
+ reportTaskChanges(config, services, images, arns, restart, backgroundDeploymentId);
for (const service of services) {
step(`${service}: desired count ${before.counts[service] ?? 0} → ${workloadDesiredCount(config, service)}`);
}
@@ -2201,6 +2337,14 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
assertAwsDeployImage(config);
before = serviceSnapshot(config, allServices);
current = currentDeploymentManifest(aws);
+ if (current?.backgroundDeploymentId && !aws.backgroundWorkControl)
+ throw new CliError("cannot disable background ownership control while a controlled deployment is recorded");
+ if (aws.backgroundWorkControl) {
+ if (!current?.backgroundDeploymentId && !services.includes("core"))
+ throw new CliError("enabling background ownership requires deploying core to allocate its identity");
+ reconcileBackgroundPreparation(aws, current, before);
+ if (services.includes("core")) await assertBackgroundCohortReplaceable(config, current);
+ }
const selected = new Set(services);
if (allServices.some((service) => !selected.has(service))) {
current = trustedDeploymentBaseline(
@@ -2271,7 +2415,21 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
images[service] = await publishWorkloadImage(config, service, plugins.get(service), stagingLabel, opts);
}
}
- const desired = reportTaskChanges(config, services, images, arns, restart);
+ let backgroundDeploymentId = current?.backgroundDeploymentId;
+ if (aws.backgroundWorkControl && !backgroundDeploymentId)
+ backgroundDeploymentId = `${aws.services.core!.ecsService}:${releaseId}`;
+ const desired = reportTaskChanges(config, services, images, arns, restart, backgroundDeploymentId);
+ const changedCore = desired.find((item) => item.service === "core" && item.changed);
+ if (aws.backgroundWorkControl && changedCore) {
+ backgroundDeploymentId = `${aws.services.core!.ecsService}:${releaseId}`;
+ changedCore.task = renderTaskDefinition(config, "core", images.core!, arns, backgroundDeploymentId);
+ prepareBackgroundDeployment(aws, {
+ id: releaseId,
+ previousManifestId: current?.id,
+ backgroundDeploymentId,
+ before,
+ });
+ }
const targets: Record<string, string> = {};
for (const item of desired) {
if (!item.changed) {
@@ -2352,6 +2510,7 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
) {
recorded = recordDeploymentManifest(aws, releaseTasks, {
id: releaseId,
+ ...(backgroundDeploymentId ? { backgroundDeploymentId } : {}),
counts: releaseCounts,
imageLabel: label,
...(dbRestorePoint ? { dbRestorePoint } : {}),
@@ -2359,6 +2518,7 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt
imageProvenance: releaseImageProvenance,
});
}
+ if (aws.backgroundWorkControl && changedCore) clearBackgroundPreparation(aws);
if (recorded && dbRestorePoint) confirmDbRestorePointCovered(config, dbRestorePoint);
for (const service of candidate ? [] : services) {
try {
@@ -2532,6 +2692,11 @@ export async function awsRollback(
} else {
targetManifest = deploymentManifestForTarget(aws, to);
}
+ if (currentManifest?.backgroundDeploymentId || targetManifest.backgroundDeploymentId) {
+ if (!aws.backgroundWorkControl || !targetManifest.backgroundDeploymentId)
+ throw new CliError("controlled rollback requires a protocol-capable target deployment");
+ await assertBackgroundCohortReplaceable(config, currentManifest);
+ }
const missing = services.filter((service) => !targetManifest.tasks[service]);
if (missing.length)
throw new CliError(`target AWS deployment manifest is missing workloads: ${missing.join(", ")}`);
@@ -2674,6 +2839,339 @@ export function taskDefinitionForBackgroundWork(
};
}
+function awsBackgroundWorkTransport(config: QmConfig): BackgroundWorkTransport {
+ const aws = requireAws(config);
+ const secret = (name: string): string =>
+ awsText(aws, [
+ "secretsmanager",
+ "get-secret-value",
+ "--secret-id",
+ `${aws.secretsPrefix}${name}`,
+ "--query",
+ "SecretString",
+ ]);
+ const signing = secret("CORE_SIGNING_SECRET");
+ const control = secret("DEPLOYMENT_CONTROL_SECRET");
+ if (isInvalidSecret("CORE_SIGNING_SECRET", signing) || control.trim().length < 32 || control === signing)
+ throw new CliError("deployment control requires a distinct secret of at least 32 characters");
+ const url = new URL(config.publicUrl);
+ url.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/background-work`;
+ url.search = "";
+ return (method, body = "") => {
+ const timestamp = Math.floor(Date.now() / 1000);
+ const signature = createHmac("sha256", signing)
+ .update(`v0:${timestamp}:${method}\n${url.pathname}\n${body}`)
+ .digest("hex");
+ return awsCoreRequest(config, url, {
+ method,
+ headers: {
+ "content-type": "application/json",
+ "x-timestamp": String(timestamp),
+ "x-signature": `v0=${signature}`,
+ authorization: `Bearer ${control}`,
+ },
+ ...(method === "POST" ? { body } : {}),
+ signal: AbortSignal.timeout(30_000),
+ });
+ };
+}
+
+interface AwsBackgroundCohort {
+ deploymentId: string;
+ taskArns: string[];
+ manifest: DeploymentManifest;
+}
+
+export function awsBackgroundWorkBootState(config: QmConfig): { enabled: boolean; deploymentId?: string } | undefined {
+ const aws = requireAws(config);
+ assertAwsCallerAccount(aws);
+ const manifest = currentDeploymentManifest(aws);
+ const taskDefinition = manifest?.tasks.core;
+ if (!taskDefinition) return undefined;
+ const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
+ "ecs",
+ "describe-task-definition",
+ "--task-definition",
+ taskDefinition,
+ ]).taskDefinition;
+ const containers = task?.containerDefinitions as Array<Record<string, unknown>> | undefined;
+ const core = containers?.filter((container) => container.name === "core");
+ if (core?.length !== 1) throw new CliError("recorded task definition must contain exactly one core container");
+ const secrets = (core[0]!.secrets ?? []) as Array<{ name: string }>;
+ if (secrets.some((secret) => secret.name === "BACKGROUND_WORK_ENABLED" || secret.name === "BACKGROUND_DEPLOYMENT_ID"))
+ throw new CliError("recorded background boot settings cannot be supplied as secrets");
+ const environment = (core[0]!.environment ?? []) as Array<{ name: string; value: string }>;
+ const identities = environment.filter((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID");
+ if (
+ manifest.backgroundDeploymentId
+ ? identities.length !== 1 || identities[0]!.value !== manifest.backgroundDeploymentId
+ : identities.length !== 0
+ )
+ throw new CliError("recorded core task does not match the manifest background deployment identity");
+ const flags = environment.filter((entry) => entry.name === "BACKGROUND_WORK_ENABLED");
+ if (flags.length !== 1 || !["0", "1"].includes(flags[0]!.value))
+ throw new CliError("recorded core task must contain one explicit BACKGROUND_WORK_ENABLED value of 0 or 1");
+ return {
+ enabled: flags[0]!.value === "1",
+ ...(manifest.backgroundDeploymentId ? { deploymentId: manifest.backgroundDeploymentId } : {}),
+ };
+}
+
+async function awsBackgroundCohort(
+ config: QmConfig,
+ configDir: string,
+ candidatePath?: string,
+): Promise<AwsBackgroundCohort> {
+ const { aws, workloads } = awsTopology(config, configDir);
+ if (!aws.backgroundWorkControl)
+ throw new CliError("background ownership control is not enabled in the AWS configuration");
+ const manifest = currentDeploymentManifest(aws);
+ if (!manifest?.backgroundDeploymentId)
+ throw new CliError("background ownership requires a manifest-bound deployment identity");
+ const candidate = candidatePath ? releaseCandidate(config, candidatePath) : undefined;
+ const states = describedServices(config, workloads);
+ assertOwnedServices(config, states, workloads);
+ const before = serviceSnapshotFromStates(states, workloads);
+ for (const workload of workloads) {
+ if (manifest.tasks[workload] !== before.tasks[workload] || manifest.counts?.[workload] !== before.counts[workload])
+ throw new CliError(`cannot control background work while ${workload} differs from the deployment manifest`);
+ const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [
+ "ecs",
+ "describe-task-definition",
+ "--task-definition",
+ before.tasks[workload]!,
+ ]).taskDefinition;
+ const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(
+ (item) => item.name === workload,
+ );
+ if (typeof container?.image !== "string" || !isPinnedWorkloadImage(config, workload, container.image))
+ throw new CliError(`${workload} does not have a trusted digest-pinned image`);
+ if (
+ candidate &&
+ (container.image !== candidate.images[workload] ||
+ !candidate.imageProvenance[workload] ||
+ canonicalJson(candidate.imageProvenance[workload]) !== canonicalJson(manifest.imageProvenance?.[workload]))
+ )
+ throw new CliError(`${workload} does not match the expected release candidate`);
+ if (workload === "core") {
+ const environment = (container.environment ?? []) as Array<{ name: string; value: string }>;
+ if (
+ environment.find((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID")?.value !==
+ manifest.backgroundDeploymentId
+ )
+ throw new CliError("core task definition does not match the recorded background deployment identity");
+ }
+ }
+ const count = before.counts.core;
+ if (!count) throw new CliError("background ownership requires running core capacity");
+ await awaitServiceTargets(config, {
+ core: { taskDefinition: before.tasks.core!, desiredCount: count, waitForDrain: false },
+ });
+ const taskArns =
+ awsJson<{ taskArns?: string[] }>(aws, [
+ "ecs",
+ "list-tasks",
+ "--cluster",
+ aws.cluster,
+ "--service-name",
+ aws.services.core!.ecsService,
+ "--desired-status",
+ "RUNNING",
+ ]).taskArns ?? [];
+ const tasks = chunks(taskArns, 100).flatMap((batch) => {
+ const response = awsJson<{
+ tasks?: Array<{ taskArn?: string; taskDefinitionArn?: string; lastStatus?: string; healthStatus?: string }>;
+ failures?: unknown[];
+ }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);
+ if (response.failures?.length) throw new CliError("cannot prove every running core task identity");
+ return response.tasks ?? [];
+ });
+ const current = tasks.filter(
+ (task) =>
+ task.taskDefinitionArn === before.tasks.core && task.lastStatus === "RUNNING" && task.healthStatus === "HEALTHY",
+ );
+ if (
+ current.length !== count ||
+ current.some((task) => !task.taskArn) ||
+ new Set(current.map((task) => task.taskArn)).size !== count
+ )
+ throw new CliError("core does not have the exact healthy task cohort recorded in its deployment manifest");
+ return { deploymentId: manifest.backgroundDeploymentId, taskArns: current.map((task) => task.taskArn!), manifest };
+}
+
+export async function awsBackgroundWorkStatus(
+ config: QmConfig,
+ configDir: string,
+ candidatePath?: string,
+): Promise<{
+ status: BackgroundWorkStatus;
+ deploymentId: string;
+ taskArns: string[];
+}> {
+ assertAwsCallerAccount(requireAws(config));
+ const cohort = await awsBackgroundCohort(config, configDir, candidatePath);
+ const status = await readBackgroundWork(awsBackgroundWorkTransport(config), cohort.deploymentId);
+ for (const taskArn of cohort.taskArns) {
+ if (
+ !status.members.some(
+ (member) => member.taskArn === taskArn && member.deploymentId === cohort.deploymentId && !member.retired,
+ )
+ )
+ throw new CliError("a running core task has not enrolled in the ownership protocol");
+ }
+ return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns };
+}
+
+export interface AwsBackgroundWorkPeer {
+ config: QmConfig;
+ configDir: string;
+ candidatePath?: string;
+}
+
+async function withBackgroundPeerLeases<T>(peers: AwsBackgroundWorkPeer[], operation: () => Promise<T>): Promise<T> {
+ const unique = new Map(
+ peers.map((peer) => {
+ const aws = requireAws(peer.config);
+ return [
+ `${aws.accountId}:${aws.region}:${deployLocksTable(aws)}:${deploymentStateKey(aws, "deploy")}`,
+ aws,
+ ] as const;
+ }),
+ );
+ if (unique.size !== peers.length)
+ throw new CliError("background ownership peers must have distinct deployment leases");
+ const leases: Array<{ aws: AwsConfig; lease: ReturnType<typeof acquireLease> }> = [];
+ try {
+ for (const [, aws] of [...unique.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+ assertAwsCallerAccount(aws);
+ leases.push({ aws, lease: acquireLease(aws) });
+ }
+ return await operation();
+ } finally {
+ for (const { aws, lease } of leases.reverse()) releaseLease(aws, lease);
+ }
+}
+
+export async function awsBootstrapBackgroundWork(
+ peers: AwsBackgroundWorkPeer[],
+ desiredDeploymentId: string | null,
+): Promise<BackgroundWorkStatus> {
+ if (!peers.length) throw new CliError("background ownership bootstrap requires every participating deployment");
+ return withBackgroundPeerLeases(peers, async () => {
+ const cohorts = [];
+ for (const peer of peers)
+ cohorts.push(await awsBackgroundWorkStatus(peer.config, peer.configDir, peer.candidatePath));
+ const first = cohorts[0]!;
+ if (cohorts.some((cohort) => cohort.status.enabled || cohort.status.generation !== 0))
+ throw new CliError("background ownership bootstrap requires disabled generation zero for every cohort");
+ if (new Set(cohorts.map((cohort) => cohort.deploymentId)).size !== cohorts.length)
+ throw new CliError("background ownership bootstrap requires distinct cohort identities");
+ const shared = (status: BackgroundWorkStatus): string =>
+ canonicalJson({
+ generation: status.generation,
+ desiredDeploymentId: status.desiredDeploymentId,
+ members: [...status.members].sort((a, b) => a.instanceId.localeCompare(b.instanceId)),
+ });
+ if (cohorts.some((cohort) => shared(cohort.status) !== shared(first.status)))
+ throw new CliError("background ownership peers do not agree on shared durable membership");
+ const taskArns = cohorts.flatMap((cohort) => cohort.taskArns);
+ if (
+ new Set(taskArns).size !== taskArns.length ||
+ first.status.members.some((member) => !member.retired && (!member.taskArn || !taskArns.includes(member.taskArn)))
+ )
+ throw new CliError("bootstrap must include every enrolled non-retired task");
+ const desired =
+ desiredDeploymentId === null ? undefined : cohorts.find((cohort) => cohort.deploymentId === desiredDeploymentId);
+ if (desiredDeploymentId !== null && !desired)
+ throw new CliError("bootstrap desired owner is not a verified peer cohort");
+ const transport = awsBackgroundWorkTransport(peers[0]!.config);
+ const mutation = { ...backgroundWorkMutation(0, desiredDeploymentId), bootstrapTaskArns: taskArns };
+ const applied = await mutateBackgroundWork(transport, first.deploymentId, mutation);
+ return awaitBackgroundWork(
+ transport,
+ first.deploymentId,
+ { generation: applied.generation, desiredDeploymentId, taskArns: desired?.taskArns ?? [] },
+ { timeoutMs: envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000), pollMs: 1000 },
+ );
+ });
+}
+
+export async function awsRetireBackgroundWorkMembers(
+ peers: AwsBackgroundWorkPeer[],
+ terminatedMembers: Array<Pick<BackgroundWorkMember, "instanceId" | "taskArn" | "generation">>,
+): Promise<BackgroundWorkStatus> {
+ if (!peers.length || !terminatedMembers.length)
+ throw new CliError("task retirement requires deployment peers and exact member identities");
+ return withBackgroundPeerLeases(peers, async () => {
+ const first = await awsBackgroundWorkStatus(peers[0]!.config, peers[0]!.configDir, peers[0]!.candidatePath);
+ for (const retired of terminatedMembers) {
+ const member = first.status.members.find(
+ (item) =>
+ item.instanceId === retired.instanceId &&
+ item.taskArn === retired.taskArn &&
+ item.generation === retired.generation,
+ );
+ if (!member?.taskArn || member.retired)
+ throw new CliError("task retirement does not match a live durable member");
+ let proved = false;
+ for (const peer of peers) {
+ const aws = requireAws(peer.config);
+ if (!member.deploymentId.startsWith(`${aws.services.core!.ecsService}:`)) continue;
+ const response = awsJson<{
+ tasks?: Array<{ taskArn?: string; taskDefinitionArn?: string; lastStatus?: string; group?: string }>;
+ failures?: unknown[];
+ }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", member.taskArn]);
+ const task = response.tasks?.find((item) => item.taskArn === member.taskArn);
+ if (
+ response.failures?.length ||
+ !task ||
+ task.lastStatus !== "STOPPED" ||
+ task.group !== `service:${aws.services.core!.ecsService}` ||
+ !task.taskDefinitionArn
+ )
+ continue;
+ const definition = awsJson<{
+ taskDefinition?: {
+ containerDefinitions?: Array<{ name?: string; environment?: Array<{ name: string; value: string }> }>;
+ };
+ }>(aws, ["ecs", "describe-task-definition", "--task-definition", task.taskDefinitionArn]).taskDefinition;
+ const identity = definition?.containerDefinitions
+ ?.find((container) => container.name === "core")
+ ?.environment?.find((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID")?.value;
+ if (identity === member.deploymentId) proved = true;
+ }
+ if (!proved)
+ throw new CliError(
+ "task retirement requires ECS STOPPED evidence bound to the exact service, task and deployment identity",
+ );
+ }
+ return mutateBackgroundWork(awsBackgroundWorkTransport(peers[0]!.config), first.deploymentId, {
+ expectedGeneration: first.status.generation,
+ requestId: randomUUID(),
+ terminatedMembers,
+ });
+ });
+}
+
+async function assertBackgroundCohortReplaceable(
+ config: QmConfig,
+ current: DeploymentManifest | undefined,
+): Promise<void> {
+ if (!current?.backgroundDeploymentId) return;
+ const status = await readBackgroundWork(awsBackgroundWorkTransport(config), current.backgroundDeploymentId);
+ if (
+ status.enabled &&
+ (status.desiredDeploymentId === current.backgroundDeploymentId ||
+ status.members.some(
+ (member) =>
+ !member.retired && member.deploymentId === current.backgroundDeploymentId && member.state !== "drained",
+ ))
+ )
+ throw new CliError(
+ "pause or hand over background ownership and wait for every member to drain before replacing the current core cohort",
+ );
+}
+
export async function awsSetBackgroundWork(
config: QmConfig,
configDir: string,
@@ -2723,6 +3221,32 @@ export async function awsSetBackgroundWork(
}
if (workload === "core") coreTask = task;
}
+ if (aws.backgroundWorkControl) {
+ const cohort = await awsBackgroundCohort(config, configDir, candidatePath);
+ const transport = awsBackgroundWorkTransport(config);
+ let status = await readBackgroundWork(transport, cohort.deploymentId);
+ if (!status.enabled)
+ throw new CliError("explicitly bootstrap all background ownership cohorts before changing ownership");
+ if (!enabled && status.desiredDeploymentId !== null && status.desiredDeploymentId !== cohort.deploymentId)
+ throw new CliError("cannot disable background work on a different deployment's current owner");
+ const desiredDeploymentId = enabled ? cohort.deploymentId : null;
+ if (status.desiredDeploymentId !== desiredDeploymentId) {
+ status = await mutateBackgroundWork(
+ transport,
+ cohort.deploymentId,
+ backgroundWorkMutation(status.generation, desiredDeploymentId),
+ );
+ }
+ await awaitBackgroundWork(
+ transport,
+ cohort.deploymentId,
+ { generation: status.generation, desiredDeploymentId, taskArns: enabled ? cohort.taskArns : [] },
+ { timeoutMs: envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000), pollMs: 1000 },
+ );
+ return;
+ }
+ if (current.backgroundDeploymentId)
+ throw new CliError("controlled background work cannot fall back to task replacement");
const desired = taskDefinitionForBackgroundWork(coreTask!, enabled);
const core = (coreTask!.containerDefinitions as Array<Record<string, unknown>>).find(
(item) => item.name === "core",
@@ -2820,6 +3344,8 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFil
}
await withAwsLease(aws, async () => {
const baseline = currentDeploymentManifest(aws);
+ if (baseline?.backgroundDeploymentId && !aws.backgroundWorkControl)
+ throw new CliError("controlled secret activation cannot fall back to a legacy task definition");
const states = describedServices(config, workloads);
assertOwnedServices(config, states, workloads);
const before = baseline ? serviceSnapshotFromStates(states, workloads) : undefined;
@@ -2846,6 +3372,24 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFil
const affected = workloads.filter((workload) =>
workloadSecrets(config, workload, uploaded).some((secret) => uploaded[secret.name]),
);
+ if (baseline?.backgroundDeploymentId) {
+ for (const secret of staged.filter((item) =>
+ ["DEPLOYMENT_CONTROL_SECRET", "CORE_SIGNING_SECRET"].includes(item.name),
+ )) {
+ const existing = awsText(aws, [
+ "secretsmanager",
+ "get-secret-value",
+ "--secret-id",
+ secret.id,
+ "--query",
+ "SecretString",
+ ]);
+ if (existing !== readFileSync(secret.file, "utf8"))
+ throw new CliError(
+ `${secret.name} cannot be rotated while controlled task cohorts are deployed; coordinated control credential rotation is required`,
+ );
+ }
+ }
for (const secret of staged) {
try {
awsText(aws, [
@@ -2870,6 +3414,12 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFil
if (affected.length) step("secret activation deferred to the first complete AWS deployment");
return;
}
+ if (aws.backgroundWorkControl && affected.includes("core")) {
+ note(
+ "core secret activation deferred: pause or hand over background ownership, then run qm up --restart core to allocate a new cohort",
+ );
+ affected.splice(affected.indexOf("core"), 1);
+ }
for (const [component, host, marker, expected] of [
["admin", "web-ui", "ADMIN_ENABLED", "1"],
["auth", "portal", "AUTH_EMBEDDED", "1"],
@@ -2912,7 +3462,7 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFil
) {
throw new CliError(`cannot rotate secrets while ${workload} lacks a trusted digest-pinned image`);
}
- const desired = renderTaskDefinition(config, workload, container.image, arns);
+ const desired = renderTaskDefinition(config, workload, container.image, arns, baseline.backgroundDeploymentId);
if (!taskDefinitionChanges(desired, live).length) continue;
const dir = mkdtempSync(join(tmpdir(), "qm-task-"));
try {
@@ -4105,7 +4655,7 @@ async function checkLive(
item.value,
]),
);
- const expectedEnv = workloadEnvironment(config, service);
+ const expectedEnv = workloadEnvironment(config, service, manifest.backgroundDeploymentId);
if (canonicalJson(liveEnv) !== canonicalJson(expectedEnv))
failures.push(`${service}: environment drift (including live-only keys)`);
const expectedSecrets = workloadSecrets(config, service, arns)
@@ -4123,9 +4673,10 @@ async function checkLive(
failures.push(`${service}: image drift (live ${container.image}, desired ${desiredImage})`);
}
const comparisonImage = desiredImage ?? `${repository}@sha256:${"0".repeat(64)}`;
- const fields = taskDefinitionDiff(renderTaskDefinition(config, service, comparisonImage, arns), live).filter(
- (field) => desiredImage || field !== `taskDefinition.containerDefinitions.${service}.image`,
- );
+ const fields = taskDefinitionDiff(
+ renderTaskDefinition(config, service, comparisonImage, arns, manifest.backgroundDeploymentId),
+ live,
+ ).filter((field) => desiredImage || field !== `taskDefinition.containerDefinitions.${service}.image`);
if (fields.length) failures.push(`${service}: task-definition drift (${fields.join(", ")})`);
}
}
diff --git a/cli/src/background-work.ts b/cli/src/background-work.ts
new file mode 100644
index 000000000..1d7e50508
--- /dev/null
+++ b/cli/src/background-work.ts
@@ -0,0 +1,211 @@
+import { randomUUID } from "node:crypto";
+import { CliError } from "./log.ts";
+import { sleep } from "./util.ts";
+
+export interface BackgroundWorkMember {
+ instanceId: string;
+ taskArn: string | null;
+ deploymentId: string;
+ generation: number;
+ state: "admitted" | "relinquished" | "drained";
+ retired: boolean;
+ ready: boolean;
+}
+
+export interface BackgroundWorkStatus {
+ protocol: 1;
+ enabled: boolean;
+ deploymentId: string;
+ instanceId: string;
+ generation: number;
+ desiredDeploymentId: string | null;
+ lastRequestId: string | null;
+ members: BackgroundWorkMember[];
+}
+
+export type BackgroundWorkTransport = (
+ method: "GET" | "POST",
+ body?: string,
+) => Promise<{ status: number; body: string }>;
+
+export type BackgroundWorkMutation = {
+ expectedGeneration: number;
+ requestId: string;
+} & (
+ | { desiredDeploymentId: string | null; bootstrapTaskArns?: string[] }
+ | { terminatedMembers: Array<Pick<BackgroundWorkMember, "instanceId" | "taskArn" | "generation">> }
+);
+
+function record(value: unknown): value is Record<string, unknown> {
+ return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function text(value: unknown): value is string {
+ return typeof value === "string" && value.length > 0;
+}
+
+function generation(value: unknown): value is number {
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+
+export function parseBackgroundWorkStatus(body: string, deploymentId: string): BackgroundWorkStatus {
+ let value: unknown;
+ try {
+ value = JSON.parse(body);
+ } catch {
+ throw new CliError("background ownership returned invalid JSON");
+ }
+ if (
+ !record(value) ||
+ value.protocol !== 1 ||
+ typeof value.enabled !== "boolean" ||
+ value.deploymentId !== deploymentId ||
+ !text(value.instanceId) ||
+ !generation(value.generation) ||
+ !(value.desiredDeploymentId === null || text(value.desiredDeploymentId)) ||
+ !(value.lastRequestId === null || text(value.lastRequestId)) ||
+ !Array.isArray(value.members)
+ ) {
+ throw new CliError("background ownership response does not match the requested deployment and protocol");
+ }
+ const members = value.members;
+ for (const member of members) {
+ if (
+ !record(member) ||
+ !text(member.instanceId) ||
+ !(member.taskArn === null || text(member.taskArn)) ||
+ !text(member.deploymentId) ||
+ !generation(member.generation) ||
+ member.generation > value.generation ||
+ !["admitted", "relinquished", "drained"].includes(String(member.state)) ||
+ typeof member.retired !== "boolean" ||
+ typeof member.ready !== "boolean"
+ ) {
+ throw new CliError("background ownership returned an invalid member");
+ }
+ }
+ const identities = members.map((member) => member.instanceId);
+ if (new Set(identities).size !== identities.length)
+ throw new CliError("background ownership returned duplicate instance identities");
+ if (
+ !members.some(
+ (member) => member.instanceId === value.instanceId && member.deploymentId === deploymentId && !member.retired,
+ )
+ )
+ throw new CliError("background ownership responder is not enrolled in its durable membership");
+ return value as unknown as BackgroundWorkStatus;
+}
+
+export async function readBackgroundWork(
+ transport: BackgroundWorkTransport,
+ deploymentId: string,
+): Promise<BackgroundWorkStatus> {
+ const response = await transport("GET");
+ if (response.status !== 200)
+ throw new CliError(`background ownership read failed with HTTP ${response.status}; legacy fallback is forbidden`);
+ return parseBackgroundWorkStatus(response.body, deploymentId);
+}
+
+function mutationCommitted(state: BackgroundWorkStatus, mutation: BackgroundWorkMutation): boolean {
+ if (state.lastRequestId !== mutation.requestId) return false;
+ if ("desiredDeploymentId" in mutation) {
+ return (
+ state.generation === mutation.expectedGeneration + 1 &&
+ state.desiredDeploymentId === mutation.desiredDeploymentId &&
+ state.enabled
+ );
+ }
+ return (
+ state.generation === mutation.expectedGeneration &&
+ mutation.terminatedMembers.every((retired) =>
+ state.members.some(
+ (member) =>
+ member.instanceId === retired.instanceId &&
+ member.taskArn === retired.taskArn &&
+ member.generation === retired.generation &&
+ member.retired,
+ ),
+ )
+ );
+}
+
+export async function mutateBackgroundWork(
+ transport: BackgroundWorkTransport,
+ deploymentId: string,
+ mutation: BackgroundWorkMutation,
+): Promise<BackgroundWorkStatus> {
+ const body = JSON.stringify(mutation);
+ let response: { status: number; body: string } | undefined;
+ try {
+ response = await transport("POST", body);
+ } catch {
+ response = undefined;
+ }
+ if (response?.status === 200) {
+ try {
+ const state = parseBackgroundWorkStatus(response.body, deploymentId);
+ if (mutationCommitted(state, mutation)) return state;
+ } catch {
+ response = undefined;
+ }
+ }
+ const observed = await readBackgroundWork(transport, deploymentId);
+ if (mutationCommitted(observed, mutation)) return observed;
+ if (observed.generation !== mutation.expectedGeneration)
+ throw new CliError("background ownership changed concurrently; refusing to replace another ownership generation");
+ throw new CliError(
+ `background ownership mutation is unconfirmed${response ? ` (HTTP ${response.status})` : ""}; retry the same request ID ${mutation.requestId} after reading ownership`,
+ );
+}
+
+export function backgroundWorkMutation(
+ expectedGeneration: number,
+ desiredDeploymentId: string | null,
+): BackgroundWorkMutation {
+ return { expectedGeneration, desiredDeploymentId, requestId: randomUUID() };
+}
+
+export async function awaitBackgroundWork(
+ transport: BackgroundWorkTransport,
+ deploymentId: string,
+ expected: { generation: number; desiredDeploymentId: string | null; taskArns: string[] },
+ options: { timeoutMs: number; pollMs: number },
+): Promise<BackgroundWorkStatus> {
+ const deadline = Date.now() + options.timeoutMs;
+ for (;;) {
+ const state = await readBackgroundWork(transport, deploymentId);
+ if (
+ !state.enabled ||
+ state.generation !== expected.generation ||
+ state.desiredDeploymentId !== expected.desiredDeploymentId
+ ) {
+ throw new CliError("background ownership changed while awaiting acknowledgment");
+ }
+ const priorOwners = state.members.filter(
+ (member) => !member.retired && member.generation < expected.generation && member.state === "admitted",
+ );
+ const ready =
+ expected.desiredDeploymentId === null
+ ? state.members.every((member) => member.retired || member.state !== "admitted")
+ : expected.taskArns.length > 0 &&
+ expected.taskArns.every((taskArn) =>
+ state.members.some(
+ (member) =>
+ member.taskArn === taskArn &&
+ member.deploymentId === expected.desiredDeploymentId &&
+ member.generation === expected.generation &&
+ member.state === "admitted" &&
+ member.ready &&
+ !member.retired,
+ ),
+ );
+ const unexpectedOwner = state.members.some(
+ (member) =>
+ !member.retired && member.state === "admitted" && member.deploymentId !== expected.desiredDeploymentId,
+ );
+ if (priorOwners.length === 0 && !unexpectedOwner && ready) return state;
+ if (Date.now() >= deadline)
+ throw new CliError("timed out awaiting background ownership acknowledgment; no member was inferred dead");
+ await sleep(options.pollMs);
+ }
+}
diff --git a/cli/src/config.ts b/cli/src/config.ts
index 02b75c123..cbea1aaef 100644
--- a/cli/src/config.ts
+++ b/cli/src/config.ts
@@ -91,6 +91,7 @@ export interface AwsConfig {
imageLabel: string;
alb?: string;
sharedAlb?: boolean;
+ backgroundWorkControl?: boolean;
rdsInstance?: string;
predeployDbSnapshot?: boolean;
dbRetentionMinDays?: number;
@@ -749,6 +750,13 @@ function validate(raw: unknown, path: string): QmConfig {
validatePortalTrust(out, path);
if (target === "aws") {
validateAwsFrontDoor(out, path);
+ if (
+ out.env.core?.BACKGROUND_DEPLOYMENT_ID !== undefined ||
+ out.secretEnv?.core?.BACKGROUND_DEPLOYMENT_ID !== undefined
+ )
+ throw new CliError(
+ `${path}: BACKGROUND_DEPLOYMENT_ID is allocated by aws.backgroundWorkControl and cannot be configured directly`,
+ );
const externalImages = [
...plugins.filter((plugin) => plugin.image).map((plugin) => plugin.name),
...Object.keys(imageOverrides).filter((service) => services.includes(service as DeclaredServiceName)),
@@ -1127,6 +1135,9 @@ function validateAws(
);
}
}
+ if (raw["backgroundWorkControl"] !== undefined && typeof raw["backgroundWorkControl"] !== "boolean") {
+ throw new CliError(`${path}: "aws.backgroundWorkControl" must be a boolean`);
+ }
if (raw["sharedAlb"] !== undefined && typeof raw["sharedAlb"] !== "boolean") {
throw new CliError(`${path}: "aws.sharedAlb" must be a boolean`);
}
@@ -1428,6 +1439,11 @@ function validateAws(
if (deploymentState) out.deploymentState = deploymentState;
if (alb) out.alb = alb;
if (raw["sharedAlb"] !== undefined) out.sharedAlb = raw["sharedAlb"] as boolean;
+ if (raw["backgroundWorkControl"] !== undefined) out.backgroundWorkControl = raw["backgroundWorkControl"] as boolean;
+ if (out.backgroundWorkControl && (services.core?.ecsService.length ?? 0) > 219)
+ throw new CliError(
+ `${path}: controlled core service names must leave room for a unique deployment identity (maximum 219 characters)`,
+ );
if (rdsInstance) out.rdsInstance = rdsInstance;
if (predeployDbSnapshot !== undefined) out.predeployDbSnapshot = predeployDbSnapshot;
if (dbRetentionMinDays !== undefined) out.dbRetentionMinDays = dbRetentionMinDays;
diff --git a/cli/src/secrets.ts b/cli/src/secrets.ts
index bc8c83806..5e6ae16b0 100644
--- a/cli/src/secrets.ts
+++ b/cli/src/secrets.ts
@@ -4,6 +4,7 @@ import { TARGET_ENV_DEFAULTS } from "./target-env-defaults.ts";
import { deploymentSecretValue } from "./util.ts";
type SecretCondition =
+ | { kind: "background-work-control" }
| { kind: "sandbox-backend"; backend: string }
| { kind: "env-equals"; service: DeclaredServiceName; name: string; value: string }
| { kind: "env-in"; service: DeclaredServiceName; name: string; values: string[] }
@@ -84,6 +85,13 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [
description: "HMAC key shared by core and surface plugins.",
generate: MINT_LOCALLY,
},
+ {
+ name: "DEPLOYMENT_CONTROL_SECRET",
+ service: "core",
+ required: { when: { kind: "background-work-control" } },
+ description: "Deployment ownership credential held only by core and the privileged deployment operator.",
+ generate: MINT_LOCALLY,
+ },
{
name: "CAPABILITY_SECRET",
service: "core",
@@ -500,6 +508,8 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [
];
function conditionMatches(config: QmConfig, condition: SecretCondition): boolean {
+ if (condition.kind === "background-work-control")
+ return config.target === "aws" && config.aws?.backgroundWorkControl === true;
if (condition.kind === "sandbox-backend") {
const backend =
config.env.core?.SANDBOX_BACKEND ??
@@ -637,6 +647,9 @@ export function computedSecrets(config: QmConfig): ComputedSecret[] {
});
}
}
+ const control = byName.get("DEPLOYMENT_CONTROL_SECRET");
+ if (control && (control.services.some((service) => service !== "core") || control.aliases?.length))
+ throw new Error("DEPLOYMENT_CONTROL_SECRET must be delivered only to core under its original name");
const secrets = [...byName.values()]
.map((secret) => ({
...secret,
@@ -735,6 +748,7 @@ function requiresOtherEmailTransport(config: QmConfig, condition: SecretConditio
}
function conditionClause(condition: SecretCondition): string {
+ if (condition.kind === "background-work-control") return "aws.backgroundWorkControl is enabled";
if (condition.kind === "sandbox-backend")
return `SANDBOX_BACKEND or SANDBOX_SCOPE_BACKENDS selects ${condition.backend}`;
if (condition.kind === "service-enabled") return `the ${condition.service} service is enabled`;
diff --git a/cli/src/util.ts b/cli/src/util.ts
index 0d527ad5a..0d4b3f97b 100644
--- a/cli/src/util.ts
+++ b/cli/src/util.ts
@@ -275,7 +275,10 @@ export function isInvalidSecret(name: string, value: string | undefined): boolea
);
}
return (
- (name === "CONNECTOR_SECRET_KEY" || name === "CORE_SIGNING_SECRET" || name === "SKILL_SIGNING_SECRET") &&
+ (name === "CONNECTOR_SECRET_KEY" ||
+ name === "CORE_SIGNING_SECRET" ||
+ name === "SKILL_SIGNING_SECRET" ||
+ name === "DEPLOYMENT_CONTROL_SECRET") &&
candidate.length < 32
);
}
diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts
index c9a7025a2..8ee229e7c 100644
--- a/cli/test/aws.test.ts
+++ b/cli/test/aws.test.ts
@@ -14,6 +14,10 @@ import {
assertAwsPublicRouting,
assertGithubDeployTrust,
awsCheckLive,
+ awsBackgroundWorkStatus,
+ awsBackgroundWorkBootState,
+ awsBootstrapBackgroundWork,
+ awsRetireBackgroundWorkMembers,
awsDeploymentLayerTransport,
awsDown,
awsLogs,
@@ -322,8 +326,8 @@ const statePath = ${JSON.stringify(state)};
const s = JSON.parse(fs.readFileSync(statePath, "utf8"));
const save = () => fs.writeFileSync(statePath, JSON.stringify(s));
const after = (flag) => args[args.indexOf(flag) + 1];
-if (a.includes("secretsmanager get-secret-value") && !a.includes("--query")) console.log(JSON.stringify({ ARN: "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf", SecretString: a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)}) }));
-else if (a.includes("secretsmanager get-secret-value") && a.includes("--query SecretString")) console.log(a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)}));
+if (a.includes("secretsmanager get-secret-value") && !a.includes("--query")) console.log(JSON.stringify({ ARN: "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf", SecretString: a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("DEPLOYMENT_CONTROL_SECRET") ? "separate-control-secret-value-0000000000" : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)})) }));
+else if (a.includes("secretsmanager get-secret-value") && a.includes("--query SecretString")) console.log(a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("DEPLOYMENT_CONTROL_SECRET") ? "separate-control-secret-value-0000000000" : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)})));
else if (a.includes("secretsmanager get-secret-value") && a.includes("--query ARN")) console.log("arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf");
else if (a.includes("ecr get-login-password")) console.log("pw");
else if (a.includes("ecr batch-get-image")) console.log(JSON.stringify({ images: [{ imageManifest: "{}", imageManifestMediaType: "application/vnd.oci.image.index.v1+json" }] }));
@@ -413,8 +417,9 @@ else if (a.includes("ecs describe-service-revisions")) {
}) }));
}
else if (a.includes("ecs list-tasks")) console.log(JSON.stringify({ taskArns: process.env.AWS_FAKE_NO_RUNNING_TASK ? [] : [...(process.env.AWS_FAKE_LARGE_ROLLOUT ? Array.from({ length: 100 }, (_, i) => "arn:aws:ecs:us-west-2:123456789012:task/old-core-" + i) : []), "arn:aws:ecs:us-west-2:123456789012:task/live-core"] }));
+else if (a.includes("ecs describe-tasks") && s.stoppedTasks?.[after("--tasks")]) console.log(JSON.stringify({tasks: [s.stoppedTasks[after("--tasks")]], failures: []}));
else if (a.includes("ecs describe-tasks") && a.includes("task/old-core-")) console.log(JSON.stringify({ tasks: [] }));
-else if (a.includes("ecs describe-tasks") && a.includes("task/live-core")) console.log(JSON.stringify({ tasks: [{ taskDefinitionArn: process.env.AWS_FAKE_STALE_CORE ? "stale-task-definition" : s.services["acme-core"].taskDefinition, lastStatus: "RUNNING", healthStatus: "HEALTHY", containers: [{ name: "core", networkInterfaces: [{ privateIpv4Address: "10.0.1.8" }] }] }] }));
+else if (a.includes("ecs describe-tasks") && a.includes("task/live-core")) console.log(JSON.stringify({ tasks: [{ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/live-core", taskDefinitionArn: process.env.AWS_FAKE_STALE_CORE ? "stale-task-definition" : s.services["acme-core"].taskDefinition, lastStatus: "RUNNING", healthStatus: "HEALTHY", containers: [{ name: "core", networkInterfaces: [{ privateIpv4Address: "10.0.1.8" }] }] }] }));
else if (a.includes("ecs run-task")) console.log(JSON.stringify({ tasks: [{ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/canary" }] }));
else if (a.includes("ecs wait tasks-stopped")) console.log("");
else if (a.includes("ecs describe-tasks")) { const exitCode = Number(process.env.AWS_FAKE_CANARY_EXIT || "0") || ${JSON.stringify(opts.migrationExitCode ?? 0)}; console.log(JSON.stringify({ tasks: [{ stoppedReason: "Essential container in task exited", containers: [{ name: "core", exitCode, reason: process.env.AWS_FAKE_CANARY_REASON ?? (exitCode && ${JSON.stringify(Boolean(opts.migrationExitCode ?? 0))} ? "migration failed" : undefined) }] }], failures: [] })); }
@@ -470,7 +475,13 @@ else if (a.includes("dynamodb transact-write-items")) {
save();
if (s.transactAttempts <= failTransactionPuts) { console.error("TransactionRejected"); process.exit(1); }
}
- for (const write of JSON.parse(after("--transact-items"))) if (write.Put) s.dynamo[write.Put.Item.lockKey.S] = write.Put.Item;
+ for (const write of JSON.parse(after("--transact-items"))) {
+ if (write.Put) {
+ if (write.Put.ConditionExpression === "attribute_not_exists(lockKey)" && s.dynamo[write.Put.Item.lockKey.S]) { console.error("ConditionalCheckFailedException"); process.exit(1); }
+ s.dynamo[write.Put.Item.lockKey.S] = write.Put.Item;
+ }
+ if (write.Delete) delete s.dynamo[write.Delete.Key.lockKey.S];
+ }
save();
console.log("");
}
@@ -1466,6 +1477,65 @@ test("AWS up coalesces a requested restart into one deployment even when the tas
}
});
+test("recorded background boot flags come from the manifest task and reject ambiguous settings", () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-background-boot-"));
+ const configured = oneServiceConfig();
+ const fake = statefulAws(dir, configured);
+ try {
+ assert.equal(awsBackgroundWorkBootState(configured), undefined);
+ const state = JSON.parse(readFileSync(fake.state, "utf8"));
+ const task = "arn:aws:ecs:us-west-2:123456789012:task-definition/acme-core:42";
+ state.dynamo = manifestItems([{ id: "recorded", tasks: { core: task } }], "recorded");
+ const manifest = JSON.parse(state.dynamo["deployment/manifest/recorded"].manifest.S);
+ const core: {
+ name: string;
+ environment: Array<{ name: string; value: string }>;
+ secrets: Array<{ name: string }>;
+ } = {
+ name: "core",
+ environment: [{ name: "BACKGROUND_WORK_ENABLED", value: "1" }],
+ secrets: [],
+ };
+ state.definitions[task] = { containerDefinitions: [core] };
+ const save = () => {
+ state.dynamo["deployment/manifest/recorded"].manifest.S = JSON.stringify(manifest);
+ writeFileSync(fake.state, JSON.stringify(state));
+ };
+ save();
+ assert.deepEqual(awsBackgroundWorkBootState(configured), { enabled: true });
+ assert.match(readFileSync(fake.log, "utf8"), /describe-task-definition --task-definition .*acme-core:42/);
+ core.environment[0]!.value = "0";
+ manifest.backgroundDeploymentId = "core:recorded";
+ core.environment.push({ name: "BACKGROUND_DEPLOYMENT_ID", value: "core:recorded" });
+ save();
+ assert.deepEqual(awsBackgroundWorkBootState(configured), { enabled: false, deploymentId: "core:recorded" });
+ core.environment[1]!.value = "wrong-identity";
+ save();
+ assert.throws(() => awsBackgroundWorkBootState(configured), /manifest background deployment identity/);
+ core.environment[1]!.value = "core:recorded";
+ for (const value of ["", "yes", "2"]) {
+ core.environment[0]!.value = value;
+ save();
+ assert.throws(() => awsBackgroundWorkBootState(configured), /explicit BACKGROUND_WORK_ENABLED/);
+ }
+ core.environment[0]!.value = "1";
+ core.environment.push({ ...core.environment[0]! });
+ save();
+ assert.throws(() => awsBackgroundWorkBootState(configured), /explicit BACKGROUND_WORK_ENABLED/);
+ core.environment.pop();
+ core.secrets.push({ name: "BACKGROUND_WORK_ENABLED" });
+ save();
+ assert.throws(() => awsBackgroundWorkBootState(configured), /supplied as secrets/);
+ core.secrets = [];
+ core.environment.shift();
+ save();
+ assert.throws(() => awsBackgroundWorkBootState(configured), /explicit BACKGROUND_WORK_ENABLED/);
+ } finally {
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
test("background mode changes preserve task settings and reject secret-controlled modes", () => {
const task = {
family: "core",
@@ -5443,3 +5513,220 @@ test("AWS deployment progress rejects invalid options before AWS calls", async (
rmSync(dir, { recursive: true, force: true });
}
});
+
+test("controlled AWS cohorts bind immutable identities and hand over without ECS replacement", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-controlled-background-"));
+ const dockerBin = join(dir, "docker");
+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);
+ chmodSync(dockerBin, 0o755);
+ const base = oneServiceConfig();
+ const single: QmConfig = { ...base, aws: { ...base.aws!, backgroundWorkControl: true } };
+ const fake = statefulAws(dir, single);
+ const priorPath = process.env.PATH;
+ process.env.PATH = `${dir}:${priorPath}`;
+ try {
+ await awsUp(single, dir, { yes: true });
+ const persisted = JSON.parse(readFileSync(fake.state, "utf8"));
+ const manifest = JSON.parse(
+ persisted.dynamo[`deployment/manifest/${persisted.dynamo["deployment/current"].manifestId.S}`].manifest.S,
+ );
+ assert.match(manifest.backgroundDeploymentId, /^acme-core:[0-9a-f-]{36}$/);
+ assert.equal(persisted.dynamo["deployment/background-preparation"], undefined);
+ const preparationCalls = readFileSync(fake.log, "utf8");
+ const preparedAt = preparationCalls.indexOf('"preparation":{"S"');
+ assert.ok(preparedAt >= 0 && preparedAt < preparationCalls.indexOf("ecs register-task-definition"));
+ const taskArn = "arn:aws:ecs:us-west-2:123456789012:task/live-core";
+ const ownership = {
+ protocol: 1,
+ enabled: false,
+ deploymentId: manifest.backgroundDeploymentId,
+ instanceId: "instance-one",
+ generation: 0,
+ desiredDeploymentId: null as string | null,
+ lastRequestId: null as string | null,
+ members: [
+ {
+ instanceId: "instance-one",
+ taskArn,
+ deploymentId: manifest.backgroundDeploymentId,
+ generation: 0,
+ state: "drained",
+ retired: false,
+ ready: false,
+ },
+ ],
+ };
+ const fetchLayer = globalThis.fetch;
+ const mutations: unknown[] = [];
+ globalThis.fetch = async (url, init) => {
+ if (!String(url).includes("/v1/background-work")) return fetchLayer(url, init);
+ const headers = new Headers(init?.headers);
+ assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");
+ const body = String(init?.body ?? "");
+ const signature = createHmac("sha256", TEST_SECRET_VALUE)
+ .update(`v0:${headers.get("x-timestamp")}:${init?.method}\n/v1/background-work\n${body}`)
+ .digest("hex");
+ assert.equal(headers.get("x-signature"), `v0=${signature}`);
+ if (init?.method === "POST") {
+ const mutation = JSON.parse(body);
+ mutations.push(mutation);
+ assert.equal(mutation.expectedGeneration, ownership.generation);
+ ownership.lastRequestId = mutation.requestId;
+ if (mutation.terminatedMembers) {
+ for (const proof of mutation.terminatedMembers) {
+ const member = ownership.members.find((item) => item.instanceId === proof.instanceId)!;
+ member.retired = true;
+ member.state = "drained";
+ member.ready = false;
+ }
+ } else {
+ ownership.enabled = true;
+ ownership.generation++;
+ ownership.desiredDeploymentId = mutation.desiredDeploymentId;
+ ownership.members[0]!.generation = ownership.generation;
+ ownership.members[0]!.state = mutation.desiredDeploymentId ? "admitted" : "relinquished";
+ ownership.members[0]!.ready = Boolean(mutation.desiredDeploymentId);
+ }
+ }
+ return new Response(JSON.stringify(ownership), { status: 200 });
+ };
+ assert.deepEqual((await awsBackgroundWorkStatus(single, dir)).taskArns, [taskArn]);
+ await assert.rejects(awsSetBackgroundWork(single, dir, true), /explicitly bootstrap/);
+ const stoppedLegacyArn = "arn:aws:ecs:us-west-2:123456789012:task/stopped-legacy-core";
+ ownership.members.push({
+ ...ownership.members[0]!,
+ instanceId: "legacy-dead",
+ taskArn: stoppedLegacyArn,
+ state: "admitted",
+ });
+ const beforeBootstrap = JSON.parse(readFileSync(fake.state, "utf8"));
+ beforeBootstrap.stoppedTasks = {
+ [stoppedLegacyArn]: {
+ taskArn: stoppedLegacyArn,
+ lastStatus: "STOPPED",
+ group: "service:acme-core",
+ taskDefinitionArn: manifest.tasks.core,
+ },
+ };
+ writeFileSync(fake.state, JSON.stringify(beforeBootstrap));
+ await assert.rejects(
+ awsBootstrapBackgroundWork([{ config: single, configDir: dir }], manifest.backgroundDeploymentId),
+ /every enrolled/,
+ );
+ const retiredLegacy = await awsRetireBackgroundWorkMembers(
+ [{ config: single, configDir: dir }],
+ [{ instanceId: "legacy-dead", taskArn: stoppedLegacyArn, generation: 0 }],
+ );
+ assert.equal(retiredLegacy.generation, 0);
+ assert.equal(retiredLegacy.enabled, false);
+ const bootstrapped = await awsBootstrapBackgroundWork(
+ [{ config: single, configDir: dir }],
+ manifest.backgroundDeploymentId,
+ );
+ assert.equal(bootstrapped.generation, 1);
+ writeFileSync(fake.log, "");
+ await assert.rejects(awsUp(single, dir, { yes: true }), /pause or hand over/);
+ assert.doesNotMatch(
+ readFileSync(fake.log, "utf8"),
+ /ecs update-service|register-task-definition|s3api put-object|ecr get-login-password/,
+ );
+ ownership.desiredDeploymentId = "another-current-owner";
+ const beforeWrongPause = mutations.length;
+ await assert.rejects(awsSetBackgroundWork(single, dir, false), /different deployment/);
+ assert.equal(mutations.length, beforeWrongPause);
+ ownership.desiredDeploymentId = manifest.backgroundDeploymentId;
+ await awsSetBackgroundWork(single, dir, false);
+ await awsSetBackgroundWork(single, dir, true);
+ assert.equal(ownership.generation, 3);
+ assert.equal(mutations.length, 4);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service|register-task-definition|run-task/);
+ const unchanged = JSON.parse(readFileSync(fake.state, "utf8"));
+ assert.deepEqual(unchanged.dynamo, persisted.dynamo);
+ await assert.rejects(
+ awsRetireBackgroundWorkMembers(
+ [{ config: single, configDir: dir }],
+ [{ instanceId: "instance-one", taskArn, generation: 3 }],
+ ),
+ /STOPPED evidence/,
+ );
+ const stoppedArn = "arn:aws:ecs:us-west-2:123456789012:task/stopped-core";
+ ownership.members.push({
+ ...ownership.members[0]!,
+ instanceId: "terminated-instance",
+ taskArn: stoppedArn,
+ generation: 1,
+ });
+ unchanged.stoppedTasks = {
+ [stoppedArn]: {
+ taskArn: stoppedArn,
+ lastStatus: "STOPPED",
+ group: "service:acme-core",
+ taskDefinitionArn: manifest.tasks.core,
+ },
+ };
+ writeFileSync(fake.state, JSON.stringify(unchanged));
+ const retired = await awsRetireBackgroundWorkMembers(
+ [{ config: single, configDir: dir }],
+ [{ instanceId: "terminated-instance", taskArn: stoppedArn, generation: 1 }],
+ );
+ assert.equal(retired.generation, 3);
+ assert.equal(retired.members.at(-1)!.retired, true);
+ await awsSetBackgroundWork(single, dir, false);
+ await assert.rejects(awsUp(single, dir, { yes: true, restart: ["core"] }), /every member to drain/);
+ await assert.rejects(awsRollback(single, manifest.id), /every member to drain/);
+ ownership.members[0]!.state = "drained";
+ await awsUp(single, dir, { yes: true, restart: ["core"] });
+ const replacement = JSON.parse(readFileSync(fake.state, "utf8"));
+ const replacementManifest = JSON.parse(
+ replacement.dynamo[`deployment/manifest/${replacement.dynamo["deployment/current"].manifestId.S}`].manifest.S,
+ );
+ assert.notEqual(replacementManifest.backgroundDeploymentId, manifest.backgroundDeploymentId);
+ assert.notEqual(replacementManifest.tasks.core, manifest.tasks.core);
+ ownership.deploymentId = replacementManifest.backgroundDeploymentId;
+ ownership.members[0]!.deploymentId = replacementManifest.backgroundDeploymentId;
+ const controlEnv = join(dir, "control.env");
+ writeFileSync(
+ controlEnv,
+ computedSecrets(single)
+ .filter((secret) => secret.managedBy !== "terraform")
+ .map(
+ (secret) =>
+ `${secret.name}=${secret.name === "DEPLOYMENT_CONTROL_SECRET" ? "new-control-secret".repeat(3) : TEST_SECRET_VALUE}`,
+ )
+ .join("\n"),
+ );
+ writeFileSync(fake.log, "");
+ await assert.rejects(awsSecretsPush(single, dir, controlEnv), /cannot be rotated/);
+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /put-secret-value|ecs update-service/);
+ replacement.dynamo["deployment/background-preparation"] = {
+ preparation: {
+ S: JSON.stringify({
+ id: "unresolved",
+ previousManifestId: "different",
+ backgroundDeploymentId: "another",
+ before: { tasks: {}, counts: {} },
+ }),
+ },
+ };
+ writeFileSync(fake.state, JSON.stringify(replacement));
+ await assert.rejects(awsUp(single, dir, { yes: true }), /remains unresolved/);
+ delete replacement.dynamo["deployment/background-preparation"];
+ writeFileSync(fake.state, JSON.stringify(replacement));
+ ownership.deploymentId = "stale-cohort";
+ await assert.rejects(awsSetBackgroundWork(single, dir, false), /requested deployment/);
+ assert.equal(mutations.length, 6);
+ ownership.deploymentId = replacementManifest.backgroundDeploymentId;
+ await awsRollback(single, manifest.id);
+ const rolledBack = JSON.parse(readFileSync(fake.state, "utf8"));
+ assert.equal(rolledBack.dynamo["deployment/current"].manifestId.S, manifest.id);
+ assert.equal(rolledBack.services["acme-core"].taskDefinition, manifest.tasks.core);
+ await assert.rejects(
+ awsSetBackgroundWork({ ...single, aws: { ...single.aws!, backgroundWorkControl: false } }, dir, true),
+ /cannot fall back/,
+ );
+ } finally {
+ process.env.PATH = priorPath;
+ fake.restore();
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
diff --git a/cli/test/background-work.test.ts b/cli/test/background-work.test.ts
new file mode 100644
index 000000000..ee34d5438
--- /dev/null
+++ b/cli/test/background-work.test.ts
@@ -0,0 +1,135 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+ awaitBackgroundWork,
+ mutateBackgroundWork,
+ parseBackgroundWorkStatus,
+ type BackgroundWorkStatus,
+ type BackgroundWorkTransport,
+} from "../src/background-work.ts";
+
+const state = (): BackgroundWorkStatus => ({
+ protocol: 1,
+ enabled: true,
+ deploymentId: "new-cohort",
+ instanceId: "new-instance",
+ generation: 2,
+ desiredDeploymentId: "new-cohort",
+ lastRequestId: "our-request",
+ members: [
+ {
+ instanceId: "old-instance",
+ taskArn: "old-task",
+ deploymentId: "old-cohort",
+ generation: 1,
+ state: "relinquished",
+ retired: false,
+ ready: true,
+ },
+ {
+ instanceId: "new-instance",
+ taskArn: "new-task",
+ deploymentId: "new-cohort",
+ generation: 2,
+ state: "admitted",
+ retired: false,
+ ready: true,
+ },
+ ],
+});
+
+const response = (value: BackgroundWorkStatus) => ({ status: 200, body: JSON.stringify(value) });
+
+test("ownership status refuses stale responders and malformed durable membership", () => {
+ assert.equal(parseBackgroundWorkStatus(JSON.stringify(state()), "new-cohort").generation, 2);
+ assert.throws(() => parseBackgroundWorkStatus(JSON.stringify(state()), "old-cohort"), /requested deployment/);
+ const duplicate = state();
+ duplicate.members.push(duplicate.members[0]!);
+ assert.throws(() => parseBackgroundWorkStatus(JSON.stringify(duplicate), "new-cohort"), /duplicate/);
+ const unenrolled = state();
+ unenrolled.instanceId = "missing";
+ assert.throws(() => parseBackgroundWorkStatus(JSON.stringify(unenrolled), "new-cohort"), /not enrolled/);
+});
+
+test("lost mutation responses require the exact request ID and generation on durable readback", async () => {
+ const calls: string[] = [];
+ const mutation = { expectedGeneration: 1, desiredDeploymentId: "new-cohort", requestId: "our-request" };
+ let observed = state();
+ const transport: BackgroundWorkTransport = async (method, body) => {
+ calls.push(method);
+ if (method === "POST") {
+ assert.deepEqual(JSON.parse(body!), mutation);
+ throw new Error("connection lost after commit");
+ }
+ return response(observed);
+ };
+ assert.equal((await mutateBackgroundWork(transport, "new-cohort", mutation)).generation, 2);
+ assert.deepEqual(calls, ["POST", "GET"]);
+ observed = { ...state(), lastRequestId: "someone-else" };
+ await assert.rejects(mutateBackgroundWork(transport, "new-cohort", mutation), /changed concurrently/);
+});
+
+test("retirement acknowledgment binds every terminated instance, task and generation", async () => {
+ const mutation = {
+ expectedGeneration: 2,
+ requestId: "retirement",
+ terminatedMembers: [{ instanceId: "old-instance", taskArn: "old-task", generation: 1 }],
+ };
+ const observed = state();
+ observed.lastRequestId = "retirement";
+ observed.members[0]!.retired = true;
+ const transport: BackgroundWorkTransport = async () => response(observed);
+ assert.equal((await mutateBackgroundWork(transport, "new-cohort", mutation)).generation, 2);
+ observed.members[0]!.taskArn = "different-task";
+ await assert.rejects(mutateBackgroundWork(transport, "new-cohort", mutation), /unconfirmed/);
+});
+
+test("activation waits for every old owner and every expected new task without waiting for old turns to drain", async () => {
+ const observed = state();
+ observed.members[0]!.state = "admitted";
+ let polls = 0;
+ const transport: BackgroundWorkTransport = async () => {
+ polls++;
+ if (polls === 2) observed.members[0]!.state = "relinquished";
+ if (polls === 3)
+ observed.members.push({
+ ...observed.members[1]!,
+ instanceId: "second-instance",
+ taskArn: "second-task",
+ ready: false,
+ });
+ if (polls === 4) observed.members[2]!.ready = true;
+ return response(observed);
+ };
+ const ready = await awaitBackgroundWork(
+ transport,
+ "new-cohort",
+ { generation: 2, desiredDeploymentId: "new-cohort", taskArns: ["new-task", "second-task"] },
+ { timeoutMs: 10_000, pollMs: 1 },
+ );
+ assert.equal(polls, 4);
+ assert.equal(ready.members[0]!.state, "relinquished");
+});
+
+test("pause never infers unacknowledged owners dead and refuses concurrent generations", async () => {
+ const observed = state();
+ observed.desiredDeploymentId = null;
+ observed.members[0]!.state = "admitted";
+ observed.members[1]!.state = "drained";
+ const transport: BackgroundWorkTransport = async () => response(observed);
+ const expected = { generation: 2, desiredDeploymentId: null, taskArns: [] };
+ await assert.rejects(
+ awaitBackgroundWork(transport, "new-cohort", expected, { timeoutMs: 0, pollMs: 1 }),
+ /no member was inferred dead/,
+ );
+ observed.members[0]!.state = "relinquished";
+ assert.equal(
+ (await awaitBackgroundWork(transport, "new-cohort", expected, { timeoutMs: 0, pollMs: 1 })).generation,
+ 2,
+ );
+ observed.generation = 3;
+ await assert.rejects(
+ awaitBackgroundWork(transport, "new-cohort", expected, { timeoutMs: 0, pollMs: 1 }),
+ /changed while awaiting/,
+ );
+});
diff --git a/cli/test/config.test.ts b/cli/test/config.test.ts
index 2b1d103e7..e4b4cf940 100644
--- a/cli/test/config.test.ts
+++ b/cli/test/config.test.ts
@@ -1309,3 +1309,28 @@ test("screening defaults off and model screening requires an explicit backend",
);
}
});
+
+test("AWS ownership control is opt-in and reserves deployment identity allocation", () => {
+ const aws = {
+ accountId: "123456789012",
+ region: "us-west-2",
+ cluster: "acme",
+ deployRoleArn: "arn:aws:iam::123456789012:role/deploy",
+ secretsPrefix: "acme/",
+ imageLabel: "release",
+ networking: { cloudMapNamespace: "acme.internal" },
+ services: { core: { ecrRepository: "core", ecsService: "acme-core", cpu: 512, memory: 1024 } },
+ };
+ withConfig({ target: "aws", aws }, ({ path }) =>
+ assert.equal(loadConfigAt(path).config.aws!.backgroundWorkControl, undefined),
+ );
+ withConfig({ target: "aws", aws: { ...aws, backgroundWorkControl: true } }, ({ path }) =>
+ assert.equal(loadConfigAt(path).config.aws!.backgroundWorkControl, true),
+ );
+ withConfig({ target: "aws", aws: { ...aws, backgroundWorkControl: "true" } }, ({ path }) =>
+ assert.throws(() => loadConfigAt(path), /backgroundWorkControl/),
+ );
+ withConfig({ target: "aws", aws, env: { core: { BACKGROUND_DEPLOYMENT_ID: "reused" } } }, ({ path }) =>
+ assert.throws(() => loadConfigAt(path), /allocated/),
+ );
+});
diff --git a/cli/test/secrets.test.ts b/cli/test/secrets.test.ts
index 131c24bee..9c7375b7e 100644
--- a/cli/test/secrets.test.ts
+++ b/cli/test/secrets.test.ts
@@ -419,3 +419,36 @@ test("shared Fly publishing requires private peers only when selected", () => {
}
}
});
+
+test("deployment control credentials are conditional and cannot be delivered to plugins or sandboxes", () => {
+ assert.equal(
+ computedSecrets(makeConfig()).some((secret) => secret.name === "DEPLOYMENT_CONTROL_SECRET"),
+ false,
+ );
+ const aws = { backgroundWorkControl: true } as NonNullable<QmConfig["aws"]>;
+ const config = makeConfig({ target: "aws", aws });
+ const control = secretByName(config, "DEPLOYMENT_CONTROL_SECRET");
+ assert.equal(control.required, true);
+ assert.deepEqual(control.services, ["core"]);
+ assert.throws(
+ () =>
+ computedSecrets({
+ ...config,
+ plugins: [{ name: "external", secrets: [{ name: "DEPLOYMENT_CONTROL_SECRET" }] }],
+ }),
+ /only to core/,
+ );
+ assert.throws(
+ () => computedSecrets({ ...config, sandbox: { app: "sandbox", secretEnv: ["DEPLOYMENT_CONTROL_SECRET"] } }),
+ /only to core/,
+ );
+ assert.throws(
+ () =>
+ computedSecrets({
+ ...config,
+ services: ["core", "portal"],
+ secretEnv: { portal: { TOKEN: "DEPLOYMENT_CONTROL_SECRET" } },
+ }),
+ /only to core/,
+ );
+});
diff --git a/src/deployment/secret-schema.ts b/src/deployment/secret-schema.ts
index 78798361f..d474798b9 100644
--- a/src/deployment/secret-schema.ts
+++ b/src/deployment/secret-schema.ts
@@ -2,6 +2,7 @@ import { isStrongSigningSecret } from "../auth/source-auth.ts";
type SecretGate =
| "production"
+ | "background-work-control"
| "codex"
| "postgres"
| "sprites"
@@ -31,6 +32,7 @@ export const CORE_SECRET_SPECS: readonly RuntimeSecretSpec[] = [
{ name: "CAPABILITY_SECRET", requiredWhen: "production" },
{ name: "CONNECTOR_SECRET_KEY", requiredWhen: "production" },
{ name: "CORE_SIGNING_SECRET", requiredWhen: "production" },
+ { name: "DEPLOYMENT_CONTROL_SECRET", requiredWhen: "background-work-control" },
{ name: "PORTAL_IDENTITY_SECRET", requiredWhen: "production" },
{ name: "SKILL_SIGNING_SECRET", requiredWhen: "production" },
{ name: "AUTH_ALLOWED_EMAILS", requiredWhen: "email-auth" },
@@ -63,6 +65,7 @@ function sandboxBackendSelected(env: NodeJS.ProcessEnv, backend: string): boolea
const GATE_PREDICATES: Readonly<Record<SecretGate, (env: NodeJS.ProcessEnv) => boolean>> = {
production: (env) => env.NODE_ENV === "production",
+ "background-work-control": (env) => Boolean(env.BACKGROUND_DEPLOYMENT_ID?.trim()),
codex: (env) => env.HARNESS?.trim() === "codex" && !env.CODEX_AUTH_FILE?.trim() && !env.CODEX_AUTH_CREDENTIAL?.trim(),
postgres: (env) => env.SESSION_STORE === "postgres" || env.RUN_STORE === "postgres",
sprites: (env) => sandboxBackendSelected(env, "sprites"),
@@ -102,6 +105,7 @@ function isInvalidSecret(name: string, value: string | undefined): boolean {
return (
(name === "CONNECTOR_SECRET_KEY" ||
name === "CORE_SIGNING_SECRET" ||
+ name === "DEPLOYMENT_CONTROL_SECRET" ||
name === "SKILL_SIGNING_SECRET" ||
name === "AWS_DEPLOY_GATE_SECRET") &&
!isStrongSigningSecret(candidate)
diff --git a/test/secret-schema-drift.test.ts b/test/secret-schema-drift.test.ts
index 6ceab52bd..2def94644 100644
--- a/test/secret-schema-drift.test.ts
+++ b/test/secret-schema-drift.test.ts
@@ -16,6 +16,26 @@ test("the standalone CLI and core agree on runtime-enforced core secret names",
assert.deepEqual([...runtime].sort(), [...cli].filter((name) => name !== "PUBLIC_API_URL").sort());
});
+test("background ownership requires strong control credentials only for configured deployments", () => {
+ assert.deepEqual(validateCoreSecretEnv({}), []);
+ for (const secret of [undefined, "", " ".repeat(32), "short"]) {
+ assert.deepEqual(
+ validateCoreSecretEnv({
+ BACKGROUND_DEPLOYMENT_ID: "core:release",
+ DEPLOYMENT_CONTROL_SECRET: secret,
+ }),
+ ["DEPLOYMENT_CONTROL_SECRET"],
+ );
+ }
+ assert.deepEqual(
+ validateCoreSecretEnv({
+ BACKGROUND_DEPLOYMENT_ID: "core:release",
+ DEPLOYMENT_CONTROL_SECRET: "separate-control-secret-value-0000000000",
+ }),
+ [],
+ );
+});
+
test('deploy/core/Dockerfile pins NODE_ENV=production — the "production" secret gate is load-bearing on that line', () => {
const dockerfile = readFileSync(new URL("../deploy/core/Dockerfile", import.meta.url), "utf8");
assert.match(dockerfile, /^ENV NODE_ENV=production$/m);