Multiplayer agent harness for work.
Transfer AWS background ownership without restarting tasks (#1292)
Regan Bell33e3bf2dab59
diff --git a/cli/README.md b/cli/README.mdindex ed7d16062..15df9d960 100644--- a/cli/README.md+++ b/cli/README.md@@ -77,6 +77,45 @@ configuration only, so it prints that timestamp as the matching data restorepoint (`aws rds restore-db-instance-to-point-in-time`);`aws.predeployDbSnapshot: false` opts out.+AWS deployments can opt into durable background ownership with+`aws.backgroundWorkControl: true`. Deploy protocol-capable core images to every+participating stack before bootstrapping ownership. The CLI allocates a unique+`BACKGROUND_DEPLOYMENT_ID` for each replacement core cohort and records it in the+deployment manifest. A no-op deployment and automatic ECS task replacement keep+that identity; an explicit core restart allocates a new one. Pending preparation+is persisted before ECS changes, and an ambiguous previous deployment must be+reconciled before another identity can be allocated.++Control requests require both `CORE_SIGNING_SECRET` and a distinct+`DEPLOYMENT_CONTROL_SECRET` of at least 32 characters. The control secret is+restricted to core. Configuring an identity preserves the legacy background flag+until explicit bootstrap verifies every participating task. The exported+`awsBootstrapBackgroundWork` adapter accepts all peer configurations and an exact+desired deployment identity, or `null` to start paused. It refuses missing cohorts+and inconsistent durable membership.++The exported `awsBackgroundWorkBootState` reads the exact manifest core task to+return its recorded boot flag and optional deployment identity. It returns+`undefined` only without a recorded core task, and rejects missing or ambiguous+boot flags or identities. Deployment wrappers can preserve the boot environment+while changing durable ownership independently.++Once bootstrapped, background mode changes use generation-checked ownership+requests without restarting ECS tasks. Activation waits for prior owners to stop+claiming and every expected task to finish activation. Pausing stops new claims;+in-flight turns can continue draining. Replacing or rolling back the active core+cohort requires an explicit pause or handover and proof that every member has+drained first. A demotion refuses to pause a different current owner. Unresponsive members are never+assumed dead: `awsRetireBackgroundWorkMembers` requires exact instance, task ARN,+and generation identities plus ECS evidence that each task stopped. This recovery+is also available before bootstrap for stopped legacy members.++Core secret uploads defer activation to a subsequent staged `up --restart core`.+The generic upload path refuses changes to either control credential while a+controlled cohort is recorded, because replacing credentials before coordinating+all running processes would break ownership control. Legacy AWS deployments keep+the task replacement path when ownership control is not configured.+Batch operators can set `QM_DEPLOY_PROGRESS_FILE` to a new absolute file path and`QM_DEPLOY_PROGRESS_TOKEN` to a unique attempt identifier for candidate `up --yes`.After all forward service updates have been submitted, the CLI atomically createsdiff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 76cb5f2fa..c9d78eb2d 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -1,6 +1,6 @@import { awsCoreHostnames, awsPortalAppsDomain, validAlbHostname } from "../aws-routing.ts";import https from "node:https";-import { createHash, randomUUID } from "node:crypto";+import { createHash, createHmac, randomUUID } from "node:crypto";import { spawnSync } from "node:child_process";import { lookup, resolveCname } from "node:dns/promises";import {@@ -88,6 +88,53 @@ import {type DeploymentLayerTransport,} from "../deployment-layer.ts";+import {+ awaitBackgroundWork,+ backgroundWorkMutation,+ mutateBackgroundWork,+ readBackgroundWork,+ type BackgroundWorkMember,+ type BackgroundWorkStatus,+ type BackgroundWorkTransport,+} from "../background-work.ts";++export async function awsCoreRequest(+ config: QmConfig,+ url: URL,+ init: RequestInit,+): Promise<{ status: number; body: string }> {+ const target = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");+ if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");+ if (awsPublicOrigin(config).protocol === "http:") {+ assertCloudFrontLayerTarget(config, url, target);+ const response = await fetch(url, init);+ return { status: response.status, body: await response.text() };+ }+ return new Promise((resolve, reject) => {+ const request = https.request(+ url,+ {+ hostname: target,+ servername: url.hostname,+ method: init.method,+ headers: { ...(init.headers as Record<string, string>), host: url.host },+ signal: init.signal ?? undefined,+ agent: false,+ },+ (response) => {+ const chunks: Buffer[] = [];+ response.on("data", (chunk: Buffer) => chunks.push(chunk));+ response.on("error", reject);+ response.on("end", () =>+ resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }),+ );+ },+ );+ request.on("error", reject);+ request.end(init.body as string | undefined);+ });+}+export const awsDeploymentLayerTransport: DeploymentLayerTransport = httpDeploymentLayerTransport({urlOf: (config) => {const url = new URL(config.apiUrl ?? config.publicUrl);@@ -96,38 +143,7 @@ export const awsDeploymentLayerTransport: DeploymentLayerTransport = httpDeploymurl.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/deployment-layer`;return url;},- request: async (config, url, init) => {- const target = awsPublicFrontDoor(config).dnsName.toLowerCase().replace(/\.$/, "");- if (!validAlbHostname(target)) throw new CliError("AWS deployment-layer ALB hostname is invalid");- if (awsPublicOrigin(config).protocol === "http:") {- assertCloudFrontLayerTarget(config, url, target);- const response = await fetch(url, init);- return { status: response.status, body: await response.text() };- }- return new Promise((resolve, reject) => {- const request = https.request(- url,- {- hostname: target,- servername: url.hostname,- method: init.method,- headers: { ...(init.headers as Record<string, string>), host: url.host },- signal: init.signal ?? undefined,- agent: false,- },- (response) => {- const chunks: Buffer[] = [];- response.on("data", (chunk: Buffer) => chunks.push(chunk));- response.on("error", reject);- response.on("end", () =>- resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") }),- );- },- );- request.on("error", reject);- request.end(init.body as string | undefined);- });- },+ request: awsCoreRequest,secretFallback: (config) =>config.aws? capture(process.env.AWS_BIN ?? "aws", [@@ -393,8 +409,20 @@ export function serviceEnvironment(config: QmConfig, service: ServiceName): Recoreturn Object.fromEntries(Object.entries(env).sort(([a], [b]) => a.localeCompare(b)));}-function workloadEnvironment(config: QmConfig, workload: string): Record<string, string> {- if (isServiceName(workload)) return serviceEnvironment(config, workload);+function workloadEnvironment(+ config: QmConfig,+ workload: string,+ backgroundDeploymentId?: string,+): Record<string, string> {+ if (isServiceName(workload)) {+ const env = serviceEnvironment(config, workload);+ if (workload === "core" && requireAws(config).backgroundWorkControl) {+ if (!backgroundDeploymentId)+ throw new CliError("controlled background work requires a manifest-bound deployment identity");+ env.BACKGROUND_DEPLOYMENT_ID = backgroundDeploymentId;+ }+ return env;+ }const plugin = config.plugins.find((entry) => entry.name === workload);return Object.fromEntries(Object.entries({@@ -434,6 +462,7 @@ export function renderTaskDefinition(service: string,image: string,secretArns?: Record<string, string>,+ backgroundDeploymentId?: string,): EcsTaskDefinition {if (!isDigestPinned(image)) throw new CliError(`aws task image for ${service} must be pinned by digest`);const aws = requireAws(config);@@ -484,7 +513,9 @@ export function renderTaskDefinition(name: service,image,essential: true,- environment: Object.entries(workloadEnvironment(config, service)).map(([name, value]) => ({ name, value })),+ environment: Object.entries(workloadEnvironment(config, service, backgroundDeploymentId)).map(+ ([name, value]) => ({ name, value }),+ ),secrets,portMappings: [{@@ -848,6 +879,7 @@ async function runAwsCandidateMigration(export function secretArns(config: QmConfig): Record<string, string> {const aws = requireAws(config);+ const controlValues = new Map<string, string>();const pairs = computedSecrets(config).flatMap((secret) => {const id = `${aws.secretsPrefix}${secret.name}`;try {@@ -871,12 +903,19 @@ export function secretArns(config: QmConfig): Record<string, string> {if (!value.ARN || isInvalidSecret(secret.name, value.SecretString)) {throw new CliError(`required AWS secret ${secret.name} has no usable, non-placeholder AWSCURRENT value`);}+ if (secret.name === "CORE_SIGNING_SECRET" || secret.name === "DEPLOYMENT_CONTROL_SECRET")+ controlValues.set(secret.name, value.SecretString!);return [[secret.name, value.ARN] as const];} catch (error) {if (!secret.required && /ResourceNotFoundException/.test(errMessage(error))) return [];throw error;}});+ if (+ aws.backgroundWorkControl &&+ controlValues.get("CORE_SIGNING_SECRET") === controlValues.get("DEPLOYMENT_CONTROL_SECRET")+ )+ throw new CliError("DEPLOYMENT_CONTROL_SECRET must differ from CORE_SIGNING_SECRET");return Object.fromEntries(pairs);}@@ -1176,6 +1215,7 @@ interface AwsReleaseCandidate {interface DeploymentManifest {id: string;+ backgroundDeploymentId?: string;previous?: string;createdAt: string;imageLabel?: string;@@ -1187,6 +1227,93 @@ interface DeploymentManifest {layer?: { key: string; sha256: string };}+interface BackgroundDeploymentPreparation {+ id: string;+ previousManifestId?: string;+ backgroundDeploymentId: string;+ before: { tasks: Record<string, string>; counts: Record<string, number> };+}++const BACKGROUND_PREPARATION_KEY = "deployment/background-preparation";++function clearBackgroundPreparation(aws: AwsConfig): void {+ awsText(aws, [+ "dynamodb",+ "transact-write-items",+ "--transact-items",+ JSON.stringify([+ {+ Delete: {+ TableName: deployLocksTable(aws),+ Key: { lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } },+ },+ },+ ]),+ ]);+}++function reconcileBackgroundPreparation(+ aws: AwsConfig,+ current: DeploymentManifest | undefined,+ before: BackgroundDeploymentPreparation["before"],+): void {+ const response = awsJson<{ Item?: Record<string, { S?: string }> }>(aws, [+ "dynamodb",+ "get-item",+ "--table-name",+ deployLocksTable(aws),+ "--key",+ JSON.stringify({ lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) } }),+ "--consistent-read",+ ]);+ if (!response.Item) return;+ let preparation: BackgroundDeploymentPreparation;+ try {+ preparation = JSON.parse(response.Item.preparation?.S ?? "");+ if (+ !preparation ||+ typeof preparation.id !== "string" ||+ typeof preparation.backgroundDeploymentId !== "string" ||+ !preparation.before?.tasks ||+ !preparation.before.counts+ )+ throw new Error("invalid preparation");+ } catch {+ throw new CliError("pending background deployment preparation is invalid; refusing a new cohort");+ }+ const committed =+ current?.id === preparation.id &&+ current.backgroundDeploymentId === preparation.backgroundDeploymentId &&+ canonicalJson(before) === canonicalJson({ tasks: current.tasks, counts: current.counts });+ const restored =+ current?.id === preparation.previousManifestId && canonicalJson(before) === canonicalJson(preparation.before);+ if (!committed && !restored)+ throw new CliError(+ `background deployment ${preparation.id} remains unresolved; reconcile its recorded tasks before retrying instead of allocating another cohort`,+ );+ clearBackgroundPreparation(aws);+}++function prepareBackgroundDeployment(aws: AwsConfig, preparation: BackgroundDeploymentPreparation): void {+ awsText(aws, [+ "dynamodb",+ "transact-write-items",+ "--transact-items",+ JSON.stringify([+ {+ Put: {+ TableName: deployLocksTable(aws),+ Item: {+ lockKey: { S: deploymentStateKey(aws, BACKGROUND_PREPARATION_KEY) },+ preparation: { S: JSON.stringify(preparation) },+ },+ ConditionExpression: "attribute_not_exists(lockKey)",+ },+ },+ ]),+ ]);+}+const DEPLOYMENT_POINTER_KEY = "deployment/current";const DEPLOYMENT_MANIFEST_PREFIX = "deployment/manifest/";const ECS_SERVICE_BATCH_SIZE = 10;@@ -1546,12 +1673,16 @@ function recordDeploymentManifest(layer?: DeploymentManifest["layer"];imageProvenance?: DeploymentManifest["imageProvenance"];counts?: DeploymentManifest["counts"];+ backgroundDeploymentId?: string;},): DeploymentManifest {const current = currentDeploymentManifest(aws);const counts = release.counts ?? current?.counts;const manifest: DeploymentManifest = {id: release.id ?? randomUUID(),+ ...((release.backgroundDeploymentId ?? current?.backgroundDeploymentId)+ ? { backgroundDeploymentId: release.backgroundDeploymentId ?? current?.backgroundDeploymentId }+ : {}),...(current ? { previous: current.id } : {}),createdAt: new Date().toISOString(),...(release.imageLabel ? { imageLabel: release.imageLabel } : {}),@@ -1852,10 +1983,11 @@ function reportTaskChanges(images: Record<string, string>,arns: Record<string, string>,restart: ReadonlySet<string> = new Set(),+ backgroundDeploymentId?: string,): Array<{ service: string; task: EcsTaskDefinition; changed: boolean }> {const desired = services.map((service) => ({service,- task: renderTaskDefinition(config, service, images[service]!, arns),+ task: renderTaskDefinition(config, service, images[service]!, arns, backgroundDeploymentId),live: liveTask(config, service),}));return desired.map((item) => {@@ -2159,7 +2291,11 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt}images[service] = plannedWorkloadImage(config, service, plugins.get(service));}- reportTaskChanges(config, services, images, arns, restart);+ const backgroundDeploymentId = aws.backgroundWorkControl+ ? (currentDeploymentManifest(aws)?.backgroundDeploymentId ??+ `${aws.services.core!.ecsService}:00000000-0000-0000-0000-000000000000`)+ : undefined;+ reportTaskChanges(config, services, images, arns, restart, backgroundDeploymentId);for (const service of services) {step(`${service}: desired count ${before.counts[service] ?? 0} → ${workloadDesiredCount(config, service)}`);}@@ -2201,6 +2337,14 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOptassertAwsDeployImage(config);before = serviceSnapshot(config, allServices);current = currentDeploymentManifest(aws);+ if (current?.backgroundDeploymentId && !aws.backgroundWorkControl)+ throw new CliError("cannot disable background ownership control while a controlled deployment is recorded");+ if (aws.backgroundWorkControl) {+ if (!current?.backgroundDeploymentId && !services.includes("core"))+ throw new CliError("enabling background ownership requires deploying core to allocate its identity");+ reconcileBackgroundPreparation(aws, current, before);+ if (services.includes("core")) await assertBackgroundCohortReplaceable(config, current);+ }const selected = new Set(services);if (allServices.some((service) => !selected.has(service))) {current = trustedDeploymentBaseline(@@ -2271,7 +2415,21 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOptimages[service] = await publishWorkloadImage(config, service, plugins.get(service), stagingLabel, opts);}}- const desired = reportTaskChanges(config, services, images, arns, restart);+ let backgroundDeploymentId = current?.backgroundDeploymentId;+ if (aws.backgroundWorkControl && !backgroundDeploymentId)+ backgroundDeploymentId = `${aws.services.core!.ecsService}:${releaseId}`;+ const desired = reportTaskChanges(config, services, images, arns, restart, backgroundDeploymentId);+ const changedCore = desired.find((item) => item.service === "core" && item.changed);+ if (aws.backgroundWorkControl && changedCore) {+ backgroundDeploymentId = `${aws.services.core!.ecsService}:${releaseId}`;+ changedCore.task = renderTaskDefinition(config, "core", images.core!, arns, backgroundDeploymentId);+ prepareBackgroundDeployment(aws, {+ id: releaseId,+ previousManifestId: current?.id,+ backgroundDeploymentId,+ before,+ });+ }const targets: Record<string, string> = {};for (const item of desired) {if (!item.changed) {@@ -2352,6 +2510,7 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt) {recorded = recordDeploymentManifest(aws, releaseTasks, {id: releaseId,+ ...(backgroundDeploymentId ? { backgroundDeploymentId } : {}),counts: releaseCounts,imageLabel: label,...(dbRestorePoint ? { dbRestorePoint } : {}),@@ -2359,6 +2518,7 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOptimageProvenance: releaseImageProvenance,});}+ if (aws.backgroundWorkControl && changedCore) clearBackgroundPreparation(aws);if (recorded && dbRestorePoint) confirmDbRestorePointCovered(config, dbRestorePoint);for (const service of candidate ? [] : services) {try {@@ -2532,6 +2692,11 @@ export async function awsRollback(} else {targetManifest = deploymentManifestForTarget(aws, to);}+ if (currentManifest?.backgroundDeploymentId || targetManifest.backgroundDeploymentId) {+ if (!aws.backgroundWorkControl || !targetManifest.backgroundDeploymentId)+ throw new CliError("controlled rollback requires a protocol-capable target deployment");+ await assertBackgroundCohortReplaceable(config, currentManifest);+ }const missing = services.filter((service) => !targetManifest.tasks[service]);if (missing.length)throw new CliError(`target AWS deployment manifest is missing workloads: ${missing.join(", ")}`);@@ -2674,6 +2839,339 @@ export function taskDefinitionForBackgroundWork(};}+function awsBackgroundWorkTransport(config: QmConfig): BackgroundWorkTransport {+ const aws = requireAws(config);+ const secret = (name: string): string =>+ awsText(aws, [+ "secretsmanager",+ "get-secret-value",+ "--secret-id",+ `${aws.secretsPrefix}${name}`,+ "--query",+ "SecretString",+ ]);+ const signing = secret("CORE_SIGNING_SECRET");+ const control = secret("DEPLOYMENT_CONTROL_SECRET");+ if (isInvalidSecret("CORE_SIGNING_SECRET", signing) || control.trim().length < 32 || control === signing)+ throw new CliError("deployment control requires a distinct secret of at least 32 characters");+ const url = new URL(config.publicUrl);+ url.pathname = `${url.pathname.replace(/\/+$/, "")}/v1/background-work`;+ url.search = "";+ return (method, body = "") => {+ const timestamp = Math.floor(Date.now() / 1000);+ const signature = createHmac("sha256", signing)+ .update(`v0:${timestamp}:${method}\n${url.pathname}\n${body}`)+ .digest("hex");+ return awsCoreRequest(config, url, {+ method,+ headers: {+ "content-type": "application/json",+ "x-timestamp": String(timestamp),+ "x-signature": `v0=${signature}`,+ authorization: `Bearer ${control}`,+ },+ ...(method === "POST" ? { body } : {}),+ signal: AbortSignal.timeout(30_000),+ });+ };+}++interface AwsBackgroundCohort {+ deploymentId: string;+ taskArns: string[];+ manifest: DeploymentManifest;+}++export function awsBackgroundWorkBootState(config: QmConfig): { enabled: boolean; deploymentId?: string } | undefined {+ const aws = requireAws(config);+ assertAwsCallerAccount(aws);+ const manifest = currentDeploymentManifest(aws);+ const taskDefinition = manifest?.tasks.core;+ if (!taskDefinition) return undefined;+ const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [+ "ecs",+ "describe-task-definition",+ "--task-definition",+ taskDefinition,+ ]).taskDefinition;+ const containers = task?.containerDefinitions as Array<Record<string, unknown>> | undefined;+ const core = containers?.filter((container) => container.name === "core");+ if (core?.length !== 1) throw new CliError("recorded task definition must contain exactly one core container");+ const secrets = (core[0]!.secrets ?? []) as Array<{ name: string }>;+ if (secrets.some((secret) => secret.name === "BACKGROUND_WORK_ENABLED" || secret.name === "BACKGROUND_DEPLOYMENT_ID"))+ throw new CliError("recorded background boot settings cannot be supplied as secrets");+ const environment = (core[0]!.environment ?? []) as Array<{ name: string; value: string }>;+ const identities = environment.filter((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID");+ if (+ manifest.backgroundDeploymentId+ ? identities.length !== 1 || identities[0]!.value !== manifest.backgroundDeploymentId+ : identities.length !== 0+ )+ throw new CliError("recorded core task does not match the manifest background deployment identity");+ const flags = environment.filter((entry) => entry.name === "BACKGROUND_WORK_ENABLED");+ if (flags.length !== 1 || !["0", "1"].includes(flags[0]!.value))+ throw new CliError("recorded core task must contain one explicit BACKGROUND_WORK_ENABLED value of 0 or 1");+ return {+ enabled: flags[0]!.value === "1",+ ...(manifest.backgroundDeploymentId ? { deploymentId: manifest.backgroundDeploymentId } : {}),+ };+}++async function awsBackgroundCohort(+ config: QmConfig,+ configDir: string,+ candidatePath?: string,+): Promise<AwsBackgroundCohort> {+ const { aws, workloads } = awsTopology(config, configDir);+ if (!aws.backgroundWorkControl)+ throw new CliError("background ownership control is not enabled in the AWS configuration");+ const manifest = currentDeploymentManifest(aws);+ if (!manifest?.backgroundDeploymentId)+ throw new CliError("background ownership requires a manifest-bound deployment identity");+ const candidate = candidatePath ? releaseCandidate(config, candidatePath) : undefined;+ const states = describedServices(config, workloads);+ assertOwnedServices(config, states, workloads);+ const before = serviceSnapshotFromStates(states, workloads);+ for (const workload of workloads) {+ if (manifest.tasks[workload] !== before.tasks[workload] || manifest.counts?.[workload] !== before.counts[workload])+ throw new CliError(`cannot control background work while ${workload} differs from the deployment manifest`);+ const task = awsJson<{ taskDefinition?: Record<string, unknown> }>(aws, [+ "ecs",+ "describe-task-definition",+ "--task-definition",+ before.tasks[workload]!,+ ]).taskDefinition;+ const container = (task?.containerDefinitions as Array<Record<string, unknown>> | undefined)?.find(+ (item) => item.name === workload,+ );+ if (typeof container?.image !== "string" || !isPinnedWorkloadImage(config, workload, container.image))+ throw new CliError(`${workload} does not have a trusted digest-pinned image`);+ if (+ candidate &&+ (container.image !== candidate.images[workload] ||+ !candidate.imageProvenance[workload] ||+ canonicalJson(candidate.imageProvenance[workload]) !== canonicalJson(manifest.imageProvenance?.[workload]))+ )+ throw new CliError(`${workload} does not match the expected release candidate`);+ if (workload === "core") {+ const environment = (container.environment ?? []) as Array<{ name: string; value: string }>;+ if (+ environment.find((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID")?.value !==+ manifest.backgroundDeploymentId+ )+ throw new CliError("core task definition does not match the recorded background deployment identity");+ }+ }+ const count = before.counts.core;+ if (!count) throw new CliError("background ownership requires running core capacity");+ await awaitServiceTargets(config, {+ core: { taskDefinition: before.tasks.core!, desiredCount: count, waitForDrain: false },+ });+ const taskArns =+ awsJson<{ taskArns?: string[] }>(aws, [+ "ecs",+ "list-tasks",+ "--cluster",+ aws.cluster,+ "--service-name",+ aws.services.core!.ecsService,+ "--desired-status",+ "RUNNING",+ ]).taskArns ?? [];+ const tasks = chunks(taskArns, 100).flatMap((batch) => {+ const response = awsJson<{+ tasks?: Array<{ taskArn?: string; taskDefinitionArn?: string; lastStatus?: string; healthStatus?: string }>;+ failures?: unknown[];+ }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);+ if (response.failures?.length) throw new CliError("cannot prove every running core task identity");+ return response.tasks ?? [];+ });+ const current = tasks.filter(+ (task) =>+ task.taskDefinitionArn === before.tasks.core && task.lastStatus === "RUNNING" && task.healthStatus === "HEALTHY",+ );+ if (+ current.length !== count ||+ current.some((task) => !task.taskArn) ||+ new Set(current.map((task) => task.taskArn)).size !== count+ )+ throw new CliError("core does not have the exact healthy task cohort recorded in its deployment manifest");+ return { deploymentId: manifest.backgroundDeploymentId, taskArns: current.map((task) => task.taskArn!), manifest };+}++export async function awsBackgroundWorkStatus(+ config: QmConfig,+ configDir: string,+ candidatePath?: string,+): Promise<{+ status: BackgroundWorkStatus;+ deploymentId: string;+ taskArns: string[];+}> {+ assertAwsCallerAccount(requireAws(config));+ const cohort = await awsBackgroundCohort(config, configDir, candidatePath);+ const status = await readBackgroundWork(awsBackgroundWorkTransport(config), cohort.deploymentId);+ for (const taskArn of cohort.taskArns) {+ if (+ !status.members.some(+ (member) => member.taskArn === taskArn && member.deploymentId === cohort.deploymentId && !member.retired,+ )+ )+ throw new CliError("a running core task has not enrolled in the ownership protocol");+ }+ return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns };+}++export interface AwsBackgroundWorkPeer {+ config: QmConfig;+ configDir: string;+ candidatePath?: string;+}++async function withBackgroundPeerLeases<T>(peers: AwsBackgroundWorkPeer[], operation: () => Promise<T>): Promise<T> {+ const unique = new Map(+ peers.map((peer) => {+ const aws = requireAws(peer.config);+ return [+ `${aws.accountId}:${aws.region}:${deployLocksTable(aws)}:${deploymentStateKey(aws, "deploy")}`,+ aws,+ ] as const;+ }),+ );+ if (unique.size !== peers.length)+ throw new CliError("background ownership peers must have distinct deployment leases");+ const leases: Array<{ aws: AwsConfig; lease: ReturnType<typeof acquireLease> }> = [];+ try {+ for (const [, aws] of [...unique.entries()].sort(([a], [b]) => a.localeCompare(b))) {+ assertAwsCallerAccount(aws);+ leases.push({ aws, lease: acquireLease(aws) });+ }+ return await operation();+ } finally {+ for (const { aws, lease } of leases.reverse()) releaseLease(aws, lease);+ }+}++export async function awsBootstrapBackgroundWork(+ peers: AwsBackgroundWorkPeer[],+ desiredDeploymentId: string | null,+): Promise<BackgroundWorkStatus> {+ if (!peers.length) throw new CliError("background ownership bootstrap requires every participating deployment");+ return withBackgroundPeerLeases(peers, async () => {+ const cohorts = [];+ for (const peer of peers)+ cohorts.push(await awsBackgroundWorkStatus(peer.config, peer.configDir, peer.candidatePath));+ const first = cohorts[0]!;+ if (cohorts.some((cohort) => cohort.status.enabled || cohort.status.generation !== 0))+ throw new CliError("background ownership bootstrap requires disabled generation zero for every cohort");+ if (new Set(cohorts.map((cohort) => cohort.deploymentId)).size !== cohorts.length)+ throw new CliError("background ownership bootstrap requires distinct cohort identities");+ const shared = (status: BackgroundWorkStatus): string =>+ canonicalJson({+ generation: status.generation,+ desiredDeploymentId: status.desiredDeploymentId,+ members: [...status.members].sort((a, b) => a.instanceId.localeCompare(b.instanceId)),+ });+ if (cohorts.some((cohort) => shared(cohort.status) !== shared(first.status)))+ throw new CliError("background ownership peers do not agree on shared durable membership");+ const taskArns = cohorts.flatMap((cohort) => cohort.taskArns);+ if (+ new Set(taskArns).size !== taskArns.length ||+ first.status.members.some((member) => !member.retired && (!member.taskArn || !taskArns.includes(member.taskArn)))+ )+ throw new CliError("bootstrap must include every enrolled non-retired task");+ const desired =+ desiredDeploymentId === null ? undefined : cohorts.find((cohort) => cohort.deploymentId === desiredDeploymentId);+ if (desiredDeploymentId !== null && !desired)+ throw new CliError("bootstrap desired owner is not a verified peer cohort");+ const transport = awsBackgroundWorkTransport(peers[0]!.config);+ const mutation = { ...backgroundWorkMutation(0, desiredDeploymentId), bootstrapTaskArns: taskArns };+ const applied = await mutateBackgroundWork(transport, first.deploymentId, mutation);+ return awaitBackgroundWork(+ transport,+ first.deploymentId,+ { generation: applied.generation, desiredDeploymentId, taskArns: desired?.taskArns ?? [] },+ { timeoutMs: envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000), pollMs: 1000 },+ );+ });+}++export async function awsRetireBackgroundWorkMembers(+ peers: AwsBackgroundWorkPeer[],+ terminatedMembers: Array<Pick<BackgroundWorkMember, "instanceId" | "taskArn" | "generation">>,+): Promise<BackgroundWorkStatus> {+ if (!peers.length || !terminatedMembers.length)+ throw new CliError("task retirement requires deployment peers and exact member identities");+ return withBackgroundPeerLeases(peers, async () => {+ const first = await awsBackgroundWorkStatus(peers[0]!.config, peers[0]!.configDir, peers[0]!.candidatePath);+ for (const retired of terminatedMembers) {+ const member = first.status.members.find(+ (item) =>+ item.instanceId === retired.instanceId &&+ item.taskArn === retired.taskArn &&+ item.generation === retired.generation,+ );+ if (!member?.taskArn || member.retired)+ throw new CliError("task retirement does not match a live durable member");+ let proved = false;+ for (const peer of peers) {+ const aws = requireAws(peer.config);+ if (!member.deploymentId.startsWith(`${aws.services.core!.ecsService}:`)) continue;+ const response = awsJson<{+ tasks?: Array<{ taskArn?: string; taskDefinitionArn?: string; lastStatus?: string; group?: string }>;+ failures?: unknown[];+ }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", member.taskArn]);+ const task = response.tasks?.find((item) => item.taskArn === member.taskArn);+ if (+ response.failures?.length ||+ !task ||+ task.lastStatus !== "STOPPED" ||+ task.group !== `service:${aws.services.core!.ecsService}` ||+ !task.taskDefinitionArn+ )+ continue;+ const definition = awsJson<{+ taskDefinition?: {+ containerDefinitions?: Array<{ name?: string; environment?: Array<{ name: string; value: string }> }>;+ };+ }>(aws, ["ecs", "describe-task-definition", "--task-definition", task.taskDefinitionArn]).taskDefinition;+ const identity = definition?.containerDefinitions+ ?.find((container) => container.name === "core")+ ?.environment?.find((entry) => entry.name === "BACKGROUND_DEPLOYMENT_ID")?.value;+ if (identity === member.deploymentId) proved = true;+ }+ if (!proved)+ throw new CliError(+ "task retirement requires ECS STOPPED evidence bound to the exact service, task and deployment identity",+ );+ }+ return mutateBackgroundWork(awsBackgroundWorkTransport(peers[0]!.config), first.deploymentId, {+ expectedGeneration: first.status.generation,+ requestId: randomUUID(),+ terminatedMembers,+ });+ });+}++async function assertBackgroundCohortReplaceable(+ config: QmConfig,+ current: DeploymentManifest | undefined,+): Promise<void> {+ if (!current?.backgroundDeploymentId) return;+ const status = await readBackgroundWork(awsBackgroundWorkTransport(config), current.backgroundDeploymentId);+ if (+ status.enabled &&+ (status.desiredDeploymentId === current.backgroundDeploymentId ||+ status.members.some(+ (member) =>+ !member.retired && member.deploymentId === current.backgroundDeploymentId && member.state !== "drained",+ ))+ )+ throw new CliError(+ "pause or hand over background ownership and wait for every member to drain before replacing the current core cohort",+ );+}+export async function awsSetBackgroundWork(config: QmConfig,configDir: string,@@ -2723,6 +3221,32 @@ export async function awsSetBackgroundWork(}if (workload === "core") coreTask = task;}+ if (aws.backgroundWorkControl) {+ const cohort = await awsBackgroundCohort(config, configDir, candidatePath);+ const transport = awsBackgroundWorkTransport(config);+ let status = await readBackgroundWork(transport, cohort.deploymentId);+ if (!status.enabled)+ throw new CliError("explicitly bootstrap all background ownership cohorts before changing ownership");+ if (!enabled && status.desiredDeploymentId !== null && status.desiredDeploymentId !== cohort.deploymentId)+ throw new CliError("cannot disable background work on a different deployment's current owner");+ const desiredDeploymentId = enabled ? cohort.deploymentId : null;+ if (status.desiredDeploymentId !== desiredDeploymentId) {+ status = await mutateBackgroundWork(+ transport,+ cohort.deploymentId,+ backgroundWorkMutation(status.generation, desiredDeploymentId),+ );+ }+ await awaitBackgroundWork(+ transport,+ cohort.deploymentId,+ { generation: status.generation, desiredDeploymentId, taskArns: enabled ? cohort.taskArns : [] },+ { timeoutMs: envNum("QM_AWS_ROLLOUT_DEADLINE_MS", 30 * 60_000), pollMs: 1000 },+ );+ return;+ }+ if (current.backgroundDeploymentId)+ throw new CliError("controlled background work cannot fall back to task replacement");const desired = taskDefinitionForBackgroundWork(coreTask!, enabled);const core = (coreTask!.containerDefinitions as Array<Record<string, unknown>>).find((item) => item.name === "core",@@ -2820,6 +3344,8 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFil}await withAwsLease(aws, async () => {const baseline = currentDeploymentManifest(aws);+ if (baseline?.backgroundDeploymentId && !aws.backgroundWorkControl)+ throw new CliError("controlled secret activation cannot fall back to a legacy task definition");const states = describedServices(config, workloads);assertOwnedServices(config, states, workloads);const before = baseline ? serviceSnapshotFromStates(states, workloads) : undefined;@@ -2846,6 +3372,24 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFilconst affected = workloads.filter((workload) =>workloadSecrets(config, workload, uploaded).some((secret) => uploaded[secret.name]),);+ if (baseline?.backgroundDeploymentId) {+ for (const secret of staged.filter((item) =>+ ["DEPLOYMENT_CONTROL_SECRET", "CORE_SIGNING_SECRET"].includes(item.name),+ )) {+ const existing = awsText(aws, [+ "secretsmanager",+ "get-secret-value",+ "--secret-id",+ secret.id,+ "--query",+ "SecretString",+ ]);+ if (existing !== readFileSync(secret.file, "utf8"))+ throw new CliError(+ `${secret.name} cannot be rotated while controlled task cohorts are deployed; coordinated control credential rotation is required`,+ );+ }+ }for (const secret of staged) {try {awsText(aws, [@@ -2870,6 +3414,12 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFilif (affected.length) step("secret activation deferred to the first complete AWS deployment");return;}+ if (aws.backgroundWorkControl && affected.includes("core")) {+ note(+ "core secret activation deferred: pause or hand over background ownership, then run qm up --restart core to allocate a new cohort",+ );+ affected.splice(affected.indexOf("core"), 1);+ }for (const [component, host, marker, expected] of [["admin", "web-ui", "ADMIN_ENABLED", "1"],["auth", "portal", "AUTH_EMBEDDED", "1"],@@ -2912,7 +3462,7 @@ export async function awsSecretsPush(config: QmConfig, configDir: string, envFil) {throw new CliError(`cannot rotate secrets while ${workload} lacks a trusted digest-pinned image`);}- const desired = renderTaskDefinition(config, workload, container.image, arns);+ const desired = renderTaskDefinition(config, workload, container.image, arns, baseline.backgroundDeploymentId);if (!taskDefinitionChanges(desired, live).length) continue;const dir = mkdtempSync(join(tmpdir(), "qm-task-"));try {@@ -4105,7 +4655,7 @@ async function checkLive(item.value,]),);- const expectedEnv = workloadEnvironment(config, service);+ const expectedEnv = workloadEnvironment(config, service, manifest.backgroundDeploymentId);if (canonicalJson(liveEnv) !== canonicalJson(expectedEnv))failures.push(`${service}: environment drift (including live-only keys)`);const expectedSecrets = workloadSecrets(config, service, arns)@@ -4123,9 +4673,10 @@ async function checkLive(failures.push(`${service}: image drift (live ${container.image}, desired ${desiredImage})`);}const comparisonImage = desiredImage ?? `${repository}@sha256:${"0".repeat(64)}`;- const fields = taskDefinitionDiff(renderTaskDefinition(config, service, comparisonImage, arns), live).filter(- (field) => desiredImage || field !== `taskDefinition.containerDefinitions.${service}.image`,- );+ const fields = taskDefinitionDiff(+ renderTaskDefinition(config, service, comparisonImage, arns, manifest.backgroundDeploymentId),+ live,+ ).filter((field) => desiredImage || field !== `taskDefinition.containerDefinitions.${service}.image`);if (fields.length) failures.push(`${service}: task-definition drift (${fields.join(", ")})`);}}diff --git a/cli/src/background-work.ts b/cli/src/background-work.tsnew file mode 100644index 000000000..1d7e50508--- /dev/null+++ b/cli/src/background-work.ts@@ -0,0 +1,211 @@+import { randomUUID } from "node:crypto";+import { CliError } from "./log.ts";+import { sleep } from "./util.ts";++export interface BackgroundWorkMember {+ instanceId: string;+ taskArn: string | null;+ deploymentId: string;+ generation: number;+ state: "admitted" | "relinquished" | "drained";+ retired: boolean;+ ready: boolean;+}++export interface BackgroundWorkStatus {+ protocol: 1;+ enabled: boolean;+ deploymentId: string;+ instanceId: string;+ generation: number;+ desiredDeploymentId: string | null;+ lastRequestId: string | null;+ members: BackgroundWorkMember[];+}++export type BackgroundWorkTransport = (+ method: "GET" | "POST",+ body?: string,+) => Promise<{ status: number; body: string }>;++export type BackgroundWorkMutation = {+ expectedGeneration: number;+ requestId: string;+} & (+ | { desiredDeploymentId: string | null; bootstrapTaskArns?: string[] }+ | { terminatedMembers: Array<Pick<BackgroundWorkMember, "instanceId" | "taskArn" | "generation">> }+);++function record(value: unknown): value is Record<string, unknown> {+ return value !== null && typeof value === "object" && !Array.isArray(value);+}++function text(value: unknown): value is string {+ return typeof value === "string" && value.length > 0;+}++function generation(value: unknown): value is number {+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;+}++export function parseBackgroundWorkStatus(body: string, deploymentId: string): BackgroundWorkStatus {+ let value: unknown;+ try {+ value = JSON.parse(body);+ } catch {+ throw new CliError("background ownership returned invalid JSON");+ }+ if (+ !record(value) ||+ value.protocol !== 1 ||+ typeof value.enabled !== "boolean" ||+ value.deploymentId !== deploymentId ||+ !text(value.instanceId) ||+ !generation(value.generation) ||+ !(value.desiredDeploymentId === null || text(value.desiredDeploymentId)) ||+ !(value.lastRequestId === null || text(value.lastRequestId)) ||+ !Array.isArray(value.members)+ ) {+ throw new CliError("background ownership response does not match the requested deployment and protocol");+ }+ const members = value.members;+ for (const member of members) {+ if (+ !record(member) ||+ !text(member.instanceId) ||+ !(member.taskArn === null || text(member.taskArn)) ||+ !text(member.deploymentId) ||+ !generation(member.generation) ||+ member.generation > value.generation ||+ !["admitted", "relinquished", "drained"].includes(String(member.state)) ||+ typeof member.retired !== "boolean" ||+ typeof member.ready !== "boolean"+ ) {+ throw new CliError("background ownership returned an invalid member");+ }+ }+ const identities = members.map((member) => member.instanceId);+ if (new Set(identities).size !== identities.length)+ throw new CliError("background ownership returned duplicate instance identities");+ if (+ !members.some(+ (member) => member.instanceId === value.instanceId && member.deploymentId === deploymentId && !member.retired,+ )+ )+ throw new CliError("background ownership responder is not enrolled in its durable membership");+ return value as unknown as BackgroundWorkStatus;+}++export async function readBackgroundWork(+ transport: BackgroundWorkTransport,+ deploymentId: string,+): Promise<BackgroundWorkStatus> {+ const response = await transport("GET");+ if (response.status !== 200)+ throw new CliError(`background ownership read failed with HTTP ${response.status}; legacy fallback is forbidden`);+ return parseBackgroundWorkStatus(response.body, deploymentId);+}++function mutationCommitted(state: BackgroundWorkStatus, mutation: BackgroundWorkMutation): boolean {+ if (state.lastRequestId !== mutation.requestId) return false;+ if ("desiredDeploymentId" in mutation) {+ return (+ state.generation === mutation.expectedGeneration + 1 &&+ state.desiredDeploymentId === mutation.desiredDeploymentId &&+ state.enabled+ );+ }+ return (+ state.generation === mutation.expectedGeneration &&+ mutation.terminatedMembers.every((retired) =>+ state.members.some(+ (member) =>+ member.instanceId === retired.instanceId &&+ member.taskArn === retired.taskArn &&+ member.generation === retired.generation &&+ member.retired,+ ),+ )+ );+}++export async function mutateBackgroundWork(+ transport: BackgroundWorkTransport,+ deploymentId: string,+ mutation: BackgroundWorkMutation,+): Promise<BackgroundWorkStatus> {+ const body = JSON.stringify(mutation);+ let response: { status: number; body: string } | undefined;+ try {+ response = await transport("POST", body);+ } catch {+ response = undefined;+ }+ if (response?.status === 200) {+ try {+ const state = parseBackgroundWorkStatus(response.body, deploymentId);+ if (mutationCommitted(state, mutation)) return state;+ } catch {+ response = undefined;+ }+ }+ const observed = await readBackgroundWork(transport, deploymentId);+ if (mutationCommitted(observed, mutation)) return observed;+ if (observed.generation !== mutation.expectedGeneration)+ throw new CliError("background ownership changed concurrently; refusing to replace another ownership generation");+ throw new CliError(+ `background ownership mutation is unconfirmed${response ? ` (HTTP ${response.status})` : ""}; retry the same request ID ${mutation.requestId} after reading ownership`,+ );+}++export function backgroundWorkMutation(+ expectedGeneration: number,+ desiredDeploymentId: string | null,+): BackgroundWorkMutation {+ return { expectedGeneration, desiredDeploymentId, requestId: randomUUID() };+}++export async function awaitBackgroundWork(+ transport: BackgroundWorkTransport,+ deploymentId: string,+ expected: { generation: number; desiredDeploymentId: string | null; taskArns: string[] },+ options: { timeoutMs: number; pollMs: number },+): Promise<BackgroundWorkStatus> {+ const deadline = Date.now() + options.timeoutMs;+ for (;;) {+ const state = await readBackgroundWork(transport, deploymentId);+ if (+ !state.enabled ||+ state.generation !== expected.generation ||+ state.desiredDeploymentId !== expected.desiredDeploymentId+ ) {+ throw new CliError("background ownership changed while awaiting acknowledgment");+ }+ const priorOwners = state.members.filter(+ (member) => !member.retired && member.generation < expected.generation && member.state === "admitted",+ );+ const ready =+ expected.desiredDeploymentId === null+ ? state.members.every((member) => member.retired || member.state !== "admitted")+ : expected.taskArns.length > 0 &&+ expected.taskArns.every((taskArn) =>+ state.members.some(+ (member) =>+ member.taskArn === taskArn &&+ member.deploymentId === expected.desiredDeploymentId &&+ member.generation === expected.generation &&+ member.state === "admitted" &&+ member.ready &&+ !member.retired,+ ),+ );+ const unexpectedOwner = state.members.some(+ (member) =>+ !member.retired && member.state === "admitted" && member.deploymentId !== expected.desiredDeploymentId,+ );+ if (priorOwners.length === 0 && !unexpectedOwner && ready) return state;+ if (Date.now() >= deadline)+ throw new CliError("timed out awaiting background ownership acknowledgment; no member was inferred dead");+ await sleep(options.pollMs);+ }+}diff --git a/cli/src/config.ts b/cli/src/config.tsindex 02b75c123..cbea1aaef 100644--- a/cli/src/config.ts+++ b/cli/src/config.ts@@ -91,6 +91,7 @@ export interface AwsConfig {imageLabel: string;alb?: string;sharedAlb?: boolean;+ backgroundWorkControl?: boolean;rdsInstance?: string;predeployDbSnapshot?: boolean;dbRetentionMinDays?: number;@@ -749,6 +750,13 @@ function validate(raw: unknown, path: string): QmConfig {validatePortalTrust(out, path);if (target === "aws") {validateAwsFrontDoor(out, path);+ if (+ out.env.core?.BACKGROUND_DEPLOYMENT_ID !== undefined ||+ out.secretEnv?.core?.BACKGROUND_DEPLOYMENT_ID !== undefined+ )+ throw new CliError(+ `${path}: BACKGROUND_DEPLOYMENT_ID is allocated by aws.backgroundWorkControl and cannot be configured directly`,+ );const externalImages = [...plugins.filter((plugin) => plugin.image).map((plugin) => plugin.name),...Object.keys(imageOverrides).filter((service) => services.includes(service as DeclaredServiceName)),@@ -1127,6 +1135,9 @@ function validateAws();}}+ if (raw["backgroundWorkControl"] !== undefined && typeof raw["backgroundWorkControl"] !== "boolean") {+ throw new CliError(`${path}: "aws.backgroundWorkControl" must be a boolean`);+ }if (raw["sharedAlb"] !== undefined && typeof raw["sharedAlb"] !== "boolean") {throw new CliError(`${path}: "aws.sharedAlb" must be a boolean`);}@@ -1428,6 +1439,11 @@ function validateAws(if (deploymentState) out.deploymentState = deploymentState;if (alb) out.alb = alb;if (raw["sharedAlb"] !== undefined) out.sharedAlb = raw["sharedAlb"] as boolean;+ if (raw["backgroundWorkControl"] !== undefined) out.backgroundWorkControl = raw["backgroundWorkControl"] as boolean;+ if (out.backgroundWorkControl && (services.core?.ecsService.length ?? 0) > 219)+ throw new CliError(+ `${path}: controlled core service names must leave room for a unique deployment identity (maximum 219 characters)`,+ );if (rdsInstance) out.rdsInstance = rdsInstance;if (predeployDbSnapshot !== undefined) out.predeployDbSnapshot = predeployDbSnapshot;if (dbRetentionMinDays !== undefined) out.dbRetentionMinDays = dbRetentionMinDays;diff --git a/cli/src/secrets.ts b/cli/src/secrets.tsindex bc8c83806..5e6ae16b0 100644--- a/cli/src/secrets.ts+++ b/cli/src/secrets.ts@@ -4,6 +4,7 @@ import { TARGET_ENV_DEFAULTS } from "./target-env-defaults.ts";import { deploymentSecretValue } from "./util.ts";type SecretCondition =+ | { kind: "background-work-control" }| { kind: "sandbox-backend"; backend: string }| { kind: "env-equals"; service: DeclaredServiceName; name: string; value: string }| { kind: "env-in"; service: DeclaredServiceName; name: string; values: string[] }@@ -84,6 +85,13 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [description: "HMAC key shared by core and surface plugins.",generate: MINT_LOCALLY,},+ {+ name: "DEPLOYMENT_CONTROL_SECRET",+ service: "core",+ required: { when: { kind: "background-work-control" } },+ description: "Deployment ownership credential held only by core and the privileged deployment operator.",+ generate: MINT_LOCALLY,+ },{name: "CAPABILITY_SECRET",service: "core",@@ -500,6 +508,8 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [];function conditionMatches(config: QmConfig, condition: SecretCondition): boolean {+ if (condition.kind === "background-work-control")+ return config.target === "aws" && config.aws?.backgroundWorkControl === true;if (condition.kind === "sandbox-backend") {const backend =config.env.core?.SANDBOX_BACKEND ??@@ -637,6 +647,9 @@ export function computedSecrets(config: QmConfig): ComputedSecret[] {});}}+ const control = byName.get("DEPLOYMENT_CONTROL_SECRET");+ if (control && (control.services.some((service) => service !== "core") || control.aliases?.length))+ throw new Error("DEPLOYMENT_CONTROL_SECRET must be delivered only to core under its original name");const secrets = [...byName.values()].map((secret) => ({...secret,@@ -735,6 +748,7 @@ function requiresOtherEmailTransport(config: QmConfig, condition: SecretConditio}function conditionClause(condition: SecretCondition): string {+ if (condition.kind === "background-work-control") return "aws.backgroundWorkControl is enabled";if (condition.kind === "sandbox-backend")return `SANDBOX_BACKEND or SANDBOX_SCOPE_BACKENDS selects ${condition.backend}`;if (condition.kind === "service-enabled") return `the ${condition.service} service is enabled`;diff --git a/cli/src/util.ts b/cli/src/util.tsindex 0d527ad5a..0d4b3f97b 100644--- a/cli/src/util.ts+++ b/cli/src/util.ts@@ -275,7 +275,10 @@ export function isInvalidSecret(name: string, value: string | undefined): boolea);}return (- (name === "CONNECTOR_SECRET_KEY" || name === "CORE_SIGNING_SECRET" || name === "SKILL_SIGNING_SECRET") &&+ (name === "CONNECTOR_SECRET_KEY" ||+ name === "CORE_SIGNING_SECRET" ||+ name === "SKILL_SIGNING_SECRET" ||+ name === "DEPLOYMENT_CONTROL_SECRET") &&candidate.length < 32);}diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex c9a7025a2..8ee229e7c 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -14,6 +14,10 @@ import {assertAwsPublicRouting,assertGithubDeployTrust,awsCheckLive,+ awsBackgroundWorkStatus,+ awsBackgroundWorkBootState,+ awsBootstrapBackgroundWork,+ awsRetireBackgroundWorkMembers,awsDeploymentLayerTransport,awsDown,awsLogs,@@ -322,8 +326,8 @@ const statePath = ${JSON.stringify(state)};const s = JSON.parse(fs.readFileSync(statePath, "utf8"));const save = () => fs.writeFileSync(statePath, JSON.stringify(s));const after = (flag) => args[args.indexOf(flag) + 1];-if (a.includes("secretsmanager get-secret-value") && !a.includes("--query")) console.log(JSON.stringify({ ARN: "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf", SecretString: a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)}) }));-else if (a.includes("secretsmanager get-secret-value") && a.includes("--query SecretString")) console.log(a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)}));+if (a.includes("secretsmanager get-secret-value") && !a.includes("--query")) console.log(JSON.stringify({ ARN: "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf", SecretString: a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("DEPLOYMENT_CONTROL_SECRET") ? "separate-control-secret-value-0000000000" : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)})) }));+else if (a.includes("secretsmanager get-secret-value") && a.includes("--query SecretString")) console.log(a.includes("PUBLIC_API_URL") ? (process.env.AWS_FAKE_PUBLIC_API_URL || ${JSON.stringify(configured.apiUrl ?? configured.publicUrl)}) : (a.includes("DEPLOYMENT_CONTROL_SECRET") ? "separate-control-secret-value-0000000000" : (a.includes("CORE_SIGNING_SECRET") && process.env.AWS_FAKE_SECRET_VALUE || ${JSON.stringify(TEST_SECRET_VALUE)})));else if (a.includes("secretsmanager get-secret-value") && a.includes("--query ARN")) console.log("arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf");else if (a.includes("ecr get-login-password")) console.log("pw");else if (a.includes("ecr batch-get-image")) console.log(JSON.stringify({ images: [{ imageManifest: "{}", imageManifestMediaType: "application/vnd.oci.image.index.v1+json" }] }));@@ -413,8 +417,9 @@ else if (a.includes("ecs describe-service-revisions")) {}) }));}else if (a.includes("ecs list-tasks")) console.log(JSON.stringify({ taskArns: process.env.AWS_FAKE_NO_RUNNING_TASK ? [] : [...(process.env.AWS_FAKE_LARGE_ROLLOUT ? Array.from({ length: 100 }, (_, i) => "arn:aws:ecs:us-west-2:123456789012:task/old-core-" + i) : []), "arn:aws:ecs:us-west-2:123456789012:task/live-core"] }));+else if (a.includes("ecs describe-tasks") && s.stoppedTasks?.[after("--tasks")]) console.log(JSON.stringify({tasks: [s.stoppedTasks[after("--tasks")]], failures: []}));else if (a.includes("ecs describe-tasks") && a.includes("task/old-core-")) console.log(JSON.stringify({ tasks: [] }));-else if (a.includes("ecs describe-tasks") && a.includes("task/live-core")) console.log(JSON.stringify({ tasks: [{ taskDefinitionArn: process.env.AWS_FAKE_STALE_CORE ? "stale-task-definition" : s.services["acme-core"].taskDefinition, lastStatus: "RUNNING", healthStatus: "HEALTHY", containers: [{ name: "core", networkInterfaces: [{ privateIpv4Address: "10.0.1.8" }] }] }] }));+else if (a.includes("ecs describe-tasks") && a.includes("task/live-core")) console.log(JSON.stringify({ tasks: [{ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/live-core", taskDefinitionArn: process.env.AWS_FAKE_STALE_CORE ? "stale-task-definition" : s.services["acme-core"].taskDefinition, lastStatus: "RUNNING", healthStatus: "HEALTHY", containers: [{ name: "core", networkInterfaces: [{ privateIpv4Address: "10.0.1.8" }] }] }] }));else if (a.includes("ecs run-task")) console.log(JSON.stringify({ tasks: [{ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/canary" }] }));else if (a.includes("ecs wait tasks-stopped")) console.log("");else if (a.includes("ecs describe-tasks")) { const exitCode = Number(process.env.AWS_FAKE_CANARY_EXIT || "0") || ${JSON.stringify(opts.migrationExitCode ?? 0)}; console.log(JSON.stringify({ tasks: [{ stoppedReason: "Essential container in task exited", containers: [{ name: "core", exitCode, reason: process.env.AWS_FAKE_CANARY_REASON ?? (exitCode && ${JSON.stringify(Boolean(opts.migrationExitCode ?? 0))} ? "migration failed" : undefined) }] }], failures: [] })); }@@ -470,7 +475,13 @@ else if (a.includes("dynamodb transact-write-items")) {save();if (s.transactAttempts <= failTransactionPuts) { console.error("TransactionRejected"); process.exit(1); }}- for (const write of JSON.parse(after("--transact-items"))) if (write.Put) s.dynamo[write.Put.Item.lockKey.S] = write.Put.Item;+ for (const write of JSON.parse(after("--transact-items"))) {+ if (write.Put) {+ if (write.Put.ConditionExpression === "attribute_not_exists(lockKey)" && s.dynamo[write.Put.Item.lockKey.S]) { console.error("ConditionalCheckFailedException"); process.exit(1); }+ s.dynamo[write.Put.Item.lockKey.S] = write.Put.Item;+ }+ if (write.Delete) delete s.dynamo[write.Delete.Key.lockKey.S];+ }save();console.log("");}@@ -1466,6 +1477,65 @@ test("AWS up coalesces a requested restart into one deployment even when the tas}});+test("recorded background boot flags come from the manifest task and reject ambiguous settings", () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-background-boot-"));+ const configured = oneServiceConfig();+ const fake = statefulAws(dir, configured);+ try {+ assert.equal(awsBackgroundWorkBootState(configured), undefined);+ const state = JSON.parse(readFileSync(fake.state, "utf8"));+ const task = "arn:aws:ecs:us-west-2:123456789012:task-definition/acme-core:42";+ state.dynamo = manifestItems([{ id: "recorded", tasks: { core: task } }], "recorded");+ const manifest = JSON.parse(state.dynamo["deployment/manifest/recorded"].manifest.S);+ const core: {+ name: string;+ environment: Array<{ name: string; value: string }>;+ secrets: Array<{ name: string }>;+ } = {+ name: "core",+ environment: [{ name: "BACKGROUND_WORK_ENABLED", value: "1" }],+ secrets: [],+ };+ state.definitions[task] = { containerDefinitions: [core] };+ const save = () => {+ state.dynamo["deployment/manifest/recorded"].manifest.S = JSON.stringify(manifest);+ writeFileSync(fake.state, JSON.stringify(state));+ };+ save();+ assert.deepEqual(awsBackgroundWorkBootState(configured), { enabled: true });+ assert.match(readFileSync(fake.log, "utf8"), /describe-task-definition --task-definition .*acme-core:42/);+ core.environment[0]!.value = "0";+ manifest.backgroundDeploymentId = "core:recorded";+ core.environment.push({ name: "BACKGROUND_DEPLOYMENT_ID", value: "core:recorded" });+ save();+ assert.deepEqual(awsBackgroundWorkBootState(configured), { enabled: false, deploymentId: "core:recorded" });+ core.environment[1]!.value = "wrong-identity";+ save();+ assert.throws(() => awsBackgroundWorkBootState(configured), /manifest background deployment identity/);+ core.environment[1]!.value = "core:recorded";+ for (const value of ["", "yes", "2"]) {+ core.environment[0]!.value = value;+ save();+ assert.throws(() => awsBackgroundWorkBootState(configured), /explicit BACKGROUND_WORK_ENABLED/);+ }+ core.environment[0]!.value = "1";+ core.environment.push({ ...core.environment[0]! });+ save();+ assert.throws(() => awsBackgroundWorkBootState(configured), /explicit BACKGROUND_WORK_ENABLED/);+ core.environment.pop();+ core.secrets.push({ name: "BACKGROUND_WORK_ENABLED" });+ save();+ assert.throws(() => awsBackgroundWorkBootState(configured), /supplied as secrets/);+ core.secrets = [];+ core.environment.shift();+ save();+ assert.throws(() => awsBackgroundWorkBootState(configured), /explicit BACKGROUND_WORK_ENABLED/);+ } finally {+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});+test("background mode changes preserve task settings and reject secret-controlled modes", () => {const task = {family: "core",@@ -5443,3 +5513,220 @@ test("AWS deployment progress rejects invalid options before AWS calls", async (rmSync(dir, { recursive: true, force: true });}});++test("controlled AWS cohorts bind immutable identities and hand over without ECS replacement", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-controlled-background-"));+ const dockerBin = join(dir, "docker");+ writeFileSync(dockerBin, `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);+ chmodSync(dockerBin, 0o755);+ const base = oneServiceConfig();+ const single: QmConfig = { ...base, aws: { ...base.aws!, backgroundWorkControl: true } };+ const fake = statefulAws(dir, single);+ const priorPath = process.env.PATH;+ process.env.PATH = `${dir}:${priorPath}`;+ try {+ await awsUp(single, dir, { yes: true });+ const persisted = JSON.parse(readFileSync(fake.state, "utf8"));+ const manifest = JSON.parse(+ persisted.dynamo[`deployment/manifest/${persisted.dynamo["deployment/current"].manifestId.S}`].manifest.S,+ );+ assert.match(manifest.backgroundDeploymentId, /^acme-core:[0-9a-f-]{36}$/);+ assert.equal(persisted.dynamo["deployment/background-preparation"], undefined);+ const preparationCalls = readFileSync(fake.log, "utf8");+ const preparedAt = preparationCalls.indexOf('"preparation":{"S"');+ assert.ok(preparedAt >= 0 && preparedAt < preparationCalls.indexOf("ecs register-task-definition"));+ const taskArn = "arn:aws:ecs:us-west-2:123456789012:task/live-core";+ const ownership = {+ protocol: 1,+ enabled: false,+ deploymentId: manifest.backgroundDeploymentId,+ instanceId: "instance-one",+ generation: 0,+ desiredDeploymentId: null as string | null,+ lastRequestId: null as string | null,+ members: [+ {+ instanceId: "instance-one",+ taskArn,+ deploymentId: manifest.backgroundDeploymentId,+ generation: 0,+ state: "drained",+ retired: false,+ ready: false,+ },+ ],+ };+ const fetchLayer = globalThis.fetch;+ const mutations: unknown[] = [];+ globalThis.fetch = async (url, init) => {+ if (!String(url).includes("/v1/background-work")) return fetchLayer(url, init);+ const headers = new Headers(init?.headers);+ assert.equal(headers.get("authorization"), "Bearer separate-control-secret-value-0000000000");+ const body = String(init?.body ?? "");+ const signature = createHmac("sha256", TEST_SECRET_VALUE)+ .update(`v0:${headers.get("x-timestamp")}:${init?.method}\n/v1/background-work\n${body}`)+ .digest("hex");+ assert.equal(headers.get("x-signature"), `v0=${signature}`);+ if (init?.method === "POST") {+ const mutation = JSON.parse(body);+ mutations.push(mutation);+ assert.equal(mutation.expectedGeneration, ownership.generation);+ ownership.lastRequestId = mutation.requestId;+ if (mutation.terminatedMembers) {+ for (const proof of mutation.terminatedMembers) {+ const member = ownership.members.find((item) => item.instanceId === proof.instanceId)!;+ member.retired = true;+ member.state = "drained";+ member.ready = false;+ }+ } else {+ ownership.enabled = true;+ ownership.generation++;+ ownership.desiredDeploymentId = mutation.desiredDeploymentId;+ ownership.members[0]!.generation = ownership.generation;+ ownership.members[0]!.state = mutation.desiredDeploymentId ? "admitted" : "relinquished";+ ownership.members[0]!.ready = Boolean(mutation.desiredDeploymentId);+ }+ }+ return new Response(JSON.stringify(ownership), { status: 200 });+ };+ assert.deepEqual((await awsBackgroundWorkStatus(single, dir)).taskArns, [taskArn]);+ await assert.rejects(awsSetBackgroundWork(single, dir, true), /explicitly bootstrap/);+ const stoppedLegacyArn = "arn:aws:ecs:us-west-2:123456789012:task/stopped-legacy-core";+ ownership.members.push({+ ...ownership.members[0]!,+ instanceId: "legacy-dead",+ taskArn: stoppedLegacyArn,+ state: "admitted",+ });+ const beforeBootstrap = JSON.parse(readFileSync(fake.state, "utf8"));+ beforeBootstrap.stoppedTasks = {+ [stoppedLegacyArn]: {+ taskArn: stoppedLegacyArn,+ lastStatus: "STOPPED",+ group: "service:acme-core",+ taskDefinitionArn: manifest.tasks.core,+ },+ };+ writeFileSync(fake.state, JSON.stringify(beforeBootstrap));+ await assert.rejects(+ awsBootstrapBackgroundWork([{ config: single, configDir: dir }], manifest.backgroundDeploymentId),+ /every enrolled/,+ );+ const retiredLegacy = await awsRetireBackgroundWorkMembers(+ [{ config: single, configDir: dir }],+ [{ instanceId: "legacy-dead", taskArn: stoppedLegacyArn, generation: 0 }],+ );+ assert.equal(retiredLegacy.generation, 0);+ assert.equal(retiredLegacy.enabled, false);+ const bootstrapped = await awsBootstrapBackgroundWork(+ [{ config: single, configDir: dir }],+ manifest.backgroundDeploymentId,+ );+ assert.equal(bootstrapped.generation, 1);+ writeFileSync(fake.log, "");+ await assert.rejects(awsUp(single, dir, { yes: true }), /pause or hand over/);+ assert.doesNotMatch(+ readFileSync(fake.log, "utf8"),+ /ecs update-service|register-task-definition|s3api put-object|ecr get-login-password/,+ );+ ownership.desiredDeploymentId = "another-current-owner";+ const beforeWrongPause = mutations.length;+ await assert.rejects(awsSetBackgroundWork(single, dir, false), /different deployment/);+ assert.equal(mutations.length, beforeWrongPause);+ ownership.desiredDeploymentId = manifest.backgroundDeploymentId;+ await awsSetBackgroundWork(single, dir, false);+ await awsSetBackgroundWork(single, dir, true);+ assert.equal(ownership.generation, 3);+ assert.equal(mutations.length, 4);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecs update-service|register-task-definition|run-task/);+ const unchanged = JSON.parse(readFileSync(fake.state, "utf8"));+ assert.deepEqual(unchanged.dynamo, persisted.dynamo);+ await assert.rejects(+ awsRetireBackgroundWorkMembers(+ [{ config: single, configDir: dir }],+ [{ instanceId: "instance-one", taskArn, generation: 3 }],+ ),+ /STOPPED evidence/,+ );+ const stoppedArn = "arn:aws:ecs:us-west-2:123456789012:task/stopped-core";+ ownership.members.push({+ ...ownership.members[0]!,+ instanceId: "terminated-instance",+ taskArn: stoppedArn,+ generation: 1,+ });+ unchanged.stoppedTasks = {+ [stoppedArn]: {+ taskArn: stoppedArn,+ lastStatus: "STOPPED",+ group: "service:acme-core",+ taskDefinitionArn: manifest.tasks.core,+ },+ };+ writeFileSync(fake.state, JSON.stringify(unchanged));+ const retired = await awsRetireBackgroundWorkMembers(+ [{ config: single, configDir: dir }],+ [{ instanceId: "terminated-instance", taskArn: stoppedArn, generation: 1 }],+ );+ assert.equal(retired.generation, 3);+ assert.equal(retired.members.at(-1)!.retired, true);+ await awsSetBackgroundWork(single, dir, false);+ await assert.rejects(awsUp(single, dir, { yes: true, restart: ["core"] }), /every member to drain/);+ await assert.rejects(awsRollback(single, manifest.id), /every member to drain/);+ ownership.members[0]!.state = "drained";+ await awsUp(single, dir, { yes: true, restart: ["core"] });+ const replacement = JSON.parse(readFileSync(fake.state, "utf8"));+ const replacementManifest = JSON.parse(+ replacement.dynamo[`deployment/manifest/${replacement.dynamo["deployment/current"].manifestId.S}`].manifest.S,+ );+ assert.notEqual(replacementManifest.backgroundDeploymentId, manifest.backgroundDeploymentId);+ assert.notEqual(replacementManifest.tasks.core, manifest.tasks.core);+ ownership.deploymentId = replacementManifest.backgroundDeploymentId;+ ownership.members[0]!.deploymentId = replacementManifest.backgroundDeploymentId;+ const controlEnv = join(dir, "control.env");+ writeFileSync(+ controlEnv,+ computedSecrets(single)+ .filter((secret) => secret.managedBy !== "terraform")+ .map(+ (secret) =>+ `${secret.name}=${secret.name === "DEPLOYMENT_CONTROL_SECRET" ? "new-control-secret".repeat(3) : TEST_SECRET_VALUE}`,+ )+ .join("\n"),+ );+ writeFileSync(fake.log, "");+ await assert.rejects(awsSecretsPush(single, dir, controlEnv), /cannot be rotated/);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /put-secret-value|ecs update-service/);+ replacement.dynamo["deployment/background-preparation"] = {+ preparation: {+ S: JSON.stringify({+ id: "unresolved",+ previousManifestId: "different",+ backgroundDeploymentId: "another",+ before: { tasks: {}, counts: {} },+ }),+ },+ };+ writeFileSync(fake.state, JSON.stringify(replacement));+ await assert.rejects(awsUp(single, dir, { yes: true }), /remains unresolved/);+ delete replacement.dynamo["deployment/background-preparation"];+ writeFileSync(fake.state, JSON.stringify(replacement));+ ownership.deploymentId = "stale-cohort";+ await assert.rejects(awsSetBackgroundWork(single, dir, false), /requested deployment/);+ assert.equal(mutations.length, 6);+ ownership.deploymentId = replacementManifest.backgroundDeploymentId;+ await awsRollback(single, manifest.id);+ const rolledBack = JSON.parse(readFileSync(fake.state, "utf8"));+ assert.equal(rolledBack.dynamo["deployment/current"].manifestId.S, manifest.id);+ assert.equal(rolledBack.services["acme-core"].taskDefinition, manifest.tasks.core);+ await assert.rejects(+ awsSetBackgroundWork({ ...single, aws: { ...single.aws!, backgroundWorkControl: false } }, dir, true),+ /cannot fall back/,+ );+ } finally {+ process.env.PATH = priorPath;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});diff --git a/cli/test/background-work.test.ts b/cli/test/background-work.test.tsnew file mode 100644index 000000000..ee34d5438--- /dev/null+++ b/cli/test/background-work.test.ts@@ -0,0 +1,135 @@+import test from "node:test";+import assert from "node:assert/strict";+import {+ awaitBackgroundWork,+ mutateBackgroundWork,+ parseBackgroundWorkStatus,+ type BackgroundWorkStatus,+ type BackgroundWorkTransport,+} from "../src/background-work.ts";++const state = (): BackgroundWorkStatus => ({+ protocol: 1,+ enabled: true,+ deploymentId: "new-cohort",+ instanceId: "new-instance",+ generation: 2,+ desiredDeploymentId: "new-cohort",+ lastRequestId: "our-request",+ members: [+ {+ instanceId: "old-instance",+ taskArn: "old-task",+ deploymentId: "old-cohort",+ generation: 1,+ state: "relinquished",+ retired: false,+ ready: true,+ },+ {+ instanceId: "new-instance",+ taskArn: "new-task",+ deploymentId: "new-cohort",+ generation: 2,+ state: "admitted",+ retired: false,+ ready: true,+ },+ ],+});++const response = (value: BackgroundWorkStatus) => ({ status: 200, body: JSON.stringify(value) });++test("ownership status refuses stale responders and malformed durable membership", () => {+ assert.equal(parseBackgroundWorkStatus(JSON.stringify(state()), "new-cohort").generation, 2);+ assert.throws(() => parseBackgroundWorkStatus(JSON.stringify(state()), "old-cohort"), /requested deployment/);+ const duplicate = state();+ duplicate.members.push(duplicate.members[0]!);+ assert.throws(() => parseBackgroundWorkStatus(JSON.stringify(duplicate), "new-cohort"), /duplicate/);+ const unenrolled = state();+ unenrolled.instanceId = "missing";+ assert.throws(() => parseBackgroundWorkStatus(JSON.stringify(unenrolled), "new-cohort"), /not enrolled/);+});++test("lost mutation responses require the exact request ID and generation on durable readback", async () => {+ const calls: string[] = [];+ const mutation = { expectedGeneration: 1, desiredDeploymentId: "new-cohort", requestId: "our-request" };+ let observed = state();+ const transport: BackgroundWorkTransport = async (method, body) => {+ calls.push(method);+ if (method === "POST") {+ assert.deepEqual(JSON.parse(body!), mutation);+ throw new Error("connection lost after commit");+ }+ return response(observed);+ };+ assert.equal((await mutateBackgroundWork(transport, "new-cohort", mutation)).generation, 2);+ assert.deepEqual(calls, ["POST", "GET"]);+ observed = { ...state(), lastRequestId: "someone-else" };+ await assert.rejects(mutateBackgroundWork(transport, "new-cohort", mutation), /changed concurrently/);+});++test("retirement acknowledgment binds every terminated instance, task and generation", async () => {+ const mutation = {+ expectedGeneration: 2,+ requestId: "retirement",+ terminatedMembers: [{ instanceId: "old-instance", taskArn: "old-task", generation: 1 }],+ };+ const observed = state();+ observed.lastRequestId = "retirement";+ observed.members[0]!.retired = true;+ const transport: BackgroundWorkTransport = async () => response(observed);+ assert.equal((await mutateBackgroundWork(transport, "new-cohort", mutation)).generation, 2);+ observed.members[0]!.taskArn = "different-task";+ await assert.rejects(mutateBackgroundWork(transport, "new-cohort", mutation), /unconfirmed/);+});++test("activation waits for every old owner and every expected new task without waiting for old turns to drain", async () => {+ const observed = state();+ observed.members[0]!.state = "admitted";+ let polls = 0;+ const transport: BackgroundWorkTransport = async () => {+ polls++;+ if (polls === 2) observed.members[0]!.state = "relinquished";+ if (polls === 3)+ observed.members.push({+ ...observed.members[1]!,+ instanceId: "second-instance",+ taskArn: "second-task",+ ready: false,+ });+ if (polls === 4) observed.members[2]!.ready = true;+ return response(observed);+ };+ const ready = await awaitBackgroundWork(+ transport,+ "new-cohort",+ { generation: 2, desiredDeploymentId: "new-cohort", taskArns: ["new-task", "second-task"] },+ { timeoutMs: 10_000, pollMs: 1 },+ );+ assert.equal(polls, 4);+ assert.equal(ready.members[0]!.state, "relinquished");+});++test("pause never infers unacknowledged owners dead and refuses concurrent generations", async () => {+ const observed = state();+ observed.desiredDeploymentId = null;+ observed.members[0]!.state = "admitted";+ observed.members[1]!.state = "drained";+ const transport: BackgroundWorkTransport = async () => response(observed);+ const expected = { generation: 2, desiredDeploymentId: null, taskArns: [] };+ await assert.rejects(+ awaitBackgroundWork(transport, "new-cohort", expected, { timeoutMs: 0, pollMs: 1 }),+ /no member was inferred dead/,+ );+ observed.members[0]!.state = "relinquished";+ assert.equal(+ (await awaitBackgroundWork(transport, "new-cohort", expected, { timeoutMs: 0, pollMs: 1 })).generation,+ 2,+ );+ observed.generation = 3;+ await assert.rejects(+ awaitBackgroundWork(transport, "new-cohort", expected, { timeoutMs: 0, pollMs: 1 }),+ /changed while awaiting/,+ );+});diff --git a/cli/test/config.test.ts b/cli/test/config.test.tsindex 2b1d103e7..e4b4cf940 100644--- a/cli/test/config.test.ts+++ b/cli/test/config.test.ts@@ -1309,3 +1309,28 @@ test("screening defaults off and model screening requires an explicit backend",);}});++test("AWS ownership control is opt-in and reserves deployment identity allocation", () => {+ const aws = {+ accountId: "123456789012",+ region: "us-west-2",+ cluster: "acme",+ deployRoleArn: "arn:aws:iam::123456789012:role/deploy",+ secretsPrefix: "acme/",+ imageLabel: "release",+ networking: { cloudMapNamespace: "acme.internal" },+ services: { core: { ecrRepository: "core", ecsService: "acme-core", cpu: 512, memory: 1024 } },+ };+ withConfig({ target: "aws", aws }, ({ path }) =>+ assert.equal(loadConfigAt(path).config.aws!.backgroundWorkControl, undefined),+ );+ withConfig({ target: "aws", aws: { ...aws, backgroundWorkControl: true } }, ({ path }) =>+ assert.equal(loadConfigAt(path).config.aws!.backgroundWorkControl, true),+ );+ withConfig({ target: "aws", aws: { ...aws, backgroundWorkControl: "true" } }, ({ path }) =>+ assert.throws(() => loadConfigAt(path), /backgroundWorkControl/),+ );+ withConfig({ target: "aws", aws, env: { core: { BACKGROUND_DEPLOYMENT_ID: "reused" } } }, ({ path }) =>+ assert.throws(() => loadConfigAt(path), /allocated/),+ );+});diff --git a/cli/test/secrets.test.ts b/cli/test/secrets.test.tsindex 131c24bee..9c7375b7e 100644--- a/cli/test/secrets.test.ts+++ b/cli/test/secrets.test.ts@@ -419,3 +419,36 @@ test("shared Fly publishing requires private peers only when selected", () => {}}});++test("deployment control credentials are conditional and cannot be delivered to plugins or sandboxes", () => {+ assert.equal(+ computedSecrets(makeConfig()).some((secret) => secret.name === "DEPLOYMENT_CONTROL_SECRET"),+ false,+ );+ const aws = { backgroundWorkControl: true } as NonNullable<QmConfig["aws"]>;+ const config = makeConfig({ target: "aws", aws });+ const control = secretByName(config, "DEPLOYMENT_CONTROL_SECRET");+ assert.equal(control.required, true);+ assert.deepEqual(control.services, ["core"]);+ assert.throws(+ () =>+ computedSecrets({+ ...config,+ plugins: [{ name: "external", secrets: [{ name: "DEPLOYMENT_CONTROL_SECRET" }] }],+ }),+ /only to core/,+ );+ assert.throws(+ () => computedSecrets({ ...config, sandbox: { app: "sandbox", secretEnv: ["DEPLOYMENT_CONTROL_SECRET"] } }),+ /only to core/,+ );+ assert.throws(+ () =>+ computedSecrets({+ ...config,+ services: ["core", "portal"],+ secretEnv: { portal: { TOKEN: "DEPLOYMENT_CONTROL_SECRET" } },+ }),+ /only to core/,+ );+});diff --git a/src/deployment/secret-schema.ts b/src/deployment/secret-schema.tsindex 78798361f..d474798b9 100644--- a/src/deployment/secret-schema.ts+++ b/src/deployment/secret-schema.ts@@ -2,6 +2,7 @@ import { isStrongSigningSecret } from "../auth/source-auth.ts";type SecretGate =| "production"+ | "background-work-control"| "codex"| "postgres"| "sprites"@@ -31,6 +32,7 @@ export const CORE_SECRET_SPECS: readonly RuntimeSecretSpec[] = [{ name: "CAPABILITY_SECRET", requiredWhen: "production" },{ name: "CONNECTOR_SECRET_KEY", requiredWhen: "production" },{ name: "CORE_SIGNING_SECRET", requiredWhen: "production" },+ { name: "DEPLOYMENT_CONTROL_SECRET", requiredWhen: "background-work-control" },{ name: "PORTAL_IDENTITY_SECRET", requiredWhen: "production" },{ name: "SKILL_SIGNING_SECRET", requiredWhen: "production" },{ name: "AUTH_ALLOWED_EMAILS", requiredWhen: "email-auth" },@@ -63,6 +65,7 @@ function sandboxBackendSelected(env: NodeJS.ProcessEnv, backend: string): booleaconst GATE_PREDICATES: Readonly<Record<SecretGate, (env: NodeJS.ProcessEnv) => boolean>> = {production: (env) => env.NODE_ENV === "production",+ "background-work-control": (env) => Boolean(env.BACKGROUND_DEPLOYMENT_ID?.trim()),codex: (env) => env.HARNESS?.trim() === "codex" && !env.CODEX_AUTH_FILE?.trim() && !env.CODEX_AUTH_CREDENTIAL?.trim(),postgres: (env) => env.SESSION_STORE === "postgres" || env.RUN_STORE === "postgres",sprites: (env) => sandboxBackendSelected(env, "sprites"),@@ -102,6 +105,7 @@ function isInvalidSecret(name: string, value: string | undefined): boolean {return ((name === "CONNECTOR_SECRET_KEY" ||name === "CORE_SIGNING_SECRET" ||+ name === "DEPLOYMENT_CONTROL_SECRET" ||name === "SKILL_SIGNING_SECRET" ||name === "AWS_DEPLOY_GATE_SECRET") &&!isStrongSigningSecret(candidate)diff --git a/test/secret-schema-drift.test.ts b/test/secret-schema-drift.test.tsindex 6ceab52bd..2def94644 100644--- a/test/secret-schema-drift.test.ts+++ b/test/secret-schema-drift.test.ts@@ -16,6 +16,26 @@ test("the standalone CLI and core agree on runtime-enforced core secret names",assert.deepEqual([...runtime].sort(), [...cli].filter((name) => name !== "PUBLIC_API_URL").sort());});+test("background ownership requires strong control credentials only for configured deployments", () => {+ assert.deepEqual(validateCoreSecretEnv({}), []);+ for (const secret of [undefined, "", " ".repeat(32), "short"]) {+ assert.deepEqual(+ validateCoreSecretEnv({+ BACKGROUND_DEPLOYMENT_ID: "core:release",+ DEPLOYMENT_CONTROL_SECRET: secret,+ }),+ ["DEPLOYMENT_CONTROL_SECRET"],+ );+ }+ assert.deepEqual(+ validateCoreSecretEnv({+ BACKGROUND_DEPLOYMENT_ID: "core:release",+ DEPLOYMENT_CONTROL_SECRET: "separate-control-secret-value-0000000000",+ }),+ [],+ );+});+test('deploy/core/Dockerfile pins NODE_ENV=production — the "production" secret gate is load-bearing on that line', () => {const dockerfile = readFileSync(new URL("../deploy/core/Dockerfile", import.meta.url), "utf8");assert.match(dockerfile, /^ENV NODE_ENV=production$/m);