Multiplayer agent harness for work.
Parallelize AWS web updates with unchanged verified routing (#1300)
Regan Bellec86d8b603a6
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex df6e3ba36..a05f42690 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -1902,6 +1902,55 @@ function adoptMissingWorkloads(return true;}+function unchangedWebRouting(+ config: QmConfig,+ manifest: DeploymentManifest | undefined,+ desired: Array<{ service: string; task: EcsTaskDefinition }>,+): boolean {+ if (!manifest) return false;+ const workloads = ["web-ui", "portal"];+ if (workloads.some((workload) => !desired.some((item) => item.service === workload))) return false;+ try {+ const states = describedServices(config, workloads);+ for (const workload of workloads) {+ const count = manifest.counts?.[workload];+ const definition = manifest.tasks[workload];+ if (!definition || !count || count !== workloadDesiredCount(config, workload)) return false;+ stableWorkloadCapacity(config, workload, states.get(workload)!, definition, count);+ const prior = awsJson<{ taskDefinition?: Record<string, unknown> }>(requireAws(config), [+ "ecs",+ "describe-task-definition",+ "--task-definition",+ definition,+ ]).taskDefinition;+ const next = desired.find((item) => item.service === workload)!.task;+ const keys = workload === "portal" ? ["WEB_UI_UPSTREAM", "ADMIN_UPSTREAM"] : ["ADMIN_ENABLED"];+ const settings = (task: Record<string, unknown> | undefined): string[] | undefined => {+ const containers = task?.containerDefinitions as Array<Record<string, unknown>> | undefined;+ const matches = containers?.filter((container) => container.name === workload);+ if (matches?.length !== 1) return undefined;+ const container = matches[0]!;+ const secrets = (container.secrets ?? []) as Array<{ name: string }>;+ const environment = (container.environment ?? []) as Array<{ name: string; value: string }>;+ const values: string[] = [];+ for (const key of keys) {+ const entries = environment.filter((entry) => entry.name === key);+ if (secrets.some((entry) => entry.name === key) || entries.length !== 1 || !entries[0]!.value)+ return undefined;+ values.push(entries[0]!.value);+ }+ return values;+ };+ const previous = settings(prior);+ const upcoming = settings(next as unknown as Record<string, unknown>);+ if (!previous || !upcoming || canonicalJson(previous) !== canonicalJson(upcoming)) return false;+ }+ return true;+ } catch {+ return false;+ }+}+async function applyServiceTargets(config: QmConfig,targets: Record<string, string>,@@ -2503,7 +2552,10 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOptObject.fromEntries(Object.keys(rolloutTargets).map((service) => [service, workloadDesiredCount(config, service)]),),- { webBeforePortal: true, onSubmitted: reportProgress ? () => reportProgress(targets) : undefined },+ {+ webBeforePortal: !unchangedWebRouting(config, current, desired),+ onSubmitted: reportProgress ? () => reportProgress(targets) : undefined,+ },);applied = true;} else {@@ -3087,6 +3139,87 @@ export interface AwsBackgroundWorkCapacity {workloads: Record<string, { taskDefinition: string; deploymentId: string; desiredCount: number; taskArns: string[] }>;}+function stableWorkloadCapacity(+ config: QmConfig,+ workload: string,+ state: EcsServiceState,+ taskDefinition: string,+ desiredCount: number | undefined,+): AwsBackgroundWorkCapacity["workloads"][string] {+ const aws = requireAws(config);+ const primary = state.deployments?.filter((deployment) => deployment.status === "PRIMARY") ?? [];+ const deployment = primary[0];+ if (+ typeof desiredCount !== "number" ||+ !Number.isSafeInteger(desiredCount) ||+ desiredCount < 0 ||+ state.status !== "ACTIVE" ||+ state.taskDefinition !== taskDefinition ||+ state.desiredCount !== desiredCount ||+ state.runningCount !== desiredCount ||+ state.pendingCount !== 0 ||+ primary.length !== 1 ||+ !deployment?.id ||+ deployment.taskDefinition !== taskDefinition ||+ deployment.runningCount !== desiredCount ||+ state.deployments?.some((item) => item.pendingCount !== 0 || (item !== deployment && item.runningCount !== 0)) ||+ (state.deploymentConfiguration?.strategy === "BLUE_GREEN"+ ? nativeBlueGreenStatus(config, workload, taskDefinition) !== "SUCCESSFUL"+ : deployment.rolloutState !== "COMPLETED")+ )+ throw new CliError(`${workload} has not reached stable deployment capacity`);+ const listed = new Set<string>();+ for (const desiredStatus of ["RUNNING", "STOPPED"]) {+ const response = awsJson<{ taskArns?: string[] }>(aws, [+ "ecs",+ "list-tasks",+ "--cluster",+ aws.cluster,+ "--service-name",+ aws.services[workload]!.ecsService,+ "--desired-status",+ desiredStatus,+ ]);+ if (!Array.isArray(response.taskArns) || response.taskArns.some((arn) => typeof arn !== "string" || !arn))+ throw new CliError(`${workload} task inventory is unavailable`);+ for (const arn of response.taskArns) listed.add(arn);+ }+ const live: string[] = [];+ for (const batch of chunks([...listed], 100)) {+ const response = awsJson<{+ tasks?: Array<{+ taskArn?: string;+ taskDefinitionArn?: string;+ lastStatus?: string;+ desiredStatus?: string;+ healthStatus?: string;+ }>;+ failures?: unknown[];+ }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);+ if (+ response.failures?.length ||+ !Array.isArray(response.tasks) ||+ response.tasks.length !== batch.length ||+ new Set(response.tasks.map((task) => task.taskArn)).size !== batch.length ||+ response.tasks.some((task) => !task.taskArn || !batch.includes(task.taskArn))+ )+ throw new CliError(`${workload} capacity requires complete task inventory`);+ for (const task of response.tasks) {+ if (task.lastStatus === "STOPPED") continue;+ if (+ task.lastStatus !== "RUNNING" ||+ task.desiredStatus !== "RUNNING" ||+ task.taskDefinitionArn !== taskDefinition ||+ (task.healthStatus !== "HEALTHY" && (isServiceName(workload) || task.healthStatus !== "UNKNOWN"))+ )+ throw new CliError(`${workload} still has a pending, unhealthy, or retiring task`);+ live.push(task.taskArn!);+ }+ }+ if (live.length !== desiredCount) throw new CliError(`${workload} does not have its exact stable task cohort`);+ return { taskDefinition, deploymentId: deployment.id, desiredCount, taskArns: live.sort() };+}+export async function awsBackgroundWorkCapacity(config: QmConfig,configDir: string,@@ -3126,80 +3259,13 @@ export async function awsBackgroundWorkCapacity(workloads: {},};for (const workload of [...workloads].sort()) {- const state = states.get(workload)!;- const taskDefinition = cohort.manifest.tasks[workload]!;- const desiredCount = cohort.manifest.counts?.[workload];- const primary = state.deployments?.filter((deployment) => deployment.status === "PRIMARY") ?? [];- const deployment = primary[0];- if (- typeof desiredCount !== "number" ||- !Number.isSafeInteger(desiredCount) ||- desiredCount < 0 ||- state.status !== "ACTIVE" ||- state.taskDefinition !== taskDefinition ||- state.desiredCount !== desiredCount ||- state.runningCount !== desiredCount ||- state.pendingCount !== 0 ||- primary.length !== 1 ||- !deployment?.id ||- deployment.taskDefinition !== taskDefinition ||- deployment.runningCount !== desiredCount ||- state.deployments?.some((item) => item.pendingCount !== 0 || (item !== deployment && item.runningCount !== 0)) ||- (state.deploymentConfiguration?.strategy === "BLUE_GREEN"- ? nativeBlueGreenStatus(config, workload, taskDefinition) !== "SUCCESSFUL"- : deployment.rolloutState !== "COMPLETED")- )- throw new CliError(`${workload} has not reached stable deployment capacity`);- const listed = new Set<string>();- for (const desiredStatus of ["RUNNING", "STOPPED"]) {- const response = awsJson<{ taskArns?: string[] }>(aws, [- "ecs",- "list-tasks",- "--cluster",- aws.cluster,- "--service-name",- aws.services[workload]!.ecsService,- "--desired-status",- desiredStatus,- ]);- if (!Array.isArray(response.taskArns) || response.taskArns.some((arn) => typeof arn !== "string" || !arn))- throw new CliError(`${workload} task inventory is unavailable`);- for (const arn of response.taskArns) listed.add(arn);- }- const live: string[] = [];- for (const batch of chunks([...listed], 100)) {- const response = awsJson<{- tasks?: Array<{- taskArn?: string;- taskDefinitionArn?: string;- lastStatus?: string;- desiredStatus?: string;- healthStatus?: string;- }>;- failures?: unknown[];- }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", ...batch]);- if (- response.failures?.length ||- !Array.isArray(response.tasks) ||- response.tasks.length !== batch.length ||- new Set(response.tasks.map((task) => task.taskArn)).size !== batch.length ||- response.tasks.some((task) => !task.taskArn || !batch.includes(task.taskArn))- )- throw new CliError(`${workload} capacity requires complete task inventory`);- for (const task of response.tasks) {- if (task.lastStatus === "STOPPED") continue;- if (- task.lastStatus !== "RUNNING" ||- task.desiredStatus !== "RUNNING" ||- task.taskDefinitionArn !== taskDefinition ||- (task.healthStatus !== "HEALTHY" && (isServiceName(workload) || task.healthStatus !== "UNKNOWN"))- )- throw new CliError(`${workload} still has a pending, unhealthy, or retiring task`);- live.push(task.taskArn!);- }- }- if (live.length !== desiredCount) throw new CliError(`${workload} does not have its exact stable task cohort`);- proof.workloads[workload] = { taskDefinition, deploymentId: deployment.id, desiredCount, taskArns: live.sort() };+ proof.workloads[workload] = stableWorkloadCapacity(+ config,+ workload,+ states.get(workload)!,+ cohort.manifest.tasks[workload]!,+ cohort.manifest.counts?.[workload],+ );}if (canonicalJson(proof.workloads.core?.taskArns) !== canonicalJson([...cohort.taskArns].sort()))throw new CliError("core task cohort changed while checking capacity");diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 3c778e2de..e90339863 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -383,7 +383,7 @@ else if (a.includes("ecs describe-services")) {{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "IN_PROGRESS", runningCount: service.desiredCount, failedTasks: 1 },{ id: "old-protected", status: "ACTIVE", taskDefinition: service.taskDefinition, rolloutState: "COMPLETED", runningCount: 1, failedTasks: 0 },]- : ${JSON.stringify(opts.rolloutFailed ?? false)}+ : (${JSON.stringify(opts.rolloutFailed ?? false)} || s.failedTaskDefinition === service.taskDefinition)? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "FAILED", runningCount: 0, failedTasks: 1 }]: ${JSON.stringify(opts.primaryFailedTasks ?? false)} || transientlyFailing? [{ id: service.deploymentId, status: "PRIMARY", taskDefinition: service.taskDefinition, rolloutState: "IN_PROGRESS", runningCount: 0, failedTasks: 1 }]@@ -427,6 +427,8 @@ else if (a.includes("ecs get-task-protection")) {const requested = args.slice(args.indexOf("--tasks") + 1).filter(arg => arg.startsWith("arn:"));console.log(JSON.stringify(s.protectionResponse || {protectedTasks: requested.map(taskArn => ({taskArn, protectionEnabled:false})), failures:[]}));}+else if (a.includes("ecs list-tasks") && s.taskInventory?.[after("--service-name")]) console.log(JSON.stringify({ taskArns: after("--desired-status") === "STOPPED" ? [] : s.taskInventory[after("--service-name")].map(task => task.taskArn) }));+else if (a.includes("ecs describe-tasks") && Object.values(s.taskInventory || {}).flat().some(task => args.includes(task.taskArn))) console.log(JSON.stringify({ tasks: Object.values(s.taskInventory).flat().filter(task => args.includes(task.taskArn)), failures: [] }));else if (a.includes("ecs list-tasks")) console.log(JSON.stringify({ taskArns: process.env.AWS_FAKE_NO_RUNNING_TASK ? [] : [...(process.env.AWS_FAKE_LARGE_ROLLOUT ? Array.from({ length: 100 }, (_, i) => "arn:aws:ecs:us-west-2:123456789012:task/old-core-" + i) : []), "arn:aws:ecs:us-west-2:123456789012:task/live-core"] }));else if (a.includes("ecs describe-tasks") && s.stoppedTasks?.[after("--tasks")]) console.log(JSON.stringify({tasks: [s.stoppedTasks[after("--tasks")]], failures: []}));else if (a.includes("ecs describe-tasks") && a.includes("task/old-core-")) console.log(JSON.stringify({ tasks: [] }));@@ -456,6 +458,7 @@ else if (a.includes("ecs update-service")) {if (!${JSON.stringify(opts.ignoreUpdate ?? false)} && args.includes("--task-definition")) service.taskDefinition = after("--task-definition");if (!${JSON.stringify(opts.ignoreUpdate ?? false)} && args.includes("--desired-count")) service.desiredCount = Number(after("--desired-count"));if (!${JSON.stringify(opts.ignoreUpdate ?? false)}) service.deploymentId = "ecs-svc/" + name + "-" + (++s.revision);+ if (s.failNextWebRollout && service.workload === "web-ui") { s.failedTaskDefinition = service.taskDefinition; s.failNextWebRollout = false; }s.updated = true;s.drainPolls = ${JSON.stringify(opts.drainPolls ?? 0)};s.blueGreenPolls = 0;@@ -6084,3 +6087,82 @@ test("controlled AWS cohorts bind immutable identities and hand over without ECSrmSync(dir, { recursive: true, force: true });}});++for (const mode of ["unchanged", "migration", "missing", "unstable", "failure"] as const) {+ test(`AWS web routing transition gate: ${mode}`, async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-web-routing-"));+ writeFileSync(join(dir, "docker"), `#!/usr/bin/env node\nconsole.log("Digest: sha256:${"a".repeat(64)}");\n`);+ chmodSync(join(dir, "docker"), 0o755);+ const priorPath = process.env.PATH;+ process.env.PATH = `${dir}:${priorPath}`;+ const fake = statefulAws(dir, config, {}, { drainPolls: 4 });+ try {+ await awsUp(config, dir, { yes: true });+ const baseline = JSON.parse(readFileSync(fake.state, "utf8"));+ baseline.drainPolls = 0;+ baseline.taskInventory = {};+ for (const workload of ["web-ui", "portal"]) {+ const service = baseline.services[`acme-${workload}`];+ const task = baseline.definitions[service.taskDefinition];+ task.containerDefinitions[0].image = task.containerDefinitions[0].image.replace(+ /sha256:[a-f0-9]+/,+ `sha256:${"b".repeat(64)}`,+ );+ baseline.taskInventory[`acme-${workload}`] = Array.from({ length: service.desiredCount }, (_, index) => ({+ taskArn: `arn:aws:ecs:us-west-2:123456789012:task/${workload}-${index}`,+ taskDefinitionArn: service.taskDefinition,+ desiredStatus: "RUNNING",+ lastStatus: "RUNNING",+ healthStatus: "HEALTHY",+ }));+ }+ const portal = baseline.definitions[baseline.services["acme-portal"].taskDefinition].containerDefinitions[0];+ if (mode === "migration")+ portal.environment.find((entry: { name: string }) => entry.name === "ADMIN_UPSTREAM").value =+ "http://admin.acme.local:8080";+ if (mode === "missing")+ portal.environment = portal.environment.filter((entry: { name: string }) => entry.name !== "ADMIN_UPSTREAM");+ if (mode === "unstable") baseline.taskInventory["acme-web-ui"][0].desiredStatus = "STOPPED";+ if (mode === "failure") baseline.failNextWebRollout = true;+ writeFileSync(fake.state, JSON.stringify(baseline));+ writeFileSync(fake.log, "");+ if (mode === "failure")+ await assert.rejects(() => awsUp(config, dir, { yes: true }), /PRIMARY rollout is FAILED/);+ else await awsUp(config, dir, { yes: true });+ const calls = readFileSync(fake.log, "utf8").trim().split("\n");+ const webUpdate = calls.findIndex(+ (line) => line.includes("ecs update-service") && line.includes("--service acme-web-ui"),+ );+ const portalUpdate = calls.findIndex(+ (line) => line.includes("ecs update-service") && line.includes("--service acme-portal"),+ );+ assert.ok(webUpdate >= 0 && portalUpdate > webUpdate);+ const between = calls.slice(webUpdate + 1, portalUpdate).filter((line) => line.includes("ecs describe-services"));+ if (mode === "unchanged" || mode === "failure")+ assert.equal(between.length, 0, "both workloads submit before rollout polling");+ else assert.ok(between.length >= 5, "uncertain routing waits for full old web retirement");+ if (mode === "failure") {+ const after = JSON.parse(readFileSync(fake.state, "utf8"));+ for (const workload of ["web-ui", "portal"]) {+ assert.equal(+ after.services[`acme-${workload}`].taskDefinition,+ baseline.services[`acme-${workload}`].taskDefinition,+ );+ assert.equal(+ calls.filter((line) => line.includes("ecs update-service") && line.includes(`--service acme-${workload}`))+ .length,+ 2,+ );+ }+ assert.equal(+ after.dynamo["deployment/current"].manifestId.S,+ baseline.dynamo["deployment/current"].manifestId.S,+ );+ }+ } finally {+ process.env.PATH = priorPath;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+ });+}diff --git a/docs/combined-services.md b/docs/combined-services.mdindex 7c27222c7..1697f0a17 100644--- a/docs/combined-services.md+++ b/docs/combined-services.md@@ -10,7 +10,7 @@ The modules retain separate handlers. Admin continues verifying signed portal id1. Record the existing release's source, image digests, configuration, secret versions, and workload sizes. Preserve database backups and the previous workloads for rollback.2. Review the combined workloads' CPU and memory allocations, execution-role access to the union of their component secrets, and network access needed by the auth mail transport. Resolve conflicting environment or secret mappings before deployment.-3. Build web-ui and portal from the same source release. The CLI orders AWS and Fly deployments with web-ui before portal. Deploy the combined web-ui before changing portal's admin upstream to its `/admin` endpoint. Deploy portal with its embedded auth configuration and both names for the broker client secret. Keep the public issuer, callback URL, session keys, token secret, and signing JWK unchanged.+3. Build web-ui and portal from the same source release. The CLI orders AWS routing transitions and Fly deployments with web-ui before portal. Ordinary AWS image updates can start both workloads together when the recorded and live cohorts are stable and their web/admin routing and enabled admin capability are unchanged. Missing or uncertain evidence preserves the ordered rollout. Deploy the combined web-ui before changing portal's admin upstream to its `/admin` endpoint. Deploy portal with its embedded auth configuration and both names for the broker client secret. Keep the public issuer, callback URL, session keys, token secret, and signing JWK unchanged.On AWS, a secrets push uploads values but defers task activation when the live tasks still use the separate-service configuration. Complete the image rollout with `qm up`; keep existing signing and session secret values during this migration.4. Verify signed-in chat, administrator and non-administrator access, login, logout, remembered sessions, and a live admin operation. Verify forged admin cookies are rejected, token endpoints are not publicly exposed, and streaming responses still work.