Multiplayer agent harness for work.
Support isolated deployments on shared AWS foundations (#1186)
Regan Bell518b68c5d84f
diff --git a/cli/src/aws-lease.ts b/cli/src/aws-lease.tsindex 2d29eff19..6995d877f 100644--- a/cli/src/aws-lease.ts+++ b/cli/src/aws-lease.ts@@ -18,10 +18,15 @@ export function awsText(aws: AwsConfig, args: string[]): string {}export function deployLocksTable(aws: AwsConfig): string {- return `${aws.cluster}-deploy-locks`;+ return aws.deploymentState?.table ?? `${aws.cluster}-deploy-locks`;+}++export function deploymentStateKey(aws: AwsConfig, key: string): string {+ return aws.deploymentState ? `${aws.deploymentState.namespace}/${key}` : key;}export function acquireAwsLease(aws: AwsConfig, key = "deploy"): AwsLease {+ key = deploymentStateKey(aws, key);const table = deployLocksTable(aws);const holder = randomUUID();const now = Math.floor(Date.now() / 1000);diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 8efcd4530..f5f7a86d1 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -9,6 +9,7 @@ import {acquireAwsLease as acquireLease,awsText,deployLocksTable,+ deploymentStateKey,releaseAwsLease as releaseLease,withAwsLease,} from "../aws-lease.ts";@@ -1431,7 +1432,7 @@ function deploymentManifest(aws: AwsConfig, id: string): DeploymentManifest {"--table-name",deployLocksTable(aws),"--key",- JSON.stringify({ lockKey: { S: `${DEPLOYMENT_MANIFEST_PREFIX}${id}` } }),+ JSON.stringify({ lockKey: { S: deploymentStateKey(aws, `${DEPLOYMENT_MANIFEST_PREFIX}${id}`) } }),"--consistent-read",]);const raw = dynamoString(response.Item, "manifest");@@ -1454,7 +1455,7 @@ function deploymentManifestAtPointer(aws: AwsConfig, key: string): DeploymentMan"--table-name",deployLocksTable(aws),"--key",- JSON.stringify({ lockKey: { S: key } }),+ JSON.stringify({ lockKey: { S: deploymentStateKey(aws, key) } }),"--consistent-read",]);const id = dynamoString(response.Item, "manifestId");@@ -1469,7 +1470,7 @@ function manifestTransaction(aws: AwsConfig, manifest: DeploymentManifest | undePut: {TableName: table,Item: {- lockKey: { S: `${DEPLOYMENT_MANIFEST_PREFIX}${manifest.id}` },+ lockKey: { S: deploymentStateKey(aws, `${DEPLOYMENT_MANIFEST_PREFIX}${manifest.id}`) },manifest: { S: JSON.stringify(manifest) },},},@@ -1479,7 +1480,7 @@ function manifestTransaction(aws: AwsConfig, manifest: DeploymentManifest | undePut: {TableName: table,Item: {- lockKey: { S: `deployment/label/${manifest.imageLabel}` },+ lockKey: { S: deploymentStateKey(aws, `deployment/label/${manifest.imageLabel}`) },manifestId: { S: manifest.id },},},@@ -1489,7 +1490,7 @@ function manifestTransaction(aws: AwsConfig, manifest: DeploymentManifest | undePut: {TableName: table,Item: {- lockKey: { S: DEPLOYMENT_POINTER_KEY },+ lockKey: { S: deploymentStateKey(aws, DEPLOYMENT_POINTER_KEY) },manifestId: { S: pointerId },},},@@ -2295,7 +2296,7 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt});}if (recorded && dbRestorePoint) confirmDbRestorePointCovered(config, dbRestorePoint);- for (const service of services) {+ for (const service of candidate ? [] : services) {try {promoteStagedImage(config, service, images[service]!, label);promotedServices.add(service);@@ -3338,14 +3339,18 @@ export function assertAwsPublicRouting(}}if (- targetGroups.length !== expectedTargetArns.size ||- targetGroups.some((group) => !group.TargetGroupArn || !expectedTargetArns.has(group.TargetGroupArn))+ !aws.sharedAlb &&+ (targetGroups.length !== expectedTargetArns.size ||+ targetGroups.some((group) => !group.TargetGroupArn || !expectedTargetArns.has(group.TargetGroupArn)))) {throw new Error("ALB has target groups for private or unknown services");}if (alternateTargets.size && alternateTargets.size !== ingress.length) {throw new Error("ALB cannot mix rolling and blue/green ingress services");}+ if (aws.sharedAlb && (!hasPortal || coreHosts.length || publicPaths.size || alternateTargets.size !== 1)) {+ throw new Error("shared ALB requires one blue/green portal ingress without additional public routes");+ }const defaults = listener.DefaultActions ?? [];if (alternateTargets.size) {if (@@ -3392,6 +3397,35 @@ export function assertAwsPublicRouting(}>;}>(aws, ["elbv2", "describe-rules", "--listener-arn", listener.ListenerArn!]).Rules ?? [];let nonDefault = rules.filter((rule) => !rule.IsDefault);+ if (aws.sharedAlb) {+ const hostname = new URL(config.publicUrl).hostname.toLowerCase();+ const ownRule = productionRules.get("portal");+ for (const rule of nonDefault) {+ const hosts = rule.Conditions?.filter((condition) => condition.Field === "host-header");+ const values = hosts?.[0]?.HostHeaderConfig?.Values ?? hosts?.[0]?.Values ?? [];+ if (hosts?.length !== 1 || !values.length || values.some((value) => !validAlbHostname(value))) {+ throw new Error("shared ALB rules require explicit non-wildcard hostnames");+ }+ const matchesHost = values.some((value) => value.toLowerCase() === hostname);+ if (rule.RuleArn === ownRule) {+ if (!matchesHost || values.length !== 1)+ throw new Error("shared ALB portal rule must match only this company hostname");+ } else {+ const referencesOwnTarget = rule.Actions?.some(+ (action) =>+ expectedTargetArns.has(action.TargetGroupArn ?? "") ||+ action.ForwardConfig?.TargetGroups?.some((target) => expectedTargetArns.has(target.TargetGroupArn ?? "")),+ );+ if (matchesHost || referencesOwnTarget) throw new Error("shared ALB has an overlapping company route");+ }+ }+ nonDefault = nonDefault+ .filter((rule) => rule.RuleArn === ownRule)+ .map((rule) => ({+ ...rule,+ Conditions: rule.Conditions?.filter((condition) => condition.Field !== "host-header"),+ }));+ }const pluginRules = new Set<(typeof nonDefault)[number]>();for (const [name, paths] of publicPaths) {const matches = nonDefault.filter((rule) =>diff --git a/cli/src/config.ts b/cli/src/config.tsindex d71494a3a..bf17e63c7 100644--- a/cli/src/config.ts+++ b/cli/src/config.ts@@ -80,6 +80,7 @@ export interface AwsServiceConfig {}export interface AwsConfig {+ deploymentState?: { table: string; namespace: string };accountId: string;region: string;cluster: string;@@ -87,6 +88,7 @@ export interface AwsConfig {secretsPrefix: string;imageLabel: string;alb?: string;+ sharedAlb?: boolean;rdsInstance?: string;predeployDbSnapshot?: boolean;dbRetentionMinDays?: number;@@ -1076,6 +1078,22 @@ function validateAws(`${path}: "aws.imageLabel" must be a valid OCI/ECR tag (1-128 letters, digits, underscores, periods, or hyphens; the first character cannot be a period or hyphen)`,);}+ let deploymentState: AwsConfig["deploymentState"];+ if (raw["deploymentState"] !== undefined) {+ const state = raw["deploymentState"];+ if (+ !isPlainObject(state) ||+ typeof state.table !== "string" ||+ !/^[A-Za-z0-9_.-]{3,255}$/.test(state.table) ||+ typeof state.namespace !== "string" ||+ !/^[a-z0-9][a-z0-9-]{0,62}$/.test(state.namespace)+ ) {+ throw new CliError(+ `${path}: "aws.deploymentState" requires a valid DynamoDB table name and a lowercase company namespace`,+ );+ }+ deploymentState = { table: state.table, namespace: state.namespace };+ }let alb: string | undefined;if (raw["alb"] !== undefined) {alb = requiredString(raw["alb"], "alb");@@ -1085,6 +1103,12 @@ function validateAws();}}+ if (raw["sharedAlb"] !== undefined && typeof raw["sharedAlb"] !== "boolean") {+ throw new CliError(`${path}: "aws.sharedAlb" must be a boolean`);+ }+ if (raw["sharedAlb"] === true && !alb) {+ throw new CliError(`${path}: "aws.sharedAlb" requires "aws.alb"`);+ }let rdsInstance: string | undefined;if (raw["rdsInstance"] !== undefined) {rdsInstance = requiredString(raw["rdsInstance"], "rdsInstance");@@ -1377,7 +1401,9 @@ function validateAws(networking: netOut,services,};+ if (deploymentState) out.deploymentState = deploymentState;if (alb) out.alb = alb;+ if (raw["sharedAlb"] !== undefined) out.sharedAlb = raw["sharedAlb"] as boolean;if (rdsInstance) out.rdsInstance = rdsInstance;if (predeployDbSnapshot !== undefined) out.predeployDbSnapshot = predeployDbSnapshot;if (dbRetentionMinDays !== undefined) out.dbRetentionMinDays = dbRetentionMinDays;diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex e02fbd675..299e0734f 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -10,6 +10,7 @@ import { dirname, join } from "node:path";import {assertAwsDeploymentStorage,assertAwsPublicListener,+ assertAwsPublicRouting,assertGithubDeployTrust,awsCheckLive,awsDeploymentLayerTransport,@@ -57,6 +58,9 @@ function fakeAws(frontService: "core" | "portal" = "portal",ingress: {blueGreen?: boolean;+ sharedHost?: string;+ siblingHost?: string;+ siblingOwnTarget?: boolean;coreHosts?: string[];targetGroups?: Partial<Record<"core" | "portal", string>>;} = {},@@ -119,6 +123,19 @@ function fakeAws(},];}+ if (ingress.sharedHost) {+ (baseRules[0]!.Conditions as unknown[]).push({+ Field: "host-header",+ HostHeaderConfig: { Values: [ingress.sharedHost] },+ });+ groups.push({ TargetGroupArn: "other-target", TargetGroupName: "other-target" });+ baseRules.push({+ RuleArn: "other-rule",+ IsDefault: false,+ Actions: [{ Type: "forward", TargetGroupArn: ingress.siblingOwnTarget ? targetArn : "other-target" }],+ Conditions: [{ Field: "host-header", HostHeaderConfig: { Values: [ingress.siblingHost ?? "other.example"] } }],+ });+ }writeFileSync(log, "");writeFileSync(bin,@@ -2665,7 +2682,7 @@ test("AWS builds one immutable candidate manifest and deploys its exact digest wwriteFileSync(fake.log, "");await awsUp(single, dir, { yes: true, candidate: candidatePath });const deployCalls = readFileSync(fake.log, "utf8");- assert.match(deployCalls, new RegExp(`ecr batch-get-image .*imageDigest=sha256:${"a".repeat(64)}`));+ assert.doesNotMatch(deployCalls, /ecr put-image|ecr batch-delete-image|ecr initiate-layer-upload/);assert.match(deployCalls, /ecs run-task .*src\/migrate-main\.ts/);assert.ok(deployCalls.indexOf("rds describe-db-instances") < deployCalls.indexOf("ecs run-task"));assert.ok(deployCalls.indexOf("ecs run-task") < deployCalls.indexOf("ecs update-service"));@@ -4981,3 +4998,96 @@ test("AWS secrets push defers activation against pre-consolidation images", asynrmSync(dir, { recursive: true, force: true });}});++test("shared deployment state isolates company manifests and leases without mutating candidate images", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-aws-inactive-candidate-"));+ const candidatePath = join(dir, "candidate.json");+ writeFileSync(+ candidatePath,+ JSON.stringify({+ contract: 1,+ accountId: "123456789012",+ region: "us-west-2",+ label: "candidate-deadbeef",+ images: {+ core: `123456789012.dkr.ecr.us-west-2.amazonaws.com/qm-core@sha256:${"a".repeat(64)}`,+ },+ imageProvenance: { core: { kind: "source-build", source: "checkout" } },+ }),+ );+ const single = oneServiceConfig();+ single.aws!.deploymentState = { table: "fleet-deploy-locks", namespace: "acme" };+ const fake = statefulAws(dir, single);+ const prior = process.env.AWS_FAKE_ALB_DNS;+ process.env.AWS_FAKE_ALB_DNS = "192.0.2.1";+ try {+ await awsUp(single, dir, { yes: true, candidate: candidatePath, inactive: true });+ assert.match(readFileSync(fake.log, "utf8"), /ecs update-service/);+ const first = JSON.parse(readFileSync(fake.state, "utf8"));+ const pointer = first.dynamo["acme/deployment/current"].manifestId.S;+ assert.ok(first.dynamo[`acme/deployment/manifest/${pointer}`]);+ assert.ok(first.dynamo["acme/deployment/label/" + single.aws!.imageLabel]);+ assert.equal(first.dynamo["deployment/current"], undefined);+ assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecr put-image|ecr batch-delete-image/);+ single.aws!.deploymentState.namespace = "second";+ await awsUp(single, dir, { yes: true, candidate: candidatePath, inactive: true });+ const second = JSON.parse(readFileSync(fake.state, "utf8"));+ assert.equal(second.dynamo["acme/deployment/current"].manifestId.S, pointer);+ assert.notEqual(second.dynamo["second/deployment/current"].manifestId.S, pointer);+ const calls = readFileSync(fake.log, "utf8");+ assert.match(calls, /"S":"acme\/deploy"/);+ assert.match(calls, /"S":"second\/deploy"/);+ } finally {+ if (prior === undefined) delete process.env.AWS_FAKE_ALB_DNS;+ else process.env.AWS_FAKE_ALB_DNS = prior;+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+});++test("shared ALB validates company routes while rejecting sibling overlap", () => {+ const company = { ...config, aws: { ...config.aws!, sharedAlb: true, alb: "shared" } };+ const name = `acme-qm-port-${createHash("sha1").update("acme-qm:portal").digest("hex").slice(0, 6)}`;+ const primary = `arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/${name}/1`;+ const alternateName = `acme-qm-port-g-${createHash("sha1").update("acme-qm:portal:alternate").digest("hex").slice(0, 6)}`;+ const alternate = `arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/${alternateName}/2`;+ const services = new Map([+ [+ "portal",+ {+ loadBalancers: [+ {+ targetGroupArn: primary,+ advancedConfiguration: {+ alternateTargetGroupArn: alternate,+ productionListenerRule:+ "arn:aws:elasticloadbalancing:us-west-2:123456789012:listener-rule/app/test/1/2/production",+ },+ },+ ],+ },+ ],+ ]);+ const hostname = new URL(company.publicUrl).hostname;+ for (const variant of ["valid", "same-host", "wildcard", "own-target", "wrong-host"]) {+ const dir = mkdtempSync(join(tmpdir(), "qm-shared-alb-"));+ const fake = fakeAws(dir, "", "portal", {+ blueGreen: true,+ sharedHost: variant === "wrong-host" ? "wrong.example" : hostname,+ siblingHost:+ ({ "same-host": hostname, wildcard: "*.example" } as Record<string, string>)[variant] ?? "other.example",+ siblingOwnTarget: variant === "own-target",+ });+ try {+ if (variant === "valid") assert.equal(assertAwsPublicRouting(company, services).get("portal"), primary);+ else assert.throws(() => assertAwsPublicRouting(company, services), /shared ALB/);+ assert.throws(+ () => assertAwsPublicRouting({ ...company, aws: { ...company.aws, sharedAlb: false } }, services),+ /unknown services/,+ );+ } finally {+ fake.restore();+ rmSync(dir, { recursive: true, force: true });+ }+ }+});diff --git a/docs/deploy-directory.md b/docs/deploy-directory.mdindex 92ad5cdbb..3a6ee4e79 100644--- a/docs/deploy-directory.md+++ b/docs/deploy-directory.md@@ -182,3 +182,11 @@ rules. Deployment preflight verifies the exact paths, target-group attachmentsand listener precedence, including native blue/green weighted routing. Updatevendored Terraform with the `public_paths` service attribute and`public_path_services` routing before enabling this setting.++### Shared AWS foundations++Existing AWS deployments retain their company-owned resources by default. For a shared DynamoDB deployment table, set `aws.deploymentState` to `{ "table": "fleet-deploy-locks", "namespace": "acme" }`. Every lease, current pointer, label pointer and immutable manifest uses that namespace. Use a unique namespace for each company and scope its item IAM permissions with `dynamodb:LeadingKeys` to `acme/*`; grant `DescribeTable` separately. Changing an existing deployment's state location requires explicitly migrating its history while deployment operations are stopped.++To use a shared ALB, set `aws.alb` and `aws.sharedAlb: true`. The supported shared topology has one blue/green portal ingress per company, with an exact host-header condition plus `/*` path, a fixed-404 listener default, separate primary/alternate target groups and a company-owned production rule. Additional public plugin/core routes are not supported in this topology. Other companies' rules must use explicit non-wildcard hostnames and must not reference this company's target groups. Set each service's `targetGroup`, `taskRoleArn`, and `executionRoleArn` explicitly when sharing an ECS cluster. Retain company-specific service permissions and allow ECS to modify only that company's listener rule.++An exact `--candidate` deployment consumes its digest-pinned images without changing ECR tags. Its publisher owns image retention; retain every active and rollback digest. Builds performed by the deployment CLI still stage and promote their images as before.