Multiplayer agent harness for work.
Deploy the egress proxy as a regular qm service (#2266)
Regan Bellaf4b106ec846
diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.ymlindex a79d77cbc..7c1af4fa3 100644--- a/.github/workflows/publish-cli.yml+++ b/.github/workflows/publish-cli.yml@@ -57,7 +57,7 @@ jobs:run: |set -euo pipefailout=$(jq -n '{sandboxBase: "", services: {}}')- for service in core web-ui admin portal auth sandbox-base; do+ for service in core web-ui admin portal auth egress-proxy sandbox-base; dorepo="yc-software/qm/$service"token=$(curl -fsS "https://ghcr.io/token?scope=repository:$repo:pull&service=ghcr.io" | jq -r .token)digest=$(curl -fsS -o /dev/null -D - -H "Authorization: Bearer $token" \@@ -78,7 +78,7 @@ jobs:printf '%s\n' "$out" > cli/manifest.jsoncat cli/manifest.jsonjq -e '[.sandboxBase] + [.services[]]- | length == 6+ | length == 7and all(test("^ghcr\\.io/yc-software/qm/[a-z-]+@sha256:[0-9a-f]{64}$"))and all(test("@sha256:(.)\\1{63}$") | not)' cli/manifest.json > /dev/nullprintf 'manifest=%s\n' "$(jq -c . cli/manifest.json)" >> "$GITHUB_OUTPUT"diff --git a/cli/manifest.json b/cli/manifest.jsonindex 3dcb8bfda..fd7db393a 100644--- a/cli/manifest.json+++ b/cli/manifest.json@@ -5,6 +5,7 @@"web-ui": "registry.invalid/qm/qm-web-ui@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","admin": "registry.invalid/qm/qm-admin@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","portal": "registry.invalid/qm/qm-portal@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",- "auth": "registry.invalid/qm/qm-auth@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ "auth": "registry.invalid/qm/qm-auth@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "egress-proxy": "registry.invalid/qm/qm-egress-proxy@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 86334e10e..4ce355a93 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -380,7 +380,12 @@ export function serviceEnvironment(config: QmConfig, service: ServiceName): Reco: {};const env: Record<string, string> = {...orgEnv(service, config.orgId, config.publicUrl, config.services.includes("portal"), brandEnvOf(config)),- ...(service === "core" ? {} : { CORE_API_URL: coreUrl }),+ ...(service === "core"+ ? {}+ : {+ CORE_API_URL:+ isServiceName(service) && serviceDef(service).tcpIngress ? (config.apiUrl ?? config.publicUrl) : coreUrl,+ }),...coreEnv,...hostedServiceEnv(config.services, config.env, service),...(service === "core" ? securityScreenEnv(config) : {}),@@ -498,7 +503,8 @@ export function renderTaskDefinition(const aws = requireAws(config);const spec = aws.services[service];if (!spec) throw new CliError(`aws.services.${service} is missing`);- const internalPort = isServiceName(service) ? serviceDef(service).docker.internalPort : 8080;+ const def = isServiceName(service) ? serviceDef(service) : undefined;+ const internalPort = def?.docker.internalPort ?? 8080;const executionRoleArn = spec.executionRoleArn ?? `arn:aws:iam::${aws.accountId}:role/${aws.cluster}-task-execution`;const managedTaskRole = spec.assumeRoleArns?.length? `${aws.cluster}-${service}-task`@@ -512,9 +518,9 @@ export function renderTaskDefinition(})),).sort((a, b) => a.name.localeCompare(b.name));- const healthCheck = isServiceName(service)+ const healthCheck = def? {- command: [+ command: def.tcpIngress?.healthCheck ?? ["CMD","node","-e",@@ -553,7 +559,7 @@ export function renderTaskDefinition(containerPort: internalPort,hostPort: internalPort,protocol: "tcp",- appProtocol: "http",+ ...(def?.tcpIngress ? {} : { appProtocol: "http" }),},],...(healthCheck ? { healthCheck } : {}),@@ -4053,7 +4059,11 @@ export function assertAwsPublicRouting(}}for (const name of deployedAwsServices(aws)) {- if (!ingress.includes(name) && (routingServices.get(name)?.loadBalancers ?? []).length) {+ if (+ !ingress.includes(name) &&+ !(isServiceName(name) && serviceDef(name).tcpIngress) &&+ (routingServices.get(name)?.loadBalancers ?? []).length+ ) {throw new Error(`private ECS service ${name} is attached to a load balancer`);}}diff --git a/cli/src/backends/dev-ci.ts b/cli/src/backends/dev-ci.tsindex 227413dcb..db5453071 100644--- a/cli/src/backends/dev-ci.ts+++ b/cli/src/backends/dev-ci.ts@@ -87,7 +87,7 @@ export async function devCiUp(): Promise<void> {const baseEnv = { ...process.env, ...overlay } as NodeJS.ProcessEnv;const coreLog = join(dir, "core.log");- const corePid = spawnBackground(process.execPath, [...serviceDef("core").dev.entry], {+ const corePid = spawnBackground(process.execPath, [...serviceDef("core").dev!.entry], {cwd: root,env: { ...baseEnv, ORG_ID: "acme", SESSION_STORE: "memory", PORT: String(port) },logFile: coreLog,diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.tsindex 1222d212c..9318ac623 100644--- a/cli/src/backends/fly.ts+++ b/cli/src/backends/fly.ts@@ -669,6 +669,24 @@ function ensureFlycast(app: string): void {note(`flycast: allocated private v6 for ${app}`);}+function ensurePublicIngress(app: string): void {+ let ips: Array<{ Type?: string }>;+ try {+ ips = JSON.parse(fly(["ips", "list", "-a", app, "--json"]) || "[]");+ } catch {+ throw new CliError(`fly ips list returned invalid JSON for ${app}`);+ }+ const types = new Set(ips.map((ip) => ip.Type));+ if (!types.has("v6")) {+ fly(["ips", "allocate-v6", "-a", app]);+ note(`public ingress: allocated v6 for ${app}`);+ }+ if (!types.has("v4") && !types.has("shared_v4")) {+ fly(["ips", "allocate-v4", "--shared", "-a", app]);+ note(`public ingress: allocated shared v4 for ${app}`);+ }+}+interface FlyImageDetail {digest: string;immutableReference: string;@@ -848,6 +866,7 @@ async function deployService(note(`\n=== deploying ${app} ===`);if (service === "core") timing.time(service, "Postgres ensure", () => ensurePostgres(ctx));if (spec.flycast) timing.time(service, "Flycast ensure", () => ensureFlycast(app));+ if (spec.publicIngress) timing.time(service, "public ingress ensure", () => ensurePublicIngress(app));const cfgPath = serviceCfgPath(ctx, service);const args = ["deploy", "--yes", "-c", cfgPath, ...spec.deployFlags];if (imageSource?.kind === "tagged") {diff --git a/cli/src/provider-scaffold.ts b/cli/src/provider-scaffold.tsindex c5c4c8361..570161ffe 100644--- a/cli/src/provider-scaffold.ts+++ b/cli/src/provider-scaffold.ts@@ -69,8 +69,9 @@ ${values.providerFields}// First-party services to run. The full set: "core" (the agent runtime and API,// always required), "slack" (the Slack surface, runs inside the core process),// "web-ui" (a browser chat UI), "admin" (the operator dashboard), "portal"- // (the public sign-in front door), and "auth" (the built-in sign-in broker that- // emails one-time links; drop it to sign in through an external OIDC provider).+ // (the public sign-in front door), "auth" (the built-in sign-in broker that+ // emails one-time links; drop it to sign in through an external OIDC provider),+ // and "egress-proxy" (the TLS proxy that enforces sandbox egress policy)."services": [${values.services.map((service) => JSON.stringify(service)).join(", ")}],// Extra plugins to run alongside the services. A prebuilt image runs withdiff --git a/cli/src/secrets.ts b/cli/src/secrets.tsindex f31e07818..b62c3b630 100644--- a/cli/src/secrets.ts+++ b/cli/src/secrets.ts@@ -402,6 +402,20 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [description: "HMAC key shared by core and surface plugins.",generate: MINT_LOCALLY,},+ {+ name: "CAPABILITY_SECRET",+ service: "egress-proxy",+ required: true,+ description: "Signing key for scoped agent capabilities and egress grants; must differ from every other key.",+ generate: MINT_LOCALLY,+ },+ {+ name: "CORE_SIGNING_SECRET",+ service: "egress-proxy",+ required: true,+ description: "HMAC key shared by core and surface plugins.",+ generate: MINT_LOCALLY,+ },{name: "AUTH_SIGNING_JWK",service: "auth",diff --git a/cli/src/services.ts b/cli/src/services.tsindex 7588c7197..05ff07a6d 100644--- a/cli/src/services.ts+++ b/cli/src/services.ts@@ -1,4 +1,4 @@-export const SERVICE_NAMES = ["core", "web-ui", "admin", "portal", "auth"] as const;+export const SERVICE_NAMES = ["core", "web-ui", "admin", "portal", "auth", "egress-proxy"] as const;export type ServiceName = (typeof SERVICE_NAMES)[number];export const isServiceName = (s: string): s is ServiceName => (SERVICE_NAMES as readonly string[]).includes(s);@@ -86,13 +86,14 @@ interface FlyServiceSpec {stackKeys: string[];deployFlags: string[];flycast?: boolean;+ publicIngress?: boolean;}export interface ServiceDef {name: ServiceName;readiness: RegExp;order: number;- dev: {+ dev?: {cwd: string;entry: string[];portEnv?: "PORT";@@ -105,6 +106,7 @@ export interface ServiceDef {hostPortOffset?: number;};fly?: FlyServiceSpec;+ tcpIngress?: { healthCheck: string[] };}export interface BrandEnv {@@ -321,6 +323,26 @@ const CATALOG: Record<ServiceName, ServiceDef> = {flycast: true,},},+ "egress-proxy": {+ name: "egress-proxy",+ readiness: /starting main dispatch loop/,+ order: 50,+ docker: { image: "egress-proxy", internalPort: 48080, portEnv: "PORT", hostPortOffset: 4 },+ fly: {+ managed: (s) => pluginWiring("egress-proxy", s),+ stackKeys: [],+ deployFlags: ["--ha=false"],+ publicIngress: true,+ },+ tcpIngress: {+ healthCheck: [+ "CMD",+ "node",+ "-e",+ "require('net').connect(48081,'127.0.0.1').on('connect',()=>require('http').get({socketPath:'/tmp/envoy-admin.sock',path:'/ready'},r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))).on('error',()=>process.exit(1))",+ ],+ },+ },};export const serviceDef = (name: ServiceName): ServiceDef => CATALOG[name];diff --git a/cli/src/terraform.ts b/cli/src/terraform.tsindex e4dc2b8e4..e04979656 100644--- a/cli/src/terraform.ts+++ b/cli/src/terraform.ts@@ -280,7 +280,8 @@ export function assertTerraformScaffoldSupportsConfig(config: QmConfig, configDiconst hasPublicPaths = services.some((service) => service?.publicPaths?.length);const hasAssumeRoles = services.some((service) => service?.assumeRoleArns !== undefined);const hasDatabaseOverrides = config.aws?.dbInstanceClass !== undefined;- if (!hasPublicPaths && !hasAssumeRoles && !hasDatabaseOverrides) return;+ const hasEgressProxy = Boolean(config.aws?.services["egress-proxy"]);+ if (!hasPublicPaths && !hasAssumeRoles && !hasDatabaseOverrides && !hasEgressProxy) return;const tfvarsPath = join(configDir, "infra", "terraform.tfvars");if (!existsSync(tfvarsPath)) return;const variablesPath = join(configDir, "infra", "variables.tf");@@ -300,6 +301,11 @@ export function assertTerraformScaffoldSupportsConfig(config: QmConfig, configDi"the vendored AWS scaffold predates aws.services.*.publicPaths; update infra/variables.tf and infra/main.tf before exposing plugins",);}+ if (hasEgressProxy && !/resource\s+"aws_lb"\s+"egress_proxy"/.test(main)) {+ throw new CliError(+ "the vendored AWS scaffold predates the egress-proxy service; update infra/main.tf and infra/outputs.tf from the current scaffold before enabling it",+ );+ }if (hasAssumeRoles &&(!/assume_role_arns\s*=\s*optional/.test(variables) ||diff --git a/cli/templates/aws/main.tf b/cli/templates/aws/main.tfindex 1a312b53a..6050c2eae 100644--- a/cli/templates/aws/main.tf+++ b/cli/templates/aws/main.tf@@ -8,6 +8,7 @@ locals {public_path_services = { for name, service in var.services : name => service.public_paths if length(service.public_paths) > 0 }public_service_names = concat(local.has_portal ? concat(["portal"], length(var.core_public_hosts) > 0 ? ["core"] : []) : ["core"], keys(local.public_path_services))ingress_services = { for name, service in var.services : name => service if contains(local.public_service_names, name) }+ egress_proxy = { for name, service in var.services : name => service if name == "egress-proxy" }direct_path_services = merge(local.has_portal ? {} : { core = ["/v1/*"] }, local.public_path_services)alb_name = "${substr(var.cluster_name, 0, 23)}-${substr(sha1(var.cluster_name), 0, 8)}"service_security_groups = [aws_security_group.services.id]@@ -830,6 +831,63 @@ resource "aws_lb_listener" "public" {}}+resource "aws_security_group" "egress_proxy" {+ for_each = local.egress_proxy+ name = "${var.cluster_name}-egress-proxy"+ vpc_id = local.vpc_id+ ingress {+ from_port = each.value.internal_port+ to_port = each.value.internal_port+ protocol = "tcp"+ cidr_blocks = [aws_vpc.this.cidr_block]+ }+ egress {+ from_port = 0+ to_port = 0+ protocol = "-1"+ cidr_blocks = ["0.0.0.0/0"]+ }+ tags = local.tags+}++resource "aws_lb" "egress_proxy" {+ for_each = local.egress_proxy+ name = "${substr(var.cluster_name, 0, 20)}-egr-${substr(sha1("${var.cluster_name}:${each.key}"), 0, 6)}"+ load_balancer_type = "network"+ subnets = local.subnet_ids+ enable_cross_zone_load_balancing = true+ tags = local.tags+}++resource "aws_lb_target_group" "egress_proxy" {+ for_each = local.egress_proxy+ name = "${substr(var.cluster_name, 0, 20)}-egr-${substr(sha1("${var.cluster_name}:${each.key}:tcp"), 0, 6)}"+ port = each.value.internal_port+ protocol = "TCP"+ target_type = "ip"+ vpc_id = local.vpc_id+ tags = local.tags+}++resource "aws_lb_listener" "egress_proxy" {+ for_each = local.egress_proxy+ load_balancer_arn = aws_lb.egress_proxy[each.key].arn+ port = 443+ protocol = "TLS"+ certificate_arn = var.certificate_arn+ ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06"+ default_action {+ type = "forward"+ target_group_arn = aws_lb_target_group.egress_proxy[each.key].arn+ }+ lifecycle {+ precondition {+ condition = var.certificate_arn != ""+ error_message = "the egress-proxy service requires certificate_arn: sandboxes present capability tokens to it, so it only listens on TLS"+ }+ }+}+resource "aws_cloudfront_distribution" "portal" {enabled = trueis_ipv6_enabled = true@@ -969,13 +1027,13 @@ resource "aws_ecs_task_definition" "bootstrap" {operating_system_family = "LINUX"cpu_architecture = each.value.architecture == "amd64" ? "X86_64" : "ARM64"}- container_definitions = jsonencode([{ name = each.key, image = "public.ecr.aws/docker/library/alpine:3.20", essential = true, command = ["sh", "-c", "while true; do nc -l -p ${each.value.internal_port} -e echo ok; done"], portMappings = [{ name = each.key, containerPort = each.value.internal_port, appProtocol = "http" }], logConfiguration = { logDriver = "awslogs", options = { awslogs-group = aws_cloudwatch_log_group.service[each.key].name, awslogs-region = var.region, awslogs-stream-prefix = each.key } } }])+ container_definitions = jsonencode([{ name = each.key, image = "public.ecr.aws/docker/library/alpine:3.20", essential = true, command = ["sh", "-c", "while true; do nc -l -p ${each.value.internal_port} -e echo ok; done"], portMappings = [merge({ name = each.key, containerPort = each.value.internal_port }, contains(keys(local.egress_proxy), each.key) ? {} : { appProtocol = "http" })], logConfiguration = { logDriver = "awslogs", options = { awslogs-group = aws_cloudwatch_log_group.service[each.key].name, awslogs-region = var.region, awslogs-stream-prefix = each.key } } }])tags = local.tags}resource "aws_ecs_service" "service" {for_each = var.services- depends_on = [aws_lb_listener.public, aws_lb_listener_rule.production, aws_iam_role_policy_attachment.ecs_load_balancer]+ depends_on = [aws_lb_listener.public, aws_lb_listener_rule.production, aws_lb_listener.egress_proxy, aws_iam_role_policy_attachment.ecs_load_balancer]name = each.value.ecs_servicecluster = aws_ecs_cluster.this.idtask_definition = aws_ecs_task_definition.bootstrap[each.key].arn@@ -983,7 +1041,7 @@ resource "aws_ecs_service" "service" {launch_type = "FARGATE"network_configuration {subnets = local.subnet_ids- security_groups = local.service_security_groups+ security_groups = contains(keys(local.egress_proxy), each.key) ? [aws_security_group.egress_proxy[each.key].id] : local.service_security_groupsassign_public_ip = true}service_connect_configuration {@@ -992,9 +1050,12 @@ resource "aws_ecs_service" "service" {service {port_name = each.keydiscovery_name = each.key- timeout {- idle_timeout_seconds = 300- per_request_timeout_seconds = 0+ dynamic "timeout" {+ for_each = contains(keys(local.egress_proxy), each.key) ? [] : [each.key]+ content {+ idle_timeout_seconds = 300+ per_request_timeout_seconds = 0+ }}client_alias {dns_name = "${each.key}.${var.cloud_map_namespace}"@@ -1008,8 +1069,8 @@ resource "aws_ecs_service" "service" {rollback = true}deployment_configuration {- strategy = "BLUE_GREEN"- bake_time_in_minutes = var.ecs_blue_green_bake_minutes+ strategy = contains(keys(local.egress_proxy), each.key) ? "ROLLING" : "BLUE_GREEN"+ bake_time_in_minutes = contains(keys(local.egress_proxy), each.key) ? null : var.ecs_blue_green_bake_minutes}dynamic "alarms" {for_each = contains(keys(local.ingress_services), each.key) ? [each.key] : []@@ -1036,6 +1097,14 @@ resource "aws_ecs_service" "service" {}}}+ dynamic "load_balancer" {+ for_each = contains(keys(local.egress_proxy), each.key) ? [each.key] : []+ content {+ target_group_arn = aws_lb_target_group.egress_proxy[load_balancer.value].arn+ container_name = each.key+ container_port = each.value.internal_port+ }+ }lifecycle {ignore_changes = [task_definition, desired_count]}diff --git a/cli/templates/aws/outputs.tf b/cli/templates/aws/outputs.tfindex 28229666d..b076b8466 100644--- a/cli/templates/aws/outputs.tf+++ b/cli/templates/aws/outputs.tf@@ -1,4 +1,5 @@output "alb_hostname" { value = aws_lb.this.dns_name }+output "egress_proxy_hostname" { value = try(aws_lb.egress_proxy["egress-proxy"].dns_name, null) }output "cloudfront_hostname" { value = aws_cloudfront_distribution.portal.domain_name }output "cluster" { value = aws_ecs_cluster.this.name }output "deploy_role_arn" { value = aws_iam_role.github_deploy.arn }diff --git a/deploy/egress-proxy-pub/fly.toml b/cli/templates/fly/egress-proxy.tomlsimilarity index 73%rename from deploy/egress-proxy-pub/fly.tomlrename to cli/templates/fly/egress-proxy.tomlindex c294237ac..2107f6833 100644--- a/deploy/egress-proxy-pub/fly.toml+++ b/cli/templates/fly/egress-proxy.toml@@ -1,11 +1,10 @@-app = "qm-egress-proxy-pub"+app = "qm-egress-proxy"primary_region = "sjc"-[build]- dockerfile = "../egress-proxy/Dockerfile"-[env]- EGRESS_TOKENLESS = "deny"+ QM_DEPLOYMENT_ID = "qm-v2:personal:acme:qm"+ CORE_API_URL = "http://qm-core.internal:8080"+ CORE_ORG_ID = "acme"[[services]]internal_port = 48080diff --git a/cli/test/auth-broker.test.ts b/cli/test/auth-broker.test.tsindex 43f37acec..cfb8de14c 100644--- a/cli/test/auth-broker.test.ts+++ b/cli/test/auth-broker.test.ts@@ -43,7 +43,7 @@ test("the auth service is a first-class, privately addressed service with its owassert.equal(def.docker.hostPortOffset, undefined, "docker must not publish the broker on a host port");const offsets = SERVICE_NAMES.map((name) => serviceDef(name).docker.hostPortOffset);assert.equal(new Set(offsets).size, offsets.length, "host port offsets stay unique");- const slots = SERVICE_NAMES.map((name) => serviceDef(name).dev.portSlotOffset);+ const slots = SERVICE_NAMES.map((name) => serviceDef(name).dev?.portSlotOffset);assert.equal(new Set(slots).size, slots.length, "dev port slots stay unique");assert.ok(def.order < serviceDef("portal").order, "the broker comes up before the portal that publishes it");assert.deepEqual(diff --git a/cli/test/aws-public-paths.test.ts b/cli/test/aws-public-paths.test.tsindex d47031bab..2cc407856 100644--- a/cli/test/aws-public-paths.test.ts+++ b/cli/test/aws-public-paths.test.ts@@ -193,3 +193,22 @@ for (const name of ["v1", "d", "key", "models", "slack"]) {}});}++test("the egress proxy may sit behind its own load balancer while other private services may not", () => {+ const f = fixture();+ try {+ const loaded = loadConfigAt(f.configPath).config;+ const proxy = { ecrRepository: "egress-proxy", ecsService: "egress-proxy", cpu: 256, memory: 512 };+ const config = {+ ...loaded,+ services: [...loaded.services, "egress-proxy" as const],+ aws: { ...loaded.aws!, services: { ...loaded.aws!.services, "egress-proxy": proxy } },+ };+ const services = new Map([...f.services, ["egress-proxy", { loadBalancers: [{ targetGroupArn: "nlb-tcp" }] }]]);+ assert.equal(assertAwsPublicRouting(config, services).has("egress-proxy"), false);+ services.set("web-ui", { loadBalancers: [{ targetGroupArn: "nlb-tcp" }] });+ assert.throws(() => assertAwsPublicRouting(config, services), /private ECS service web-ui is attached/);+ } finally {+ f.close();+ }+});diff --git a/cli/test/e2e/aws-rds-config.e2e.test.ts b/cli/test/e2e/aws-terraform.e2e.test.tssimilarity index 66%rename from cli/test/e2e/aws-rds-config.e2e.test.tsrename to cli/test/e2e/aws-terraform.e2e.test.tsindex f57e0d30d..26a1ac700 100644--- a/cli/test/e2e/aws-rds-config.e2e.test.ts+++ b/cli/test/e2e/aws-terraform.e2e.test.ts@@ -165,3 +165,72 @@ test(}},);++test(+ "AWS init plans the egress proxy behind its own TLS network load balancer",+ { skip: !terraformRequested && !terraformBin ? "Terraform is unavailable" : false },+ () => {+ assert.ok(terraformBin, `QM_TEST_TERRAFORM is not executable: ${process.env.QM_TEST_TERRAFORM}`);+ const dir = tmp("aws-egress-proxy");+ try {+ const initialized = runCli(["init", dir, "--org", "egress", "--target", "aws", "--model-provider", "anthropic"]);+ assert.equal(initialized.code, 0, initialized.out);+ configureAwsCoordinates(dir);+ const path = join(dir, "qm.config.jsonc");+ writeFileSync(+ path,+ readFileSync(path, "utf8")+ .replace('"services": [', '"services": ["egress-proxy", ')+ .replace(+ '"services": {',+ '"services": { "egress-proxy": { "ecrRepository": "egress-proxy", "ecsService": "egress-proxy", "cpu": 512, "memory": 1024 },',+ ),+ );+ configureTerraform(dir);+ const rendered = runCli(["infra", "render", "--config", path]);+ assert.equal(rendered.code, 0, rendered.out);+ writeFileSync(+ join(dir, "infra", "egress.tftest.hcl"),+ `mock_provider "aws" {+ mock_data "aws_availability_zones" {+ defaults = { names = ["us-west-2a", "us-west-2b"] }+ }+}+mock_provider "random" {}+run "tls" {+ command = plan+ variables { certificate_arn = "arn:aws:acm:us-west-2:123456789012:certificate/00000000-0000-0000-0000-000000000000" }+ assert {+ condition = aws_lb.egress_proxy["egress-proxy"].load_balancer_type == "network" && aws_lb.egress_proxy["egress-proxy"].enable_cross_zone_load_balancing && aws_lb_listener.egress_proxy["egress-proxy"].port == 443 && aws_lb_listener.egress_proxy["egress-proxy"].protocol == "TLS"+ error_message = "the proxy is not behind a TLS network load balancer"+ }+ assert {+ condition = aws_lb_target_group.egress_proxy["egress-proxy"].protocol == "TCP" && aws_lb_target_group.egress_proxy["egress-proxy"].port == 48080 && length(aws_ecs_service.service["egress-proxy"].load_balancer) == 1 && !contains(keys(aws_lb_target_group.service), "egress-proxy")+ error_message = "the proxy is not attached to its TCP target group alone"+ }+ assert {+ condition = aws_ecs_service.service["egress-proxy"].deployment_configuration[0].strategy == "ROLLING" && aws_ecs_service.service["core"].deployment_configuration[0].strategy == "BLUE_GREEN"+ error_message = "unexpected rollout strategies"+ }+ assert {+ condition = length(aws_ecs_service.service["egress-proxy"].service_connect_configuration[0].service[0].timeout) == 0 && aws_ecs_service.service["core"].service_connect_configuration[0].service[0].timeout[0].per_request_timeout_seconds == 0+ error_message = "ECS rejects a per-request timeout on the proxy's TCP Service Connect port"+ }+ assert {+ condition = !can(jsondecode(aws_ecs_task_definition.bootstrap["egress-proxy"].container_definitions)[0].portMappings[0].appProtocol) && jsondecode(aws_ecs_task_definition.bootstrap["core"].container_definitions)[0].portMappings[0].appProtocol == "http"+ error_message = "the proxy's bootstrap port mapping must be plain TCP"+ }+}+run "plaintext" {+ command = plan+ expect_failures = [aws_lb_listener.egress_proxy]+}+`,+ );+ terraform(dir, terraformBin, ["init", "-backend=false"]);+ assert.match(terraform(dir, terraformBin, ["test", "-verbose"]), /Success!.*2 passed, 0 failed/s);+ } finally {+ rmDir(dir);+ }+ },+);diff --git a/cli/test/egress-proxy.test.ts b/cli/test/egress-proxy.test.tsnew file mode 100644index 000000000..bf3cbda16--- /dev/null+++ b/cli/test/egress-proxy.test.ts@@ -0,0 +1,105 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { spawn } from "node:child_process";+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { createServer as createHttpServer } from "node:http";+import { createServer as createTcpServer, type Server } from "node:net";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { renderTaskDefinition } from "../src/backends/aws.ts";+import type { QmConfig } from "../src/config.ts";+import { renderTerraformVars } from "../src/terraform.ts";++const service = (name: string) => ({ ecrRepository: `qm-${name}`, ecsService: `acme-${name}`, cpu: 512, memory: 1024 });++const config: QmConfig = {+ contract: 1,+ orgId: "acme",+ publicUrl: "https://agent.acme.example",+ apiUrl: "https://api.acme.example",+ target: "aws",+ services: ["core", "egress-proxy"],+ plugins: [],+ skills: [],+ imageOverrides: {},+ env: { core: { HARNESS: "pi", AWS_DEPLOY_IMAGE: "acme-qm-sandbox" } },+ aws: {+ accountId: "123456789012",+ region: "us-west-2",+ cluster: "acme-qm",+ deployRoleArn: "arn:aws:iam::123456789012:role/acme-deploy",+ imageLabel: "release",+ secretsPrefix: "acme/qm/",+ networking: { cloudMapNamespace: "acme.internal" },+ services: { core: service("core"), "egress-proxy": service("egress-proxy") },+ },+};++const image = `123456789012.dkr.ecr.us-west-2.amazonaws.com/qm-egress-proxy@sha256:${"a".repeat(64)}`;+const proxyContainer = (deployment = config) =>+ renderTaskDefinition(deployment, "egress-proxy", image).containerDefinitions[0]!;+const coreApiUrl = (deployment: QmConfig) =>+ (proxyContainer(deployment).environment as Array<{ name: string; value: string }>).find(+ (entry) => entry.name === "CORE_API_URL",+ )?.value;++test("AWS runs the egress proxy as plain TCP with the keys it verifies and relays with", () => {+ const container = proxyContainer();+ assert.deepEqual(container.portMappings, [+ { name: "egress-proxy", containerPort: 48080, hostPort: 48080, protocol: "tcp" },+ ]);+ assert.deepEqual(+ (container.secrets as Array<{ name: string }>).map((secret) => secret.name),+ ["CAPABILITY_SECRET", "CORE_SIGNING_SECRET"],+ );+ assert.equal(coreApiUrl(config), "https://api.acme.example");+ const { apiUrl: _apiUrl, ...withoutApiUrl } = config;+ assert.equal(coreApiUrl(withoutApiUrl), "https://agent.acme.example");+});++const adminSocket = "/tmp/envoy-admin.sock";++function listen(server: Server, ...args: unknown[]): Promise<void> {+ return new Promise((resolve) => (server.listen as (...a: unknown[]) => void)(...args, resolve));+}++test(+ "the AWS proxy health check passes only while the decision service listens and Envoy reports ready",+ { skip: existsSync(adminSocket) ? `${adminSocket} is in use` : false },+ async () => {+ const [, ...command] = (proxyContainer().healthCheck as { command: string[] }).command;+ const probe = () =>+ new Promise<number | null>((resolve) =>+ spawn(command[0]!, command.slice(1), { timeout: 10_000 }).on("exit", resolve),+ );+ assert.notEqual(await probe(), 0);+ let ready = 200;+ const authz = createTcpServer((socket) => socket.end());+ const envoy = createHttpServer((_req, res) => res.writeHead(ready).end());+ await listen(authz, 48081, "127.0.0.1");+ try {+ assert.notEqual(await probe(), 0);+ await listen(envoy, adminSocket);+ assert.equal(await probe(), 0);+ ready = 503;+ assert.notEqual(await probe(), 0);+ } finally {+ authz.close();+ envoy.close();+ }+ },+);++test("a vendored AWS scaffold without the proxy load balancer refuses the egress proxy", () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-legacy-egress-"));+ try {+ const infra = join(dir, "infra");+ mkdirSync(infra);+ writeFileSync(join(infra, "terraform.tfvars"), "services = {}\n");+ writeFileSync(join(infra, "variables.tf"), 'variable "services" { type = map(any) }\n');+ writeFileSync(join(infra, "main.tf"), 'resource "aws_lb" "this" {}\n');+ assert.throws(() => renderTerraformVars(config, dir), /AWS scaffold predates the egress-proxy service/);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});diff --git a/cli/test/fly-up.test.ts b/cli/test/fly-up.test.tsindex 8367eb853..29a55adb7 100644--- a/cli/test/fly-up.test.ts+++ b/cli/test/fly-up.test.ts@@ -166,3 +166,66 @@ test('--only "slack" explains the virtual service runs in-process on the core',rmSync(dir, { recursive: true, force: true });}});++test("fly up gives the egress proxy public ingress once, before deploying it", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-fly-egress-"));+ const fly = join(dir, "fly");+ const log = join(dir, "fly.log");+ const ipsFile = join(dir, "ips");+ writeFileSync(+ fly,+ `#!/usr/bin/env node+const fs = require("node:fs");+const a = process.argv.slice(2);+fs.appendFileSync(${JSON.stringify(log)}, a.join(" ") + "\\n");+if (a[0] === "apps" && a[1] === "create") console.log("already been taken");+else if (a[0] === "apps" && a[1] === "list") console.log(JSON.stringify([{ Name: "acme-core" }, { Name: "acme-egress-proxy" }]));+else if (a[0] === "secrets" && a[1] === "list") console.log("QM_OWNER_CD83933C6C53374D digest\\nCAPABILITY_SECRET digest\\nCORE_SIGNING_SECRET digest");+else if (a[0] === "ips" && a[1] === "list") console.log(fs.existsSync(${JSON.stringify(ipsFile)}) ? fs.readFileSync(${JSON.stringify(ipsFile)}, "utf8") : "[]");+else console.log("ok");+`,+ );+ chmodSync(fly, 0o755);+ const prior = process.env.FLY_BIN;+ process.env.FLY_BIN = fly;+ const config: QmConfig = {+ contract: 1,+ orgId: "acme",+ publicUrl: "https://acme.fly.dev",+ target: "fly",+ appPrefix: "acme",+ region: "sjc",+ flyOrg: "personal",+ services: ["core", "egress-proxy"],+ plugins: [],+ skills: [],+ env: {},+ imageOverrides: { "egress-proxy": `registry.fly.io/acme-egress-proxy@sha256:${"c".repeat(64)}` },+ sandbox: { app: "acme-sandboxes" },+ };+ try {+ await flyUp(config, dir, { only: ["egress-proxy"] });+ let calls = readFileSync(log, "utf8");+ assert.match(calls, /^ips allocate-v6 -a acme-egress-proxy$/m);+ assert.match(calls, /^ips allocate-v4 --shared -a acme-egress-proxy$/m);+ assert.ok(calls.indexOf("ips allocate-v4") < calls.indexOf("deploy --yes"));++ writeFileSync(log, "");+ writeFileSync(+ ipsFile,+ JSON.stringify([+ { Type: "private_v6", Address: "fdaa::3" },+ { Type: "v6", Address: "2a09:8280:1::1:0" },+ { Type: "shared_v4", Address: "66.241.124.1" },+ ]),+ );+ await flyUp(config, dir, { only: ["egress-proxy"] });+ calls = readFileSync(log, "utf8");+ assert.doesNotMatch(calls, /ips allocate/);+ assert.match(calls, /^deploy --yes .*acme-egress-proxy/m);+ } finally {+ if (prior === undefined) delete process.env.FLY_BIN;+ else process.env.FLY_BIN = prior;+ rmSync(dir, { recursive: true, force: true });+ }+});diff --git a/cli/test/terraform.test.ts b/cli/test/terraform.test.tsindex 386f6229d..5ae36fdce 100644--- a/cli/test/terraform.test.ts+++ b/cli/test/terraform.test.ts@@ -763,7 +763,7 @@ test("AWS ECS services wait until target groups are attached to the ALB", () =>const service = mainTf.match(/resource "aws_ecs_service" "service" \{([\s\S]*?)\n\}/)?.[1] ?? "";assert.match(service,- /depends_on\s*= \[aws_lb_listener\.public, aws_lb_listener_rule\.production, aws_iam_role_policy_attachment\.ecs_load_balancer\]/,+ /depends_on\s*= \[aws_lb_listener\.public, aws_lb_listener_rule\.production, aws_lb_listener\.egress_proxy, aws_iam_role_policy_attachment\.ecs_load_balancer\]/,);});@@ -773,7 +773,7 @@ test("AWS ECS services use native blue-green rollouts with rollback signals andassert.match(versions, /version\s*= ">= 6\.6, < 7\.0"/);assert.match(service,- /deployment_configuration \{[\s\S]*strategy\s*= "BLUE_GREEN"[\s\S]*bake_time_in_minutes = var\.ecs_blue_green_bake_minutes/,+ /deployment_configuration \{[\s\S]*strategy\s*= .*"BLUE_GREEN"[\s\S]*bake_time_in_minutes = .*var\.ecs_blue_green_bake_minutes/,);assert.match(service,@@ -790,7 +790,7 @@ test("AWS ECS services use native blue-green rollouts with rollback signals andassert.match(service, /sigint_rollback = true/);assert.match(mainTf,- /portMappings = \[\{ name = each\.key, containerPort = each\.value\.internal_port, appProtocol = "http" \}\]/,+ /portMappings = \[merge\(\{ name = each\.key, containerPort = each\.value\.internal_port \}, contains\(keys\(local\.egress_proxy\), each\.key\) \? \{\} : \{ appProtocol = "http" \}\)\]/,);});diff --git a/deploy/egress-proxy/fly.toml b/deploy/egress-proxy/fly.tomldeleted file mode 100644index 507e936a4..000000000--- a/deploy/egress-proxy/fly.toml+++ /dev/null@@ -1,35 +0,0 @@-# The forced egress proxy (Fly realization). Published PRIVATELY — flycast only,-# no public IP (`fly ips allocate-v6 --private` before first deploy; never `fly ips allocate-*` a-# public one). Sandboxes reach it at `<app>.flycast:<port>` via HTTPS_PROXY; a default-deny egress-# Network Policy on each sandbox allows ONLY this port, forcing all egress through here.-#-# Port note: 48080 is deliberately OBSCURE. The Fly Network Policy can only allow egress by PORT-# (not by destination), so whatever port we allow is also reachable on ANY host directly — the-# "port leak". An obscure port keeps that leak small (a denied host is unlikely to serve on it);-# direct egress on 443/80 — i.e. ~all real egress — stays fully blocked.-app = "qm-egress-proxy"-primary_region = "sjc"---[[services]]- internal_port = 48080- protocol = "tcp"- auto_stop_machines = false- min_machines_running = 1-- [[services.ports]]- port = 48080-- [[services.tcp_checks]]- interval = "15s"- timeout = "2s"- grace_period = "10s"--# Envoy + the Node decision service sit at ~130mb resident before serving anything,-# which leaves almost no headroom on 256mb under real fleet traffic.-[[vm]]- size = "shared-cpu-1x"- memory = "512mb"--kill_signal = "SIGTERM"-kill_timeout = "10s"diff --git a/deploy/stacks/acme/qm.config.jsonc b/deploy/stacks/acme/qm.config.jsoncindex a9bbf90cd..576de4f10 100644--- a/deploy/stacks/acme/qm.config.jsonc+++ b/deploy/stacks/acme/qm.config.jsonc@@ -10,7 +10,7 @@"sandbox": {"app": "acme-sandboxes",},- "services": ["core", "slack", "admin", "web-ui", "portal"],+ "services": ["core", "slack", "admin", "web-ui", "portal", "egress-proxy"],"env": {"admin": {"ADMIN_BASE_PATH": "/admin",diff --git a/docs/deploy-directory.md b/docs/deploy-directory.mdindex 6dcf8b368..dbda08631 100644--- a/docs/deploy-directory.md+++ b/docs/deploy-directory.md@@ -191,6 +191,25 @@ and listener precedence, including native blue/green weighted routing. Updatevendored Terraform with the `public_paths` service attribute and`public_path_services` routing before enabling this setting.+### Sandbox egress proxy++Add `"egress-proxy"` to `services` to deploy the sandbox egress proxy+(`deploy/egress-proxy`) with every `qm up`, from the same release as core. It+receives `CAPABILITY_SECRET` and `CORE_SIGNING_SECRET` from the computed secret+set and relays its audit records to core.++On AWS the reference module puts it behind its own network load balancer, which+terminates TLS on 443 with the module's `certificate_arn` (required) and forwards+plain TCP to the proxy. The proxy runs in its own security group, outside the+services' private network, so it relays audit records through `apiUrl` (or+`publicUrl`). Point a hostname covered by the certificate at the+`egress_proxy_hostname` output. On Fly it serves TLS on 443 at+`<appPrefix>-egress-proxy.fly.dev`. Docker publishes the plain listener on a host+port and leaves TLS to the operator. Then set the sandbox backend's proxy URL, for+example `env.core.SPRITES_EGRESS_PROXY_URL` or `MODAL_EGRESS_PROXY_URL`, to+`https://<proxy hostname>`. Update vendored Terraform from the current scaffold+before enabling it on AWS.+### Shared AWS foundationsExisting AWS deployments retain their company-owned resources by default. For a shared DynamoDB deployment table, set `aws.deploymentState` to `{ "table": "fleet-deploy-locks", "namespace": "acme" }`. Every lease, current pointer, label pointer and immutable manifest uses that namespace. Use a unique namespace for each company and scope its item IAM permissions with `dynamodb:LeadingKeys` to `acme/*`; grant `DescribeTable` separately. Changing an existing deployment's state location requires explicitly migrating its history while deployment operations are stopped.diff --git a/docs/e2b-template.md b/docs/e2b-template.mdindex 5c6cc20cc..794dcd860 100644--- a/docs/e2b-template.md+++ b/docs/e2b-template.md@@ -94,7 +94,7 @@ point and uses E2B's rules only to make that proxy unavoidable:does neither.Set `E2B_EGRESS_PROXY_URL` to a QM egress proxy reachable on port 443, for example-`deploy/egress-proxy-pub` (TLS on 443 in front of the proxy's port 48080). Core then+the `egress-proxy` service (TLS on 443 in front of the proxy's port 48080). Core thencreates every agent computer with `allowOut: [<proxy host>]` and `denyOut: ["0.0.0.0/0"]`and points the computer's proxy settings at it with the turn's token. Without it, corewarns at boot and computers have open egress.diff --git a/test/release-workflows.test.ts b/test/release-workflows.test.tsindex 1e4831ce7..a1d052de1 100644--- a/test/release-workflows.test.ts+++ b/test/release-workflows.test.ts@@ -84,7 +84,7 @@ test("the published package pins real image digests, never the checked-in sentinworkflow.indexOf("Pin published image digests") < workflow.indexOf("npm publish"),"digests are resolved before the package is published",);- assert.match(workflow, /for service in core web-ui admin portal auth sandbox-base; do/);+ assert.match(workflow, /for service in core web-ui admin portal auth egress-proxy sandbox-base; do/);assert.match(workflow, /printf '%s\\n' "\$out" > cli\/manifest\.json/);assert.match(workflow, /no published image for \$repo at \$IMAGES_REF/);assert.match(workflow, /\{63\}\$"\) \| not\)/);@@ -94,7 +94,7 @@ test("the published package pins real image digests, never the checked-in sentinservices: Record<string, string>;};const refs = [sentinel.sandboxBase, ...Object.values(sentinel.services)];- assert.equal(refs.length, 6);+ assert.equal(refs.length, 7);assert.ok(refs.every((ref) => ref.startsWith("registry.invalid/")),"the checked-in manifest stays a sentinel so a source checkout never pulls a stale digest",