Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Deploy the egress proxy as a regular qm service (#2266)

Regan Bellaf4b106ec846
diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml
index a79d77cbc..7c1af4fa3 100644
--- a/.github/workflows/publish-cli.yml
+++ b/.github/workflows/publish-cli.yml
@@ -57,7 +57,7 @@ jobs:
run: |
set -euo pipefail
out=$(jq -n '{sandboxBase: "", services: {}}')
- for service in core web-ui admin portal auth sandbox-base; do
+ for service in core web-ui admin portal auth egress-proxy sandbox-base; do
repo="yc-software/qm/$service"
token=$(curl -fsS "https://ghcr.io/token?scope=repository:$repo:pull&service=ghcr.io" | jq -r .token)
digest=$(curl -fsS -o /dev/null -D - -H "Authorization: Bearer $token" \
@@ -78,7 +78,7 @@ jobs:
printf '%s\n' "$out" > cli/manifest.json
cat cli/manifest.json
jq -e '[.sandboxBase] + [.services[]]
- | length == 6
+ | length == 7
and all(test("^ghcr\\.io/yc-software/qm/[a-z-]+@sha256:[0-9a-f]{64}$"))
and all(test("@sha256:(.)\\1{63}$") | not)' cli/manifest.json > /dev/null
printf 'manifest=%s\n' "$(jq -c . cli/manifest.json)" >> "$GITHUB_OUTPUT"
diff --git a/cli/manifest.json b/cli/manifest.json
index 3dcb8bfda..fd7db393a 100644
--- a/cli/manifest.json
+++ b/cli/manifest.json
@@ -5,6 +5,7 @@
"web-ui": "registry.invalid/qm/qm-web-ui@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"admin": "registry.invalid/qm/qm-admin@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"portal": "registry.invalid/qm/qm-portal@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
- "auth": "registry.invalid/qm/qm-auth@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ "auth": "registry.invalid/qm/qm-auth@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "egress-proxy": "registry.invalid/qm/qm-egress-proxy@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
}
}
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index 86334e10e..4ce355a93 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -380,7 +380,12 @@ export function serviceEnvironment(config: QmConfig, service: ServiceName): Reco
: {};
const env: Record<string, string> = {
...orgEnv(service, config.orgId, config.publicUrl, config.services.includes("portal"), brandEnvOf(config)),
- ...(service === "core" ? {} : { CORE_API_URL: coreUrl }),
+ ...(service === "core"
+ ? {}
+ : {
+ CORE_API_URL:
+ isServiceName(service) && serviceDef(service).tcpIngress ? (config.apiUrl ?? config.publicUrl) : coreUrl,
+ }),
...coreEnv,
...hostedServiceEnv(config.services, config.env, service),
...(service === "core" ? securityScreenEnv(config) : {}),
@@ -498,7 +503,8 @@ export function renderTaskDefinition(
const aws = requireAws(config);
const spec = aws.services[service];
if (!spec) throw new CliError(`aws.services.${service} is missing`);
- const internalPort = isServiceName(service) ? serviceDef(service).docker.internalPort : 8080;
+ const def = isServiceName(service) ? serviceDef(service) : undefined;
+ const internalPort = def?.docker.internalPort ?? 8080;
const executionRoleArn = spec.executionRoleArn ?? `arn:aws:iam::${aws.accountId}:role/${aws.cluster}-task-execution`;
const managedTaskRole = spec.assumeRoleArns?.length
? `${aws.cluster}-${service}-task`
@@ -512,9 +518,9 @@ export function renderTaskDefinition(
})),
)
.sort((a, b) => a.name.localeCompare(b.name));
- const healthCheck = isServiceName(service)
+ const healthCheck = def
? {
- command: [
+ command: def.tcpIngress?.healthCheck ?? [
"CMD",
"node",
"-e",
@@ -553,7 +559,7 @@ export function renderTaskDefinition(
containerPort: internalPort,
hostPort: internalPort,
protocol: "tcp",
- appProtocol: "http",
+ ...(def?.tcpIngress ? {} : { appProtocol: "http" }),
},
],
...(healthCheck ? { healthCheck } : {}),
@@ -4053,7 +4059,11 @@ export function assertAwsPublicRouting(
}
}
for (const name of deployedAwsServices(aws)) {
- if (!ingress.includes(name) && (routingServices.get(name)?.loadBalancers ?? []).length) {
+ if (
+ !ingress.includes(name) &&
+ !(isServiceName(name) && serviceDef(name).tcpIngress) &&
+ (routingServices.get(name)?.loadBalancers ?? []).length
+ ) {
throw new Error(`private ECS service ${name} is attached to a load balancer`);
}
}
diff --git a/cli/src/backends/dev-ci.ts b/cli/src/backends/dev-ci.ts
index 227413dcb..db5453071 100644
--- a/cli/src/backends/dev-ci.ts
+++ b/cli/src/backends/dev-ci.ts
@@ -87,7 +87,7 @@ export async function devCiUp(): Promise<void> {
const baseEnv = { ...process.env, ...overlay } as NodeJS.ProcessEnv;
const coreLog = join(dir, "core.log");
- const corePid = spawnBackground(process.execPath, [...serviceDef("core").dev.entry], {
+ const corePid = spawnBackground(process.execPath, [...serviceDef("core").dev!.entry], {
cwd: root,
env: { ...baseEnv, ORG_ID: "acme", SESSION_STORE: "memory", PORT: String(port) },
logFile: coreLog,
diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.ts
index 1222d212c..9318ac623 100644
--- a/cli/src/backends/fly.ts
+++ b/cli/src/backends/fly.ts
@@ -669,6 +669,24 @@ function ensureFlycast(app: string): void {
note(`flycast: allocated private v6 for ${app}`);
}
+function ensurePublicIngress(app: string): void {
+ let ips: Array<{ Type?: string }>;
+ try {
+ ips = JSON.parse(fly(["ips", "list", "-a", app, "--json"]) || "[]");
+ } catch {
+ throw new CliError(`fly ips list returned invalid JSON for ${app}`);
+ }
+ const types = new Set(ips.map((ip) => ip.Type));
+ if (!types.has("v6")) {
+ fly(["ips", "allocate-v6", "-a", app]);
+ note(`public ingress: allocated v6 for ${app}`);
+ }
+ if (!types.has("v4") && !types.has("shared_v4")) {
+ fly(["ips", "allocate-v4", "--shared", "-a", app]);
+ note(`public ingress: allocated shared v4 for ${app}`);
+ }
+}
+
interface FlyImageDetail {
digest: string;
immutableReference: string;
@@ -848,6 +866,7 @@ async function deployService(
note(`\n=== deploying ${app} ===`);
if (service === "core") timing.time(service, "Postgres ensure", () => ensurePostgres(ctx));
if (spec.flycast) timing.time(service, "Flycast ensure", () => ensureFlycast(app));
+ if (spec.publicIngress) timing.time(service, "public ingress ensure", () => ensurePublicIngress(app));
const cfgPath = serviceCfgPath(ctx, service);
const args = ["deploy", "--yes", "-c", cfgPath, ...spec.deployFlags];
if (imageSource?.kind === "tagged") {
diff --git a/cli/src/provider-scaffold.ts b/cli/src/provider-scaffold.ts
index c5c4c8361..570161ffe 100644
--- a/cli/src/provider-scaffold.ts
+++ b/cli/src/provider-scaffold.ts
@@ -69,8 +69,9 @@ ${values.providerFields}
// First-party services to run. The full set: "core" (the agent runtime and API,
// always required), "slack" (the Slack surface, runs inside the core process),
// "web-ui" (a browser chat UI), "admin" (the operator dashboard), "portal"
- // (the public sign-in front door), and "auth" (the built-in sign-in broker that
- // emails one-time links; drop it to sign in through an external OIDC provider).
+ // (the public sign-in front door), "auth" (the built-in sign-in broker that
+ // emails one-time links; drop it to sign in through an external OIDC provider),
+ // and "egress-proxy" (the TLS proxy that enforces sandbox egress policy).
"services": [${values.services.map((service) => JSON.stringify(service)).join(", ")}],
// Extra plugins to run alongside the services. A prebuilt image runs with
diff --git a/cli/src/secrets.ts b/cli/src/secrets.ts
index f31e07818..b62c3b630 100644
--- a/cli/src/secrets.ts
+++ b/cli/src/secrets.ts
@@ -402,6 +402,20 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [
description: "HMAC key shared by core and surface plugins.",
generate: MINT_LOCALLY,
},
+ {
+ name: "CAPABILITY_SECRET",
+ service: "egress-proxy",
+ required: true,
+ description: "Signing key for scoped agent capabilities and egress grants; must differ from every other key.",
+ generate: MINT_LOCALLY,
+ },
+ {
+ name: "CORE_SIGNING_SECRET",
+ service: "egress-proxy",
+ required: true,
+ description: "HMAC key shared by core and surface plugins.",
+ generate: MINT_LOCALLY,
+ },
{
name: "AUTH_SIGNING_JWK",
service: "auth",
diff --git a/cli/src/services.ts b/cli/src/services.ts
index 7588c7197..05ff07a6d 100644
--- a/cli/src/services.ts
+++ b/cli/src/services.ts
@@ -1,4 +1,4 @@
-export const SERVICE_NAMES = ["core", "web-ui", "admin", "portal", "auth"] as const;
+export const SERVICE_NAMES = ["core", "web-ui", "admin", "portal", "auth", "egress-proxy"] as const;
export type ServiceName = (typeof SERVICE_NAMES)[number];
export const isServiceName = (s: string): s is ServiceName => (SERVICE_NAMES as readonly string[]).includes(s);
@@ -86,13 +86,14 @@ interface FlyServiceSpec {
stackKeys: string[];
deployFlags: string[];
flycast?: boolean;
+ publicIngress?: boolean;
}
export interface ServiceDef {
name: ServiceName;
readiness: RegExp;
order: number;
- dev: {
+ dev?: {
cwd: string;
entry: string[];
portEnv?: "PORT";
@@ -105,6 +106,7 @@ export interface ServiceDef {
hostPortOffset?: number;
};
fly?: FlyServiceSpec;
+ tcpIngress?: { healthCheck: string[] };
}
export interface BrandEnv {
@@ -321,6 +323,26 @@ const CATALOG: Record<ServiceName, ServiceDef> = {
flycast: true,
},
},
+ "egress-proxy": {
+ name: "egress-proxy",
+ readiness: /starting main dispatch loop/,
+ order: 50,
+ docker: { image: "egress-proxy", internalPort: 48080, portEnv: "PORT", hostPortOffset: 4 },
+ fly: {
+ managed: (s) => pluginWiring("egress-proxy", s),
+ stackKeys: [],
+ deployFlags: ["--ha=false"],
+ publicIngress: true,
+ },
+ tcpIngress: {
+ healthCheck: [
+ "CMD",
+ "node",
+ "-e",
+ "require('net').connect(48081,'127.0.0.1').on('connect',()=>require('http').get({socketPath:'/tmp/envoy-admin.sock',path:'/ready'},r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))).on('error',()=>process.exit(1))",
+ ],
+ },
+ },
};
export const serviceDef = (name: ServiceName): ServiceDef => CATALOG[name];
diff --git a/cli/src/terraform.ts b/cli/src/terraform.ts
index e4dc2b8e4..e04979656 100644
--- a/cli/src/terraform.ts
+++ b/cli/src/terraform.ts
@@ -280,7 +280,8 @@ export function assertTerraformScaffoldSupportsConfig(config: QmConfig, configDi
const hasPublicPaths = services.some((service) => service?.publicPaths?.length);
const hasAssumeRoles = services.some((service) => service?.assumeRoleArns !== undefined);
const hasDatabaseOverrides = config.aws?.dbInstanceClass !== undefined;
- if (!hasPublicPaths && !hasAssumeRoles && !hasDatabaseOverrides) return;
+ const hasEgressProxy = Boolean(config.aws?.services["egress-proxy"]);
+ if (!hasPublicPaths && !hasAssumeRoles && !hasDatabaseOverrides && !hasEgressProxy) return;
const tfvarsPath = join(configDir, "infra", "terraform.tfvars");
if (!existsSync(tfvarsPath)) return;
const variablesPath = join(configDir, "infra", "variables.tf");
@@ -300,6 +301,11 @@ export function assertTerraformScaffoldSupportsConfig(config: QmConfig, configDi
"the vendored AWS scaffold predates aws.services.*.publicPaths; update infra/variables.tf and infra/main.tf before exposing plugins",
);
}
+ if (hasEgressProxy && !/resource\s+"aws_lb"\s+"egress_proxy"/.test(main)) {
+ throw new CliError(
+ "the vendored AWS scaffold predates the egress-proxy service; update infra/main.tf and infra/outputs.tf from the current scaffold before enabling it",
+ );
+ }
if (
hasAssumeRoles &&
(!/assume_role_arns\s*=\s*optional/.test(variables) ||
diff --git a/cli/templates/aws/main.tf b/cli/templates/aws/main.tf
index 1a312b53a..6050c2eae 100644
--- a/cli/templates/aws/main.tf
+++ b/cli/templates/aws/main.tf
@@ -8,6 +8,7 @@ locals {
public_path_services = { for name, service in var.services : name => service.public_paths if length(service.public_paths) > 0 }
public_service_names = concat(local.has_portal ? concat(["portal"], length(var.core_public_hosts) > 0 ? ["core"] : []) : ["core"], keys(local.public_path_services))
ingress_services = { for name, service in var.services : name => service if contains(local.public_service_names, name) }
+ egress_proxy = { for name, service in var.services : name => service if name == "egress-proxy" }
direct_path_services = merge(local.has_portal ? {} : { core = ["/v1/*"] }, local.public_path_services)
alb_name = "${substr(var.cluster_name, 0, 23)}-${substr(sha1(var.cluster_name), 0, 8)}"
service_security_groups = [aws_security_group.services.id]
@@ -830,6 +831,63 @@ resource "aws_lb_listener" "public" {
}
}
+resource "aws_security_group" "egress_proxy" {
+ for_each = local.egress_proxy
+ name = "${var.cluster_name}-egress-proxy"
+ vpc_id = local.vpc_id
+ ingress {
+ from_port = each.value.internal_port
+ to_port = each.value.internal_port
+ protocol = "tcp"
+ cidr_blocks = [aws_vpc.this.cidr_block]
+ }
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+ tags = local.tags
+}
+
+resource "aws_lb" "egress_proxy" {
+ for_each = local.egress_proxy
+ name = "${substr(var.cluster_name, 0, 20)}-egr-${substr(sha1("${var.cluster_name}:${each.key}"), 0, 6)}"
+ load_balancer_type = "network"
+ subnets = local.subnet_ids
+ enable_cross_zone_load_balancing = true
+ tags = local.tags
+}
+
+resource "aws_lb_target_group" "egress_proxy" {
+ for_each = local.egress_proxy
+ name = "${substr(var.cluster_name, 0, 20)}-egr-${substr(sha1("${var.cluster_name}:${each.key}:tcp"), 0, 6)}"
+ port = each.value.internal_port
+ protocol = "TCP"
+ target_type = "ip"
+ vpc_id = local.vpc_id
+ tags = local.tags
+}
+
+resource "aws_lb_listener" "egress_proxy" {
+ for_each = local.egress_proxy
+ load_balancer_arn = aws_lb.egress_proxy[each.key].arn
+ port = 443
+ protocol = "TLS"
+ certificate_arn = var.certificate_arn
+ ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06"
+ default_action {
+ type = "forward"
+ target_group_arn = aws_lb_target_group.egress_proxy[each.key].arn
+ }
+ lifecycle {
+ precondition {
+ condition = var.certificate_arn != ""
+ error_message = "the egress-proxy service requires certificate_arn: sandboxes present capability tokens to it, so it only listens on TLS"
+ }
+ }
+}
+
resource "aws_cloudfront_distribution" "portal" {
enabled = true
is_ipv6_enabled = true
@@ -969,13 +1027,13 @@ resource "aws_ecs_task_definition" "bootstrap" {
operating_system_family = "LINUX"
cpu_architecture = each.value.architecture == "amd64" ? "X86_64" : "ARM64"
}
- container_definitions = jsonencode([{ name = each.key, image = "public.ecr.aws/docker/library/alpine:3.20", essential = true, command = ["sh", "-c", "while true; do nc -l -p ${each.value.internal_port} -e echo ok; done"], portMappings = [{ name = each.key, containerPort = each.value.internal_port, appProtocol = "http" }], logConfiguration = { logDriver = "awslogs", options = { awslogs-group = aws_cloudwatch_log_group.service[each.key].name, awslogs-region = var.region, awslogs-stream-prefix = each.key } } }])
+ container_definitions = jsonencode([{ name = each.key, image = "public.ecr.aws/docker/library/alpine:3.20", essential = true, command = ["sh", "-c", "while true; do nc -l -p ${each.value.internal_port} -e echo ok; done"], portMappings = [merge({ name = each.key, containerPort = each.value.internal_port }, contains(keys(local.egress_proxy), each.key) ? {} : { appProtocol = "http" })], logConfiguration = { logDriver = "awslogs", options = { awslogs-group = aws_cloudwatch_log_group.service[each.key].name, awslogs-region = var.region, awslogs-stream-prefix = each.key } } }])
tags = local.tags
}
resource "aws_ecs_service" "service" {
for_each = var.services
- depends_on = [aws_lb_listener.public, aws_lb_listener_rule.production, aws_iam_role_policy_attachment.ecs_load_balancer]
+ depends_on = [aws_lb_listener.public, aws_lb_listener_rule.production, aws_lb_listener.egress_proxy, aws_iam_role_policy_attachment.ecs_load_balancer]
name = each.value.ecs_service
cluster = aws_ecs_cluster.this.id
task_definition = aws_ecs_task_definition.bootstrap[each.key].arn
@@ -983,7 +1041,7 @@ resource "aws_ecs_service" "service" {
launch_type = "FARGATE"
network_configuration {
subnets = local.subnet_ids
- security_groups = local.service_security_groups
+ security_groups = contains(keys(local.egress_proxy), each.key) ? [aws_security_group.egress_proxy[each.key].id] : local.service_security_groups
assign_public_ip = true
}
service_connect_configuration {
@@ -992,9 +1050,12 @@ resource "aws_ecs_service" "service" {
service {
port_name = each.key
discovery_name = each.key
- timeout {
- idle_timeout_seconds = 300
- per_request_timeout_seconds = 0
+ dynamic "timeout" {
+ for_each = contains(keys(local.egress_proxy), each.key) ? [] : [each.key]
+ content {
+ idle_timeout_seconds = 300
+ per_request_timeout_seconds = 0
+ }
}
client_alias {
dns_name = "${each.key}.${var.cloud_map_namespace}"
@@ -1008,8 +1069,8 @@ resource "aws_ecs_service" "service" {
rollback = true
}
deployment_configuration {
- strategy = "BLUE_GREEN"
- bake_time_in_minutes = var.ecs_blue_green_bake_minutes
+ strategy = contains(keys(local.egress_proxy), each.key) ? "ROLLING" : "BLUE_GREEN"
+ bake_time_in_minutes = contains(keys(local.egress_proxy), each.key) ? null : var.ecs_blue_green_bake_minutes
}
dynamic "alarms" {
for_each = contains(keys(local.ingress_services), each.key) ? [each.key] : []
@@ -1036,6 +1097,14 @@ resource "aws_ecs_service" "service" {
}
}
}
+ dynamic "load_balancer" {
+ for_each = contains(keys(local.egress_proxy), each.key) ? [each.key] : []
+ content {
+ target_group_arn = aws_lb_target_group.egress_proxy[load_balancer.value].arn
+ container_name = each.key
+ container_port = each.value.internal_port
+ }
+ }
lifecycle {
ignore_changes = [task_definition, desired_count]
}
diff --git a/cli/templates/aws/outputs.tf b/cli/templates/aws/outputs.tf
index 28229666d..b076b8466 100644
--- a/cli/templates/aws/outputs.tf
+++ b/cli/templates/aws/outputs.tf
@@ -1,4 +1,5 @@
output "alb_hostname" { value = aws_lb.this.dns_name }
+output "egress_proxy_hostname" { value = try(aws_lb.egress_proxy["egress-proxy"].dns_name, null) }
output "cloudfront_hostname" { value = aws_cloudfront_distribution.portal.domain_name }
output "cluster" { value = aws_ecs_cluster.this.name }
output "deploy_role_arn" { value = aws_iam_role.github_deploy.arn }
diff --git a/deploy/egress-proxy-pub/fly.toml b/cli/templates/fly/egress-proxy.toml
similarity index 73%
rename from deploy/egress-proxy-pub/fly.toml
rename to cli/templates/fly/egress-proxy.toml
index c294237ac..2107f6833 100644
--- a/deploy/egress-proxy-pub/fly.toml
+++ b/cli/templates/fly/egress-proxy.toml
@@ -1,11 +1,10 @@
-app = "qm-egress-proxy-pub"
+app = "qm-egress-proxy"
primary_region = "sjc"
-[build]
- dockerfile = "../egress-proxy/Dockerfile"
-
[env]
- EGRESS_TOKENLESS = "deny"
+ QM_DEPLOYMENT_ID = "qm-v2:personal:acme:qm"
+ CORE_API_URL = "http://qm-core.internal:8080"
+ CORE_ORG_ID = "acme"
[[services]]
internal_port = 48080
diff --git a/cli/test/auth-broker.test.ts b/cli/test/auth-broker.test.ts
index 43f37acec..cfb8de14c 100644
--- a/cli/test/auth-broker.test.ts
+++ b/cli/test/auth-broker.test.ts
@@ -43,7 +43,7 @@ test("the auth service is a first-class, privately addressed service with its ow
assert.equal(def.docker.hostPortOffset, undefined, "docker must not publish the broker on a host port");
const offsets = SERVICE_NAMES.map((name) => serviceDef(name).docker.hostPortOffset);
assert.equal(new Set(offsets).size, offsets.length, "host port offsets stay unique");
- const slots = SERVICE_NAMES.map((name) => serviceDef(name).dev.portSlotOffset);
+ const slots = SERVICE_NAMES.map((name) => serviceDef(name).dev?.portSlotOffset);
assert.equal(new Set(slots).size, slots.length, "dev port slots stay unique");
assert.ok(def.order < serviceDef("portal").order, "the broker comes up before the portal that publishes it");
assert.deepEqual(
diff --git a/cli/test/aws-public-paths.test.ts b/cli/test/aws-public-paths.test.ts
index d47031bab..2cc407856 100644
--- a/cli/test/aws-public-paths.test.ts
+++ b/cli/test/aws-public-paths.test.ts
@@ -193,3 +193,22 @@ for (const name of ["v1", "d", "key", "models", "slack"]) {
}
});
}
+
+test("the egress proxy may sit behind its own load balancer while other private services may not", () => {
+ const f = fixture();
+ try {
+ const loaded = loadConfigAt(f.configPath).config;
+ const proxy = { ecrRepository: "egress-proxy", ecsService: "egress-proxy", cpu: 256, memory: 512 };
+ const config = {
+ ...loaded,
+ services: [...loaded.services, "egress-proxy" as const],
+ aws: { ...loaded.aws!, services: { ...loaded.aws!.services, "egress-proxy": proxy } },
+ };
+ const services = new Map([...f.services, ["egress-proxy", { loadBalancers: [{ targetGroupArn: "nlb-tcp" }] }]]);
+ assert.equal(assertAwsPublicRouting(config, services).has("egress-proxy"), false);
+ services.set("web-ui", { loadBalancers: [{ targetGroupArn: "nlb-tcp" }] });
+ assert.throws(() => assertAwsPublicRouting(config, services), /private ECS service web-ui is attached/);
+ } finally {
+ f.close();
+ }
+});
diff --git a/cli/test/e2e/aws-rds-config.e2e.test.ts b/cli/test/e2e/aws-terraform.e2e.test.ts
similarity index 66%
rename from cli/test/e2e/aws-rds-config.e2e.test.ts
rename to cli/test/e2e/aws-terraform.e2e.test.ts
index f57e0d30d..26a1ac700 100644
--- a/cli/test/e2e/aws-rds-config.e2e.test.ts
+++ b/cli/test/e2e/aws-terraform.e2e.test.ts
@@ -165,3 +165,72 @@ test(
}
},
);
+
+test(
+ "AWS init plans the egress proxy behind its own TLS network load balancer",
+ { skip: !terraformRequested && !terraformBin ? "Terraform is unavailable" : false },
+ () => {
+ assert.ok(terraformBin, `QM_TEST_TERRAFORM is not executable: ${process.env.QM_TEST_TERRAFORM}`);
+ const dir = tmp("aws-egress-proxy");
+ try {
+ const initialized = runCli(["init", dir, "--org", "egress", "--target", "aws", "--model-provider", "anthropic"]);
+ assert.equal(initialized.code, 0, initialized.out);
+ configureAwsCoordinates(dir);
+ const path = join(dir, "qm.config.jsonc");
+ writeFileSync(
+ path,
+ readFileSync(path, "utf8")
+ .replace('"services": [', '"services": ["egress-proxy", ')
+ .replace(
+ '"services": {',
+ '"services": { "egress-proxy": { "ecrRepository": "egress-proxy", "ecsService": "egress-proxy", "cpu": 512, "memory": 1024 },',
+ ),
+ );
+ configureTerraform(dir);
+ const rendered = runCli(["infra", "render", "--config", path]);
+ assert.equal(rendered.code, 0, rendered.out);
+ writeFileSync(
+ join(dir, "infra", "egress.tftest.hcl"),
+ `mock_provider "aws" {
+ mock_data "aws_availability_zones" {
+ defaults = { names = ["us-west-2a", "us-west-2b"] }
+ }
+}
+mock_provider "random" {}
+run "tls" {
+ command = plan
+ variables { certificate_arn = "arn:aws:acm:us-west-2:123456789012:certificate/00000000-0000-0000-0000-000000000000" }
+ assert {
+ condition = aws_lb.egress_proxy["egress-proxy"].load_balancer_type == "network" && aws_lb.egress_proxy["egress-proxy"].enable_cross_zone_load_balancing && aws_lb_listener.egress_proxy["egress-proxy"].port == 443 && aws_lb_listener.egress_proxy["egress-proxy"].protocol == "TLS"
+ error_message = "the proxy is not behind a TLS network load balancer"
+ }
+ assert {
+ condition = aws_lb_target_group.egress_proxy["egress-proxy"].protocol == "TCP" && aws_lb_target_group.egress_proxy["egress-proxy"].port == 48080 && length(aws_ecs_service.service["egress-proxy"].load_balancer) == 1 && !contains(keys(aws_lb_target_group.service), "egress-proxy")
+ error_message = "the proxy is not attached to its TCP target group alone"
+ }
+ assert {
+ condition = aws_ecs_service.service["egress-proxy"].deployment_configuration[0].strategy == "ROLLING" && aws_ecs_service.service["core"].deployment_configuration[0].strategy == "BLUE_GREEN"
+ error_message = "unexpected rollout strategies"
+ }
+ assert {
+ condition = length(aws_ecs_service.service["egress-proxy"].service_connect_configuration[0].service[0].timeout) == 0 && aws_ecs_service.service["core"].service_connect_configuration[0].service[0].timeout[0].per_request_timeout_seconds == 0
+ error_message = "ECS rejects a per-request timeout on the proxy's TCP Service Connect port"
+ }
+ assert {
+ condition = !can(jsondecode(aws_ecs_task_definition.bootstrap["egress-proxy"].container_definitions)[0].portMappings[0].appProtocol) && jsondecode(aws_ecs_task_definition.bootstrap["core"].container_definitions)[0].portMappings[0].appProtocol == "http"
+ error_message = "the proxy's bootstrap port mapping must be plain TCP"
+ }
+}
+run "plaintext" {
+ command = plan
+ expect_failures = [aws_lb_listener.egress_proxy]
+}
+`,
+ );
+ terraform(dir, terraformBin, ["init", "-backend=false"]);
+ assert.match(terraform(dir, terraformBin, ["test", "-verbose"]), /Success!.*2 passed, 0 failed/s);
+ } finally {
+ rmDir(dir);
+ }
+ },
+);
diff --git a/cli/test/egress-proxy.test.ts b/cli/test/egress-proxy.test.ts
new file mode 100644
index 000000000..bf3cbda16
--- /dev/null
+++ b/cli/test/egress-proxy.test.ts
@@ -0,0 +1,105 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { createServer as createHttpServer } from "node:http";
+import { createServer as createTcpServer, type Server } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { renderTaskDefinition } from "../src/backends/aws.ts";
+import type { QmConfig } from "../src/config.ts";
+import { renderTerraformVars } from "../src/terraform.ts";
+
+const service = (name: string) => ({ ecrRepository: `qm-${name}`, ecsService: `acme-${name}`, cpu: 512, memory: 1024 });
+
+const config: QmConfig = {
+ contract: 1,
+ orgId: "acme",
+ publicUrl: "https://agent.acme.example",
+ apiUrl: "https://api.acme.example",
+ target: "aws",
+ services: ["core", "egress-proxy"],
+ plugins: [],
+ skills: [],
+ imageOverrides: {},
+ env: { core: { HARNESS: "pi", AWS_DEPLOY_IMAGE: "acme-qm-sandbox" } },
+ aws: {
+ accountId: "123456789012",
+ region: "us-west-2",
+ cluster: "acme-qm",
+ deployRoleArn: "arn:aws:iam::123456789012:role/acme-deploy",
+ imageLabel: "release",
+ secretsPrefix: "acme/qm/",
+ networking: { cloudMapNamespace: "acme.internal" },
+ services: { core: service("core"), "egress-proxy": service("egress-proxy") },
+ },
+};
+
+const image = `123456789012.dkr.ecr.us-west-2.amazonaws.com/qm-egress-proxy@sha256:${"a".repeat(64)}`;
+const proxyContainer = (deployment = config) =>
+ renderTaskDefinition(deployment, "egress-proxy", image).containerDefinitions[0]!;
+const coreApiUrl = (deployment: QmConfig) =>
+ (proxyContainer(deployment).environment as Array<{ name: string; value: string }>).find(
+ (entry) => entry.name === "CORE_API_URL",
+ )?.value;
+
+test("AWS runs the egress proxy as plain TCP with the keys it verifies and relays with", () => {
+ const container = proxyContainer();
+ assert.deepEqual(container.portMappings, [
+ { name: "egress-proxy", containerPort: 48080, hostPort: 48080, protocol: "tcp" },
+ ]);
+ assert.deepEqual(
+ (container.secrets as Array<{ name: string }>).map((secret) => secret.name),
+ ["CAPABILITY_SECRET", "CORE_SIGNING_SECRET"],
+ );
+ assert.equal(coreApiUrl(config), "https://api.acme.example");
+ const { apiUrl: _apiUrl, ...withoutApiUrl } = config;
+ assert.equal(coreApiUrl(withoutApiUrl), "https://agent.acme.example");
+});
+
+const adminSocket = "/tmp/envoy-admin.sock";
+
+function listen(server: Server, ...args: unknown[]): Promise<void> {
+ return new Promise((resolve) => (server.listen as (...a: unknown[]) => void)(...args, resolve));
+}
+
+test(
+ "the AWS proxy health check passes only while the decision service listens and Envoy reports ready",
+ { skip: existsSync(adminSocket) ? `${adminSocket} is in use` : false },
+ async () => {
+ const [, ...command] = (proxyContainer().healthCheck as { command: string[] }).command;
+ const probe = () =>
+ new Promise<number | null>((resolve) =>
+ spawn(command[0]!, command.slice(1), { timeout: 10_000 }).on("exit", resolve),
+ );
+ assert.notEqual(await probe(), 0);
+ let ready = 200;
+ const authz = createTcpServer((socket) => socket.end());
+ const envoy = createHttpServer((_req, res) => res.writeHead(ready).end());
+ await listen(authz, 48081, "127.0.0.1");
+ try {
+ assert.notEqual(await probe(), 0);
+ await listen(envoy, adminSocket);
+ assert.equal(await probe(), 0);
+ ready = 503;
+ assert.notEqual(await probe(), 0);
+ } finally {
+ authz.close();
+ envoy.close();
+ }
+ },
+);
+
+test("a vendored AWS scaffold without the proxy load balancer refuses the egress proxy", () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-legacy-egress-"));
+ try {
+ const infra = join(dir, "infra");
+ mkdirSync(infra);
+ writeFileSync(join(infra, "terraform.tfvars"), "services = {}\n");
+ writeFileSync(join(infra, "variables.tf"), 'variable "services" { type = map(any) }\n');
+ writeFileSync(join(infra, "main.tf"), 'resource "aws_lb" "this" {}\n');
+ assert.throws(() => renderTerraformVars(config, dir), /AWS scaffold predates the egress-proxy service/);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
diff --git a/cli/test/fly-up.test.ts b/cli/test/fly-up.test.ts
index 8367eb853..29a55adb7 100644
--- a/cli/test/fly-up.test.ts
+++ b/cli/test/fly-up.test.ts
@@ -166,3 +166,66 @@ test('--only "slack" explains the virtual service runs in-process on the core',
rmSync(dir, { recursive: true, force: true });
}
});
+
+test("fly up gives the egress proxy public ingress once, before deploying it", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "qm-fly-egress-"));
+ const fly = join(dir, "fly");
+ const log = join(dir, "fly.log");
+ const ipsFile = join(dir, "ips");
+ writeFileSync(
+ fly,
+ `#!/usr/bin/env node
+const fs = require("node:fs");
+const a = process.argv.slice(2);
+fs.appendFileSync(${JSON.stringify(log)}, a.join(" ") + "\\n");
+if (a[0] === "apps" && a[1] === "create") console.log("already been taken");
+else if (a[0] === "apps" && a[1] === "list") console.log(JSON.stringify([{ Name: "acme-core" }, { Name: "acme-egress-proxy" }]));
+else if (a[0] === "secrets" && a[1] === "list") console.log("QM_OWNER_CD83933C6C53374D digest\\nCAPABILITY_SECRET digest\\nCORE_SIGNING_SECRET digest");
+else if (a[0] === "ips" && a[1] === "list") console.log(fs.existsSync(${JSON.stringify(ipsFile)}) ? fs.readFileSync(${JSON.stringify(ipsFile)}, "utf8") : "[]");
+else console.log("ok");
+`,
+ );
+ chmodSync(fly, 0o755);
+ const prior = process.env.FLY_BIN;
+ process.env.FLY_BIN = fly;
+ const config: QmConfig = {
+ contract: 1,
+ orgId: "acme",
+ publicUrl: "https://acme.fly.dev",
+ target: "fly",
+ appPrefix: "acme",
+ region: "sjc",
+ flyOrg: "personal",
+ services: ["core", "egress-proxy"],
+ plugins: [],
+ skills: [],
+ env: {},
+ imageOverrides: { "egress-proxy": `registry.fly.io/acme-egress-proxy@sha256:${"c".repeat(64)}` },
+ sandbox: { app: "acme-sandboxes" },
+ };
+ try {
+ await flyUp(config, dir, { only: ["egress-proxy"] });
+ let calls = readFileSync(log, "utf8");
+ assert.match(calls, /^ips allocate-v6 -a acme-egress-proxy$/m);
+ assert.match(calls, /^ips allocate-v4 --shared -a acme-egress-proxy$/m);
+ assert.ok(calls.indexOf("ips allocate-v4") < calls.indexOf("deploy --yes"));
+
+ writeFileSync(log, "");
+ writeFileSync(
+ ipsFile,
+ JSON.stringify([
+ { Type: "private_v6", Address: "fdaa::3" },
+ { Type: "v6", Address: "2a09:8280:1::1:0" },
+ { Type: "shared_v4", Address: "66.241.124.1" },
+ ]),
+ );
+ await flyUp(config, dir, { only: ["egress-proxy"] });
+ calls = readFileSync(log, "utf8");
+ assert.doesNotMatch(calls, /ips allocate/);
+ assert.match(calls, /^deploy --yes .*acme-egress-proxy/m);
+ } finally {
+ if (prior === undefined) delete process.env.FLY_BIN;
+ else process.env.FLY_BIN = prior;
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
diff --git a/cli/test/terraform.test.ts b/cli/test/terraform.test.ts
index 386f6229d..5ae36fdce 100644
--- a/cli/test/terraform.test.ts
+++ b/cli/test/terraform.test.ts
@@ -763,7 +763,7 @@ test("AWS ECS services wait until target groups are attached to the ALB", () =>
const service = mainTf.match(/resource "aws_ecs_service" "service" \{([\s\S]*?)\n\}/)?.[1] ?? "";
assert.match(
service,
- /depends_on\s*= \[aws_lb_listener\.public, aws_lb_listener_rule\.production, aws_iam_role_policy_attachment\.ecs_load_balancer\]/,
+ /depends_on\s*= \[aws_lb_listener\.public, aws_lb_listener_rule\.production, aws_lb_listener\.egress_proxy, aws_iam_role_policy_attachment\.ecs_load_balancer\]/,
);
});
@@ -773,7 +773,7 @@ test("AWS ECS services use native blue-green rollouts with rollback signals and
assert.match(versions, /version\s*= ">= 6\.6, < 7\.0"/);
assert.match(
service,
- /deployment_configuration \{[\s\S]*strategy\s*= "BLUE_GREEN"[\s\S]*bake_time_in_minutes = var\.ecs_blue_green_bake_minutes/,
+ /deployment_configuration \{[\s\S]*strategy\s*= .*"BLUE_GREEN"[\s\S]*bake_time_in_minutes = .*var\.ecs_blue_green_bake_minutes/,
);
assert.match(
service,
@@ -790,7 +790,7 @@ test("AWS ECS services use native blue-green rollouts with rollback signals and
assert.match(service, /sigint_rollback = true/);
assert.match(
mainTf,
- /portMappings = \[\{ name = each\.key, containerPort = each\.value\.internal_port, appProtocol = "http" \}\]/,
+ /portMappings = \[merge\(\{ name = each\.key, containerPort = each\.value\.internal_port \}, contains\(keys\(local\.egress_proxy\), each\.key\) \? \{\} : \{ appProtocol = "http" \}\)\]/,
);
});
diff --git a/deploy/egress-proxy/fly.toml b/deploy/egress-proxy/fly.toml
deleted file mode 100644
index 507e936a4..000000000
--- a/deploy/egress-proxy/fly.toml
+++ /dev/null
@@ -1,35 +0,0 @@
-# The forced egress proxy (Fly realization). Published PRIVATELY — flycast only,
-# no public IP (`fly ips allocate-v6 --private` before first deploy; never `fly ips allocate-*` a
-# public one). Sandboxes reach it at `<app>.flycast:<port>` via HTTPS_PROXY; a default-deny egress
-# Network Policy on each sandbox allows ONLY this port, forcing all egress through here.
-#
-# Port note: 48080 is deliberately OBSCURE. The Fly Network Policy can only allow egress by PORT
-# (not by destination), so whatever port we allow is also reachable on ANY host directly — the
-# "port leak". An obscure port keeps that leak small (a denied host is unlikely to serve on it);
-# direct egress on 443/80 — i.e. ~all real egress — stays fully blocked.
-app = "qm-egress-proxy"
-primary_region = "sjc"
-
-
-[[services]]
- internal_port = 48080
- protocol = "tcp"
- auto_stop_machines = false
- min_machines_running = 1
-
- [[services.ports]]
- port = 48080
-
- [[services.tcp_checks]]
- interval = "15s"
- timeout = "2s"
- grace_period = "10s"
-
-# Envoy + the Node decision service sit at ~130mb resident before serving anything,
-# which leaves almost no headroom on 256mb under real fleet traffic.
-[[vm]]
- size = "shared-cpu-1x"
- memory = "512mb"
-
-kill_signal = "SIGTERM"
-kill_timeout = "10s"
diff --git a/deploy/stacks/acme/qm.config.jsonc b/deploy/stacks/acme/qm.config.jsonc
index a9bbf90cd..576de4f10 100644
--- a/deploy/stacks/acme/qm.config.jsonc
+++ b/deploy/stacks/acme/qm.config.jsonc
@@ -10,7 +10,7 @@
"sandbox": {
"app": "acme-sandboxes",
},
- "services": ["core", "slack", "admin", "web-ui", "portal"],
+ "services": ["core", "slack", "admin", "web-ui", "portal", "egress-proxy"],
"env": {
"admin": {
"ADMIN_BASE_PATH": "/admin",
diff --git a/docs/deploy-directory.md b/docs/deploy-directory.md
index 6dcf8b368..dbda08631 100644
--- a/docs/deploy-directory.md
+++ b/docs/deploy-directory.md
@@ -191,6 +191,25 @@ and listener precedence, including native blue/green weighted routing. Update
vendored Terraform with the `public_paths` service attribute and
`public_path_services` routing before enabling this setting.
+### Sandbox egress proxy
+
+Add `"egress-proxy"` to `services` to deploy the sandbox egress proxy
+(`deploy/egress-proxy`) with every `qm up`, from the same release as core. It
+receives `CAPABILITY_SECRET` and `CORE_SIGNING_SECRET` from the computed secret
+set and relays its audit records to core.
+
+On AWS the reference module puts it behind its own network load balancer, which
+terminates TLS on 443 with the module's `certificate_arn` (required) and forwards
+plain TCP to the proxy. The proxy runs in its own security group, outside the
+services' private network, so it relays audit records through `apiUrl` (or
+`publicUrl`). Point a hostname covered by the certificate at the
+`egress_proxy_hostname` output. On Fly it serves TLS on 443 at
+`<appPrefix>-egress-proxy.fly.dev`. Docker publishes the plain listener on a host
+port and leaves TLS to the operator. Then set the sandbox backend's proxy URL, for
+example `env.core.SPRITES_EGRESS_PROXY_URL` or `MODAL_EGRESS_PROXY_URL`, to
+`https://<proxy hostname>`. Update vendored Terraform from the current scaffold
+before enabling it on AWS.
+
### Shared AWS foundations
Existing AWS deployments retain their company-owned resources by default. For a shared DynamoDB deployment table, set `aws.deploymentState` to `{ "table": "fleet-deploy-locks", "namespace": "acme" }`. Every lease, current pointer, label pointer and immutable manifest uses that namespace. Use a unique namespace for each company and scope its item IAM permissions with `dynamodb:LeadingKeys` to `acme/*`; grant `DescribeTable` separately. Changing an existing deployment's state location requires explicitly migrating its history while deployment operations are stopped.
diff --git a/docs/e2b-template.md b/docs/e2b-template.md
index 5c6cc20cc..794dcd860 100644
--- a/docs/e2b-template.md
+++ b/docs/e2b-template.md
@@ -94,7 +94,7 @@ point and uses E2B's rules only to make that proxy unavoidable:
does neither.
Set `E2B_EGRESS_PROXY_URL` to a QM egress proxy reachable on port 443, for example
-`deploy/egress-proxy-pub` (TLS on 443 in front of the proxy's port 48080). Core then
+the `egress-proxy` service (TLS on 443 in front of the proxy's port 48080). Core then
creates every agent computer with `allowOut: [<proxy host>]` and `denyOut: ["0.0.0.0/0"]`
and points the computer's proxy settings at it with the turn's token. Without it, core
warns at boot and computers have open egress.
diff --git a/test/release-workflows.test.ts b/test/release-workflows.test.ts
index 1e4831ce7..a1d052de1 100644
--- a/test/release-workflows.test.ts
+++ b/test/release-workflows.test.ts
@@ -84,7 +84,7 @@ test("the published package pins real image digests, never the checked-in sentin
workflow.indexOf("Pin published image digests") < workflow.indexOf("npm publish"),
"digests are resolved before the package is published",
);
- assert.match(workflow, /for service in core web-ui admin portal auth sandbox-base; do/);
+ assert.match(workflow, /for service in core web-ui admin portal auth egress-proxy sandbox-base; do/);
assert.match(workflow, /printf '%s\\n' "\$out" > cli\/manifest\.json/);
assert.match(workflow, /no published image for \$repo at \$IMAGES_REF/);
assert.match(workflow, /\{63\}\$"\) \| not\)/);
@@ -94,7 +94,7 @@ test("the published package pins real image digests, never the checked-in sentin
services: Record<string, string>;
};
const refs = [sentinel.sandboxBase, ...Object.values(sentinel.services)];
- assert.equal(refs.length, 6);
+ assert.equal(refs.length, 7);
assert.ok(
refs.every((ref) => ref.startsWith("registry.invalid/")),
"the checked-in manifest stays a sentinel so a source checkout never pulls a stale digest",