Multiplayer agent harness for work.
egress: unconditional loopback deny, Envoy admin (#399)
Regan Bell624b23570dda
diff --git a/deploy/egress-proxy/Dockerfile b/deploy/egress-proxy/Dockerfileindex 6c9b077a1..038e143a5 100644--- a/deploy/egress-proxy/Dockerfile+++ b/deploy/egress-proxy/Dockerfile@@ -14,6 +14,9 @@ COPY src ./srcCOPY tsconfig.json ./COPY deploy/egress-proxy/envoy.yaml deploy/egress-proxy/start.sh ./RUN chmod +x start.sh+# A rejected config is a fleet-wide outage: start.sh kills the container when Envoy dies, so an+# invalid envoy.yaml has to fail the build, not the deploy.+RUN envoy --mode validate -c /app/envoy.yamlENV NODE_ENV=productionEXPOSE 48080diff --git a/deploy/egress-proxy/envoy.yaml b/deploy/egress-proxy/envoy.yamlindex a2074c6a0..1e2ca2b37 100644--- a/deploy/egress-proxy/envoy.yaml+++ b/deploy/egress-proxy/envoy.yaml@@ -6,6 +6,16 @@## Cloud metadata authorities (IMDS 169.254.169.254, GCP metadata, AWS IPv6 IMDS) are ALSO denied# statically here at the Envoy layer — belt and braces.++# A UNIX SOCKET, never a TCP port — not even loopback. This process is a forward proxy: a+# sandbox that asks it to `CONNECT 127.0.0.1:<port>` makes it dial its own namespace, so a+# loopback admin listener is reachable by every sandbox and /quitquitquit would drop fleet+# egress on demand. The decision service denies loopback destinations too; this is the other half.+# Read it with `curl --unix-socket /tmp/envoy-admin.sock http://localhost/stats`. mode 384 = 0600.+admin:+ address:+ pipe: { path: /tmp/envoy-admin.sock, mode: 384 }+static_resources:listeners:- name: egress_proxy@@ -17,6 +27,17 @@ static_resources:typed_config:"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManagerstat_prefix: egress_proxy+ # One line per connection, on stdout (`fly logs`). %RESPONSE_FLAGS% and+ # %RESPONSE_CODE_DETAILS% are the point: they name WHY a request failed+ # (lua_response, upstream_reset_before_response_started{…}), which the+ # 503's body carries but a failed CONNECT throws away.+ access_log:+ - name: envoy.access_loggers.stdout+ typed_config:+ "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog+ log_format:+ text_format_source:+ inline_string: "[envoy] %REQ(:METHOD)% %REQ(:AUTHORITY):256% %RESPONSE_CODE% flags=%RESPONSE_FLAGS% detail=%RESPONSE_CODE_DETAILS% upstream=%UPSTREAM_HOST% dur=%DURATION%\n"http_protocol_options:allow_absolute_url: trueupgrade_configs:diff --git a/deploy/egress-proxy/fly.toml b/deploy/egress-proxy/fly.tomlindex e039ca0af..507e936a4 100644--- a/deploy/egress-proxy/fly.toml+++ b/deploy/egress-proxy/fly.toml@@ -25,9 +25,11 @@ primary_region = "sjc"timeout = "2s"grace_period = "10s"+# Envoy + the Node decision service sit at ~130mb resident before serving anything,+# which leaves almost no headroom on 256mb under real fleet traffic.[[vm]]size = "shared-cpu-1x"- memory = "256mb"+ memory = "512mb"kill_signal = "SIGTERM"kill_timeout = "10s"diff --git a/src/egress-authz-main.ts b/src/egress-authz-main.tsindex c5cbe5c91..e484e80c9 100644--- a/src/egress-authz-main.ts+++ b/src/egress-authz-main.ts@@ -18,12 +18,16 @@ const DENY_ALL: EgressPolicy = { allowedHosts: ["deny.invalid"], deniedHosts: []const METADATA_HOSTS = ["metadata.google.internal", "metadata.goog"];-const LINK_LOCAL = new BlockList();-LINK_LOCAL.addSubnet("169.254.0.0", 16, "ipv4");-LINK_LOCAL.addSubnet("fe80::", 10, "ipv6");-LINK_LOCAL.addAddress("fd00:ec2::254", "ipv6");--export function isLinkLocalOrMetadataIp(ip: string): boolean {+const BLOCKED = new BlockList();+BLOCKED.addSubnet("169.254.0.0", 16, "ipv4");+BLOCKED.addSubnet("fe80::", 10, "ipv6");+BLOCKED.addAddress("fd00:ec2::254", "ipv6");+BLOCKED.addSubnet("127.0.0.0", 8, "ipv4");+BLOCKED.addSubnet("0.0.0.0", 8, "ipv4");+BLOCKED.addAddress("::1", "ipv6");+BLOCKED.addAddress("::", "ipv6");++export function isBlockedDestinationIp(ip: string): boolean {const s = ip.trim().toLowerCase()@@ -31,7 +35,7 @@ export function isLinkLocalOrMetadataIp(ip: string): boolean {.replace(/%.*$/, "");const fam = isIP(s);if (!fam) return false;- return LINK_LOCAL.check(s, fam === 4 ? "ipv4" : "ipv6");+ return BLOCKED.check(s, fam === 4 ? "ipv4" : "ipv6");}function isAlwaysBlockedHost(host: string): boolean {@@ -40,7 +44,7 @@ function isAlwaysBlockedHost(host: string): boolean {.toLowerCase().replace(/^\[(.*)\]$/, "$1").replace(/\.$/, "");- if (isIP(h)) return isLinkLocalOrMetadataIp(h);+ if (isIP(h)) return isBlockedDestinationIp(h);return METADATA_HOSTS.some((m) => h === m || h.endsWith(`.${m}`));}@@ -108,7 +112,7 @@ async function decide(if (ips.some((ip) =>- isLinkLocalOrMetadataIp(ip) ||+ isBlockedDestinationIp(ip) ||isHostDenied(ip, policy?.deniedHosts) ||(policy?.denyPrivateNetworks === true && !privateAllowed && isPrivateNetworkIp(ip)),)diff --git a/test/egress-authz.test.ts b/test/egress-authz.test.tsindex 2715a42fa..0f851b604 100644--- a/test/egress-authz.test.ts+++ b/test/egress-authz.test.ts@@ -7,7 +7,7 @@ import {createRelayAuditSink,tokenFromRequest,hostFromAuthority,- isLinkLocalOrMetadataIp,+ isBlockedDestinationIp,type EgressAuthzDeps,} from "../src/egress-authz-main.ts";import { verifySignature } from "../src/auth/source-auth.ts";@@ -111,26 +111,31 @@ test("hostFromAuthority strips ports and IPv6 brackets", () => {assert.equal(hostFromAuthority(""), null);});-test("isLinkLocalOrMetadataIp covers IMDS, 169.254/16, fe80::/10, v4-mapped, AWS v6 IMDS", () => {- assert.equal(isLinkLocalOrMetadataIp("169.254.169.254"), true);- assert.equal(isLinkLocalOrMetadataIp("169.254.1.1"), true);- assert.equal(isLinkLocalOrMetadataIp("::ffff:169.254.169.254"), true);- assert.equal(isLinkLocalOrMetadataIp("fe80::1"), true);- assert.equal(isLinkLocalOrMetadataIp("febf::1"), true);- assert.equal(isLinkLocalOrMetadataIp("fd00:ec2::254"), true);- assert.equal(isLinkLocalOrMetadataIp("[fd00:ec2::254]"), true);- assert.equal(isLinkLocalOrMetadataIp("8.8.8.8"), false);- assert.equal(isLinkLocalOrMetadataIp("2606:4700::1111"), false);- assert.equal(isLinkLocalOrMetadataIp("fec0::1"), false);+test("isBlockedDestinationIp covers IMDS, 169.254/16, fe80::/10, v4-mapped, AWS v6 IMDS", () => {+ assert.equal(isBlockedDestinationIp("169.254.169.254"), true);+ assert.equal(isBlockedDestinationIp("169.254.1.1"), true);+ assert.equal(isBlockedDestinationIp("::ffff:169.254.169.254"), true);+ assert.equal(isBlockedDestinationIp("fe80::1"), true);+ assert.equal(isBlockedDestinationIp("febf::1"), true);+ assert.equal(isBlockedDestinationIp("fd00:ec2::254"), true);+ assert.equal(isBlockedDestinationIp("[fd00:ec2::254]"), true);+ assert.equal(isBlockedDestinationIp("8.8.8.8"), false);+ assert.equal(isBlockedDestinationIp("2606:4700::1111"), false);+ assert.equal(isBlockedDestinationIp("fec0::1"), false);});-test("isLinkLocalOrMetadataIp catches non-canonical spellings of blocked addresses", () => {- assert.equal(isLinkLocalOrMetadataIp("::ffff:a9fe:a9fe"), true);- assert.equal(isLinkLocalOrMetadataIp("[::ffff:a9fe:a9fe]"), true);- assert.equal(isLinkLocalOrMetadataIp("fd00:ec2:0:0:0:0:0:254"), true);- assert.equal(isLinkLocalOrMetadataIp("fd00:0ec2::0254"), true);- assert.equal(isLinkLocalOrMetadataIp("FE80::1"), true);- assert.equal(isLinkLocalOrMetadataIp("fe80::1%eth0"), true);+test("isBlockedDestinationIp catches non-canonical spellings of blocked addresses", () => {+ assert.equal(isBlockedDestinationIp("::ffff:a9fe:a9fe"), true);+ assert.equal(isBlockedDestinationIp("[::ffff:a9fe:a9fe]"), true);+ assert.equal(isBlockedDestinationIp("fd00:ec2:0:0:0:0:0:254"), true);+ assert.equal(isBlockedDestinationIp("fd00:0ec2::0254"), true);+ assert.equal(isBlockedDestinationIp("FE80::1"), true);+ assert.equal(isBlockedDestinationIp("fe80::1%eth0"), true);+ assert.equal(isBlockedDestinationIp("0.0.0.0"), true);+ assert.equal(isBlockedDestinationIp("0.0.0.7"), true);+ assert.equal(isBlockedDestinationIp("::"), true);+ assert.equal(isBlockedDestinationIp("::ffff:0:0"), true);+ assert.equal(isBlockedDestinationIp("[::]"), true);});test("valid token, open policy => 200 + audited as ok with scope/principal", async () => {@@ -263,6 +268,23 @@ test("a name RESOLVING to link-local / denied IP is denied (rebind guard)", asyn}});+// A forward proxy dials from ITS namespace, so an allowed `CONNECT 127.0.0.1:<port>` would bridge+// any caller to the services colocated with the proxy — this decision service, Envoy's admin.+test("loopback destinations are denied even under an open policy, by literal and by resolved IP", async () => {+ const { server, records } = boot({ lookup: async () => ["127.0.0.1"] });+ const port = await listen(server);+ try {+ const open = await egressToken({ allowedHosts: [] });+ for (const authority of ["127.0.0.1:9901", "127.0.0.1:48081", "[::1]:9901", "127.1.2.3:80"]) {+ assert.equal(await check(port, authority, open), 403, authority);+ }+ assert.equal(await check(port, "rebind.example.com:9901", open), 403);+ assert.equal(records.at(-1)?.verdict, "denied");+ } finally {+ await close(server);+ }+});+test("a name resolving to a policy-denied IP is denied", async () => {const { server } = boot({ lookup: async () => ["10.9.9.9"] });const port = await listen(server);