Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

egress: unconditional loopback deny, Envoy admin (#399)

Regan Bell624b23570dda
diff --git a/deploy/egress-proxy/Dockerfile b/deploy/egress-proxy/Dockerfile
index 6c9b077a1..038e143a5 100644
--- a/deploy/egress-proxy/Dockerfile
+++ b/deploy/egress-proxy/Dockerfile
@@ -14,6 +14,9 @@ COPY src ./src
COPY tsconfig.json ./
COPY deploy/egress-proxy/envoy.yaml deploy/egress-proxy/start.sh ./
RUN chmod +x start.sh
+# A rejected config is a fleet-wide outage: start.sh kills the container when Envoy dies, so an
+# invalid envoy.yaml has to fail the build, not the deploy.
+RUN envoy --mode validate -c /app/envoy.yaml
ENV NODE_ENV=production
EXPOSE 48080
diff --git a/deploy/egress-proxy/envoy.yaml b/deploy/egress-proxy/envoy.yaml
index a2074c6a0..1e2ca2b37 100644
--- a/deploy/egress-proxy/envoy.yaml
+++ b/deploy/egress-proxy/envoy.yaml
@@ -6,6 +6,16 @@
#
# Cloud metadata authorities (IMDS 169.254.169.254, GCP metadata, AWS IPv6 IMDS) are ALSO denied
# statically here at the Envoy layer — belt and braces.
+
+# A UNIX SOCKET, never a TCP port — not even loopback. This process is a forward proxy: a
+# sandbox that asks it to `CONNECT 127.0.0.1:<port>` makes it dial its own namespace, so a
+# loopback admin listener is reachable by every sandbox and /quitquitquit would drop fleet
+# egress on demand. The decision service denies loopback destinations too; this is the other half.
+# Read it with `curl --unix-socket /tmp/envoy-admin.sock http://localhost/stats`. mode 384 = 0600.
+admin:
+ address:
+ pipe: { path: /tmp/envoy-admin.sock, mode: 384 }
+
static_resources:
listeners:
- name: egress_proxy
@@ -17,6 +27,17 @@ static_resources:
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: egress_proxy
+ # One line per connection, on stdout (`fly logs`). %RESPONSE_FLAGS% and
+ # %RESPONSE_CODE_DETAILS% are the point: they name WHY a request failed
+ # (lua_response, upstream_reset_before_response_started{…}), which the
+ # 503's body carries but a failed CONNECT throws away.
+ access_log:
+ - name: envoy.access_loggers.stdout
+ typed_config:
+ "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+ log_format:
+ text_format_source:
+ inline_string: "[envoy] %REQ(:METHOD)% %REQ(:AUTHORITY):256% %RESPONSE_CODE% flags=%RESPONSE_FLAGS% detail=%RESPONSE_CODE_DETAILS% upstream=%UPSTREAM_HOST% dur=%DURATION%\n"
http_protocol_options:
allow_absolute_url: true
upgrade_configs:
diff --git a/deploy/egress-proxy/fly.toml b/deploy/egress-proxy/fly.toml
index e039ca0af..507e936a4 100644
--- a/deploy/egress-proxy/fly.toml
+++ b/deploy/egress-proxy/fly.toml
@@ -25,9 +25,11 @@ primary_region = "sjc"
timeout = "2s"
grace_period = "10s"
+# Envoy + the Node decision service sit at ~130mb resident before serving anything,
+# which leaves almost no headroom on 256mb under real fleet traffic.
[[vm]]
size = "shared-cpu-1x"
- memory = "256mb"
+ memory = "512mb"
kill_signal = "SIGTERM"
kill_timeout = "10s"
diff --git a/src/egress-authz-main.ts b/src/egress-authz-main.ts
index c5cbe5c91..e484e80c9 100644
--- a/src/egress-authz-main.ts
+++ b/src/egress-authz-main.ts
@@ -18,12 +18,16 @@ const DENY_ALL: EgressPolicy = { allowedHosts: ["deny.invalid"], deniedHosts: []
const METADATA_HOSTS = ["metadata.google.internal", "metadata.goog"];
-const LINK_LOCAL = new BlockList();
-LINK_LOCAL.addSubnet("169.254.0.0", 16, "ipv4");
-LINK_LOCAL.addSubnet("fe80::", 10, "ipv6");
-LINK_LOCAL.addAddress("fd00:ec2::254", "ipv6");
-
-export function isLinkLocalOrMetadataIp(ip: string): boolean {
+const BLOCKED = new BlockList();
+BLOCKED.addSubnet("169.254.0.0", 16, "ipv4");
+BLOCKED.addSubnet("fe80::", 10, "ipv6");
+BLOCKED.addAddress("fd00:ec2::254", "ipv6");
+BLOCKED.addSubnet("127.0.0.0", 8, "ipv4");
+BLOCKED.addSubnet("0.0.0.0", 8, "ipv4");
+BLOCKED.addAddress("::1", "ipv6");
+BLOCKED.addAddress("::", "ipv6");
+
+export function isBlockedDestinationIp(ip: string): boolean {
const s = ip
.trim()
.toLowerCase()
@@ -31,7 +35,7 @@ export function isLinkLocalOrMetadataIp(ip: string): boolean {
.replace(/%.*$/, "");
const fam = isIP(s);
if (!fam) return false;
- return LINK_LOCAL.check(s, fam === 4 ? "ipv4" : "ipv6");
+ return BLOCKED.check(s, fam === 4 ? "ipv4" : "ipv6");
}
function isAlwaysBlockedHost(host: string): boolean {
@@ -40,7 +44,7 @@ function isAlwaysBlockedHost(host: string): boolean {
.toLowerCase()
.replace(/^\[(.*)\]$/, "$1")
.replace(/\.$/, "");
- if (isIP(h)) return isLinkLocalOrMetadataIp(h);
+ if (isIP(h)) return isBlockedDestinationIp(h);
return METADATA_HOSTS.some((m) => h === m || h.endsWith(`.${m}`));
}
@@ -108,7 +112,7 @@ async function decide(
if (
ips.some(
(ip) =>
- isLinkLocalOrMetadataIp(ip) ||
+ isBlockedDestinationIp(ip) ||
isHostDenied(ip, policy?.deniedHosts) ||
(policy?.denyPrivateNetworks === true && !privateAllowed && isPrivateNetworkIp(ip)),
)
diff --git a/test/egress-authz.test.ts b/test/egress-authz.test.ts
index 2715a42fa..0f851b604 100644
--- a/test/egress-authz.test.ts
+++ b/test/egress-authz.test.ts
@@ -7,7 +7,7 @@ import {
createRelayAuditSink,
tokenFromRequest,
hostFromAuthority,
- isLinkLocalOrMetadataIp,
+ isBlockedDestinationIp,
type EgressAuthzDeps,
} from "../src/egress-authz-main.ts";
import { verifySignature } from "../src/auth/source-auth.ts";
@@ -111,26 +111,31 @@ test("hostFromAuthority strips ports and IPv6 brackets", () => {
assert.equal(hostFromAuthority(""), null);
});
-test("isLinkLocalOrMetadataIp covers IMDS, 169.254/16, fe80::/10, v4-mapped, AWS v6 IMDS", () => {
- assert.equal(isLinkLocalOrMetadataIp("169.254.169.254"), true);
- assert.equal(isLinkLocalOrMetadataIp("169.254.1.1"), true);
- assert.equal(isLinkLocalOrMetadataIp("::ffff:169.254.169.254"), true);
- assert.equal(isLinkLocalOrMetadataIp("fe80::1"), true);
- assert.equal(isLinkLocalOrMetadataIp("febf::1"), true);
- assert.equal(isLinkLocalOrMetadataIp("fd00:ec2::254"), true);
- assert.equal(isLinkLocalOrMetadataIp("[fd00:ec2::254]"), true);
- assert.equal(isLinkLocalOrMetadataIp("8.8.8.8"), false);
- assert.equal(isLinkLocalOrMetadataIp("2606:4700::1111"), false);
- assert.equal(isLinkLocalOrMetadataIp("fec0::1"), false);
+test("isBlockedDestinationIp covers IMDS, 169.254/16, fe80::/10, v4-mapped, AWS v6 IMDS", () => {
+ assert.equal(isBlockedDestinationIp("169.254.169.254"), true);
+ assert.equal(isBlockedDestinationIp("169.254.1.1"), true);
+ assert.equal(isBlockedDestinationIp("::ffff:169.254.169.254"), true);
+ assert.equal(isBlockedDestinationIp("fe80::1"), true);
+ assert.equal(isBlockedDestinationIp("febf::1"), true);
+ assert.equal(isBlockedDestinationIp("fd00:ec2::254"), true);
+ assert.equal(isBlockedDestinationIp("[fd00:ec2::254]"), true);
+ assert.equal(isBlockedDestinationIp("8.8.8.8"), false);
+ assert.equal(isBlockedDestinationIp("2606:4700::1111"), false);
+ assert.equal(isBlockedDestinationIp("fec0::1"), false);
});
-test("isLinkLocalOrMetadataIp catches non-canonical spellings of blocked addresses", () => {
- assert.equal(isLinkLocalOrMetadataIp("::ffff:a9fe:a9fe"), true);
- assert.equal(isLinkLocalOrMetadataIp("[::ffff:a9fe:a9fe]"), true);
- assert.equal(isLinkLocalOrMetadataIp("fd00:ec2:0:0:0:0:0:254"), true);
- assert.equal(isLinkLocalOrMetadataIp("fd00:0ec2::0254"), true);
- assert.equal(isLinkLocalOrMetadataIp("FE80::1"), true);
- assert.equal(isLinkLocalOrMetadataIp("fe80::1%eth0"), true);
+test("isBlockedDestinationIp catches non-canonical spellings of blocked addresses", () => {
+ assert.equal(isBlockedDestinationIp("::ffff:a9fe:a9fe"), true);
+ assert.equal(isBlockedDestinationIp("[::ffff:a9fe:a9fe]"), true);
+ assert.equal(isBlockedDestinationIp("fd00:ec2:0:0:0:0:0:254"), true);
+ assert.equal(isBlockedDestinationIp("fd00:0ec2::0254"), true);
+ assert.equal(isBlockedDestinationIp("FE80::1"), true);
+ assert.equal(isBlockedDestinationIp("fe80::1%eth0"), true);
+ assert.equal(isBlockedDestinationIp("0.0.0.0"), true);
+ assert.equal(isBlockedDestinationIp("0.0.0.7"), true);
+ assert.equal(isBlockedDestinationIp("::"), true);
+ assert.equal(isBlockedDestinationIp("::ffff:0:0"), true);
+ assert.equal(isBlockedDestinationIp("[::]"), true);
});
test("valid token, open policy => 200 + audited as ok with scope/principal", async () => {
@@ -263,6 +268,23 @@ test("a name RESOLVING to link-local / denied IP is denied (rebind guard)", asyn
}
});
+// A forward proxy dials from ITS namespace, so an allowed `CONNECT 127.0.0.1:<port>` would bridge
+// any caller to the services colocated with the proxy — this decision service, Envoy's admin.
+test("loopback destinations are denied even under an open policy, by literal and by resolved IP", async () => {
+ const { server, records } = boot({ lookup: async () => ["127.0.0.1"] });
+ const port = await listen(server);
+ try {
+ const open = await egressToken({ allowedHosts: [] });
+ for (const authority of ["127.0.0.1:9901", "127.0.0.1:48081", "[::1]:9901", "127.1.2.3:80"]) {
+ assert.equal(await check(port, authority, open), 403, authority);
+ }
+ assert.equal(await check(port, "rebind.example.com:9901", open), 403);
+ assert.equal(records.at(-1)?.verdict, "denied");
+ } finally {
+ await close(server);
+ }
+});
+
test("a name resolving to a policy-denied IP is denied", async () => {
const { server } = boot({ lookup: async () => ["10.9.9.9"] });
const port = await listen(server);