Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

docs: trim verbose comments across the egress MITM fix and e2e tests

高然88f3fc55b807
diff --git a/components/egress/nft.go b/components/egress/nft.go
index 3f29b138f..3a394c46c 100644
--- a/components/egress/nft.go
+++ b/components/egress/nft.go
@@ -114,9 +114,8 @@ func parseNftOptions(upstream *mitmproxy.UpstreamProxySpec) (nftables.Options, e
BlockDoT: true,
QuiesceOnApplyFailure: constants.IsTruthy(os.Getenv(constants.EnvExperimentalRevisionRuntime)),
}
- // Transparent MITM rewrites intercepted dports to its listener port before
- // the egress chain runs; the chain must exempt that rewritten tuple ahead
- // of port-scoped rules. Same derivation as startMitmproxyTransparentIfEnabled.
+ // Transparent MITM: exempt the REDIRECT target dport ahead of port-scoped
+ // rules (issue #2423). Same port derivation as startMitmproxyTransparentIfEnabled.
if constants.IsTruthy(os.Getenv(constants.EnvMitmproxyTransparent)) {
opts.MitmRedirectPort = constants.EnvIntOrDefault(constants.EnvMitmproxyPort, constants.DefaultMitmproxyPort)
}
diff --git a/components/egress/pkg/nftables/manager.go b/components/egress/pkg/nftables/manager.go
index e1e7fcaf3..5d351d238 100644
--- a/components/egress/pkg/nftables/manager.go
+++ b/components/egress/pkg/nftables/manager.go
@@ -65,13 +65,9 @@ type Options struct {
// sandbox workloads CONNECT it directly and bounce to denied destinations.
UpstreamProxy *UpstreamProxyEndpoint
// MitmRedirectPort, when > 0, is the transparent-MITM listener port the
- // nat OUTPUT REDIRECT rewrites intercepted dports (80/443 + extras) to.
- // The redirect runs at a lower priority than this filter chain and leaves
- // oif on the original egress interface, so the rewritten dport would
- // otherwise be judged as if the workload had dialed the listener itself:
- // the port floor complement, user port-scoped rules, and the final
- // default-deny drop would blackhole every intercepted flow. 0 (transparent
- // MITM disabled) emits nothing.
+ // nat OUTPUT REDIRECT rewrites intercepted dports to; the chain must
+ // exempt that rewritten tuple ahead of port-scoped rules or every
+ // intercepted flow is dropped (see the rule below). 0 emits nothing.
MitmRedirectPort int
}
@@ -303,13 +299,11 @@ func buildRuleset(p *policy.NetworkPolicy, opts Options) (string, error) {
// The ip6 OUTPUT REDIRECTs (DNS → :15353, MITM → :18081) land on ::1 with the original egress
// interface still selected, so oifname "lo" does not match them: ::1 is the v6 loopback rule.
fmt.Fprintf(&b, "add rule inet %s %s ip6 daddr ::1 accept\n", tableName, chainName)
- // The v4 MITM redirect lands on 127.0.0.1 with the original egress interface still
- // selected, so oifname "lo" does not match either; exempt the redirect target ahead
- // of every port-scoped rule (mirrors the dnsproxy rules above). Redirected flows
- // are still policed through mitmproxy's own re-originated connections, which are
- // not exempt. Direct workload dials of the listener already pass via oifname "lo"
- // (the route to 127.0.0.1 is lo), so this adds no new reachability; v6 is covered
- // by the ::1 rule above.
+ // The v4 MITM redirect lands on 127.0.0.1 with the original egress interface
+ // still selected, so oifname "lo" does not match: exempt the rewritten dport
+ // ahead of port-scoped rules (mirrors the dnsproxy rules above). Enforcement
+ // is intact — mitmproxy's own re-originated connections are not exempt — and
+ // direct workload dials already pass via oifname "lo"; v6 is covered by ::1.
if opts.MitmRedirectPort > 0 {
fmt.Fprintf(&b, "add rule inet %s %s ip daddr 127.0.0.1 tcp dport %d accept\n", tableName, chainName, opts.MitmRedirectPort)
}
diff --git a/components/egress/pkg/nftables/manager_test.go b/components/egress/pkg/nftables/manager_test.go
index 611f10209..2f4b13e39 100644
--- a/components/egress/pkg/nftables/manager_test.go
+++ b/components/egress/pkg/nftables/manager_test.go
@@ -823,21 +823,17 @@ func TestApplyStatic_PortFloorRangesDenyBeforeAllows(t *testing.T) {
"the platform floor must be evaluated before any user port allow")
}
-// Issue #2423: the nat OUTPUT REDIRECT (transparent MITM) rewrites intercepted
-// dports to the listener port BEFORE this chain runs (oif still on the egress
-// interface), so the rewritten dport must be accepted ahead of the port floor
-// complement, user port-scoped rules, and the final default-deny drop — or
-// every intercepted flow is silently blackholed even when the original port
-// (e.g. 443) is allowed by both the cap and the user policy.
+// Issue #2423: the MITM REDIRECT rewrites the intercepted dport to the
+// listener port before this chain runs, so the exemption must precede the
+// floor drop, user port rules, and the final default-deny drop.
func TestApplyStatic_MitmRedirectExemptedBeforePortRules(t *testing.T) {
var rendered string
m := NewManagerWithRunnerAndOptions(func(_ context.Context, script string) ([]byte, error) {
rendered = script
return nil, nil
}, Options{MitmRedirectPort: 18081})
- // Default-deny user policy that only opens 443, with the platform port
- // floor (whitelisting 80/443) as an always-deny overlay: neither covers
- // the rewritten dport 18081.
+ // Default-deny policy allowing only 443, plus the port floor: neither
+ // covers the rewritten dport 18081.
p, err := policy.ParsePolicy(`{"defaultAction":"deny","egress":[{"action":"allow","ports":[443]}]}`)
require.NoError(t, err)
p = policy.MergeAlwaysOverlay(p, []policy.EgressRule{
diff --git a/components/egress/revision_nft_options_test.go b/components/egress/revision_nft_options_test.go
index 0d3cf2037..c93d2ffd4 100644
--- a/components/egress/revision_nft_options_test.go
+++ b/components/egress/revision_nft_options_test.go
@@ -57,9 +57,7 @@ func TestNftOptionsRevisionQuiescence(t *testing.T) {
}
}
-// The egress chain exempts the MITM redirect target ahead of port-scoped
-// rules (issue #2423); the port must be derived from the same env the
-// transparent launcher uses, and only when transparent MITM is enabled.
+// The exemption port must follow the transparent-MITM env, and only when enabled.
func TestNftOptionsMitmRedirectPortFollowsTransparentMode(t *testing.T) {
opts, err := parseNftOptions(nil)
require.NoError(t, err)
diff --git a/scripts/egress-mitm-e2e-target.sh b/scripts/egress-mitm-e2e-target.sh
index 5f50bc075..1e4ccf929 100644
--- a/scripts/egress-mitm-e2e-target.sh
+++ b/scripts/egress-mitm-e2e-target.sh
@@ -14,20 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Plain-HTTP listener on a non-80/443 port used by the egress-MITM E2E tests
-# (tests/python/tests/test_egress_mitm_e2e.py) as a connectivity probe target:
-# it must LISTEN on its port so "blocked by policy/floor" (connect times out)
-# stays distinguishable from "nothing there" (instant refusal).
+# HTTP probe target for the egress-MITM E2E tests: it must LISTEN, so
+# "blocked by policy/floor" stays distinguishable from "nothing there".
: "${OPENSANDBOX_EGRESS_MITM_E2E_TARGET_CONTAINER:=opensandbox-e2e-egress-mitm-target}"
: "${OPENSANDBOX_EGRESS_MITM_E2E_TARGET_IMAGE:=python:3.11-alpine}"
: "${OPENSANDBOX_EGRESS_MITM_E2E_LABEL_KEY:=opensandbox.e2e}"
: "${OPENSANDBOX_EGRESS_MITM_E2E_LABEL_VALUE:=egress-mitm}"
-# The listener port is deliberately fixed at 8080: it must match the tests'
-# PROBE_PORT (tests/python/tests/test_egress_mitm_e2e.py) — a non-80/443 port
-# that is neither MITM-intercepted nor whitelisted by the reachable.ports.cap
-# floor used there. Not env-overridable so the two can never drift.
+# Fixed at 8080 to match the tests' PROBE_PORT; not overridable, so they
+# cannot drift.
setup_egress_mitm_e2e_target() {
local label="${OPENSANDBOX_EGRESS_MITM_E2E_LABEL_KEY}=${OPENSANDBOX_EGRESS_MITM_E2E_LABEL_VALUE}"
diff --git a/tests/python/tests/test_egress_mitm_e2e.py b/tests/python/tests/test_egress_mitm_e2e.py
index 81a0363b9..90ca31acb 100644
--- a/tests/python/tests/test_egress_mitm_e2e.py
+++ b/tests/python/tests/test_egress_mitm_e2e.py
@@ -13,28 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-"""Egress x transparent-MITM interaction E2E (docker runtime, dns+nft).
-
-Regressions for the class of bugs where the transparent MITM REDIRECT (the
-destination is rewritten to the mitmproxy listener port before the nft egress
-chain runs) is judged as if the workload had dialed the listener itself, so
-every intercepted HTTPS flow is silently blackholed even when the policy
-allows the original destination port:
-
-- Scenario A: operator port floor (``reachable.ports.cap`` with only 80/443)
- plus Credential Vault (transparent MITM) with a default-allow policy —
- HTTPS must still work, and non-whitelisted ports must stay blocked
- (issue #2423).
-- Scenario B: no floor, plus an allow-443/deny-all user policy with
- Credential Vault enabled — HTTPS must still work through the default-deny
- tail instead of dying on the rewritten port.
-
-The floor file is injected into the sandbox's egress sidecar via
-``docker exec`` (docker runtime only) and takes effect through the sidecar's
-periodic always-rules reload (~1 minute), which the test polls for. The tests
-are env-gated on ``OPENSANDBOX_EGRESS_MITM_E2E_TARGET_IP``, set by
-``scripts/egress-mitm-e2e-target.sh`` (wired into ``scripts/python-e2e.sh``);
-without it they skip.
+"""Egress x transparent-MITM E2E (docker runtime, dns+nft).
+
+The MITM REDIRECT rewrites intercepted dports to the mitmproxy listener
+before the nft chain runs; without an exemption the rewritten port hits the
+port floor or the default-deny tail and blackholes allowed HTTPS.
+
+- Scenario A: reachable.ports.cap = 80/443 + vault + default-allow — HTTPS
+ works and non-whitelisted ports stay blocked (issue #2423).
+- Scenario B: no floor + allow-443/deny-all + vault — HTTPS works.
+
+The floor file is docker-exec'd into the sidecar and picked up by the
+~1-minute always-rules reload the tests poll for. Skips without
+OPENSANDBOX_EGRESS_MITM_E2E_TARGET_IP (set by scripts/egress-mitm-e2e-target.sh).
"""
from __future__ import annotations
@@ -67,11 +58,9 @@ E2E_LABEL_KEY = os.getenv(
E2E_LABEL_VALUE = os.getenv(
"OPENSANDBOX_EGRESS_MITM_E2E_LABEL_VALUE", "egress-mitm"
)
-# The probe target listens here; the port is deliberately NOT 80/443 so it is
-# neither MITM-intercepted nor whitelisted by the port floor.
+# Not 80/443: neither MITM-intercepted nor whitelisted by the floor.
PROBE_PORT = 8080
-# The sidecar reloads operator rule files at most once per minute; poll a bit
-# beyond that before concluding the floor never took effect.
+# Always-rules reload runs at most once per minute.
FLOOR_RELOAD_TIMEOUT_S = 150
@@ -88,13 +77,10 @@ def mitm_target_ip() -> str:
def test_port_floor_with_vault_keeps_https_working(
mitm_target_ip: str,
) -> None:
- """Issue #2423: reachable.ports.cap (80/443) + vault must not blackhole HTTPS.
+ """Issue #2423: the cap whitelist (80/443) must not blackhole MITM'd HTTPS.
- The floor complement is derived from whitelisted ports, but the MITM
- REDIRECT rewrites an intercepted SYN's destination port to the mitmproxy
- listener (e.g. 18081) before the nft chain evaluates it — which lands in
- the complement. Pre-fix, every intercepted HTTPS flow timed out even
- though 443 is whitelisted and the policy is default-allow.
+ The REDIRECT-rewritten dport lands in the floor complement, so pre-fix
+ every intercepted flow timed out under a default-allow policy.
"""
cfg, sandbox = _create_sandbox(
network_policy=NetworkPolicy(defaultAction="allow"),
@@ -102,21 +88,17 @@ def test_port_floor_with_vault_keeps_https_working(
try:
_write_reachable_ports_cap(sandbox.id, ["80", "443"])
- # Phase 1 — precondition: the probe target is reachable before the
- # floor lands (it listens, the policy is default-allow).
+ # Precondition: probe target reachable before the floor lands.
assert _wait_until_tcp(
sandbox, mitm_target_ip, PROBE_PORT, expect_connected=True, timeout_s=60
), f"probe target {mitm_target_ip}:{PROBE_PORT} unreachable even before the floor"
- # Phase 2 — the injected cap file takes effect via the sidecar's
- # periodic always-rules reload (~1 minute): the non-whitelisted port
- # must flip to blocked.
+ # Floor takes effect via the ~1-minute always-rules reload.
assert _wait_until_tcp(
sandbox, mitm_target_ip, PROBE_PORT, expect_connected=False
), (
- "reachable.ports.cap floor did not take effect: a non-whitelisted "
- f"port ({PROBE_PORT}) stayed reachable under a default-allow policy; "
- "the sidecar either never reloaded the injected cap file or the floor is not enforced"
+ "floor never took effect: "
+ f"non-whitelisted port ({PROBE_PORT}) stayed reachable (cap file not reloaded?)"
)
_assert_https_via_mitm(sandbox, "https://example.com", "Example Domain")
@@ -128,13 +110,10 @@ def test_port_floor_with_vault_keeps_https_working(
def test_allow_443_deny_all_with_vault_keeps_https_working(
mitm_target_ip: str,
) -> None:
- """allow-443/deny-all + transparent MITM must let HTTPS out.
+ """allow-443/deny-all + MITM must let HTTPS out; other ports stay blocked.
- Pre-fix, the MITM REDIRECT rewrote the SYN's dport 443 to the listener
- port, which matches no allow rule, so the final default-deny drop
- blackholed every intercepted flow even though the user explicitly allowed
- 443. Enforcement must also stay intact: a port the policy does not allow
- stays blocked.
+ Pre-fix the rewritten dport matched no allow rule and hit the final
+ default-deny drop.
"""
cfg, sandbox = _create_sandbox(
network_policy=NetworkPolicy(
@@ -143,15 +122,12 @@ def test_allow_443_deny_all_with_vault_keeps_https_working(
),
)
try:
- # Gate: the MITM path is live at all. Pre-fix this connect timed out
- # because the rewritten dport hit the default-deny drop before
- # mitmproxy ever saw the flow.
+ # Gate: MITM path live (pre-fix this timed out on the rewritten dport).
assert _wait_until_tcp(
sandbox, "1.1.1.1", 443, expect_connected=True, timeout_s=60
), "TCP/443 was blackholed by the MITM redirect under allow-443/deny-all"
- # The probe target listens on PROBE_PORT, so the block below proves
- # the deny-all default really bites (not just "nothing there").
+ # Target listens on PROBE_PORT, so the block proves deny-all bites.
assert _wait_until_tcp(
sandbox, mitm_target_ip, PROBE_PORT, expect_connected=False, timeout_s=45
), "deny-all policy did not block a non-allowed port"
@@ -198,11 +174,7 @@ def _close_sandbox(cfg: object, sandbox: SandboxSync) -> None:
def _write_reachable_ports_cap(sandbox_id: str, ports: list[str]) -> None:
- """Inject the operator port floor into this sandbox's egress sidecar.
-
- The sidecar re-reads /var/egress rules periodically (~1 minute) and
- re-applies the nft static policy, so the floor takes effect without a
- restart.
+ """Write the floor file into the sidecar; picked up by the ~1-minute reload.
"""
body = "\\n".join(ports) + "\\n"
_sidecar_exec(
@@ -243,11 +215,7 @@ def _wait_until_tcp(
expect_connected: bool,
timeout_s: int = FLOOR_RELOAD_TIMEOUT_S,
) -> bool:
- """Poll the TCP probe until it matches the expectation, or give up.
-
- Covers sidecar startup ordering (nft apply, MITM readiness) and the
- always-rules reload latency without fixed sleeps.
- """
+ """Poll until the probe matches the expectation or the budget expires."""
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if _tcp_connects(sandbox, host, port) == expect_connected:
@@ -257,10 +225,10 @@ def _wait_until_tcp(
def _assert_https_via_mitm(sandbox: SandboxSync, url_and_args: str, expect: str) -> None:
- """Assert a real HTTPS request succeeds through the MITM path, with retries.
+ """Retry a curl until the expected marker shows up; TLS verification stays on.
- TLS verification stays ON: the sandbox trusts the mitm CA installed by the
- execd bootstrap, and mitmproxy verifies the upstream against public roots.
+ The sandbox trusts the mitm CA installed by the execd bootstrap, and
+ mitmproxy verifies the upstream against public roots.
"""
last = ""
for _ in range(3):