Secure, Fast, and Extensible Sandbox runtime for AI agents.
test(egress): cover gateway DNS lifecycle retries
leo pein2d6e5096cd5a
diff --git a/components/egress/fastsandbox_server_test.go b/components/egress/fastsandbox_server_test.goindex dd9637fa9..d86377eff 100644--- a/components/egress/fastsandbox_server_test.go+++ b/components/egress/fastsandbox_server_test.go@@ -50,6 +50,53 @@ type fakeNft struct {removeErr error}+// fakeGatewayDNSRedirectApplier models the entire owned nft table, not only+// callback counts. A failure can occur either before the replacement or+// after it took effect (a lost/ambiguous result).+type fakeGatewayDNSRedirectApplier struct {+ gateways map[netip.Addr]struct{}+ calls int+ failAt int+ failErr error+ applyThenFail bool+}++func newFakeGatewayDNSRedirectApplier() *fakeGatewayDNSRedirectApplier {+ return &fakeGatewayDNSRedirectApplier{gateways: make(map[netip.Addr]struct{})}+}++func (f *fakeGatewayDNSRedirectApplier) apply(gateways []netip.Addr, _ int) error {+ f.calls+++ fail := f.calls == f.failAt+ if fail && !f.applyThenFail {+ return f.failErr+ }+ f.gateways = make(map[netip.Addr]struct{}, len(gateways))+ for _, gateway := range gateways {+ f.gateways[gateway.Unmap()] = struct{}{}+ }+ if fail {+ return f.failErr+ }+ return nil+}++func (f *fakeGatewayDNSRedirectApplier) failOnCall(call int, err error, afterApply bool) {+ f.failAt, f.failErr, f.applyThenFail = call, err, afterApply+}++func (f *fakeGatewayDNSRedirectApplier) clearFailures() {+ f.failAt, f.failErr, f.applyThenFail = 0, nil, false+}++func (f *fakeGatewayDNSRedirectApplier) snapshot() map[netip.Addr]struct{} {+ return f.gateways+}++func (f *fakeGatewayDNSRedirectApplier) callCount() int {+ return f.calls+}+func (f *fakeNft) ApplyDenyFirst(_ context.Context, s subject.Subject, _ actionhandler.NetworkAttachment) error {f.mu.Lock()defer f.mu.Unlock()@@ -93,8 +140,7 @@ func fastSandboxTestServer(t *testing.T) (*fastSandboxPolicyServer, *subject.Memnft := &fakeNft{}srv := newFastSandboxPolicyServer(context.Background(), reg, nft)// no-op the gateway DNS redirect (no iptables/nft in unit tests)- srv.dnsRedirectInstall = func(netip.Addr, int) error { return nil }- srv.dnsRedirectRemove = func() error { return nil }+ srv.dnsRedirectApply = func([]netip.Addr, int) error { return nil }// Policy persistence is REQUIRED in the fast-sandbox profile — every// test server carries a store, mirroring the production posture (no// memory-only mode). The file seam defaults to a cross-platform fake@@ -696,9 +742,8 @@ func TestActionsSetBindingTakeoverGatewayRefcountConsistent(t *testing.T) {nft := &fakeNft{}srv := newFastSandboxPolicyServer(context.Background(), reg, nft)wireTestPolicyPersistence(t, srv)- var installs, removes int- srv.dnsRedirectInstall = func(netip.Addr, int) error { installs++; return nil }- srv.dnsRedirectRemove = func() error { removes++; return nil }+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.applyinst := &fakeMitmInstaller{}srv.mitmInstall = inst.installsrv.mitmRemove = inst.remove@@ -707,13 +752,13 @@ func TestActionsSetBindingTakeoverGatewayRefcountConsistent(t *testing.T) {input := `{"defaultAction":"deny"}`rec := doAction(t, srv, bindBody(t, staleUID, testIP, testFenceR, testFenceA, 1, &input))require.Equal(t, http.StatusOK, rec.Code)- require.Equal(t, 1, installs)+ require.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())rec = doAction(t, srv, bindBody(t, takeUID, testIP, "runtime-2", "attachment-2", 1, &input))require.Equal(t, http.StatusOK, rec.Code)- assert.Equal(t, 2, installs, "the successor reinstalls the shared gateway redirect")- assert.Equal(t, 1, removes, "the stale subject releases its refcount exactly once")+ assert.Equal(t, 3, dns.callCount(), "stale release and successor install each reconcile the full set")+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())srv.gwMu.Lock()_, hasStale := srv.gatewayDNSRefs[subject.FromSandboxUID(staleUID)]succ, hasSucc := srv.gatewayDNSRefs[subject.FromSandboxUID(takeUID)]@@ -1092,8 +1137,7 @@ func TestFastSandboxServerAlwaysRulesReachNft(t *testing.T) {nft := &fakeNft{}srv := newFastSandboxPolicyServer(context.Background(), reg, nft)wireTestPolicyPersistence(t, srv)- srv.dnsRedirectInstall = func(netip.Addr, int) error { return nil }- srv.dnsRedirectRemove = func() error { return nil }+ srv.dnsRedirectApply = func([]netip.Addr, int) error { return nil }// activation through the action protocol; the binding policy reaches nft// merged with the always overlay@@ -1152,17 +1196,15 @@ func TestFastSandboxServerNftFailureKeepsRegistryState(t *testing.T) {assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("example.com"))}-// TestFastSandboxServerGatewayRedirectRefcounted: the shared prerouting REDIRECT-// installs once per gateway and is removed when the last subject using it is-// unloaded.+// TestFastSandboxServerGatewayRedirectRefcounted: the shared prerouting table+// keeps every active gateway and removes only gateways without owners.func TestFastSandboxServerGatewayRedirectRefcounted(t *testing.T) {reg := subject.NewRegistry(nil, nil)nft := &fakeNft{}srv := newFastSandboxPolicyServer(context.Background(), reg, nft)wireTestPolicyPersistence(t, srv)- var installs, removes int- srv.dnsRedirectInstall = func(netip.Addr, int) error { installs++; return nil }- srv.dnsRedirectRemove = func() error { removes++; return nil }+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.applyattA := mitmAtt(t, "10.0.0.5", "10.10.0.1")attB := mitmAtt(t, "10.0.0.6", "10.10.0.1")@@ -1171,14 +1213,147 @@ func TestFastSandboxServerGatewayRedirectRefcounted(t *testing.T) {require.NoError(t, srv.OnRegistered(subject.FromSandboxUID("a"), attA))require.NoError(t, srv.OnRegistered(subject.FromSandboxUID("b"), attB))require.NoError(t, srv.OnRegistered(subject.FromSandboxUID("c"), attC))- assert.Equal(t, 2, installs, "shared gateway installs once; distinct gateway installs again")+ wantBoth := map[netip.Addr]struct{}{+ netip.MustParseAddr("10.10.0.1"): {},+ netip.MustParseAddr("10.20.0.1"): {},+ }+ assert.Equal(t, wantBoth, dns.snapshot())require.NoError(t, srv.OnUnloaded(subject.FromSandboxUID("a"), attA))+ assert.Equal(t, wantBoth, dns.snapshot(), "a non-final shared-gateway release preserves the whole table")require.NoError(t, srv.OnUnloaded(subject.FromSandboxUID("b"), attB))- assert.Equal(t, 1, removes, "remove only when the LAST subject of the shared gateway unloads")+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.20.0.1"): {}}, dns.snapshot(), "removing gw1 must preserve gw2")require.NoError(t, srv.OnUnloaded(subject.FromSandboxUID("c"), attC))- assert.Equal(t, 2, removes)+ assert.Empty(t, dns.snapshot(), "the final release removes the DNS table contents")+}++func TestFastSandboxServerReleaseCleansUncommittedInstallAfterLostResult(t *testing.T) {+ srv, _, _ := fastSandboxTestServer(t)+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.apply+ subj, gw := subject.FromSandboxUID("uncommitted"), netip.MustParseAddr("10.10.0.1")+ dns.failOnCall(1, errors.New("result lost"), true)+ require.Error(t, srv.installGatewayDNSRedirect(subj, gw))+ srv.gwMu.Lock()+ assert.Empty(t, srv.gatewayDNSRefs)+ srv.gwMu.Unlock()+ assert.Equal(t, map[netip.Addr]struct{}{gw: {}}, dns.snapshot())++ require.NoError(t, srv.releaseGatewayDNSRedirect(subj))+ assert.Empty(t, dns.snapshot(), "release reconciles even though install ownership was not committed")+}++func TestActionsSetAndRemoveRetryAfterGatewayDNSFailures(t *testing.T) {+ srv, reg, _ := fastSandboxTestServer(t)+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.apply+ input := `{"defaultAction":"deny"}`+ subj := subject.FromSandboxUID("dns-remove")+ dns.failOnCall(1, errors.New("nft busy"), false)+ rec := doAction(t, srv, bindBody(t, "dns-remove", testIP, testFenceR, testFenceA, 1, strPtr(input)))+ require.Equal(t, http.StatusInternalServerError, rec.Code)+ assert.Equal(t, subject.StateDenying, mustState(reg.Get(subj)))+ assert.Empty(t, dns.snapshot())+ srv.gwMu.Lock()+ assert.Empty(t, srv.gatewayDNSRefs, "failed install does not commit ownership")+ srv.gwMu.Unlock()++ dns.clearFailures()+ rec = doAction(t, srv, bindBody(t, "dns-remove", testIP, testFenceR, testFenceA, 1, strPtr(input)))+ require.Equal(t, http.StatusOK, rec.Code)+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())+ dns.failOnCall(3, errors.New("nft result lost"), true)++ rec = doAction(t, srv, removeBody(t, "dns-remove", testFenceR, testFenceA))+ require.Equal(t, http.StatusInternalServerError, rec.Code)+ state, ok := reg.Get(subj)+ require.True(t, ok, "REMOVE_BINDING must not unregister after a failed DNS reconcile")+ assert.Equal(t, subject.StateDenying, state)+ srv.gwMu.Lock()+ assert.Equal(t, netip.MustParseAddr(testGW), srv.gatewayDNSRefs[subj], "failed release retains retry ownership")+ srv.gwMu.Unlock()+ assert.Empty(t, dns.snapshot(), "the apply-then-error fake makes the kernel result ambiguous")++ dns.clearFailures()+ rec = doAction(t, srv, removeBody(t, "dns-remove", testFenceR, testFenceA))+ require.Equal(t, http.StatusOK, rec.Code)+ _, ok = reg.Get(subj)+ assert.False(t, ok)+ assert.Empty(t, dns.snapshot())+ srv.gwMu.Lock()+ assert.Empty(t, srv.gatewayDNSRefs)+ srv.gwMu.Unlock()+}++func TestActionsSetBindingTakeoverRetriesAfterGatewayDNSFailure(t *testing.T) {+ srv, reg, _ := fastSandboxTestServer(t)+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.apply+ staleUID, successorUID := "dns-stale", "dns-successor"+ stale, successor := subject.FromSandboxUID(staleUID), subject.FromSandboxUID(successorUID)+ input := `{"defaultAction":"deny"}`+ setBindingAndReady(t, srv, reg, staleUID, testIP, &input)+ dns.failOnCall(2, errors.New("nft result lost"), true)++ rec := doAction(t, srv, bindBody(t, successorUID, testIP, "runtime-2", "attachment-2", 1, strPtr(input)))+ require.Equal(t, http.StatusInternalServerError, rec.Code)+ require.Equal(t, subject.StateActive, mustState(reg.Get(stale)), "failed DNS release must not evict the old owner")+ _, ok := reg.Get(successor)+ assert.False(t, ok, "takeover must not publish its successor before DNS cleanup succeeds")+ srv.gwMu.Lock()+ assert.Equal(t, netip.MustParseAddr(testGW), srv.gatewayDNSRefs[stale])+ srv.gwMu.Unlock()++ dns.clearFailures()+ rec = doAction(t, srv, bindBody(t, successorUID, testIP, "runtime-2", "attachment-2", 1, strPtr(input)))+ require.Equal(t, http.StatusOK, rec.Code)+ _, ok = reg.Get(stale)+ assert.False(t, ok)+ require.Equal(t, subject.StateDenying, mustState(reg.Get(successor)))+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())+}++func TestActionsMITMFailureRollbackAndTerminalCleanup(t *testing.T) {+ srv, reg, _ := fastSandboxTestServer(t)+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.apply+ srv.mitmInstall = func([]iptables.MitmRedirectEntry) error { return errors.New("mitm failed") }+ input := `{"defaultAction":"deny"}`++ rec := doAction(t, srv, bindBody(t, "mitm-dns", testIP, testFenceR, testFenceA, 1, strPtr(input)))+ require.Equal(t, http.StatusInternalServerError, rec.Code)+ assert.Contains(t, rec.Body.String(), "mitm failed")+ subj := subject.FromSandboxUID("mitm-dns")+ state, ok := reg.Get(subj)+ require.True(t, ok, "failed registration remains denying and retryable")+ assert.Equal(t, subject.StateDenying, state)+ srv.gwMu.Lock()+ assert.Empty(t, srv.gatewayDNSRefs, "successful rollback commits the empty ownership map")+ srv.gwMu.Unlock()+ assert.Empty(t, dns.snapshot(), "successful rollback removes the newly installed redirect")++ // Retry the same deny-first registration; this time rollback applies but+ // returns an error, so ownership remains available to terminal cleanup.+ dns.failOnCall(4, errors.New("DNS rollback failed"), true)+ rec = doAction(t, srv, bindBody(t, "mitm-dns", testIP, testFenceR, testFenceA, 1, strPtr(input)))+ require.Equal(t, http.StatusInternalServerError, rec.Code)+ assert.Contains(t, rec.Body.String(), "mitm failed")+ assert.Contains(t, rec.Body.String(), "DNS rollback failed")+ srv.gwMu.Lock()+ assert.Equal(t, netip.MustParseAddr(testGW), srv.gatewayDNSRefs[subj], "failed rollback retains ownership")+ srv.gwMu.Unlock()+ assert.Empty(t, dns.snapshot(), "the rollback result is ambiguous")++ dns.clearFailures()+ rec = doAction(t, srv, removeBody(t, "mitm-dns", testFenceR, testFenceA))+ require.Equal(t, http.StatusOK, rec.Code)+ _, ok = reg.Get(subj)+ assert.False(t, ok)+ assert.Empty(t, dns.snapshot())+ srv.gwMu.Lock()+ assert.Empty(t, srv.gatewayDNSRefs)+ srv.gwMu.Unlock()}func strPtr(s string) *string { return &s }@@ -1290,18 +1465,25 @@ func TestFastSandboxServerGatewayRedirectDuplicateSetBindingIdempotent(t *testinnft := &fakeNft{}srv := newFastSandboxPolicyServer(context.Background(), reg, nft)wireTestPolicyPersistence(t, srv)- var installs, removes int- srv.dnsRedirectInstall = func(netip.Addr, int) error { installs++; return nil }- srv.dnsRedirectRemove = func() error { removes++; return nil }+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.applyatt := mitmAtt(t, "10.0.0.5", "10.10.0.1")s := subject.FromSandboxUID("a")- require.NoError(t, srv.OnRegistered(s, att))+ dns.failOnCall(1, errors.New("result lost"), true)+ require.Error(t, srv.OnRegistered(s, att))+ srv.gwMu.Lock()+ assert.Empty(t, srv.gatewayDNSRefs, "failed install does not commit the map")+ srv.gwMu.Unlock()+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot())+ dns.clearFailures()+ require.NoError(t, srv.OnRegistered(s, att), "the same desired set must be reconciled after an ambiguous error")require.NoError(t, srv.OnRegistered(s, att), "duplicate SET_BINDING delivery")- assert.Equal(t, 1, installs, "duplicate registration must not re-install the redirect")+ assert.Equal(t, 3, dns.callCount(), "duplicate registration still reconciles the complete desired set")+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot())require.NoError(t, srv.OnUnloaded(s, att))- assert.Equal(t, 1, removes, "one unload must fully release the gateway")+ assert.Empty(t, dns.snapshot(), "one unload must fully release the gateway")}// TestFastSandboxServerGatewayRedirectRebindMovesGateway: a rebind that moved the@@ -1311,17 +1493,24 @@ func TestFastSandboxServerGatewayRedirectRebindMovesGateway(t *testing.T) {nft := &fakeNft{}srv := newFastSandboxPolicyServer(context.Background(), reg, nft)wireTestPolicyPersistence(t, srv)- var installs, removes int- srv.dnsRedirectInstall = func(netip.Addr, int) error { installs++; return nil }- srv.dnsRedirectRemove = func() error { removes++; return nil }+ dns := newFakeGatewayDNSRedirectApplier()+ srv.dnsRedirectApply = dns.applys := subject.FromSandboxUID("a")att1 := mitmAtt(t, "10.0.0.5", "10.10.0.1")att2 := mitmAtt(t, "10.0.0.5", "10.20.0.1")require.NoError(t, srv.OnRegistered(s, att1))+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot())+ dns.failOnCall(2, errors.New("replacement rejected"), false)+ require.Error(t, srv.OnRegistered(s, att2))+ srv.gwMu.Lock()+ assert.Equal(t, netip.MustParseAddr("10.10.0.1"), srv.gatewayDNSRefs[s], "failed replacement does not commit the new gateway")+ srv.gwMu.Unlock()+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot(), "a rejected batch keeps the prior table")+ dns.clearFailures()require.NoError(t, srv.OnRegistered(s, att2), "rebind with a new gateway")- assert.Equal(t, 2, installs)+ assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.20.0.1"): {}}, dns.snapshot(), "one replacement drops the old gateway")require.NoError(t, srv.OnUnloaded(s, att2))- assert.Equal(t, 2, removes, "both gateways must be released")+ assert.Empty(t, dns.snapshot(), "the re-bound gateway must be released")}