Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(egress): reconcile gateway DNS from subject ownership
leo peinc64ff9edc711
diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.goindex bb5a0c552..8563fd7bb 100644--- a/components/egress/fastsandbox_server.go+++ b/components/egress/fastsandbox_server.go@@ -111,16 +111,13 @@ type fastSandboxPolicyServer struct {// each other or with a concurrent SET_BINDING commit).policyCommitMu sync.Mutex- // gatewayDNSRefs maps each subject to its gateway so the shared- // prerouting REDIRECT (sandbox DNS -> loopback proxy) is installed once- // per gateway and removed when the last subject using it is gone. Keyed- // per SUBJECT (not per gateway) so at-least-once SET_BINDING delivery is- // idempotent: a duplicate registration is a map no-op instead of a- // double refcount. Injected fns keep the hooks testable without iptables.- gwMu sync.Mutex- gatewayDNSRefs map[subject.Subject]netip.Addr- dnsRedirectInstall func(gateway netip.Addr, port int) error- dnsRedirectRemove func() error+ // gatewayDNSRefs maps each subject to its gateway; the shared prerouting+ // REDIRECT table is rebuilt from its complete gateway set. The staged map+ // is published only after the nft replacement succeeds, so failures retain+ // ownership for a later lifecycle retry. gwMu serializes map/table updates.+ gwMu sync.Mutex+ gatewayDNSRefs map[subject.Subject]netip.Addr+ dnsRedirectApply func(gateways []netip.Addr, port int) error// mitmMu guards the per-subject interception entries; the Pod-netns table// is rebuilt wholesale from this map on every change (see@@ -142,21 +139,20 @@ type fastSandboxPolicyServer struct {func newFastSandboxPolicyServer(ctx context.Context, reg *subject.MemoryRegistry, nft fastSandboxNftApplier) *fastSandboxPolicyServer {return &fastSandboxPolicyServer{- ctx: ctx,- reg: reg,- nft: nft,- actionGate: make(chan struct{}, 1),- rebinds: make(map[subject.Subject]*subjectRebind),- vaults: make(map[subject.Subject]vaultEntry),- pendingPolicies: make(map[subject.Subject]*policy.NetworkPolicy),- policies: make(map[subject.Subject]*policyEntry),- subjAtt: make(map[subject.Subject]actionhandler.NetworkAttachment),- subjFence: make(map[subject.Subject]subject.Fencing),- gatewayDNSRefs: make(map[subject.Subject]netip.Addr),- dnsRedirectInstall: iptables.SetupGatewayDNSRedirect,- dnsRedirectRemove: iptables.RemoveGatewayDNSRedirect,- mitmEntries: make(map[subject.Subject]iptables.MitmRedirectEntry),- instanceID: newHandlerInstanceID(),+ ctx: ctx,+ reg: reg,+ nft: nft,+ actionGate: make(chan struct{}, 1),+ rebinds: make(map[subject.Subject]*subjectRebind),+ vaults: make(map[subject.Subject]vaultEntry),+ pendingPolicies: make(map[subject.Subject]*policy.NetworkPolicy),+ policies: make(map[subject.Subject]*policyEntry),+ subjAtt: make(map[subject.Subject]actionhandler.NetworkAttachment),+ subjFence: make(map[subject.Subject]subject.Fencing),+ gatewayDNSRefs: make(map[subject.Subject]netip.Addr),+ dnsRedirectApply: iptables.ReplaceGatewayDNSRedirects,+ mitmEntries: make(map[subject.Subject]iptables.MitmRedirectEntry),+ instanceID: newHandlerInstanceID(),}}@@ -199,70 +195,63 @@ const fastSandboxDNSProxyPort = 15353// that runs under policyCommitMu (see applyPolicy).const policyApplyTimeout = 10 * time.Second-// installGatewayDNSRedirect records a subject's gateway and installs (once)-// the prerouting REDIRECT for it. Idempotent under at-least-once SET_BINDING-// delivery: a duplicate registration (or a retried deny-first install) is a-// no-op. A rebind that moved the subject to a different gateway releases the-// old gateway first. Fails closed: a subject whose DNS cannot reach the-// proxy must not register as usable.+// installGatewayDNSRedirect stages a subject's gateway and atomically+// rebuilds the shared prerouting table from the complete desired set. Always+// reconcile, including duplicate delivery: a prior command may have applied+// but returned an ambiguous error. Fails closed: a subject whose DNS cannot+// reach the proxy must not register as usable.func (s *fastSandboxPolicyServer) installGatewayDNSRedirect(subj subject.Subject, gateway netip.Addr) error {s.gwMu.Lock()defer s.gwMu.Unlock()- if old, ok := s.gatewayDNSRefs[subj]; ok {- if old == gateway {- return nil // duplicate delivery: already counted for this gateway- }- // The subject moved gateways (rebind): release the old one first.- delete(s.gatewayDNSRefs, subj)- if s.countGatewayUsersLocked(old) == 0 && s.dnsRedirectRemove != nil {- _ = s.dnsRedirectRemove()- }- }- s.gatewayDNSRefs[subj] = gateway- if s.dnsRedirectInstall == nil {- return nil- }- if s.countGatewayUsersLocked(gateway) > 1 {- return nil // already installed for this gateway- }- if err := s.dnsRedirectInstall(gateway, fastSandboxDNSProxyPort); err != nil {- delete(s.gatewayDNSRefs, subj)- return err+ next := cloneGatewayDNSRefs(s.gatewayDNSRefs)+ next[subj] = gateway.Unmap()+ if err := s.reconcileGatewayDNSRedirectsLocked(next); err != nil {+ return fmt.Errorf("gateway DNS redirect install: %w", err)}+ s.gatewayDNSRefs = nextreturn nil}-// countGatewayUsersLocked counts the subjects currently mapped to a gateway.-// Callers hold gwMu.-func (s *fastSandboxPolicyServer) countGatewayUsersLocked(gateway netip.Addr) int {- n := 0- for _, g := range s.gatewayDNSRefs {- if g == gateway {- n++- }+// releaseGatewayDNSRedirect stages ownership removal and reconciles the full+// set even when the subject is absent. A previous apply may have succeeded+// while its result was lost, leaving no committed map entry to drive cleanup.+// On failure, retain the old map for a retry.+func (s *fastSandboxPolicyServer) releaseGatewayDNSRedirect(subj subject.Subject) error {+ s.gwMu.Lock()+ defer s.gwMu.Unlock()+ next := cloneGatewayDNSRefs(s.gatewayDNSRefs)+ delete(next, subj)+ if err := s.reconcileGatewayDNSRedirectsLocked(next); err != nil {+ return fmt.Errorf("gateway DNS redirect release: %w", err)}- return n+ s.gatewayDNSRefs = next+ return nil}-// releaseGatewayDNSRedirect drops the subject's gateway mapping and removes-// the shared REDIRECT table when the last subject using that gateway is gone.-// Idempotent: a duplicate unload is a no-op.-func (s *fastSandboxPolicyServer) releaseGatewayDNSRedirect(subj subject.Subject) {- s.gwMu.Lock()- defer s.gwMu.Unlock()- gateway, ok := s.gatewayDNSRefs[subj]- if !ok {- return+// reconcileGatewayDNSRedirectsLocked derives a normalized gateway set from+// the staged subject map. Callers hold gwMu; the applier atomically replaces+// the table and treats an empty set as removal of the table.+func (s *fastSandboxPolicyServer) reconcileGatewayDNSRedirectsLocked(refs map[subject.Subject]netip.Addr) error {+ if s.dnsRedirectApply == nil {+ return nil}- delete(s.gatewayDNSRefs, subj)- if s.countGatewayUsersLocked(gateway) > 0 {- return+ set := make(map[netip.Addr]struct{}, len(refs))+ for _, gateway := range refs {+ set[gateway.Unmap()] = struct{}{}}- if s.dnsRedirectRemove != nil {- if err := s.dnsRedirectRemove(); err != nil {- log.Warnf("gateway DNS redirect remove (ignored): %v", err)- }+ gateways := make([]netip.Addr, 0, len(set))+ for gateway := range set {+ gateways = append(gateways, gateway)}+ return s.dnsRedirectApply(gateways, fastSandboxDNSProxyPort)+}++func cloneGatewayDNSRefs(refs map[subject.Subject]netip.Addr) map[subject.Subject]netip.Addr {+ clone := make(map[subject.Subject]netip.Addr, len(refs))+ for subj, gateway := range refs {+ clone[subj] = gateway+ }+ return clone}// Handler returns the fast-sandbox-profile HTTP mux: the Sandbox Actions@@ -507,7 +496,9 @@ func (s *fastSandboxPolicyServer) OnRegistered(subj subject.Subject, att actionh// traffic the MITM layer is responsible for). Roll back the gateway// DNS redirect installed above — the caller retries OnRegistered,// and an unreleased mapping would keep the gateway redirect forever.- s.releaseGatewayDNSRedirect(subj)+ if rollbackErr := s.releaseGatewayDNSRedirect(subj); rollbackErr != nil {+ return fmt.Errorf("mitm redirect: %w (DNS rollback failed: %v)", err, rollbackErr)+ }return err}log.Infof("subject %s deny-first enforced (nft + gateway redirect + mitm redirect) total=%dms denyFirst=%dms dnsRedirect=%dms mitm=%dms",@@ -548,12 +539,12 @@ func (s *fastSandboxPolicyServer) setMitmRedirect(subj subject.Subject, att acti// OnUnloaded implements subject.LifecycleHooks: remove enforcement only.// The caller unregisters and then purges local state outside the registry lock.-// The gateway refcount is released when the last subject using it goes away.+// Gateway membership is reconciled from the remaining subject owners; a+// failed DNS update is returned so terminal cleanup can retry.//-// Enforcement removal runs FIRST: a transient nft failure leaves every other-// teardown step undone, and the caller keeps the subject registered so the-// retried terminal cleanup re-runs everything (no double gateway release, no-// stale rules).+// Enforcement removal runs FIRST. A transient nft failure leaves later+// teardown steps undone; a DNS reconcile failure also prevents the caller from+// unregistering the subject and is retried with the same terminal cleanup.func (s *fastSandboxPolicyServer) OnUnloaded(subj subject.Subject, _ actionhandler.NetworkAttachment) error {if err := s.unloadEnforcement(subj); err != nil {return err@@ -565,14 +556,17 @@ func (s *fastSandboxPolicyServer) OnUnloaded(subj subject.Subject, _ actionhandl// unloadEnforcement tears down kernel/redirect state only. It must never// acquire s.mu or read the registry: takeover calls it under the registry// write lock. Registry calls and their callbacks never span s.mu acquisition.-// nft removal goes first; failure retains the existing cleanup retry boundary.+// nft removal goes first; nft or DNS failure retains the existing cleanup+// retry boundary, and DNS ownership is committed only after reconciliation.func (s *fastSandboxPolicyServer) unloadEnforcement(subj subject.Subject) error {nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)defer cancel()if err := s.nft.Remove(nftCtx, subj); err != nil {return err}- s.releaseGatewayDNSRedirect(subj)+ if err := s.releaseGatewayDNSRedirect(subj); err != nil {+ return err+ }s.removeMitmRedirect(subj)return nil}