Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(egress): fence queued vault requests across subject rebinds
leo pein6046302bc884
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.goindex 5e291766c..c9cd23991 100644--- a/components/egress/fastsandbox_actions.go+++ b/components/egress/fastsandbox_actions.go@@ -212,13 +212,14 @@ func (s *fastSandboxPolicyServer) applySetBinding(ctx context.Context, env *acti// Serialize registration and rebind cleanup with policy and vault commits.s.policyCommitMu.Lock()defer s.policyCommitMu.Unlock()+ old, registered := s.reg.Fence(subj)+ oldState, _ := s.reg.Get(subj)s.mu.Lock()// Waiting for an in-progress Vault commit may outlive the request too.if err := s.actionCanceled(ctx); err != nil {s.mu.Unlock()return http.StatusServiceUnavailable, fmt.Errorf("action canceled: %w", err)}- old, registered := s.reg.Fence(subj)oldAtt, knownNetwork := s.subjAtt[subj]knownNetwork = knownNetwork && oldAtt.IP.IsValid()token := s.rebinds[subj]@@ -228,7 +229,7 @@ func (s *fastSandboxPolicyServer) applySetBinding(ctx context.Context, env *actioldAtt, knownNetwork = token.oldNetwork, true}if token == nil && registered && old.Matches(fence) && !knownNetwork {- if state, _ := s.reg.Get(subj); state == subject.StateActive {+ if oldState == subject.StateActive {// Successful registration always records an attempt before becoming// active. Missing metadata here is an internal inconsistency, not a// reason to mutate a live subject or acknowledge an uninstalled update.@@ -480,13 +481,11 @@ func (s *fastSandboxPolicyServer) applyRemoveBinding(env *actionhandler.Envelopereturn http.StatusInternalServerError, fmt.Errorf("enforcement removal: %w", err)}prev := s.reg.Unregister(subj)- // Preserve the existing retry opportunity for a transient file-delete- // failure. Both attempts follow Unregister in one Vault transaction, so a- // valid pre-SET import cannot land between cleanup and its retry.+ // Terminal file cleanup remains best-effort after Unregister. A failed+ // delete leaves the file fenced to the old identity; this acknowledgement+ // does not promise automatic cleanup retry.s.mu.Lock()- if err := s.dropSubjectStateLocked(subj); err != nil {- s.purgeVaultFile(subj)- }+ s.dropSubjectStateLocked(subj)cleared := token != nil && s.rebinds[subj] == tokenif cleared {delete(s.rebinds, subj)diff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.goindex b942826d2..7a3c0f9fd 100644--- a/components/egress/fastsandbox_policy.go+++ b/components/egress/fastsandbox_policy.go@@ -92,8 +92,8 @@ func (s *fastSandboxPolicyServer) handlePolicyRelay(w http.ResponseWriter, r *ht// and the recorded policy. Held across the whole read so the policy body// and its revision come from the same commit (the server's GET→merge→PUT// PATCH uses the revision as its precondition). Lock order:-// policyCommitMu → registry → s.mu (see fastsandbox_server.go's takeover-// path; the reverse order deadlocks the DNS hot path).+// policyCommitMu spans the transaction; registry reads finish before s.mu.+// Registry callbacks never acquire s.mu, and no registry call runs under it.func (s *fastSandboxPolicyServer) selectRelayPolicy(r *http.Request, subj subject.Subject) (*policyRelayDocument, int, string) {if s.subjectRebinding(subj) {return nil, http.StatusServiceUnavailable, "subject rebind in progress"diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.goindex bc880e2ba..f6bbb8533 100644--- a/components/egress/fastsandbox_server.go+++ b/components/egress/fastsandbox_server.go@@ -62,6 +62,9 @@ type fastSandboxPolicyServer struct {// for a mutex holder. Vault transactions use mu, never this gate.actionGate chan struct{}+ // Never acquire mu while holding a registry lock (including callbacks).+ // Registry lookups also stay outside mu so kernel I/O cannot pin mu.+ // Pre-read identities are validated against subjFence under mu.mu sync.Mutexrebinds map[subject.Subject]*subjectRebind// pendingPolicies holds the SET_BINDING input of a still-denying subject@@ -74,7 +77,8 @@ type fastSandboxPolicyServer struct {// failed enforcement, for same-fence comparison and old-IP cleanup.subjAtt map[subject.Subject]actionhandler.NetworkAttachment// subjFence mirrors the registered fence per subject: the relay paths- // pre-read the registry before s.mu (lock order: registry → s.mu), and+ // finish registry pre-reads before taking s.mu; no registry lock spans mu.+ // The mirror is published after successful registration, and// this in-lock comparison rejects requests whose pre-read went stale// across a rebind.subjFence map[subject.Subject]subject.Fencing@@ -368,11 +372,12 @@ func (s *fastSandboxPolicyServer) handleCredentialVaultActive(w http.ResponseWrihttp.Error(w, fmt.Sprintf("invalid clientIp %q", raw), http.StatusBadRequest)return}+ subj, resolved := s.reg.Resolve(subject.SubjectKey{SourceIP: ip})+ fence, registered := s.reg.Fence(subj)respond := func() func(http.ResponseWriter) {s.mu.Lock()defer s.mu.Unlock()- subj, ok := s.reg.Resolve(subject.SubjectKey{SourceIP: ip})- if !ok {+ if !resolved {return func(w http.ResponseWriter) { http.Error(w, "no subject for clientIp", http.StatusNotFound) }}if s.subjectRebindingLocked(subj) {@@ -380,7 +385,9 @@ func (s *fastSandboxPolicyServer) handleCredentialVaultActive(w http.ResponseWrihttp.Error(w, "subject rebind in progress", http.StatusServiceUnavailable)}}- fence, _ := s.reg.Fence(subj)+ if !registered || !s.fenceMatchesLocked(subj, fence) || s.subjAtt[subj].IP != ip {+ return func(w http.ResponseWriter) { http.Error(w, "no subject for clientIp", http.StatusNotFound) }+ }return captureActiveVaultSnapshot(r, s.vaultForLocked(subj, fence))}()respond(w)@@ -435,14 +442,17 @@ type vaultEntry struct {// (same-UID rebind without REMOVE_BINDING, or a TOCTOU store materialized// against an unregistered subject) is dropped, never inherited — a reset// never carries old credentials into a new sandbox (OSEP-0022). The fence is-// read under s.mu, serialized with the rebind purge in applySetBinding.+// pre-read outside s.mu and checked against the published mirror under s.mu.func (s *fastSandboxPolicyServer) vaultFor(subj subject.Subject) *credentialvault.Store {+ fence, registered := s.reg.Fence(subj)s.mu.Lock()defer s.mu.Unlock()if s.subjectRebindingLocked(subj) {return nil}- fence, _ := s.reg.Fence(subj)+ if !registered || !s.fenceMatchesLocked(subj, fence) {+ return nil+ }return s.vaultForLocked(subj, fence)}@@ -549,7 +559,7 @@ func (s *fastSandboxPolicyServer) OnUnloaded(subj subject.Subject, _ actionhandl// unloadEnforcement tears down kernel/redirect state only. It must never// acquire s.mu or read the registry: takeover calls it under the registry-// write lock, while vault transactions acquire s.mu before registry reads.+// write lock. Registry calls and their callbacks never span s.mu acquisition.// nft removal goes first; failure retains the existing cleanup retry boundary.func (s *fastSandboxPolicyServer) unloadEnforcement(subj subject.Subject) error {nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)diff --git a/components/egress/fastsandbox_transition.go b/components/egress/fastsandbox_transition.goindex a0b080c12..223d8bd68 100644--- a/components/egress/fastsandbox_transition.go+++ b/components/egress/fastsandbox_transition.go@@ -109,3 +109,13 @@ func (s *fastSandboxPolicyServer) subjectRebinding(subj subject.Subject) bool {defer s.mu.Unlock()return s.subjectRebindingLocked(subj)}++// subjectRebindingNow is only an early rejection hint. It never queues before+// the request's registry pre-read; the authoritative guard is repeated under mu.+func (s *fastSandboxPolicyServer) subjectRebindingNow(subj subject.Subject) bool {+ if !s.mu.TryLock() {+ return false+ }+ defer s.mu.Unlock()+ return s.subjectRebindingLocked(subj)+}diff --git a/components/egress/fastsandbox_transition_actions_test.go b/components/egress/fastsandbox_transition_actions_test.goindex d4cd5c54f..f45d675dc 100644--- a/components/egress/fastsandbox_transition_actions_test.go+++ b/components/egress/fastsandbox_transition_actions_test.go@@ -92,20 +92,13 @@ func TestTransitionTakeoverRelayLockChain(t *testing.T) {takeover := transitionRequest(srv, context.Background(), http.MethodPost, constants.ActionsDispatchPath, bindBody(t, "successor", testIP, "new-r", "new-a", 1, vaultAllowPolicy()))<-entered // callback owns registry write lock, with kernel removal pausedrelay := transitionRequest(srv, context.Background(), http.MethodGet, constants.VaultRelayPathPrefix+testUID, "")- // The callback cannot yet own s.mu. Once TryLock fails, the real relay has- // acquired it and is waiting for registry.Fence: the reverse lock edge.- deadline := time.Now().Add(time.Second)- for {- if !srv.mu.TryLock() {- break- }+ // A relay waiting behind kernel I/O must not pin the shared state lock.+ transitionWaitForMutex(t, "readRelayVault")+ unlocked := srv.mu.TryLock()+ if unlocked {srv.mu.Unlock()- if time.Now().After(deadline) {- close(release)- t.Fatal("relay did not reach registry read while holding s.mu")- }- time.Sleep(time.Millisecond)}+ assert.True(t, unlocked, "registry wait must not hold s.mu")close(release)require.Equal(t, http.StatusNotFound, transitionResult(t, relay).Code)require.Equal(t, http.StatusOK, transitionResult(t, takeover).Code)diff --git a/components/egress/fastsandbox_transition_vault_test.go b/components/egress/fastsandbox_transition_vault_test.goindex 41f4f6c86..ca9b9e759 100644--- a/components/egress/fastsandbox_transition_vault_test.go+++ b/components/egress/fastsandbox_transition_vault_test.go@@ -15,10 +15,13 @@package mainimport (+ "context""errors""net/http""net/http/httptest"+ "net/netip""path/filepath"+ "runtime""strings""sync""testing"@@ -218,7 +221,7 @@ type transitionDeleteProbe struct {func (f *transitionDeleteProbe) Delete() (policy.FileMutationState, error) { return f.remove() }-func TestTransitionTerminalPurgeRetryStaysAfterUnregisterUnderLock(t *testing.T) {+func TestTransitionTerminalPurgeIsBestEffortAfterUnregister(t *testing.T) {h := newVaultPersistHarness(t)subj := h.snapshotSubject(t)file := h.fileFor(snapshotTestUID)@@ -243,7 +246,7 @@ func TestTransitionTerminalPurgeRetryStaysAfterUnregisterUnderLock(t *testing.T)h.vp.files[subj] = probeh.vp.mu.Unlock()require.Equal(t, http.StatusOK, doAction(t, h.srv, removeBody(t, snapshotTestUID, testFenceR, testFenceA)).Code)- require.Equal(t, 2, calls, "terminal cleanup is a bounded retry, not a loop")+ require.Equal(t, 1, calls, "terminal cleanup makes one best-effort attempt")require.True(t, file.exists)require.Empty(t, h.srv.vaults)vf, _, err := credentialvault.DecodeVaultFile(file.data)@@ -254,3 +257,86 @@ func TestTransitionTerminalPurgeRetryStaysAfterUnregisterUnderLock(t *testing.T)require.Equal(t, http.StatusOK, doAction(t, h.srv, removeBody(t, snapshotTestUID, testFenceR, testFenceA)).Code)require.False(t, file.exists)}++// Wait for the actual HTTP handler to queue on a held mutex, not an arbitrary+// sleep. The enclosing test controls publication while that request is parked.+func transitionWaitForMutex(t *testing.T, handler string) {+ t.Helper()+ deadline := time.Now().Add(2 * time.Second)+ for time.Now().Before(deadline) {+ buf := make([]byte, 1<<20)+ n := runtime.Stack(buf, true)+ for _, stack := range strings.Split(string(buf[:n]), "\n\n") {+ if strings.Contains(stack, handler) && (strings.Contains(stack, "sync.Mutex.Lock") || strings.Contains(stack, "sync.RWMutex.RLock")) {+ return+ }+ }+ runtime.Gosched()+ }+ t.Fatal("handler did not queue on the controlled mutex")+}++func TestTransitionHeaderlessRelayCannotAdoptSuccessor(t *testing.T) {+ for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete} {+ t.Run(method, func(t *testing.T) {+ h := newVaultPersistHarness(t)+ subj := h.snapshotSubject(t)+ before := append([]byte(nil), h.fileFor(snapshotTestUID).data...)+ body := ""+ if method == http.MethodPost {+ body = vaultCredential()+ }+ if method == http.MethodPut {+ body = vaultPut("1")+ }+ if method == http.MethodGet {+ h.srv.mu.Lock()+ } else {+ h.srv.policyCommitMu.Lock()+ }+ unlock := func() {+ if method == http.MethodGet {+ h.srv.mu.Unlock()+ } else {+ h.srv.policyCommitMu.Unlock()+ }+ }+ locked := true+ defer func() {+ if locked {+ unlock()+ }+ }()+ result := transitionRequest(h.srv, context.Background(), method, constants.VaultRelayPathPrefix+snapshotTestUID, body)+ handler := "readRelayVault"+ if method != http.MethodGet {+ handler = "handleVaultRelayWrite.func1"+ }+ transitionWaitForMutex(t, handler)+ // Model the completed publication while the request is queued: registry,+ // mirror and vault all belong to the successor, and no rebind token remains.+ // Registry changes here have no callbacks; this is a controlled test fixture.+ next := subject.Fencing{RuntimeInstanceID: "successor", AttachmentID: "successor-att"}+ h.reg.Unregister(subj)+ h.reg.Register(subj, subject.SubjectKey{SourceIP: netip.MustParseAddr(testIP)}, next)+ if method != http.MethodGet {+ h.srv.mu.Lock()+ }+ h.srv.subjFence[subj] = next+ entry := h.srv.vaults[subj]+ entry.fence = next+ h.srv.vaults[subj] = entry+ delete(h.srv.rebinds, subj)+ if method != http.MethodGet {+ h.srv.mu.Unlock()+ }+ unlock()+ locked = false+ rec := transitionResult(t, result)+ require.Equal(t, http.StatusNotFound, rec.Code)+ require.NotContains(t, rec.Body.String(), "credentials")+ require.Equal(t, before, h.fileFor(snapshotTestUID).data)+ require.True(t, h.fileFor(snapshotTestUID).exists)+ })+ }+}diff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.goindex 3bf716f92..370d47349 100644--- a/components/egress/fastsandbox_vault.go+++ b/components/egress/fastsandbox_vault.go@@ -122,6 +122,13 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r// The body has been consumed; a late response must not try to read it.r.Body.Close()+ if s.subjectRebindingNow(subj) {+ http.Error(w, "subject rebind in progress", http.StatusServiceUnavailable)+ return+ }+ // Capture the request identity before either serialization lock. A queued+ // headerless write must not adopt a successor published while it waits.+ fence, registered := s.reg.Fence(subj)// Capture a response while committing; no network output holds s.mu.respond := func() func() {if s.subjectRebinding(subj) {@@ -135,8 +142,7 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, rif s.subjectRebindingLocked(subj) {return func() { http.Error(w, "subject rebind in progress", http.StatusServiceUnavailable) }}- fence, ok := s.reg.Fence(subj)- if !ok {+ if !registered || !s.fenceMatchesLocked(subj, fence) {return func() { http.Error(w, "unknown subject", http.StatusNotFound) }}if inc := strings.TrimSpace(r.Header.Get(constants.EgressVaultIncarnationHeader)); inc != "" && inc != fence.RuntimeInstanceID {@@ -268,13 +274,16 @@ func writeVaultRelayWriteResponse(w http.ResponseWriter, r *http.Request, state// readRelayVault captures an independent inline result under the subject lock;// sending it afterwards cannot delay lifecycle state changes.func (s *fastSandboxPolicyServer) readRelayVault(r *http.Request, subj subject.Subject) (credentialvault.FullRevision, int, string, error) {+ if s.subjectRebindingNow(subj) {+ return credentialvault.FullRevision{}, http.StatusServiceUnavailable, "subject rebind in progress", nil+ }+ fence, registered := s.reg.Fence(subj)s.mu.Lock()defer s.mu.Unlock()if s.subjectRebindingLocked(subj) {return credentialvault.FullRevision{}, http.StatusServiceUnavailable, "subject rebind in progress", nil}- fence, ok := s.reg.Fence(subj)- if !ok {+ if !registered || !s.fenceMatchesLocked(subj, fence) {return credentialvault.FullRevision{}, http.StatusNotFound, "unknown subject", nil}if inc := strings.TrimSpace(r.Header.Get(constants.EgressVaultIncarnationHeader)); inc != "" && inc != fence.RuntimeInstanceID {diff --git a/components/egress/fastsandbox_vault_persist_test.go b/components/egress/fastsandbox_vault_persist_test.goindex 6a6de809d..68dd3e257 100644--- a/components/egress/fastsandbox_vault_persist_test.go+++ b/components/egress/fastsandbox_vault_persist_test.go@@ -660,14 +660,17 @@ func TestVaultPersistRemoveBindingPurgesFile(t *testing.T) {require.Equal(t, http.StatusCreated, rec.Code)require.True(t, h.fileFor(vaultTestUID).exists)- // Cleanup runs after Unregister and retains its bounded second attempt- // on failure, without unlocking between attempts.+ // A failed best-effort cleanup is still acknowledged; an explicit later+ // REMOVE can retry it, but no automatic retry is promised.h.fileFor(vaultTestUID).failFirstDelete = errors.New("unlink EIO")rec = doAction(t, h.srv, removeBody(t, vaultTestUID, testFenceR, testFenceA))require.Equal(t, http.StatusOK, rec.Code)- assert.False(t, h.fileFor(vaultTestUID).exists, "REMOVE_BINDING purges memory + file (terminal)")- assert.Equal(t, 2, h.fileFor(vaultTestUID).deleteCalls, "the post-unregister cleanup retries its failed first purge")+ assert.True(t, h.fileFor(vaultTestUID).exists, "failed cleanup leaves the old fenced file")+ assert.Equal(t, 1, h.fileFor(vaultTestUID).deleteCalls)+ rec = doAction(t, h.srv, removeBody(t, vaultTestUID, testFenceR, testFenceA))+ require.Equal(t, http.StatusOK, rec.Code)+ assert.False(t, h.fileFor(vaultTestUID).exists)// REMOVE_BINDING for unknown Handler state also drops any cached file.h.seedVaultFile(vaultTestUID, "runtime-x", "attachment-x", 1, reloadVaultBody)