Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): fence queued vault requests across subject rebinds

leo pein6046302bc884
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index 5e291766c..c9cd23991 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -212,13 +212,14 @@ func (s *fastSandboxPolicyServer) applySetBinding(ctx context.Context, env *acti
// Serialize registration and rebind cleanup with policy and vault commits.
s.policyCommitMu.Lock()
defer s.policyCommitMu.Unlock()
+ old, registered := s.reg.Fence(subj)
+ oldState, _ := s.reg.Get(subj)
s.mu.Lock()
// Waiting for an in-progress Vault commit may outlive the request too.
if err := s.actionCanceled(ctx); err != nil {
s.mu.Unlock()
return http.StatusServiceUnavailable, fmt.Errorf("action canceled: %w", err)
}
- old, registered := s.reg.Fence(subj)
oldAtt, knownNetwork := s.subjAtt[subj]
knownNetwork = knownNetwork && oldAtt.IP.IsValid()
token := s.rebinds[subj]
@@ -228,7 +229,7 @@ func (s *fastSandboxPolicyServer) applySetBinding(ctx context.Context, env *acti
oldAtt, knownNetwork = token.oldNetwork, true
}
if token == nil && registered && old.Matches(fence) && !knownNetwork {
- if state, _ := s.reg.Get(subj); state == subject.StateActive {
+ if oldState == subject.StateActive {
// Successful registration always records an attempt before becoming
// active. Missing metadata here is an internal inconsistency, not a
// reason to mutate a live subject or acknowledge an uninstalled update.
@@ -480,13 +481,11 @@ func (s *fastSandboxPolicyServer) applyRemoveBinding(env *actionhandler.Envelope
return http.StatusInternalServerError, fmt.Errorf("enforcement removal: %w", err)
}
prev := s.reg.Unregister(subj)
- // Preserve the existing retry opportunity for a transient file-delete
- // failure. Both attempts follow Unregister in one Vault transaction, so a
- // valid pre-SET import cannot land between cleanup and its retry.
+ // Terminal file cleanup remains best-effort after Unregister. A failed
+ // delete leaves the file fenced to the old identity; this acknowledgement
+ // does not promise automatic cleanup retry.
s.mu.Lock()
- if err := s.dropSubjectStateLocked(subj); err != nil {
- s.purgeVaultFile(subj)
- }
+ s.dropSubjectStateLocked(subj)
cleared := token != nil && s.rebinds[subj] == token
if cleared {
delete(s.rebinds, subj)
diff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.go
index b942826d2..7a3c0f9fd 100644
--- a/components/egress/fastsandbox_policy.go
+++ b/components/egress/fastsandbox_policy.go
@@ -92,8 +92,8 @@ func (s *fastSandboxPolicyServer) handlePolicyRelay(w http.ResponseWriter, r *ht
// and the recorded policy. Held across the whole read so the policy body
// and its revision come from the same commit (the server's GET→merge→PUT
// PATCH uses the revision as its precondition). Lock order:
-// policyCommitMu → registry → s.mu (see fastsandbox_server.go's takeover
-// path; the reverse order deadlocks the DNS hot path).
+// policyCommitMu spans the transaction; registry reads finish before s.mu.
+// Registry callbacks never acquire s.mu, and no registry call runs under it.
func (s *fastSandboxPolicyServer) selectRelayPolicy(r *http.Request, subj subject.Subject) (*policyRelayDocument, int, string) {
if s.subjectRebinding(subj) {
return nil, http.StatusServiceUnavailable, "subject rebind in progress"
diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.go
index bc880e2ba..f6bbb8533 100644
--- a/components/egress/fastsandbox_server.go
+++ b/components/egress/fastsandbox_server.go
@@ -62,6 +62,9 @@ type fastSandboxPolicyServer struct {
// for a mutex holder. Vault transactions use mu, never this gate.
actionGate chan struct{}
+ // Never acquire mu while holding a registry lock (including callbacks).
+ // Registry lookups also stay outside mu so kernel I/O cannot pin mu.
+ // Pre-read identities are validated against subjFence under mu.
mu sync.Mutex
rebinds map[subject.Subject]*subjectRebind
// pendingPolicies holds the SET_BINDING input of a still-denying subject
@@ -74,7 +77,8 @@ type fastSandboxPolicyServer struct {
// failed enforcement, for same-fence comparison and old-IP cleanup.
subjAtt map[subject.Subject]actionhandler.NetworkAttachment
// subjFence mirrors the registered fence per subject: the relay paths
- // pre-read the registry before s.mu (lock order: registry → s.mu), and
+ // finish registry pre-reads before taking s.mu; no registry lock spans mu.
+ // The mirror is published after successful registration, and
// this in-lock comparison rejects requests whose pre-read went stale
// across a rebind.
subjFence map[subject.Subject]subject.Fencing
@@ -368,11 +372,12 @@ func (s *fastSandboxPolicyServer) handleCredentialVaultActive(w http.ResponseWri
http.Error(w, fmt.Sprintf("invalid clientIp %q", raw), http.StatusBadRequest)
return
}
+ subj, resolved := s.reg.Resolve(subject.SubjectKey{SourceIP: ip})
+ fence, registered := s.reg.Fence(subj)
respond := func() func(http.ResponseWriter) {
s.mu.Lock()
defer s.mu.Unlock()
- subj, ok := s.reg.Resolve(subject.SubjectKey{SourceIP: ip})
- if !ok {
+ if !resolved {
return func(w http.ResponseWriter) { http.Error(w, "no subject for clientIp", http.StatusNotFound) }
}
if s.subjectRebindingLocked(subj) {
@@ -380,7 +385,9 @@ func (s *fastSandboxPolicyServer) handleCredentialVaultActive(w http.ResponseWri
http.Error(w, "subject rebind in progress", http.StatusServiceUnavailable)
}
}
- fence, _ := s.reg.Fence(subj)
+ if !registered || !s.fenceMatchesLocked(subj, fence) || s.subjAtt[subj].IP != ip {
+ return func(w http.ResponseWriter) { http.Error(w, "no subject for clientIp", http.StatusNotFound) }
+ }
return captureActiveVaultSnapshot(r, s.vaultForLocked(subj, fence))
}()
respond(w)
@@ -435,14 +442,17 @@ type vaultEntry struct {
// (same-UID rebind without REMOVE_BINDING, or a TOCTOU store materialized
// against an unregistered subject) is dropped, never inherited — a reset
// never carries old credentials into a new sandbox (OSEP-0022). The fence is
-// read under s.mu, serialized with the rebind purge in applySetBinding.
+// pre-read outside s.mu and checked against the published mirror under s.mu.
func (s *fastSandboxPolicyServer) vaultFor(subj subject.Subject) *credentialvault.Store {
+ fence, registered := s.reg.Fence(subj)
s.mu.Lock()
defer s.mu.Unlock()
if s.subjectRebindingLocked(subj) {
return nil
}
- fence, _ := s.reg.Fence(subj)
+ if !registered || !s.fenceMatchesLocked(subj, fence) {
+ return nil
+ }
return s.vaultForLocked(subj, fence)
}
@@ -549,7 +559,7 @@ func (s *fastSandboxPolicyServer) OnUnloaded(subj subject.Subject, _ actionhandl
// unloadEnforcement tears down kernel/redirect state only. It must never
// acquire s.mu or read the registry: takeover calls it under the registry
-// write lock, while vault transactions acquire s.mu before registry reads.
+// write lock. Registry calls and their callbacks never span s.mu acquisition.
// nft removal goes first; failure retains the existing cleanup retry boundary.
func (s *fastSandboxPolicyServer) unloadEnforcement(subj subject.Subject) error {
nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
diff --git a/components/egress/fastsandbox_transition.go b/components/egress/fastsandbox_transition.go
index a0b080c12..223d8bd68 100644
--- a/components/egress/fastsandbox_transition.go
+++ b/components/egress/fastsandbox_transition.go
@@ -109,3 +109,13 @@ func (s *fastSandboxPolicyServer) subjectRebinding(subj subject.Subject) bool {
defer s.mu.Unlock()
return s.subjectRebindingLocked(subj)
}
+
+// subjectRebindingNow is only an early rejection hint. It never queues before
+// the request's registry pre-read; the authoritative guard is repeated under mu.
+func (s *fastSandboxPolicyServer) subjectRebindingNow(subj subject.Subject) bool {
+ if !s.mu.TryLock() {
+ return false
+ }
+ defer s.mu.Unlock()
+ return s.subjectRebindingLocked(subj)
+}
diff --git a/components/egress/fastsandbox_transition_actions_test.go b/components/egress/fastsandbox_transition_actions_test.go
index d4cd5c54f..f45d675dc 100644
--- a/components/egress/fastsandbox_transition_actions_test.go
+++ b/components/egress/fastsandbox_transition_actions_test.go
@@ -92,20 +92,13 @@ func TestTransitionTakeoverRelayLockChain(t *testing.T) {
takeover := transitionRequest(srv, context.Background(), http.MethodPost, constants.ActionsDispatchPath, bindBody(t, "successor", testIP, "new-r", "new-a", 1, vaultAllowPolicy()))
<-entered // callback owns registry write lock, with kernel removal paused
relay := transitionRequest(srv, context.Background(), http.MethodGet, constants.VaultRelayPathPrefix+testUID, "")
- // The callback cannot yet own s.mu. Once TryLock fails, the real relay has
- // acquired it and is waiting for registry.Fence: the reverse lock edge.
- deadline := time.Now().Add(time.Second)
- for {
- if !srv.mu.TryLock() {
- break
- }
+ // A relay waiting behind kernel I/O must not pin the shared state lock.
+ transitionWaitForMutex(t, "readRelayVault")
+ unlocked := srv.mu.TryLock()
+ if unlocked {
srv.mu.Unlock()
- if time.Now().After(deadline) {
- close(release)
- t.Fatal("relay did not reach registry read while holding s.mu")
- }
- time.Sleep(time.Millisecond)
}
+ assert.True(t, unlocked, "registry wait must not hold s.mu")
close(release)
require.Equal(t, http.StatusNotFound, transitionResult(t, relay).Code)
require.Equal(t, http.StatusOK, transitionResult(t, takeover).Code)
diff --git a/components/egress/fastsandbox_transition_vault_test.go b/components/egress/fastsandbox_transition_vault_test.go
index 41f4f6c86..ca9b9e759 100644
--- a/components/egress/fastsandbox_transition_vault_test.go
+++ b/components/egress/fastsandbox_transition_vault_test.go
@@ -15,10 +15,13 @@
package main
import (
+ "context"
"errors"
"net/http"
"net/http/httptest"
+ "net/netip"
"path/filepath"
+ "runtime"
"strings"
"sync"
"testing"
@@ -218,7 +221,7 @@ type transitionDeleteProbe struct {
func (f *transitionDeleteProbe) Delete() (policy.FileMutationState, error) { return f.remove() }
-func TestTransitionTerminalPurgeRetryStaysAfterUnregisterUnderLock(t *testing.T) {
+func TestTransitionTerminalPurgeIsBestEffortAfterUnregister(t *testing.T) {
h := newVaultPersistHarness(t)
subj := h.snapshotSubject(t)
file := h.fileFor(snapshotTestUID)
@@ -243,7 +246,7 @@ func TestTransitionTerminalPurgeRetryStaysAfterUnregisterUnderLock(t *testing.T)
h.vp.files[subj] = probe
h.vp.mu.Unlock()
require.Equal(t, http.StatusOK, doAction(t, h.srv, removeBody(t, snapshotTestUID, testFenceR, testFenceA)).Code)
- require.Equal(t, 2, calls, "terminal cleanup is a bounded retry, not a loop")
+ require.Equal(t, 1, calls, "terminal cleanup makes one best-effort attempt")
require.True(t, file.exists)
require.Empty(t, h.srv.vaults)
vf, _, err := credentialvault.DecodeVaultFile(file.data)
@@ -254,3 +257,86 @@ func TestTransitionTerminalPurgeRetryStaysAfterUnregisterUnderLock(t *testing.T)
require.Equal(t, http.StatusOK, doAction(t, h.srv, removeBody(t, snapshotTestUID, testFenceR, testFenceA)).Code)
require.False(t, file.exists)
}
+
+// Wait for the actual HTTP handler to queue on a held mutex, not an arbitrary
+// sleep. The enclosing test controls publication while that request is parked.
+func transitionWaitForMutex(t *testing.T, handler string) {
+ t.Helper()
+ deadline := time.Now().Add(2 * time.Second)
+ for time.Now().Before(deadline) {
+ buf := make([]byte, 1<<20)
+ n := runtime.Stack(buf, true)
+ for _, stack := range strings.Split(string(buf[:n]), "\n\n") {
+ if strings.Contains(stack, handler) && (strings.Contains(stack, "sync.Mutex.Lock") || strings.Contains(stack, "sync.RWMutex.RLock")) {
+ return
+ }
+ }
+ runtime.Gosched()
+ }
+ t.Fatal("handler did not queue on the controlled mutex")
+}
+
+func TestTransitionHeaderlessRelayCannotAdoptSuccessor(t *testing.T) {
+ for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete} {
+ t.Run(method, func(t *testing.T) {
+ h := newVaultPersistHarness(t)
+ subj := h.snapshotSubject(t)
+ before := append([]byte(nil), h.fileFor(snapshotTestUID).data...)
+ body := ""
+ if method == http.MethodPost {
+ body = vaultCredential()
+ }
+ if method == http.MethodPut {
+ body = vaultPut("1")
+ }
+ if method == http.MethodGet {
+ h.srv.mu.Lock()
+ } else {
+ h.srv.policyCommitMu.Lock()
+ }
+ unlock := func() {
+ if method == http.MethodGet {
+ h.srv.mu.Unlock()
+ } else {
+ h.srv.policyCommitMu.Unlock()
+ }
+ }
+ locked := true
+ defer func() {
+ if locked {
+ unlock()
+ }
+ }()
+ result := transitionRequest(h.srv, context.Background(), method, constants.VaultRelayPathPrefix+snapshotTestUID, body)
+ handler := "readRelayVault"
+ if method != http.MethodGet {
+ handler = "handleVaultRelayWrite.func1"
+ }
+ transitionWaitForMutex(t, handler)
+ // Model the completed publication while the request is queued: registry,
+ // mirror and vault all belong to the successor, and no rebind token remains.
+ // Registry changes here have no callbacks; this is a controlled test fixture.
+ next := subject.Fencing{RuntimeInstanceID: "successor", AttachmentID: "successor-att"}
+ h.reg.Unregister(subj)
+ h.reg.Register(subj, subject.SubjectKey{SourceIP: netip.MustParseAddr(testIP)}, next)
+ if method != http.MethodGet {
+ h.srv.mu.Lock()
+ }
+ h.srv.subjFence[subj] = next
+ entry := h.srv.vaults[subj]
+ entry.fence = next
+ h.srv.vaults[subj] = entry
+ delete(h.srv.rebinds, subj)
+ if method != http.MethodGet {
+ h.srv.mu.Unlock()
+ }
+ unlock()
+ locked = false
+ rec := transitionResult(t, result)
+ require.Equal(t, http.StatusNotFound, rec.Code)
+ require.NotContains(t, rec.Body.String(), "credentials")
+ require.Equal(t, before, h.fileFor(snapshotTestUID).data)
+ require.True(t, h.fileFor(snapshotTestUID).exists)
+ })
+ }
+}
diff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.go
index 3bf716f92..370d47349 100644
--- a/components/egress/fastsandbox_vault.go
+++ b/components/egress/fastsandbox_vault.go
@@ -122,6 +122,13 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
// The body has been consumed; a late response must not try to read it.
r.Body.Close()
+ if s.subjectRebindingNow(subj) {
+ http.Error(w, "subject rebind in progress", http.StatusServiceUnavailable)
+ return
+ }
+ // Capture the request identity before either serialization lock. A queued
+ // headerless write must not adopt a successor published while it waits.
+ fence, registered := s.reg.Fence(subj)
// Capture a response while committing; no network output holds s.mu.
respond := func() func() {
if s.subjectRebinding(subj) {
@@ -135,8 +142,7 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
if s.subjectRebindingLocked(subj) {
return func() { http.Error(w, "subject rebind in progress", http.StatusServiceUnavailable) }
}
- fence, ok := s.reg.Fence(subj)
- if !ok {
+ if !registered || !s.fenceMatchesLocked(subj, fence) {
return func() { http.Error(w, "unknown subject", http.StatusNotFound) }
}
if inc := strings.TrimSpace(r.Header.Get(constants.EgressVaultIncarnationHeader)); inc != "" && inc != fence.RuntimeInstanceID {
@@ -268,13 +274,16 @@ func writeVaultRelayWriteResponse(w http.ResponseWriter, r *http.Request, state
// readRelayVault captures an independent inline result under the subject lock;
// sending it afterwards cannot delay lifecycle state changes.
func (s *fastSandboxPolicyServer) readRelayVault(r *http.Request, subj subject.Subject) (credentialvault.FullRevision, int, string, error) {
+ if s.subjectRebindingNow(subj) {
+ return credentialvault.FullRevision{}, http.StatusServiceUnavailable, "subject rebind in progress", nil
+ }
+ fence, registered := s.reg.Fence(subj)
s.mu.Lock()
defer s.mu.Unlock()
if s.subjectRebindingLocked(subj) {
return credentialvault.FullRevision{}, http.StatusServiceUnavailable, "subject rebind in progress", nil
}
- fence, ok := s.reg.Fence(subj)
- if !ok {
+ if !registered || !s.fenceMatchesLocked(subj, fence) {
return credentialvault.FullRevision{}, http.StatusNotFound, "unknown subject", nil
}
if inc := strings.TrimSpace(r.Header.Get(constants.EgressVaultIncarnationHeader)); inc != "" && inc != fence.RuntimeInstanceID {
diff --git a/components/egress/fastsandbox_vault_persist_test.go b/components/egress/fastsandbox_vault_persist_test.go
index 6a6de809d..68dd3e257 100644
--- a/components/egress/fastsandbox_vault_persist_test.go
+++ b/components/egress/fastsandbox_vault_persist_test.go
@@ -660,14 +660,17 @@ func TestVaultPersistRemoveBindingPurgesFile(t *testing.T) {
require.Equal(t, http.StatusCreated, rec.Code)
require.True(t, h.fileFor(vaultTestUID).exists)
- // Cleanup runs after Unregister and retains its bounded second attempt
- // on failure, without unlocking between attempts.
+ // A failed best-effort cleanup is still acknowledged; an explicit later
+ // REMOVE can retry it, but no automatic retry is promised.
h.fileFor(vaultTestUID).failFirstDelete = errors.New("unlink EIO")
rec = doAction(t, h.srv, removeBody(t, vaultTestUID, testFenceR, testFenceA))
require.Equal(t, http.StatusOK, rec.Code)
- assert.False(t, h.fileFor(vaultTestUID).exists, "REMOVE_BINDING purges memory + file (terminal)")
- assert.Equal(t, 2, h.fileFor(vaultTestUID).deleteCalls, "the post-unregister cleanup retries its failed first purge")
+ assert.True(t, h.fileFor(vaultTestUID).exists, "failed cleanup leaves the old fenced file")
+ assert.Equal(t, 1, h.fileFor(vaultTestUID).deleteCalls)
+ rec = doAction(t, h.srv, removeBody(t, vaultTestUID, testFenceR, testFenceA))
+ require.Equal(t, http.StatusOK, rec.Code)
+ assert.False(t, h.fileFor(vaultTestUID).exists)
// REMOVE_BINDING for unknown Handler state also drops any cached file.
h.seedVaultFile(vaultTestUID, "runtime-x", "attachment-x", 1, reloadVaultBody)