Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): enforce registry-first lock order, serialize purges, harden snapshot targets

高然15422c369434
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index bd057502e..68d31ef32 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -114,7 +114,7 @@ func (s *fastSandboxPolicyServer) handleActionsStatus(w http.ResponseWriter, r *
// persistence there is no recorded policy to hand off, so the
// transfer degrades (skips) rather than failing the checkpoint.
PolicyTransfer: s.policyFiles != nil,
- PolicyRecovery: s.policyRecoveryActive(),
+ PolicyRecovery: policyRecovery,
DegradedVaults: degraded,
}
if s.mitmGate != nil {
@@ -201,13 +201,21 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (
// and the recorded policy must go too (memory + persisted files), or the
// old instance's credentials and policy would survive into the
// successor's flows. Same-UID actions serialize per subject; the vault
- // and policy relay writes serialize against the same state.
+ // and policy relay writes serialize against the same state — the purge
+ // runs under policyCommitMu so an in-flight relay commit against the
+ // OLD fence cannot resurrect a purged record after the rebind (its
+ // commit-time fence re-check aborts it). Callers are never under the
+ // registry write lock here, so taking policyCommitMu is safe (the
+ // takeover path, which IS under the registry lock, purges without it —
+ // its relay races are fenced out by the commit-time re-check).
if old, ok := s.reg.Fence(subj); ok && !old.Matches(fence) {
+ s.policyCommitMu.Lock()
s.mu.Lock()
s.dropVault(subj)
delete(s.policies, subj)
s.purgePolicyFile(subj)
s.mu.Unlock()
+ s.policyCommitMu.Unlock()
log.Infof("subject %s: rebind (%s -> %s), vault slice and policy record purged", subj, old.RuntimeInstanceID, fence.RuntimeInstanceID)
}
// Egress-restart recovery (OSEP-0022 stage 2): a subject with no live
diff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.go
index b5cec60db..9b09391e1 100644
--- a/components/egress/fastsandbox_policy.go
+++ b/components/egress/fastsandbox_policy.go
@@ -27,8 +27,9 @@
//
// Sidecar-parity guards on every op: unregistered subject → 404, never
// cached; expectedIncarnation mismatch → 412; sealed (exported for a
-// pause checkpoint) → 409; policy persistence recovery latch → 503.
-// Validation failure → 400 NACK keeping last-good. With persistence enabled
+// pause checkpoint) → 409; policy persistence recovery latch → 503 on
+// writes (GET keeps serving the in-memory record). Validation failure →
+// 400 NACK keeping last-good. With persistence enabled
// every write runs the commit protocol — persist → apply → ack — so the ack
// means durable AND applied (see fastsandbox_policy_persist.go).
package main
@@ -101,11 +102,13 @@ func (s *fastSandboxPolicyServer) handlePolicyRelay(w http.ResponseWriter, r *ht
}
// selectRelayPolicy decides registration, the expectedIncarnation
-// precondition, the seal, and the recorded policy under one s.mu hold. A
-// status other than zero means the caller must write (status, msg).
+// precondition, the seal, and the recorded policy. Registry reads happen
+// BEFORE s.mu — the lock order is registry → s.mu everywhere (the takeover
+// path holds the registry write lock while purging subject state under
+// s.mu; the reverse order deadlocks the DNS hot path). A fence change
+// between the read and the lock is bounded: the purge removes the record,
+// and the write paths re-validate the fence at commit time.
func (s *fastSandboxPolicyServer) selectRelayPolicy(r *http.Request, subj subject.Subject) (*policyRelayDocument, int, string) {
- s.mu.Lock()
- defer s.mu.Unlock()
fence, ok := s.reg.Fence(subj)
if !ok {
return nil, http.StatusNotFound, "unknown subject"
@@ -113,10 +116,12 @@ func (s *fastSandboxPolicyServer) selectRelayPolicy(r *http.Request, subj subjec
if inc := strings.TrimSpace(r.Header.Get(constants.EgressVaultIncarnationHeader)); inc != "" && inc != fence.RuntimeInstanceID {
return nil, http.StatusPreconditionFailed, "expectedIncarnation mismatch"
}
+ pol := s.reg.UserPolicy(subj)
+ s.mu.Lock()
+ defer s.mu.Unlock()
if e := s.policies[subj]; e != nil && e.sealed {
return nil, http.StatusConflict, "subject policy is sealed (paused); the bundle copy is already taken"
}
- pol := s.reg.UserPolicy(subj)
if pol == nil {
pol = s.pendingPolicies[subj]
}
@@ -191,20 +196,20 @@ func (s *fastSandboxPolicyServer) handlePolicyRelayWrite(w http.ResponseWriter,
s.policyCommitMu.Lock()
defer s.policyCommitMu.Unlock()
- // Guards + decision under one s.mu hold, serialized against rebinds and
- // terminal purges.
- s.mu.Lock()
+ // Registry reads before s.mu (lock order: registry → s.mu, see
+ // selectRelayPolicy); the commit-time target re-check closes the TOCTOU.
fence, ok := s.reg.Fence(subj)
if !ok {
- s.mu.Unlock()
http.Error(w, "unknown subject", http.StatusNotFound)
return
}
if inc := strings.TrimSpace(r.Header.Get(constants.EgressVaultIncarnationHeader)); inc != "" && inc != fence.RuntimeInstanceID {
- s.mu.Unlock()
http.Error(w, "expectedIncarnation mismatch", http.StatusPreconditionFailed)
return
}
+ // Guards + decision under one s.mu hold, serialized against rebinds and
+ // terminal purges (the whole write already holds policyCommitMu).
+ s.mu.Lock()
entry := s.policies[subj]
if entry != nil && entry.sealed {
s.mu.Unlock()
@@ -306,7 +311,8 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.
if s.policyFiles == nil {
// Memory-only (persistence disabled): direct commit.
if err := apply(); err != nil {
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ log.Errorf("subject %s: policy relay write failed: %v", subj, err)
+ http.Error(w, "policy commit failure", http.StatusInternalServerError)
return
}
writeJSON(w, putResponseStatus(created), doc)
@@ -339,14 +345,13 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.
writeJSON(w, putResponseStatus(created), doc)
}
-// policyCommitTarget re-reads the subject's lifecycle state under s.mu and
-// verifies the registration fence still matches the commit's — the
-// data-plane-ready activation and rebinds run outside the commit
-// serialization, so a stale apply must be aborted (the commit protocol
-// rolls the file back) instead of landing on the wrong identity or state.
+// policyCommitTarget re-reads the subject's lifecycle state and verifies
+// the registration fence still matches the commit's — the data-plane-ready
+// activation and rebinds run outside the commit serialization, so a stale
+// apply must be aborted (the commit protocol rolls the file back) instead
+// of landing on the wrong identity or state. Registry reads only: no s.mu
+// (lock order: registry → s.mu, see selectRelayPolicy).
func (s *fastSandboxPolicyServer) policyCommitTarget(subj subject.Subject, fence subject.Fencing) (subject.State, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
regFence, ok := s.reg.Fence(subj)
if !ok || regFence != fence {
return subject.StateAbsent, fmt.Errorf("stale policy commit (fence changed) for subject %s", subj)
@@ -399,7 +404,8 @@ func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subj
if s.policyFiles == nil {
if err := apply(); err != nil {
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ log.Errorf("subject %s: policy relay delete failed: %v", subj, err)
+ http.Error(w, "policy commit failure", http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
@@ -434,17 +440,3 @@ func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subj
log.Infof("subject %s: policy deleted (deny-first), record purged", subj)
w.WriteHeader(http.StatusNoContent)
}
-
-// writePolicyRelayWriteResponse maps the committed mutation onto the relay
-// contract: PUT → 200 (201 when it created), DELETE → 204.
-func writePolicyRelayWriteResponse(w http.ResponseWriter, r *http.Request, revision int64, created bool) {
- if r.Method == http.MethodDelete {
- w.WriteHeader(http.StatusNoContent)
- return
- }
- status := http.StatusOK
- if created {
- status = http.StatusCreated
- }
- writeJSON(w, status, policyRelayDocument{Revision: revision, Policy: json.RawMessage("null")})
-}
diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.go
index dbebee026..9c60fa9b3 100644
--- a/components/egress/fastsandbox_policy_persist.go
+++ b/components/egress/fastsandbox_policy_persist.go
@@ -179,6 +179,21 @@ func (s *fastSandboxPolicyServer) SetPolicyFiles(pp *policyPersistence) {
s.policyFiles = pp
}
+// reservesStatePath reports whether a driver-provided snapshot path
+// resolves inside ANY per-subject state directory — the vault and policy
+// directories are siblings on one volume, and an export target under the
+// other kind's directory could clobber its state (the rehydrate would then
+// quarantine the corrupted file).
+func (s *fastSandboxPolicyServer) reservesStatePath(path string) bool {
+ if s.vaultFiles != nil && s.vaultFiles.reservesPath(path) {
+ return true
+ }
+ if s.policyFiles != nil && s.policyFiles.reservesPath(path) {
+ return true
+ }
+ return false
+}
+
// purgePolicyFile removes the subject's persisted policy (rebind or terminal
// cleanup). Best effort: a failed purge leaves a file that its recorded
// fence keeps out of every later registration.
@@ -279,8 +294,12 @@ func (s *fastSandboxPolicyServer) policyHealthIssue() string {
// on a still-live sandbox): the entry goes and the persisted file is purged,
// so the deny-first state is also the durable state — a restart cannot
// resurrect a removed policy, and a later registration starts from its own
-// binding input.
+// input. Serialized with the relay commits (policyCommitMu) so an in-flight
+// write against the removed binding cannot recreate the record after the
+// purge; callers are never under the registry write lock.
func (s *fastSandboxPolicyServer) clearPolicyRecord(subj subject.Subject) {
+ s.policyCommitMu.Lock()
+ defer s.policyCommitMu.Unlock()
s.mu.Lock()
delete(s.policies, subj)
s.mu.Unlock()
@@ -391,6 +410,22 @@ func (s *fastSandboxPolicyServer) applyBindingPolicy(subj subject.Subject, fence
s.mu.Lock()
entry := s.policies[subj]
+ if entry != nil && entry.fence != fence {
+ // A stale record (its purge outcome was uncertain, or a rebind
+ // interleaved the purge) must never supersede the fresh
+ // registration's input: purge it durably and seed from the input.
+ delete(s.policies, subj)
+ pp := s.policyFiles
+ s.mu.Unlock()
+ if pp != nil {
+ if err := pp.purge(subj); err != nil {
+ log.Errorf("subject %s: stale policy record purge failed (leftover stays fenced out): %v", subj, err)
+ }
+ }
+ log.Infof("subject %s: stale policy record discarded (fence mismatch), binding input seeds", subj)
+ s.mu.Lock()
+ entry = nil
+ }
pp := s.policyFiles
s.mu.Unlock()
if entry != nil {
diff --git a/components/egress/fastsandbox_policy_persist_test.go b/components/egress/fastsandbox_policy_persist_test.go
index b2d7a874a..cc75e0e51 100644
--- a/components/egress/fastsandbox_policy_persist_test.go
+++ b/components/egress/fastsandbox_policy_persist_test.go
@@ -502,6 +502,36 @@ func TestPolicyPersistDeleteTombstoneSurvivesRestart(t *testing.T) {
assert.Equal(t, "deny", last.Evaluate("other.example.com"))
}
+// TestPolicyPersistStaleRecordNeverSupersedesFreshRegistration: a leftover
+// record under a foreign fence (purge outcome uncertain) must not make
+// applyBindingPolicy supersede the fresh registration's binding input.
+func TestPolicyPersistStaleRecordNeverSupersedesFreshRegistration(t *testing.T) {
+ h := newPolicyPersistHarness(t)
+
+ // A stale record under a foreign fence survives on disk (its purge
+ // failed) and in memory.
+ h.readySubject(t, policyTestUID, vaultTestIP)
+ rec := doPolicy(t, h.srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllowAlt(), 1, false, true), "")
+ require.Equal(t, http.StatusOK, rec.Code)
+ h.fileFor(policyTestUID).failFirstDelete = errors.New("unlink EIO")
+ // Rebind with an uncertain purge outcome: the relayed policy must not
+ // leak into the successor.
+ const newRuntime = "runtime-2"
+ rec = doAction(t, h.srv, bindBody(t, policyTestUID, vaultTestIP, newRuntime, "attachment-2", 2, strPtr(policyAllow())))
+ require.Equal(t, http.StatusOK, rec.Code)
+ rec = doAction(t, h.srv, hookBody(t, policyTestUID, newRuntime, "attachment-2", constants.HookDataPlaneReady))
+ require.Equal(t, http.StatusOK, rec.Code)
+
+ // The successor was seeded from its OWN binding input at revision 1.
+ rec = doPolicy(t, h.srv, http.MethodGet, policyTestUID, "", "")
+ rev, doc := mustPolicyRevision(t, rec)
+ assert.Equal(t, int64(1), rev)
+ assert.Contains(t, string(doc.Policy), "api.example.com")
+ assert.NotContains(t, string(doc.Policy), "other.example.com", "the foreign-fence record must not supersede the fresh input")
+ pf, _ := decodePolicyFile(t, h.fileFor(policyTestUID))
+ assert.Equal(t, newRuntime, pf.Incarnation)
+}
+
// ---------------------------------------------------------------------------
// Capability + healthy posture
// ---------------------------------------------------------------------------
diff --git a/components/egress/fastsandbox_policy_snapshot.go b/components/egress/fastsandbox_policy_snapshot.go
index 2458deb7e..a3dcbfdb9 100644
--- a/components/egress/fastsandbox_policy_snapshot.go
+++ b/components/egress/fastsandbox_policy_snapshot.go
@@ -68,14 +68,17 @@ func (s *fastSandboxPolicyServer) applyPolicyExport(env *actionhandler.Envelope)
subj := subject.FromSandboxUID(env.Sandbox.UID)
fence := subject.FromRevision(env)
target := strings.TrimSpace(env.Snapshot.Path)
- if s.policyFiles.reservesPath(target) {
- return http.StatusBadRequest, fmt.Errorf("policy bundle target %q must not live under the policy persistence directory", target)
+ if s.reservesStatePath(target) {
+ return http.StatusBadRequest, fmt.Errorf("policy bundle target %q must not live under a per-subject state directory", target)
}
// A subject that never registered (a sandbox created without an egress
// binding, or after terminal cleanup) has no policy to carry: 200 so the
// checkpoint proceeds — a 404 would fail every pause of a binding-less
// sandbox. A registered subject under a stale fence still fails below.
- if _, ok := s.reg.Fence(subj); !ok {
+ // Registry reads before the lock acquisitions (lock order: registry →
+ // policyCommitMu → s.mu, see selectRelayPolicy).
+ regFence, registered := s.reg.Fence(subj)
+ if !registered {
log.Infof("policy export: subject %s not registered; no policy to carry, checkpoint proceeds without it", subj)
return http.StatusOK, nil
}
@@ -87,7 +90,7 @@ func (s *fastSandboxPolicyServer) applyPolicyExport(env *actionhandler.Envelope)
// commit "under" the seal.
s.policyCommitMu.Lock()
s.mu.Lock()
- snap, status, err := s.policyExportSourceImageLocked(subj, fence)
+ snap, status, err := s.policyExportSourceImageLocked(subj, fence, regFence, registered)
s.mu.Unlock()
s.policyCommitMu.Unlock()
if err != nil {
@@ -110,12 +113,15 @@ func (s *fastSandboxPolicyServer) applyPolicyExport(env *actionhandler.Envelope)
// Phase 3: seal only against the exact copied image, again serialized
// with the relay commits — between the re-check and the seal no writer
- // can pass the guards (they now observe the seal) or commit.
+ // can pass the guards (they now observe the seal) or commit. The fence
+ // is re-read fresh (registry before s.mu) so a rebind during the copy
+ // fails the export instead of sealing the wrong subject's bytes.
+ regFence, registered = s.reg.Fence(subj)
s.policyCommitMu.Lock()
defer s.policyCommitMu.Unlock()
s.mu.Lock()
defer s.mu.Unlock()
- live, status, err := s.policyExportSourceImageLocked(subj, fence)
+ live, status, err := s.policyExportSourceImageLocked(subj, fence, regFence, registered)
if err != nil {
return status, err
}
@@ -155,10 +161,11 @@ func (s *fastSandboxPolicyServer) applyPolicyExport(env *actionhandler.Envelope)
}
// policyExportSourceImageLocked runs the export guards and returns a
-// consistent image of the subject's policy file. Callers hold s.mu.
-func (s *fastSandboxPolicyServer) policyExportSourceImageLocked(subj subject.Subject, fence subject.Fencing) (*policy.PolicyFileSnapshot, int, error) {
- regFence, ok := s.reg.Fence(subj)
- if !ok {
+// consistent image of the subject's policy file. regFence/registered are
+// the caller's pre-read of the registration (lock order: registry →
+// policyCommitMu → s.mu, see applyPolicyExport). Callers hold s.mu.
+func (s *fastSandboxPolicyServer) policyExportSourceImageLocked(subj subject.Subject, fence subject.Fencing, regFence subject.Fencing, registered bool) (*policy.PolicyFileSnapshot, int, error) {
+ if !registered {
return nil, http.StatusNotFound, fmt.Errorf("policy export for unknown subject %s", subj)
}
if !regFence.Matches(fence) {
@@ -215,8 +222,8 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)
subj := subject.FromSandboxUID(env.Sandbox.UID)
fence := subject.FromRevision(env)
source := strings.TrimSpace(env.Snapshot.Path)
- if s.policyFiles.reservesPath(source) {
- return http.StatusBadRequest, fmt.Errorf("policy bundle source %q must not live under the policy persistence directory", source)
+ if s.reservesStatePath(source) {
+ return http.StatusBadRequest, fmt.Errorf("policy bundle source %q must not live under a per-subject state directory", source)
}
// Read before locking; a missing/oversized/unreadable source is
@@ -250,6 +257,9 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)
s.policyCommitMu.Lock()
defer s.policyCommitMu.Unlock()
+ // Registry reads before s.mu (lock order: registry → policyCommitMu →
+ // s.mu, see applyPolicyExport).
+ regFence, registered := s.reg.Fence(subj)
s.mu.Lock()
// The latch freezes every policy file mutation; only a restart
// reconciles (relay-write parity).
@@ -272,7 +282,6 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)
}
}
}
- regFence, registered := s.reg.Fence(subj)
if registered && !regFence.Matches(fence) {
s.mu.Unlock()
return http.StatusConflict, fmt.Errorf("stale policy import (fence mismatch) for subject %s", subj)
diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.go
index 24a2e002c..d5a02097e 100644
--- a/components/egress/fastsandbox_server.go
+++ b/components/egress/fastsandbox_server.go
@@ -405,12 +405,16 @@ type vaultEntry struct {
// A store is only valid under the fence it was created with: a mismatch
// (same-UID rebind without REMOVE_BINDING, or a TOCTOU store materialized
// against an unregistered subject) is dropped, never inherited — a reset
-// never carries old credentials into a new sandbox (OSEP-0022). The fence is
-// read under s.mu, serialized with the rebind purge in applySetBinding.
+// never carries old credentials into a new sandbox (OSEP-0022). The fence
+// is PRE-READ by the caller (lock order: registry → s.mu); a fence change
+// between the read and the lock is bounded by the rebind purge — the stale
+// store is dropped on the next selection either way.
func (s *fastSandboxPolicyServer) vaultFor(subj subject.Subject) *credentialvault.Store {
+ // Registry read before s.mu (lock order: registry → s.mu, see
+ // handleVaultRelayWrite).
+ fence, _ := s.reg.Fence(subj)
s.mu.Lock()
defer s.mu.Unlock()
- fence, _ := s.reg.Fence(subj)
return s.vaultForLocked(subj, fence)
}
@@ -545,6 +549,12 @@ func (s *fastSandboxPolicyServer) takeoverStaleSubject(stale subject.Subject) er
// policy record, pending policy, attachment) plus the persisted vault and
// policy files — terminal cleanup (OSEP-0022: REMOVE_BINDING purges memory +
// file).
+//
+// Deliberately WITHOUT policyCommitMu: one caller (takeoverStaleSubject)
+// runs under the registry WRITE lock, and taking policyCommitMu there would
+// invert against relay commits (policyCommitMu → registry via applyPolicy).
+// Safety comes from the commit paths' fence re-check (policyCommitTarget)
+// and the recorded fence fencing out stale files at rehydration.
func (s *fastSandboxPolicyServer) dropSubjectState(subj subject.Subject) {
s.mu.Lock()
delete(s.vaults, subj)
diff --git a/components/egress/fastsandbox_server_test.go b/components/egress/fastsandbox_server_test.go
index 5cd13cbe9..5bc32f26c 100644
--- a/components/egress/fastsandbox_server_test.go
+++ b/components/egress/fastsandbox_server_test.go
@@ -696,21 +696,31 @@ func TestActionsInvalidEnvelopesRejected(t *testing.T) {
// must stay gone.
func TestFastSandboxServerSupersededRoutesGone(t *testing.T) {
srv, _, _ := fastSandboxTestServer(t)
+ // The bare /policy path redirects into the relay subtree; the old
+ // UID-header surface stays gone. The exact 3xx code varies by method
+ // (and Go version) — assert the redirect class and the Location target.
for _, tc := range []struct {
method, path string
- want int
}{
- {http.MethodGet, "/policy", http.StatusTemporaryRedirect},
- {http.MethodPut, "/policy", http.StatusTemporaryRedirect},
- {http.MethodPost, "/policy", http.StatusTemporaryRedirect},
- {http.MethodDelete, "/policy", http.StatusTemporaryRedirect},
- {http.MethodGet, "/credential-vault", http.StatusNotFound},
- {http.MethodPost, "/credential-vault", http.StatusNotFound},
- {http.MethodPatch, "/credential-vault", http.StatusNotFound},
- {http.MethodDelete, "/credential-vault", http.StatusNotFound},
+ {http.MethodGet, "/policy"},
+ {http.MethodPut, "/policy"},
+ {http.MethodPost, "/policy"},
+ {http.MethodDelete, "/policy"},
} {
rec := doRequest(t, srv, tc.method, tc.path, `{}`)
- require.Equal(t, tc.want, rec.Code, "%s %s", tc.method, tc.path)
+ require.True(t, rec.Code >= 300 && rec.Code < 400, "%s %s: want a redirect, got %d", tc.method, tc.path, rec.Code)
+ assert.Equal(t, "/policy/", rec.Header().Get("Location"), "%s %s", tc.method, tc.path)
+ }
+ for _, tc := range []struct {
+ method, path string
+ }{
+ {http.MethodGet, "/credential-vault"},
+ {http.MethodPost, "/credential-vault"},
+ {http.MethodPatch, "/credential-vault"},
+ {http.MethodDelete, "/credential-vault"},
+ } {
+ rec := doRequest(t, srv, tc.method, tc.path, `{}`)
+ require.Equal(t, http.StatusNotFound, rec.Code, "%s %s must be gone", tc.method, tc.path)
}
}
diff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.go
index 8a3be7777..2045ecc9d 100644
--- a/components/egress/fastsandbox_vault.go
+++ b/components/egress/fastsandbox_vault.go
@@ -123,8 +123,8 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
// The body has been consumed; a late response must not try to read it.
r.Body.Close()
- s.mu.Lock()
- defer s.mu.Unlock()
+ // Registry reads before s.mu (lock order: registry → s.mu, see
+ // selectRelayVault).
fence, ok := s.reg.Fence(subj)
if !ok {
http.Error(w, "unknown subject", http.StatusNotFound)
@@ -134,6 +134,12 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
http.Error(w, "expectedIncarnation mismatch", http.StatusPreconditionFailed)
return
}
+ effective := s.reg.EffectivePolicy(subj)
+
+ // The rest of the mutation holds s.mu, serializing per-subject writers
+ // and fencing rebind/terminal purges.
+ s.mu.Lock()
+ defer s.mu.Unlock()
if s.vaultSealedLocked(subj) {
http.Error(w, "subject vault is sealed (paused); the bundle copy is already taken", http.StatusConflict)
return
@@ -143,7 +149,7 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
return
}
store := s.vaultForLocked(subj, fence)
- pol := s.vaultValidationPolicyLocked(subj)
+ pol := s.vaultValidationPolicyLocked(effective, subj)
var (
candidate *credentialvault.MutationCandidate
@@ -260,15 +266,14 @@ func writeVaultRelayWriteResponse(w http.ResponseWriter, r *http.Request, state
}
// selectRelayVault decides registration, the expectedIncarnation
-// precondition, and the fence-bound store selection under one s.mu hold
-// (see vaultForLocked). A concurrent rebind then either purges before
-// selection — the stale incarnation mismatches the successor's fresh fence —
-// or purges after — the stale request holds the old store, which the purge
-// discards. Either way a stale writer never reaches the successor's store.
+// precondition, and the fence-bound store selection. Registry reads happen
+// BEFORE s.mu (lock order: registry → s.mu, see handleVaultRelayWrite); a
+// concurrent rebind then either purges before selection — the stale
+// incarnation mismatches the successor's fresh fence — or purges after —
+// the stale request holds the old store, which the purge discards. Either
+// way a stale writer never reaches the successor's store.
// A nil store means the caller must write (status, msg).
func (s *fastSandboxPolicyServer) selectRelayVault(r *http.Request, subj subject.Subject) (*credentialvault.Store, int, string) {
- s.mu.Lock()
- defer s.mu.Unlock()
fence, ok := s.reg.Fence(subj)
if !ok {
return nil, http.StatusNotFound, "unknown subject"
@@ -276,6 +281,8 @@ func (s *fastSandboxPolicyServer) selectRelayVault(r *http.Request, subj subject
if inc := strings.TrimSpace(r.Header.Get(constants.EgressVaultIncarnationHeader)); inc != "" && inc != fence.RuntimeInstanceID {
return nil, http.StatusPreconditionFailed, "expectedIncarnation mismatch"
}
+ s.mu.Lock()
+ defer s.mu.Unlock()
if s.vaultSealedLocked(subj) {
return nil, http.StatusConflict, "subject vault is sealed (paused); the bundle copy is already taken"
}
@@ -293,15 +300,15 @@ func (s *fastSandboxPolicyServer) vaultSealedLocked(subj subject.Subject) bool {
// against: the effective policy of an active subject, or — while the subject
// is still denying — its pending binding policy (the op is held to the
// data-plane-ready barrier, so validation must not depend on hook timing).
-// Callers hold s.mu.
-func (s *fastSandboxPolicyServer) vaultValidationPolicyLocked(subj subject.Subject) *policy.NetworkPolicy {
- if pol := s.reg.EffectivePolicy(subj); pol != nil {
- return pol
- }
- if pending := s.pendingPolicies[subj]; pending != nil {
- return s.reg.EffectiveOf(pending)
+// The effective policy is PRE-READ by the caller before s.mu (lock order:
+// registry → s.mu, see handleVaultRelayWrite); the pending fallback uses
+// the raw user policy — the always overlay is enforcement-side and
+// validation only needs the explicit allow cover. Callers hold s.mu.
+func (s *fastSandboxPolicyServer) vaultValidationPolicyLocked(effective *policy.NetworkPolicy, subj subject.Subject) *policy.NetworkPolicy {
+ if effective != nil {
+ return effective
}
- return nil
+ return s.pendingPolicies[subj]
}
// writeVaultRelayError maps store errors onto the relay contract: revision
diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.go
index cdd02fd10..787ac16f1 100644
--- a/components/egress/fastsandbox_vault_persist.go
+++ b/components/egress/fastsandbox_vault_persist.go
@@ -143,11 +143,23 @@ func sweepStateTempFiles(dir, noun string) error {
return nil
}
+// knownStateDirNames are the per-subject state directories that may share
+// one egress volume root (the vault and the recorded policy). A sibling
+// state directory is ours, not a foreign entry — without this, whichever
+// directory initializes second would see the first as foreign, skip the
+// volume-root restriction, and the atomic machinery would then reject every
+// file op on the untrusted ancestor.
+var knownStateDirNames = map[string]bool{
+ "vault": true,
+ "policy": true,
+}
+
// restrictVolumeRoot clears group/world-write bits from the vault volume
// root (the vault directory's parent): kubelet creates emptyDir roots 0777,
-// which the atomic machinery rejects as an untrusted ancestor. A parent
-// holding foreign entries is left untouched — that is the manifest's job —
-// and already-restrictive roots are a no-op.
+// which the atomic machinery rejects as an untrusted ancestor. Sibling
+// state directories (knownStateDirNames) belong to us; a root holding any
+// other entry is left untouched — that is the manifest's job — and
+// already-restrictive roots are a no-op.
func restrictVolumeRoot(dir string) error {
parent := filepath.Dir(dir)
info, err := os.Stat(parent)
@@ -166,10 +178,11 @@ func restrictVolumeRoot(dir string) error {
}
base := filepath.Base(dir)
for _, entry := range entries {
- if entry.Name() != base {
- log.Infof("vault persistence: volume root %s holds foreign entry %q; not restricting (fix permissions in the manifest)", parent, entry.Name())
- return nil
+ if entry.Name() == base || (entry.IsDir() && knownStateDirNames[entry.Name()]) {
+ continue
}
+ log.Infof("vault persistence: volume root %s holds foreign entry %q; not restricting (fix permissions in the manifest)", parent, entry.Name())
+ return nil
}
if err := os.Chmod(parent, info.Mode().Perm()&^0o022); err != nil {
return fmt.Errorf("restrict vault volume root %s: %w", parent, err)
@@ -178,12 +191,6 @@ func restrictVolumeRoot(dir string) error {
return nil
}
-// sweepVaultTempFiles unlinks staging files a crashed writer left behind
-// (rename never attempted — unconfirmed candidate state).
-func sweepVaultTempFiles(dir string) error {
- return sweepStateTempFiles(dir, "vault")
-}
-
// vaultFileUID strips the internal "s-" prefix: the file contract names files
// <sandboxUid>.json.
func vaultFileUID(subj subject.Subject) string {
diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.go
index 8cc5c3918..f2035cf25 100644
--- a/components/egress/fastsandbox_vault_snapshot.go
+++ b/components/egress/fastsandbox_vault_snapshot.go
@@ -87,21 +87,24 @@ func (s *fastSandboxPolicyServer) applyVaultExport(env *actionhandler.Envelope)
subj := subject.FromSandboxUID(env.Sandbox.UID)
fence := subject.FromRevision(env)
target := strings.TrimSpace(env.Snapshot.Path)
- if s.vaultFiles.reservesPath(target) {
- return http.StatusBadRequest, fmt.Errorf("vault bundle target %q must not live under the vault persistence directory", target)
+ if s.reservesStatePath(target) {
+ return http.StatusBadRequest, fmt.Errorf("vault bundle target %q must not live under a per-subject state directory", target)
}
// A subject that never registered (a sandbox created without an egress
// binding, or after terminal cleanup) has no vault to carry: 200 so the
// checkpoint proceeds — a 404 would fail every pause of a binding-less
// sandbox. A registered subject under a stale fence still fails below.
- if _, ok := s.reg.Fence(subj); !ok {
+ // Registry reads before the lock acquisitions (lock order: registry →
+ // s.mu, see handleVaultRelayWrite).
+ regFence, registered := s.reg.Fence(subj)
+ if !registered {
log.Infof("vault export: subject %s not registered; no vault to carry, checkpoint proceeds without it", subj)
return http.StatusOK, nil
}
// Phase 1: guards + consistent source image.
s.mu.Lock()
- snap, status, err := s.exportSourceImageLocked(subj, fence)
+ snap, status, err := s.exportSourceImageLocked(subj, fence, regFence, registered)
if err != nil {
s.mu.Unlock()
return status, err
@@ -120,10 +123,13 @@ func (s *fastSandboxPolicyServer) applyVaultExport(env *actionhandler.Envelope)
}
}
- // Phase 3: seal only against the exact copied image.
+ // Phase 3: seal only against the exact copied image; the fence is
+ // re-read fresh (registry before s.mu) so a rebind during the copy
+ // fails the export.
+ regFence, registered = s.reg.Fence(subj)
s.mu.Lock()
defer s.mu.Unlock()
- live, status, err := s.exportSourceImageLocked(subj, fence)
+ live, status, err := s.exportSourceImageLocked(subj, fence, regFence, registered)
if err != nil {
return status, err
}
@@ -161,10 +167,11 @@ func (s *fastSandboxPolicyServer) applyVaultExport(env *actionhandler.Envelope)
}
// exportSourceImageLocked runs the export guards and returns a consistent
-// image of the subject's vault file. Callers hold s.mu.
-func (s *fastSandboxPolicyServer) exportSourceImageLocked(subj subject.Subject, fence subject.Fencing) (*policy.PolicyFileSnapshot, int, error) {
- regFence, ok := s.reg.Fence(subj)
- if !ok {
+// image of the subject's vault file. regFence/registered are the caller's
+// pre-read of the registration (lock order: registry → s.mu, see
+// applyVaultExport). Callers hold s.mu.
+func (s *fastSandboxPolicyServer) exportSourceImageLocked(subj subject.Subject, fence subject.Fencing, regFence subject.Fencing, registered bool) (*policy.PolicyFileSnapshot, int, error) {
+ if !registered {
return nil, http.StatusNotFound, fmt.Errorf("vault export for unknown subject %s", subj)
}
if !regFence.Matches(fence) {
@@ -221,8 +228,8 @@ func (s *fastSandboxPolicyServer) applyVaultImport(env *actionhandler.Envelope)
subj := subject.FromSandboxUID(env.Sandbox.UID)
fence := subject.FromRevision(env)
source := strings.TrimSpace(env.Snapshot.Path)
- if s.vaultFiles.reservesPath(source) {
- return http.StatusBadRequest, fmt.Errorf("vault bundle source %q must not live under the vault persistence directory", source)
+ if s.reservesStatePath(source) {
+ return http.StatusBadRequest, fmt.Errorf("vault bundle source %q must not live under a per-subject state directory", source)
}
// Read before locking; a missing/oversized/unreadable source is
@@ -247,6 +254,9 @@ func (s *fastSandboxPolicyServer) applyVaultImport(env *actionhandler.Envelope)
return http.StatusInternalServerError, fmt.Errorf("vault bundle re-stamp failed: %w", err)
}
+ // Registry reads before s.mu (lock order: registry → s.mu, see
+ // handleVaultRelayWrite).
+ effective := s.reg.EffectivePolicy(subj)
s.mu.Lock()
defer s.mu.Unlock()
// The latch freezes every vault file mutation; only a restart
@@ -275,7 +285,7 @@ func (s *fastSandboxPolicyServer) applyVaultImport(env *actionhandler.Envelope)
}
// Validate before any durable effect — a permanently broken bundle
// is a 400 with last-good kept, never half-open.
- if err := validateImportedVault(vf, body, s.vaultValidationPolicyLocked(subj)); err != nil {
+ if err := validateImportedVault(vf, body, s.vaultValidationPolicyLocked(effective, subj)); err != nil {
return http.StatusBadRequest, fmt.Errorf("vault bundle rejected: %v", err)
}
}
@@ -289,7 +299,7 @@ func (s *fastSandboxPolicyServer) applyVaultImport(env *actionhandler.Envelope)
// Before SET_BINDING: registration reloads and validates.
return s.vaultFiles.unseal(subj)
}
- return s.installImportedVaultLocked(subj, fence, vf, body)
+ return s.installImportedVaultLocked(subj, fence, vf, body, effective)
},
)
if err != nil {
@@ -314,8 +324,10 @@ func (s *fastSandboxPolicyServer) applyVaultImport(env *actionhandler.Envelope)
// per-subject slice under the restored fence, replacing any stale entry.
// The slice is only replaced once Reload accepts — a failure leaves the live
// entry untouched while the commit protocol rolls the file back.
-func (s *fastSandboxPolicyServer) installImportedVaultLocked(subj subject.Subject, fence subject.Fencing, vf credentialvault.VaultFile, body credentialvault.VaultFileBody) error {
- pol := s.vaultValidationPolicyLocked(subj)
+// effective is the caller's pre-read of the subject's effective policy
+// (lock order: registry → s.mu, see handleVaultRelayWrite).
+func (s *fastSandboxPolicyServer) installImportedVaultLocked(subj subject.Subject, fence subject.Fencing, vf credentialvault.VaultFile, body credentialvault.VaultFileBody, effective *policy.NetworkPolicy) error {
+ pol := s.vaultValidationPolicyLocked(effective, subj)
entry, ok := s.vaults[subj]
if !ok || entry.fence != fence {
entry = vaultEntry{store: credentialvault.NewStore(nil, func() bool { return true }), fence: fence}
diff --git a/components/egress/pkg/policy/policyfile.go b/components/egress/pkg/policy/policyfile.go
index afad8cf2c..41896d311 100644
--- a/components/egress/pkg/policy/policyfile.go
+++ b/components/egress/pkg/policy/policyfile.go
@@ -93,6 +93,11 @@ func EncodePolicyFile(incarnation, attachmentID string, revision int64, body []b
if err != nil {
return nil, fmt.Errorf("encode policy file envelope: %w", err)
}
+ if len(out)+1 > MaxEncodedPolicyFileBytes {
+ // Enforce the cap at encode time: an oversized fence field or body
+ // must not produce a file our own importer would reject.
+ return nil, fmt.Errorf("encoded policy file exceeds %d bytes", MaxEncodedPolicyFileBytes)
+ }
return append(out, '\n'), nil
}
diff --git a/components/egress/tests/smoke-fast-sandbox.sh b/components/egress/tests/smoke-fast-sandbox.sh
index 0c5833390..9267c899c 100755
--- a/components/egress/tests/smoke-fast-sandbox.sh
+++ b/components/egress/tests/smoke-fast-sandbox.sh
@@ -183,6 +183,20 @@ bind_and_ready() {
lifecycle_hook "${uid}" "${runtime}" "${att}" sandbox.data-plane-ready
}
+# relay_put_policy <uid> <input>: a runtime policy change through the
+# handler-relay surface (/policy/{uid}). The binding input only ever seeds a
+# fresh subject's policy — every later change rides this surface, which
+# applies in place on an active subject and holds pending on a denying one.
+relay_put_policy() {
+ local uid="$1" input="$2" code
+ code="$(curl -s -o /dev/null -w '%{http_code}' -XPUT \
+ "http://127.0.0.1:${POLICY_PORT}/policy/${uid}" -d "{\"policy\":${input}}")"
+ case "${code}" in
+ 200|201) ;;
+ *) fail "relay PUT /policy/${uid} failed: ${code}" ;;
+ esac
+}
+
dns_query() {
# dns_query <netns-or-host> <name>
local where="$1" name="$2"
@@ -374,10 +388,13 @@ expect_rcode osb-sandbox-a allow.test 3
pass "deny-first registered (nft + NXDOMAIN, policy pending)"
###############################################################################
-info "Test 2: data-plane-ready activates the subject (dns+nft)"
-set_binding a 10.10.0.5 2 runtime-a att-a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"10.99.0.2"}]}'
+info "Test 2: data-plane-ready activates the subject; the relay owns runtime changes"
lifecycle_hook a runtime-a att-a sandbox.runtime-ready
lifecycle_hook a runtime-a att-a sandbox.data-plane-ready
+# The binding input seeded deny at spec 1; the allow policy is a RUNTIME
+# change through the handler-relay surface, applied in place on the active
+# subject. A same-fence SET_BINDING with a new input would be superseded.
+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"10.99.0.2"}]}'
expect_answers osb-sandbox-a allow.test 1.1.1.1
expect_answers osb-sandbox-a other.test 1.1.1.2
expect_rcode osb-sandbox-a nope.test 3
@@ -410,8 +427,8 @@ fi
pass "forward drop (default deny)"
###############################################################################
-info "Test 4: deny CIDR overrides allow (nft layer, in-place update)"
-set_binding a 10.10.0.5 3 runtime-a att-a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"deny","target":"10.99.0.0/24"}]}'
+info "Test 4: deny CIDR overrides allow (nft layer, relay update)"
+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"deny","target":"10.99.0.0/24"}]}'
if ip netns exec osb-sandbox-a curl -s -m 3 -o /dev/null http://10.99.0.2:8080/ 2>/dev/null; then
fail "deny CIDR must block 10.99.0.2"
fi
@@ -430,8 +447,11 @@ http_code="$(curl -s -o /dev/null -w '%{http_code}' -XPOST \
"http://127.0.0.1:${POLICY_PORT}/credential-vault" -d '{}')"
[ "${http_code}" = "404" ] || fail "superseded /credential-vault must be gone (404), got ${http_code}"
http_code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${POLICY_PORT}/policy")"
-[ "${http_code}" = "404" ] || fail "superseded /policy must be gone (404), got ${http_code}"
-pass "superseded proxy-route surfaces removed (policy rides the binding only)"
+case "${http_code}" in
+ 301|307|308) ;; # redirects into the /policy/{uid} relay subtree
+ *) fail "bare /policy must redirect into the relay subtree, got ${http_code}" ;;
+esac
+pass "superseded proxy-route surfaces removed (relay surfaces only)"
set_up_netns b 10.10.0.6
bind_and_ready b 10.10.0.6 1 runtime-b att-b '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"}]}'
@@ -592,8 +612,10 @@ else
# rejected), so the sandbox curls the ext netns with an explicit Host
# header; the policy keeps allowing the real daddr 10.99.0.2 so the nft
# layers pass the packet.
- set_binding a 10.10.0.5 8 runtime-a-2 att-a-2 '{"defaultAction":"deny","egress":[{"action":"allow","target":"10.99.0.2"},{"action":"allow","target":"ext.test"}]}'
lifecycle_hook a runtime-a-2 att-a-2 sandbox.runtime-ready
+ # A relay write while denying is held to the data-plane-ready barrier
+ # (pending-policy validation semantics, same as the vault).
+ relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"10.99.0.2"},{"action":"allow","target":"ext.test"}]}'
lifecycle_hook a runtime-a-2 att-a-2 sandbox.data-plane-ready
code="$(curl -s -o /dev/null -w '%{http_code}' --unix-socket "${ACTIVE_SOCK}" 'http://localhost/credential-vault/_active?clientIp=10.10.0.5')"
[ "${code}" = "404" ] || fail "active API: subject a has no vault yet (relay not wired), got ${code}"
@@ -803,10 +825,9 @@ else
# Default-allow subject: still dropped — the containment is profile-wide,
# above every per-subject rule. Control first: non-proxy traffic flows, so
# only the proxy endpoint is unreachable. Subject a is still active, so
- # this in-place binding update applies immediately — no data-plane-ready
- # replay (the protocol sends no Hook for updates; a redundant one 409s on
- # the fail-closed "no pending policy" guard).
- set_binding a 10.10.0.5 11 runtime-a-2 att-a-2 '{"defaultAction":"allow"}'
+ # this relay update applies immediately (no data-plane-ready replay —
+ # relay writes apply in place on an active subject).
+ relay_put_policy a '{"defaultAction":"allow"}'
out="$(ip netns exec osb-sandbox-a curl -s -m 5 -H 'Host: ext.test' http://10.99.0.2:8080/)"
echo "${out}" | grep -qi "client=10.10.0.5" || fail "default-allow control request failed (subject inactive?); got: ${out}"
if ip netns exec osb-sandbox-a curl -s -m 3 -o /dev/null --proxytunnel -x http://10.99.0.2:3128 http://10.99.0.2:8080/ 2>/dev/null; then