Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(egress): close the rebind/relay activation race and harden state persistence
高然06f1b75c6338
diff --git a/components/egress/fastsandbox.go b/components/egress/fastsandbox.goindex ea5f6b2c2..39ebd0459 100644--- a/components/egress/fastsandbox.go+++ b/components/egress/fastsandbox.go@@ -43,6 +43,7 @@ import ("net/http""net/netip""os"+ "path/filepath""strings""time"@@ -141,6 +142,23 @@ func runFastSandboxProfile(ctx context.Context, upstreamSpec *mitmproxy.Upstreamif policyDir == "" {log.Fatalf("fast-sandbox policy persistence: OPENSANDBOX_EGRESS_POLICY_DIR is required (the recorded policy is egress-held state)")}+ // Register the configured state-directory basenames BEFORE either+ // constructor runs: the volume-root restriction must recognize the+ // configured sibling as ours even on custom-named deployments.+ registerStateDirName(policyDir)+ if vaultDir := strings.TrimSpace(os.Getenv(constants.EnvEgressVaultDir)); vaultDir != "" {+ registerStateDirName(vaultDir)+ }+ // The two state directories are siblings holding same-named per-subject+ // files (<uid>.json) and seal markers: aliasing them would make each+ // persister clobber the other's state (a restart-time rehydrate of one+ // kind quarantines the other kind's file). Reject before any state is+ // written.+ if vaultDir := strings.TrimSpace(os.Getenv(constants.EnvEgressVaultDir)); vaultDir != "" {+ if sameStateDir(vaultDir, policyDir) {+ log.Fatalf("fast-sandbox policy persistence: OPENSANDBOX_EGRESS_POLICY_DIR must differ from OPENSANDBOX_EGRESS_VAULT_DIR (both resolve to %s)", policyDir)+ }+ }pp, err := newPolicyPersistence(policyDir)if err != nil {log.Fatalf("fast-sandbox policy persistence: %v", err)@@ -299,6 +317,18 @@ func runFastSandboxProfile(ctx context.Context, upstreamSpec *mitmproxy.Upstream_ = os.Stderr.Sync()}+// sameStateDir reports whether two state-directory paths resolve to the+// same directory (lexical cleaning plus best-effort symlink resolution).+func sameStateDir(a, b string) bool {+ a, b = filepath.Clean(a), filepath.Clean(b)+ if a == b {+ return true+ }+ ra, erra := filepath.EvalSymlinks(a)+ rb, errb := filepath.EvalSymlinks(b)+ return erra == nil && errb == nil && ra == rb+}+// fastSandboxNftOptions assembles the fast-sandbox nft options: the shared// DoH-443 blocking env (OPENSANDBOX_EGRESS_BLOCK_DOH_443 strict all-443 drop// when the blocklist is empty + OPENSANDBOX_EGRESS_DOH_BLOCKLISTdiff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.goindex 17e20a83f..29eb1953b 100644--- a/components/egress/fastsandbox_actions.go+++ b/components/egress/fastsandbox_actions.go@@ -197,26 +197,29 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (return http.StatusBadRequest, fmt.Errorf("invalid binding input policy: %v", err)}}+ // The WHOLE mutation runs under policyCommitMu — rebind purge,+ // registration, and the seed/apply are one serialized unit against the+ // relay commits and the hook activation. Without this, a+ // predecessor-identity relay write holding policyCommitMu could+ // straddle the rebind's fence flip: its apply closure passes the fence+ // checks pre-registration, then its reg.ApplyPolicy lands on the+ // SUCCESSOR's registry entry after RegisterAndEnforce — the successor+ // would go active under the predecessor's policy. Lock order:+ // policyCommitMu → registry → s.mu (matches the commit paths).+ s.policyCommitMu.Lock()+ defer s.policyCommitMu.Unlock()+// A fence change is a rebind: discard ALL prior per-subject state. The// registry drops its entry inside RegisterAndEnforce; the vault slice// and the recorded policy must go too (memory + persisted files), or the// old instance's credentials and policy would survive into the- // successor's flows. Same-UID actions serialize per subject; the vault- // and policy relay writes serialize against the same state — the purge- // runs under policyCommitMu so an in-flight relay commit against the- // OLD fence cannot resurrect a purged record after the rebind (its- // commit-time fence re-check aborts it). Callers are never under the- // registry write lock here, so taking policyCommitMu is safe (the- // takeover path, which IS under the registry lock, purges without it —- // its relay races are fenced out by the commit-time re-check).+ // successor's flows.if old, ok := s.reg.Fence(subj); ok && !old.Matches(fence) {- s.policyCommitMu.Lock()s.mu.Lock()s.dropVault(subj)delete(s.policies, subj)s.purgePolicyFile(subj)s.mu.Unlock()- s.policyCommitMu.Unlock()log.Infof("subject %s: rebind (%s -> %s), vault slice and policy record purged", subj, old.RuntimeInstanceID, fence.RuntimeInstanceID)}// Egress-restart recovery (OSEP-0022 stage 2): a subject with no live@@ -244,7 +247,7 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (// binding's policy to a policy-less subject) and purge the recorded// policy, so deny-first is also the durable state.s.clearPendingPolicy(subj)- if err := s.clearPolicyRecord(subj); err != nil {+ if err := s.clearPolicyRecordLocked(subj); err != nil {// The durable removal did not confirm: fail the action so the// Fastlet's retry re-runs the purge/tombstone instead of acking// a removal whose file may still carry the removed policy.@@ -260,7 +263,7 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (}// Install the binding policy (or honor the file-backed record that// supersedes it) through the policy commit protocol.- if err := s.applyBindingPolicy(subj, fence, pol, state == subject.StateActive); err != nil {+ if err := s.applyBindingPolicyLocked(subj, fence, pol, state == subject.StateActive); err != nil {log.Errorf("subject %s: binding policy commit failed: %v", subj, err)return http.StatusInternalServerError, fmt.Errorf("binding policy commit: %w", err)}diff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.goindex 5aee46b11..75cb3ddda 100644--- a/components/egress/fastsandbox_policy.go+++ b/components/egress/fastsandbox_policy.go@@ -194,6 +194,10 @@ func (s *fastSandboxPolicyServer) handlePolicyRelayWrite(w http.ResponseWriter,http.Error(w, "invalid policy: "+err.Error(), http.StatusBadRequest)return}+ if len(doc.Policy) > policy.MaxPolicyBodyBytes {+ http.Error(w, fmt.Sprintf("policy body exceeds %d bytes", policy.MaxPolicyBodyBytes), http.StatusBadRequest)+ return+ }expected, perr := relayPolicyPrecondition(&doc)if perr != nil {http.Error(w, perr.Error(), http.StatusBadRequest)@@ -444,6 +448,6 @@ func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subjhttp.Error(w, "policy persistence failure", status)return}- log.Infof("subject %s: policy deleted (deny-first), record purged", subj)+ log.Infof("subject %s: policy deleted (deny tombstone enforced and persisted at revision %d)", subj, newRev)w.WriteHeader(http.StatusNoContent)}diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.goindex a6ef67256..c164c476b 100644--- a/components/egress/fastsandbox_policy_persist.go+++ b/components/egress/fastsandbox_policy_persist.go@@ -172,8 +172,9 @@ func (p *policyPersistence) reservesPath(path string) bool {return reservesPathUnder(p.dir, path)}-// SetPolicyFiles wires persistence under the relay handlers; nil (the-// default) keeps the policy memory-only.+// SetPolicyFiles wires persistence under the relay handlers. Policy+// persistence is REQUIRED in the fast-sandbox profile (startup fails+// without it), so every assembled server carries a store.func (s *fastSandboxPolicyServer) SetPolicyFiles(pp *policyPersistence) {s.mu.Lock()defer s.mu.Unlock()@@ -300,10 +301,9 @@ func (s *fastSandboxPolicyServer) policyHealthIssue() string {// away policy. A tombstone persist failure latches recovery (writes frozen// until restart). Serialized with the relay commits (policyCommitMu) so an// in-flight write against the removed binding cannot recreate the record-// after the purge; callers are never under the registry write lock.-func (s *fastSandboxPolicyServer) clearPolicyRecord(subj subject.Subject) error {- s.policyCommitMu.Lock()- defer s.policyCommitMu.Unlock()+// after the purge. The caller holds policyCommitMu (the SET_BINDING+// mutation hold); never call under the registry write lock.+func (s *fastSandboxPolicyServer) clearPolicyRecordLocked(subj subject.Subject) error {s.mu.Lock()entry := s.policies[subj]pp := s.policyFiles@@ -397,7 +397,17 @@ func (s *fastSandboxPolicyServer) rehydratePolicyReload(subj subject.Subject, fereturn}if !snap.Exists() {- // Fresh subject: nothing persisted, the binding input seeds.+ // Fresh subject: nothing persisted, the binding input seeds. A+ // durable seal marker (an export that sealed a file-less subject)+ // must still gate the relay ops — the seed entry carries it.+ if s.policyFiles.isSealed(subj) {+ s.mu.Lock()+ s.policies[subj] = &policyEntry{fence: fence, sealed: true}+ s.pendingPolicies[subj] = policy.DefaultDenyPolicy()+ s.mu.Unlock()+ log.Infof("subject %s: seal restored from marker on a file-less subject (binding input seeds pending default-deny)", subj)+ return+ }return}pf, body, decErr := policy.DecodePolicyFile(snap.Data())@@ -449,17 +459,15 @@ func (s *fastSandboxPolicyServer) rehydratePolicyReload(subj subject.Subject, felog.Infof("subject %s: policy reloaded from file (revision %d, incarnation %s); binding input superseded", subj, pf.Revision, pf.Incarnation)}-// applyBindingPolicy seeds the binding's policy for a freshly registered-// subject (record absent) through the same commit protocol as the relay-// writes. A subject with an existing record never reads its binding input-// again: the recorded policy — seeded once, then owned by the relay writes+// applyBindingPolicyLocked seeds the binding's policy for a freshly+// registered subject (record absent) through the same commit protocol as+// the relay writes. The caller holds policyCommitMu (the SET_BINDING+// mutation hold). A subject with an existing record never reads its+// binding input again: the recorded policy — seeded once, then owned by the relay writes// and the snapshot transfer — is authoritative, exactly like the vault// slice. Binding replays (at-least-once delivery, egress restarts, and the// superseded CRD-driven update path) are therefore policy no-ops.-func (s *fastSandboxPolicyServer) applyBindingPolicy(subj subject.Subject, fence subject.Fencing, pol *policy.NetworkPolicy, stateActive bool) error {- s.policyCommitMu.Lock()- defer s.policyCommitMu.Unlock()-+func (s *fastSandboxPolicyServer) applyBindingPolicyLocked(subj subject.Subject, fence subject.Fencing, pol *policy.NetworkPolicy, stateActive bool) error {s.mu.Lock()entry := s.policies[subj]if entry != nil && entry.fence != fence {diff --git a/components/egress/fastsandbox_policy_relay_test.go b/components/egress/fastsandbox_policy_relay_test.goindex c0efe8632..9e2deeb85 100644--- a/components/egress/fastsandbox_policy_relay_test.go+++ b/components/egress/fastsandbox_policy_relay_test.go@@ -15,17 +15,21 @@package mainimport (+ "context""encoding/json""net/http""net/http/httptest"+ "net/netip""strconv""strings""testing"+ "time""github.com/stretchr/testify/assert""github.com/stretchr/testify/require""github.com/alibaba/opensandbox/egress/pkg/constants"+ "github.com/alibaba/opensandbox/egress/pkg/policy""github.com/alibaba/opensandbox/egress/pkg/subject")@@ -301,6 +305,120 @@ func TestPolicyRelayPutReplacesStalePendingTombstone(t *testing.T) {assert.Equal(t, "deny", reg_UserEvaluate(srv, s, "api.example.com"))}+// hookNft forces an interleaving at the relay apply's enforcement step: the+// injected callback runs while the relay commit holds policyCommitMu, which+// is exactly the window a concurrent rebind would race through.+type hookNft struct {+ fakeNft+ onPolicy func()+}++func (h *hookNft) ApplyPolicy(ctx context.Context, s subject.Subject, pol *policy.NetworkPolicy) error {+ if h.onPolicy != nil {+ fn := h.onPolicy+ h.onPolicy = nil+ fn()+ }+ return h.fakeNft.ApplyPolicy(ctx, s, pol)+}++// TestRelayPutRacingRebindNeverActivatesPredecessorPolicy (review H3): a+// relay PUT for the PREDECESSOR identity whose apply lands after the+// rebind's registration must not activate the successor under the+// predecessor's policy. The rebind runs on another goroutine from inside+// the relay's enforcement step, forcing the fence flip into the apply+// window; the fixed serialization (the whole SET_BINDING mutation holds+// policyCommitMu) makes the rebind wait, the relay commit completes against+// the old instance, and the purge discards it.+func TestRelayPutRacingRebindNeverActivatesPredecessorPolicy(t *testing.T) {+ reg := subject.NewRegistry(nil, nil)+ hnft := &hookNft{}+ srv := newFastSandboxPolicyServer(context.Background(), reg, hnft)+ srv.dnsRedirectInstall = func(netip.Addr, int) error { return nil }+ srv.dnsRedirectRemove = func() error { return nil }+ pp, err := newPolicyPersistence(t.TempDir())+ require.NoError(t, err)+ files := make(map[string]*fakeVaultFile)+ pp.newFile = func(path string) (vaultFileStore, error) {+ f, ok := files[path]+ if !ok {+ f = newFakeVaultFile(path)+ files[path] = f+ }+ return f, nil+ }+ srv.SetPolicyFiles(pp)++ const uid, ip = "race-u", "10.0.0.5"+ const oldPolicy = `{"defaultAction":"deny","egress":[{"action":"allow","target":"old.com"}]}`+ const newPolicy = `{"defaultAction":"deny","egress":[{"action":"allow","target":"new.com"}]}`++ // Generation one: old fence, active with old policy.+ rec := doAction(t, srv, bindBody(t, uid, ip, testFenceR, testFenceA, 1, strPtr(oldPolicy)))+ require.Equal(t, http.StatusOK, rec.Code)+ rec = doAction(t, srv, hookBody(t, uid, testFenceR, testFenceA, constants.HookRuntimeReady))+ require.Equal(t, http.StatusOK, rec.Code)+ rec = doAction(t, srv, hookBody(t, uid, testFenceR, testFenceA, constants.HookDataPlaneReady))+ require.Equal(t, http.StatusOK, rec.Code)+ s := subject.FromSandboxUID(uid)+ require.Equal(t, subject.StateActive, mustState(reg.Get(s)))++ // A queued relay PUT for the OLD identity: its apply BLOCKS inside the+ // enforcement step while the rebind (same UID, new fence) completes on+ // another goroutine — forcing the fence flip into the apply window. With+ // the serialization fix the rebind cannot proceed (policyCommitMu is+ // held by this very commit), so the block times out and the rebind runs+ // to completion afterwards; either way the successor must converge to+ // its own policy.+ started := make(chan struct{})+ done := make(chan struct{})+ hnft.onPolicy = func() {+ close(started)+ select {+ case <-done:+ case <-time.After(2 * time.Second):+ // Serialized: the rebind runs after this commit.+ }+ }+ go func() {+ <-started+ defer close(done)+ r := doAction(t, srv, bindBody(t, uid, ip, "runtime-2", "attachment-2", 2, strPtr(newPolicy)))+ assert.Equal(t, http.StatusOK, r.Code)+ }()++ // The relay write itself passes its guards (pre-read fence = old) and+ // must NOT leak its policy into the successor.+ rec = doPolicy(t, srv, http.MethodPut, uid, policyPutDoc(oldPolicy, 1, false, true), "")+ require.Equal(t, http.StatusOK, rec.Code)++ select {+ case <-done:+ case <-time.After(10 * time.Second):+ t.Fatal("the rebind never completed (serialization deadlock?)")+ }++ // The successor is deny-first with ITS OWN binding input — never active+ // under the predecessor's policy, whichever way the interleaving went.+ state, ok := reg.Get(s)+ require.True(t, ok)+ if state == subject.StateActive {+ // Activated only legitimately: the successor's own policy enforced.+ assert.Equal(t, "allow", reg.UserPolicy(s).Evaluate("new.com"))+ assert.Equal(t, "deny", reg.UserPolicy(s).Evaluate("old.com"))+ } else {+ assert.Equal(t, subject.StateDenying, state)+ assert.Nil(t, reg.EffectivePolicy(s), "deny-first until the successor's own data-plane-ready")+ }++ // The successor's own lifecycle completes with its own policy.+ rec = doAction(t, srv, hookBody(t, uid, "runtime-2", "attachment-2", constants.HookDataPlaneReady))+ require.Equal(t, http.StatusOK, rec.Code)+ require.Equal(t, subject.StateActive, mustState(reg.Get(s)))+ assert.Equal(t, "allow", reg.UserPolicy(s).Evaluate("new.com"))+ assert.Equal(t, "deny", reg.UserPolicy(s).Evaluate("old.com"), "the predecessor's policy never reaches the successor")+}+// reg_UserEvaluate evaluates the subject's recorded user policy (test aid to// keep assertions readable).func reg_UserEvaluate(srv *fastSandboxPolicyServer, s subject.Subject, domain string) string {diff --git a/components/egress/fastsandbox_policy_snapshot.go b/components/egress/fastsandbox_policy_snapshot.goindex 5a14c420c..07b6f78d0 100644--- a/components/egress/fastsandbox_policy_snapshot.go+++ b/components/egress/fastsandbox_policy_snapshot.go@@ -170,12 +170,10 @@ func (s *fastSandboxPolicyServer) policyExportSourceImageLocked(subj subject.Sub}if !s.fenceMatchesLocked(subj, fence) {// Purged between the pre-read and the lock — the stale envelope- // lost the race with a rebind.+ // lost the race with a rebind. The registered-fence mismatch above+ // already covers the plain stale case.return nil, http.StatusNotFound, fmt.Errorf("policy export for unknown subject %s", subj)}- if !regFence.Matches(fence) {- return nil, http.StatusConflict, fmt.Errorf("stale policy export (fence mismatch) for subject %s", subj)- }if recovery := s.policyRecovery.Load(); recovery != nil {return nil, http.StatusServiceUnavailable, fmt.Errorf("policy persistence recovery required: %s", recovery.reason)}@@ -311,7 +309,17 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)s.mu.Unlock()return s.policyFiles.unseal(subj)}- return s.installImportedPolicyLocked(subj, fence, pf.Revision, pol, sealed)+ err := s.installImportedPolicyLocked(subj, fence, pf.Revision, pol, sealed)+ if err != nil && sealed {+ // The unseal already happened; restore the durable seal so+ // disk and memory agree the bundle is still the truth (the+ // commit protocol rolls the file back to the sealed image;+ // the in-memory entry was never modified).+ if sealErr := s.policyFiles.seal(subj); sealErr != nil {+ log.Errorf("subject %s: policy seal marker restore failed after import apply failure: %v", subj, sealErr)+ }+ }+ return err},)if err != nil {diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.goindex c9a6378fd..855e8599a 100644--- a/components/egress/fastsandbox_server.go+++ b/components/egress/fastsandbox_server.go@@ -187,6 +187,10 @@ func (s *fastSandboxPolicyServer) mitmRedirectRebuild() error {// per-subject gateway REDIRECT forwards sandbox DNS here.const fastSandboxDNSProxyPort = 15353+// policyApplyTimeout bounds one kernel policy transaction on every path+// that runs under policyCommitMu (see applyPolicy).+const policyApplyTimeout = 10 * time.Second+// installGatewayDNSRedirect records a subject's gateway and installs (once)// the prerouting REDIRECT for it. Idempotent under at-least-once SET_BINDING// delivery: a duplicate registration (or a retried deny-first install) is a@@ -384,7 +388,14 @@ func (s *fastSandboxPolicyServer) applyPolicy(subj subject.Subject, pol *policy.s.policyMu.Lock()defer s.policyMu.Unlock()eff := s.reg.EffectiveOf(pol)- nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)+ // policyApplyTimeout bounds the kernel transaction for every caller+ // that runs under policyCommitMu (relay commits, the SET_BINDING seed,+ // and the data-plane-ready hook all serialize on it): an unbounded+ // deadline would let one slow nft apply head-of-line-block every other+ // subject's policy write past the listener's write timeout. The+ // registration deny-first install is NOT under that serialization and+ // keeps its own 30s budget.+ nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)defer cancel()if err := s.nft.ApplyPolicy(nftCtx, subj, eff); err != nil {return fmt.Errorf("nft policy apply: %w", err)@@ -636,7 +647,7 @@ func (s *fastSandboxPolicyServer) pendingPolicy(subj subject.Subject) *policy.Ne// sandbox must be fully blocked again). nft commits before the registry// state, so the transition stays fail-closed.func (s *fastSandboxPolicyServer) revertToDenyFirst(subj subject.Subject, att actionhandler.NetworkAttachment) error {- nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)+ nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)defer cancel()if err := s.nft.ApplyDenyFirst(nftCtx, subj, att); err != nil {return fmt.Errorf("revert to deny-first: %w", err)diff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.goindex 111be5b2e..17b8ef004 100644--- a/components/egress/fastsandbox_vault.go+++ b/components/egress/fastsandbox_vault.go@@ -134,19 +134,11 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, rhttp.Error(w, "expectedIncarnation mismatch", http.StatusPreconditionFailed)return}- effective := s.reg.EffectivePolicy(subj)- if effective == nil {- // Pre-read the pending policy and merge the always overlay BEFORE- // s.mu (registry → s.mu order): a vault op held to the- // data-plane-ready barrier must validate against the SAME policy- // the hook will enforce, operator always-rules included.- s.mu.Lock()- pending := s.pendingPolicies[subj]- s.mu.Unlock()- if pending != nil {- effective = s.reg.EffectiveOf(pending)- }- }+ // Pre-compute the validation policy BEFORE s.mu (registry → s.mu+ // order): a vault op held to the data-plane-ready barrier must validate+ // against the SAME policy the hook will enforce, operator always-rules+ // included.+ effective := s.vaultValidationPolicy(subj)// The rest of the mutation holds s.mu, serializing per-subject writers// and fencing rebind/terminal purges. The validation policy was fullydiff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.goindex 787ac16f1..5432da9ee 100644--- a/components/egress/fastsandbox_vault_persist.go+++ b/components/egress/fastsandbox_vault_persist.go@@ -114,7 +114,7 @@ func prepareStateDir(dir, noun string) error {}// The volume root must pass the atomic machinery's trusted-ancestor// check; kubelet creates emptyDir roots 0777, which fails it.- if err := restrictVolumeRoot(dir); err != nil {+ if err := restrictVolumeRoot(dir, noun); err != nil {return err}if err := sweepStateTempFiles(dir, noun); err != nil {@@ -143,28 +143,36 @@ func sweepStateTempFiles(dir, noun string) error {return nil}-// knownStateDirNames are the per-subject state directories that may share-// one egress volume root (the vault and the recorded policy). A sibling-// state directory is ours, not a foreign entry — without this, whichever-// directory initializes second would see the first as foreign, skip the-// volume-root restriction, and the atomic machinery would then reject every-// file op on the untrusted ancestor.-var knownStateDirNames = map[string]bool{+// stateDirNames are the basenames of the per-subject state directories that+// share one egress volume root, registered at assembly from the CONFIGURED+// paths (the basename is deployment-owned, so it cannot be hardcoded; the+// conventional names stay pre-seeded). restrictVolumeRoot treats registered+// siblings as ours — without this, whichever directory initializes second+// would see the first as foreign, skip the volume-root restriction, and the+// atomic machinery would then reject every file op on the untrusted+// ancestor.+var stateDirNames = map[string]bool{"vault": true,"policy": true,}-// restrictVolumeRoot clears group/world-write bits from the vault volume-// root (the vault directory's parent): kubelet creates emptyDir roots 0777,+// registerStateDirName records a configured state directory basename so the+// volume-root restriction recognizes it as a sibling.+func registerStateDirName(dir string) {+ stateDirNames[filepath.Base(filepath.Clean(dir))] = true+}++// restrictVolumeRoot clears group/world-write bits from the state volume+// root (the state directory's parent): kubelet creates emptyDir roots 0777,// which the atomic machinery rejects as an untrusted ancestor. Sibling-// state directories (knownStateDirNames) belong to us; a root holding any+// state directories (stateDirNames) belong to us; a root holding any// other entry is left untouched — that is the manifest's job — and// already-restrictive roots are a no-op.-func restrictVolumeRoot(dir string) error {+func restrictVolumeRoot(dir, noun string) error {parent := filepath.Dir(dir)info, err := os.Stat(parent)if err != nil {- return fmt.Errorf("inspect vault volume root %s: %w", parent, err)+ return fmt.Errorf("inspect %s volume root %s: %w", noun, parent, err)}if !info.IsDir() || info.Mode().Perm()&0o022 == 0 {return nil@@ -178,16 +186,16 @@ func restrictVolumeRoot(dir string) error {}base := filepath.Base(dir)for _, entry := range entries {- if entry.Name() == base || (entry.IsDir() && knownStateDirNames[entry.Name()]) {+ if entry.Name() == base || (entry.IsDir() && stateDirNames[entry.Name()]) {continue}- log.Infof("vault persistence: volume root %s holds foreign entry %q; not restricting (fix permissions in the manifest)", parent, entry.Name())+ log.Infof("%s persistence: volume root %s holds foreign entry %q; not restricting (fix permissions in the manifest)", noun, parent, entry.Name())return nil}if err := os.Chmod(parent, info.Mode().Perm()&^0o022); err != nil {- return fmt.Errorf("restrict vault volume root %s: %w", parent, err)+ return fmt.Errorf("restrict %s volume root %s: %w", noun, parent, err)}- log.Infof("vault persistence: restricted group/world-writable volume root %s (kubelet emptyDir default)", parent)+ log.Infof("%s persistence: restricted group/world-writable volume root %s (kubelet emptyDir default)", noun, parent)return nil}diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.goindex 1d5e441e3..52dcee10f 100644--- a/components/egress/fastsandbox_vault_snapshot.go+++ b/components/egress/fastsandbox_vault_snapshot.go@@ -179,12 +179,10 @@ func (s *fastSandboxPolicyServer) exportSourceImageLocked(subj subject.Subject,}if !s.fenceMatchesLocked(subj, fence) {// Purged between the pre-read and the lock — the stale envelope- // lost the race with a rebind.+ // lost the race with a rebind. The registered-fence mismatch above+ // already covers the plain stale case.return nil, http.StatusNotFound, fmt.Errorf("vault export for unknown subject %s", subj)}- if !regFence.Matches(fence) {- return nil, http.StatusConflict, fmt.Errorf("stale vault export (fence mismatch) for subject %s", subj)- }if recovery := s.vaultRecovery.Load(); recovery != nil {return nil, http.StatusServiceUnavailable, fmt.Errorf("vault persistence recovery required: %s", recovery.reason)}diff --git a/components/egress/tests/e2e-fast-sandbox-firecracker.sh b/components/egress/tests/e2e-fast-sandbox-firecracker.shindex b6a39ed35..eefc30c54 100755--- a/components/egress/tests/e2e-fast-sandbox-firecracker.sh+++ b/components/egress/tests/e2e-fast-sandbox-firecracker.sh@@ -408,6 +408,7 @@ EGRESS_CT_ID="$(docker run -d --name "${EGRESS_CT}" \-v "${MITM_HOME}:/var/lib/mitmproxy" \-v "${SEED_DIR}:/var/egress/mitm-ca-seed:ro" \-e OPENSANDBOX_EGRESS_PROFILE=fast-sandbox \+ -e OPENSANDBOX_EGRESS_POLICY_DIR=/var/lib/opensandbox-egress/policy \-e OPENSANDBOX_EGRESS_DNS_UPSTREAM="${BRIDGE_GW}:${DNS_UPSTREAM_PORT}" \-e OPENSANDBOX_EGRESS_DNS_UPSTREAM_PROBE=allow.test \-e OPENSANDBOX_EGRESS_HTTP_ADDR="127.0.0.1:${POLICY_PORT}" \diff --git a/components/egress/tests/smoke-fast-sandbox.sh b/components/egress/tests/smoke-fast-sandbox.shindex b620d3ec8..d94808b63 100755--- a/components/egress/tests/smoke-fast-sandbox.sh+++ b/components/egress/tests/smoke-fast-sandbox.sh@@ -460,6 +460,8 @@ case "${http_code}" in301|307|308) ;; # redirects into the /policy/{uid} relay subtree*) fail "bare /policy must redirect into the relay subtree, got ${http_code}" ;;esac+location="$(curl -s -o /dev/null -w '%{header_json}' "http://127.0.0.1:${POLICY_PORT}/policy" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("location","")[0])' 2>/dev/null || true)"+[ "${location}" = "/policy/" ] || fail "bare /policy redirect target must be /policy/, got ${location:-none}"pass "superseded proxy-route surfaces removed (relay surfaces only)"set_up_netns b 10.10.0.6@@ -488,6 +490,11 @@ info "Test 9: restart recovery (reset -> action replay -> active)"# Refresh the DNS-learned dyn lease (1.1.1.1) right before the restart, so# the post-restart "stale wiped" assertions are meaningful.expect_answers osb-sandbox-a allow.test 1.1.1.1+# Durability probe: relay a policy DISTINCT from the replayed binding input;+# after the restart + SET_BINDING replay the recorded file must supersede+# the input — asserted by the marker target surviving the GET below+# (vault parity, Test 14e).+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"durability.test"}]}'wait_for 10 "dyn lease present before restart" nft_has '1.1.1.1'kill "${EGRESS_PID}" 2>/dev/nullwait "${EGRESS_PID}" 2>/dev/null || true@@ -498,7 +505,11 @@ expect_rcode osb-sandbox-a allow.test 3# The Fastlet's instanceId replay: SET_BINDING + reached Hooks.bind_and_ready a 10.10.0.5 5 runtime-a-2 att-a-2 '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"}]}'expect_answers osb-sandbox-a allow.test 1.1.1.1-pass "restart recovery (stale wiped, action replay reactivates)"+# The durability probe's relay write survived the restart: the recorded+# file superseded the replayed binding input.+recorded="$(curl -s "http://127.0.0.1:${POLICY_PORT}/policy/a")"+echo "${recorded}" | grep -q 'durability.test' || fail "the relayed policy must survive the restart (file supersedes the replayed binding input), got: ${recorded}"+pass "restart durability (recorded policy supersedes the replayed binding input)"###############################################################################info "Test 10: strict DoH mode (no blocklist) drops all tcp 443 globally"@@ -857,6 +868,10 @@ elsewait_for 15 "drop set re-seeded after restart" bash -c \"nft list set inet opensandbox-fast-sandbox upstream_proxy_v4 2>/dev/null | grep -q '10.99.0.2'"bind_and_ready a 10.10.0.5 12 runtime-a-2 att-a-2 '{"defaultAction":"deny","egress":[{"action":"allow","target":"10.99.0.2"},{"action":"allow","target":"ext.test"}]}'+ # The default-allow control write (spec 11) survived this restart — the+ # file supersedes the replayed input — so re-assert the allow-list+ # through the relay before the containment assertions.+ relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"10.99.0.2"},{"action":"allow","target":"ext.test"}]}'if ip netns exec osb-sandbox-a curl -s -m 3 -o /dev/null --proxytunnel -x http://10.99.0.2:3128 http://10.99.0.2:8080/ 2>/dev/null; thenfail "containment must survive egress restart"fidiff --git a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yamlindex f71129599..9974da180 100644--- a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml+++ b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml@@ -78,12 +78,11 @@ spec:# keep the vault memory-only (lost on egress restart).- name: OPENSANDBOX_EGRESS_VAULT_DIRvalue: /var/lib/opensandbox-egress/vault- # Policy persistence: one per-subject file under this directory backs- # the /policy/{uid} relay ack the same way (persist → apply → ack)- # and carries the POLICY_EXPORT / POLICY_IMPORT snapshot transfer.- # Lives on the same Pod-scoped volume as the vault files. Unset- # would keep the recorded policy memory-only and omit the- # policyTransfer capability.+ # Policy persistence: REQUIRED for the fast-sandbox profile — the+ # egress fails startup without it. One per-subject policy file under+ # this directory backs the /policy/{uid} relay ack (persist → apply+ # → ack) and carries the POLICY_EXPORT / POLICY_IMPORT snapshot+ # transfer. Lives on the same Pod-scoped volume as the vault files.- name: OPENSANDBOX_EGRESS_POLICY_DIRvalue: /var/lib/opensandbox-egress/policyvolumeMounts:diff --git a/server/opensandbox_server/services/fast_sandbox/policy_relay.py b/server/opensandbox_server/services/fast_sandbox/policy_relay.pyindex 628a94ba6..696b18f30 100644--- a/server/opensandbox_server/services/fast_sandbox/policy_relay.py+++ b/server/opensandbox_server/services/fast_sandbox/policy_relay.py@@ -44,7 +44,6 @@ from opensandbox_server.services.fast_sandbox.fastpath_client import (from opensandbox_server.services.fast_sandbox.network_policy import (delete_policy_rules,merge_policy_rules,- normalized_policy,policy_status,)from opensandbox_server.services.fast_sandbox.vault import EGRESS_HANDLER@@ -197,8 +196,3 @@ def relay_delete_rules(service, sandbox_id: str, targets: list[str]) -> dict:ack = Nonepolicy = (ack or {}).get("policy")return policy_status(policy if isinstance(policy, dict) else kept)---def normalized_for_create(policy) -> dict:- """Create-path normalization (the binding input seed)."""- return normalized_policy(policy)diff --git a/specs/sandbox-lifecycle.yml b/specs/sandbox-lifecycle.ymlindex 998ff12d8..2231d092a 100644--- a/specs/sandbox-lifecycle.yml+++ b/specs/sandbox-lifecycle.yml@@ -143,9 +143,10 @@ paths:take priority over existing rules with the same target and replacethem in place; within one patch payload, the first rule for a targetwins; existing rules for other targets and the current defaultAction- are preserved. Fsb commits the merged binding through FastPath with- UID/generation conflict protection. Other backends proxy the- sandbox-side egress service.+ are preserved. Fsb composes the merge server-side (relay GET → merge+ → relay PUT carrying the read revision) — a concurrent writer wins+ and the stale merge is a 409. Other backends proxy the sandbox-side+ egress service.requestBody:required: truecontent: