Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): close the rebind/relay activation race and harden state persistence

高然06f1b75c6338
diff --git a/components/egress/fastsandbox.go b/components/egress/fastsandbox.go
index ea5f6b2c2..39ebd0459 100644
--- a/components/egress/fastsandbox.go
+++ b/components/egress/fastsandbox.go
@@ -43,6 +43,7 @@ import (
"net/http"
"net/netip"
"os"
+ "path/filepath"
"strings"
"time"
@@ -141,6 +142,23 @@ func runFastSandboxProfile(ctx context.Context, upstreamSpec *mitmproxy.Upstream
if policyDir == "" {
log.Fatalf("fast-sandbox policy persistence: OPENSANDBOX_EGRESS_POLICY_DIR is required (the recorded policy is egress-held state)")
}
+ // Register the configured state-directory basenames BEFORE either
+ // constructor runs: the volume-root restriction must recognize the
+ // configured sibling as ours even on custom-named deployments.
+ registerStateDirName(policyDir)
+ if vaultDir := strings.TrimSpace(os.Getenv(constants.EnvEgressVaultDir)); vaultDir != "" {
+ registerStateDirName(vaultDir)
+ }
+ // The two state directories are siblings holding same-named per-subject
+ // files (<uid>.json) and seal markers: aliasing them would make each
+ // persister clobber the other's state (a restart-time rehydrate of one
+ // kind quarantines the other kind's file). Reject before any state is
+ // written.
+ if vaultDir := strings.TrimSpace(os.Getenv(constants.EnvEgressVaultDir)); vaultDir != "" {
+ if sameStateDir(vaultDir, policyDir) {
+ log.Fatalf("fast-sandbox policy persistence: OPENSANDBOX_EGRESS_POLICY_DIR must differ from OPENSANDBOX_EGRESS_VAULT_DIR (both resolve to %s)", policyDir)
+ }
+ }
pp, err := newPolicyPersistence(policyDir)
if err != nil {
log.Fatalf("fast-sandbox policy persistence: %v", err)
@@ -299,6 +317,18 @@ func runFastSandboxProfile(ctx context.Context, upstreamSpec *mitmproxy.Upstream
_ = os.Stderr.Sync()
}
+// sameStateDir reports whether two state-directory paths resolve to the
+// same directory (lexical cleaning plus best-effort symlink resolution).
+func sameStateDir(a, b string) bool {
+ a, b = filepath.Clean(a), filepath.Clean(b)
+ if a == b {
+ return true
+ }
+ ra, erra := filepath.EvalSymlinks(a)
+ rb, errb := filepath.EvalSymlinks(b)
+ return erra == nil && errb == nil && ra == rb
+}
+
// fastSandboxNftOptions assembles the fast-sandbox nft options: the shared
// DoH-443 blocking env (OPENSANDBOX_EGRESS_BLOCK_DOH_443 strict all-443 drop
// when the blocklist is empty + OPENSANDBOX_EGRESS_DOH_BLOCKLIST
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index 17e20a83f..29eb1953b 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -197,26 +197,29 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (
return http.StatusBadRequest, fmt.Errorf("invalid binding input policy: %v", err)
}
}
+ // The WHOLE mutation runs under policyCommitMu — rebind purge,
+ // registration, and the seed/apply are one serialized unit against the
+ // relay commits and the hook activation. Without this, a
+ // predecessor-identity relay write holding policyCommitMu could
+ // straddle the rebind's fence flip: its apply closure passes the fence
+ // checks pre-registration, then its reg.ApplyPolicy lands on the
+ // SUCCESSOR's registry entry after RegisterAndEnforce — the successor
+ // would go active under the predecessor's policy. Lock order:
+ // policyCommitMu → registry → s.mu (matches the commit paths).
+ s.policyCommitMu.Lock()
+ defer s.policyCommitMu.Unlock()
+
// A fence change is a rebind: discard ALL prior per-subject state. The
// registry drops its entry inside RegisterAndEnforce; the vault slice
// and the recorded policy must go too (memory + persisted files), or the
// old instance's credentials and policy would survive into the
- // successor's flows. Same-UID actions serialize per subject; the vault
- // and policy relay writes serialize against the same state — the purge
- // runs under policyCommitMu so an in-flight relay commit against the
- // OLD fence cannot resurrect a purged record after the rebind (its
- // commit-time fence re-check aborts it). Callers are never under the
- // registry write lock here, so taking policyCommitMu is safe (the
- // takeover path, which IS under the registry lock, purges without it —
- // its relay races are fenced out by the commit-time re-check).
+ // successor's flows.
if old, ok := s.reg.Fence(subj); ok && !old.Matches(fence) {
- s.policyCommitMu.Lock()
s.mu.Lock()
s.dropVault(subj)
delete(s.policies, subj)
s.purgePolicyFile(subj)
s.mu.Unlock()
- s.policyCommitMu.Unlock()
log.Infof("subject %s: rebind (%s -> %s), vault slice and policy record purged", subj, old.RuntimeInstanceID, fence.RuntimeInstanceID)
}
// Egress-restart recovery (OSEP-0022 stage 2): a subject with no live
@@ -244,7 +247,7 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (
// binding's policy to a policy-less subject) and purge the recorded
// policy, so deny-first is also the durable state.
s.clearPendingPolicy(subj)
- if err := s.clearPolicyRecord(subj); err != nil {
+ if err := s.clearPolicyRecordLocked(subj); err != nil {
// The durable removal did not confirm: fail the action so the
// Fastlet's retry re-runs the purge/tombstone instead of acking
// a removal whose file may still carry the removed policy.
@@ -260,7 +263,7 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (
}
// Install the binding policy (or honor the file-backed record that
// supersedes it) through the policy commit protocol.
- if err := s.applyBindingPolicy(subj, fence, pol, state == subject.StateActive); err != nil {
+ if err := s.applyBindingPolicyLocked(subj, fence, pol, state == subject.StateActive); err != nil {
log.Errorf("subject %s: binding policy commit failed: %v", subj, err)
return http.StatusInternalServerError, fmt.Errorf("binding policy commit: %w", err)
}
diff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.go
index 5aee46b11..75cb3ddda 100644
--- a/components/egress/fastsandbox_policy.go
+++ b/components/egress/fastsandbox_policy.go
@@ -194,6 +194,10 @@ func (s *fastSandboxPolicyServer) handlePolicyRelayWrite(w http.ResponseWriter,
http.Error(w, "invalid policy: "+err.Error(), http.StatusBadRequest)
return
}
+ if len(doc.Policy) > policy.MaxPolicyBodyBytes {
+ http.Error(w, fmt.Sprintf("policy body exceeds %d bytes", policy.MaxPolicyBodyBytes), http.StatusBadRequest)
+ return
+ }
expected, perr := relayPolicyPrecondition(&doc)
if perr != nil {
http.Error(w, perr.Error(), http.StatusBadRequest)
@@ -444,6 +448,6 @@ func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subj
http.Error(w, "policy persistence failure", status)
return
}
- log.Infof("subject %s: policy deleted (deny-first), record purged", subj)
+ log.Infof("subject %s: policy deleted (deny tombstone enforced and persisted at revision %d)", subj, newRev)
w.WriteHeader(http.StatusNoContent)
}
diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.go
index a6ef67256..c164c476b 100644
--- a/components/egress/fastsandbox_policy_persist.go
+++ b/components/egress/fastsandbox_policy_persist.go
@@ -172,8 +172,9 @@ func (p *policyPersistence) reservesPath(path string) bool {
return reservesPathUnder(p.dir, path)
}
-// SetPolicyFiles wires persistence under the relay handlers; nil (the
-// default) keeps the policy memory-only.
+// SetPolicyFiles wires persistence under the relay handlers. Policy
+// persistence is REQUIRED in the fast-sandbox profile (startup fails
+// without it), so every assembled server carries a store.
func (s *fastSandboxPolicyServer) SetPolicyFiles(pp *policyPersistence) {
s.mu.Lock()
defer s.mu.Unlock()
@@ -300,10 +301,9 @@ func (s *fastSandboxPolicyServer) policyHealthIssue() string {
// away policy. A tombstone persist failure latches recovery (writes frozen
// until restart). Serialized with the relay commits (policyCommitMu) so an
// in-flight write against the removed binding cannot recreate the record
-// after the purge; callers are never under the registry write lock.
-func (s *fastSandboxPolicyServer) clearPolicyRecord(subj subject.Subject) error {
- s.policyCommitMu.Lock()
- defer s.policyCommitMu.Unlock()
+// after the purge. The caller holds policyCommitMu (the SET_BINDING
+// mutation hold); never call under the registry write lock.
+func (s *fastSandboxPolicyServer) clearPolicyRecordLocked(subj subject.Subject) error {
s.mu.Lock()
entry := s.policies[subj]
pp := s.policyFiles
@@ -397,7 +397,17 @@ func (s *fastSandboxPolicyServer) rehydratePolicyReload(subj subject.Subject, fe
return
}
if !snap.Exists() {
- // Fresh subject: nothing persisted, the binding input seeds.
+ // Fresh subject: nothing persisted, the binding input seeds. A
+ // durable seal marker (an export that sealed a file-less subject)
+ // must still gate the relay ops — the seed entry carries it.
+ if s.policyFiles.isSealed(subj) {
+ s.mu.Lock()
+ s.policies[subj] = &policyEntry{fence: fence, sealed: true}
+ s.pendingPolicies[subj] = policy.DefaultDenyPolicy()
+ s.mu.Unlock()
+ log.Infof("subject %s: seal restored from marker on a file-less subject (binding input seeds pending default-deny)", subj)
+ return
+ }
return
}
pf, body, decErr := policy.DecodePolicyFile(snap.Data())
@@ -449,17 +459,15 @@ func (s *fastSandboxPolicyServer) rehydratePolicyReload(subj subject.Subject, fe
log.Infof("subject %s: policy reloaded from file (revision %d, incarnation %s); binding input superseded", subj, pf.Revision, pf.Incarnation)
}
-// applyBindingPolicy seeds the binding's policy for a freshly registered
-// subject (record absent) through the same commit protocol as the relay
-// writes. A subject with an existing record never reads its binding input
-// again: the recorded policy — seeded once, then owned by the relay writes
+// applyBindingPolicyLocked seeds the binding's policy for a freshly
+// registered subject (record absent) through the same commit protocol as
+// the relay writes. The caller holds policyCommitMu (the SET_BINDING
+// mutation hold). A subject with an existing record never reads its
+// binding input again: the recorded policy — seeded once, then owned by the relay writes
// and the snapshot transfer — is authoritative, exactly like the vault
// slice. Binding replays (at-least-once delivery, egress restarts, and the
// superseded CRD-driven update path) are therefore policy no-ops.
-func (s *fastSandboxPolicyServer) applyBindingPolicy(subj subject.Subject, fence subject.Fencing, pol *policy.NetworkPolicy, stateActive bool) error {
- s.policyCommitMu.Lock()
- defer s.policyCommitMu.Unlock()
-
+func (s *fastSandboxPolicyServer) applyBindingPolicyLocked(subj subject.Subject, fence subject.Fencing, pol *policy.NetworkPolicy, stateActive bool) error {
s.mu.Lock()
entry := s.policies[subj]
if entry != nil && entry.fence != fence {
diff --git a/components/egress/fastsandbox_policy_relay_test.go b/components/egress/fastsandbox_policy_relay_test.go
index c0efe8632..9e2deeb85 100644
--- a/components/egress/fastsandbox_policy_relay_test.go
+++ b/components/egress/fastsandbox_policy_relay_test.go
@@ -15,17 +15,21 @@
package main
import (
+ "context"
"encoding/json"
"net/http"
"net/http/httptest"
+ "net/netip"
"strconv"
"strings"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/alibaba/opensandbox/egress/pkg/constants"
+ "github.com/alibaba/opensandbox/egress/pkg/policy"
"github.com/alibaba/opensandbox/egress/pkg/subject"
)
@@ -301,6 +305,120 @@ func TestPolicyRelayPutReplacesStalePendingTombstone(t *testing.T) {
assert.Equal(t, "deny", reg_UserEvaluate(srv, s, "api.example.com"))
}
+// hookNft forces an interleaving at the relay apply's enforcement step: the
+// injected callback runs while the relay commit holds policyCommitMu, which
+// is exactly the window a concurrent rebind would race through.
+type hookNft struct {
+ fakeNft
+ onPolicy func()
+}
+
+func (h *hookNft) ApplyPolicy(ctx context.Context, s subject.Subject, pol *policy.NetworkPolicy) error {
+ if h.onPolicy != nil {
+ fn := h.onPolicy
+ h.onPolicy = nil
+ fn()
+ }
+ return h.fakeNft.ApplyPolicy(ctx, s, pol)
+}
+
+// TestRelayPutRacingRebindNeverActivatesPredecessorPolicy (review H3): a
+// relay PUT for the PREDECESSOR identity whose apply lands after the
+// rebind's registration must not activate the successor under the
+// predecessor's policy. The rebind runs on another goroutine from inside
+// the relay's enforcement step, forcing the fence flip into the apply
+// window; the fixed serialization (the whole SET_BINDING mutation holds
+// policyCommitMu) makes the rebind wait, the relay commit completes against
+// the old instance, and the purge discards it.
+func TestRelayPutRacingRebindNeverActivatesPredecessorPolicy(t *testing.T) {
+ reg := subject.NewRegistry(nil, nil)
+ hnft := &hookNft{}
+ srv := newFastSandboxPolicyServer(context.Background(), reg, hnft)
+ srv.dnsRedirectInstall = func(netip.Addr, int) error { return nil }
+ srv.dnsRedirectRemove = func() error { return nil }
+ pp, err := newPolicyPersistence(t.TempDir())
+ require.NoError(t, err)
+ files := make(map[string]*fakeVaultFile)
+ pp.newFile = func(path string) (vaultFileStore, error) {
+ f, ok := files[path]
+ if !ok {
+ f = newFakeVaultFile(path)
+ files[path] = f
+ }
+ return f, nil
+ }
+ srv.SetPolicyFiles(pp)
+
+ const uid, ip = "race-u", "10.0.0.5"
+ const oldPolicy = `{"defaultAction":"deny","egress":[{"action":"allow","target":"old.com"}]}`
+ const newPolicy = `{"defaultAction":"deny","egress":[{"action":"allow","target":"new.com"}]}`
+
+ // Generation one: old fence, active with old policy.
+ rec := doAction(t, srv, bindBody(t, uid, ip, testFenceR, testFenceA, 1, strPtr(oldPolicy)))
+ require.Equal(t, http.StatusOK, rec.Code)
+ rec = doAction(t, srv, hookBody(t, uid, testFenceR, testFenceA, constants.HookRuntimeReady))
+ require.Equal(t, http.StatusOK, rec.Code)
+ rec = doAction(t, srv, hookBody(t, uid, testFenceR, testFenceA, constants.HookDataPlaneReady))
+ require.Equal(t, http.StatusOK, rec.Code)
+ s := subject.FromSandboxUID(uid)
+ require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
+
+ // A queued relay PUT for the OLD identity: its apply BLOCKS inside the
+ // enforcement step while the rebind (same UID, new fence) completes on
+ // another goroutine — forcing the fence flip into the apply window. With
+ // the serialization fix the rebind cannot proceed (policyCommitMu is
+ // held by this very commit), so the block times out and the rebind runs
+ // to completion afterwards; either way the successor must converge to
+ // its own policy.
+ started := make(chan struct{})
+ done := make(chan struct{})
+ hnft.onPolicy = func() {
+ close(started)
+ select {
+ case <-done:
+ case <-time.After(2 * time.Second):
+ // Serialized: the rebind runs after this commit.
+ }
+ }
+ go func() {
+ <-started
+ defer close(done)
+ r := doAction(t, srv, bindBody(t, uid, ip, "runtime-2", "attachment-2", 2, strPtr(newPolicy)))
+ assert.Equal(t, http.StatusOK, r.Code)
+ }()
+
+ // The relay write itself passes its guards (pre-read fence = old) and
+ // must NOT leak its policy into the successor.
+ rec = doPolicy(t, srv, http.MethodPut, uid, policyPutDoc(oldPolicy, 1, false, true), "")
+ require.Equal(t, http.StatusOK, rec.Code)
+
+ select {
+ case <-done:
+ case <-time.After(10 * time.Second):
+ t.Fatal("the rebind never completed (serialization deadlock?)")
+ }
+
+ // The successor is deny-first with ITS OWN binding input — never active
+ // under the predecessor's policy, whichever way the interleaving went.
+ state, ok := reg.Get(s)
+ require.True(t, ok)
+ if state == subject.StateActive {
+ // Activated only legitimately: the successor's own policy enforced.
+ assert.Equal(t, "allow", reg.UserPolicy(s).Evaluate("new.com"))
+ assert.Equal(t, "deny", reg.UserPolicy(s).Evaluate("old.com"))
+ } else {
+ assert.Equal(t, subject.StateDenying, state)
+ assert.Nil(t, reg.EffectivePolicy(s), "deny-first until the successor's own data-plane-ready")
+ }
+
+ // The successor's own lifecycle completes with its own policy.
+ rec = doAction(t, srv, hookBody(t, uid, "runtime-2", "attachment-2", constants.HookDataPlaneReady))
+ require.Equal(t, http.StatusOK, rec.Code)
+ require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
+ assert.Equal(t, "allow", reg.UserPolicy(s).Evaluate("new.com"))
+ assert.Equal(t, "deny", reg.UserPolicy(s).Evaluate("old.com"), "the predecessor's policy never reaches the successor")
+}
+
// reg_UserEvaluate evaluates the subject's recorded user policy (test aid to
// keep assertions readable).
func reg_UserEvaluate(srv *fastSandboxPolicyServer, s subject.Subject, domain string) string {
diff --git a/components/egress/fastsandbox_policy_snapshot.go b/components/egress/fastsandbox_policy_snapshot.go
index 5a14c420c..07b6f78d0 100644
--- a/components/egress/fastsandbox_policy_snapshot.go
+++ b/components/egress/fastsandbox_policy_snapshot.go
@@ -170,12 +170,10 @@ func (s *fastSandboxPolicyServer) policyExportSourceImageLocked(subj subject.Sub
}
if !s.fenceMatchesLocked(subj, fence) {
// Purged between the pre-read and the lock — the stale envelope
- // lost the race with a rebind.
+ // lost the race with a rebind. The registered-fence mismatch above
+ // already covers the plain stale case.
return nil, http.StatusNotFound, fmt.Errorf("policy export for unknown subject %s", subj)
}
- if !regFence.Matches(fence) {
- return nil, http.StatusConflict, fmt.Errorf("stale policy export (fence mismatch) for subject %s", subj)
- }
if recovery := s.policyRecovery.Load(); recovery != nil {
return nil, http.StatusServiceUnavailable, fmt.Errorf("policy persistence recovery required: %s", recovery.reason)
}
@@ -311,7 +309,17 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)
s.mu.Unlock()
return s.policyFiles.unseal(subj)
}
- return s.installImportedPolicyLocked(subj, fence, pf.Revision, pol, sealed)
+ err := s.installImportedPolicyLocked(subj, fence, pf.Revision, pol, sealed)
+ if err != nil && sealed {
+ // The unseal already happened; restore the durable seal so
+ // disk and memory agree the bundle is still the truth (the
+ // commit protocol rolls the file back to the sealed image;
+ // the in-memory entry was never modified).
+ if sealErr := s.policyFiles.seal(subj); sealErr != nil {
+ log.Errorf("subject %s: policy seal marker restore failed after import apply failure: %v", subj, sealErr)
+ }
+ }
+ return err
},
)
if err != nil {
diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.go
index c9a6378fd..855e8599a 100644
--- a/components/egress/fastsandbox_server.go
+++ b/components/egress/fastsandbox_server.go
@@ -187,6 +187,10 @@ func (s *fastSandboxPolicyServer) mitmRedirectRebuild() error {
// per-subject gateway REDIRECT forwards sandbox DNS here.
const fastSandboxDNSProxyPort = 15353
+// policyApplyTimeout bounds one kernel policy transaction on every path
+// that runs under policyCommitMu (see applyPolicy).
+const policyApplyTimeout = 10 * time.Second
+
// installGatewayDNSRedirect records a subject's gateway and installs (once)
// the prerouting REDIRECT for it. Idempotent under at-least-once SET_BINDING
// delivery: a duplicate registration (or a retried deny-first install) is a
@@ -384,7 +388,14 @@ func (s *fastSandboxPolicyServer) applyPolicy(subj subject.Subject, pol *policy.
s.policyMu.Lock()
defer s.policyMu.Unlock()
eff := s.reg.EffectiveOf(pol)
- nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
+ // policyApplyTimeout bounds the kernel transaction for every caller
+ // that runs under policyCommitMu (relay commits, the SET_BINDING seed,
+ // and the data-plane-ready hook all serialize on it): an unbounded
+ // deadline would let one slow nft apply head-of-line-block every other
+ // subject's policy write past the listener's write timeout. The
+ // registration deny-first install is NOT under that serialization and
+ // keeps its own 30s budget.
+ nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)
defer cancel()
if err := s.nft.ApplyPolicy(nftCtx, subj, eff); err != nil {
return fmt.Errorf("nft policy apply: %w", err)
@@ -636,7 +647,7 @@ func (s *fastSandboxPolicyServer) pendingPolicy(subj subject.Subject) *policy.Ne
// sandbox must be fully blocked again). nft commits before the registry
// state, so the transition stays fail-closed.
func (s *fastSandboxPolicyServer) revertToDenyFirst(subj subject.Subject, att actionhandler.NetworkAttachment) error {
- nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
+ nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)
defer cancel()
if err := s.nft.ApplyDenyFirst(nftCtx, subj, att); err != nil {
return fmt.Errorf("revert to deny-first: %w", err)
diff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.go
index 111be5b2e..17b8ef004 100644
--- a/components/egress/fastsandbox_vault.go
+++ b/components/egress/fastsandbox_vault.go
@@ -134,19 +134,11 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
http.Error(w, "expectedIncarnation mismatch", http.StatusPreconditionFailed)
return
}
- effective := s.reg.EffectivePolicy(subj)
- if effective == nil {
- // Pre-read the pending policy and merge the always overlay BEFORE
- // s.mu (registry → s.mu order): a vault op held to the
- // data-plane-ready barrier must validate against the SAME policy
- // the hook will enforce, operator always-rules included.
- s.mu.Lock()
- pending := s.pendingPolicies[subj]
- s.mu.Unlock()
- if pending != nil {
- effective = s.reg.EffectiveOf(pending)
- }
- }
+ // Pre-compute the validation policy BEFORE s.mu (registry → s.mu
+ // order): a vault op held to the data-plane-ready barrier must validate
+ // against the SAME policy the hook will enforce, operator always-rules
+ // included.
+ effective := s.vaultValidationPolicy(subj)
// The rest of the mutation holds s.mu, serializing per-subject writers
// and fencing rebind/terminal purges. The validation policy was fully
diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.go
index 787ac16f1..5432da9ee 100644
--- a/components/egress/fastsandbox_vault_persist.go
+++ b/components/egress/fastsandbox_vault_persist.go
@@ -114,7 +114,7 @@ func prepareStateDir(dir, noun string) error {
}
// The volume root must pass the atomic machinery's trusted-ancestor
// check; kubelet creates emptyDir roots 0777, which fails it.
- if err := restrictVolumeRoot(dir); err != nil {
+ if err := restrictVolumeRoot(dir, noun); err != nil {
return err
}
if err := sweepStateTempFiles(dir, noun); err != nil {
@@ -143,28 +143,36 @@ func sweepStateTempFiles(dir, noun string) error {
return nil
}
-// knownStateDirNames are the per-subject state directories that may share
-// one egress volume root (the vault and the recorded policy). A sibling
-// state directory is ours, not a foreign entry — without this, whichever
-// directory initializes second would see the first as foreign, skip the
-// volume-root restriction, and the atomic machinery would then reject every
-// file op on the untrusted ancestor.
-var knownStateDirNames = map[string]bool{
+// stateDirNames are the basenames of the per-subject state directories that
+// share one egress volume root, registered at assembly from the CONFIGURED
+// paths (the basename is deployment-owned, so it cannot be hardcoded; the
+// conventional names stay pre-seeded). restrictVolumeRoot treats registered
+// siblings as ours — without this, whichever directory initializes second
+// would see the first as foreign, skip the volume-root restriction, and the
+// atomic machinery would then reject every file op on the untrusted
+// ancestor.
+var stateDirNames = map[string]bool{
"vault": true,
"policy": true,
}
-// restrictVolumeRoot clears group/world-write bits from the vault volume
-// root (the vault directory's parent): kubelet creates emptyDir roots 0777,
+// registerStateDirName records a configured state directory basename so the
+// volume-root restriction recognizes it as a sibling.
+func registerStateDirName(dir string) {
+ stateDirNames[filepath.Base(filepath.Clean(dir))] = true
+}
+
+// restrictVolumeRoot clears group/world-write bits from the state volume
+// root (the state directory's parent): kubelet creates emptyDir roots 0777,
// which the atomic machinery rejects as an untrusted ancestor. Sibling
-// state directories (knownStateDirNames) belong to us; a root holding any
+// state directories (stateDirNames) belong to us; a root holding any
// other entry is left untouched — that is the manifest's job — and
// already-restrictive roots are a no-op.
-func restrictVolumeRoot(dir string) error {
+func restrictVolumeRoot(dir, noun string) error {
parent := filepath.Dir(dir)
info, err := os.Stat(parent)
if err != nil {
- return fmt.Errorf("inspect vault volume root %s: %w", parent, err)
+ return fmt.Errorf("inspect %s volume root %s: %w", noun, parent, err)
}
if !info.IsDir() || info.Mode().Perm()&0o022 == 0 {
return nil
@@ -178,16 +186,16 @@ func restrictVolumeRoot(dir string) error {
}
base := filepath.Base(dir)
for _, entry := range entries {
- if entry.Name() == base || (entry.IsDir() && knownStateDirNames[entry.Name()]) {
+ if entry.Name() == base || (entry.IsDir() && stateDirNames[entry.Name()]) {
continue
}
- log.Infof("vault persistence: volume root %s holds foreign entry %q; not restricting (fix permissions in the manifest)", parent, entry.Name())
+ log.Infof("%s persistence: volume root %s holds foreign entry %q; not restricting (fix permissions in the manifest)", noun, parent, entry.Name())
return nil
}
if err := os.Chmod(parent, info.Mode().Perm()&^0o022); err != nil {
- return fmt.Errorf("restrict vault volume root %s: %w", parent, err)
+ return fmt.Errorf("restrict %s volume root %s: %w", noun, parent, err)
}
- log.Infof("vault persistence: restricted group/world-writable volume root %s (kubelet emptyDir default)", parent)
+ log.Infof("%s persistence: restricted group/world-writable volume root %s (kubelet emptyDir default)", noun, parent)
return nil
}
diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.go
index 1d5e441e3..52dcee10f 100644
--- a/components/egress/fastsandbox_vault_snapshot.go
+++ b/components/egress/fastsandbox_vault_snapshot.go
@@ -179,12 +179,10 @@ func (s *fastSandboxPolicyServer) exportSourceImageLocked(subj subject.Subject,
}
if !s.fenceMatchesLocked(subj, fence) {
// Purged between the pre-read and the lock — the stale envelope
- // lost the race with a rebind.
+ // lost the race with a rebind. The registered-fence mismatch above
+ // already covers the plain stale case.
return nil, http.StatusNotFound, fmt.Errorf("vault export for unknown subject %s", subj)
}
- if !regFence.Matches(fence) {
- return nil, http.StatusConflict, fmt.Errorf("stale vault export (fence mismatch) for subject %s", subj)
- }
if recovery := s.vaultRecovery.Load(); recovery != nil {
return nil, http.StatusServiceUnavailable, fmt.Errorf("vault persistence recovery required: %s", recovery.reason)
}
diff --git a/components/egress/tests/e2e-fast-sandbox-firecracker.sh b/components/egress/tests/e2e-fast-sandbox-firecracker.sh
index b6a39ed35..eefc30c54 100755
--- a/components/egress/tests/e2e-fast-sandbox-firecracker.sh
+++ b/components/egress/tests/e2e-fast-sandbox-firecracker.sh
@@ -408,6 +408,7 @@ EGRESS_CT_ID="$(docker run -d --name "${EGRESS_CT}" \
-v "${MITM_HOME}:/var/lib/mitmproxy" \
-v "${SEED_DIR}:/var/egress/mitm-ca-seed:ro" \
-e OPENSANDBOX_EGRESS_PROFILE=fast-sandbox \
+ -e OPENSANDBOX_EGRESS_POLICY_DIR=/var/lib/opensandbox-egress/policy \
-e OPENSANDBOX_EGRESS_DNS_UPSTREAM="${BRIDGE_GW}:${DNS_UPSTREAM_PORT}" \
-e OPENSANDBOX_EGRESS_DNS_UPSTREAM_PROBE=allow.test \
-e OPENSANDBOX_EGRESS_HTTP_ADDR="127.0.0.1:${POLICY_PORT}" \
diff --git a/components/egress/tests/smoke-fast-sandbox.sh b/components/egress/tests/smoke-fast-sandbox.sh
index b620d3ec8..d94808b63 100755
--- a/components/egress/tests/smoke-fast-sandbox.sh
+++ b/components/egress/tests/smoke-fast-sandbox.sh
@@ -460,6 +460,8 @@ case "${http_code}" in
301|307|308) ;; # redirects into the /policy/{uid} relay subtree
*) fail "bare /policy must redirect into the relay subtree, got ${http_code}" ;;
esac
+location="$(curl -s -o /dev/null -w '%{header_json}' "http://127.0.0.1:${POLICY_PORT}/policy" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("location","")[0])' 2>/dev/null || true)"
+[ "${location}" = "/policy/" ] || fail "bare /policy redirect target must be /policy/, got ${location:-none}"
pass "superseded proxy-route surfaces removed (relay surfaces only)"
set_up_netns b 10.10.0.6
@@ -488,6 +490,11 @@ info "Test 9: restart recovery (reset -> action replay -> active)"
# Refresh the DNS-learned dyn lease (1.1.1.1) right before the restart, so
# the post-restart "stale wiped" assertions are meaningful.
expect_answers osb-sandbox-a allow.test 1.1.1.1
+# Durability probe: relay a policy DISTINCT from the replayed binding input;
+# after the restart + SET_BINDING replay the recorded file must supersede
+# the input — asserted by the marker target surviving the GET below
+# (vault parity, Test 14e).
+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"durability.test"}]}'
wait_for 10 "dyn lease present before restart" nft_has '1.1.1.1'
kill "${EGRESS_PID}" 2>/dev/null
wait "${EGRESS_PID}" 2>/dev/null || true
@@ -498,7 +505,11 @@ expect_rcode osb-sandbox-a allow.test 3
# The Fastlet's instanceId replay: SET_BINDING + reached Hooks.
bind_and_ready a 10.10.0.5 5 runtime-a-2 att-a-2 '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"}]}'
expect_answers osb-sandbox-a allow.test 1.1.1.1
-pass "restart recovery (stale wiped, action replay reactivates)"
+# The durability probe's relay write survived the restart: the recorded
+# file superseded the replayed binding input.
+recorded="$(curl -s "http://127.0.0.1:${POLICY_PORT}/policy/a")"
+echo "${recorded}" | grep -q 'durability.test' || fail "the relayed policy must survive the restart (file supersedes the replayed binding input), got: ${recorded}"
+pass "restart durability (recorded policy supersedes the replayed binding input)"
###############################################################################
info "Test 10: strict DoH mode (no blocklist) drops all tcp 443 globally"
@@ -857,6 +868,10 @@ else
wait_for 15 "drop set re-seeded after restart" bash -c \
"nft list set inet opensandbox-fast-sandbox upstream_proxy_v4 2>/dev/null | grep -q '10.99.0.2'"
bind_and_ready a 10.10.0.5 12 runtime-a-2 att-a-2 '{"defaultAction":"deny","egress":[{"action":"allow","target":"10.99.0.2"},{"action":"allow","target":"ext.test"}]}'
+ # The default-allow control write (spec 11) survived this restart — the
+ # file supersedes the replayed input — so re-assert the allow-list
+ # through the relay before the containment assertions.
+ relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"10.99.0.2"},{"action":"allow","target":"ext.test"}]}'
if ip netns exec osb-sandbox-a curl -s -m 3 -o /dev/null --proxytunnel -x http://10.99.0.2:3128 http://10.99.0.2:8080/ 2>/dev/null; then
fail "containment must survive egress restart"
fi
diff --git a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml
index f71129599..9974da180 100644
--- a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml
+++ b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml
@@ -78,12 +78,11 @@ spec:
# keep the vault memory-only (lost on egress restart).
- name: OPENSANDBOX_EGRESS_VAULT_DIR
value: /var/lib/opensandbox-egress/vault
- # Policy persistence: one per-subject file under this directory backs
- # the /policy/{uid} relay ack the same way (persist → apply → ack)
- # and carries the POLICY_EXPORT / POLICY_IMPORT snapshot transfer.
- # Lives on the same Pod-scoped volume as the vault files. Unset
- # would keep the recorded policy memory-only and omit the
- # policyTransfer capability.
+ # Policy persistence: REQUIRED for the fast-sandbox profile — the
+ # egress fails startup without it. One per-subject policy file under
+ # this directory backs the /policy/{uid} relay ack (persist → apply
+ # → ack) and carries the POLICY_EXPORT / POLICY_IMPORT snapshot
+ # transfer. Lives on the same Pod-scoped volume as the vault files.
- name: OPENSANDBOX_EGRESS_POLICY_DIR
value: /var/lib/opensandbox-egress/policy
volumeMounts:
diff --git a/server/opensandbox_server/services/fast_sandbox/policy_relay.py b/server/opensandbox_server/services/fast_sandbox/policy_relay.py
index 628a94ba6..696b18f30 100644
--- a/server/opensandbox_server/services/fast_sandbox/policy_relay.py
+++ b/server/opensandbox_server/services/fast_sandbox/policy_relay.py
@@ -44,7 +44,6 @@ from opensandbox_server.services.fast_sandbox.fastpath_client import (
from opensandbox_server.services.fast_sandbox.network_policy import (
delete_policy_rules,
merge_policy_rules,
- normalized_policy,
policy_status,
)
from opensandbox_server.services.fast_sandbox.vault import EGRESS_HANDLER
@@ -197,8 +196,3 @@ def relay_delete_rules(service, sandbox_id: str, targets: list[str]) -> dict:
ack = None
policy = (ack or {}).get("policy")
return policy_status(policy if isinstance(policy, dict) else kept)
-
-
-def normalized_for_create(policy) -> dict:
- """Create-path normalization (the binding input seed)."""
- return normalized_policy(policy)
diff --git a/specs/sandbox-lifecycle.yml b/specs/sandbox-lifecycle.yml
index 998ff12d8..2231d092a 100644
--- a/specs/sandbox-lifecycle.yml
+++ b/specs/sandbox-lifecycle.yml
@@ -143,9 +143,10 @@ paths:
take priority over existing rules with the same target and replace
them in place; within one patch payload, the first rule for a target
wins; existing rules for other targets and the current defaultAction
- are preserved. Fsb commits the merged binding through FastPath with
- UID/generation conflict protection. Other backends proxy the
- sandbox-side egress service.
+ are preserved. Fsb composes the merge server-side (relay GET → merge
+ → relay PUT carrying the read revision) — a concurrent writer wins
+ and the stale merge is a 409. Other backends proxy the sandbox-side
+ egress service.
requestBody:
required: true
content: