Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(egress): serialize policy commits with lifecycle hooks, tombstone deletes
高然6cca228c6691
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.goindex 2f6524973..bd057502e 100644--- a/components/egress/fastsandbox_actions.go+++ b/components/egress/fastsandbox_actions.go@@ -277,6 +277,15 @@ func (s *fastSandboxPolicyServer) applyLifecycleHook(env *actionhandler.Envelopecase constants.HookRuntimeReady:return http.StatusOK, nilcase constants.HookDataPlaneReady:+ // Serialized with the policy relay commits (policyCommitMu): a+ // relay write that passed its guards before this hook must either+ // land first (the activation below then applies the relayed+ // pending policy) or after (its apply-time state re-read then sees+ // Active and updates enforcement in place) — it can never land+ // after the activation as a pending-only write that nothing will+ // ever consume.+ s.policyCommitMu.Lock()+ defer s.policyCommitMu.Unlock()// Peek, do not consume: a transient nft failure must leave the// pending policy in place so the Fastlet's retry of the same Hook// succeeds instead of 409-ing forever with the subject deny-first.diff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.goindex 163decb24..b5cec60db 100644--- a/components/egress/fastsandbox_policy.go+++ b/components/egress/fastsandbox_policy.go@@ -220,8 +220,6 @@ func (s *fastSandboxPolicyServer) handlePolicyRelayWrite(w http.ResponseWriter,if entry != nil {cur = entry.revision}- state, _ := s.reg.Get(subj)- stateActive := state == subject.StateActiveatt, attOK := s.subjAtt[subj]s.mu.Unlock()@@ -230,7 +228,8 @@ func (s *fastSandboxPolicyServer) handlePolicyRelayWrite(w http.ResponseWriter,http.Error(w, "subject has no recorded policy", http.StatusNotFound)return}- s.deleteRelayPolicy(w, subj, fence, att, attOK, stateActive)+ // Deletion is an authoritative tombstone at the successor revision.+ s.deleteRelayPolicy(w, subj, fence, att, attOK, cur+1)return}@@ -246,7 +245,7 @@ func (s *fastSandboxPolicyServer) handlePolicyRelayWrite(w http.ResponseWriter,}created := entry == nilnewRev := cur + 1- s.putRelayPolicy(w, r, subj, fence, pol, newRev, created, stateActive)+ s.putRelayPolicy(w, r, subj, fence, pol, newRev, created)}// relayPolicyPrecondition extracts the effective precondition. The revision@@ -269,8 +268,12 @@ func relayPolicyPrecondition(doc *policyRelayDocument) (expected *int64, err err// putRelayPolicy commits a full replace: persist → apply → ack, so the ack// means durable AND applied. The caller holds policyCommitMu (the decision-// and the commit are one serialized unit).-func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.Request, subj subject.Subject, fence subject.Fencing, pol *policy.NetworkPolicy, newRev int64, created bool, stateActive bool) {+// and the commit are one serialized unit, and the data-plane-ready hook+// serializes on the same mutex). The apply closure re-reads the lifecycle+// state and the registration fence — both can change between the guard+// phase and this commit (a concurrent hook activation, a rebind) — so the+// recorded state can never diverge from what was actually enforced.+func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.Request, subj subject.Subject, fence subject.Fencing, pol *policy.NetworkPolicy, newRev int64, created bool) {raw, err := json.Marshal(pol)if err != nil {log.Errorf("subject %s: policy marshal failed: %v", subj, err)@@ -280,7 +283,11 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.doc := policyRelayDocument{Revision: newRev, Policy: raw}apply := func() error {- if stateActive {+ state, err := s.policyCommitTarget(subj, fence)+ if err != nil {+ return err+ }+ if state == subject.StateActive {// Enforcement first: a failed apply leaves the record untouched// (the commit protocol rolls the file back).if err := s.applyPolicy(subj, pol); err != nil {@@ -288,7 +295,7 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.}}s.mu.Lock()- if !stateActive {+ if state != subject.StateActive {s.pendingPolicies[subj] = pol}s.policies[subj] = &policyEntry{revision: newRev, fence: fence}@@ -299,7 +306,7 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.if s.policyFiles == nil {// Memory-only (persistence disabled): direct commit.if err := apply(); err != nil {- http.Error(w, "policy apply failure", http.StatusInternalServerError)+ http.Error(w, err.Error(), http.StatusInternalServerError)return}writeJSON(w, putResponseStatus(created), doc)@@ -332,6 +339,22 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.writeJSON(w, putResponseStatus(created), doc)}+// policyCommitTarget re-reads the subject's lifecycle state under s.mu and+// verifies the registration fence still matches the commit's — the+// data-plane-ready activation and rebinds run outside the commit+// serialization, so a stale apply must be aborted (the commit protocol+// rolls the file back) instead of landing on the wrong identity or state.+func (s *fastSandboxPolicyServer) policyCommitTarget(subj subject.Subject, fence subject.Fencing) (subject.State, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ regFence, ok := s.reg.Fence(subj)+ if !ok || regFence != fence {+ return subject.StateAbsent, fmt.Errorf("stale policy commit (fence changed) for subject %s", subj)+ }+ state, _ := s.reg.Get(subj)+ return state, nil+}+// putResponseStatus maps the committed replace onto the relay contract:// 201 when it created the record, 200 otherwise.func putResponseStatus(created bool) int {@@ -341,12 +364,20 @@ func putResponseStatus(created bool) int {return http.StatusOK}-// deleteRelayPolicy commits the revert to deny-first: memory and file go-// together, so a restart cannot resurrect a deleted policy. The caller-// holds policyCommitMu.-func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subj subject.Subject, fence subject.Fencing, att actionhandler.NetworkAttachment, attOK, stateActive bool) {+// deleteRelayPolicy commits the deletion as an authoritative default-deny+// tombstone: enforcement reverts to deny-first AND the record persists at+// the successor revision (memory + file), so an egress restart reloads the+// deletion instead of letting the replayed bootstrap binding resurrect the+// deleted policy. GET serves the deny record; a later PUT replaces it at+// the successor revision. The caller holds policyCommitMu.+func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subj subject.Subject, fence subject.Fencing, att actionhandler.NetworkAttachment, attOK bool, newRev int64) {+ deny := policy.DefaultDenyPolicy()apply := func() error {- if stateActive {+ state, err := s.policyCommitTarget(subj, fence)+ if err != nil {+ return err+ }+ if state == subject.StateActive {// Enforcement first: nft deny-first commits before the registry// state (fail-closed transition), mirroring SET_BINDING removal.if !attOK {@@ -357,8 +388,11 @@ func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subj}}s.mu.Lock()- delete(s.pendingPolicies, subj)- delete(s.policies, subj)+ // The activation barrier keeps working across a hook replay: a+ // replayed data-plane-ready activates the deny record, never a+ // stale policy.+ s.pendingPolicies[subj] = deny+ s.policies[subj] = &policyEntry{revision: newRev, fence: fence}s.mu.Unlock()return nil}@@ -371,10 +405,20 @@ func (s *fastSandboxPolicyServer) deleteRelayPolicy(w http.ResponseWriter, subjw.WriteHeader(http.StatusNoContent)return}- err := s.persistPolicyCandidateLocked(+ raw, err := json.Marshal(deny)+ if err != nil {+ http.Error(w, "policy persistence failure", http.StatusInternalServerError)+ return+ }+ payload, err := policy.EncodePolicyFile(fence.RuntimeInstanceID, fence.AttachmentID, newRev, raw)+ if err != nil {+ http.Error(w, "policy persistence failure", http.StatusInternalServerError)+ return+ }+ err = s.persistPolicyCandidateLocked(subj,func(fs vaultFileStore) (policy.FileMutationState, error) {- return fs.Delete()+ return fs.SaveRaw(payload)},apply,)diff --git a/components/egress/fastsandbox_policy_persist_test.go b/components/egress/fastsandbox_policy_persist_test.goindex 6e6c6e9df..b2d7a874a 100644--- a/components/egress/fastsandbox_policy_persist_test.go+++ b/components/egress/fastsandbox_policy_persist_test.go@@ -453,6 +453,55 @@ func mustReadRevision(t *testing.T, h *policyPersistHarness, uid string) int64 {return rev}+// TestPolicyPersistDeleteTombstoneSurvivesRestart: a relay DELETE persists+// an authoritative default-deny record, so an egress restart's binding+// replay loads the tombstone instead of resurrecting the deleted policy —+// and a replayed data-plane-ready activates the deny record.+func TestPolicyPersistDeleteTombstoneSurvivesRestart(t *testing.T) {+ h := newPolicyPersistHarness(t)+ h.readySubject(t, policyTestUID, vaultTestIP)++ // Runtime mutation to rev 2, then delete → deny tombstone at rev 3.+ rec := doPolicy(t, h.srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllowAlt(), 1, false, true), "")+ require.Equal(t, http.StatusOK, rec.Code)+ rec = doPolicy(t, h.srv, http.MethodDelete, policyTestUID, "", "")+ require.Equal(t, http.StatusNoContent, rec.Code)+ pf, body := decodePolicyFile(t, h.fileFor(policyTestUID))+ assert.Equal(t, int64(3), pf.Revision)+ assert.Contains(t, string(body), `"defaultAction":"deny"`)++ // Fresh server over the same persistence state: the binding replay must+ // load the tombstone, not reseed the bootstrap policy.+ srv2, reg2, nft2 := fastSandboxTestServer(t)+ pp2, err := newPolicyPersistence(h.dir)+ require.NoError(t, err)+ pp2.newFile = h.pp.newFile+ srv2.SetPolicyFiles(pp2)++ rec = doAction(t, srv2, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 1, strPtr(policyAllow())))+ require.Equal(t, http.StatusOK, rec.Code)+ rec = doAction(t, srv2, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookRuntimeReady))+ require.Equal(t, http.StatusOK, rec.Code)+ rec = doAction(t, srv2, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookDataPlaneReady))+ require.Equal(t, http.StatusOK, rec.Code)++ rec = doPolicy(t, srv2, http.MethodGet, policyTestUID, "", "")+ rev, doc := mustPolicyRevision(t, rec)+ assert.Equal(t, int64(3), rev)+ assert.Contains(t, string(doc.Policy), `"defaultAction":"deny"`)+ assert.NotContains(t, string(doc.Policy), "api.example.com", "the deleted policy must not resurrect")++ // Enforcement is the deny tombstone, not the bootstrap policy.+ s := subject.FromSandboxUID(policyTestUID)+ assert.Equal(t, subject.StateActive, mustState(reg2.Get(s)))+ nft2.mu.Lock()+ last := nft2.lastPolicy+ nft2.mu.Unlock()+ require.NotNil(t, last)+ assert.Equal(t, "deny", last.Evaluate("api.example.com"))+ assert.Equal(t, "deny", last.Evaluate("other.example.com"))+}+// ---------------------------------------------------------------------------// Capability + healthy posture// ---------------------------------------------------------------------------diff --git a/components/egress/fastsandbox_policy_relay_test.go b/components/egress/fastsandbox_policy_relay_test.goindex bc14e2c4f..8f02f817f 100644--- a/components/egress/fastsandbox_policy_relay_test.go+++ b/components/egress/fastsandbox_policy_relay_test.go@@ -160,18 +160,19 @@ func TestPolicyRelayPreconditions(t *testing.T) {rec = doPolicy(t, srv, http.MethodPut, policyTestUID, `{"policy":`+policyAllow()+`,"revision":1,"expectedRevision":2}`, "")require.Equal(t, http.StatusBadRequest, rec.Code)- // DELETE, then a conditional PUT against the missing record: 412, not a- // create.+ // DELETE lands a deny tombstone at the successor revision; a conditional+ // PUT against the pre-delete revision is a 412.rec = doPolicy(t, srv, http.MethodDelete, policyTestUID, "", "")require.Equal(t, http.StatusNoContent, rec.Code)rec = doPolicy(t, srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllow(), 4, false, true), "")require.Equal(t, http.StatusPreconditionFailed, rec.Code)- // Unconditional PUT recreates from scratch at revision 1.+ // Unconditional PUT replaces the tombstone at the successor revision.rec = doPolicy(t, srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllow(), 0, false, false), "")- require.Equal(t, http.StatusCreated, rec.Code)- rev, _ = mustPolicyRevision(t, rec)- assert.Equal(t, int64(1), rev)+ require.Equal(t, http.StatusOK, rec.Code)+ rev, doc := mustPolicyRevision(t, rec)+ assert.Equal(t, int64(6), rev)+ assert.Contains(t, string(doc.Policy), "api.example.com")}func TestPolicyRelayInvalidPolicyKeepsLastGood(t *testing.T) {@@ -197,30 +198,63 @@ func TestPolicyRelayInvalidPolicyKeepsLastGood(t *testing.T) {}func TestPolicyRelayDeleteRevertsDenyFirst(t *testing.T) {- srv, reg, _ := fastSandboxTestServer(t)+ srv, reg, nft := fastSandboxTestServer(t)s := setBindingAndReady(t, srv, reg, policyTestUID, vaultTestIP, strPtr(policyAllow()))require.Equal(t, subject.StateActive, mustState(reg.Get(s)))- // DELETE removes the seeded record and reverts the subject to deny-first- // — the binding policy goes with it (runtime truth is relay-owned).+ // DELETE removes the seeded policy (revision 1 → tombstone at 2) and+ // reverts the subject to deny-first.rec := doPolicy(t, srv, http.MethodDelete, policyTestUID, "", "")require.Equal(t, http.StatusNoContent, rec.Code)assert.Equal(t, subject.StateDenying, mustState(reg.Get(s)))assert.Nil(t, reg.UserPolicy(s))assert.Nil(t, reg.EffectivePolicy(s))++ // The tombstone is authoritative: GET serves the default-deny record at+ // the successor revision, and a binding replay cannot resurrect the+ // deleted policy.+ rec = doPolicy(t, srv, http.MethodGet, policyTestUID, "", "")+ rev, doc := mustPolicyRevision(t, rec)+ assert.Equal(t, int64(2), rev)+ assert.Contains(t, string(doc.Policy), `"defaultAction":"deny"`)+ assert.NotContains(t, string(doc.Policy), "api.example.com")+ rec = doAction(t, srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 2, strPtr(policyAllow())))+ require.Equal(t, http.StatusOK, rec.Code)rec = doPolicy(t, srv, http.MethodGet, policyTestUID, "", "")- assert.Equal(t, http.StatusNotFound, rec.Code)+ rev, doc = mustPolicyRevision(t, rec)+ assert.Equal(t, int64(2), rev, "the tombstone survives the binding replay")+ assert.NotContains(t, string(doc.Policy), "api.example.com")- // Duplicate DELETE is a 404.+ // A replayed data-plane-ready activates the deny record (the barrier+ // consumes the tombstone's pending policy) — never a stale policy.+ rec = doAction(t, srv, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookDataPlaneReady))+ require.Equal(t, http.StatusOK, rec.Code)+ assert.Equal(t, subject.StateActive, mustState(reg.Get(s)))+ nft.mu.Lock()+ last := nft.lastPolicy+ nft.mu.Unlock()+ require.NotNil(t, last)+ assert.Equal(t, "deny", last.Evaluate("api.example.com"), "the enforced policy is the deny tombstone")++ // DELETE is idempotent (the tombstone replaces itself at the successor+ // revision).rec = doPolicy(t, srv, http.MethodDelete, policyTestUID, "", "")- assert.Equal(t, http.StatusNotFound, rec.Code)-- // A re-add via SET_BINDING + data-plane-ready reactivates.- rec = doAction(t, srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 2, strPtr(policyAllow())))+ assert.Equal(t, http.StatusNoContent, rec.Code)+ rev, _ = mustPolicyRevision(t, doPolicy(t, srv, http.MethodGet, policyTestUID, "", ""))+ assert.Equal(t, int64(3), rev)++ // Re-add goes through the relay: the PUT replaces the tombstone. The+ // subject was reverted to deny-first by the second DELETE, so the write+ // is held pending until a data-plane-ready replay activates it.+ rec = doPolicy(t, srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllow(), 3, false, true), "")require.Equal(t, http.StatusOK, rec.Code)+ rev, doc = mustPolicyRevision(t, rec)+ assert.Equal(t, int64(4), rev)+ assert.Equal(t, subject.StateDenying, mustState(reg.Get(s)))rec = doAction(t, srv, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookDataPlaneReady))require.Equal(t, http.StatusOK, rec.Code)assert.Equal(t, subject.StateActive, mustState(reg.Get(s)))+ assert.Equal(t, "allow", reg_UserEvaluate(srv, s, "api.example.com"))}func TestPolicyRelayDenyingSubjectHoldsPending(t *testing.T) {diff --git a/components/egress/fastsandbox_policy_snapshot.go b/components/egress/fastsandbox_policy_snapshot.goindex 9bb9e9f9d..2458deb7e 100644--- a/components/egress/fastsandbox_policy_snapshot.go+++ b/components/egress/fastsandbox_policy_snapshot.go@@ -80,16 +80,23 @@ func (s *fastSandboxPolicyServer) applyPolicyExport(env *actionhandler.Envelope)return http.StatusOK, nil}- // Phase 1: guards + consistent source image.+ // Phase 1: guards + consistent source image, serialized with the policy+ // relay commits (policyCommitMu). A relay writer that passed its guards+ // but has not committed yet is either flushed before the snapshot (the+ // copy carries its write) or blocked until phase 3 — it can never+ // commit "under" the seal.+ s.policyCommitMu.Lock()s.mu.Lock()snap, status, err := s.policyExportSourceImageLocked(subj, fence)+ s.mu.Unlock()+ s.policyCommitMu.Unlock()if err != nil {- s.mu.Unlock()return status, err}- s.mu.Unlock()- // Phase 2: no lock held across the driver-owned write.+ // Phase 2: no lock held across the driver-owned write. A relay commit+ // here is caught by the phase-3 re-check (the copied image no longer+ // matches the live file → 409, no seal).if snap.Exists() {bundle, err := s.policyFiles.newFile(target)if err != nil {@@ -101,7 +108,11 @@ func (s *fastSandboxPolicyServer) applyPolicyExport(env *actionhandler.Envelope)}}- // Phase 3: seal only against the exact copied image.+ // Phase 3: seal only against the exact copied image, again serialized+ // with the relay commits — between the re-check and the seal no writer+ // can pass the guards (they now observe the seal) or commit.+ s.policyCommitMu.Lock()+ defer s.policyCommitMu.Unlock()s.mu.Lock()defer s.mu.Unlock()live, status, err := s.policyExportSourceImageLocked(subj, fence)@@ -266,8 +277,6 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)s.mu.Unlock()return http.StatusConflict, fmt.Errorf("stale policy import (fence mismatch) for subject %s", subj)}- state, _ := s.reg.Get(subj)- stateActive := state == subject.StateActivesealed := falseif e := s.policies[subj]; e != nil && e.sealed {sealed = true@@ -287,7 +296,7 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)s.mu.Unlock()return s.policyFiles.unseal(subj)}- return s.installImportedPolicyLocked(subj, fence, pf.Revision, pol, stateActive, sealed)+ return s.installImportedPolicyLocked(subj, fence, pf.Revision, pol, sealed)},)if err != nil {@@ -310,32 +319,35 @@ func (s *fastSandboxPolicyServer) applyPolicyImport(env *actionhandler.Envelope)// installImportedPolicyLocked binds the imported revision into the live// per-subject record under the restored fence, replacing any stale record,-// and unseals it. The record is only replaced once enforcement (or the-// pending store) accepted — a failure leaves the live state untouched while-// the commit protocol rolls the file back. Enforcement runs WITHOUT s.mu-// (it takes the registry write lock); the s.mu suffix is the docstring's-// "Locked" sense: map mutations under s.mu, unseal after acceptance.-func (s *fastSandboxPolicyServer) installImportedPolicyLocked(subj subject.Subject, fence subject.Fencing, revision int64, pol *policy.NetworkPolicy, stateActive, wasSealed bool) error {- if stateActive {+// and unseals it. Ordering inside the commit protocol's apply step: the+// durable unseal FIRST (a failure here mutates nothing live — the commit+// protocol rolls the file back and the seal stays), then the state/fence+// re-read, then enforcement, then the record. Enforcement runs WITHOUT+// s.mu (it takes the registry write lock); the s.mu suffix is the+// docstring's "Locked" sense: map mutations under s.mu.+func (s *fastSandboxPolicyServer) installImportedPolicyLocked(subj subject.Subject, fence subject.Fencing, revision int64, pol *policy.NetworkPolicy, wasSealed bool) error {+ if wasSealed {+ if err := s.policyFiles.unseal(subj); err != nil {+ return err+ }+ }+ state, err := s.policyCommitTarget(subj, fence)+ if err != nil {+ return err+ }+ if state == subject.StateActive {// Enforcement first: a failed apply leaves the record untouched- // (the commit protocol rolls the file back and keeps the seal).+ // (the commit protocol rolls the file back).if err := s.applyPolicy(subj, pol); err != nil {return err}}s.mu.Lock()- if !stateActive {+ if state != subject.StateActive {s.pendingPolicies[subj] = pol}s.policies[subj] = &policyEntry{revision: revision, fence: fence}s.mu.Unlock()- // Unseal only after the state accepted — a failed apply rolls the file- // back and keeps the seal.- if wasSealed {- if err := s.policyFiles.unseal(subj); err != nil {- return err- }- }return nil}diff --git a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yamlindex 0392da8b9..f71129599 100644--- a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml+++ b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml@@ -78,6 +78,14 @@ spec:# keep the vault memory-only (lost on egress restart).- name: OPENSANDBOX_EGRESS_VAULT_DIRvalue: /var/lib/opensandbox-egress/vault+ # Policy persistence: one per-subject file under this directory backs+ # the /policy/{uid} relay ack the same way (persist → apply → ack)+ # and carries the POLICY_EXPORT / POLICY_IMPORT snapshot transfer.+ # Lives on the same Pod-scoped volume as the vault files. Unset+ # would keep the recorded policy memory-only and omit the+ # policyTransfer capability.+ - name: OPENSANDBOX_EGRESS_POLICY_DIR+ value: /var/lib/opensandbox-egress/policyvolumeMounts:- name: egress-vault# Plaintext v1 posture: the vault volume joins the credential