Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(egress): same-commit policy reads, durable removal, pending consistency
高然f870dd3bf38a
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.goindex a595e29ff..17e20a83f 100644--- a/components/egress/fastsandbox_actions.go+++ b/components/egress/fastsandbox_actions.go@@ -244,7 +244,12 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (// binding's policy to a policy-less subject) and purge the recorded// policy, so deny-first is also the durable state.s.clearPendingPolicy(subj)- s.clearPolicyRecord(subj)+ if err := s.clearPolicyRecord(subj); err != nil {+ // The durable removal did not confirm: fail the action so the+ // Fastlet's retry re-runs the purge/tombstone instead of acking+ // a removal whose file may still carry the removed policy.+ return http.StatusInternalServerError, fmt.Errorf("policy record removal: %w", err)+ }if state == subject.StateActive {if err := s.revertToDenyFirst(subj, att); err != nil {return http.StatusInternalServerError, errdiff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.goindex 080e81d4f..5aee46b11 100644--- a/components/egress/fastsandbox_policy.go+++ b/components/egress/fastsandbox_policy.go@@ -112,6 +112,14 @@ func (s *fastSandboxPolicyServer) handlePolicyRelay(w http.ResponseWriter, r *ht// between the read and the lock is bounded: the purge removes the record,// and the write paths re-validate the fence at commit time.func (s *fastSandboxPolicyServer) selectRelayPolicy(r *http.Request, subj subject.Subject) (*policyRelayDocument, int, string) {+ // Serialized with the policy commits: the policy body and its revision+ // must come from the SAME commit, or a GET overlapping a PUT would pair+ // the old policy with the new revision — and the server's GET→merge→PUT+ // PATCH would then present that revision and overwrite the concurrent+ // replacement. Lock order: policyCommitMu → registry → s.mu (matches+ // the commit paths).+ s.policyCommitMu.Lock()+ defer s.policyCommitMu.Unlock()fence, ok := s.reg.Fence(subj)if !ok {return nil, http.StatusNotFound, "unknown subject"@@ -310,9 +318,11 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.}}s.mu.Lock()- if state != subject.StateActive {- s.pendingPolicies[subj] = pol- }+ // The pending slot mirrors what is enforced in BOTH states: on a+ // denying subject it is the barrier's payload; on an active one it+ // keeps an at-least-once data-plane-ready replay idempotent (a+ // stale deny tombstone must never re-enforce over the replacement).+ s.pendingPolicies[subj] = pol// A fresh entry also clears a degraded mark: the successful write// IS the heal.s.policies[subj] = &policyEntry{revision: newRev, fence: fence}diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.goindex e211a39a8..a6ef67256 100644--- a/components/egress/fastsandbox_policy_persist.go+++ b/components/egress/fastsandbox_policy_persist.go@@ -291,19 +291,67 @@ func (s *fastSandboxPolicyServer) policyHealthIssue() string {}// clearPolicyRecord removes the subject's recorded policy (binding removal-// on a still-live sandbox): the entry goes and the persisted file is purged,-// so the deny-first state is also the durable state — a restart cannot-// resurrect a removed policy, and a later registration starts from its own-// input. Serialized with the relay commits (policyCommitMu) so an in-flight-// write against the removed binding cannot recreate the record after the-// purge; callers are never under the registry write lock.-func (s *fastSandboxPolicyServer) clearPolicyRecord(subj subject.Subject) {+// on a still-live sandbox): the in-memory record and pending policy go, and+// the persisted file is purged so the deny-first state is also the durable+// state. An uncertain/failed purge falls back to persisting a default-deny+// tombstone at the successor revision — a leftover file whose recorded+// content still matches the removed policy would otherwise be loaded by a+// later same-fence registration or restart, resurrecting an acknowledged-+// away policy. A tombstone persist failure latches recovery (writes frozen+// until restart). Serialized with the relay commits (policyCommitMu) so an+// in-flight write against the removed binding cannot recreate the record+// after the purge; callers are never under the registry write lock.+func (s *fastSandboxPolicyServer) clearPolicyRecord(subj subject.Subject) error {s.policyCommitMu.Lock()defer s.policyCommitMu.Unlock()s.mu.Lock()+ entry := s.policies[subj]+ pp := s.policyFiles+ s.mu.Unlock()+ if entry == nil {+ return nil+ }+ s.mu.Lock()delete(s.policies, subj)+ delete(s.pendingPolicies, subj)s.mu.Unlock()- s.purgePolicyFile(subj)+ if pp == nil {+ return nil+ }+ if err := pp.purge(subj); err == nil {+ return nil+ } else {+ log.Warnf("subject %s: policy file purge failed (%v); persisting a deny tombstone instead", subj, err)+ }+ deny := policy.DefaultDenyPolicy()+ raw, err := json.Marshal(deny)+ if err != nil {+ return fmt.Errorf("deny tombstone marshal: %w", err)+ }+ rev := entry.revision + 1+ payload, err := policy.EncodePolicyFile(entry.fence.RuntimeInstanceID, entry.fence.AttachmentID, rev, raw)+ if err != nil {+ return fmt.Errorf("deny tombstone encode: %w", err)+ }+ err = s.persistPolicyCandidateLocked(+ subj,+ func(fs vaultFileStore) (policy.FileMutationState, error) {+ return fs.SaveRaw(payload)+ },+ func() error {+ s.mu.Lock()+ s.pendingPolicies[subj] = deny+ s.policies[subj] = &policyEntry{revision: rev, fence: entry.fence}+ s.mu.Unlock()+ return nil+ },+ )+ if err != nil {+ log.Errorf("subject %s: deny tombstone persist failed: %v", subj, err)+ return fmt.Errorf("deny tombstone persist: %w", err)+ }+ log.Infof("subject %s: deny tombstone persisted after purge failure (revision %d)", subj, rev)+ return nil}// rehydratePolicyReload is the egress-restart recovery path: atdiff --git a/components/egress/fastsandbox_policy_persist_test.go b/components/egress/fastsandbox_policy_persist_test.goindex 72ce10a8f..e94005d41 100644--- a/components/egress/fastsandbox_policy_persist_test.go+++ b/components/egress/fastsandbox_policy_persist_test.go@@ -547,6 +547,38 @@ func TestPolicyPersistStaleRecordNeverSupersedesFreshRegistration(t *testing.T)assert.Equal(t, newRuntime, pf.Incarnation)}+// TestPolicyPersistRemovalPurgeFailureLandsDenyTombstone (review F3): a+// binding removal whose durable purge fails must not leave the removed+// policy's file recoverable — a deny tombstone lands instead, so a later+// same-fence registration loads deny, never the removed policy.+func TestPolicyPersistRemovalPurgeFailureLandsDenyTombstone(t *testing.T) {+ h := newPolicyPersistHarness(t)+ h.readySubject(t, policyTestUID, vaultTestIP)++ // Runtime mutation to rev 2, then a removal whose unlink fails.+ rec := doPolicy(t, h.srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllowAlt(), 1, false, true), "")+ require.Equal(t, http.StatusOK, rec.Code)+ h.fileFor(policyTestUID).failFirstDelete = errors.New("unlink EIO")++ rec = doAction(t, h.srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 2, nil))+ require.Equal(t, http.StatusOK, rec.Code)++ // The tombstone replaced the removed policy durably.+ pf, body := decodePolicyFile(t, h.fileFor(policyTestUID))+ assert.Equal(t, int64(3), pf.Revision)+ assert.Contains(t, string(body), `"defaultAction":"deny"`)+ assert.NotContains(t, string(body), "other.example.com", "the removed policy must not survive the removal")++ // A later same-fence registration loads the tombstone — never the+ // removed policy.+ rec = doAction(t, h.srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 3, strPtr(policyAllow())))+ require.Equal(t, http.StatusOK, rec.Code)+ rec = doPolicy(t, h.srv, http.MethodGet, policyTestUID, "", "")+ rev, doc := mustPolicyRevision(t, rec)+ assert.Equal(t, int64(3), rev)+ assert.Contains(t, string(doc.Policy), `"defaultAction":"deny"`)+}+// ---------------------------------------------------------------------------// Capability + healthy posture// ---------------------------------------------------------------------------diff --git a/components/egress/fastsandbox_policy_relay_test.go b/components/egress/fastsandbox_policy_relay_test.goindex 8ed73ba18..c0efe8632 100644--- a/components/egress/fastsandbox_policy_relay_test.go+++ b/components/egress/fastsandbox_policy_relay_test.go@@ -276,6 +276,31 @@ func TestPolicyRelayDenyingSubjectHoldsPending(t *testing.T) {assert.Equal(t, "allow", reg_UserEvaluate(srv, s, "other.example.com"))}+// TestPolicyRelayPutReplacesStalePendingTombstone (review F4): after a+// DELETE on an active subject, the pending slot holds the deny tombstone; a+// later relay PUT must replace it, so an at-least-once data-plane-ready+// replay re-applies the REPLACEMENT, never the stale deny.+func TestPolicyRelayPutReplacesStalePendingTombstone(t *testing.T) {+ srv, reg, _ := fastSandboxTestServer(t)+ s := setBindingAndReady(t, srv, reg, policyTestUID, vaultTestIP, strPtr(policyAllow()))++ // DELETE lands the deny tombstone (active, enforced in place).+ rec := doPolicy(t, srv, http.MethodDelete, policyTestUID, "", "")+ require.Equal(t, http.StatusNoContent, rec.Code)++ // The relay PUT applies in place AND replaces the pending tombstone.+ rec = doPolicy(t, srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllowAlt(), 2, false, true), "")+ require.Equal(t, http.StatusOK, rec.Code)++ // A replayed data-plane-ready consumes the pending policy: it must be+ // the replacement, not the stale deny.+ rec = doAction(t, srv, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookDataPlaneReady))+ require.Equal(t, http.StatusOK, rec.Code)+ assert.Equal(t, subject.StateActive, mustState(reg.Get(s)))+ assert.Equal(t, "allow", reg_UserEvaluate(srv, s, "other.example.com"))+ assert.Equal(t, "deny", reg_UserEvaluate(srv, s, "api.example.com"))+}+// reg_UserEvaluate evaluates the subject's recorded user policy (test aid to// keep assertions readable).func reg_UserEvaluate(srv *fastSandboxPolicyServer, s subject.Subject, domain string) string {diff --git a/components/egress/fastsandbox_policy_snapshot.go b/components/egress/fastsandbox_policy_snapshot.goindex 6ce9b7c87..5a14c420c 100644--- a/components/egress/fastsandbox_policy_snapshot.go+++ b/components/egress/fastsandbox_policy_snapshot.go@@ -358,9 +358,9 @@ func (s *fastSandboxPolicyServer) installImportedPolicyLocked(subj subject.Subje}}s.mu.Lock()- if state != subject.StateActive {- s.pendingPolicies[subj] = pol- }+ // The pending slot mirrors what is enforced (same rationale as the+ // relay PUT: a replayed data-plane-ready must be idempotent).+ s.pendingPolicies[subj] = pols.policies[subj] = &policyEntry{revision: revision, fence: fence}s.mu.Unlock()return nildiff --git a/server/opensandbox_server/services/fast_sandbox/policy_relay.py b/server/opensandbox_server/services/fast_sandbox/policy_relay.pyindex 6eb84ca55..628a94ba6 100644--- a/server/opensandbox_server/services/fast_sandbox/policy_relay.py+++ b/server/opensandbox_server/services/fast_sandbox/policy_relay.py@@ -56,7 +56,18 @@ _POLICY_PATH_PREFIX = "/policy/"def policy_relay_error(status_code: int, message: str) -> HTTPException:- """Map an egress relay status onto the public HTTP contract."""+ """Map an egress relay status onto the public HTTP contract.++ The egress reports revision conflicts as 412 (handler-relay internal+ contract); the public network-policy operations declare 409, so the+ conflict is remapped here instead of leaking an undocumented status to+ generated clients.+ """+ if status_code == status.HTTP_412_PRECONDITION_FAILED:+ return HTTPException(+ status_code=status.HTTP_409_CONFLICT,+ detail={"code": SandboxErrorCodes.FSB_API_ERROR, "message": message},+ )if status_code == 400:return HTTPException(status_code=status.HTTP_400_BAD_REQUEST,diff --git a/server/tests/test_fsb_service.py b/server/tests/test_fsb_service.pyindex 44dad9802..b5f0c4c45 100644--- a/server/tests/test_fsb_service.py+++ b/server/tests/test_fsb_service.py@@ -951,7 +951,8 @@ def test_http_policy_patch_and_delete_match_sidecar_semantics(persisted_fsb):# Conflict protection still applies to merged commits: the merged PUT# carries the read revision as expectedRevision — a concurrent relay# write landing between the merge-read and the commit turns the stale- # merge into a 412.+ # merge into a 409 (the egress 412 is remapped to the public conflict+ # status).from opensandbox_server.services.fast_sandbox import policy_relay as policy_relay_modulereal_get_full = policy_relay_module._relay_get_full@@ -966,7 +967,7 @@ def test_http_policy_patch_and_delete_match_sidecar_semantics(persisted_fsb):response = client.patch(url, json=[{"action": "deny", "target": "d.com"}])finally:policy_relay_module._relay_get_full = real_get_full- assert response.status_code == 412+ assert response.status_code == 409def test_policy_rejects_invalid_input_and_other_tenant(persisted_fsb):diff --git a/specs/sandbox-lifecycle.yml b/specs/sandbox-lifecycle.ymlindex 04ce4c63c..4b19376de 100644--- a/specs/sandbox-lifecycle.yml+++ b/specs/sandbox-lifecycle.yml@@ -59,15 +59,10 @@ paths:operationId: getSandboxNetworkPolicysummary: Read sandbox network policydescription: |- Fsb reads the egress-held record — the response reflects exactly what- the egress enforces, not the create-time binding input. Other backends- proxy the sandbox-side egress service.- description: |- For Fsb, reads persisted egress Action Binding intent in the tenant's- Sandbox CR, not live enforcement state. An absent binding defaults to- deny-first when an egress handler is configured; this is not proof of- enforcement on a pool without that handler. Other backends proxy the- sandbox-side egress service. Requires lifecycle API authentication.+ Fsb reads the egress-held policy record — the response reflects+ exactly what the egress enforces, not the create-time binding input.+ Other backends proxy the sandbox-side egress service. Requires+ lifecycle API authentication.responses:'200':description: Policy intent (Fsb) or sidecar policy (other backends).