Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): same-commit policy reads, durable removal, pending consistency

高然f870dd3bf38a
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index a595e29ff..17e20a83f 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -244,7 +244,12 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (
// binding's policy to a policy-less subject) and purge the recorded
// policy, so deny-first is also the durable state.
s.clearPendingPolicy(subj)
- s.clearPolicyRecord(subj)
+ if err := s.clearPolicyRecord(subj); err != nil {
+ // The durable removal did not confirm: fail the action so the
+ // Fastlet's retry re-runs the purge/tombstone instead of acking
+ // a removal whose file may still carry the removed policy.
+ return http.StatusInternalServerError, fmt.Errorf("policy record removal: %w", err)
+ }
if state == subject.StateActive {
if err := s.revertToDenyFirst(subj, att); err != nil {
return http.StatusInternalServerError, err
diff --git a/components/egress/fastsandbox_policy.go b/components/egress/fastsandbox_policy.go
index 080e81d4f..5aee46b11 100644
--- a/components/egress/fastsandbox_policy.go
+++ b/components/egress/fastsandbox_policy.go
@@ -112,6 +112,14 @@ func (s *fastSandboxPolicyServer) handlePolicyRelay(w http.ResponseWriter, r *ht
// between the read and the lock is bounded: the purge removes the record,
// and the write paths re-validate the fence at commit time.
func (s *fastSandboxPolicyServer) selectRelayPolicy(r *http.Request, subj subject.Subject) (*policyRelayDocument, int, string) {
+ // Serialized with the policy commits: the policy body and its revision
+ // must come from the SAME commit, or a GET overlapping a PUT would pair
+ // the old policy with the new revision — and the server's GET→merge→PUT
+ // PATCH would then present that revision and overwrite the concurrent
+ // replacement. Lock order: policyCommitMu → registry → s.mu (matches
+ // the commit paths).
+ s.policyCommitMu.Lock()
+ defer s.policyCommitMu.Unlock()
fence, ok := s.reg.Fence(subj)
if !ok {
return nil, http.StatusNotFound, "unknown subject"
@@ -310,9 +318,11 @@ func (s *fastSandboxPolicyServer) putRelayPolicy(w http.ResponseWriter, r *http.
}
}
s.mu.Lock()
- if state != subject.StateActive {
- s.pendingPolicies[subj] = pol
- }
+ // The pending slot mirrors what is enforced in BOTH states: on a
+ // denying subject it is the barrier's payload; on an active one it
+ // keeps an at-least-once data-plane-ready replay idempotent (a
+ // stale deny tombstone must never re-enforce over the replacement).
+ s.pendingPolicies[subj] = pol
// A fresh entry also clears a degraded mark: the successful write
// IS the heal.
s.policies[subj] = &policyEntry{revision: newRev, fence: fence}
diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.go
index e211a39a8..a6ef67256 100644
--- a/components/egress/fastsandbox_policy_persist.go
+++ b/components/egress/fastsandbox_policy_persist.go
@@ -291,19 +291,67 @@ func (s *fastSandboxPolicyServer) policyHealthIssue() string {
}
// clearPolicyRecord removes the subject's recorded policy (binding removal
-// on a still-live sandbox): the entry goes and the persisted file is purged,
-// so the deny-first state is also the durable state — a restart cannot
-// resurrect a removed policy, and a later registration starts from its own
-// input. Serialized with the relay commits (policyCommitMu) so an in-flight
-// write against the removed binding cannot recreate the record after the
-// purge; callers are never under the registry write lock.
-func (s *fastSandboxPolicyServer) clearPolicyRecord(subj subject.Subject) {
+// on a still-live sandbox): the in-memory record and pending policy go, and
+// the persisted file is purged so the deny-first state is also the durable
+// state. An uncertain/failed purge falls back to persisting a default-deny
+// tombstone at the successor revision — a leftover file whose recorded
+// content still matches the removed policy would otherwise be loaded by a
+// later same-fence registration or restart, resurrecting an acknowledged-
+// away policy. A tombstone persist failure latches recovery (writes frozen
+// until restart). Serialized with the relay commits (policyCommitMu) so an
+// in-flight write against the removed binding cannot recreate the record
+// after the purge; callers are never under the registry write lock.
+func (s *fastSandboxPolicyServer) clearPolicyRecord(subj subject.Subject) error {
s.policyCommitMu.Lock()
defer s.policyCommitMu.Unlock()
s.mu.Lock()
+ entry := s.policies[subj]
+ pp := s.policyFiles
+ s.mu.Unlock()
+ if entry == nil {
+ return nil
+ }
+ s.mu.Lock()
delete(s.policies, subj)
+ delete(s.pendingPolicies, subj)
s.mu.Unlock()
- s.purgePolicyFile(subj)
+ if pp == nil {
+ return nil
+ }
+ if err := pp.purge(subj); err == nil {
+ return nil
+ } else {
+ log.Warnf("subject %s: policy file purge failed (%v); persisting a deny tombstone instead", subj, err)
+ }
+ deny := policy.DefaultDenyPolicy()
+ raw, err := json.Marshal(deny)
+ if err != nil {
+ return fmt.Errorf("deny tombstone marshal: %w", err)
+ }
+ rev := entry.revision + 1
+ payload, err := policy.EncodePolicyFile(entry.fence.RuntimeInstanceID, entry.fence.AttachmentID, rev, raw)
+ if err != nil {
+ return fmt.Errorf("deny tombstone encode: %w", err)
+ }
+ err = s.persistPolicyCandidateLocked(
+ subj,
+ func(fs vaultFileStore) (policy.FileMutationState, error) {
+ return fs.SaveRaw(payload)
+ },
+ func() error {
+ s.mu.Lock()
+ s.pendingPolicies[subj] = deny
+ s.policies[subj] = &policyEntry{revision: rev, fence: entry.fence}
+ s.mu.Unlock()
+ return nil
+ },
+ )
+ if err != nil {
+ log.Errorf("subject %s: deny tombstone persist failed: %v", subj, err)
+ return fmt.Errorf("deny tombstone persist: %w", err)
+ }
+ log.Infof("subject %s: deny tombstone persisted after purge failure (revision %d)", subj, rev)
+ return nil
}
// rehydratePolicyReload is the egress-restart recovery path: at
diff --git a/components/egress/fastsandbox_policy_persist_test.go b/components/egress/fastsandbox_policy_persist_test.go
index 72ce10a8f..e94005d41 100644
--- a/components/egress/fastsandbox_policy_persist_test.go
+++ b/components/egress/fastsandbox_policy_persist_test.go
@@ -547,6 +547,38 @@ func TestPolicyPersistStaleRecordNeverSupersedesFreshRegistration(t *testing.T)
assert.Equal(t, newRuntime, pf.Incarnation)
}
+// TestPolicyPersistRemovalPurgeFailureLandsDenyTombstone (review F3): a
+// binding removal whose durable purge fails must not leave the removed
+// policy's file recoverable — a deny tombstone lands instead, so a later
+// same-fence registration loads deny, never the removed policy.
+func TestPolicyPersistRemovalPurgeFailureLandsDenyTombstone(t *testing.T) {
+ h := newPolicyPersistHarness(t)
+ h.readySubject(t, policyTestUID, vaultTestIP)
+
+ // Runtime mutation to rev 2, then a removal whose unlink fails.
+ rec := doPolicy(t, h.srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllowAlt(), 1, false, true), "")
+ require.Equal(t, http.StatusOK, rec.Code)
+ h.fileFor(policyTestUID).failFirstDelete = errors.New("unlink EIO")
+
+ rec = doAction(t, h.srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 2, nil))
+ require.Equal(t, http.StatusOK, rec.Code)
+
+ // The tombstone replaced the removed policy durably.
+ pf, body := decodePolicyFile(t, h.fileFor(policyTestUID))
+ assert.Equal(t, int64(3), pf.Revision)
+ assert.Contains(t, string(body), `"defaultAction":"deny"`)
+ assert.NotContains(t, string(body), "other.example.com", "the removed policy must not survive the removal")
+
+ // A later same-fence registration loads the tombstone — never the
+ // removed policy.
+ rec = doAction(t, h.srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 3, strPtr(policyAllow())))
+ require.Equal(t, http.StatusOK, rec.Code)
+ rec = doPolicy(t, h.srv, http.MethodGet, policyTestUID, "", "")
+ rev, doc := mustPolicyRevision(t, rec)
+ assert.Equal(t, int64(3), rev)
+ assert.Contains(t, string(doc.Policy), `"defaultAction":"deny"`)
+}
+
// ---------------------------------------------------------------------------
// Capability + healthy posture
// ---------------------------------------------------------------------------
diff --git a/components/egress/fastsandbox_policy_relay_test.go b/components/egress/fastsandbox_policy_relay_test.go
index 8ed73ba18..c0efe8632 100644
--- a/components/egress/fastsandbox_policy_relay_test.go
+++ b/components/egress/fastsandbox_policy_relay_test.go
@@ -276,6 +276,31 @@ func TestPolicyRelayDenyingSubjectHoldsPending(t *testing.T) {
assert.Equal(t, "allow", reg_UserEvaluate(srv, s, "other.example.com"))
}
+// TestPolicyRelayPutReplacesStalePendingTombstone (review F4): after a
+// DELETE on an active subject, the pending slot holds the deny tombstone; a
+// later relay PUT must replace it, so an at-least-once data-plane-ready
+// replay re-applies the REPLACEMENT, never the stale deny.
+func TestPolicyRelayPutReplacesStalePendingTombstone(t *testing.T) {
+ srv, reg, _ := fastSandboxTestServer(t)
+ s := setBindingAndReady(t, srv, reg, policyTestUID, vaultTestIP, strPtr(policyAllow()))
+
+ // DELETE lands the deny tombstone (active, enforced in place).
+ rec := doPolicy(t, srv, http.MethodDelete, policyTestUID, "", "")
+ require.Equal(t, http.StatusNoContent, rec.Code)
+
+ // The relay PUT applies in place AND replaces the pending tombstone.
+ rec = doPolicy(t, srv, http.MethodPut, policyTestUID, policyPutDoc(policyAllowAlt(), 2, false, true), "")
+ require.Equal(t, http.StatusOK, rec.Code)
+
+ // A replayed data-plane-ready consumes the pending policy: it must be
+ // the replacement, not the stale deny.
+ rec = doAction(t, srv, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookDataPlaneReady))
+ require.Equal(t, http.StatusOK, rec.Code)
+ assert.Equal(t, subject.StateActive, mustState(reg.Get(s)))
+ assert.Equal(t, "allow", reg_UserEvaluate(srv, s, "other.example.com"))
+ assert.Equal(t, "deny", reg_UserEvaluate(srv, s, "api.example.com"))
+}
+
// reg_UserEvaluate evaluates the subject's recorded user policy (test aid to
// keep assertions readable).
func reg_UserEvaluate(srv *fastSandboxPolicyServer, s subject.Subject, domain string) string {
diff --git a/components/egress/fastsandbox_policy_snapshot.go b/components/egress/fastsandbox_policy_snapshot.go
index 6ce9b7c87..5a14c420c 100644
--- a/components/egress/fastsandbox_policy_snapshot.go
+++ b/components/egress/fastsandbox_policy_snapshot.go
@@ -358,9 +358,9 @@ func (s *fastSandboxPolicyServer) installImportedPolicyLocked(subj subject.Subje
}
}
s.mu.Lock()
- if state != subject.StateActive {
- s.pendingPolicies[subj] = pol
- }
+ // The pending slot mirrors what is enforced (same rationale as the
+ // relay PUT: a replayed data-plane-ready must be idempotent).
+ s.pendingPolicies[subj] = pol
s.policies[subj] = &policyEntry{revision: revision, fence: fence}
s.mu.Unlock()
return nil
diff --git a/server/opensandbox_server/services/fast_sandbox/policy_relay.py b/server/opensandbox_server/services/fast_sandbox/policy_relay.py
index 6eb84ca55..628a94ba6 100644
--- a/server/opensandbox_server/services/fast_sandbox/policy_relay.py
+++ b/server/opensandbox_server/services/fast_sandbox/policy_relay.py
@@ -56,7 +56,18 @@ _POLICY_PATH_PREFIX = "/policy/"
def policy_relay_error(status_code: int, message: str) -> HTTPException:
- """Map an egress relay status onto the public HTTP contract."""
+ """Map an egress relay status onto the public HTTP contract.
+
+ The egress reports revision conflicts as 412 (handler-relay internal
+ contract); the public network-policy operations declare 409, so the
+ conflict is remapped here instead of leaking an undocumented status to
+ generated clients.
+ """
+ if status_code == status.HTTP_412_PRECONDITION_FAILED:
+ return HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail={"code": SandboxErrorCodes.FSB_API_ERROR, "message": message},
+ )
if status_code == 400:
return HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
diff --git a/server/tests/test_fsb_service.py b/server/tests/test_fsb_service.py
index 44dad9802..b5f0c4c45 100644
--- a/server/tests/test_fsb_service.py
+++ b/server/tests/test_fsb_service.py
@@ -951,7 +951,8 @@ def test_http_policy_patch_and_delete_match_sidecar_semantics(persisted_fsb):
# Conflict protection still applies to merged commits: the merged PUT
# carries the read revision as expectedRevision — a concurrent relay
# write landing between the merge-read and the commit turns the stale
- # merge into a 412.
+ # merge into a 409 (the egress 412 is remapped to the public conflict
+ # status).
from opensandbox_server.services.fast_sandbox import policy_relay as policy_relay_module
real_get_full = policy_relay_module._relay_get_full
@@ -966,7 +967,7 @@ def test_http_policy_patch_and_delete_match_sidecar_semantics(persisted_fsb):
response = client.patch(url, json=[{"action": "deny", "target": "d.com"}])
finally:
policy_relay_module._relay_get_full = real_get_full
- assert response.status_code == 412
+ assert response.status_code == 409
def test_policy_rejects_invalid_input_and_other_tenant(persisted_fsb):
diff --git a/specs/sandbox-lifecycle.yml b/specs/sandbox-lifecycle.yml
index 04ce4c63c..4b19376de 100644
--- a/specs/sandbox-lifecycle.yml
+++ b/specs/sandbox-lifecycle.yml
@@ -59,15 +59,10 @@ paths:
operationId: getSandboxNetworkPolicy
summary: Read sandbox network policy
description: |
- Fsb reads the egress-held record — the response reflects exactly what
- the egress enforces, not the create-time binding input. Other backends
- proxy the sandbox-side egress service.
- description: |
- For Fsb, reads persisted egress Action Binding intent in the tenant's
- Sandbox CR, not live enforcement state. An absent binding defaults to
- deny-first when an egress handler is configured; this is not proof of
- enforcement on a pool without that handler. Other backends proxy the
- sandbox-side egress service. Requires lifecycle API authentication.
+ Fsb reads the egress-held policy record — the response reflects
+ exactly what the egress enforces, not the create-time binding input.
+ Other backends proxy the sandbox-side egress service. Requires
+ lifecycle API authentication.
responses:
'200':
description: Policy intent (Fsb) or sidecar policy (other backends).