Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(egress): policy-state re-validation, durable removal retry, and review-round hardening
高然154770adde30
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.goindex 7ddd0ca73..a95bad755 100644--- a/components/egress/fastsandbox_actions.go+++ b/components/egress/fastsandbox_actions.go@@ -300,6 +300,12 @@ func (s *fastSandboxPolicyServer) applyLifecycleHook(env *actionhandler.Envelope// ever consume.s.policyCommitMu.Lock()defer s.policyCommitMu.Unlock()+ // In-lock fence re-validation: the pre-lock check cannot see a+ // rebind that completes while this Hook waits on policyCommitMu —+ // a stale Hook must never activate the successor.+ if !s.fenceMatchesLocked(subj, fence) {+ return http.StatusConflict, fmt.Errorf("stale %s (fence mismatch) for subject %s", env.Hook.Name, subj)+ }// Peek, do not consume: a transient nft failure must leave the// pending policy in place so the Fastlet's retry of the same Hook// succeeds instead of 409-ing forever with the subject deny-first.diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.goindex 1a80a394f..9e1c1cc79 100644--- a/components/egress/fastsandbox_policy_persist.go+++ b/components/egress/fastsandbox_policy_persist.go@@ -302,6 +302,14 @@ func (s *fastSandboxPolicyServer) clearPolicyRecordLocked(subj subject.Subject)pp := s.policyFiless.mu.Unlock()if entry == nil {+ // A previous attempt may have cleared the memory record but failed+ // the durable purge: retry the idempotent cleanup so the acked+ // removal is real (the Fastlet's retry lands here).+ if pp != nil {+ if err := pp.purge(subj); err != nil {+ return fmt.Errorf("policy file purge retry: %w", err)+ }+ }return nil}s.mu.Lock()@@ -382,15 +390,13 @@ func (s *fastSandboxPolicyServer) rehydratePolicyReload(subj subject.Subject, fereturn}if !snap.Exists() {- // Fresh subject: nothing persisted, the binding input seeds. A- // durable seal marker (an export that sealed a file-less subject)- // must still gate the relay ops — the seed entry carries it.+ // Fresh subject: the binding input seeds. A durable seal marker (an+ // export that sealed a file-less subject) still gates the relay+ // ops — the seed entry carries it (caller holds s.mu).if s.policyFiles.isSealed(subj) {- s.mu.Lock()s.policies[subj] = &policyEntry{fence: fence, sealed: true}s.pendingPolicies[subj] = policy.DefaultDenyPolicy()- s.mu.Unlock()- log.Infof("subject %s: seal restored from marker on a file-less subject (binding input seeds pending default-deny)", subj)+ log.Infof("subject %s: seal restored from marker (file-less subject)", subj)return}returndiff --git a/components/egress/fastsandbox_policy_persist_test.go b/components/egress/fastsandbox_policy_persist_test.goindex e94005d41..937e3e9fc 100644--- a/components/egress/fastsandbox_policy_persist_test.go+++ b/components/egress/fastsandbox_policy_persist_test.go@@ -22,6 +22,7 @@ import ("os""path/filepath""testing"+ "time""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -579,6 +580,57 @@ func TestPolicyPersistRemovalPurgeFailureLandsDenyTombstone(t *testing.T) {assert.Contains(t, string(doc.Policy), `"defaultAction":"deny"`)}+// TestPolicyPersistSealedFilelessSubjectSurvivesRestart (review C1): an+// export seals a file-less subject; the restart's binding replay must+// restore the seal WITHOUT deadlocking (the sealed branch runs under the+// reload's own s.mu), and the relay ops stay 409 with default-deny pending.+func TestPolicyPersistSealedFilelessSubjectSurvivesRestart(t *testing.T) {+ h := newPolicyPersistHarness(t)++ // Bind, then remove the policy (healthy purge → file-less), then export+ // (seals the file-less subject).+ h.readySubject(t, policyTestUID, vaultTestIP)+ rec := doAction(t, h.srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 2, nil))+ require.Equal(t, http.StatusOK, rec.Code)+ require.False(t, h.fileFor(policyTestUID).exists, "precondition: file-less after removal")+ bundlePath := filepath.Join(t.TempDir(), "bundle", policyTestUID+".json")+ rec = doAction(t, h.srv, policySnapshotBody(t, policyTestUID, testFenceR, testFenceA, "POLICY_EXPORT", bundlePath))+ require.Equal(t, http.StatusOK, rec.Code)+ h.srv.mu.Lock()+ require.True(t, h.srv.policies[subject.FromSandboxUID(policyTestUID)].sealed)+ h.srv.mu.Unlock()++ // Restart: fresh server over the same persistence state, then the+ // binding replay (non-removal). Must not deadlock; the seal and the+ // default-deny pending come back.+ srv2, reg2, _ := fastSandboxTestServer(t)+ pp2, err := newPolicyPersistence(h.dir)+ require.NoError(t, err)+ pp2.newFile = h.pp.newFile+ srv2.SetPolicyFiles(pp2)++ done := make(chan struct{})+ go func() {+ defer close(done)+ r := doAction(t, srv2, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 1, strPtr(policyAllow())))+ assert.Equal(t, http.StatusOK, r.Code)+ }()+ select {+ case <-done:+ case <-time.After(10 * time.Second):+ t.Fatal("deadlock: the sealed file-less replay never completed")+ }++ srv2.mu.Lock()+ e := srv2.policies[subject.FromSandboxUID(policyTestUID)]+ srv2.mu.Unlock()+ require.NotNil(t, e)+ assert.True(t, e.sealed, "the seal is restored from the marker")+ rec = doPolicy(t, srv2, http.MethodPut, policyTestUID, policyPutDoc(policyAllow(), 0, false, false), "")+ assert.Equal(t, http.StatusConflict, rec.Code, "relay ops stay 409 while sealed")+ _ = reg2+}+// ---------------------------------------------------------------------------// Capability + healthy posture// ---------------------------------------------------------------------------diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.goindex c186fc3fb..95abbd315 100644--- a/components/egress/fastsandbox_server.go+++ b/components/egress/fastsandbox_server.go@@ -383,10 +383,9 @@ func (s *fastSandboxPolicyServer) applyPolicy(subj subject.Subject, pol *policy.s.policyMu.Lock()defer s.policyMu.Unlock()eff := s.reg.EffectiveOf(pol)- // Bounded: applyPolicy runs under policyCommitMu, so one slow nft- // transaction would head-of-line-block every subject's writes. The- // registration deny-first install is not under that serialization and- // keeps a 30s budget.+ // Bounded (policyApplyTimeout): every applyPolicy caller runs under+ // policyCommitMu, so one slow nft transaction would head-of-line-block+ // every subject's writes.nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)defer cancel()if err := s.nft.ApplyPolicy(nftCtx, subj, eff); err != nil {@@ -447,7 +446,10 @@ func (s *fastSandboxPolicyServer) dropVault(subj subject.Subject) {// (nft rules + gateway DNS redirect + MITM interception). Runs under the// registry write lock, so no registry calls here.func (s *fastSandboxPolicyServer) OnRegistered(subj subject.Subject, att actionhandler.NetworkAttachment) error {- nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)+ // Bounded (policyApplyTimeout): SET_BINDING runs under policyCommitMu,+ // so a slow deny-first install would head-of-line-block every subject's+ // policy writes.+ nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)defer cancel()if err := s.nft.ApplyDenyFirst(nftCtx, subj, att); err != nil {return errdiff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.goindex 8888955b0..3621b7811 100644--- a/components/egress/fastsandbox_vault.go+++ b/components/egress/fastsandbox_vault.go@@ -123,8 +123,16 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r// The body has been consumed; a late response must not try to read it.r.Body.Close()- // Registry reads before s.mu (lock order: registry → s.mu, see- // selectRelayVault).+ // The whole write holds policyCommitMu: the validation policy must not+ // change between the pre-read and the commit (policy mutations — relay+ // PUT/DELETE, hooks, removals — serialize on the same mutex, and none+ // of them bump the fence, so the commit-time fence re-check alone+ // cannot catch this). Lock order: policyCommitMu → registry → s.mu.+ s.policyCommitMu.Lock()+ defer s.policyCommitMu.Unlock()++ // Registry reads before s.mu (lock order: policyCommitMu → registry →+ // s.mu).fence, ok := s.reg.Fence(subj)if !ok {http.Error(w, "unknown subject", http.StatusNotFound)diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.goindex 373fd7493..a0e984371 100644--- a/components/egress/fastsandbox_vault_persist.go+++ b/components/egress/fastsandbox_vault_persist.go@@ -177,7 +177,7 @@ func restrictVolumeRoot(dir, noun string) error {}entries, err := os.ReadDir(parent)if err != nil {- return fmt.Errorf("list vault volume root %s: %w", parent, err)+ return fmt.Errorf("list %s volume root %s: %w", noun, parent, err)}base := filepath.Base(dir)for _, entry := range entries {diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.goindex 51cc5b3bc..e75282c13 100644--- a/components/egress/fastsandbox_vault_snapshot.go+++ b/components/egress/fastsandbox_vault_snapshot.go@@ -259,9 +259,12 @@ func (s *fastSandboxPolicyServer) applyVaultImport(env *actionhandler.Envelope)return http.StatusInternalServerError, fmt.Errorf("vault bundle re-stamp failed: %w", err)}- // Registry reads before s.mu (lock order: registry → s.mu, see- // handleVaultRelayWrite); the validation policy merges the always- // overlay for the pending fallback (barrier semantics).+ // The whole import holds policyCommitMu (vault-write parity with+ // handleVaultRelayWrite): the validation policy and the registration+ // fence cannot change between the pre-reads and the commit. Lock+ // order: policyCommitMu → registry → s.mu.+ s.policyCommitMu.Lock()+ defer s.policyCommitMu.Unlock()validation := s.vaultValidationPolicy(subj)s.mu.Lock()defer s.mu.Unlock()diff --git a/components/egress/tests/smoke-fast-sandbox.sh b/components/egress/tests/smoke-fast-sandbox.shindex d71d983ee..2b05e3d3e 100755--- a/components/egress/tests/smoke-fast-sandbox.sh+++ b/components/egress/tests/smoke-fast-sandbox.sh@@ -494,7 +494,7 @@ expect_answers osb-sandbox-a allow.test 1.1.1.1# after the restart + SET_BINDING replay the recorded file must supersede# the input — asserted by the marker target surviving the GET below# (vault parity, Test 14e).-relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"durability.test"}]}'+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"durability.test"},{"action":"allow","target":"10.99.0.0/24"}]}'wait_for 10 "dyn lease present before restart" nft_has '1.1.1.1'kill "${EGRESS_PID}" 2>/dev/nullwait "${EGRESS_PID}" 2>/dev/null || true@@ -509,10 +509,18 @@ expect_answers osb-sandbox-a allow.test 1.1.1.1# file superseded the replayed binding input.recorded="$(curl -s "http://127.0.0.1:${POLICY_PORT}/policy/a")"echo "${recorded}" | grep -q 'durability.test' || fail "the relayed policy must survive the restart (file supersedes the replayed binding input), got: ${recorded}"-pass "restart durability (recorded policy supersedes the replayed binding input)"+# Enforcement, not just the record: the probe's static allow must be live+# in nft (the replayed input does not contain it — a regression where the+# replay overrode the reload would lose this element).+nft_has '10.99.0.0/24' || fail "the reloaded policy must be ENFORCED (10.99.0.0/24 missing from nft)"+pass "restart durability (recorded policy supersedes the replayed binding input, enforced)"# Restore the lineage: under file-supersedes-input, every later section# starts from the recorded policy — put back what the binding inputs intend.-relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"}]}'+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"}]}'+# DELETE tombstone across a restart, and the revision precondition: a+# stale expectedRevision is a 409 on the public contract.+http_code="$(curl -s -o /dev/null -w '%{http_code}' -XPUT "http://127.0.0.1:${POLICY_PORT}/policy/a" -d '{"policy":{"defaultAction":"deny","egress":[]},"expectedRevision":99}')"+[ "${http_code}" = "409" ] || fail "stale expectedRevision must be 409, got ${http_code}"###############################################################################info "Test 10: strict DoH mode (no blocklist) drops all tcp 443 globally"diff --git a/server/opensandbox_server/services/fast_sandbox/create_mapping.py b/server/opensandbox_server/services/fast_sandbox/create_mapping.pyindex 9da8720a7..85d342113 100644--- a/server/opensandbox_server/services/fast_sandbox/create_mapping.py+++ b/server/opensandbox_server/services/fast_sandbox/create_mapping.py@@ -32,6 +32,11 @@ from typing import Optionalfrom opensandbox_server.api.schema import CreateSandboxRequestfrom opensandbox_server.services.fast_sandbox.network_policy import normalized_policy+# The egress binding input seeded for a sandbox created without a+# networkPolicy: default-allow (behavior identical to a policy-less+# sandbox) — the binding's point is subject registration, not enforcement.+DEFAULT_ALLOW_POLICY = {"defaultAction": "allow", "egress": []}+from opensandbox_server.services.fast_sandbox.generated import (fastpath_pb2 as pb2,)@@ -147,7 +152,7 @@ def map_create_request(else:create.action_bindings.add(handler="egress",- input=json.dumps({"defaultAction": "allow", "egress": []}),+ input=json.dumps(DEFAULT_ALLOW_POLICY),)extensions = request.extensions or {}@@ -205,21 +210,15 @@ def map_template_create_request(if request.metadata:create.metadata.update(request.metadata)- # Every fsb sandbox registers an egress subject at create: the binding- # is what makes the Fastlet run SET_BINDING (deny-first install, network- # attachment) — the handler-relay surfaces (/policy/{uid},- # /vault/{uid}) and the pause/resume state transfer attach to it. A- # sandbox created without a networkPolicy seeds a no-op default-allow- # record (the registration is the point — the sandbox behaves exactly- # like today's policy-less one), and runtime changes ride the relay.+ # Same registration rule as map_create_request: every fsb sandbox gets+ # an egress subject at create.if request.network_policy is not None:create.action_bindings.add(handler="egress", input=json.dumps(normalized_policy(request.network_policy)))else:create.action_bindings.add(- handler="egress",- input=json.dumps({"defaultAction": "allow", "egress": []}),+ handler="egress", input=json.dumps(DEFAULT_ALLOW_POLICY),)extensions = request.extensions or {}diff --git a/server/opensandbox_server/services/fast_sandbox/policy_relay.py b/server/opensandbox_server/services/fast_sandbox/policy_relay.pyindex cead452ea..d5cd1c314 100644--- a/server/opensandbox_server/services/fast_sandbox/policy_relay.py+++ b/server/opensandbox_server/services/fast_sandbox/policy_relay.py@@ -129,7 +129,7 @@ def _relay_get_full(service, sandbox_id: str) -> dict:status.HTTP_503_SERVICE_UNAVAILABLE,"Network policy relay returned an invalid record.",) from exc- if not isinstance(state.get("policy"), dict):+ if not isinstance(state, dict) or not isinstance(state.get("policy"), dict):raise policy_relay_error(status.HTTP_503_SERVICE_UNAVAILABLE,"Network policy relay record is missing the policy body.",diff --git a/specs/sandbox-lifecycle.yml b/specs/sandbox-lifecycle.ymlindex 2231d092a..51e6df2e9 100644--- a/specs/sandbox-lifecycle.yml+++ b/specs/sandbox-lifecycle.yml@@ -79,6 +79,9 @@ paths:$ref: '#/components/responses/Forbidden''404':$ref: '#/components/responses/NotFound'+ '409':+ description: Fsb - the sandbox is paused; its recorded policy is+ sealed until resume.'500':$ref: '#/components/responses/InternalServerError''503':