Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): policy-state re-validation, durable removal retry, and review-round hardening

高然154770adde30
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index 7ddd0ca73..a95bad755 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -300,6 +300,12 @@ func (s *fastSandboxPolicyServer) applyLifecycleHook(env *actionhandler.Envelope
// ever consume.
s.policyCommitMu.Lock()
defer s.policyCommitMu.Unlock()
+ // In-lock fence re-validation: the pre-lock check cannot see a
+ // rebind that completes while this Hook waits on policyCommitMu —
+ // a stale Hook must never activate the successor.
+ if !s.fenceMatchesLocked(subj, fence) {
+ return http.StatusConflict, fmt.Errorf("stale %s (fence mismatch) for subject %s", env.Hook.Name, subj)
+ }
// Peek, do not consume: a transient nft failure must leave the
// pending policy in place so the Fastlet's retry of the same Hook
// succeeds instead of 409-ing forever with the subject deny-first.
diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.go
index 1a80a394f..9e1c1cc79 100644
--- a/components/egress/fastsandbox_policy_persist.go
+++ b/components/egress/fastsandbox_policy_persist.go
@@ -302,6 +302,14 @@ func (s *fastSandboxPolicyServer) clearPolicyRecordLocked(subj subject.Subject)
pp := s.policyFiles
s.mu.Unlock()
if entry == nil {
+ // A previous attempt may have cleared the memory record but failed
+ // the durable purge: retry the idempotent cleanup so the acked
+ // removal is real (the Fastlet's retry lands here).
+ if pp != nil {
+ if err := pp.purge(subj); err != nil {
+ return fmt.Errorf("policy file purge retry: %w", err)
+ }
+ }
return nil
}
s.mu.Lock()
@@ -382,15 +390,13 @@ func (s *fastSandboxPolicyServer) rehydratePolicyReload(subj subject.Subject, fe
return
}
if !snap.Exists() {
- // Fresh subject: nothing persisted, the binding input seeds. A
- // durable seal marker (an export that sealed a file-less subject)
- // must still gate the relay ops — the seed entry carries it.
+ // Fresh subject: the binding input seeds. A durable seal marker (an
+ // export that sealed a file-less subject) still gates the relay
+ // ops — the seed entry carries it (caller holds s.mu).
if s.policyFiles.isSealed(subj) {
- s.mu.Lock()
s.policies[subj] = &policyEntry{fence: fence, sealed: true}
s.pendingPolicies[subj] = policy.DefaultDenyPolicy()
- s.mu.Unlock()
- log.Infof("subject %s: seal restored from marker on a file-less subject (binding input seeds pending default-deny)", subj)
+ log.Infof("subject %s: seal restored from marker (file-less subject)", subj)
return
}
return
diff --git a/components/egress/fastsandbox_policy_persist_test.go b/components/egress/fastsandbox_policy_persist_test.go
index e94005d41..937e3e9fc 100644
--- a/components/egress/fastsandbox_policy_persist_test.go
+++ b/components/egress/fastsandbox_policy_persist_test.go
@@ -22,6 +22,7 @@ import (
"os"
"path/filepath"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -579,6 +580,57 @@ func TestPolicyPersistRemovalPurgeFailureLandsDenyTombstone(t *testing.T) {
assert.Contains(t, string(doc.Policy), `"defaultAction":"deny"`)
}
+// TestPolicyPersistSealedFilelessSubjectSurvivesRestart (review C1): an
+// export seals a file-less subject; the restart's binding replay must
+// restore the seal WITHOUT deadlocking (the sealed branch runs under the
+// reload's own s.mu), and the relay ops stay 409 with default-deny pending.
+func TestPolicyPersistSealedFilelessSubjectSurvivesRestart(t *testing.T) {
+ h := newPolicyPersistHarness(t)
+
+ // Bind, then remove the policy (healthy purge → file-less), then export
+ // (seals the file-less subject).
+ h.readySubject(t, policyTestUID, vaultTestIP)
+ rec := doAction(t, h.srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 2, nil))
+ require.Equal(t, http.StatusOK, rec.Code)
+ require.False(t, h.fileFor(policyTestUID).exists, "precondition: file-less after removal")
+ bundlePath := filepath.Join(t.TempDir(), "bundle", policyTestUID+".json")
+ rec = doAction(t, h.srv, policySnapshotBody(t, policyTestUID, testFenceR, testFenceA, "POLICY_EXPORT", bundlePath))
+ require.Equal(t, http.StatusOK, rec.Code)
+ h.srv.mu.Lock()
+ require.True(t, h.srv.policies[subject.FromSandboxUID(policyTestUID)].sealed)
+ h.srv.mu.Unlock()
+
+ // Restart: fresh server over the same persistence state, then the
+ // binding replay (non-removal). Must not deadlock; the seal and the
+ // default-deny pending come back.
+ srv2, reg2, _ := fastSandboxTestServer(t)
+ pp2, err := newPolicyPersistence(h.dir)
+ require.NoError(t, err)
+ pp2.newFile = h.pp.newFile
+ srv2.SetPolicyFiles(pp2)
+
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ r := doAction(t, srv2, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 1, strPtr(policyAllow())))
+ assert.Equal(t, http.StatusOK, r.Code)
+ }()
+ select {
+ case <-done:
+ case <-time.After(10 * time.Second):
+ t.Fatal("deadlock: the sealed file-less replay never completed")
+ }
+
+ srv2.mu.Lock()
+ e := srv2.policies[subject.FromSandboxUID(policyTestUID)]
+ srv2.mu.Unlock()
+ require.NotNil(t, e)
+ assert.True(t, e.sealed, "the seal is restored from the marker")
+ rec = doPolicy(t, srv2, http.MethodPut, policyTestUID, policyPutDoc(policyAllow(), 0, false, false), "")
+ assert.Equal(t, http.StatusConflict, rec.Code, "relay ops stay 409 while sealed")
+ _ = reg2
+}
+
// ---------------------------------------------------------------------------
// Capability + healthy posture
// ---------------------------------------------------------------------------
diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.go
index c186fc3fb..95abbd315 100644
--- a/components/egress/fastsandbox_server.go
+++ b/components/egress/fastsandbox_server.go
@@ -383,10 +383,9 @@ func (s *fastSandboxPolicyServer) applyPolicy(subj subject.Subject, pol *policy.
s.policyMu.Lock()
defer s.policyMu.Unlock()
eff := s.reg.EffectiveOf(pol)
- // Bounded: applyPolicy runs under policyCommitMu, so one slow nft
- // transaction would head-of-line-block every subject's writes. The
- // registration deny-first install is not under that serialization and
- // keeps a 30s budget.
+ // Bounded (policyApplyTimeout): every applyPolicy caller runs under
+ // policyCommitMu, so one slow nft transaction would head-of-line-block
+ // every subject's writes.
nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)
defer cancel()
if err := s.nft.ApplyPolicy(nftCtx, subj, eff); err != nil {
@@ -447,7 +446,10 @@ func (s *fastSandboxPolicyServer) dropVault(subj subject.Subject) {
// (nft rules + gateway DNS redirect + MITM interception). Runs under the
// registry write lock, so no registry calls here.
func (s *fastSandboxPolicyServer) OnRegistered(subj subject.Subject, att actionhandler.NetworkAttachment) error {
- nftCtx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
+ // Bounded (policyApplyTimeout): SET_BINDING runs under policyCommitMu,
+ // so a slow deny-first install would head-of-line-block every subject's
+ // policy writes.
+ nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)
defer cancel()
if err := s.nft.ApplyDenyFirst(nftCtx, subj, att); err != nil {
return err
diff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.go
index 8888955b0..3621b7811 100644
--- a/components/egress/fastsandbox_vault.go
+++ b/components/egress/fastsandbox_vault.go
@@ -123,8 +123,16 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
// The body has been consumed; a late response must not try to read it.
r.Body.Close()
- // Registry reads before s.mu (lock order: registry → s.mu, see
- // selectRelayVault).
+ // The whole write holds policyCommitMu: the validation policy must not
+ // change between the pre-read and the commit (policy mutations — relay
+ // PUT/DELETE, hooks, removals — serialize on the same mutex, and none
+ // of them bump the fence, so the commit-time fence re-check alone
+ // cannot catch this). Lock order: policyCommitMu → registry → s.mu.
+ s.policyCommitMu.Lock()
+ defer s.policyCommitMu.Unlock()
+
+ // Registry reads before s.mu (lock order: policyCommitMu → registry →
+ // s.mu).
fence, ok := s.reg.Fence(subj)
if !ok {
http.Error(w, "unknown subject", http.StatusNotFound)
diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.go
index 373fd7493..a0e984371 100644
--- a/components/egress/fastsandbox_vault_persist.go
+++ b/components/egress/fastsandbox_vault_persist.go
@@ -177,7 +177,7 @@ func restrictVolumeRoot(dir, noun string) error {
}
entries, err := os.ReadDir(parent)
if err != nil {
- return fmt.Errorf("list vault volume root %s: %w", parent, err)
+ return fmt.Errorf("list %s volume root %s: %w", noun, parent, err)
}
base := filepath.Base(dir)
for _, entry := range entries {
diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.go
index 51cc5b3bc..e75282c13 100644
--- a/components/egress/fastsandbox_vault_snapshot.go
+++ b/components/egress/fastsandbox_vault_snapshot.go
@@ -259,9 +259,12 @@ func (s *fastSandboxPolicyServer) applyVaultImport(env *actionhandler.Envelope)
return http.StatusInternalServerError, fmt.Errorf("vault bundle re-stamp failed: %w", err)
}
- // Registry reads before s.mu (lock order: registry → s.mu, see
- // handleVaultRelayWrite); the validation policy merges the always
- // overlay for the pending fallback (barrier semantics).
+ // The whole import holds policyCommitMu (vault-write parity with
+ // handleVaultRelayWrite): the validation policy and the registration
+ // fence cannot change between the pre-reads and the commit. Lock
+ // order: policyCommitMu → registry → s.mu.
+ s.policyCommitMu.Lock()
+ defer s.policyCommitMu.Unlock()
validation := s.vaultValidationPolicy(subj)
s.mu.Lock()
defer s.mu.Unlock()
diff --git a/components/egress/tests/smoke-fast-sandbox.sh b/components/egress/tests/smoke-fast-sandbox.sh
index d71d983ee..2b05e3d3e 100755
--- a/components/egress/tests/smoke-fast-sandbox.sh
+++ b/components/egress/tests/smoke-fast-sandbox.sh
@@ -494,7 +494,7 @@ expect_answers osb-sandbox-a allow.test 1.1.1.1
# after the restart + SET_BINDING replay the recorded file must supersede
# the input — asserted by the marker target surviving the GET below
# (vault parity, Test 14e).
-relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"durability.test"}]}'
+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"},{"action":"allow","target":"durability.test"},{"action":"allow","target":"10.99.0.0/24"}]}'
wait_for 10 "dyn lease present before restart" nft_has '1.1.1.1'
kill "${EGRESS_PID}" 2>/dev/null
wait "${EGRESS_PID}" 2>/dev/null || true
@@ -509,10 +509,18 @@ expect_answers osb-sandbox-a allow.test 1.1.1.1
# file superseded the replayed binding input.
recorded="$(curl -s "http://127.0.0.1:${POLICY_PORT}/policy/a")"
echo "${recorded}" | grep -q 'durability.test' || fail "the relayed policy must survive the restart (file supersedes the replayed binding input), got: ${recorded}"
-pass "restart durability (recorded policy supersedes the replayed binding input)"
+# Enforcement, not just the record: the probe's static allow must be live
+# in nft (the replayed input does not contain it — a regression where the
+# replay overrode the reload would lose this element).
+nft_has '10.99.0.0/24' || fail "the reloaded policy must be ENFORCED (10.99.0.0/24 missing from nft)"
+pass "restart durability (recorded policy supersedes the replayed binding input, enforced)"
# Restore the lineage: under file-supersedes-input, every later section
# starts from the recorded policy — put back what the binding inputs intend.
-relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"}]}'
+relay_put_policy a '{"defaultAction":"deny","egress":[{"action":"allow","target":"*.test"}]}'
+# DELETE tombstone across a restart, and the revision precondition: a
+# stale expectedRevision is a 409 on the public contract.
+http_code="$(curl -s -o /dev/null -w '%{http_code}' -XPUT "http://127.0.0.1:${POLICY_PORT}/policy/a" -d '{"policy":{"defaultAction":"deny","egress":[]},"expectedRevision":99}')"
+[ "${http_code}" = "409" ] || fail "stale expectedRevision must be 409, got ${http_code}"
###############################################################################
info "Test 10: strict DoH mode (no blocklist) drops all tcp 443 globally"
diff --git a/server/opensandbox_server/services/fast_sandbox/create_mapping.py b/server/opensandbox_server/services/fast_sandbox/create_mapping.py
index 9da8720a7..85d342113 100644
--- a/server/opensandbox_server/services/fast_sandbox/create_mapping.py
+++ b/server/opensandbox_server/services/fast_sandbox/create_mapping.py
@@ -32,6 +32,11 @@ from typing import Optional
from opensandbox_server.api.schema import CreateSandboxRequest
from opensandbox_server.services.fast_sandbox.network_policy import normalized_policy
+# The egress binding input seeded for a sandbox created without a
+# networkPolicy: default-allow (behavior identical to a policy-less
+# sandbox) — the binding's point is subject registration, not enforcement.
+DEFAULT_ALLOW_POLICY = {"defaultAction": "allow", "egress": []}
+
from opensandbox_server.services.fast_sandbox.generated import (
fastpath_pb2 as pb2,
)
@@ -147,7 +152,7 @@ def map_create_request(
else:
create.action_bindings.add(
handler="egress",
- input=json.dumps({"defaultAction": "allow", "egress": []}),
+ input=json.dumps(DEFAULT_ALLOW_POLICY),
)
extensions = request.extensions or {}
@@ -205,21 +210,15 @@ def map_template_create_request(
if request.metadata:
create.metadata.update(request.metadata)
- # Every fsb sandbox registers an egress subject at create: the binding
- # is what makes the Fastlet run SET_BINDING (deny-first install, network
- # attachment) — the handler-relay surfaces (/policy/{uid},
- # /vault/{uid}) and the pause/resume state transfer attach to it. A
- # sandbox created without a networkPolicy seeds a no-op default-allow
- # record (the registration is the point — the sandbox behaves exactly
- # like today's policy-less one), and runtime changes ride the relay.
+ # Same registration rule as map_create_request: every fsb sandbox gets
+ # an egress subject at create.
if request.network_policy is not None:
create.action_bindings.add(
handler="egress", input=json.dumps(normalized_policy(request.network_policy))
)
else:
create.action_bindings.add(
- handler="egress",
- input=json.dumps({"defaultAction": "allow", "egress": []}),
+ handler="egress", input=json.dumps(DEFAULT_ALLOW_POLICY),
)
extensions = request.extensions or {}
diff --git a/server/opensandbox_server/services/fast_sandbox/policy_relay.py b/server/opensandbox_server/services/fast_sandbox/policy_relay.py
index cead452ea..d5cd1c314 100644
--- a/server/opensandbox_server/services/fast_sandbox/policy_relay.py
+++ b/server/opensandbox_server/services/fast_sandbox/policy_relay.py
@@ -129,7 +129,7 @@ def _relay_get_full(service, sandbox_id: str) -> dict:
status.HTTP_503_SERVICE_UNAVAILABLE,
"Network policy relay returned an invalid record.",
) from exc
- if not isinstance(state.get("policy"), dict):
+ if not isinstance(state, dict) or not isinstance(state.get("policy"), dict):
raise policy_relay_error(
status.HTTP_503_SERVICE_UNAVAILABLE,
"Network policy relay record is missing the policy body.",
diff --git a/specs/sandbox-lifecycle.yml b/specs/sandbox-lifecycle.yml
index 2231d092a..51e6df2e9 100644
--- a/specs/sandbox-lifecycle.yml
+++ b/specs/sandbox-lifecycle.yml
@@ -79,6 +79,9 @@ paths:
$ref: '#/components/responses/Forbidden'
'404':
$ref: '#/components/responses/NotFound'
+ '409':
+ description: Fsb - the sandbox is paused; its recorded policy is
+ sealed until resume.
'500':
$ref: '#/components/responses/InternalServerError'
'503':