Secure, Fast, and Extensible Sandbox runtime for AI agents.
test(env): assert the restore-side vault reload; tighten comments
高然a55fc8ec4174
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.goindex 691cf9e6b..a14ed50ee 100644--- a/components/egress/fastsandbox_actions.go+++ b/components/egress/fastsandbox_actions.go@@ -102,9 +102,9 @@ func (s *fastSandboxPolicyServer) handleActionsStatus(w http.ResponseWriter, r *InstanceID: s.instanceID,Recovery: recovery,DegradedVaults: degraded,- // Capability signal for the Fastlet's pause/resume vault transfer:- // without vault persistence the Handler cannot hold vault state at- // all, so the transfer must degrade (skip), not fail the checkpoint.+ // Capability signal for the Fastlet's vault transfer: without vault+ // persistence there is no vault state to hand off, so the transfer+ // degrades (skips) rather than failing the checkpoint.VaultTransfer: s.vaultFiles != nil,}if s.mitmGate != nil {diff --git a/components/egress/fastsandbox_server_test.go b/components/egress/fastsandbox_server_test.goindex 1cc614af3..5760ab8c7 100644--- a/components/egress/fastsandbox_server_test.go+++ b/components/egress/fastsandbox_server_test.go@@ -238,8 +238,7 @@ func TestActionsStatus(t *testing.T) {assert.Equal(t, constants.ActionsAPIVersion, status.APIVersion)assert.True(t, status.Ready)assert.NotEmpty(t, status.InstanceID)- // The test server configures no vault persistence: the capability- // signal stays false, so the Fastlet skips pause/resume vault transfer.+ // No vault persistence configured: the capability stays false.assert.False(t, status.VaultTransfer)assert.NotContains(t, rec.Body.String(), "vaultTransfer", "the capability field stays omitted when false")diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.goindex 62a5d333c..0c7b23c54 100644--- a/components/egress/fastsandbox_vault_persist.go+++ b/components/egress/fastsandbox_vault_persist.go@@ -96,12 +96,8 @@ func newVaultPersistence(dir string) (*vaultPersistence, error) {if err := os.Chmod(dir, 0o700); err != nil {return nil, fmt.Errorf("restrict vault directory %s: %w", dir, err)}- // Kubelet creates emptyDir roots root-owned and 0777; the per-file atomic- // machinery validates every ancestor and rejects group/world-writable- // ones ("policy ancestor is not a trusted directory"), so the documented- // layout — a volume dedicated to vault state, mounted at the vault- // directory's parent — would fail every vault op at runtime. Restrict- // that root here, bounded to volumes that hold nothing else.+ // The volume root must pass the atomic machinery's trusted-ancestor+ // check; kubelet creates emptyDir roots 0777, which fails it.if err := restrictVolumeRoot(dir); err != nil {return nil, err}@@ -116,13 +112,10 @@ func newVaultPersistence(dir string) (*vaultPersistence, error) {}// restrictVolumeRoot clears group/world-write bits from the vault volume-// root (the vault directory's immediate parent) when the volume is dedicated-// to vault state: kubelet creates emptyDir roots root:root 0777, which the-// per-file atomic machinery rejects as an untrusted ancestor, so without-// this every vault op fails at runtime on the documented layout. A parent-// holding foreign entries is left untouched — a shared-volume layout fixes-// its permissions in the manifest, and the per-file machinery keeps failing-// closed and loudly. Already-restrictive parents are a no-op.+// root (the vault directory's parent): kubelet creates emptyDir roots 0777,+// which the atomic machinery rejects as an untrusted ancestor. A parent+// holding foreign entries is left untouched — that is the manifest's job —+// and already-restrictive roots are a no-op.func restrictVolumeRoot(dir string) error {parent := filepath.Dir(dir)info, err := os.Stat(parent)diff --git a/components/egress/fastsandbox_vault_persist_test.go b/components/egress/fastsandbox_vault_persist_test.goindex b85e0ef67..269f4779b 100644--- a/components/egress/fastsandbox_vault_persist_test.go+++ b/components/egress/fastsandbox_vault_persist_test.go@@ -385,12 +385,9 @@ func TestVaultPersistApplyFailureRollsBackFile(t *testing.T) {const reloadVaultBody = `{"credentials":[{"name":"tok","source":{"type":"inline","value":"sekret"}}],` +`"bindings":[{"name":"api","match":{"hosts":["api.example.com"]},"auth":{"type":"bearer","credential":"tok"}}]}`-// Kubelet creates emptyDir volume roots root:root 0777, and the per-file-// atomic machinery rejects group/world-writable ancestors — which failed-// every vault op on the documented layout (volume mounted at the vault-// directory's parent). Construction restricts a dedicated volume root,-// leaves one holding foreign entries alone, and keeps already-restrictive-// roots as they are.+// Construction restricts a dedicated 0777 emptyDir volume root (an+// untrusted ancestor for the atomic machinery), leaves one holding foreign+// entries alone, and keeps already-restrictive roots as they are.func TestVaultPersistRestrictsDedicatedVolumeRoot(t *testing.T) {t.Run("dedicated root is restricted", func(t *testing.T) {root := t.TempDir()diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.goindex 5c5503cf9..42180b82f 100644--- a/components/egress/fastsandbox_vault_snapshot.go+++ b/components/egress/fastsandbox_vault_snapshot.go@@ -84,13 +84,10 @@ func (s *fastSandboxPolicyServer) applyVaultExport(env *actionhandler.Envelope)if s.vaultFiles.reservesPath(target) {return http.StatusBadRequest, fmt.Errorf("vault bundle target %q must not live under the vault persistence directory", target)}- // A subject that never registered has no vault state to carry and- // nothing to seal: a sandbox created without an egress binding (e.g. a- // template-backed create that never set a network policy) is a legal,- // vault-free shape, and a terminal cleanup looks identical. Answer 200- // so the checkpoint proceeds without a vault — a 404 here would fail- // every pause of a binding-less sandbox forever. A REGISTERED subject- // under a stale fence still fails below.+ // A subject that never registered (a sandbox created without an egress+ // binding, or after terminal cleanup) has no vault to carry: 200 so the+ // checkpoint proceeds — a 404 would fail every pause of a binding-less+ // sandbox. A registered subject under a stale fence still fails below.if _, ok := s.reg.Fence(subj); !ok {log.Infof("vault export: subject %s not registered; no vault to carry, checkpoint proceeds without it", subj)return http.StatusOK, nildiff --git a/components/egress/fastsandbox_vault_snapshot_test.go b/components/egress/fastsandbox_vault_snapshot_test.goindex 0c3a3ce27..96bedf99d 100644--- a/components/egress/fastsandbox_vault_snapshot_test.go+++ b/components/egress/fastsandbox_vault_snapshot_test.go@@ -160,12 +160,9 @@ func TestVaultSnapshotExportWithoutVaultFileStillSeals(t *testing.T) {assert.Equal(t, http.StatusConflict, doVault(t, h.srv, http.MethodPost, snapshotTestUID, vaultCredential(), "").Code)}-// A subject that never registered (a sandbox created without an egress-// binding — e.g. a template-backed create that never set a network policy)-// has no vault state to carry: the export is a 200 no-op so the checkpoint-// proceeds. A 404 here would fail every pause of a binding-less sandbox-// forever; terminal cleanup looks identical and is equally vault-free, and-// a registered subject under a stale fence still fails (409) below.+// An unregistered subject (no egress binding, or after terminal cleanup)+// has no vault to carry: the export is a 200 no-op. A registered subject+// under a stale fence still fails (409).func TestVaultSnapshotExportUnregisteredSubjectIsNoOp(t *testing.T) {h := newVaultPersistHarness(t)bundlePath := filepath.Join(t.TempDir(), "bundle", "unregistered.json")@@ -210,9 +207,7 @@ func TestVaultSnapshotExportGuards(t *testing.T) {t.Run("unknown subject", func(t *testing.T) {h := newVaultPersistHarness(t)rec := doAction(t, h.srv, snapshotBody(t, "ghost", testFenceR, testFenceA, "VAULT_EXPORT", "/tmp/bundle.json"))- // An unregistered subject has no vault state to carry (a sandbox- // without an egress binding, or after terminal cleanup): 200 no-op- // so the checkpoint proceeds. See the dedicated no-op test.+ // 200 no-op: nothing to carry. See the dedicated no-op test.assert.Equal(t, http.StatusOK, rec.Code)assert.False(t, h.fakeForPath("/tmp/bundle.json").exists)})diff --git a/components/egress/pkg/actionhandler/actionhandler.go b/components/egress/pkg/actionhandler/actionhandler.goindex 4d1cbc9d2..7e7d5d40b 100644--- a/components/egress/pkg/actionhandler/actionhandler.go+++ b/components/egress/pkg/actionhandler/actionhandler.go@@ -286,11 +286,9 @@ type StatusResponse struct {// (interception fails closed for them; a successful write clears it).Recovery bool `json:"vaultRecovery,omitempty"`DegradedVaults []string `json:"degradedVaults,omitempty"`- // VaultTransfer is the vault-transfer capability signal (OSEP-0022,- // additive): true only when this Handler serves VAULT_EXPORT /- // VAULT_IMPORT AND persists vault files (OPENSANDBOX_EGRESS_VAULT_DIR- // configured). Absent on older Handler builds, which decodes as false:- // the Fastlet then skips the pause/resume vault transfer and the- // checkpoint proceeds without a vault — the pre-transfer semantics.+ // VaultTransfer reports that this Handler serves VAULT_EXPORT /+ // VAULT_IMPORT and persists vault files. Absent on older builds decodes+ // as false: the Fastlet then skips the transfer (checkpoint proceeds+ // without a vault) instead of failing it.VaultTransfer bool `json:"vaultTransfer,omitempty"`}diff --git a/scripts/fast-sandbox-env/integration-env.sh b/scripts/fast-sandbox-env/integration-env.shindex 584158559..199ee3b96 100755--- a/scripts/fast-sandbox-env/integration-env.sh+++ b/scripts/fast-sandbox-env/integration-env.sh@@ -202,9 +202,7 @@ ACTION=""log() { printf '\033[1;34m[fast-sandbox-env]\033[0m %s\n' "$*" | tee -a "$WORK/run.log" >&2; }die() { printf '\033[1;31m[fast-sandbox-env] ERROR:\033[0m %s\n' "$*" >&2; exit 1; }pass() { printf '\033[1;32m[fast-sandbox-env] PASS\033[0m %s\n' "$*" | tee -a "$WORK/run.log" >&2; }-# fail dumps before exiting: an explicit `exit` does NOT fire the ERR trap-# (only a failing command does), so verify-stage failures reached through-# wait_for -> fail would otherwise tear down the cluster with no pod logs.+# fail dumps before exiting: an explicit `exit` does not fire the ERR trap.fail() { printf '\033[1;31m[fast-sandbox-env] FAIL\033[0m %s\n' "$*" >&2; failure_dump "${STAGE_CUR_LABEL:-$ACTION}" || true; exit 1; }highlight() { printf '\033[1;36m%s\033[0m\n' "$*"; }@@ -328,12 +326,9 @@ failure_dump() {kubectl logs -n "$NS" deploy/fast-sandbox-controller --tail=80 2>&1 || trueecho "--- firecracker-runtime logs (tail) ---"kubectl logs -n "$NS" daemonset/firecracker-runtime --all-containers --tail=80 2>&1 || true- # --all-containers is the point: `kubectl logs -l` without it logs- # only the first container (fastlet) and silently drops egress /- # fastlet-proxy, which is where pause/snapshot and policy failures- # actually surface. The window is deep because a failure loop (a- # per-second pause retry) floods the tail and buries the- # pre-incident history that explains how the failure started.+ # --all-containers: the label selector alone logs only the first+ # container and drops egress. Deep tail: failure loops flood the+ # window and bury the pre-incident history.echo "--- fastlet pod logs, every container (deep tail) ---"kubectl logs -n "$RESOURCE_NS" -l app=sandbox-fastlet \--all-containers --prefix --timestamps --tail=15000 2>&1 || true@@ -357,10 +352,8 @@ failure_dump() {echo "--- Sandbox CRs (status only) ---"kubectl get sandboxes -n "$RESOURCE_NS" -o json 2>/dev/null \| jq -c '.items[] | {name: .metadata.name, phase: .status.phase, conditions: .status.conditions, infra: .status.infraComponents}' 2>&1 || true- # Cross-container visibility of the snapshot staging area: the vault- # export hands the DRIVER's bundle path to the egress container, and- # a mount-namespace mismatch shows up as ENOENT on the egress side- # only. Compare both views.+ # A mount-namespace mismatch (driver staging path invisible to+ # egress) shows up as ENOENT on the egress side only.for pod in $(kubectl -n "$RESOURCE_NS" get pods -l app=sandbox-fastlet -o name 2>/dev/null); doecho "### ${pod##*/}: snapshot staging visibility"echo "-- fastlet view:"@@ -1708,9 +1701,8 @@ vault_revision_is() { # <expected-revision>vault_lifecycle_verify() {local created vault_body ack rev sandbox_uidlog "verify (vault): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID)"- # The policy rides the create (the vault validates its bindings against- # it, and a create WITHOUT a policy produces no egress binding at all —- # the subject would never register).+ # The policy rides the create: a policy-less create produces no egress+ # binding at all, so the subject would never register.jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "ext.test"}]}' \> "$GEN_DIR/vault-policy.json"created="$(osb_api sandbox create --template "$TEMPLATE_ID" --timeout 1h \@@ -1722,16 +1714,13 @@ vault_lifecycle_verify() {wait_for "vault sandbox Running" 150 sandbox_runningsandbox_uid="$(kubectl -n "$RESOURCE_NS" get sandbox "$VERIFY_ID" -o jsonpath='{.metadata.uid}' 2>/dev/null)"[[ -n "$sandbox_uid" ]] || fail "vault verify: sandbox CR carried no uid"- # The subject must be registered AND activated (deny-first install, then- # the data-plane-ready hook flips it) before the vault POST; otherwise- # the relay 404s (create -> ready -> set vault). The second- # "SET_BINDING applied (state=active)" line only appears after a policy- # UPDATE; a create-carried policy activates without it.+ # Registered and activated before the vault POST, else the relay 404s.+ # (The "(state=active)" re-delivery only follows a policy UPDATE; a+ # create-carried policy activates without it.)wait_for "vault subject registered on egress" 120 \vault_log_contains "subject s-$sandbox_uid: data-plane-ready, policy active"- # 1) create-only: 201 ack carries the egress-validated revision; a- # duplicate POST is 409 (atomic across server replicas).+ # 1) create-only: ack carries the revision; duplicate POST is 409.vault_body="$(jq -n '{credentials: [{name: "tok", source: {type: "inline", value: "sekrit-vault-e2e"}}], bindings: [{name: "ext", match: {hosts: ["ext.test"]}, auth: {type: "apiKey", name: "X-Api-Key", credential: "tok"}}]}')"ack="$(server_api POST "/sandboxes/$VERIFY_ID/credential-vault" "$vault_body")" \|| fail "vault verify: POST credential-vault failed"@@ -1745,9 +1734,8 @@ vault_lifecycle_verify() {|| fail "vault verify: duplicate POST was not 409"pass "duplicate POST rejected (create-only 409)"- # 2) pause/resume carries the REAL vault: the egress copies the vault- # file into the checkpoint bundle (export) and re-imports it on- # restore; the relay GET still reports the same revision.+ # 2) pause/resume carries the real vault: export -> bundle -> import,+ # with the relay GET reporting the same revision.osb_api sandbox pause "$VERIFY_ID" >/dev/null 2>&1 || fail "vault verify: pause failed"wait_for "vault sandbox Paused (checkpoint durable)" 240 sandbox_pausedwait_for "vault exported into the checkpoint bundle" 120 \@@ -1755,15 +1743,16 @@ vault_lifecycle_verify() {pass "pause exported the vault bundle (subject sealed)"osb_api sandbox resume "$VERIFY_ID" --skip-health-check >/dev/null 2>&1 || fail "vault verify: resume failed"wait_for "vault sandbox Running (restored)" 240 sandbox_running- wait_for "vault re-imported from the bundle" 120 \- vault_log_contains "subject s-$sandbox_uid: vault imported from snapshot bundle"+ # The pause's REMOVE_BINDING purges the subject, so the restore import+ # lands through the unregistered branch and the file reloads at the+ # SET_BINDING replay (the reload line carries the restored revision).+ wait_for "vault bundle reloaded at re-registration" 120 \+ vault_log_contains "subject s-$sandbox_uid: vault reloaded from file"wait_for "vault revision preserved across pause/resume" 120 vault_revision_is "$rev"pass "vault survived pause/resume (revision $rev preserved)"- # 3) egress-restart recovery: crash the worker (the supervisor restarts- # it with a fresh instanceId); the fastlet replays SET_BINDING and the- # vault file reloads by incarnation. The GET succeeding with the same- # revision proves the whole chain (relay up, replay done, file loaded).+ # 3) egress-restart recovery: crash the worker (fresh instanceId ->+ # SET_BINDING replay -> vault file reload by incarnation).local pod restarted=0for pod in $(pool_pods); doif kubectl -n "$RESOURCE_NS" exec "$pod" -c egress -- sh -c 'command -v pkill >/dev/null' 2>/dev/null; then@@ -1779,8 +1768,7 @@ vault_lifecycle_verify() {pass "vault reloaded after egress restart (revision $rev preserved)"fi- # 4) delete + re-create: DELETE is terminal, GET 404s, a fresh POST is a- # new vault at revision 1.+ # 4) delete + re-create.server_api DELETE "/sandboxes/$VERIFY_ID/credential-vault" >/dev/null \|| fail "vault verify: DELETE failed"[[ "$(server_api_status GET "/sandboxes/$VERIFY_ID/credential-vault")" == "404" ]] \diff --git a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yamlindex f4ef931f5..0392da8b9 100644--- a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml+++ b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml@@ -84,13 +84,9 @@ spec:# trust surface. Swap to `medium: Memory` (tmpfs) to keep the# per-subject vault files off node disk.mountPath: /var/lib/opensandbox-egress- # Vault bundle handoff (OSEP-0022): the export/import envelope- # carries the DRIVER's snapshot staging path, which lives under the- # runtime state root. Mount the same runtime-injected hostPath- # volume the fastlet container gets (the controller adds it for the- # first container and merges it into the pod spec), so the Handler- # resolves the path in its own mount namespace. Read-write: the- # Handler copies exports out and reads imports in.+ # Vault bundle handoff (OSEP-0022): the envelope carries the+ # driver's staging path under the runtime state root; mount the+ # same runtime-injected volume so the path resolves here.- name: firecracker-statemountPath: /var/lib/fast-sandbox/firecracker# Egress installs nft/iptables rules (DNS redirect + per-subjectdiff --git a/server/configuration.md b/server/configuration.mdindex 7ab9e0328..a926dbc7d 100644--- a/server/configuration.md+++ b/server/configuration.md@@ -167,7 +167,7 @@ The kubernetes runtime also serves fsb (`fsb-`) sandboxes side by side; these fi| `fastpath_wait_ready_seconds` | number | `30.0` | Bounded readiness wait for DataPlaneReady after Create. || `fastpath_resource_pool` | string | `"default-pool"` | SandboxPool when `extensions.poolRef` is unset. || `template_s3_publish_secret` | string | `"sandbox-oss-credentials"` | Object-store credentials secret for server-created SandboxTemplates. |-| `template_mitm_ca_secret` | string \| omitted | `null` | Secret holding the pinned cluster MITM CA certificate (OSEP-0022). When set, server-created SandboxTemplates bake `spec.egressMitmCertificate` from it so guests trust the interception CA from first boot; the referenced entry must carry the certificate only, never the private key. The Secret must exist in each template's namespace (ops-runbook distribution) — a missed copy fails the build observably. Unset keeps templates CA-free. |+| `template_mitm_ca_secret` | string \| omitted | `null` | Secret holding the pinned cluster MITM CA certificate (OSEP-0022). When set, server-created SandboxTemplates bake `spec.egressMitmCertificate` from it (certificate entry only, never the private key); unset keeps templates CA-free. The Secret must exist in each template's namespace (ops-runbook distribution); a missed copy fails the build observably. || `template_mitm_ca_secret_key` | string | `"ca.crt"` | Secret entry holding the CA certificate when `template_mitm_ca_secret` is set. |---diff --git a/server/opensandbox_server/config.py b/server/opensandbox_server/config.pyindex b944b808a..c10b304bd 100644--- a/server/opensandbox_server/config.py+++ b/server/opensandbox_server/config.py@@ -818,12 +818,10 @@ class KubernetesRuntimeConfig(BaseModel):description=("Secret holding the pinned cluster MITM CA certificate ""(OSEP-0022). When set, server-created SandboxTemplates bake "- "spec.egressMitmCertificate from it so guests trust the "- "interception CA from first boot; the referenced entry must "- "carry the certificate only, never the private key. The Secret "- "must exist in each template's namespace (ops-runbook "- "distribution) — a missed copy fails the build observably. "- "Unset keeps templates CA-free."+ "spec.egressMitmCertificate from it (certificate entry only, "+ "never the private key); unset keeps templates CA-free. The "+ "Secret must exist in each template's namespace (ops-runbook "+ "distribution); a missed copy fails the build observably."),)template_mitm_ca_secret_key: str = Field(diff --git a/server/opensandbox_server/services/templates/template_service.py b/server/opensandbox_server/services/templates/template_service.pyindex 1a1769f0f..2b21d80e9 100644--- a/server/opensandbox_server/services/templates/template_service.py+++ b/server/opensandbox_server/services/templates/template_service.py@@ -397,13 +397,11 @@ class FastSandboxTemplateService:env = record.spec.get("env") or {}if env:spec["envs"] = [{"name": name, "value": value} for name, value in env.items()]- # OSEP-0022 CA wiring: when the operator configured the pinned MITM CA- # Secret, every server-created template bakes it — the certificate- # entry only, never the private key (the builder rejects key material- # and installs the certificate into the guest trust store at the- # snapshot boot). Secret refs are namespace-local, so the copy must- # exist in the template's namespace (ops-runbook distribution); a- # missed copy fails the build observably instead of degrading.+ # OSEP-0022 CA wiring: bake the pinned MITM CA Secret's certificate+ # entry (never the private key) into every server-created template+ # when configured. The Secret must exist in the template's+ # namespace (ops-runbook distribution); a missed copy fails the+ # build observably.if self._k8s_config.template_mitm_ca_secret:mitm_certificate: dict = {"secretRef": {"name": self._k8s_config.template_mitm_ca_secret}