Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

test(env): assert the restore-side vault reload; tighten comments

高然a55fc8ec4174
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index 691cf9e6b..a14ed50ee 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -102,9 +102,9 @@ func (s *fastSandboxPolicyServer) handleActionsStatus(w http.ResponseWriter, r *
InstanceID: s.instanceID,
Recovery: recovery,
DegradedVaults: degraded,
- // Capability signal for the Fastlet's pause/resume vault transfer:
- // without vault persistence the Handler cannot hold vault state at
- // all, so the transfer must degrade (skip), not fail the checkpoint.
+ // Capability signal for the Fastlet's vault transfer: without vault
+ // persistence there is no vault state to hand off, so the transfer
+ // degrades (skips) rather than failing the checkpoint.
VaultTransfer: s.vaultFiles != nil,
}
if s.mitmGate != nil {
diff --git a/components/egress/fastsandbox_server_test.go b/components/egress/fastsandbox_server_test.go
index 1cc614af3..5760ab8c7 100644
--- a/components/egress/fastsandbox_server_test.go
+++ b/components/egress/fastsandbox_server_test.go
@@ -238,8 +238,7 @@ func TestActionsStatus(t *testing.T) {
assert.Equal(t, constants.ActionsAPIVersion, status.APIVersion)
assert.True(t, status.Ready)
assert.NotEmpty(t, status.InstanceID)
- // The test server configures no vault persistence: the capability
- // signal stays false, so the Fastlet skips pause/resume vault transfer.
+ // No vault persistence configured: the capability stays false.
assert.False(t, status.VaultTransfer)
assert.NotContains(t, rec.Body.String(), "vaultTransfer", "the capability field stays omitted when false")
diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.go
index 62a5d333c..0c7b23c54 100644
--- a/components/egress/fastsandbox_vault_persist.go
+++ b/components/egress/fastsandbox_vault_persist.go
@@ -96,12 +96,8 @@ func newVaultPersistence(dir string) (*vaultPersistence, error) {
if err := os.Chmod(dir, 0o700); err != nil {
return nil, fmt.Errorf("restrict vault directory %s: %w", dir, err)
}
- // Kubelet creates emptyDir roots root-owned and 0777; the per-file atomic
- // machinery validates every ancestor and rejects group/world-writable
- // ones ("policy ancestor is not a trusted directory"), so the documented
- // layout — a volume dedicated to vault state, mounted at the vault
- // directory's parent — would fail every vault op at runtime. Restrict
- // that root here, bounded to volumes that hold nothing else.
+ // The volume root must pass the atomic machinery's trusted-ancestor
+ // check; kubelet creates emptyDir roots 0777, which fails it.
if err := restrictVolumeRoot(dir); err != nil {
return nil, err
}
@@ -116,13 +112,10 @@ func newVaultPersistence(dir string) (*vaultPersistence, error) {
}
// restrictVolumeRoot clears group/world-write bits from the vault volume
-// root (the vault directory's immediate parent) when the volume is dedicated
-// to vault state: kubelet creates emptyDir roots root:root 0777, which the
-// per-file atomic machinery rejects as an untrusted ancestor, so without
-// this every vault op fails at runtime on the documented layout. A parent
-// holding foreign entries is left untouched — a shared-volume layout fixes
-// its permissions in the manifest, and the per-file machinery keeps failing
-// closed and loudly. Already-restrictive parents are a no-op.
+// root (the vault directory's parent): kubelet creates emptyDir roots 0777,
+// which the atomic machinery rejects as an untrusted ancestor. A parent
+// holding foreign entries is left untouched — that is the manifest's job —
+// and already-restrictive roots are a no-op.
func restrictVolumeRoot(dir string) error {
parent := filepath.Dir(dir)
info, err := os.Stat(parent)
diff --git a/components/egress/fastsandbox_vault_persist_test.go b/components/egress/fastsandbox_vault_persist_test.go
index b85e0ef67..269f4779b 100644
--- a/components/egress/fastsandbox_vault_persist_test.go
+++ b/components/egress/fastsandbox_vault_persist_test.go
@@ -385,12 +385,9 @@ func TestVaultPersistApplyFailureRollsBackFile(t *testing.T) {
const reloadVaultBody = `{"credentials":[{"name":"tok","source":{"type":"inline","value":"sekret"}}],` +
`"bindings":[{"name":"api","match":{"hosts":["api.example.com"]},"auth":{"type":"bearer","credential":"tok"}}]}`
-// Kubelet creates emptyDir volume roots root:root 0777, and the per-file
-// atomic machinery rejects group/world-writable ancestors — which failed
-// every vault op on the documented layout (volume mounted at the vault
-// directory's parent). Construction restricts a dedicated volume root,
-// leaves one holding foreign entries alone, and keeps already-restrictive
-// roots as they are.
+// Construction restricts a dedicated 0777 emptyDir volume root (an
+// untrusted ancestor for the atomic machinery), leaves one holding foreign
+// entries alone, and keeps already-restrictive roots as they are.
func TestVaultPersistRestrictsDedicatedVolumeRoot(t *testing.T) {
t.Run("dedicated root is restricted", func(t *testing.T) {
root := t.TempDir()
diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.go
index 5c5503cf9..42180b82f 100644
--- a/components/egress/fastsandbox_vault_snapshot.go
+++ b/components/egress/fastsandbox_vault_snapshot.go
@@ -84,13 +84,10 @@ func (s *fastSandboxPolicyServer) applyVaultExport(env *actionhandler.Envelope)
if s.vaultFiles.reservesPath(target) {
return http.StatusBadRequest, fmt.Errorf("vault bundle target %q must not live under the vault persistence directory", target)
}
- // A subject that never registered has no vault state to carry and
- // nothing to seal: a sandbox created without an egress binding (e.g. a
- // template-backed create that never set a network policy) is a legal,
- // vault-free shape, and a terminal cleanup looks identical. Answer 200
- // so the checkpoint proceeds without a vault — a 404 here would fail
- // every pause of a binding-less sandbox forever. A REGISTERED subject
- // under a stale fence still fails below.
+ // A subject that never registered (a sandbox created without an egress
+ // binding, or after terminal cleanup) has no vault to carry: 200 so the
+ // checkpoint proceeds — a 404 would fail every pause of a binding-less
+ // sandbox. A registered subject under a stale fence still fails below.
if _, ok := s.reg.Fence(subj); !ok {
log.Infof("vault export: subject %s not registered; no vault to carry, checkpoint proceeds without it", subj)
return http.StatusOK, nil
diff --git a/components/egress/fastsandbox_vault_snapshot_test.go b/components/egress/fastsandbox_vault_snapshot_test.go
index 0c3a3ce27..96bedf99d 100644
--- a/components/egress/fastsandbox_vault_snapshot_test.go
+++ b/components/egress/fastsandbox_vault_snapshot_test.go
@@ -160,12 +160,9 @@ func TestVaultSnapshotExportWithoutVaultFileStillSeals(t *testing.T) {
assert.Equal(t, http.StatusConflict, doVault(t, h.srv, http.MethodPost, snapshotTestUID, vaultCredential(), "").Code)
}
-// A subject that never registered (a sandbox created without an egress
-// binding — e.g. a template-backed create that never set a network policy)
-// has no vault state to carry: the export is a 200 no-op so the checkpoint
-// proceeds. A 404 here would fail every pause of a binding-less sandbox
-// forever; terminal cleanup looks identical and is equally vault-free, and
-// a registered subject under a stale fence still fails (409) below.
+// An unregistered subject (no egress binding, or after terminal cleanup)
+// has no vault to carry: the export is a 200 no-op. A registered subject
+// under a stale fence still fails (409).
func TestVaultSnapshotExportUnregisteredSubjectIsNoOp(t *testing.T) {
h := newVaultPersistHarness(t)
bundlePath := filepath.Join(t.TempDir(), "bundle", "unregistered.json")
@@ -210,9 +207,7 @@ func TestVaultSnapshotExportGuards(t *testing.T) {
t.Run("unknown subject", func(t *testing.T) {
h := newVaultPersistHarness(t)
rec := doAction(t, h.srv, snapshotBody(t, "ghost", testFenceR, testFenceA, "VAULT_EXPORT", "/tmp/bundle.json"))
- // An unregistered subject has no vault state to carry (a sandbox
- // without an egress binding, or after terminal cleanup): 200 no-op
- // so the checkpoint proceeds. See the dedicated no-op test.
+ // 200 no-op: nothing to carry. See the dedicated no-op test.
assert.Equal(t, http.StatusOK, rec.Code)
assert.False(t, h.fakeForPath("/tmp/bundle.json").exists)
})
diff --git a/components/egress/pkg/actionhandler/actionhandler.go b/components/egress/pkg/actionhandler/actionhandler.go
index 4d1cbc9d2..7e7d5d40b 100644
--- a/components/egress/pkg/actionhandler/actionhandler.go
+++ b/components/egress/pkg/actionhandler/actionhandler.go
@@ -286,11 +286,9 @@ type StatusResponse struct {
// (interception fails closed for them; a successful write clears it).
Recovery bool `json:"vaultRecovery,omitempty"`
DegradedVaults []string `json:"degradedVaults,omitempty"`
- // VaultTransfer is the vault-transfer capability signal (OSEP-0022,
- // additive): true only when this Handler serves VAULT_EXPORT /
- // VAULT_IMPORT AND persists vault files (OPENSANDBOX_EGRESS_VAULT_DIR
- // configured). Absent on older Handler builds, which decodes as false:
- // the Fastlet then skips the pause/resume vault transfer and the
- // checkpoint proceeds without a vault — the pre-transfer semantics.
+ // VaultTransfer reports that this Handler serves VAULT_EXPORT /
+ // VAULT_IMPORT and persists vault files. Absent on older builds decodes
+ // as false: the Fastlet then skips the transfer (checkpoint proceeds
+ // without a vault) instead of failing it.
VaultTransfer bool `json:"vaultTransfer,omitempty"`
}
diff --git a/scripts/fast-sandbox-env/integration-env.sh b/scripts/fast-sandbox-env/integration-env.sh
index 584158559..199ee3b96 100755
--- a/scripts/fast-sandbox-env/integration-env.sh
+++ b/scripts/fast-sandbox-env/integration-env.sh
@@ -202,9 +202,7 @@ ACTION=""
log() { printf '\033[1;34m[fast-sandbox-env]\033[0m %s\n' "$*" | tee -a "$WORK/run.log" >&2; }
die() { printf '\033[1;31m[fast-sandbox-env] ERROR:\033[0m %s\n' "$*" >&2; exit 1; }
pass() { printf '\033[1;32m[fast-sandbox-env] PASS\033[0m %s\n' "$*" | tee -a "$WORK/run.log" >&2; }
-# fail dumps before exiting: an explicit `exit` does NOT fire the ERR trap
-# (only a failing command does), so verify-stage failures reached through
-# wait_for -> fail would otherwise tear down the cluster with no pod logs.
+# fail dumps before exiting: an explicit `exit` does not fire the ERR trap.
fail() { printf '\033[1;31m[fast-sandbox-env] FAIL\033[0m %s\n' "$*" >&2; failure_dump "${STAGE_CUR_LABEL:-$ACTION}" || true; exit 1; }
highlight() { printf '\033[1;36m%s\033[0m\n' "$*"; }
@@ -328,12 +326,9 @@ failure_dump() {
kubectl logs -n "$NS" deploy/fast-sandbox-controller --tail=80 2>&1 || true
echo "--- firecracker-runtime logs (tail) ---"
kubectl logs -n "$NS" daemonset/firecracker-runtime --all-containers --tail=80 2>&1 || true
- # --all-containers is the point: `kubectl logs -l` without it logs
- # only the first container (fastlet) and silently drops egress /
- # fastlet-proxy, which is where pause/snapshot and policy failures
- # actually surface. The window is deep because a failure loop (a
- # per-second pause retry) floods the tail and buries the
- # pre-incident history that explains how the failure started.
+ # --all-containers: the label selector alone logs only the first
+ # container and drops egress. Deep tail: failure loops flood the
+ # window and bury the pre-incident history.
echo "--- fastlet pod logs, every container (deep tail) ---"
kubectl logs -n "$RESOURCE_NS" -l app=sandbox-fastlet \
--all-containers --prefix --timestamps --tail=15000 2>&1 || true
@@ -357,10 +352,8 @@ failure_dump() {
echo "--- Sandbox CRs (status only) ---"
kubectl get sandboxes -n "$RESOURCE_NS" -o json 2>/dev/null \
| jq -c '.items[] | {name: .metadata.name, phase: .status.phase, conditions: .status.conditions, infra: .status.infraComponents}' 2>&1 || true
- # Cross-container visibility of the snapshot staging area: the vault
- # export hands the DRIVER's bundle path to the egress container, and
- # a mount-namespace mismatch shows up as ENOENT on the egress side
- # only. Compare both views.
+ # A mount-namespace mismatch (driver staging path invisible to
+ # egress) shows up as ENOENT on the egress side only.
for pod in $(kubectl -n "$RESOURCE_NS" get pods -l app=sandbox-fastlet -o name 2>/dev/null); do
echo "### ${pod##*/}: snapshot staging visibility"
echo "-- fastlet view:"
@@ -1708,9 +1701,8 @@ vault_revision_is() { # <expected-revision>
vault_lifecycle_verify() {
local created vault_body ack rev sandbox_uid
log "verify (vault): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID)"
- # The policy rides the create (the vault validates its bindings against
- # it, and a create WITHOUT a policy produces no egress binding at all —
- # the subject would never register).
+ # The policy rides the create: a policy-less create produces no egress
+ # binding at all, so the subject would never register.
jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "ext.test"}]}' \
> "$GEN_DIR/vault-policy.json"
created="$(osb_api sandbox create --template "$TEMPLATE_ID" --timeout 1h \
@@ -1722,16 +1714,13 @@ vault_lifecycle_verify() {
wait_for "vault sandbox Running" 150 sandbox_running
sandbox_uid="$(kubectl -n "$RESOURCE_NS" get sandbox "$VERIFY_ID" -o jsonpath='{.metadata.uid}' 2>/dev/null)"
[[ -n "$sandbox_uid" ]] || fail "vault verify: sandbox CR carried no uid"
- # The subject must be registered AND activated (deny-first install, then
- # the data-plane-ready hook flips it) before the vault POST; otherwise
- # the relay 404s (create -> ready -> set vault). The second
- # "SET_BINDING applied (state=active)" line only appears after a policy
- # UPDATE; a create-carried policy activates without it.
+ # Registered and activated before the vault POST, else the relay 404s.
+ # (The "(state=active)" re-delivery only follows a policy UPDATE; a
+ # create-carried policy activates without it.)
wait_for "vault subject registered on egress" 120 \
vault_log_contains "subject s-$sandbox_uid: data-plane-ready, policy active"
- # 1) create-only: 201 ack carries the egress-validated revision; a
- # duplicate POST is 409 (atomic across server replicas).
+ # 1) create-only: ack carries the revision; duplicate POST is 409.
vault_body="$(jq -n '{credentials: [{name: "tok", source: {type: "inline", value: "sekrit-vault-e2e"}}], bindings: [{name: "ext", match: {hosts: ["ext.test"]}, auth: {type: "apiKey", name: "X-Api-Key", credential: "tok"}}]}')"
ack="$(server_api POST "/sandboxes/$VERIFY_ID/credential-vault" "$vault_body")" \
|| fail "vault verify: POST credential-vault failed"
@@ -1745,9 +1734,8 @@ vault_lifecycle_verify() {
|| fail "vault verify: duplicate POST was not 409"
pass "duplicate POST rejected (create-only 409)"
- # 2) pause/resume carries the REAL vault: the egress copies the vault
- # file into the checkpoint bundle (export) and re-imports it on
- # restore; the relay GET still reports the same revision.
+ # 2) pause/resume carries the real vault: export -> bundle -> import,
+ # with the relay GET reporting the same revision.
osb_api sandbox pause "$VERIFY_ID" >/dev/null 2>&1 || fail "vault verify: pause failed"
wait_for "vault sandbox Paused (checkpoint durable)" 240 sandbox_paused
wait_for "vault exported into the checkpoint bundle" 120 \
@@ -1755,15 +1743,16 @@ vault_lifecycle_verify() {
pass "pause exported the vault bundle (subject sealed)"
osb_api sandbox resume "$VERIFY_ID" --skip-health-check >/dev/null 2>&1 || fail "vault verify: resume failed"
wait_for "vault sandbox Running (restored)" 240 sandbox_running
- wait_for "vault re-imported from the bundle" 120 \
- vault_log_contains "subject s-$sandbox_uid: vault imported from snapshot bundle"
+ # The pause's REMOVE_BINDING purges the subject, so the restore import
+ # lands through the unregistered branch and the file reloads at the
+ # SET_BINDING replay (the reload line carries the restored revision).
+ wait_for "vault bundle reloaded at re-registration" 120 \
+ vault_log_contains "subject s-$sandbox_uid: vault reloaded from file"
wait_for "vault revision preserved across pause/resume" 120 vault_revision_is "$rev"
pass "vault survived pause/resume (revision $rev preserved)"
- # 3) egress-restart recovery: crash the worker (the supervisor restarts
- # it with a fresh instanceId); the fastlet replays SET_BINDING and the
- # vault file reloads by incarnation. The GET succeeding with the same
- # revision proves the whole chain (relay up, replay done, file loaded).
+ # 3) egress-restart recovery: crash the worker (fresh instanceId ->
+ # SET_BINDING replay -> vault file reload by incarnation).
local pod restarted=0
for pod in $(pool_pods); do
if kubectl -n "$RESOURCE_NS" exec "$pod" -c egress -- sh -c 'command -v pkill >/dev/null' 2>/dev/null; then
@@ -1779,8 +1768,7 @@ vault_lifecycle_verify() {
pass "vault reloaded after egress restart (revision $rev preserved)"
fi
- # 4) delete + re-create: DELETE is terminal, GET 404s, a fresh POST is a
- # new vault at revision 1.
+ # 4) delete + re-create.
server_api DELETE "/sandboxes/$VERIFY_ID/credential-vault" >/dev/null \
|| fail "vault verify: DELETE failed"
[[ "$(server_api_status GET "/sandboxes/$VERIFY_ID/credential-vault")" == "404" ]] \
diff --git a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml
index f4ef931f5..0392da8b9 100644
--- a/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml
+++ b/scripts/fast-sandbox-env/manifests/pool/firecracker-egress-pool.yaml
@@ -84,13 +84,9 @@ spec:
# trust surface. Swap to `medium: Memory` (tmpfs) to keep the
# per-subject vault files off node disk.
mountPath: /var/lib/opensandbox-egress
- # Vault bundle handoff (OSEP-0022): the export/import envelope
- # carries the DRIVER's snapshot staging path, which lives under the
- # runtime state root. Mount the same runtime-injected hostPath
- # volume the fastlet container gets (the controller adds it for the
- # first container and merges it into the pod spec), so the Handler
- # resolves the path in its own mount namespace. Read-write: the
- # Handler copies exports out and reads imports in.
+ # Vault bundle handoff (OSEP-0022): the envelope carries the
+ # driver's staging path under the runtime state root; mount the
+ # same runtime-injected volume so the path resolves here.
- name: firecracker-state
mountPath: /var/lib/fast-sandbox/firecracker
# Egress installs nft/iptables rules (DNS redirect + per-subject
diff --git a/server/configuration.md b/server/configuration.md
index 7ab9e0328..a926dbc7d 100644
--- a/server/configuration.md
+++ b/server/configuration.md
@@ -167,7 +167,7 @@ The kubernetes runtime also serves fsb (`fsb-`) sandboxes side by side; these fi
| `fastpath_wait_ready_seconds` | number | `30.0` | Bounded readiness wait for DataPlaneReady after Create. |
| `fastpath_resource_pool` | string | `"default-pool"` | SandboxPool when `extensions.poolRef` is unset. |
| `template_s3_publish_secret` | string | `"sandbox-oss-credentials"` | Object-store credentials secret for server-created SandboxTemplates. |
-| `template_mitm_ca_secret` | string \| omitted | `null` | Secret holding the pinned cluster MITM CA certificate (OSEP-0022). When set, server-created SandboxTemplates bake `spec.egressMitmCertificate` from it so guests trust the interception CA from first boot; the referenced entry must carry the certificate only, never the private key. The Secret must exist in each template's namespace (ops-runbook distribution) — a missed copy fails the build observably. Unset keeps templates CA-free. |
+| `template_mitm_ca_secret` | string \| omitted | `null` | Secret holding the pinned cluster MITM CA certificate (OSEP-0022). When set, server-created SandboxTemplates bake `spec.egressMitmCertificate` from it (certificate entry only, never the private key); unset keeps templates CA-free. The Secret must exist in each template's namespace (ops-runbook distribution); a missed copy fails the build observably. |
| `template_mitm_ca_secret_key` | string | `"ca.crt"` | Secret entry holding the CA certificate when `template_mitm_ca_secret` is set. |
---
diff --git a/server/opensandbox_server/config.py b/server/opensandbox_server/config.py
index b944b808a..c10b304bd 100644
--- a/server/opensandbox_server/config.py
+++ b/server/opensandbox_server/config.py
@@ -818,12 +818,10 @@ class KubernetesRuntimeConfig(BaseModel):
description=(
"Secret holding the pinned cluster MITM CA certificate "
"(OSEP-0022). When set, server-created SandboxTemplates bake "
- "spec.egressMitmCertificate from it so guests trust the "
- "interception CA from first boot; the referenced entry must "
- "carry the certificate only, never the private key. The Secret "
- "must exist in each template's namespace (ops-runbook "
- "distribution) — a missed copy fails the build observably. "
- "Unset keeps templates CA-free."
+ "spec.egressMitmCertificate from it (certificate entry only, "
+ "never the private key); unset keeps templates CA-free. The "
+ "Secret must exist in each template's namespace (ops-runbook "
+ "distribution); a missed copy fails the build observably."
),
)
template_mitm_ca_secret_key: str = Field(
diff --git a/server/opensandbox_server/services/templates/template_service.py b/server/opensandbox_server/services/templates/template_service.py
index 1a1769f0f..2b21d80e9 100644
--- a/server/opensandbox_server/services/templates/template_service.py
+++ b/server/opensandbox_server/services/templates/template_service.py
@@ -397,13 +397,11 @@ class FastSandboxTemplateService:
env = record.spec.get("env") or {}
if env:
spec["envs"] = [{"name": name, "value": value} for name, value in env.items()]
- # OSEP-0022 CA wiring: when the operator configured the pinned MITM CA
- # Secret, every server-created template bakes it — the certificate
- # entry only, never the private key (the builder rejects key material
- # and installs the certificate into the guest trust store at the
- # snapshot boot). Secret refs are namespace-local, so the copy must
- # exist in the template's namespace (ops-runbook distribution); a
- # missed copy fails the build observably instead of degrading.
+ # OSEP-0022 CA wiring: bake the pinned MITM CA Secret's certificate
+ # entry (never the private key) into every server-created template
+ # when configured. The Secret must exist in the template's
+ # namespace (ops-runbook distribution); a missed copy fails the
+ # build observably.
if self._k8s_config.template_mitm_ca_secret:
mitm_certificate: dict = {
"secretRef": {"name": self._k8s_config.template_mitm_ca_secret}