Secure, Fast, and Extensible Sandbox runtime for AI agents.
perf(egress,verify): create-latency evidence pipeline + zero-fsync policy bootstrap (#2323) (#2327)
epha5feb5be3d502
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.goindex b45cb82f8..71f0333ec 100644--- a/components/egress/fastsandbox_actions.go+++ b/components/egress/fastsandbox_actions.go@@ -241,12 +241,15 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (}s.rehydrateVaultReload(subj, fence, seed, policyDegraded)}+ setBindingStarted := time.Now()+ installStarted := time.Now()state, err := s.reg.RegisterAndEnforce(subj, subject.SubjectKey{SourceIP: att.IP}, fence, func() error {return s.OnRegistered(subj, att)}, s.takeoverStaleSubject)if err != nil {return http.StatusInternalServerError, fmt.Errorf("deny-first install: %w", err)}+ denyFirstInstall := time.Since(installStarted)s.recordBindingState(subj, att, fence)if env.Binding.IsRemoval() {// Binding removed from a still-live sandbox: drop the pending policy@@ -270,11 +273,14 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (}// Install the binding policy (or honor the file-backed record that// supersedes it) through the policy commit protocol.+ policyStarted := time.Now()if err := s.applyBindingPolicyLocked(subj, fence, pol, state == subject.StateActive); err != nil {log.Errorf("subject %s: binding policy commit failed: %v", subj, err)return http.StatusInternalServerError, fmt.Errorf("binding policy commit: %w", err)}- log.Infof("subject %s: SET_BINDING applied (state=%s)", subj, state)+ policyCommit := time.Since(policyStarted)+ log.Infof("subject %s: SET_BINDING applied (state=%s) total=%dms denyFirstInstall=%dms policyCommit=%dms",+ subj, state, time.Since(setBindingStarted).Milliseconds(), denyFirstInstall.Milliseconds(), policyCommit.Milliseconds())return http.StatusOK, nil}@@ -331,11 +337,12 @@ func (s *fastSandboxPolicyServer) applyLifecycleHook(env *actionhandler.Envelope// the subject must never activate without its current policy.return http.StatusConflict, fmt.Errorf("data-plane-ready for subject %s with no pending policy", subj)}+ applyStarted := time.Now()if err := s.applyPolicy(subj, pol); err != nil {return http.StatusInternalServerError, fmt.Errorf("policy apply: %w", err)}s.clearPendingPolicy(subj)- log.Infof("subject %s: data-plane-ready, policy active", subj)+ log.Infof("subject %s: data-plane-ready, policy active applyPolicy=%dms", subj, time.Since(applyStarted).Milliseconds())return http.StatusOK, nildefault:// Unreachable: ParseEnvelope validated the Hook name.diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.goindex ae4327399..0c2569a20 100644--- a/components/egress/fastsandbox_policy_persist.go+++ b/components/egress/fastsandbox_policy_persist.go@@ -18,9 +18,17 @@// source for an egress restart — at SET_BINDING replay the committed file// supersedes the replayed binding input, which carries only the original// bootstrap policy — and the bundle source for the POLICY_EXPORT /-// POLICY_IMPORT snapshot transfer. Ordering is persist → apply → ack-// (acked ⇒ durable and applied), with the same three-state classification,-// snapshot restore, and recovery latch as the vault files.+// POLICY_IMPORT snapshot transfer. Ordering is persist → apply → ack.+//+// Durability is tiered by replayability: the bootstrap candidate+// (applyBindingPolicyLocked) publishes non-durably — rev 1 is byte-identical+// to the binding input, which the fastlet re-issues on every egress instance+// change, so a lost rename is reconstructed exactly by replay; relay writes+// and the removal tombstone record state that outruns the binding input and+// stay fully durable (acked ⇒ durable for them). For the bootstrap ack the+// guarantee is "recorded and enforced (deny-first)", not "durable". The+// three-state classification, snapshot restore, and recovery latch remain+// the same as the vault files.//// TODO(encryption-at-rest): template-borne state blobs are stored as-is.// When encryption lands it protects the store-side reader only — CLONE@@ -515,10 +523,17 @@ func (s *fastSandboxPolicyServer) applyBindingPolicyLocked(subj subject.Subject,if err != nil {return fmt.Errorf("policy file encode: %w", err)}+ // The bootstrap seed publishes NON-durably: rev 1 is byte-identical to+ // the binding input, which the fastlet re-issues on every egress+ // instance change — a crash that loses the rename is reconstructed+ // exactly by that replay (deny-first nft already enforces meanwhile).+ // Everything that outruns the binding input stays durable: relay+ // writes (SaveRaw) and the removal tombstone in+ // clearPolicyRecordLocked.return s.persistPolicyCandidateLocked(subj,func(fs vaultFileStore) (policy.FileMutationState, error) {- return fs.SaveRaw(payload)+ return fs.SaveRawNonDurable(payload)},apply,)diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.goindex 5a979e0cc..f62a6c049 100644--- a/components/egress/fastsandbox_server.go+++ b/components/egress/fastsandbox_server.go@@ -453,15 +453,19 @@ func (s *fastSandboxPolicyServer) OnRegistered(subj subject.Subject, att actionh// policy writes.nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)defer cancel()+ denyFirstStarted := time.Now()if err := s.nft.ApplyDenyFirst(nftCtx, subj, att); err != nil {return err}+ dnsStarted := time.Now()if err := s.installGatewayDNSRedirect(subj, att.Gateway); err != nil {// Fail closed: sandbox DNS addressed to gateway:53 must reach the// proxy; without the redirect the sandbox would fall back to a// resolver the policy cannot see.return err}+ dnsRedirect := time.Since(dnsStarted)+ mitmStarted := time.Now()if err := s.setMitmRedirect(subj, att, false); err != nil {// Fail closed: a sandbox whose HTTP(S) is not intercepted must not// register as usable (it could exfiltrate credentials-bearing@@ -471,7 +475,8 @@ func (s *fastSandboxPolicyServer) OnRegistered(subj subject.Subject, att actionhs.releaseGatewayDNSRedirect(subj)return err}- log.Infof("subject %s deny-first enforced (nft + gateway redirect + mitm redirect)", subj)+ log.Infof("subject %s deny-first enforced (nft + gateway redirect + mitm redirect) total=%dms denyFirst=%dms dnsRedirect=%dms mitm=%dms",+ subj, time.Since(denyFirstStarted).Milliseconds(), dnsStarted.Sub(denyFirstStarted).Milliseconds(), dnsRedirect.Milliseconds(), time.Since(mitmStarted).Milliseconds())return nil}diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.goindex 3036551a8..69959e3c3 100644--- a/components/egress/fastsandbox_vault_persist.go+++ b/components/egress/fastsandbox_vault_persist.go@@ -53,6 +53,11 @@ type vaultRecoveryState struct {type vaultFileStore interface {Snapshot() (*policy.PolicyFileSnapshot, error)SaveRaw(data []byte) (policy.FileMutationState, error)+ // SaveRawNonDurable publishes by rename without durability barriers —+ // only valid for content the caller can reconstruct by replay (the+ // atomic store documents the contract). Everything that outruns its+ // replay source must use SaveRaw.+ SaveRawNonDurable(data []byte) (policy.FileMutationState, error)Restore(*policy.PolicyFileSnapshot) (policy.FileMutationState, error)Delete() (policy.FileMutationState, error)}diff --git a/components/egress/fastsandbox_vault_persist_test.go b/components/egress/fastsandbox_vault_persist_test.goindex a2938de57..cb4d5685f 100644--- a/components/egress/fastsandbox_vault_persist_test.go+++ b/components/egress/fastsandbox_vault_persist_test.go@@ -104,6 +104,12 @@ func (f *fakeVaultFile) SaveRaw(data []byte) (policy.FileMutationState, error) {return f.saveState, nil}+func (f *fakeVaultFile) SaveRawNonDurable(data []byte) (policy.FileMutationState, error) {+ // Same observable effect as SaveRaw in the fake: the durability+ // difference lives in the real atomic store's fsync behavior.+ return f.SaveRaw(data)+}+func (f *fakeVaultFile) Restore(s *policy.PolicyFileSnapshot) (policy.FileMutationState, error) {f.restoreCalls++f.restored = append(f.restored, s)diff --git a/components/egress/pkg/policy/atomic_store_linux.go b/components/egress/pkg/policy/atomic_store_linux.goindex b6d24718c..e9f615935 100644--- a/components/egress/pkg/policy/atomic_store_linux.go+++ b/components/egress/pkg/policy/atomic_store_linux.go@@ -162,6 +162,83 @@ func (s *AtomicPolicyFile) SaveRaw(data []byte) (FileMutationState, error) {})}+// SaveRawNonDurable publishes exact bytes by rename without any durability+// barrier: no temporary-file sync, no security/IMA finalize, no directory+// sync. The three-state classification still describes the namespace effect+// (rename done = FileCommitted; pre-rename failure = FileUnchanged; a rename+// error = FileUnknown — the rename may have taken effect).+//+// Only valid for content the caller can reconstruct by replay: a crash may+// roll the file back to its previous content (or lose it entirely), so every+// publisher here must have an authority that re-derives the bytes — e.g. the+// fastlet re-issuing SET_BINDING with the binding input. Content that+// outruns its replay source must go through SaveRaw.+func (s *AtomicPolicyFile) SaveRawNonDurable(data []byte) (FileMutationState, error) {+ return s.withDirectory(func(dir *os.File) (FileMutationState, error) {+ current, err := s.ops.snapshot(dir, s.name, s.path)+ if err != nil {+ return FileUnchanged, atomicFileError("snapshot", err)+ }+ return s.replaceNonDurable(dir, data, current)+ })+}++// replaceNonDurable is replace minus the durability barriers: the temporary+// inode keeps the same mode/ownership rules and the same deferred cleanup,+// but the publish is a bare rename — the kernel may lose it (or keep the+// previous content) across a power failure. No security/IMA finalize: the+// check exists to guard the durable record; a replayable candidate tolerates+// a label/digest difference until a durable SaveRaw re-publishes.+func (s *AtomicPolicyFile) replaceNonDurable(dir *os.File, data []byte, metadata *PolicyFileSnapshot) (state FileMutationState, err error) {+ temp, err := s.ops.create(dir)+ if err != nil {+ return FileUnchanged, atomicFileError("create", err)+ }+ tempName := temp.Name()+ closed := false+ defer func() {+ if !closed {+ if closeErr := temp.Close(); closeErr != nil {+ err = errors.Join(err, atomicFileError("temporary-close", closeErr))+ }+ }+ if tempName != "" {+ if removeErr := s.ops.remove(dir, tempName); removeErr != nil && !errors.Is(removeErr, fs.ErrNotExist) {+ err = errors.Join(err, atomicFileError("temporary-cleanup", removeErr))+ }+ }+ }()+ if n, writeErr := temp.Write(data); writeErr != nil || n != len(data) {+ if writeErr == nil {+ writeErr = io.ErrShortWrite+ }+ return FileUnchanged, atomicFileError("write", writeErr)+ }+ mode := fs.FileMode(0o600)+ if metadata.exists {+ // Chown can clear set-id bits, so apply mode after ownership.+ if err := temp.Chown(metadata.uid, metadata.gid); err != nil {+ return FileUnchanged, atomicFileError("ownership", err)+ }+ mode = metadata.mode+ }+ if err := temp.Chmod(mode); err != nil {+ return FileUnchanged, atomicFileError("mode", err)+ }+ closed = true+ if err := temp.Close(); err != nil {+ return FileUnchanged, atomicFileError("file-close", err)+ }+ // From the first publication attempt onward an error is never classified as+ // unchanged. In particular, remote filesystems can report a failed rename+ // after the server has already committed it.+ if err := s.ops.rename(dir, tempName, s.name); err != nil {+ return FileUnknown, atomicFileError("rename", err)+ }+ tempName = ""+ return FileCommitted, nil+}+// Delete durably unlinks the authoritative path: an already-absent path is// FileUnchanged with a nil error; a failed unlink or directory sync is// FileUnknown (the unlink may still have taken effect).diff --git a/components/egress/pkg/policy/atomic_store_other.go b/components/egress/pkg/policy/atomic_store_other.goindex b663f38cf..ee44f350f 100644--- a/components/egress/pkg/policy/atomic_store_other.go+++ b/components/egress/pkg/policy/atomic_store_other.go@@ -35,6 +35,9 @@ func (*AtomicPolicyFile) Save(*NetworkPolicy) (FileMutationState, error) {func (*AtomicPolicyFile) SaveRaw([]byte) (FileMutationState, error) {return FileUnchanged, atomicPolicyPlatformError()}+func (*AtomicPolicyFile) SaveRawNonDurable([]byte) (FileMutationState, error) {+ return FileUnchanged, atomicPolicyPlatformError()+}func (*AtomicPolicyFile) Restore(*PolicyFileSnapshot) (FileMutationState, error) {return FileUnchanged, atomicPolicyPlatformError()}diff --git a/components/egress/pkg/policy/atomic_store_raw_nondurable_linux_test.go b/components/egress/pkg/policy/atomic_store_raw_nondurable_linux_test.gonew file mode 100644index 000000000..21bf4785e--- /dev/null+++ b/components/egress/pkg/policy/atomic_store_raw_nondurable_linux_test.go@@ -0,0 +1,79 @@+//go:build linux++// Copyright 2026 The OpenSandbox Authors+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package policy++import (+ "os"+ "path/filepath"+ "testing"++ "github.com/stretchr/testify/require"+)++// TestAtomicPolicyFileSaveRawNonDurable exercises the replayable-candidate+// surface: identical namespace effect and three-state classification as+// SaveRaw (byte-exact publish, atomic replace, full temp cleanup), with the+// durability barriers left out — which is not directly observable in-process,+// so the assertions pin the observable contract the replay invariant relies on.+func TestAtomicPolicyFileSaveRawNonDurable(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "s-1.json")+ s, err := NewAtomicPolicyFile(path)+ require.NoError(t, err)++ first := []byte(`{"v":1,"incarnation":"rt-1","revision":1,"vault":{}}` + "\n")+ state, err := s.SaveRawNonDurable(first)+ require.NoError(t, err)+ require.Equal(t, FileCommitted, state)+ got, err := os.ReadFile(path)+ require.NoError(t, err)+ require.Equal(t, string(first), string(got))+ require.Equal(t, 0o600, fileMode(t, path), "a new file is 0600")+ assertNoAtomicTemps(t, dir)++ // A second non-durable save replaces atomically like SaveRaw.+ second := []byte(`{"v":1,"incarnation":"rt-1","revision":2,"vault":{}}` + "\n")+ state, err = s.SaveRawNonDurable(second)+ require.NoError(t, err)+ require.Equal(t, FileCommitted, state)+ got, err = os.ReadFile(path)+ require.NoError(t, err)+ require.Equal(t, string(second), string(got))+ assertNoAtomicTemps(t, dir)++ // Existing mode/ownership are preserved on replacement, like SaveRaw.+ require.NoError(t, os.Chmod(path, 0o640))+ third := []byte(`{"v":1,"incarnation":"rt-1","revision":3,"vault":{}}` + "\n")+ state, err = s.SaveRawNonDurable(third)+ require.NoError(t, err)+ require.Equal(t, FileCommitted, state)+ got, err = os.ReadFile(path)+ require.NoError(t, err)+ require.Equal(t, string(third), string(got))+ require.Equal(t, 0o640, fileMode(t, path), "replacement copies the existing mode")+ assertNoAtomicTemps(t, dir)++ // The durable path still works after non-durable publishes.+ fourth := []byte(`{"v":1,"incarnation":"rt-1","revision":4,"vault":{}}` + "\n")+ state, err = s.SaveRaw(fourth)+ require.NoError(t, err)+ require.Equal(t, FileCommitted, state)+ got, err = os.ReadFile(path)+ require.NoError(t, err)+ require.Equal(t, string(fourth), string(got))+ assertNoAtomicTemps(t, dir)+}diff --git a/scripts/fast-sandbox-env/integration-env.sh b/scripts/fast-sandbox-env/integration-env.shindex 199ee3b96..c0ad582f3 100755--- a/scripts/fast-sandbox-env/integration-env.sh+++ b/scripts/fast-sandbox-env/integration-env.sh@@ -291,6 +291,10 @@ rc() {on_error() {local rc=$? task="$1"+ # Record WHERE it died before anything else: the dump alone cannot+ # distinguish a silent command failure from an environment problem+ # (e.g. the RustFS stage once died in ~1s with zero output).+ log "up failed: task=$task rc=$rc command=${BASH_COMMAND:-unknown} at ${BASH_SOURCE[1]:-?}:${BASH_LINENO[0]:-?}" || true# Subshells inherit ERR: propagate the error instead of tearing down# main-process resources; ignore SIGPIPE (cancelled runners).trap - ERR@@ -503,7 +507,7 @@ preflight() {if (( avail_kb < min_free_kb )); thendie "$target has $((avail_kb / 1024 / 1024))G free; at least 20G is required (built images, XFS StateRoot, RustFS artifacts). Free space (docker system prune / old kind clusters) or point WORK / RUSTFS_DATA / XFS_LOOP_FILE at a bigger volume"fi- log "disk headroom: $target has $((avail_kb / 1024 / 1024 / 1024))G free"+ log "disk headroom: $target has $((avail_kb / 1024 / 1024))G free"done# Fail fast on busy host ports instead of a cryptic docker/kind bind error.local port@@ -1464,43 +1468,83 @@ verify_policy_updated() {[[ "$(printf '%s' "$out" | jq -r '.policy.egress[0].target // empty')" == "github.com" ]]}-# Full wire-up: CLI create (fsb) -> FastPath -> fastlet -> firecracker+# Full wire-up: raw-API create (fsb) -> FastPath -> fastlet -> firecracker# sandbox -> signed gateway route -> fastlet-proxy -> guest execd /ping -># delete. One sandbox per call; availability is measured from the client-# (poll /ping until 200, wall-clock budget — each route poll is a CLI call,-# ~0.5s startup), not from CR status.+# (poll /ping until 200, wall-clock budget — every poll is curl-only, so+# readings resolve ~10ms), not from CR status. The timed windows wrap HTTP+# round trips only: osb invocations would bill ~0.5s of interpreter startup+# to each reading, which is exactly the measurement pollution #2323 called+# out. CLI create coverage lives in the pause-resume/vault/snapshot verifies+# and the lifecycle ops below.verify_one_sandbox() { # <label> <ping-budget-ms>- local label="$1" budget="$2" created t0 t1 t2 route code deadline+ local label="$1" budget="$2" response t0 t1 t2 route code deadline payloadlocal policy_file="$GEN_DIR/verify-policy.json"t0="$(now_ms)"# Egress policy rides on every create (binding -> nft rules), so the# chain is exercised each time; /ping is inbound and unaffected.jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "example.com"}, {action: "allow", target: "*.opensandbox.ai"}]}' \> "$policy_file"- log "verify ($label): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID)"- created="$(osb_api sandbox create --template "$TEMPLATE_ID" --timeout 1h \- --metadata origin=fast-sandbox-env-verify \- --network-policy-file "$policy_file" \- --skip-health-check -o json 2>/dev/null)" \- || fail "osb sandbox create failed against $SERVER_URL: $(osb_api sandbox create \- --template "$TEMPLATE_ID" --timeout 1h --skip-health-check 2>&1 >/dev/null | head -c 400)"- VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"+ log "verify ($label): creating a sandbox via the raw API (templateId=$TEMPLATE_ID; the timed window is curl-only)"+ # The timed windows wrap HTTP round trips ONLY: an osb invocation here+ # would bill ~0.5s of Python startup to every reading. The CLI surface+ # stays covered by the lifecycle ops and the osb create smoke.+ local payload+ payload="$(jq -n --arg tpl "$TEMPLATE_ID" --slurpfile pol "$policy_file" \+ '{templateId: $tpl, timeout: 3600, metadata: {origin: "fast-sandbox-env-verify"}, networkPolicy: $pol[0]}')"+ response="$(server_api POST "/sandboxes" "$payload")" \+ || fail "POST /sandboxes failed against $SERVER_URL (payload: $(printf '%s' "$payload" | head -c 200))"+ VERIFY_ID="$(printf '%s' "$response" | jq -r '.id')"[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"t1="$(now_ms)"- # Availability = /ping 200 through the signed gateway route (port 44772).- deadline=$(( t1 + budget * 1000000 ))+ # Availability = /ping 200 through the signed gateway route (port 44772),+ # measured at ~1ms resolution:+ # phase 1 — route publication, curl-only busy poll (the curl spawn+ # itself paces ~5ms/attempt; /ping cannot 200 before the route exists,+ # so phase-1 cadence delays detection, never pollutes the reading);+ # phase 2 — /ping hammered over a bash /dev/tcp socket: no process+ # spawn per attempt, EPOCHREALTIME keeps the hot loop fork-free.+ local gw gw_host gw_port line deadline_ms+ gw="${GATEWAY_URL#*://}"; gw="${gw%%/*}"+ gw_host="${gw%%:*}"+ gw_port="${gw##*:}"+ [[ "$gw_port" == "$gw" ]] && gw_port=80+ deadline_ms=$(( t1 / 1000000 + budget ))while :; do- route="$(osb_api sandbox endpoint "$VERIFY_ID" --port 44772 -o json 2>/dev/null \+ route="$( { server_api GET "/sandboxes/$VERIFY_ID/endpoints/44772" 2>/dev/null || true; } \| jq -r '.headers["OpenSandbox-Ingress-To"] // empty')"- if [[ -n "$route" ]]; then- code="$(curl -sS -m 5 -o /dev/null -w '%{http_code}' \- -H "OpenSandbox-Ingress-To: $route" "$GATEWAY_URL/ping" 2>/dev/null || true)"- [[ "$code" == "200" ]] && break+ [[ -n "$route" ]] && break+ if (( $(now_ms) / 1000000 >= deadline_ms )); then+ fail "gateway route for $VERIFY_ID did not publish within ${budget}ms"+ fi+ done+ code=""+ while :; do+ line=""+ # NOTE: no 2>/dev/null here — exec-without-command applies+ # redirections to the CURRENT shell, so that would permanently+ # swallow the script's stderr (every later log/pass goes dark).+ # A refused connect prints one honest bash message instead.+ if exec 3<>"/dev/tcp/$gw_host/$gw_port"; then+ printf 'GET /ping HTTP/1.1\r\nHost: %s\r\nOpenSandbox-Ingress-To: %s\r\nConnection: close\r\n\r\n' \+ "$gw" "$route" >&3+ IFS= read -r -t 1 line <&3 || true+ exec 3>&- 3<&-+ else+ sleep 0.01+ fi+ if [[ "$line" == *" 200 "* ]]; then+ code=200+ breakfi- if (( $(now_ms) >= deadline )); then- fail "execd /ping did not return 200 within ${budget}ms (last code=${code:-none}, route=${route:-none})"+ if [[ -n "${EPOCHREALTIME:-}" ]]; then+ now=$((( ${EPOCHREALTIME/./} ) / 1000))+ else+ now=$(( $(now_ms) / 1000000 ))+ fi+ if (( now >= deadline_ms )); then+ fail "execd /ping did not return 200 within ${budget}ms (last status: ${line:-none})"fi- sleep 0.2donet2="$(now_ms)"# jq filter in a variable: literal parens trip older bash's $( ) parser.@@ -1574,6 +1618,39 @@ verify_lifecycle_ops() { # <sandbox-id>pass "lifecycle: osb sandbox renew 2h ($expected_expires -> $renewed_expires)"}+# dump_latency_evidence collects the per-create timing instrumentation from+# every component on the create critical path into one greppable file:+# fastlet phase breakdown (fastlet create timing:), fastlet->egress action+# invocations (fastlet action invoke:), egress sidecar SET_BINDING / hook+# steps (timing fields on its subject lines), and fastpath create totals.+# Append-only evidence for create-latency analysis (issue #2323 style).+dump_latency_evidence() {+ local out="$LOGS_DIR/latency-evidence-$(date +%s).txt"+ {+ echo "=== latency evidence $(date -u +%FT%TZ) ==="+ echo+ echo "--- fastlet: create phase + action-invoke timing ---"+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet --tail=3000 2>/dev/null \+ | grep -E "fastlet create timing|fastlet action invoke" || true+ echo+ echo "--- egress sidecar: SET_BINDING / hook timing ---"+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet -c egress --tail=3000 2>/dev/null \+ | grep -E "deny-first enforced|SET_BINDING applied|data-plane-ready" || true+ echo+ echo "--- server: fsb create served (authoritative control-plane timing) ---"+ kubectl -n "$OSB_NS" logs deploy/opensandbox-server --tail=3000 2>/dev/null \+ | grep "fsb create served" | tail -30 || true+ echo+ echo "--- fastpath create totals (controller) ---"+ kubectl -n "$NS" logs deploy/fast-sandbox-controller --tail=3000 2>/dev/null \+ | grep "fastpath sandbox created" | tail -30 || true+ } > "$out" 2>&1 || true+ log "latency evidence: $out"+ # CI runners discard $WORK between runs, so the content must also land+ # in the job log to be analyzable.+ cat "$out"+}+opensandbox_verify() {# Cold create pulls the golden image through DART (slowest; generous# budget). Warm creates: the second may still pull on the other node via@@ -1588,6 +1665,7 @@ opensandbox_verify() {ids+=("$VERIFY_ID")donepass "end-to-end: opensandbox CLI -> server -> FastPath -> fastlet -> sandbox execd -> gateway route OK (6 sandboxes)"+ dump_latency_evidenceverify_lifecycle_ops "${ids[0]}"for id in "${ids[@]}"; doVERIFY_ID="$id"@@ -2344,7 +2422,7 @@ case "$ACTION" inrun_stage "OpenSandbox server + ingress gateway" opensandbox_uprun_stage "SandboxTemplate build (opensandbox CLI)" template_uprun_stage "SandboxPool $POOL_NAME (egress + P2P)" pool_up- run_stage "end-to-end verify (CLI create -> gateway -> execd /ping)" opensandbox_verify+ run_stage "end-to-end verify (API create -> gateway -> execd /ping)" opensandbox_verifyrun_stage "pause/resume verify (CLI -> FastPath checkpoint)" pause_resume_verifyrun_stage "vault lifecycle verify (CLI -> relay -> egress-held vault)" vault_lifecycle_verifyrun_stage "idle-pause / wake-on-access verify (OSEP-0024 profile)" idle_pause_verifydiff --git a/scripts/fast-sandbox-env/verify/sandbox.sh b/scripts/fast-sandbox-env/verify/sandbox.shindex d1828f961..57c750995 100755--- a/scripts/fast-sandbox-env/verify/sandbox.sh+++ b/scripts/fast-sandbox-env/verify/sandbox.sh@@ -13,11 +13,12 @@# See the License for the specific language governing permissions and# limitations under the License.-# verify/sandbox.sh — end-to-end sandbox verify (CLI create -> gateway -> execd /ping)+# verify/sandbox.sh — end-to-end sandbox verify (API create -> gateway -> execd /ping)## Coverage:#-# 1. Full create wire-up, 6 sandboxes: `osb sandbox create` -> lifecycle+# 1. Full create wire-up, 6 sandboxes: POST /sandboxes (raw API — the+# timed windows are curl-only so readings are not billed CLI startup)# server (fsb runtime) -> FastPath gRPC -> fastlet -> firecracker# microVM -> signed gateway route -> fastlet-proxy -> guest execd# /ping 200. Cold #1 pulls the golden image set through DART (600s@@ -61,43 +62,83 @@ verify_policy_updated() {[[ "$(printf '%s' "$out" | jq -r '.policy.egress[0].target // empty')" == "github.com" ]]}-# Full wire-up: CLI create (fsb) -> FastPath -> fastlet -> firecracker+# Full wire-up: raw-API create (fsb) -> FastPath -> fastlet -> firecracker# sandbox -> signed gateway route -> fastlet-proxy -> guest execd /ping -># delete. One sandbox per call; availability is measured from the client-# (poll /ping until 200, wall-clock budget — each route poll is a CLI call,-# ~0.5s startup), not from CR status.+# (poll /ping until 200, wall-clock budget — every poll is curl-only, so+# readings resolve ~10ms), not from CR status. The timed windows wrap HTTP+# round trips only: osb invocations would bill ~0.5s of interpreter startup+# to each reading, which is exactly the measurement pollution #2323 called+# out. CLI create coverage lives in the pause-resume/vault/snapshot verifies+# and the lifecycle ops below.verify_one_sandbox() { # <label> <ping-budget-ms>- local label="$1" budget="$2" created t0 t1 t2 route code deadline+ local label="$1" budget="$2" response t0 t1 t2 route code deadline payloadlocal policy_file="$GEN_DIR/verify-policy.json"t0="$(now_ms)"# Egress policy rides on every create (binding -> nft rules), so the# chain is exercised each time; /ping is inbound and unaffected.jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "example.com"}, {action: "allow", target: "*.opensandbox.ai"}]}' \> "$policy_file"- log "verify ($label): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID)"- created="$(osb_api sandbox create --template "$TEMPLATE_ID" --timeout 1h \- --metadata origin=fast-sandbox-env-verify \- --network-policy-file "$policy_file" \- --skip-health-check -o json 2>/dev/null)" \- || fail "osb sandbox create failed against $SERVER_URL: $(osb_api sandbox create \- --template "$TEMPLATE_ID" --timeout 1h --skip-health-check 2>&1 >/dev/null | head -c 400)"- VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"+ log "verify ($label): creating a sandbox via the raw API (templateId=$TEMPLATE_ID; the timed window is curl-only)"+ # The timed windows wrap HTTP round trips ONLY: an osb invocation here+ # would bill ~0.5s of Python startup to every reading. The CLI surface+ # stays covered by the lifecycle ops and the osb create smoke.+ local payload+ payload="$(jq -n --arg tpl "$TEMPLATE_ID" --slurpfile pol "$policy_file" \+ '{templateId: $tpl, timeout: 3600, metadata: {origin: "fast-sandbox-env-verify"}, networkPolicy: $pol[0]}')"+ response="$(server_api POST "/sandboxes" "$payload")" \+ || fail "POST /sandboxes failed against $SERVER_URL (payload: $(printf '%s' "$payload" | head -c 200))"+ VERIFY_ID="$(printf '%s' "$response" | jq -r '.id')"[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"t1="$(now_ms)"- # Availability = /ping 200 through the signed gateway route (port 44772).- deadline=$(( t1 + budget * 1000000 ))+ # Availability = /ping 200 through the signed gateway route (port 44772),+ # measured at ~1ms resolution:+ # phase 1 — route publication, curl-only busy poll (the curl spawn+ # itself paces ~5ms/attempt; /ping cannot 200 before the route exists,+ # so phase-1 cadence delays detection, never pollutes the reading);+ # phase 2 — /ping hammered over a bash /dev/tcp socket: no process+ # spawn per attempt, EPOCHREALTIME keeps the hot loop fork-free.+ local gw gw_host gw_port line deadline_ms+ gw="${GATEWAY_URL#*://}"; gw="${gw%%/*}"+ gw_host="${gw%%:*}"+ gw_port="${gw##*:}"+ [[ "$gw_port" == "$gw" ]] && gw_port=80+ deadline_ms=$(( t1 / 1000000 + budget ))while :; do- route="$(osb_api sandbox endpoint "$VERIFY_ID" --port 44772 -o json 2>/dev/null \+ route="$( { server_api GET "/sandboxes/$VERIFY_ID/endpoints/44772" 2>/dev/null || true; } \| jq -r '.headers["OpenSandbox-Ingress-To"] // empty')"- if [[ -n "$route" ]]; then- code="$(curl -sS -m 5 -o /dev/null -w '%{http_code}' \- -H "OpenSandbox-Ingress-To: $route" "$GATEWAY_URL/ping" 2>/dev/null || true)"- [[ "$code" == "200" ]] && break+ [[ -n "$route" ]] && break+ if (( $(now_ms) / 1000000 >= deadline_ms )); then+ fail "gateway route for $VERIFY_ID did not publish within ${budget}ms"fi- if (( $(now_ms) >= deadline )); then- fail "execd /ping did not return 200 within ${budget}ms (last code=${code:-none}, route=${route:-none})"+ done+ code=""+ while :; do+ line=""+ # NOTE: no 2>/dev/null here — exec-without-command applies+ # redirections to the CURRENT shell, so that would permanently+ # swallow the script's stderr (every later log/pass goes dark).+ # A refused connect prints one honest bash message instead.+ if exec 3<>"/dev/tcp/$gw_host/$gw_port"; then+ printf 'GET /ping HTTP/1.1\r\nHost: %s\r\nOpenSandbox-Ingress-To: %s\r\nConnection: close\r\n\r\n' \+ "$gw" "$route" >&3+ IFS= read -r -t 1 line <&3 || true+ exec 3>&- 3<&-+ else+ sleep 0.01+ fi+ if [[ "$line" == *" 200 "* ]]; then+ code=200+ break+ fi+ if [[ -n "${EPOCHREALTIME:-}" ]]; then+ now=$((( ${EPOCHREALTIME/./} ) / 1000))+ else+ now=$(( $(now_ms) / 1000000 ))+ fi+ if (( now >= deadline_ms )); then+ fail "execd /ping did not return 200 within ${budget}ms (last status: ${line:-none})"fi- sleep 0.2donet2="$(now_ms)"# jq filter in a variable: literal parens trip older bash's $( ) parser.@@ -171,6 +212,39 @@ verify_lifecycle_ops() { # <sandbox-id>pass "lifecycle: osb sandbox renew 2h ($expected_expires -> $renewed_expires)"}+# dump_latency_evidence collects the per-create timing instrumentation from+# every component on the create critical path into one greppable file:+# fastlet phase breakdown (fastlet create timing:), fastlet->egress action+# invocations (fastlet action invoke:), egress sidecar SET_BINDING / hook+# steps (timing fields on its subject lines), and fastpath create totals.+# Append-only evidence for create-latency analysis (issue #2323 style).+dump_latency_evidence() {+ local out="$LOGS_DIR/latency-evidence-$(date +%s).txt"+ {+ echo "=== latency evidence $(date -u +%FT%TZ) ==="+ echo+ echo "--- fastlet: create phase + action-invoke timing ---"+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet --tail=3000 2>/dev/null \+ | grep -E "fastlet create timing|fastlet action invoke" || true+ echo+ echo "--- egress sidecar: SET_BINDING / hook timing ---"+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet -c egress --tail=3000 2>/dev/null \+ | grep -E "deny-first enforced|SET_BINDING applied|data-plane-ready" || true+ echo+ echo "--- server: fsb create served (authoritative control-plane timing) ---"+ kubectl -n "$OSB_NS" logs deploy/opensandbox-server --tail=3000 2>/dev/null \+ | grep "fsb create served" | tail -30 || true+ echo+ echo "--- fastpath create totals (controller) ---"+ kubectl -n "$NS" logs deploy/fast-sandbox-controller --tail=3000 2>/dev/null \+ | grep "fastpath sandbox created" | tail -30 || true+ } > "$out" 2>&1 || true+ log "latency evidence: $out"+ # CI runners discard $WORK between runs, so the content must also land+ # in the job log to be analyzable.+ cat "$out"+}+opensandbox_verify() {# Cold create pulls the golden image through DART (slowest; generous# budget). Warm creates: the second may still pull on the other node via@@ -185,6 +259,7 @@ opensandbox_verify() {ids+=("$VERIFY_ID")donepass "end-to-end: opensandbox CLI -> server -> FastPath -> fastlet -> sandbox execd -> gateway route OK (6 sandboxes)"+ dump_latency_evidenceverify_lifecycle_ops "${ids[0]}"for id in "${ids[@]}"; doVERIFY_ID="$id"@@ -204,5 +279,5 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; thenresolve_osbrequire_template_idtrap 'on_error verify-sandbox' ERR- run_stage "end-to-end verify (CLI create -> gateway -> execd /ping)" opensandbox_verify+ run_stage "end-to-end verify (API create -> gateway -> execd /ping)" opensandbox_verifyfidiff --git a/server/opensandbox_server/services/fast_sandbox/service.py b/server/opensandbox_server/services/fast_sandbox/service.pyindex e10fd9b10..a09223c57 100644--- a/server/opensandbox_server/services/fast_sandbox/service.py+++ b/server/opensandbox_server/services/fast_sandbox/service.py@@ -26,6 +26,7 @@ from __future__ import annotationsimport asyncioimport jsonimport logging+import timefrom datetime import datetime, timezonefrom threading import Lockfrom typing import Optional@@ -432,12 +433,16 @@ class FastSandboxService(SandboxService, ExtensionService):namespace = create_request.namespacesandbox_id = create_request.request_id+ sync_started = time.perf_counter()+ rpc_ms = -1try:+ rpc_started = time.perf_counter()response = self._fastpath.create_sandbox(create_request,wait_timeout_millis=int(self._k8s.fastpath_wait_ready_seconds * 1000),)+ rpc_ms = int((time.perf_counter() - rpc_started) * 1000)except (FastPathInvalidArgument, FastPathConflict, FastPathNotFound) as exc:raise self._fastpath_http_error(exc) from excexcept FastPathError as exc:@@ -453,6 +458,12 @@ class FastSandboxService(SandboxService, ExtensionService):info = response.sandboxfinally:self._cr_reader.invalidate(namespace)+ logger.info(+ "fsb create served: sandbox=%s total=%dms fastpath_rpc=%dms",+ sandbox_id,+ int((time.perf_counter() - sync_started) * 1000),+ rpc_ms,+ )# Durable intent exists (accepted or recovery-confirmed): restart the# idle clock from this transition (OSEP-0024).