Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

perf(egress,verify): create-latency evidence pipeline + zero-fsync policy bootstrap (#2323) (#2327)

epha5feb5be3d502
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index b45cb82f8..71f0333ec 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -241,12 +241,15 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (
}
s.rehydrateVaultReload(subj, fence, seed, policyDegraded)
}
+ setBindingStarted := time.Now()
+ installStarted := time.Now()
state, err := s.reg.RegisterAndEnforce(subj, subject.SubjectKey{SourceIP: att.IP}, fence, func() error {
return s.OnRegistered(subj, att)
}, s.takeoverStaleSubject)
if err != nil {
return http.StatusInternalServerError, fmt.Errorf("deny-first install: %w", err)
}
+ denyFirstInstall := time.Since(installStarted)
s.recordBindingState(subj, att, fence)
if env.Binding.IsRemoval() {
// Binding removed from a still-live sandbox: drop the pending policy
@@ -270,11 +273,14 @@ func (s *fastSandboxPolicyServer) applySetBinding(env *actionhandler.Envelope) (
}
// Install the binding policy (or honor the file-backed record that
// supersedes it) through the policy commit protocol.
+ policyStarted := time.Now()
if err := s.applyBindingPolicyLocked(subj, fence, pol, state == subject.StateActive); err != nil {
log.Errorf("subject %s: binding policy commit failed: %v", subj, err)
return http.StatusInternalServerError, fmt.Errorf("binding policy commit: %w", err)
}
- log.Infof("subject %s: SET_BINDING applied (state=%s)", subj, state)
+ policyCommit := time.Since(policyStarted)
+ log.Infof("subject %s: SET_BINDING applied (state=%s) total=%dms denyFirstInstall=%dms policyCommit=%dms",
+ subj, state, time.Since(setBindingStarted).Milliseconds(), denyFirstInstall.Milliseconds(), policyCommit.Milliseconds())
return http.StatusOK, nil
}
@@ -331,11 +337,12 @@ func (s *fastSandboxPolicyServer) applyLifecycleHook(env *actionhandler.Envelope
// the subject must never activate without its current policy.
return http.StatusConflict, fmt.Errorf("data-plane-ready for subject %s with no pending policy", subj)
}
+ applyStarted := time.Now()
if err := s.applyPolicy(subj, pol); err != nil {
return http.StatusInternalServerError, fmt.Errorf("policy apply: %w", err)
}
s.clearPendingPolicy(subj)
- log.Infof("subject %s: data-plane-ready, policy active", subj)
+ log.Infof("subject %s: data-plane-ready, policy active applyPolicy=%dms", subj, time.Since(applyStarted).Milliseconds())
return http.StatusOK, nil
default:
// Unreachable: ParseEnvelope validated the Hook name.
diff --git a/components/egress/fastsandbox_policy_persist.go b/components/egress/fastsandbox_policy_persist.go
index ae4327399..0c2569a20 100644
--- a/components/egress/fastsandbox_policy_persist.go
+++ b/components/egress/fastsandbox_policy_persist.go
@@ -18,9 +18,17 @@
// source for an egress restart — at SET_BINDING replay the committed file
// supersedes the replayed binding input, which carries only the original
// bootstrap policy — and the bundle source for the POLICY_EXPORT /
-// POLICY_IMPORT snapshot transfer. Ordering is persist → apply → ack
-// (acked ⇒ durable and applied), with the same three-state classification,
-// snapshot restore, and recovery latch as the vault files.
+// POLICY_IMPORT snapshot transfer. Ordering is persist → apply → ack.
+//
+// Durability is tiered by replayability: the bootstrap candidate
+// (applyBindingPolicyLocked) publishes non-durably — rev 1 is byte-identical
+// to the binding input, which the fastlet re-issues on every egress instance
+// change, so a lost rename is reconstructed exactly by replay; relay writes
+// and the removal tombstone record state that outruns the binding input and
+// stay fully durable (acked ⇒ durable for them). For the bootstrap ack the
+// guarantee is "recorded and enforced (deny-first)", not "durable". The
+// three-state classification, snapshot restore, and recovery latch remain
+// the same as the vault files.
//
// TODO(encryption-at-rest): template-borne state blobs are stored as-is.
// When encryption lands it protects the store-side reader only — CLONE
@@ -515,10 +523,17 @@ func (s *fastSandboxPolicyServer) applyBindingPolicyLocked(subj subject.Subject,
if err != nil {
return fmt.Errorf("policy file encode: %w", err)
}
+ // The bootstrap seed publishes NON-durably: rev 1 is byte-identical to
+ // the binding input, which the fastlet re-issues on every egress
+ // instance change — a crash that loses the rename is reconstructed
+ // exactly by that replay (deny-first nft already enforces meanwhile).
+ // Everything that outruns the binding input stays durable: relay
+ // writes (SaveRaw) and the removal tombstone in
+ // clearPolicyRecordLocked.
return s.persistPolicyCandidateLocked(
subj,
func(fs vaultFileStore) (policy.FileMutationState, error) {
- return fs.SaveRaw(payload)
+ return fs.SaveRawNonDurable(payload)
},
apply,
)
diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.go
index 5a979e0cc..f62a6c049 100644
--- a/components/egress/fastsandbox_server.go
+++ b/components/egress/fastsandbox_server.go
@@ -453,15 +453,19 @@ func (s *fastSandboxPolicyServer) OnRegistered(subj subject.Subject, att actionh
// policy writes.
nftCtx, cancel := context.WithTimeout(s.ctx, policyApplyTimeout)
defer cancel()
+ denyFirstStarted := time.Now()
if err := s.nft.ApplyDenyFirst(nftCtx, subj, att); err != nil {
return err
}
+ dnsStarted := time.Now()
if err := s.installGatewayDNSRedirect(subj, att.Gateway); err != nil {
// Fail closed: sandbox DNS addressed to gateway:53 must reach the
// proxy; without the redirect the sandbox would fall back to a
// resolver the policy cannot see.
return err
}
+ dnsRedirect := time.Since(dnsStarted)
+ mitmStarted := time.Now()
if err := s.setMitmRedirect(subj, att, false); err != nil {
// Fail closed: a sandbox whose HTTP(S) is not intercepted must not
// register as usable (it could exfiltrate credentials-bearing
@@ -471,7 +475,8 @@ func (s *fastSandboxPolicyServer) OnRegistered(subj subject.Subject, att actionh
s.releaseGatewayDNSRedirect(subj)
return err
}
- log.Infof("subject %s deny-first enforced (nft + gateway redirect + mitm redirect)", subj)
+ log.Infof("subject %s deny-first enforced (nft + gateway redirect + mitm redirect) total=%dms denyFirst=%dms dnsRedirect=%dms mitm=%dms",
+ subj, time.Since(denyFirstStarted).Milliseconds(), dnsStarted.Sub(denyFirstStarted).Milliseconds(), dnsRedirect.Milliseconds(), time.Since(mitmStarted).Milliseconds())
return nil
}
diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.go
index 3036551a8..69959e3c3 100644
--- a/components/egress/fastsandbox_vault_persist.go
+++ b/components/egress/fastsandbox_vault_persist.go
@@ -53,6 +53,11 @@ type vaultRecoveryState struct {
type vaultFileStore interface {
Snapshot() (*policy.PolicyFileSnapshot, error)
SaveRaw(data []byte) (policy.FileMutationState, error)
+ // SaveRawNonDurable publishes by rename without durability barriers —
+ // only valid for content the caller can reconstruct by replay (the
+ // atomic store documents the contract). Everything that outruns its
+ // replay source must use SaveRaw.
+ SaveRawNonDurable(data []byte) (policy.FileMutationState, error)
Restore(*policy.PolicyFileSnapshot) (policy.FileMutationState, error)
Delete() (policy.FileMutationState, error)
}
diff --git a/components/egress/fastsandbox_vault_persist_test.go b/components/egress/fastsandbox_vault_persist_test.go
index a2938de57..cb4d5685f 100644
--- a/components/egress/fastsandbox_vault_persist_test.go
+++ b/components/egress/fastsandbox_vault_persist_test.go
@@ -104,6 +104,12 @@ func (f *fakeVaultFile) SaveRaw(data []byte) (policy.FileMutationState, error) {
return f.saveState, nil
}
+func (f *fakeVaultFile) SaveRawNonDurable(data []byte) (policy.FileMutationState, error) {
+ // Same observable effect as SaveRaw in the fake: the durability
+ // difference lives in the real atomic store's fsync behavior.
+ return f.SaveRaw(data)
+}
+
func (f *fakeVaultFile) Restore(s *policy.PolicyFileSnapshot) (policy.FileMutationState, error) {
f.restoreCalls++
f.restored = append(f.restored, s)
diff --git a/components/egress/pkg/policy/atomic_store_linux.go b/components/egress/pkg/policy/atomic_store_linux.go
index b6d24718c..e9f615935 100644
--- a/components/egress/pkg/policy/atomic_store_linux.go
+++ b/components/egress/pkg/policy/atomic_store_linux.go
@@ -162,6 +162,83 @@ func (s *AtomicPolicyFile) SaveRaw(data []byte) (FileMutationState, error) {
})
}
+// SaveRawNonDurable publishes exact bytes by rename without any durability
+// barrier: no temporary-file sync, no security/IMA finalize, no directory
+// sync. The three-state classification still describes the namespace effect
+// (rename done = FileCommitted; pre-rename failure = FileUnchanged; a rename
+// error = FileUnknown — the rename may have taken effect).
+//
+// Only valid for content the caller can reconstruct by replay: a crash may
+// roll the file back to its previous content (or lose it entirely), so every
+// publisher here must have an authority that re-derives the bytes — e.g. the
+// fastlet re-issuing SET_BINDING with the binding input. Content that
+// outruns its replay source must go through SaveRaw.
+func (s *AtomicPolicyFile) SaveRawNonDurable(data []byte) (FileMutationState, error) {
+ return s.withDirectory(func(dir *os.File) (FileMutationState, error) {
+ current, err := s.ops.snapshot(dir, s.name, s.path)
+ if err != nil {
+ return FileUnchanged, atomicFileError("snapshot", err)
+ }
+ return s.replaceNonDurable(dir, data, current)
+ })
+}
+
+// replaceNonDurable is replace minus the durability barriers: the temporary
+// inode keeps the same mode/ownership rules and the same deferred cleanup,
+// but the publish is a bare rename — the kernel may lose it (or keep the
+// previous content) across a power failure. No security/IMA finalize: the
+// check exists to guard the durable record; a replayable candidate tolerates
+// a label/digest difference until a durable SaveRaw re-publishes.
+func (s *AtomicPolicyFile) replaceNonDurable(dir *os.File, data []byte, metadata *PolicyFileSnapshot) (state FileMutationState, err error) {
+ temp, err := s.ops.create(dir)
+ if err != nil {
+ return FileUnchanged, atomicFileError("create", err)
+ }
+ tempName := temp.Name()
+ closed := false
+ defer func() {
+ if !closed {
+ if closeErr := temp.Close(); closeErr != nil {
+ err = errors.Join(err, atomicFileError("temporary-close", closeErr))
+ }
+ }
+ if tempName != "" {
+ if removeErr := s.ops.remove(dir, tempName); removeErr != nil && !errors.Is(removeErr, fs.ErrNotExist) {
+ err = errors.Join(err, atomicFileError("temporary-cleanup", removeErr))
+ }
+ }
+ }()
+ if n, writeErr := temp.Write(data); writeErr != nil || n != len(data) {
+ if writeErr == nil {
+ writeErr = io.ErrShortWrite
+ }
+ return FileUnchanged, atomicFileError("write", writeErr)
+ }
+ mode := fs.FileMode(0o600)
+ if metadata.exists {
+ // Chown can clear set-id bits, so apply mode after ownership.
+ if err := temp.Chown(metadata.uid, metadata.gid); err != nil {
+ return FileUnchanged, atomicFileError("ownership", err)
+ }
+ mode = metadata.mode
+ }
+ if err := temp.Chmod(mode); err != nil {
+ return FileUnchanged, atomicFileError("mode", err)
+ }
+ closed = true
+ if err := temp.Close(); err != nil {
+ return FileUnchanged, atomicFileError("file-close", err)
+ }
+ // From the first publication attempt onward an error is never classified as
+ // unchanged. In particular, remote filesystems can report a failed rename
+ // after the server has already committed it.
+ if err := s.ops.rename(dir, tempName, s.name); err != nil {
+ return FileUnknown, atomicFileError("rename", err)
+ }
+ tempName = ""
+ return FileCommitted, nil
+}
+
// Delete durably unlinks the authoritative path: an already-absent path is
// FileUnchanged with a nil error; a failed unlink or directory sync is
// FileUnknown (the unlink may still have taken effect).
diff --git a/components/egress/pkg/policy/atomic_store_other.go b/components/egress/pkg/policy/atomic_store_other.go
index b663f38cf..ee44f350f 100644
--- a/components/egress/pkg/policy/atomic_store_other.go
+++ b/components/egress/pkg/policy/atomic_store_other.go
@@ -35,6 +35,9 @@ func (*AtomicPolicyFile) Save(*NetworkPolicy) (FileMutationState, error) {
func (*AtomicPolicyFile) SaveRaw([]byte) (FileMutationState, error) {
return FileUnchanged, atomicPolicyPlatformError()
}
+func (*AtomicPolicyFile) SaveRawNonDurable([]byte) (FileMutationState, error) {
+ return FileUnchanged, atomicPolicyPlatformError()
+}
func (*AtomicPolicyFile) Restore(*PolicyFileSnapshot) (FileMutationState, error) {
return FileUnchanged, atomicPolicyPlatformError()
}
diff --git a/components/egress/pkg/policy/atomic_store_raw_nondurable_linux_test.go b/components/egress/pkg/policy/atomic_store_raw_nondurable_linux_test.go
new file mode 100644
index 000000000..21bf4785e
--- /dev/null
+++ b/components/egress/pkg/policy/atomic_store_raw_nondurable_linux_test.go
@@ -0,0 +1,79 @@
+//go:build linux
+
+// Copyright 2026 The OpenSandbox Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// TestAtomicPolicyFileSaveRawNonDurable exercises the replayable-candidate
+// surface: identical namespace effect and three-state classification as
+// SaveRaw (byte-exact publish, atomic replace, full temp cleanup), with the
+// durability barriers left out — which is not directly observable in-process,
+// so the assertions pin the observable contract the replay invariant relies on.
+func TestAtomicPolicyFileSaveRawNonDurable(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "s-1.json")
+ s, err := NewAtomicPolicyFile(path)
+ require.NoError(t, err)
+
+ first := []byte(`{"v":1,"incarnation":"rt-1","revision":1,"vault":{}}` + "\n")
+ state, err := s.SaveRawNonDurable(first)
+ require.NoError(t, err)
+ require.Equal(t, FileCommitted, state)
+ got, err := os.ReadFile(path)
+ require.NoError(t, err)
+ require.Equal(t, string(first), string(got))
+ require.Equal(t, 0o600, fileMode(t, path), "a new file is 0600")
+ assertNoAtomicTemps(t, dir)
+
+ // A second non-durable save replaces atomically like SaveRaw.
+ second := []byte(`{"v":1,"incarnation":"rt-1","revision":2,"vault":{}}` + "\n")
+ state, err = s.SaveRawNonDurable(second)
+ require.NoError(t, err)
+ require.Equal(t, FileCommitted, state)
+ got, err = os.ReadFile(path)
+ require.NoError(t, err)
+ require.Equal(t, string(second), string(got))
+ assertNoAtomicTemps(t, dir)
+
+ // Existing mode/ownership are preserved on replacement, like SaveRaw.
+ require.NoError(t, os.Chmod(path, 0o640))
+ third := []byte(`{"v":1,"incarnation":"rt-1","revision":3,"vault":{}}` + "\n")
+ state, err = s.SaveRawNonDurable(third)
+ require.NoError(t, err)
+ require.Equal(t, FileCommitted, state)
+ got, err = os.ReadFile(path)
+ require.NoError(t, err)
+ require.Equal(t, string(third), string(got))
+ require.Equal(t, 0o640, fileMode(t, path), "replacement copies the existing mode")
+ assertNoAtomicTemps(t, dir)
+
+ // The durable path still works after non-durable publishes.
+ fourth := []byte(`{"v":1,"incarnation":"rt-1","revision":4,"vault":{}}` + "\n")
+ state, err = s.SaveRaw(fourth)
+ require.NoError(t, err)
+ require.Equal(t, FileCommitted, state)
+ got, err = os.ReadFile(path)
+ require.NoError(t, err)
+ require.Equal(t, string(fourth), string(got))
+ assertNoAtomicTemps(t, dir)
+}
diff --git a/scripts/fast-sandbox-env/integration-env.sh b/scripts/fast-sandbox-env/integration-env.sh
index 199ee3b96..c0ad582f3 100755
--- a/scripts/fast-sandbox-env/integration-env.sh
+++ b/scripts/fast-sandbox-env/integration-env.sh
@@ -291,6 +291,10 @@ rc() {
on_error() {
local rc=$? task="$1"
+ # Record WHERE it died before anything else: the dump alone cannot
+ # distinguish a silent command failure from an environment problem
+ # (e.g. the RustFS stage once died in ~1s with zero output).
+ log "up failed: task=$task rc=$rc command=${BASH_COMMAND:-unknown} at ${BASH_SOURCE[1]:-?}:${BASH_LINENO[0]:-?}" || true
# Subshells inherit ERR: propagate the error instead of tearing down
# main-process resources; ignore SIGPIPE (cancelled runners).
trap - ERR
@@ -503,7 +507,7 @@ preflight() {
if (( avail_kb < min_free_kb )); then
die "$target has $((avail_kb / 1024 / 1024))G free; at least 20G is required (built images, XFS StateRoot, RustFS artifacts). Free space (docker system prune / old kind clusters) or point WORK / RUSTFS_DATA / XFS_LOOP_FILE at a bigger volume"
fi
- log "disk headroom: $target has $((avail_kb / 1024 / 1024 / 1024))G free"
+ log "disk headroom: $target has $((avail_kb / 1024 / 1024))G free"
done
# Fail fast on busy host ports instead of a cryptic docker/kind bind error.
local port
@@ -1464,43 +1468,83 @@ verify_policy_updated() {
[[ "$(printf '%s' "$out" | jq -r '.policy.egress[0].target // empty')" == "github.com" ]]
}
-# Full wire-up: CLI create (fsb) -> FastPath -> fastlet -> firecracker
+# Full wire-up: raw-API create (fsb) -> FastPath -> fastlet -> firecracker
# sandbox -> signed gateway route -> fastlet-proxy -> guest execd /ping ->
# delete. One sandbox per call; availability is measured from the client
-# (poll /ping until 200, wall-clock budget — each route poll is a CLI call,
-# ~0.5s startup), not from CR status.
+# (poll /ping until 200, wall-clock budget — every poll is curl-only, so
+# readings resolve ~10ms), not from CR status. The timed windows wrap HTTP
+# round trips only: osb invocations would bill ~0.5s of interpreter startup
+# to each reading, which is exactly the measurement pollution #2323 called
+# out. CLI create coverage lives in the pause-resume/vault/snapshot verifies
+# and the lifecycle ops below.
verify_one_sandbox() { # <label> <ping-budget-ms>
- local label="$1" budget="$2" created t0 t1 t2 route code deadline
+ local label="$1" budget="$2" response t0 t1 t2 route code deadline payload
local policy_file="$GEN_DIR/verify-policy.json"
t0="$(now_ms)"
# Egress policy rides on every create (binding -> nft rules), so the
# chain is exercised each time; /ping is inbound and unaffected.
jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "example.com"}, {action: "allow", target: "*.opensandbox.ai"}]}' \
> "$policy_file"
- log "verify ($label): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID)"
- created="$(osb_api sandbox create --template "$TEMPLATE_ID" --timeout 1h \
- --metadata origin=fast-sandbox-env-verify \
- --network-policy-file "$policy_file" \
- --skip-health-check -o json 2>/dev/null)" \
- || fail "osb sandbox create failed against $SERVER_URL: $(osb_api sandbox create \
- --template "$TEMPLATE_ID" --timeout 1h --skip-health-check 2>&1 >/dev/null | head -c 400)"
- VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"
+ log "verify ($label): creating a sandbox via the raw API (templateId=$TEMPLATE_ID; the timed window is curl-only)"
+ # The timed windows wrap HTTP round trips ONLY: an osb invocation here
+ # would bill ~0.5s of Python startup to every reading. The CLI surface
+ # stays covered by the lifecycle ops and the osb create smoke.
+ local payload
+ payload="$(jq -n --arg tpl "$TEMPLATE_ID" --slurpfile pol "$policy_file" \
+ '{templateId: $tpl, timeout: 3600, metadata: {origin: "fast-sandbox-env-verify"}, networkPolicy: $pol[0]}')"
+ response="$(server_api POST "/sandboxes" "$payload")" \
+ || fail "POST /sandboxes failed against $SERVER_URL (payload: $(printf '%s' "$payload" | head -c 200))"
+ VERIFY_ID="$(printf '%s' "$response" | jq -r '.id')"
[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"
t1="$(now_ms)"
- # Availability = /ping 200 through the signed gateway route (port 44772).
- deadline=$(( t1 + budget * 1000000 ))
+ # Availability = /ping 200 through the signed gateway route (port 44772),
+ # measured at ~1ms resolution:
+ # phase 1 — route publication, curl-only busy poll (the curl spawn
+ # itself paces ~5ms/attempt; /ping cannot 200 before the route exists,
+ # so phase-1 cadence delays detection, never pollutes the reading);
+ # phase 2 — /ping hammered over a bash /dev/tcp socket: no process
+ # spawn per attempt, EPOCHREALTIME keeps the hot loop fork-free.
+ local gw gw_host gw_port line deadline_ms
+ gw="${GATEWAY_URL#*://}"; gw="${gw%%/*}"
+ gw_host="${gw%%:*}"
+ gw_port="${gw##*:}"
+ [[ "$gw_port" == "$gw" ]] && gw_port=80
+ deadline_ms=$(( t1 / 1000000 + budget ))
while :; do
- route="$(osb_api sandbox endpoint "$VERIFY_ID" --port 44772 -o json 2>/dev/null \
+ route="$( { server_api GET "/sandboxes/$VERIFY_ID/endpoints/44772" 2>/dev/null || true; } \
| jq -r '.headers["OpenSandbox-Ingress-To"] // empty')"
- if [[ -n "$route" ]]; then
- code="$(curl -sS -m 5 -o /dev/null -w '%{http_code}' \
- -H "OpenSandbox-Ingress-To: $route" "$GATEWAY_URL/ping" 2>/dev/null || true)"
- [[ "$code" == "200" ]] && break
+ [[ -n "$route" ]] && break
+ if (( $(now_ms) / 1000000 >= deadline_ms )); then
+ fail "gateway route for $VERIFY_ID did not publish within ${budget}ms"
+ fi
+ done
+ code=""
+ while :; do
+ line=""
+ # NOTE: no 2>/dev/null here — exec-without-command applies
+ # redirections to the CURRENT shell, so that would permanently
+ # swallow the script's stderr (every later log/pass goes dark).
+ # A refused connect prints one honest bash message instead.
+ if exec 3<>"/dev/tcp/$gw_host/$gw_port"; then
+ printf 'GET /ping HTTP/1.1\r\nHost: %s\r\nOpenSandbox-Ingress-To: %s\r\nConnection: close\r\n\r\n' \
+ "$gw" "$route" >&3
+ IFS= read -r -t 1 line <&3 || true
+ exec 3>&- 3<&-
+ else
+ sleep 0.01
+ fi
+ if [[ "$line" == *" 200 "* ]]; then
+ code=200
+ break
fi
- if (( $(now_ms) >= deadline )); then
- fail "execd /ping did not return 200 within ${budget}ms (last code=${code:-none}, route=${route:-none})"
+ if [[ -n "${EPOCHREALTIME:-}" ]]; then
+ now=$((( ${EPOCHREALTIME/./} ) / 1000))
+ else
+ now=$(( $(now_ms) / 1000000 ))
+ fi
+ if (( now >= deadline_ms )); then
+ fail "execd /ping did not return 200 within ${budget}ms (last status: ${line:-none})"
fi
- sleep 0.2
done
t2="$(now_ms)"
# jq filter in a variable: literal parens trip older bash's $( ) parser.
@@ -1574,6 +1618,39 @@ verify_lifecycle_ops() { # <sandbox-id>
pass "lifecycle: osb sandbox renew 2h ($expected_expires -> $renewed_expires)"
}
+# dump_latency_evidence collects the per-create timing instrumentation from
+# every component on the create critical path into one greppable file:
+# fastlet phase breakdown (fastlet create timing:), fastlet->egress action
+# invocations (fastlet action invoke:), egress sidecar SET_BINDING / hook
+# steps (timing fields on its subject lines), and fastpath create totals.
+# Append-only evidence for create-latency analysis (issue #2323 style).
+dump_latency_evidence() {
+ local out="$LOGS_DIR/latency-evidence-$(date +%s).txt"
+ {
+ echo "=== latency evidence $(date -u +%FT%TZ) ==="
+ echo
+ echo "--- fastlet: create phase + action-invoke timing ---"
+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet --tail=3000 2>/dev/null \
+ | grep -E "fastlet create timing|fastlet action invoke" || true
+ echo
+ echo "--- egress sidecar: SET_BINDING / hook timing ---"
+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet -c egress --tail=3000 2>/dev/null \
+ | grep -E "deny-first enforced|SET_BINDING applied|data-plane-ready" || true
+ echo
+ echo "--- server: fsb create served (authoritative control-plane timing) ---"
+ kubectl -n "$OSB_NS" logs deploy/opensandbox-server --tail=3000 2>/dev/null \
+ | grep "fsb create served" | tail -30 || true
+ echo
+ echo "--- fastpath create totals (controller) ---"
+ kubectl -n "$NS" logs deploy/fast-sandbox-controller --tail=3000 2>/dev/null \
+ | grep "fastpath sandbox created" | tail -30 || true
+ } > "$out" 2>&1 || true
+ log "latency evidence: $out"
+ # CI runners discard $WORK between runs, so the content must also land
+ # in the job log to be analyzable.
+ cat "$out"
+}
+
opensandbox_verify() {
# Cold create pulls the golden image through DART (slowest; generous
# budget). Warm creates: the second may still pull on the other node via
@@ -1588,6 +1665,7 @@ opensandbox_verify() {
ids+=("$VERIFY_ID")
done
pass "end-to-end: opensandbox CLI -> server -> FastPath -> fastlet -> sandbox execd -> gateway route OK (6 sandboxes)"
+ dump_latency_evidence
verify_lifecycle_ops "${ids[0]}"
for id in "${ids[@]}"; do
VERIFY_ID="$id"
@@ -2344,7 +2422,7 @@ case "$ACTION" in
run_stage "OpenSandbox server + ingress gateway" opensandbox_up
run_stage "SandboxTemplate build (opensandbox CLI)" template_up
run_stage "SandboxPool $POOL_NAME (egress + P2P)" pool_up
- run_stage "end-to-end verify (CLI create -> gateway -> execd /ping)" opensandbox_verify
+ run_stage "end-to-end verify (API create -> gateway -> execd /ping)" opensandbox_verify
run_stage "pause/resume verify (CLI -> FastPath checkpoint)" pause_resume_verify
run_stage "vault lifecycle verify (CLI -> relay -> egress-held vault)" vault_lifecycle_verify
run_stage "idle-pause / wake-on-access verify (OSEP-0024 profile)" idle_pause_verify
diff --git a/scripts/fast-sandbox-env/verify/sandbox.sh b/scripts/fast-sandbox-env/verify/sandbox.sh
index d1828f961..57c750995 100755
--- a/scripts/fast-sandbox-env/verify/sandbox.sh
+++ b/scripts/fast-sandbox-env/verify/sandbox.sh
@@ -13,11 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# verify/sandbox.sh — end-to-end sandbox verify (CLI create -> gateway -> execd /ping)
+# verify/sandbox.sh — end-to-end sandbox verify (API create -> gateway -> execd /ping)
#
# Coverage:
#
-# 1. Full create wire-up, 6 sandboxes: `osb sandbox create` -> lifecycle
+# 1. Full create wire-up, 6 sandboxes: POST /sandboxes (raw API — the
+# timed windows are curl-only so readings are not billed CLI startup)
# server (fsb runtime) -> FastPath gRPC -> fastlet -> firecracker
# microVM -> signed gateway route -> fastlet-proxy -> guest execd
# /ping 200. Cold #1 pulls the golden image set through DART (600s
@@ -61,43 +62,83 @@ verify_policy_updated() {
[[ "$(printf '%s' "$out" | jq -r '.policy.egress[0].target // empty')" == "github.com" ]]
}
-# Full wire-up: CLI create (fsb) -> FastPath -> fastlet -> firecracker
+# Full wire-up: raw-API create (fsb) -> FastPath -> fastlet -> firecracker
# sandbox -> signed gateway route -> fastlet-proxy -> guest execd /ping ->
# delete. One sandbox per call; availability is measured from the client
-# (poll /ping until 200, wall-clock budget — each route poll is a CLI call,
-# ~0.5s startup), not from CR status.
+# (poll /ping until 200, wall-clock budget — every poll is curl-only, so
+# readings resolve ~10ms), not from CR status. The timed windows wrap HTTP
+# round trips only: osb invocations would bill ~0.5s of interpreter startup
+# to each reading, which is exactly the measurement pollution #2323 called
+# out. CLI create coverage lives in the pause-resume/vault/snapshot verifies
+# and the lifecycle ops below.
verify_one_sandbox() { # <label> <ping-budget-ms>
- local label="$1" budget="$2" created t0 t1 t2 route code deadline
+ local label="$1" budget="$2" response t0 t1 t2 route code deadline payload
local policy_file="$GEN_DIR/verify-policy.json"
t0="$(now_ms)"
# Egress policy rides on every create (binding -> nft rules), so the
# chain is exercised each time; /ping is inbound and unaffected.
jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "example.com"}, {action: "allow", target: "*.opensandbox.ai"}]}' \
> "$policy_file"
- log "verify ($label): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID)"
- created="$(osb_api sandbox create --template "$TEMPLATE_ID" --timeout 1h \
- --metadata origin=fast-sandbox-env-verify \
- --network-policy-file "$policy_file" \
- --skip-health-check -o json 2>/dev/null)" \
- || fail "osb sandbox create failed against $SERVER_URL: $(osb_api sandbox create \
- --template "$TEMPLATE_ID" --timeout 1h --skip-health-check 2>&1 >/dev/null | head -c 400)"
- VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"
+ log "verify ($label): creating a sandbox via the raw API (templateId=$TEMPLATE_ID; the timed window is curl-only)"
+ # The timed windows wrap HTTP round trips ONLY: an osb invocation here
+ # would bill ~0.5s of Python startup to every reading. The CLI surface
+ # stays covered by the lifecycle ops and the osb create smoke.
+ local payload
+ payload="$(jq -n --arg tpl "$TEMPLATE_ID" --slurpfile pol "$policy_file" \
+ '{templateId: $tpl, timeout: 3600, metadata: {origin: "fast-sandbox-env-verify"}, networkPolicy: $pol[0]}')"
+ response="$(server_api POST "/sandboxes" "$payload")" \
+ || fail "POST /sandboxes failed against $SERVER_URL (payload: $(printf '%s' "$payload" | head -c 200))"
+ VERIFY_ID="$(printf '%s' "$response" | jq -r '.id')"
[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"
t1="$(now_ms)"
- # Availability = /ping 200 through the signed gateway route (port 44772).
- deadline=$(( t1 + budget * 1000000 ))
+ # Availability = /ping 200 through the signed gateway route (port 44772),
+ # measured at ~1ms resolution:
+ # phase 1 — route publication, curl-only busy poll (the curl spawn
+ # itself paces ~5ms/attempt; /ping cannot 200 before the route exists,
+ # so phase-1 cadence delays detection, never pollutes the reading);
+ # phase 2 — /ping hammered over a bash /dev/tcp socket: no process
+ # spawn per attempt, EPOCHREALTIME keeps the hot loop fork-free.
+ local gw gw_host gw_port line deadline_ms
+ gw="${GATEWAY_URL#*://}"; gw="${gw%%/*}"
+ gw_host="${gw%%:*}"
+ gw_port="${gw##*:}"
+ [[ "$gw_port" == "$gw" ]] && gw_port=80
+ deadline_ms=$(( t1 / 1000000 + budget ))
while :; do
- route="$(osb_api sandbox endpoint "$VERIFY_ID" --port 44772 -o json 2>/dev/null \
+ route="$( { server_api GET "/sandboxes/$VERIFY_ID/endpoints/44772" 2>/dev/null || true; } \
| jq -r '.headers["OpenSandbox-Ingress-To"] // empty')"
- if [[ -n "$route" ]]; then
- code="$(curl -sS -m 5 -o /dev/null -w '%{http_code}' \
- -H "OpenSandbox-Ingress-To: $route" "$GATEWAY_URL/ping" 2>/dev/null || true)"
- [[ "$code" == "200" ]] && break
+ [[ -n "$route" ]] && break
+ if (( $(now_ms) / 1000000 >= deadline_ms )); then
+ fail "gateway route for $VERIFY_ID did not publish within ${budget}ms"
fi
- if (( $(now_ms) >= deadline )); then
- fail "execd /ping did not return 200 within ${budget}ms (last code=${code:-none}, route=${route:-none})"
+ done
+ code=""
+ while :; do
+ line=""
+ # NOTE: no 2>/dev/null here — exec-without-command applies
+ # redirections to the CURRENT shell, so that would permanently
+ # swallow the script's stderr (every later log/pass goes dark).
+ # A refused connect prints one honest bash message instead.
+ if exec 3<>"/dev/tcp/$gw_host/$gw_port"; then
+ printf 'GET /ping HTTP/1.1\r\nHost: %s\r\nOpenSandbox-Ingress-To: %s\r\nConnection: close\r\n\r\n' \
+ "$gw" "$route" >&3
+ IFS= read -r -t 1 line <&3 || true
+ exec 3>&- 3<&-
+ else
+ sleep 0.01
+ fi
+ if [[ "$line" == *" 200 "* ]]; then
+ code=200
+ break
+ fi
+ if [[ -n "${EPOCHREALTIME:-}" ]]; then
+ now=$((( ${EPOCHREALTIME/./} ) / 1000))
+ else
+ now=$(( $(now_ms) / 1000000 ))
+ fi
+ if (( now >= deadline_ms )); then
+ fail "execd /ping did not return 200 within ${budget}ms (last status: ${line:-none})"
fi
- sleep 0.2
done
t2="$(now_ms)"
# jq filter in a variable: literal parens trip older bash's $( ) parser.
@@ -171,6 +212,39 @@ verify_lifecycle_ops() { # <sandbox-id>
pass "lifecycle: osb sandbox renew 2h ($expected_expires -> $renewed_expires)"
}
+# dump_latency_evidence collects the per-create timing instrumentation from
+# every component on the create critical path into one greppable file:
+# fastlet phase breakdown (fastlet create timing:), fastlet->egress action
+# invocations (fastlet action invoke:), egress sidecar SET_BINDING / hook
+# steps (timing fields on its subject lines), and fastpath create totals.
+# Append-only evidence for create-latency analysis (issue #2323 style).
+dump_latency_evidence() {
+ local out="$LOGS_DIR/latency-evidence-$(date +%s).txt"
+ {
+ echo "=== latency evidence $(date -u +%FT%TZ) ==="
+ echo
+ echo "--- fastlet: create phase + action-invoke timing ---"
+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet --tail=3000 2>/dev/null \
+ | grep -E "fastlet create timing|fastlet action invoke" || true
+ echo
+ echo "--- egress sidecar: SET_BINDING / hook timing ---"
+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet -c egress --tail=3000 2>/dev/null \
+ | grep -E "deny-first enforced|SET_BINDING applied|data-plane-ready" || true
+ echo
+ echo "--- server: fsb create served (authoritative control-plane timing) ---"
+ kubectl -n "$OSB_NS" logs deploy/opensandbox-server --tail=3000 2>/dev/null \
+ | grep "fsb create served" | tail -30 || true
+ echo
+ echo "--- fastpath create totals (controller) ---"
+ kubectl -n "$NS" logs deploy/fast-sandbox-controller --tail=3000 2>/dev/null \
+ | grep "fastpath sandbox created" | tail -30 || true
+ } > "$out" 2>&1 || true
+ log "latency evidence: $out"
+ # CI runners discard $WORK between runs, so the content must also land
+ # in the job log to be analyzable.
+ cat "$out"
+}
+
opensandbox_verify() {
# Cold create pulls the golden image through DART (slowest; generous
# budget). Warm creates: the second may still pull on the other node via
@@ -185,6 +259,7 @@ opensandbox_verify() {
ids+=("$VERIFY_ID")
done
pass "end-to-end: opensandbox CLI -> server -> FastPath -> fastlet -> sandbox execd -> gateway route OK (6 sandboxes)"
+ dump_latency_evidence
verify_lifecycle_ops "${ids[0]}"
for id in "${ids[@]}"; do
VERIFY_ID="$id"
@@ -204,5 +279,5 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
resolve_osb
require_template_id
trap 'on_error verify-sandbox' ERR
- run_stage "end-to-end verify (CLI create -> gateway -> execd /ping)" opensandbox_verify
+ run_stage "end-to-end verify (API create -> gateway -> execd /ping)" opensandbox_verify
fi
diff --git a/server/opensandbox_server/services/fast_sandbox/service.py b/server/opensandbox_server/services/fast_sandbox/service.py
index e10fd9b10..a09223c57 100644
--- a/server/opensandbox_server/services/fast_sandbox/service.py
+++ b/server/opensandbox_server/services/fast_sandbox/service.py
@@ -26,6 +26,7 @@ from __future__ import annotations
import asyncio
import json
import logging
+import time
from datetime import datetime, timezone
from threading import Lock
from typing import Optional
@@ -432,12 +433,16 @@ class FastSandboxService(SandboxService, ExtensionService):
namespace = create_request.namespace
sandbox_id = create_request.request_id
+ sync_started = time.perf_counter()
+ rpc_ms = -1
try:
+ rpc_started = time.perf_counter()
response = self._fastpath.create_sandbox(
create_request,
wait_timeout_millis=int(self._k8s.fastpath_wait_ready_seconds * 1000),
)
+ rpc_ms = int((time.perf_counter() - rpc_started) * 1000)
except (FastPathInvalidArgument, FastPathConflict, FastPathNotFound) as exc:
raise self._fastpath_http_error(exc) from exc
except FastPathError as exc:
@@ -453,6 +458,12 @@ class FastSandboxService(SandboxService, ExtensionService):
info = response.sandbox
finally:
self._cr_reader.invalidate(namespace)
+ logger.info(
+ "fsb create served: sandbox=%s total=%dms fastpath_rpc=%dms",
+ sandbox_id,
+ int((time.perf_counter() - sync_started) * 1000),
+ rpc_ms,
+ )
# Durable intent exists (accepted or recovery-confirmed): restart the
# idle clock from this transition (OSEP-0024).