Secure, Fast, and Extensible Sandbox runtime for AI agents.
test(fsb-env): pause/resume verify carries the recorded policy end to end
高然09013faf7f2d
diff --git a/scripts/fast-sandbox-env/verify/pause-resume.sh b/scripts/fast-sandbox-env/verify/pause-resume.shindex 30a70a2ac..bc41d33d1 100755--- a/scripts/fast-sandbox-env/verify/pause-resume.sh+++ b/scripts/fast-sandbox-env/verify/pause-resume.sh@@ -13,41 +13,114 @@# See the License for the specific language governing permissions and# limitations under the License.-# verify/pause-resume.sh — pause/resume round trip (CLI -> FastPath checkpoint)+# verify/pause-resume.sh — pause/resume round trip with recorded-policy carry+# (CLI -> FastPath checkpoint -> artifact store)+#+# Test shape: the sandbox is created WITH an egress policy (default deny;+# allow example.com + *.opensandbox.ai), so every pause checkpoint must+# carry the subject's recorded policy file — policy.json rides the bundle+# as an opaque artifact, uploaded next to the checkpoint manifest and+# re-delivered to egress (POLICY_IMPORT) before the data plane is ready on+# resume. The checkpoint bundle is addressed through the Sandbox CR+# (status.runtime.checkpoint.manifestRef) and read from the artifact store+# with the rc CLI; the recorded policy.json is plain JSON (revision ++# embedded policy body), so its rules are asserted directly.## Coverage:#-# 1. Baseline: create a sandbox via `osb sandbox create` (template,-# skip-health-check), poll it to Running, and confirm execd /ping-# returns 200 through a freshly resolved signed gateway route.-# 2. Pause semantics (durable-first): POST pause, then poll GET until-# state=Paused. Paused means the checkpoint is fully durable in the-# artifact store AND the fastlet capacity is released — not merely-# that the call was accepted. The pause -> Paused latency is logged.-# 3. Route release (negative assertion): after Paused, the OLD signed-# route must no longer serve /ping — the gateway must reject a-# route whose sandbox is not running. A still-serving paused-# sandbox fails the verify.-# 4. Resume: POST resume, poll GET back to Running (checkpoint-# restored from the artifact store), then /ping must return 200-# through a FRESH route — resume advances the route generation, so-# re-resolving the endpoint is mandatory and the old route staying-# valid would indicate a generation bug.-# 5. Cleanup: kill the sandbox and wait until GET 404s.+# 1. Baseline: create via `osb sandbox create` carrying the policy file,+# poll to Running, confirm execd /ping 200 through a signed gateway+# route, and confirm the policy is ENFORCING pre-pause+# (networkPolicy -> egress action binding -> nft convergence) — the+# recorded policy file must exist before the checkpoint can carry it.+# 2. Pause semantics (durable-first): POST pause, poll GET until+# state=Paused (checkpoint durable in the artifact store + fastlet+# capacity released); pause -> Paused latency logged.+# 3. ARTIFACT (first pause): the egress logs "policy exported to+# snapshot bundle" for the subject, and the published bundle under+# the checkpoint manifestRef contains policy.json whose recorded+# policy still allows example.com + *.opensandbox.ai (revision >= 1).+# 4. Route release (negative assertion): after Paused, the OLD signed+# route must no longer serve /ping — a still-serving paused sandbox+# fails the verify.+# 5. Resume: POST resume, poll back to Running (checkpoint restored),+# /ping 200 through a FRESH route (resume advances the route+# generation — the old route staying valid would be a bug), the+# egress logs "policy imported from snapshot bundle" for the+# subject, and the policy is STILL ENFORCING with the original rules.+# 6. Policy update: PUT /networkpolicy replaces the rule set with a+# single allow github.com rule; the policy converges to enforcing+# with the new target.+# 7. ARTIFACT (second pause): pause again; the NEW checkpoint bundle's+# policy.json must carry the UPDATED policy — github.com present,+# example.com replaced (absent), and a revision strictly greater+# than the first bundle's.+# 8. Cleanup: kill the sandbox and wait until GET 404s.## Runs as part of `integration.sh up` (the entry chains every verify# script) or standalone against a live stack from `env.sh up`.source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib.sh"+# --- recorded-policy probes -----------------------------------------------------+# The checkpoint bundle is located through the Sandbox CR (status.runtime.+# checkpoint.manifestRef) and read from the artifact store via the rc CLI;+# policy.json is the egress policy-file envelope: a plain-JSON document+# carrying the revision and the verbatim policy body.++checkpoint_policy_bundle() { # -> policy.json envelope of the CURRENT pause checkpoint+ local ref key+ ref="$(kubectl_get "sandbox/$VERIFY_ID" '{.status.runtime.checkpoint.manifestRef}' 2>/dev/null)" || return 1+ [[ "$ref" == "s3://$RUSTFS_BUCKET/"* ]] || return 1+ key="${ref#s3://$RUSTFS_BUCKET/}"+ key="${key%/*}/policy.json"+ rc cat "chain/$RUSTFS_BUCKET/$key" 2>/dev/null+}++pause_bundle_allows() { # <target> — the bundle's recorded policy allows <target>+ checkpoint_policy_bundle | jq -e --arg t "$1" '[.policy.egress[]?.target] | index($t) != null' >/dev/null+}++pause_bundle_lacks() { # <target> — the bundle's recorded policy no longer has <target>+ checkpoint_policy_bundle | jq -e --arg t "$1" '[.policy.egress[]?.target] | index($t) == null' >/dev/null+}++pause_bundle_revision() { # -> the bundle's recorded revision (0 when unreadable)+ local bundle+ bundle="$(checkpoint_policy_bundle 2>/dev/null)" || { printf '0'; return 0; }+ printf '%s' "$bundle" | jq '.revision // 0' 2>/dev/null || printf '0'+}++pause_policy_enforced() { # <target> — GET /networkpolicy enforcing with <target>+ local out+ out="$(server_api GET "/sandboxes/$VERIFY_ID/networkpolicy" 2>/dev/null)" || return 1+ [[ "$(printf '%s' "$out" | jq -r '.mode // empty')" == "enforcing" ]] || return 1+ printf '%s' "$out" | jq -e --arg t "$1" '[.policy.egress[]?.target] | index($t) != null' >/dev/null+}++pause_egress_log_contains() { # <fixed-substring> — marker in the pool egress logs+ kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet -c egress --tail=800 2>/dev/null | grep -qF "$1"+}++# --- the verify -------------------------------------------------------------------+pause_resume_verify() {- # Accepted + poll: Paused == checkpoint durable + capacity released;- # resume advances the route generation, so /ping needs a fresh route.- local created t0 t1 t2 t3- log "verify (pause): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID)"+ # Test shape: create WITH an egress policy -> pause -> the checkpoint+ # bundle in the artifact store must carry the recorded policy file ->+ # resume -> the sandbox serves again AND the policy is still enforced ->+ # PUT a new rule set -> pause again -> the NEW checkpoint bundle must+ # carry the UPDATED policy (new rule present, replaced rule gone).+ local created t0 t1 t2 t3 rev1 rev2 sandbox_uid policy_file+ policy_file="$GEN_DIR/pause-policy.json"+ jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "example.com"}, {action: "allow", target: "*.opensandbox.ai"}]}' \+ > "$policy_file"++ log "verify (pause): creating a sandbox via the opensandbox CLI (templateId=$TEMPLATE_ID, with an egress policy)"t0="$(now_ms)"created="$(osb_api sandbox create --template "$TEMPLATE_ID" --timeout 1h \- --metadata origin=fast-sandbox-env-pause --skip-health-check -o json 2>/dev/null)" \+ --metadata origin=fast-sandbox-env-pause \+ --network-policy-file "$policy_file" \+ --skip-health-check -o json 2>/dev/null)" \|| fail "osb sandbox create failed against $SERVER_URL"VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"@@ -55,7 +128,13 @@ pause_resume_verify() {t1="$(now_ms)"log "verify (pause): $VERIFY_ID create->Running $(( (t1 - t0) / 1000000 ))ms"+ sandbox_uid="$(kubectl_get "sandbox/$VERIFY_ID" '{.metadata.uid}' 2>/dev/null)"+ [[ -n "$sandbox_uid" ]] || fail "sandbox CR carried no uid"+wait_for "pre-pause execd /ping 200 through the gateway" 100 execd_ping_ok+ # The recorded policy file must exist before the checkpoint can carry it.+ wait_for "egress policy enforcing pre-pause (networkPolicy -> binding -> nft)" 120 \+ pause_policy_enforced example.comlog "verify (pause): osb sandbox pause $VERIFY_ID"osb_api sandbox pause "$VERIFY_ID" >/dev/null 2>&1 \@@ -70,6 +149,16 @@ pause_resume_verify() {fipass "paused: execd /ping no longer served (runtime released, signed route rejected at the gateway)"+ # Artifact check (first pause): the published checkpoint set must carry+ # the recorded policy file with the pre-pause rules.+ wait_for "egress exported the recorded policy into the checkpoint bundle" 120 \+ pause_egress_log_contains "subject s-$sandbox_uid: policy exported to snapshot bundle"+ wait_for "checkpoint bundle policy.json allows example.com" 60 pause_bundle_allows example.com+ wait_for "checkpoint bundle policy.json allows *.opensandbox.ai" 60 pause_bundle_allows '*.opensandbox.ai'+ rev1="$(pause_bundle_revision)"+ [[ "${rev1:-0}" -ge 1 ]] || fail "checkpoint policy.json carries no positive revision (got ${rev1:-0})"+ pass "checkpoint artifact carries the recorded policy (policy.json, revision $rev1, pre-pause rules intact)"+log "verify (pause): osb sandbox resume $VERIFY_ID"t2="$(now_ms)"osb_api sandbox resume "$VERIFY_ID" --skip-health-check >/dev/null 2>&1 \@@ -78,8 +167,38 @@ pause_resume_verify() {t3="$(now_ms)"log "verify (pause): $VERIFY_ID resume->Running (checkpoint restored) $(( (t3 - t2) / 1000000 ))ms"wait_for "post-resume execd /ping 200 through a fresh route" 100 execd_ping_ok+ # The bundle is delivered to egress (POLICY_IMPORT) before the data plane+ # is ready, so the restored subject keeps the checkpoint's policy.+ wait_for "egress imported the policy from the checkpoint bundle" 120 \+ pause_egress_log_contains "subject s-$sandbox_uid: policy imported from snapshot bundle"+ wait_for "policy still enforcing after resume (example.com)" 120 pause_policy_enforced example.com+ pass "policy survived the resume (imported from the bundle, still enforcing)"pass "pause/resume round-trip timings: pause->Paused $(( (t2 - t1) / 1000000 ))ms, resume->Running $(( (t3 - t2) / 1000000 ))ms (opensandbox CLI -> FastPath -> artifact store)"+ # Policy update (raw PUT: the CLI only has merge-style patches), then a+ # second pause whose checkpoint must publish the UPDATED policy.+ local put_body+ put_body="$(jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "github.com"}]}')"+ server_api PUT "/sandboxes/$VERIFY_ID/networkpolicy" "$put_body" >/dev/null \+ || fail "PUT networkpolicy failed for $VERIFY_ID"+ wait_for "policy update converged (PUT -> ReplaceActionBindings -> nft)" 120 \+ pause_policy_enforced github.com++ log "verify (pause): osb sandbox pause $VERIFY_ID (second round, after the policy update)"+ osb_api sandbox pause "$VERIFY_ID" >/dev/null 2>&1 \+ || fail "second pause failed for $VERIFY_ID"+ wait_for "poll GET until Paused (second checkpoint durable)" 240 sandbox_paused++ # Artifact check (second pause): the NEW checkpoint bundle must carry the+ # UPDATED policy — the new rule present, the replaced rule gone, and a+ # strictly higher revision than the first bundle.+ wait_for "updated checkpoint bundle policy.json allows github.com" 60 pause_bundle_allows github.com+ wait_for "updated checkpoint bundle policy.json dropped example.com" 60 pause_bundle_lacks example.com+ rev2="$(pause_bundle_revision)"+ [[ "${rev2:-0}" -gt "${rev1:-0}" ]] \+ || fail "checkpoint policy.json revision did not advance across the policy update ($rev1 -> $rev2)"+ pass "checkpoint artifact updated: policy.json revision $rev1 -> $rev2, github.com added, example.com replaced"+osb_api sandbox kill "$VERIFY_ID" >/dev/null 2>&1 \|| log "verify cleanup: kill failed; remove $VERIFY_ID manually"wait_for "pause/resume sandbox deleted" 120 verify_sandbox_gone