Secure, Fast, and Extensible Sandbox runtime for AI agents.
#!/usr/bin/env bash
# Copyright 2026 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# integration-env.sh — one-command fast-sandbox integration environment,
# driven from the OpenSandbox repository.
#
# On a bare-metal Linux KVM host it builds the full stack — fast-sandbox at
# the commit pinned in manifests/third-party/fast-sandbox.commit, a two-node
# kind cluster with KVM passthrough, the OpenSandbox Helm charts (base CRDs,
# fast-sandbox control plane, server, ingress-gateway), a RustFS artifact
# store, and the firecracker-egress-pool SandboxPool with the OpenSandbox
# egress sidecar — and the verify-native matrix then proves the stack end to
# end: sandbox create via the opensandbox CLI, execd /ping through the signed
# gateway route, delete, a pause/resume round trip, a snapshot ->
# restore-from-snapshotId boot, the egress-held credential vault lifecycle
# (OSEP-0022 FastPath relay), the full MITM CA chain (OSEP-0022 trust
# anchor): a self-signed CA is provisioned as the versioned Secret, the
# server bakes it into the golden image, the pool pins the keypair into
# mitmdump, and a guest HTTPS request proves the template↔pool fingerprint
# pinning plus per-subject credential injection against a CA-signed TLS echo
# endpoint, and the execd-as-init in-guest process verify (OSEP-0018:
# `execd --init` as a subreaper under the guest init — topology, reaping,
# and the hardening report pinned in-guest).
#
# All Kubernetes resources come from the charts via `helm template` + plain
# `kubectl apply` (no helm release state); the only env-owned manifests are
# the kind cluster config and the SandboxPool. The verify stages drive the
# server through the published opensandbox CLI ("osb", installed fresh from
# PyPI; OSB_BIN overrides); raw curl remains only for the calls the CLI
# cannot express (networkpolicy replace PUT, metadata merge-patch, snapshot
# re-entry fence) plus the gateway /ping probes.
#
# Usage:
# ./scripts/fast-sandbox-env/integration-env.sh up # environment: stack + pool + server/ingress + native template
# ./scripts/fast-sandbox-env/integration-env.sh verify-native # verify matrix on the native template (needs up)
# ./scripts/fast-sandbox-env/integration-env.sh verify-overlaybd # overlaybd template build + verify matrix (needs up)
# ./scripts/fast-sandbox-env/integration-env.sh pool # re-apply the pool only
# ./scripts/fast-sandbox-env/integration-env.sh status # component/pool/DART/OpenSandbox health
# ./scripts/fast-sandbox-env/integration-env.sh down # teardown, host left clean
# ./scripts/fast-sandbox-env/integration-env.sh up --auto-clean # down on failure
#
# Environment overrides (all optional):
# WORK workspace root (default /data/fast-sandbox-env when /data exists, else $PWD/.fast-sandbox-env)
# FSB_DIR fast-sandbox checkout (default $WORK/fast-sandbox; source pinned in manifests/third-party/fast-sandbox.commit)
# KIND_CLUSTER / KIND_NODE_IMAGE / KIND_RETAIN / KIND_SINGLE
# DOCKER_MIRROR comma list injected as docker.io containerd mirrors
# RUSTFS_PORT / RUSTFS_CONSOLE_PORT / RUSTFS_AK / RUSTFS_SK / RUSTFS_IMAGE / RC_IMAGE / RUSTFS_ENDPOINT
# IMAGE_<NAME> fast-sandbox component image tags
# EGRESS_IMAGE egress image tag (default docker.io/opensandbox/egress:latest)
# SERVER_IMAGE / INGRESS_IMAGE OpenSandbox server/ingress image tags
# SERVER_HOST_PORT / GATEWAY_HOST_PORT host-side publishes (default 18080/18081)
# WARM_IMAGES=1 preheat pool warmImages (default: on-demand first-sandbox pull)
# SBX_IMAGE / EXECD template build inputs (default opensandbox/fsb-sandbox-golden:latest / opensandbox/execd:latest)
# EXECD_RUN_AS_INIT OSEP-0018 execd-as-init (default true); false =
# classic topology (the init-mode verify stage fails)
# POOL_MIN / POOL_MAX pool capacity (default 2/2; auto 1/1 when KIND_SINGLE=1)
# MAX_SANDBOXES_PER_POD per-fastlet sandbox capacity (default 8)
# XFS_STATEROOT / XFS_SIZE reflink StateRoot on/off and virtual size
# OSB_PACKAGE / OSB_BIN opensandbox CLI for the verify stages (latest from PyPI / preinstalled binary)
# SKIP_TOOL_INSTALL / SKIP_LEFTOVER_CLEAN / INOTIFY_VALUE
# IDLE_PAUSE=1 OSEP-0024 idle-pause/wake profile: Redis, server
# [automation.idle_pause], gateway activity+wake flags,
# idle-pause verify stage
# IDLE_PAUSE_ROUNDS auto-pause/wake rounds in the verify stage and idle-e2e (default 3)
# IDLE_PAUSE_PARK_BUDGET gateway wake park budget (default 5s; set e.g. 100ms to
# force every wake through the marked-503 + client-retry path)
# REDIS_IMAGE Redis image for the idle-pause profile (default redis:7-alpine)
# OCI_STACK OverlayBD OCI profile, default 1: in-cluster TLS
# registry (CA baked into the fsb images), the chart's
# runtime.oci node stack, and a full overlaybd verify
# pass beside native. Nodes need ublk support. 0 = native-only.
#
# Every stage logs to $WORK/logs/; failures dump component logs to
# logs/failure-<task>-<ts>.txt before exiting (never silently).
# -E (errtrace): stage functions must inherit the ERR trap, or on_error
# (and its failure dump) never fires — every stage runs inside a function.
set -euEo pipefail
ENV_PID=$BASHPID
OSB_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MANIFESTS_DIR="$SCRIPT_DIR/manifests"
# Heavy run-state belongs on a data volume, not the repo/root disk.
if [[ -d /data ]] && [[ -z "${WORK:-}" ]]; then
WORK="/data/fast-sandbox-env"
else
WORK="${WORK:-$PWD/.fast-sandbox-env}"
fi
LOGS_DIR="$WORK/logs"
GEN_DIR="$WORK/gen"
# Env-owned fast-sandbox clone; the source is exactly the commit pinned in
# manifests/third-party/fast-sandbox.commit (no ref override — bump the pin).
FSB_DIR="${FSB_DIR:-$WORK/fast-sandbox}"
FSB_REPO="$(sed -n 's/^repo:[[:space:]]*//p' "$OSB_ROOT/manifests/third-party/fast-sandbox.commit")"
FSB_COMMIT="$(sed -n 's/^commit:[[:space:]]*//p' "$OSB_ROOT/manifests/third-party/fast-sandbox.commit")"
KIND_CLUSTER="${KIND_CLUSTER:-fast-sandbox-integration}"
KIND_SINGLE="${KIND_SINGLE:-0}"
KIND_RETAIN="${KIND_RETAIN:-0}"
# Control plane + node runtime (controller/fastpath, firecracker-runtime,
# agent credentials, artifact-store config) ...
NS="opensandbox-system"
# ... while the SandboxPool/Template/Sandbox resources and the fastlet and
# builder Pods they spawn live in the dataplane namespace.
RESOURCE_NS="opensandbox-dataplane"
RUSTFS_IMAGE="${RUSTFS_IMAGE:-rustfs/rustfs:latest}"
RC_IMAGE="${RC_IMAGE:-rustfs/rc:latest}"
RUSTFS_PORT="${RUSTFS_PORT:-19000}"
# In-container ports are fixed (9000 S3 / 9001 console); these vars only move
# the host-side 127.0.0.1 publishes (defaults avoid 9000/9001 collisions).
RUSTFS_CONTAINER_PORT=9000
RUSTFS_CONSOLE_PORT="${RUSTFS_CONSOLE_PORT:-19001}"
RUSTFS_AK="${RUSTFS_AK:-integration-env}"
RUSTFS_SK="${RUSTFS_SK:-integration-env-secret}"
RUSTFS_BUCKET="sandbox-images"
RUSTFS_CONTAINER="${RUSTFS_CONTAINER:-fast-sandbox-env-rustfs}"
RUSTFS_DATA="$WORK/rustfs-data"
RUSTFS_ENDPOINT="${RUSTFS_ENDPOINT:-}" # auto-derived from the kind network
RC_CONFIG_DIR="$WORK/rc-config"
SBX_IMAGE="${SBX_IMAGE:-opensandbox/fsb-sandbox-golden:latest}"
EXECD="${EXECD:-opensandbox/execd:latest}"
# OSEP-0018: bake EXECD_INIT=1 into templates (the init-mode verify stage
# pins the in-guest topology); false restores the classic topology.
EXECD_RUN_AS_INIT="${EXECD_RUN_AS_INIT:-true}"
# WARM_IMAGES=1 preheats pool warmImages (keyed by template id); the default
# is the on-demand first-sandbox pull through DART.
WARM_IMAGES="${WARM_IMAGES:-0}"
POOL_NAME="${POOL_NAME:-firecracker-egress-pool}"
MAX_SANDBOXES_PER_POD="${MAX_SANDBOXES_PER_POD:-8}"
if [[ -n "${POOL_MIN:-}" && -n "${POOL_MAX:-}" ]]; then
:
elif [[ "$KIND_SINGLE" == "1" ]]; then
# Cache-only topology: a single fastlet is enough (no peer traffic).
POOL_MIN="${POOL_MIN:-1}"
POOL_MAX="${POOL_MAX:-1}"
else
POOL_MIN="${POOL_MIN:-2}"
POOL_MAX="${POOL_MAX:-2}"
fi
# Image tags (env-overridable per component).
IMG_CONTROLLER="${IMAGE_CONTROLLER:-opensandbox/fsb-controller:dev}"
IMG_FASTLET="${IMAGE_FASTLET:-opensandbox/fsb-fastlet:dev}"
IMG_FASTLET_PROXY="${IMAGE_FASTLET_PROXY:-opensandbox/fsb-fastlet-proxy:dev}"
IMG_JANITOR="${IMAGE_JANITOR:-opensandbox/fsb-janitor:dev}"
IMG_BUILDER="${IMAGE_BUILDER:-opensandbox/fsb-sandboxtemplate-builder:dev}"
# Shared build base for the builder image (built by build-fast-sandbox.sh
# during build_images; rebuilt here only if missing).
IMG_OCI_TOOLS="${IMAGE_OCI_TOOLS:-opensandbox/oci-tools:dev}"
IMG_RUNTIME="${IMAGE_RUNTIME:-opensandbox/fsb-firecracker-runtime:dev}"
IMG_EGRESS="${EGRESS_IMAGE:-docker.io/opensandbox/egress:latest}"
# OverlayBD OCI stack, default on: one cluster serves both template paths.
# Requires node ublk support (CONFIG_BLK_DEV_UBLK). Enables the in-cluster
# TLS registry (CA baked into the fsb images), the chart's runtime.oci node
# stack, the registry credentials/server setting, and the overlaybd verify
# pass. OCI_STACK=0 is the native-only escape hatch.
OCI_STACK="${OCI_STACK:-1}"
OCI_REGISTRY_IMAGE="${OCI_REGISTRY_IMAGE:-registry:2}"
OCI_HTPASSWD_IMAGE="${OCI_HTPASSWD_IMAGE:-httpd:2.4-alpine}"
OCI_REGISTRY_USER="${OCI_REGISTRY_USER:-envregistry}"
OCI_REGISTRY_PASSWORD="${OCI_REGISTRY_PASSWORD:-envregistry-secret}"
OCI_REGISTRY_SECRET="registry-env"
OCI_REGISTRY_REPO="sandbox/templates"
OCI_REGISTRY_HOST="registry.${RESOURCE_NS}.svc.cluster.local"
CA_DIR="$WORK/registry-ca"
# OSEP-0022 MITM trust anchor — always on in this environment: a self-signed
# CA is provisioned as the versioned Secret (ops-runbook shape), the server
# bakes it into the golden-image template, the egress seeds it into mitmdump,
# and a dedicated verify stage proves the whole chain (fingerprint pinning +
# per-subject credential injection over guest HTTPS). A broken chain fails
# closed and observably (MITM addon down on /healthz, template bake, pinning
# check) instead of silently degrading.
MITM_CA_SECRET="${MITM_CA_SECRET:-opensandbox-mitm-ca-v1}"
MITM_CA_DIR="$WORK/mitm-ca"
MITM_CA_FINGERPRINT_FILE="$MITM_CA_DIR/fingerprint"
MITM_CA_FINGERPRINT=""
# TLS echo endpoint (certificate signed by the pinned CA) as a container on
# the kind network (same shape as RustFS): the guest reaches it by IP, the
# vault binding matches by SNI.
MITM_ECHO_CONTAINER="${MITM_ECHO_CONTAINER:-fast-sandbox-env-mitm-echo}"
MITM_ECHO_DOMAIN="${MITM_ECHO_DOMAIN:-echo.test}"
MITM_ECHO_PORT=443
MITM_ECHO_IP=""
MITM_ECHO_NET=""
image_repo() { printf '%s' "${1%:*}"; }
image_tag() { printf '%s' "${1##*:}"; }
# --- OpenSandbox server + ingress gateway (source-built; always deployed) ------
OSB_NS="opensandbox-system"
IMG_SERVER="${SERVER_IMAGE:-docker.io/opensandbox/server:env}"
IMG_INGRESS="${INGRESS_IMAGE:-docker.io/opensandbox/ingress:env}"
FASTPATH_ENDPOINT="fast-sandbox-fastpath.opensandbox-system.svc:9090"
SERVER_API_KEY="fast-sandbox-env"
# OSEP-0024 profile: Redis + server [automation.idle_pause] + gateway activity/wake flags.
IDLE_PAUSE="${IDLE_PAUSE:-0}"
REDIS_IMAGE="${REDIS_IMAGE:-redis:7-alpine}"
REDIS_RELEASE="idle-pause-redis"
REDIS_DSN="redis://$REDIS_RELEASE.$OSB_NS.svc.cluster.local:6379/0"
# Per-workdir f1.* route-scope signing key: re-applies must keep old routes
# verifiable.
SIGNING_KEY_FILE="$WORK/opensandbox-signing-key"
# Host publish via kind extraPortMappings (loopback only; overridable on
# port collisions).
SERVER_HOST_PORT="${SERVER_HOST_PORT:-18080}"
GATEWAY_HOST_PORT="${GATEWAY_HOST_PORT:-18081}"
SERVER_NODEPORT=30880
GATEWAY_NODEPORT=30881
GATEWAY_ADDRESS="127.0.0.1:$GATEWAY_HOST_PORT"
SERVER_URL="http://127.0.0.1:$SERVER_HOST_PORT"
GATEWAY_URL="http://127.0.0.1:$GATEWAY_HOST_PORT"
# Applied by the firecracker-runtime readiness loop itself (never manually).
KVM_NODE_LABEL="sandbox.fast.io/kvm"
FC_NODE_LABEL="fast-sandbox.io/firecracker-node"
INOTIFY_VALUE="${INOTIFY_VALUE:-8192}"
SYSCTL_BACKUP="$WORK/sysctl-backup"
SKIP_TOOL_INSTALL="${SKIP_TOOL_INSTALL:-0}"
SKIP_LEFTOVER_CLEAN="${SKIP_LEFTOVER_CLEAN:-0}"
KIND_VERSION="${KIND_VERSION:-v0.24.0}"
KUBECTL_VERSION="${KUBECTL_VERSION:-v1.31.0}"
HELM_VERSION="${HELM_VERSION:-v3.16.4}"
# opensandbox CLI for the verify stages; installed by ensure_osb.
OSB_PACKAGE="${OSB_PACKAGE:-opensandbox-cli}"
OSB_BIN="${OSB_BIN:-}"
OSB_VENV="$WORK/tools/osb-venv"
# Internal goproxy mirrors can 500 on shared hosts; direct VCS just works there.
FSB_GOPROXY="${FSB_GOPROXY:-direct}"
export GOPROXY="$FSB_GOPROXY"
AUTO_CLEAN=0
ACTION=""
# --- logging / stage machinery --------------------------------------------------
log() { printf '\033[1;34m[fast-sandbox-env]\033[0m %s\n' "$*" | tee -a "$WORK/run.log" >&2; }
die() { printf '\033[1;31m[fast-sandbox-env] ERROR:\033[0m %s\n' "$*" >&2; exit 1; }
pass() { printf '\033[1;32m[fast-sandbox-env] PASS\033[0m %s\n' "$*" | tee -a "$WORK/run.log" >&2; }
# fail dumps before exiting: an explicit `exit` does not fire the ERR trap.
fail() { printf '\033[1;31m[fast-sandbox-env] FAIL\033[0m %s\n' "$*" >&2; failure_dump "${STAGE_CUR_LABEL:-$ACTION}" || true; exit 1; }
highlight() { printf '\033[1;36m%s\033[0m\n' "$*"; }
now_ms() { date +%s%N; } # GNU date (Linux); the script targets a Linux KVM host
ms2s() { awk -v ms="$1" 'BEGIN { printf "%.1f", ms / 1000 }'; }
declare -a STAGE_ORDER=()
declare -a STAGE_MS_LIST=()
STAGE_CUR=""
STAGE_CUR_LABEL=""
STAGE_START_NS=0
STAGE_N=0
stage_begin() {
STAGE_N=$((STAGE_N + 1))
STAGE_CUR="$1"
STAGE_CUR_LABEL="$(printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_')"
STAGE_START_NS="$(now_ms)"
printf '\n\033[1;36m==> [%d] %s\033[0m\n' "$STAGE_N" "$1"
}
stage_done() {
local ms
ms=$(( ($(now_ms) - STAGE_START_NS) / 1000000 ))
STAGE_ORDER+=("$STAGE_CUR")
STAGE_MS_LIST+=("$ms")
printf '\033[1;32m OK in %ss\033[0m %s\n' "$(ms2s "$ms")" "${1:-}"
}
run_stage() {
local description="$1" func="$2"
shift 2
stage_begin "$description"
"$func" "$@"
stage_done
}
stage_summary() {
local index name ms total=0
highlight "== stage timings =="
printf ' \033[1m%-46s %10s\033[0m\n' "stage" "duration"
for index in "${!STAGE_ORDER[@]}"; do
name="${STAGE_ORDER[$index]}"
ms="${STAGE_MS_LIST[$index]}"
total=$((total + ms))
printf ' %-46s %9ss\n' "$name" "$(ms2s "$ms")"
done
printf ' \033[1m%-46s %9ss\033[0m\n' "TOTAL" "$(ms2s "$total")"
}
wait_for() { # description attempts command [args...]
local description="$1" attempts="$2" attempt=0
shift 2
while ! "$@" >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [[ "$attempt" -ge "$attempts" ]]; then
fail "$description (after $attempts attempts)"
fi
sleep 2
done
pass "$description"
}
kubectl_get() { kubectl -n "$RESOURCE_NS" get "$1" -o jsonpath="$2"; }
sudo_() { if [[ "$(id -u)" == 0 ]]; then "$@"; else sudo "$@"; fi; }
# --- helpers ---------------------------------------------------------------------
kind_node() { kind get nodes --name "$KIND_CLUSTER" 2>/dev/null | head -1; }
kind_network() { # docker network of the first node container
local node
node="$(kind_node)" || return 1
docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$node" | tr ' ' '\n' | grep -v '^$' | head -1
}
rc() {
docker run --rm --network host --user "$(id -u):$(id -g)" -e RC_CONFIG_DIR=/config \
-e HTTP_PROXY= -e HTTPS_PROXY= -e ALL_PROXY= \
-e http_proxy= -e https_proxy= -e all_proxy= \
-v "$RC_CONFIG_DIR:/config" "$RC_IMAGE" "$@"
}
# --- failure dump ------------------------------------------------------------------
on_error() {
local rc=$? task="$1"
# Record WHERE it died before anything else: the dump alone cannot
# distinguish a silent command failure from an environment problem
# (e.g. the RustFS stage once died in ~1s with zero output).
log "up failed: task=$task rc=$rc command=${BASH_COMMAND:-unknown} at ${BASH_SOURCE[1]:-?}:${BASH_LINENO[0]:-?}" || true
# Subshells inherit ERR: propagate the error instead of tearing down
# main-process resources; ignore SIGPIPE (cancelled runners).
trap - ERR
[[ "$BASHPID" == "$ENV_PID" ]] || exit "$rc"
trap '' PIPE
# Dump before --auto-clean teardown destroys the evidence.
failure_dump "$task" || true
if [[ "$AUTO_CLEAN" == 1 ]]; then
log "$ACTION failed at $task; --auto-clean: running down" || true
(down) >/dev/null 2>&1 || true
fi
printf '\033[1;31m[fast-sandbox-env] FAILED at %s; dump: %s\033[0m\n' \
"$task" "$LOGS_DIR/failure-$task-*.txt" >&2 || true
exit "$rc"
}
failure_dump() {
local task="$1"
local dump="$LOGS_DIR/failure-$task-$(date +%s).txt"
mkdir -p "$LOGS_DIR"
{
echo "=== fast-sandbox-env failure: $task ($(date -u +%FT%TZ)) ==="
env | grep -E '^(KIND_(CLUSTER|SINGLE|RETAIN|NODE_IMAGE)=|FSB_DIR=|SBX_IMAGE=|IMG_[A-Z_]+=|EGRESS_IMAGE=|EXECD=|WORK=|POOL_(NAME|MIN|MAX)=|INGRESS_IMAGE=|WARM_IMAGES=|XFS_(STATEROOT|SIZE)=|RUSTFS_(IMAGE|PORT|CONSOLE_PORT|BUCKET|CONTAINER|DATA|ENDPOINT)=|RC_IMAGE=|OCI_STACK=|MITM_(CA_SECRET|CA_DIR|ECHO_CONTAINER|ECHO_DOMAIN)=)' || true
echo "--- fast-sandbox checkout ---"
git -C "$FSB_DIR" rev-parse HEAD 2>&1 || true
echo "--- kind-create.log (tail) ---"
tail -40 "$LOGS_DIR/kind-create.log" 2>&1 || true
echo "--- nodes ---"
kubectl get nodes -o wide 2>&1 || true
echo "--- pods ---"
kubectl get pods -n "$NS" -o wide 2>&1 || true
echo "--- controller logs (tail) ---"
kubectl logs -n "$NS" deploy/fast-sandbox-controller --tail=80 2>&1 || true
echo "--- firecracker-runtime logs (every pod + container, tail) ---"
# `kubectl logs daemonset/...` resolves to ONE pod only, so a wedged
# ublkd on the other node stays invisible. Loop the pods instead,
# and add the ublkd audit log plus the kernel device table: the
# daemon reports device-start failures to the audit log, not to
# stdout ("device failed to start (see daemon log)").
local rpod rcontainer
for rpod in $(runtime_pods 2>/dev/null || true); do
for rcontainer in $(kubectl -n "$NS" get pod "$rpod" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do
echo "### $rpod container=$rcontainer"
kubectl logs -n "$NS" "$rpod" -c "$rcontainer" --tail=80 2>&1 || true
done
echo "### $rpod overlaybd-ublkd audit log + kernel ublk state"
kubectl exec -n "$NS" "$rpod" -c overlaybd-ublkd -- sh -c '
tail -100 /var/log/overlaybd-audit.log 2>/dev/null
echo "-- /dev/ublk*:"
ls -la /dev/ublk* 2>&1
echo "-- /proc/devices (ublk):"
grep -i ublk /proc/devices 2>&1
echo "-- ublkd attached devices, attributed by owner (the daemon list is the only record that outlives a dead builder/fastlet pod):"
list=$(curl -sS -m 10 --noproxy "*" --unix-socket /run/fast-sandbox/overlaybd/ublkd.sock http://ublkd/v1/list 2>&1)
printf "%s\n" "$list" | tr "{," "\n" | grep -o "\"config\":\"[^\"]*\"" | sed "s/^\"config\":\"//;s/\"\$//" | sort > /tmp/fsb-dev-configs.$$
builder=0; shared=0; sandbox=0; other=0
while read -r cfg; do
[ -n "$cfg" ] || continue
case "$cfg" in
/build/*) builder=$((builder+1)); tag=builder ;;
*.snapfiles/*) shared=$((shared+1)); tag=fastlet-shared ;;
*/sandboxes/*) sandbox=$((sandbox+1)); tag=fastlet-sandbox ;;
*) other=$((other+1)); tag=other ;;
esac
echo " [$tag] $cfg"
done < /tmp/fsb-dev-configs.$$
echo " summary: builder=$builder fastlet-shared=$shared fastlet-sandbox=$sandbox other=$other"
rm -f /tmp/fsb-dev-configs.$$' 2>&1 || true
done
echo "--- fastlet pod logs, every container (deep tail) ---"
kubectl logs -n "$RESOURCE_NS" -l app=sandbox-fastlet \
--all-containers --prefix --timestamps --tail=15000 2>&1 || true
echo "--- fastlet pod logs, previous container instances (crashes) ---"
for pod in $(kubectl -n "$RESOURCE_NS" get pods -l app=sandbox-fastlet -o name 2>/dev/null); do
for container in $(kubectl -n "$RESOURCE_NS" get "$pod" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do
prev="$(kubectl -n "$RESOURCE_NS" logs "$pod" -c "$container" --previous --timestamps --tail=200 2>&1 || true)"
case "$prev" in *"no previous container"*) continue ;; esac
echo "### ${pod##*/} container=$container (previous)"
printf '%s\n' "$prev"
done
done
echo "--- builder pods + logs (deep tail: fsb-vm phase + the ublkd daemon errors it reports) ---"
kubectl get pods -n "$RESOURCE_NS" -l sandbox.fast.io/sandboxtemplate --show-labels 2>&1 || true
kubectl logs -n "$RESOURCE_NS" -l sandbox.fast.io/sandboxtemplate --tail=2000 2>&1 || true
if [[ "$OCI_STACK" == 1 ]]; then
echo "--- OCI registry (pod + logs tail) ---"
kubectl get pods -n "$RESOURCE_NS" -l app=fast-sandbox-env-registry -o wide 2>&1 || true
kubectl logs -n "$RESOURCE_NS" -l app=fast-sandbox-env-registry --tail=60 2>&1 || true
fi
echo "--- SandboxTemplates (status carries the build failure reason) ---"
kubectl get sandboxtemplates -n "$RESOURCE_NS" -o yaml 2>&1 || true
# Sandbox CR states: a pause/snapshot/resume failure shows up as a
# stuck or failed status condition here, with the phase the poll
# never reached.
echo "--- Sandbox CRs (status only) ---"
kubectl get sandboxes -n "$RESOURCE_NS" -o json 2>/dev/null \
| jq -c '.items[] | {name: .metadata.name, phase: .status.phase, conditions: .status.conditions, infra: .status.infraComponents}' 2>&1 || true
# A mount-namespace mismatch (driver staging path invisible to
# egress) shows up as ENOENT on the egress side only.
for pod in $(kubectl -n "$RESOURCE_NS" get pods -l app=sandbox-fastlet -o name 2>/dev/null); do
echo "### ${pod##*/}: snapshot staging visibility"
echo "-- fastlet view:"
kubectl -n "$RESOURCE_NS" exec "$pod" -c fastlet -- \
ls -la /var/lib/fast-sandbox/firecracker/snapshots 2>&1 | tail -8 || true
echo "-- egress view:"
kubectl -n "$RESOURCE_NS" exec "$pod" -c egress -- \
ls -la /var/lib/fast-sandbox/firecracker/snapshots 2>&1 | tail -8 || true
done
# MITM CA chain (OSEP-0022) failure scene: the addon-down reason
# lives on /healthz; seed/confdir visibility separates "Secret never
# mounted" from "mitmdump failed to start". The echo container and
# the guest curl stderr are only capturable here (down deletes both).
echo "--- MITM CA chain diagnostics ---"
if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "$MITM_ECHO_CONTAINER"; then
echo "-- echo container --"
docker ps -a --filter "name=$MITM_ECHO_CONTAINER" --format '{{.Names}} {{.Status}}' 2>&1 || true
docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}={{$v.IPAddress}} {{end}}' "$MITM_ECHO_CONTAINER" 2>&1 || true
docker logs "$MITM_ECHO_CONTAINER" --tail=50 2>&1 || true
fi
if [[ -d "$MITM_CA_DIR" ]]; then
echo "-- pinned CA material (names + fingerprint only, never key bytes) --"
ls -l "$MITM_CA_DIR" 2>&1 || true
echo "pinned fingerprint: $(cat "$MITM_CA_FINGERPRINT_FILE" 2>/dev/null || echo missing)"
fi
if [[ -f "$LOGS_DIR/mitm-ca-openssl.err" ]]; then
echo "-- openssl stderr (last MITM CA stage failure) --"
tail -20 "$LOGS_DIR/mitm-ca-openssl.err" 2>&1 || true
fi
if [[ -f "$LOGS_DIR/mitm-echo-probe.err" ]]; then
echo "-- echo endpoint probe stderr --"
tail -20 "$LOGS_DIR/mitm-echo-probe.err" 2>&1 || true
fi
if [[ -s "$MITM_CA_DIR/tls.crt" && -s "$MITM_CA_DIR/echo.crt" ]]; then
echo "-- certificate extensions (public material) --"
for cert in tls echo; do
openssl x509 -in "$MITM_CA_DIR/$cert.crt" -noout -subject -issuer -text 2>/dev/null \
| grep -E "subject=|issuer=|Basic Constraints|CA:|Key Usage|Alternative|Extended" || true
done
fi
if [[ -f "$LOGS_DIR/mitm-guest-curl.err" ]]; then
echo "-- last guest curl stderr ($LOGS_DIR/mitm-guest-curl.err) --"
tail -20 "$LOGS_DIR/mitm-guest-curl.err" 2>&1 || true
fi
local mitm_pod
mitm_pod="$(first_pool_pod 2>/dev/null || true)"
if [[ -n "$mitm_pod" ]]; then
echo "-- egress MITM surfaces (pod ${mitm_pod##*/}) --"
kubectl -n "$RESOURCE_NS" exec "pod/$mitm_pod" -c egress -- \
curl -fsS -m 5 "http://127.0.0.1:18080/healthz" 2>&1 || true
echo
kubectl -n "$RESOURCE_NS" exec "pod/$mitm_pod" -c egress -- \
curl -fsS -m 5 "http://127.0.0.1:18080/_fastlet/v1/actions/status" 2>&1 || true
echo
echo "-- egress seed mount + mitmdump confdir visibility --"
kubectl -n "$RESOURCE_NS" exec "pod/$mitm_pod" -c egress -- \
ls -la /var/egress/mitm-ca-seed 2>&1 || true
kubectl -n "$RESOURCE_NS" exec "pod/$mitm_pod" -c egress -- \
ls -la /var/lib/mitmproxy/.mitmproxy 2>&1 || true
kubectl -n "$RESOURCE_NS" exec "pod/$mitm_pod" -c egress -- \
pgrep -af mitmdump 2>&1 || true
fi
if [[ -f "$GEN_DIR/mitm-ca-policy.json" ]]; then
echo "-- mitm-ca verify sandbox policy --"
cat "$GEN_DIR/mitm-ca-policy.json" 2>&1 || true
fi
echo "--- recent events ($RESOURCE_NS) ---"
kubectl get events -n "$RESOURCE_NS" --sort-by=.lastTimestamp 2>&1 | tail -30 || true
echo "--- OpenSandbox pods ($OSB_NS) ---"
kubectl get pods -n "$OSB_NS" -o wide 2>&1 || true
echo "--- server logs (tail) ---"
kubectl logs -n "$OSB_NS" deploy/opensandbox-server --tail=80 2>&1 || true
echo "--- ingress gateway logs (tail) ---"
kubectl logs -n "$OSB_NS" deploy/opensandbox-ingress-gateway --tail=80 2>&1 || true
echo "--- pool ---"
kubectl get sandboxpool -n "$RESOURCE_NS" -o yaml 2>&1 || true
echo "--- rustfs docker logs (tail) ---"
docker logs "$RUSTFS_CONTAINER" --tail=80 2>&1 || true
} > "$dump" 2>&1 || true
log "failure dump: $dump"
}
# --- stage: preflight + tooling ------------------------------------------------------
install_release_binary() { # name version url
local name="$1" version="$2" url="$3" tmp
log "installing $name $version -> /usr/local/bin/$name"
tmp="$(mktemp -d)"
curl -fL --retry 3 -o "$tmp/$name" "$url" \
|| die "download $name failed ($url); install it manually or retry"
sudo_ install -m 0755 "$tmp/$name" "/usr/local/bin/$name"
rm -rf "$tmp"
}
ensure_tool() { # name
local name="$1"
if command -v "$name" >/dev/null 2>&1; then
return 0
fi
if [[ "$SKIP_TOOL_INSTALL" == 1 ]]; then
die "$name is required (SKIP_TOOL_INSTALL=1: install it manually)"
fi
case "$name" in
kind)
install_release_binary kind "$KIND_VERSION" \
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
;;
kubectl)
install_release_binary kubectl "${KUBECTL_VERSION#v}" \
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
;;
helm)
local tmp
log "installing helm $HELM_VERSION -> /usr/local/bin/helm"
tmp="$(mktemp -d)"
curl -fL --retry 3 -o "$tmp/helm.tgz" \
"https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \
|| die "download helm failed; install it manually or retry"
sudo_ tar -xzf "$tmp/helm.tgz" -C "$tmp" linux-amd64/helm
sudo_ install -m 0755 "$tmp/linux-amd64/helm" /usr/local/bin/helm
rm -rf "$tmp"
;;
jq)
log "installing jq via package manager"
if command -v apt-get >/dev/null 2>&1; then
sudo_ apt-get install -y jq >/dev/null
elif command -v yum >/dev/null 2>&1; then
sudo_ yum install -y jq >/dev/null
elif command -v apk >/dev/null 2>&1; then
sudo_ apk add --no-cache jq >/dev/null
else
die "no supported package manager to install jq; install it manually"
fi
;;
*)
die "$name is required; install it manually"
;;
esac
command -v "$name" >/dev/null 2>&1 || die "$name installation failed"
}
ensure_osb() {
if [[ -n "$OSB_BIN" ]]; then
[[ -x "$OSB_BIN" ]] || die "OSB_BIN=$OSB_BIN is not executable"
log "using osb from OSB_BIN: $OSB_BIN"
else
if [[ "$SKIP_TOOL_INSTALL" == 1 ]] && [[ ! -x "$OSB_VENV/bin/osb" ]]; then
die "osb is required (SKIP_TOOL_INSTALL=1: install opensandbox-cli manually or set OSB_BIN)"
fi
command -v uv >/dev/null 2>&1 \
|| die "uv is required to install the opensandbox CLI (or set OSB_BIN)"
log "installing the latest $OSB_PACKAGE into $OSB_VENV (uv)"
: > "$LOGS_DIR/osb-install.log"
rm -rf "$OSB_VENV"
# Fresh venv at its final path: venv console-script shebangs embed the
# interpreter's absolute path, so create-elsewhere-and-move breaks them.
uv venv --python 3.12 "$OSB_VENV" >>"$LOGS_DIR/osb-install.log" 2>&1 \
|| { tail -n 20 "$LOGS_DIR/osb-install.log" >&2 || true; die "uv venv --python 3.12 failed (full log: $LOGS_DIR/osb-install.log; or set OSB_BIN)"; }
uv pip install --python "$OSB_VENV/bin/python" --upgrade "$OSB_PACKAGE" \
>>"$LOGS_DIR/osb-install.log" 2>&1 \
|| { tail -n 20 "$LOGS_DIR/osb-install.log" >&2 || true; die "uv pip install --upgrade $OSB_PACKAGE failed (index unreachable? full log: $LOGS_DIR/osb-install.log; or set OSB_BIN)"; }
OSB_BIN="$OSB_VENV/bin/osb"
fi
log "osb version: $("$OSB_BIN" --version 2>/dev/null | tail -n1 || echo unknown)"
pass "opensandbox CLI ready ($OSB_BIN)"
}
host_port_busy() { # port -> 0 when something already listens on 127.0.0.1:<port>
(exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null
}
# Pull only when absent locally (*_IMAGE overrides cover private registries).
ensure_image() { # image -> pull only when missing locally
docker image inspect "$1" >/dev/null 2>&1 && return 0
docker pull -q "$1" >/dev/null || die "image $1 is not available locally and the pull failed (pre-load it with docker load, or override the *_IMAGE variable)"
}
preflight() {
[[ "$(uname -s)" == "Linux" ]] \
|| die "this environment requires a Linux host with KVM (run it on the remote development VM)"
command -v docker >/dev/null || die "docker is required"
command -v go >/dev/null || die "go is required (>=1.25, used by make images and gen-registry)"
command -v openssl >/dev/null || die "openssl is required (signing key + pinned MITM CA generation)"
ensure_tool kind
ensure_tool kubectl
ensure_tool helm
ensure_tool jq
docker info >/dev/null 2>&1 || die "docker daemon is not reachable"
local cgver
cgver="$(docker info --format '{{.CgroupVersion}}' 2>/dev/null || true)"
log "docker cgroup version=$cgver"
# kind requires cgroup v2; v1 hosts fail kubepods cgroup creation.
if [[ "$cgver" == "1" ]]; then
die "docker cgroup Version is 1; kind requires cgroup v2. Enable it with the kernel cmdline 'systemd.unified_cgroup_hierarchy=1' and reboot"
fi
[[ -e /dev/kvm ]] || die "/dev/kvm is missing on this host (KVM required)"
# Fail fast per heavy-data target instead of ENOSPC mid-run.
local min_free_kb=$((20 * 1024 * 1024)) avail_kb target xfs_dir
xfs_dir="${XFS_LOOP_FILE%/*}"
mkdir -p "$WORK" "$RUSTFS_DATA" "$xfs_dir" 2>/dev/null || true
local -a targets=("$WORK" "$RUSTFS_DATA" "$xfs_dir")
for target in "${targets[@]}"; do
avail_kb="$(df -Pk "$target" 2>/dev/null | awk 'NR==2 {print $4}')"
[[ "$avail_kb" =~ ^[0-9]+$ ]] || die "cannot determine free disk space on $target"
if (( avail_kb < min_free_kb )); then
die "$target has $((avail_kb / 1024 / 1024))G free; at least 20G is required (built images, XFS StateRoot, RustFS artifacts). Free space (docker system prune / old kind clusters) or point WORK / RUSTFS_DATA / XFS_LOOP_FILE at a bigger volume"
fi
log "disk headroom: $target has $((avail_kb / 1024 / 1024))G free"
done
# Fail fast on busy host ports instead of a cryptic docker/kind bind error.
local port
for port in "$RUSTFS_PORT" "$RUSTFS_CONSOLE_PORT" "$SERVER_HOST_PORT" "$GATEWAY_HOST_PORT"; do
if host_port_busy "$port"; then
die "127.0.0.1:$port is already in use (check 'ss -ltnp' / 'docker ps'); free it, or set RUSTFS_PORT / RUSTFS_CONSOLE_PORT / SERVER_HOST_PORT / GATEWAY_HOST_PORT"
fi
done
ensure_image "$RUSTFS_IMAGE"
ensure_image "$RC_IMAGE"
pass "preflight"
}
sysctl_set() {
local current
current="$(sysctl -n fs.inotify.max_user_instances)"
[[ "$current" -ge "$INOTIFY_VALUE" ]] && return 0
echo "$current" > "$SYSCTL_BACKUP"
log "sysctl fs.inotify.max_user_instances: $current -> $INOTIFY_VALUE"
sudo sysctl -w fs.inotify.max_user_instances="$INOTIFY_VALUE" >/dev/null
}
sysctl_restore() {
[[ -f "$SYSCTL_BACKUP" ]] || return 0
local previous
previous="$(cat "$SYSCTL_BACKUP")"
log "restoring fs.inotify.max_user_instances -> $previous"
sudo sysctl -w fs.inotify.max_user_instances="$previous" >/dev/null || true
rm -f "$SYSCTL_BACKUP"
}
# --- stage: fast-sandbox checkout @ master ---------------------------------------------
# Scratch Go tool run inside the fast-sandbox module (imports
# internal/registryconfig); removed before dirty checks and on down.
FSB_GEN_DIR="$FSB_DIR/.fast-sandbox-env-gen"
ensure_fsb() {
if [[ ! -d "$FSB_DIR/.git" ]]; then
log "cloning $FSB_REPO into $FSB_DIR"
git clone "$FSB_REPO" "$FSB_DIR" || die "clone failed; check network"
fi
rm -rf "$FSB_GEN_DIR"
[[ -z "$(git -C "$FSB_DIR" status --porcelain)" ]] \
|| die "fast-sandbox checkout at $FSB_DIR has local changes; delete it to re-clone or point FSB_DIR at a clean checkout"
# Raw SHA fetch needs allow-reachable-sha1-in-want; full fetch otherwise.
if ! git -C "$FSB_DIR" fetch -q origin "$FSB_COMMIT" 2>/dev/null; then
git -C "$FSB_DIR" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' \
|| die "git fetch failed for $FSB_REPO"
fi
git -C "$FSB_DIR" rev-parse --verify --quiet "$FSB_COMMIT^{commit}" >/dev/null \
|| die "pinned commit $FSB_COMMIT is not reachable from $FSB_REPO"
if [[ "$(git -C "$FSB_DIR" rev-parse HEAD)" != "$FSB_COMMIT" ]]; then
git -C "$FSB_DIR" clean -ffdx
fi
git -C "$FSB_DIR" checkout --force -q "$FSB_COMMIT" \
|| die "git checkout $FSB_COMMIT failed"
[[ "$(git -C "$FSB_DIR" rev-parse HEAD)" == "$FSB_COMMIT" ]] \
|| die "fast-sandbox checkout is not at the pinned commit $FSB_COMMIT"
log "fast-sandbox @ pinned $(git -C "$FSB_DIR" rev-parse --short HEAD) (manifests/third-party/fast-sandbox.commit)"
pass "fast-sandbox checkout ready"
}
# --- stage: images ---------------------------------------------------------------------
build_images() {
log "building fast-sandbox images (pinned commit, firecracker scope) via manifests/release/build-fast-sandbox.sh"
# Same checkout and defaults as the standalone builder: env and published
# build paths cannot drift.
FSB_SRC_DIR="$FSB_DIR" "$OSB_ROOT/manifests/release/build-fast-sandbox.sh" \
|| die "fast-sandbox image build failed"
if [[ "$OCI_STACK" == 1 ]]; then
bake_registry_ca_into_images
fi
log "building the OpenSandbox egress image ($IMG_EGRESS)"
# Context is the repo root (COPYs components/egress + components/internal).
# shellcheck disable=SC2086
docker build ${DOCKER_BUILD_FLAGS:-} --quiet \
-f "$OSB_ROOT/components/egress/Dockerfile" -t "$IMG_EGRESS" "$OSB_ROOT" >/dev/null \
|| die "egress image build failed"
log "building the OpenSandbox server image ($IMG_SERVER)"
# Context is server/ (self-contained Dockerfile).
# shellcheck disable=SC2086
docker build ${DOCKER_BUILD_FLAGS:-} --quiet \
-f "$OSB_ROOT/server/Dockerfile" -t "$IMG_SERVER" "$OSB_ROOT/server" >/dev/null \
|| die "server image build failed"
log "building the OpenSandbox ingress image ($IMG_INGRESS)"
# shellcheck disable=SC2086
docker build ${DOCKER_BUILD_FLAGS:-} --quiet \
-f "$OSB_ROOT/components/ingress/Dockerfile" -t "$IMG_INGRESS" "$OSB_ROOT" >/dev/null \
|| die "ingress image build failed"
pass "images built (fast-sandbox + OpenSandbox)"
}
# --- stage: XFS StateRoot (reflink CoW per-sandbox rootfs) ------------------------------
XFS_STATEROOT="${XFS_STATEROOT:-1}"
XFS_LOOP_FILE="${XFS_LOOP_FILE:-$WORK/fast-sandbox.img}"
XFS_SIZE="${XFS_SIZE:-24G}"
XFS_MOUNT_POINT="${XFS_MOUNT_POINT:-/var/lib/fast-sandbox}"
ensure_xfsprogs() {
command -v mkfs.xfs >/dev/null 2>&1 && return 0
log "installing xfsprogs (mkfs.xfs)"
if command -v apt-get >/dev/null 2>&1; then
sudo_ apt-get install -y xfsprogs >/dev/null
elif command -v yum >/dev/null 2>&1; then
sudo_ yum install -y xfsprogs >/dev/null
else
die "xfsprogs not installed and no supported package manager"
fi
}
stateroot_xfs_up() {
[[ "$XFS_STATEROOT" == 1 ]] || {
sudo_ mkdir -p "$XFS_MOUNT_POINT"
log "XFS StateRoot disabled (XFS_STATEROOT=0); per-sandbox rootfs pays a full copy"
return 0
}
if findmnt -no FSTYPE "$XFS_MOUNT_POINT" 2>/dev/null | grep -qx xfs; then
# Leftover from a run killed before teardown: scrub, and recreate the
# image if still below the agent's min-free floor (it strips the kvm
# node label and template builds stop scheduling).
stateroot_scrub
if (( $(stateroot_free_gib) >= XFS_MIN_FREE_GIB )); then
pass "XFS StateRoot ready (reflink CoW rootfs, scrubbed)"
return 0
fi
log "StateRoot below ${XFS_MIN_FREE_GIB}GiB free after scrub; recreating the XFS image"
sudo_ umount "$XFS_MOUNT_POINT" || die "unmount $XFS_MOUNT_POINT failed"
rm -f "$XFS_LOOP_FILE"
fi
ensure_xfsprogs
if [[ ! -f "$XFS_LOOP_FILE" ]]; then
log "creating sparse XFS image $XFS_LOOP_FILE (virtual $XFS_SIZE)"
truncate -s "$XFS_SIZE" "$XFS_LOOP_FILE"
sudo_ mkfs.xfs -f "$XFS_LOOP_FILE" >/dev/null 2>&1 || die "mkfs.xfs failed on $XFS_LOOP_FILE"
fi
sudo_ mkdir -p "$XFS_MOUNT_POINT"
sudo_ mount -o noatime "$XFS_LOOP_FILE" "$XFS_MOUNT_POINT" \
|| die "mount $XFS_LOOP_FILE at $XFS_MOUNT_POINT failed (loop support? try XFS_STATEROOT=0)"
local a b
a="$XFS_MOUNT_POINT/.reflink-a"
b="$XFS_MOUNT_POINT/.reflink-b"
printf 'probe' | sudo_ tee "$a" >/dev/null
if sudo_ cp --reflink=always "$a" "$b"; then
sudo_ rm -f "$a" "$b"
pass "XFS StateRoot ready (reflink CoW rootfs)"
else
sudo_ rm -f "$a" "$b"
die "reflink probe failed on $XFS_MOUNT_POINT (CoW rootfs would not work)"
fi
}
# Purge per-node runtime caches in the mounted StateRoot (shared by down()
# and the reuse branch of stateroot_xfs_up); snapshots/ leaks across runs.
stateroot_scrub() {
local node_dir
for node_dir in "$XFS_MOUNT_POINT"/*/; do
[[ -d "${node_dir}firecracker" ]] || continue
log "purging node runtime cache under ${node_dir%/}"
sudo_ rm -rf "${node_dir}firecracker/images" \
"${node_dir}firecracker/agent" \
"${node_dir}firecracker/jails" \
"${node_dir}firecracker/cache" \
"${node_dir}firecracker/snapshots" 2>/dev/null || true
done
}
# Free space on the mounted StateRoot, in GiB (GNU df).
stateroot_free_gib() {
df -BG --output=avail "$XFS_MOUNT_POINT" 2>/dev/null | tail -1 | tr -dc '0-9' || echo 0
}
XFS_MIN_FREE_GIB="${XFS_MIN_FREE_GIB:-12}"
stateroot_xfs_down() {
[[ "$XFS_STATEROOT" == 1 ]] || return 0
if findmnt -no SOURCE "$XFS_MOUNT_POINT" 2>/dev/null | grep -q "$(basename "$XFS_LOOP_FILE")"; then
log "unmounting XFS StateRoot $XFS_MOUNT_POINT"
sudo_ umount "$XFS_MOUNT_POINT"
fi
rm -f "$XFS_LOOP_FILE"
}
# --- stage: kind cluster -----------------------------------------------------------------
render_kind_config() { # > $GEN_DIR/kind-cluster.yaml
local src="$MANIFESTS_DIR/cluster/kind-cluster.yaml" out="$GEN_DIR/kind-cluster.yaml"
mkdir -p "$GEN_DIR"
cp "$src" "$out"
# OCI_STACK=1 binds the host /dev into every node (FIRST extraMount, so
# the private /dev/shm mount below it shadows the host shm again): the
# kernel creates ublk device nodes on the host devtmpfs at add time and
# this kernel registers no block major, so a static node list cannot
# track them.
if [[ "$OCI_STACK" == 1 ]]; then
awk '
/^ # @UBLK_DEV_MOUNT@$/ {
print " - hostPath: /dev"
print " containerPath: /dev"
next }
{ print }
' "$out" > "$out.tmp" && mv "$out.tmp" "$out"
else
grep -v '^ # @UBLK_DEV_MOUNT@$' "$out" > "$out.tmp" && mv "$out.tmp" "$out"
fi
if [[ "$KIND_SINGLE" == "1" ]]; then
# Strip the worker node: cache-only topology, no peer traffic.
awk '/^- role: worker/ {exit} {print}' "$out" > "$out.tmp" && mv "$out.tmp" "$out"
log "single-node topology (KIND_SINGLE=1: no worker, no peer traffic)"
fi
if [[ -n "${DOCKER_MIRROR:-}" ]]; then
# Host-specific, hence opt-in; one TOML endpoint ARRAY (repeated
# endpoint keys would override each other).
local block_file="$GEN_DIR/docker-mirror-block.yaml" mirror trimmed endpoints=""
IFS=',' read -ra mirrors <<<"$DOCKER_MIRROR"
for mirror in "${mirrors[@]}"; do
trimmed="$(printf '%s' "$mirror" | tr -d '[:space:]')"
[[ -n "$trimmed" ]] || continue
[[ -z "$endpoints" ]] && endpoints="\"$trimmed\"" || endpoints+=", \"$trimmed\""
done
[[ -n "$endpoints" ]] || die "DOCKER_MIRROR produced no usable endpoints"
{
echo 'containerdConfigPatches:'
echo '- |-'
echo ' [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]'
echo " endpoint = [$endpoints]"
} > "$block_file"
awk -v block_file="$block_file" '
NR == FNR { block = block $0 "\n"; next }
/^nodes:/ && !done { printf "%s", block; done = 1 }
{ print }
' "$block_file" "$out" > "$out.tmp" && mv "$out.tmp" "$out"
rm -f "$block_file"
log "docker.io containerd mirrors injected: $endpoints"
fi
# extraPortMappings publish the server/gateway on the host; they exist
# only at cluster creation, so a cluster reused without them stays
# cluster-internal.
local ports_file="$GEN_DIR/osb-ports-block.yaml"
{
echo ' extraPortMappings:'
echo " - containerPort: $SERVER_NODEPORT"
echo " hostPort: $SERVER_HOST_PORT"
echo ' listenAddress: 127.0.0.1'
echo ' protocol: TCP'
echo " - containerPort: $GATEWAY_NODEPORT"
echo " hostPort: $GATEWAY_HOST_PORT"
echo ' listenAddress: 127.0.0.1'
echo ' protocol: TCP'
} > "$ports_file"
awk -v block_file="$ports_file" '
NR == FNR { block = block $0 "\n"; next }
/^- role: control-plane/ && !done { print; printf "%s", block; done = 1; next }
{ print }
' "$ports_file" "$out" > "$out.tmp" && mv "$out.tmp" "$out"
rm -f "$ports_file"
log "server on 127.0.0.1:$SERVER_HOST_PORT, gateway on 127.0.0.1:$GATEWAY_HOST_PORT (NodePorts $SERVER_NODEPORT/$GATEWAY_NODEPORT)"
}
kind_up() {
local create_args=() kind_config="$GEN_DIR/kind-cluster.yaml"
[[ -f "$kind_config" ]] || die "kind config not rendered ($kind_config)"
[[ "$KIND_RETAIN" == 1 ]] && create_args+=(--retain)
if [[ -n "$(kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" || true)" ]]; then
log "cluster $KIND_CLUSTER already exists; reusing (run down first for a clean rebuild)"
else
if [[ -n "${KIND_NODE_IMAGE:-}" ]]; then
log "pulling kind node image $KIND_NODE_IMAGE (this can take minutes)"
ensure_image "$KIND_NODE_IMAGE" || die "kind node image unavailable locally and pull failed (KIND_NODE_IMAGE=$KIND_NODE_IMAGE)"
create_args+=(--image "$KIND_NODE_IMAGE")
else
log "creating cluster (pulling kindest/node may take minutes; set KIND_NODE_IMAGE to a mirror if it fails)"
fi
kind create cluster --name "$KIND_CLUSTER" \
${create_args+"${create_args[@]}"} --config "$kind_config" > "$LOGS_DIR/kind-create.log" 2>&1 || {
local rc=$?
tail -40 "$LOGS_DIR/kind-create.log" >&2 || true
log "kind create failed (full log: $LOGS_DIR/kind-create.log)" || true
return "$rc" # through ERR, so diagnostics + --auto-clean run
}
pass "kind cluster created"
fi
local node
for node in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do
docker exec "$node" sh -c 'test -e /dev/kvm' || die "KVM not visible inside the kind node container $node"
log "node $node: /dev/kvm visible"
done
if [[ "$KIND_SINGLE" != "1" ]]; then
# Untaint the control plane: agent/fastlet/builder must schedule on
# BOTH nodes for P2P peer traffic.
kubectl taint nodes --all node-role.kubernetes.io/control-plane- >/dev/null 2>&1 || true
log "control-plane taint removed (both nodes schedulable for the P2P topology)"
fi
pass "kind cluster ready (KVM passthrough + labels on every node)"
}
# --- stage: RustFS + credentials ------------------------------------------------------------
rustfs_up() {
docker rm -f "$RUSTFS_CONTAINER" >/dev/null 2>&1 || true
# RustFS runs as UID/GID 10001 and owns the bind-mounted store.
sudo_ rm -rf "$RUSTFS_DATA"
mkdir -p "$RUSTFS_DATA"
sudo_ chown -R 10001:10001 "$RUSTFS_DATA"
mkdir -p "$RC_CONFIG_DIR"
chmod 700 "$RC_CONFIG_DIR"
local net
net="$(kind_network)"
# On the kind network: pods reach the container IP directly (no
# docker-proxy/hairpin issues); 127.0.0.1 publishes serve the host.
docker run -d --name "$RUSTFS_CONTAINER" --network "$net" \
-p 127.0.0.1:"$RUSTFS_PORT":"$RUSTFS_CONTAINER_PORT" -p 127.0.0.1:"$RUSTFS_CONSOLE_PORT":9001 \
-e RUSTFS_ACCESS_KEY="$RUSTFS_AK" -e RUSTFS_SECRET_KEY="$RUSTFS_SK" \
-e RUSTFS_ADDRESS=":$RUSTFS_CONTAINER_PORT" \
-e RUSTFS_CONSOLE_ADDRESS=:9001 -e RUSTFS_CONSOLE_ENABLE=true \
-v "$RUSTFS_DATA:/data" \
"$RUSTFS_IMAGE" >/dev/null
local attempt
for attempt in $(seq 1 30); do
if curl -fsS "http://127.0.0.1:$RUSTFS_PORT/health" >/dev/null 2>&1; then break; fi
sleep 1
[[ "$attempt" == 30 ]] && die "RustFS did not become healthy"
done
rc alias set chain "http://127.0.0.1:$RUSTFS_PORT" "$RUSTFS_AK" "$RUSTFS_SK" >/dev/null 2>&1 \
|| die "could not configure the RustFS rc alias"
for attempt in $(seq 1 30); do
if rc ls chain/ >/dev/null 2>&1; then break; fi
sleep 1
[[ "$attempt" == 30 ]] && die "RustFS S3 API did not become ready for authenticated bucket listing"
done
rc mb "chain/$RUSTFS_BUCKET" >/dev/null
pass "RustFS up (bucket=$RUSTFS_BUCKET)"
}
resolve_rustfs_endpoint() {
if [[ -n "$RUSTFS_ENDPOINT" ]]; then
log "RustFS endpoint (env): $RUSTFS_ENDPOINT"
else
local net ips ip
net="$(kind_network)"
ips="$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{$v.IPAddress}} {{end}}' "$RUSTFS_CONTAINER")"
ip="$(printf '%s' "$ips" | tr ' ' '\n' | grep -A1 -x "^$net$" | tail -1)"
[[ -n "$ip" ]] || die "could not find the RustFS IP on network $net (inspect: $ips)"
# Container port, not the host publish: kind-network clients hit the
# container IP directly.
RUSTFS_ENDPOINT="http://$ip:$RUSTFS_CONTAINER_PORT"
log "RustFS endpoint (kind network IP): $RUSTFS_ENDPOINT"
fi
local net
net="$(kind_network)"
docker run --rm --network "$net" --user "$(id -u):$(id -g)" -e RC_CONFIG_DIR=/config \
-e HTTP_PROXY= -e HTTPS_PROXY= -e ALL_PROXY= \
-e http_proxy= -e https_proxy= -e all_proxy= \
-v "$RC_CONFIG_DIR:/config" "$RC_IMAGE" alias set kind-rustfs \
"$RUSTFS_ENDPOINT" "$RUSTFS_AK" "$RUSTFS_SK" >/dev/null 2>&1 \
|| die "RustFS unreachable from the kind network at $RUSTFS_ENDPOINT (override RUSTFS_ENDPOINT)"
docker run --rm --network "$net" --user "$(id -u):$(id -g)" -e RC_CONFIG_DIR=/config \
-e HTTP_PROXY= -e HTTPS_PROXY= -e ALL_PROXY= \
-e http_proxy= -e https_proxy= -e all_proxy= \
-v "$RC_CONFIG_DIR:/config" "$RC_IMAGE" ls "kind-rustfs/$RUSTFS_BUCKET" >/dev/null 2>&1 \
|| die "RustFS authenticated bucket access failed from the kind network at $RUSTFS_ENDPOINT"
pass "RustFS reachable from the kind network"
}
# gen_registry builds the agent registry JSON via fast-sandbox's
# registryconfig package; the write pair covers checkpoint publication.
gen_registry() { # host username password endpoint [write-username write-password] > registry.json
mkdir -p "$FSB_GEN_DIR"
cat > "$FSB_GEN_DIR/gen-registry.go" <<'EOF'
package main
import (
"fmt"
"os"
"fast-sandbox/internal/registryconfig"
)
func main() {
if len(os.Args) != 5 && len(os.Args) != 7 {
fmt.Fprintln(os.Stderr, "usage: gen-registry <host> <username> <password> <endpoint> [write-username write-password]")
os.Exit(1)
}
credential := registryconfig.Credential{
Host: os.Args[1], Username: os.Args[2], Password: os.Args[3], Endpoint: os.Args[4],
}
if len(os.Args) == 7 {
// Optional publish (write) pair: empty keeps the store read-only.
credential.WriteUsername, credential.WritePassword = os.Args[5], os.Args[6]
}
compiled, err := registryconfig.NewCompiled([]registryconfig.Credential{credential})
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
payload, err := compiled.Marshal()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
os.Stdout.Write(payload)
}
EOF
(cd "$FSB_DIR" && GOTOOLCHAIN=local go run .fast-sandbox-env-gen/gen-registry.go "$@")
}
credentials_up() {
# Namespaces exist even after a partial up (resume support).
kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
kubectl create namespace "$RESOURCE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
local host
host="${RUSTFS_ENDPOINT#http://}"
host="${host#https://}"
# Publish credentials for the builder Pod (dataplane namespace).
kubectl -n "$RESOURCE_NS" create secret generic sandbox-oss-credentials \
--from-literal=accessKeyId="$RUSTFS_AK" \
--from-literal=secretAccessKey="$RUSTFS_SK" \
--from-literal=endpoint="$RUSTFS_ENDPOINT" \
--from-literal=region=us-east-1 \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
# Agent pull+publish credentials (the write pair covers checkpoints).
gen_registry "$host" "$RUSTFS_AK" "$RUSTFS_SK" "$RUSTFS_ENDPOINT" "$RUSTFS_AK" "$RUSTFS_SK" \
> "$WORK/agent-registry.json"
jq -e '.credentials[0].writeUsername' "$WORK/agent-registry.json" >/dev/null \
|| die "generated agent registry carries no write credential"
kubectl -n "$NS" create secret generic fast-sandbox-agent-registry \
--from-file=registry.json="$WORK/agent-registry.json" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
# Pull credentials for the fastlet; the artifact-store ConfigMap is
# rendered by charts/fast-sandbox at install time.
kubectl -n "$RESOURCE_NS" create secret docker-registry registry-rustfs \
--docker-server="$host" --docker-username="$RUSTFS_AK" --docker-password="$RUSTFS_SK" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
local registries_yaml="registries:
- host: $host
secretRef:
name: registry-rustfs
"
if [[ "$OCI_STACK" == 1 ]]; then
# OverlayBD entries: repositoryPrefix scopes the account to the
# sandbox/* subtree (templates + checkpoints share one repository
# account — the node ublkd has a service-wide credential context).
registries_yaml+=" - host: $OCI_REGISTRY_HOST
repositoryPrefix: ${OCI_REGISTRY_REPO%%/*}
secretRef:
name: $OCI_REGISTRY_SECRET
"
kubectl -n "$RESOURCE_NS" create secret docker-registry "$OCI_REGISTRY_SECRET" \
--docker-server="$OCI_REGISTRY_HOST" \
--docker-username="$OCI_REGISTRY_USER" \
--docker-password="$OCI_REGISTRY_PASSWORD" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
fi
kubectl -n "$RESOURCE_NS" create configmap fast-sandbox-registry \
--from-literal="registries.yaml=$registries_yaml" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
pass "credentials written (publish/pull)"
}
# --- stage: MITM CA material (OSEP-0022 trust anchor) -------------------------
# mitm_ca_fingerprint <cert.pem> -> lowercase hex SHA-256 over the DER — the
# exact fingerprint form the egress /healthz and the template manifest
# lineage expose.
mitm_ca_fingerprint() {
openssl x509 -in "$1" -noout -fingerprint -sha256 2>/dev/null | sed 's/^.*=//' | tr -d ':' | tr 'A-Z' 'a-z'
}
load_mitm_ca_fingerprint() {
if [[ -z "$MITM_CA_FINGERPRINT" && -s "$MITM_CA_FINGERPRINT_FILE" ]]; then
MITM_CA_FINGERPRINT="$(cat "$MITM_CA_FINGERPRINT_FILE")"
fi
}
# Cross-invocation companion of load_mitm_ca_fingerprint: the echo endpoint's
# kind-network IP is a docker-container fact, re-derived (not persisted) so a
# verify action sees the live container rather than a stale address.
load_mitm_echo_endpoint() {
MITM_ECHO_NET="$(kind_network 2>/dev/null || true)"
MITM_ECHO_IP="$(docker inspect -f \
'{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' \
"$MITM_ECHO_CONTAINER" 2>/dev/null || true)"
[[ -n "$MITM_ECHO_IP" ]] \
|| die "TLS echo container $MITM_ECHO_CONTAINER is not running (run up first; mitm_echo_up starts it)"
}
mitm_ca_up() {
local stored_fp openssl_err
openssl_err="$LOGS_DIR/mitm-ca-openssl.err"
mkdir -p "$MITM_CA_DIR"
# Toolchain evidence: the chain guard depends on the host openssl, so a
# failure must always carry the version + stderr (never swallowed).
openssl version >> "$LOGS_DIR/environment.txt" 2>&1 || true
log "openssl: $(command -v openssl) ($(openssl version 2>&1))"
# Fresh material every run: the env owns no CA state worth carrying
# across runs (the template that bakes it is rebuilt each time anyway);
# down() removes the directory so a stale CA can never resurface.
rm -f "$MITM_CA_DIR"/tls.crt "$MITM_CA_DIR"/tls.key "$MITM_CA_DIR"/tls.srl \
"$MITM_CA_DIR"/ca.cnf "$MITM_CA_DIR"/echo.crt "$MITM_CA_DIR"/echo.key "$MITM_CA_DIR"/echo.csr "$MITM_CA_DIR"/echo.ext \
"$MITM_CA_FINGERPRINT_FILE"
# Versioned, immutable-by-convention CA (the v1 of opensandbox-mitm-ca-*):
# a real signing CA (keyCertSign), what the egress seed validator demands.
# Extensions ride an explicit config file (-config + -extensions), not
# -addext: the runner-family openssl builds seen in CI (1.1.1k-FIPS)
# have verifier quirks, so generation sticks to the most portable form
# (config sections work identically from 1.0.2 through 3.x). Every
# openssl call captures stderr into $openssl_err so a failure carries
# the toolchain's own diagnosis, not just our message.
cat > "$MITM_CA_DIR/ca.cnf" <<CNF
[req]
distinguished_name = dn
prompt = no
[dn]
CN = $MITM_CA_SECRET
[v3_ca]
basicConstraints = critical, CA:TRUE
keyUsage = critical, keyCertSign, cRLSign
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer
[v3_echo]
subjectAltName = DNS:$MITM_ECHO_DOMAIN
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid, issuer
CNF
# -days 30 (not 2): the verify actions support running against an
# environment created earlier, so short-lived material would time-bomb
# every TLS handshake there. up regenerates fresh material per run
# anyway; the window only has to outlive delayed verify runs.
if ! openssl req -x509 -new -newkey rsa:2048 -nodes -days 30 \
-keyout "$MITM_CA_DIR/tls.key" -out "$MITM_CA_DIR/tls.crt" \
-config "$MITM_CA_DIR/ca.cnf" -extensions v3_ca 2>"$openssl_err"; then
die "pinned MITM CA generation failed (openssl $(openssl version 2>&1)): $(tail -3 "$openssl_err" 2>/dev/null | tr '\n' ' ')"
fi
# Generator self-check: a build that silently drops the CA constraint
# would seed cleanly and fail every downstream TLS handshake with an
# opaque issuer-lookup error — refuse it here, with the evidence.
if ! openssl x509 -in "$MITM_CA_DIR/tls.crt" -noout -text 2>/dev/null | grep -q "CA:TRUE"; then
die "generated CA certificate lacks BasicConstraints CA:TRUE (openssl $(openssl version 2>&1) misbehaved): $(openssl x509 -in "$MITM_CA_DIR/tls.crt" -noout -text 2>/dev/null | grep -A1 -E 'Basic Constraints|Key Usage' | tr '\n' ' ')"
fi
log "pinned CA extensions: $(openssl x509 -in "$MITM_CA_DIR/tls.crt" -noout -text 2>/dev/null | grep -A1 -E 'Basic Constraints|Key Usage' | tr '\n' ';' | tr -s ' ')"
MITM_CA_FINGERPRINT="$(mitm_ca_fingerprint "$MITM_CA_DIR/tls.crt")"
[[ "$MITM_CA_FINGERPRINT" =~ ^[0-9a-f]{64}$ ]] \
|| die "generated MITM CA fingerprint is malformed: $MITM_CA_FINGERPRINT"
printf '%s' "$MITM_CA_FINGERPRINT" > "$MITM_CA_FINGERPRINT_FILE"
# Leaf certificate for the TLS echo endpoint, signed by the pinned CA —
# both trust legs of the chain test terminate at it (the guest trusts it
# through the MITM-signed certificate, mitmdump trusts it upstream).
if ! openssl req -new -newkey rsa:2048 -nodes \
-keyout "$MITM_CA_DIR/echo.key" -out "$MITM_CA_DIR/echo.csr" \
-subj "/CN=$MITM_ECHO_DOMAIN" 2>>"$openssl_err"; then
die "echo certificate request generation failed: $(tail -3 "$openssl_err" 2>/dev/null | tr '\n' ' ')"
fi
if ! openssl x509 -req -in "$MITM_CA_DIR/echo.csr" \
-CA "$MITM_CA_DIR/tls.crt" -CAkey "$MITM_CA_DIR/tls.key" -CAcreateserial \
-out "$MITM_CA_DIR/echo.crt" -days 30 \
-extfile "$MITM_CA_DIR/ca.cnf" -extensions v3_echo 2>>"$openssl_err"; then
die "echo certificate signing failed: $(tail -3 "$openssl_err" 2>/dev/null | tr '\n' ' ')"
fi
# Structural chain check (issuer == CA subject): deliberately NOT
# `openssl verify`, whose trust-store behavior diverges across the
# openssl builds seen on runners (1.1.1k-FIPS failed a textbook-valid
# chain with error 20). The chain itself is genuinely verified later at
# runtime — the guest curls without -k against the baked CA, and
# mitmdump verifies the upstream against the pinned CA — so this early
# check only needs to catch a mis-signed leaf, which a DN comparison
# does on every openssl version.
local leaf_issuer ca_subject
leaf_issuer="$(openssl x509 -in "$MITM_CA_DIR/echo.crt" -noout -issuer 2>>"$openssl_err" | sed 's/^issuer=//')"
ca_subject="$(openssl x509 -in "$MITM_CA_DIR/tls.crt" -noout -subject 2>>"$openssl_err" | sed 's/^subject=//')"
[[ -n "$leaf_issuer" && "$leaf_issuer" == "$ca_subject" ]] \
|| die "echo certificate issuer does not match the pinned CA subject: issuer='$leaf_issuer' subject='$ca_subject'"
# Ops-runbook distribution, collapsed into one namespace here (the pool
# and template-build namespaces coincide): the keypair entries feed the
# egress seed mount, the certificate-only entry (ca.crt) feeds the server
# template bake. A real deployment keeps the certificate-only copy in the
# template namespace and never carries the key.
kubectl -n "$RESOURCE_NS" create secret generic "$MITM_CA_SECRET" \
--from-file=tls.crt="$MITM_CA_DIR/tls.crt" \
--from-file=tls.key="$MITM_CA_DIR/tls.key" \
--from-file=ca.crt="$MITM_CA_DIR/tls.crt" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
# Content, not name: the stored certificate must fingerprint identically
# (the same-named-copy-wrong-content drift the design calls out).
stored_fp="$(kubectl -n "$RESOURCE_NS" get secret "$MITM_CA_SECRET" \
-o jsonpath='{.data.tls\.crt}' | base64 -d | mitm_ca_fingerprint /dev/stdin)"
[[ "$stored_fp" == "$MITM_CA_FINGERPRINT" ]] \
|| die "Secret $MITM_CA_SECRET certificate fingerprint mismatch: stored=$stored_fp pinned=$MITM_CA_FINGERPRINT"
pass "pinned MITM CA provisioned ($MITM_CA_SECRET, fingerprint ${MITM_CA_FINGERPRINT:0:16}…)"
}
# --- stage: TLS echo endpoint (certificate signed by the pinned CA) -----------
# The echo reflects the request headers it observed as the response body —
# the injected credential header must appear there, which is the per-subject
# injection assertion. Runs as a plain container on the kind network (same
# shape as RustFS): reuses the locally built egress image (python3) so no
# new image dependency is introduced.
mitm_echo_up() {
local net script probe
net="$(kind_network)"
mkdir -p "$GEN_DIR"
script="$GEN_DIR/mitm-echo-server.py"
cat > "$script" <<'PY'
import http.server
import json
import ssl
import sys
port, cert, key = int(sys.argv[1]), sys.argv[2], sys.argv[3]
class Handler(http.server.BaseHTTPRequestHandler):
def _echo(self):
body = json.dumps({k.lower(): v for k, v in self.headers.items()}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
do_GET = _echo
do_POST = _echo
def log_message(self, *args):
pass
httpd = http.server.ThreadingHTTPServer(("0.0.0.0", port), Handler)
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
ctx.load_cert_chain(cert, key)
httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
httpd.serve_forever()
PY
docker rm -f "$MITM_ECHO_CONTAINER" >/dev/null 2>&1 || true
# --entrypoint: the egress image's supervisor ENTRYPOINT must not wrap
# this process; root (image default) binds :443.
docker run -d --name "$MITM_ECHO_CONTAINER" --network "$net" \
--entrypoint python3 \
-v "$MITM_CA_DIR/echo.crt":/etc/mitm-echo/tls.crt:ro \
-v "$MITM_CA_DIR/echo.key":/etc/mitm-echo/tls.key:ro \
-v "$script":/srv/mitm-echo-server.py:ro \
"$IMG_EGRESS" /srv/mitm-echo-server.py "$MITM_ECHO_PORT" \
/etc/mitm-echo/tls.crt /etc/mitm-echo/tls.key >/dev/null \
|| die "TLS echo container failed to start"
MITM_ECHO_IP="$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' "$MITM_ECHO_CONTAINER")"
MITM_ECHO_NET="$net"
[[ -n "$MITM_ECHO_IP" ]] || die "could not resolve the echo container IP on network $net"
wait_for "TLS echo endpoint answering on $MITM_ECHO_IP:$MITM_ECHO_PORT" 30 mitm_echo_ready
probe="$(mitm_echo_headers 'X-Mitm-Echo-Probe: env-up')"
# Whitespace-tolerant: the body is python json.dumps output (": " separators).
printf '%s' "$probe" | grep -Eq '"x-mitm-echo-probe":[[:space:]]*"env-up"' \
|| die "echo endpoint did not reflect headers: $probe"
pass "TLS echo endpoint up ($MITM_ECHO_DOMAIN @ $MITM_ECHO_IP:$MITM_ECHO_PORT, cert signed by the pinned CA)"
}
mitm_echo_ready() {
[[ -n "$MITM_ECHO_IP" && -n "$MITM_ECHO_NET" ]] || return 1
mitm_echo_curl >/dev/null 2>&1
}
# mitm_echo_curl [extra-header]: HTTPS probe of the echo endpoint from a
# container ON the kind network (the resolve_rustfs_endpoint precedent) —
# the host->docker-bridge route is not a path this environment relies on
# anywhere else. Runs the locally built egress image's curl, verifying the
# chain against the pinned CA (probe stderr lands in
# $LOGS_DIR/mitm-echo-probe.err for the failure dump).
mitm_echo_curl() {
local extra=()
[[ -n "${1:-}" ]] && extra=(-H "$1")
docker run --rm --network "$MITM_ECHO_NET" \
--entrypoint curl \
-v "$MITM_CA_DIR/tls.crt":/probe/ca.crt:ro \
-e HOME=/tmp \
"$IMG_EGRESS" \
"${extra[@]}" \
-fsS -m 10 --noproxy '*' --cacert /probe/ca.crt \
--resolve "$MITM_ECHO_DOMAIN:$MITM_ECHO_PORT:$MITM_ECHO_IP" \
"https://$MITM_ECHO_DOMAIN:$MITM_ECHO_PORT/echo" 2>"$LOGS_DIR/mitm-echo-probe.err"
}
# mitm_echo_headers [extra-header] -> reflected request headers (JSON body).
mitm_echo_headers() {
mitm_echo_curl "${1:-}"
}
# --- stage: control plane -------------------------------------------------------------------
# helm is only a renderer; the cluster keeps no helm release state, so
# re-runs stay plain idempotent `kubectl apply`.
helm_render() { # release chart ns out [set-args...]
local release="$1" chart="$2" ns="$3" out="$4"
shift 4
helm template "$release" "$chart" --namespace "$ns" "$@" > "$out" \
|| die "helm template $chart failed"
}
apply_ns() { # ns -> ensure the namespace exists (idempotent)
kubectl create namespace "$1" --dry-run=client -o yaml 2>/dev/null |
kubectl apply -f - >/dev/null
}
control_plane_up() {
# Charts are the source of truth (CRDs + RBAC + control plane), replacing
# the fast-sandbox checkout's kustomize applies and env-owned node manifests.
apply_ns "$NS"
helm_render fsb-base "$OSB_ROOT/manifests/charts/base" "$NS" "$GEN_DIR/fsb-base.yaml"
kubectl apply -f "$GEN_DIR/fsb-base.yaml" >/dev/null
helm_render fast-sandbox "$OSB_ROOT/manifests/charts/fast-sandbox" "$NS" \
"$GEN_DIR/fast-sandbox.yaml" \
--set controller.image.repository="$(image_repo "$IMG_CONTROLLER")" \
--set controller.image.tag="$(image_tag "$IMG_CONTROLLER")" \
--set controller.sandboxtemplateBuilderImage="$IMG_BUILDER" \
--set controller.fastletProxyImage="$IMG_FASTLET_PROXY" \
--set janitor.image.repository="$(image_repo "$IMG_JANITOR")" \
--set janitor.image.tag="$(image_tag "$IMG_JANITOR")" \
--set runtime.image.repository="$(image_repo "$IMG_RUNTIME")" \
--set runtime.image.tag="$(image_tag "$IMG_RUNTIME")" \
--set artifactStore.store="s3://$RUSTFS_BUCKET/publish" \
--set artifactStore.endpoint="$RUSTFS_ENDPOINT" \
--set runtime.oci.enabled="$([[ "$OCI_STACK" == 1 ]] && echo true || echo false)"
kubectl apply -f "$GEN_DIR/fast-sandbox.yaml" >/dev/null
local image
for image in "$IMG_CONTROLLER" "$IMG_FASTLET" "$IMG_FASTLET_PROXY" \
"$IMG_JANITOR" "$IMG_RUNTIME" "$IMG_EGRESS"; do
kind load docker-image "$image" --name "$KIND_CLUSTER" >/dev/null
done
wait_for "controller deployment ready" 120 \
kubectl -n "$NS" rollout status deploy/fast-sandbox-controller --timeout=10s
local crd
for crd in sandboxpools sandboxtemplates sandboxes sandboxsnapshots; do
kubectl get crd "$crd.sandbox.fast.io" >/dev/null 2>&1 || die "CRD $crd missing"
done
kubectl get crd batchsandboxes.sandbox.opensandbox.io >/dev/null 2>&1 \
|| die "CRD batchsandboxes.sandbox.opensandbox.io missing"
pass "CRDs + control plane ready (charts @ pinned $(git -C "$FSB_DIR" rev-parse --short HEAD))"
}
# --- stage: firecracker runtime (node readiness + DART P2P) -----------------------------------
runtime_node_labeled() { # node -> 0 when the agent applied both labels + condition
local node="$1"
kubectl get node "$node" -o json 2>/dev/null | jq -e \
--arg fc "$FC_NODE_LABEL" --arg kvm "$KVM_NODE_LABEL" '
(.metadata.labels[$fc] == "true") and
(.metadata.labels[$kvm] == "true") and
([(.status.conditions // [])[]?
| select(.type == "FirecrackerReady" and .status == "True")] | length > 0)
' >/dev/null
}
runtime_pods() {
kubectl -n "$NS" get pods -l component=firecracker-runtime -o jsonpath='{.items[*].metadata.name}' 2>/dev/null
}
dart_roster_ready() { # pod expected-members
local pod="$1" expected="$2" members
members="$(kubectl exec -n "$NS" "$pod" -- sh -c \
'curl -fsS --noproxy "*" http://127.0.0.1:8147/admin/members' 2>/dev/null || true)"
[[ "$(printf '%s' "$members" | grep -o '"id":' | wc -l | tr -d ' ')" == "$expected" ]]
}
runtime_up() {
# DaemonSet + dart Service ship with charts/fast-sandbox; credentials land
# in credentials_up before this stage.
wait_for "firecracker-runtime DaemonSet ready" 120 \
kubectl -n "$NS" rollout status daemonset/firecracker-runtime --timeout=10s
# Positive wiring assertions: DART admin plane up, agent p2pUp=true
# (a missing P2P daemon only degrades pulls to direct S3).
local pod uid node pods
pods="$(runtime_pods)"
for pod in $pods; do
uid="$(kubectl -n "$NS" get pod "$pod" -o jsonpath='{.metadata.uid}')"
node="$(kubectl -n "$NS" get pod "$pod" -o jsonpath='{.spec.nodeName}')"
wait_for "dart admin /healthz on $node" 30 \
kubectl exec -n "$NS" "$pod" -- sh -c \
"curl -fsS --noproxy '*' http://127.0.0.1:8147/healthz | grep -q ok"
wait_for "agent health p2pUp on $node" 30 \
kubectl exec -n "$NS" "$pod" -- sh -c \
"curl -fsS --noproxy '*' --unix-socket /run/fast-sandbox/firecracker/runtime.sock -H 'Content-Type: application/json' -d '{\"podUid\":\"$uid\",\"namespace\":\"$NS\"}' http://firecracker-agent/v1/health | grep -q '\"p2pUp\":true'"
log "dart: $node dart pid=$(kubectl exec -n "$NS" "$pod" -- sh -c 'pgrep -x dart')"
done
# P2P roster: every daemon must see every peer before any pull, so later
# nodes are fed by peers instead of the origin.
local expected_members
expected_members="$(printf '%s' "$pods" | wc -w | tr -d ' ')"
for pod in $pods; do
node="$(kubectl -n "$NS" get pod "$pod" -o jsonpath='{.spec.nodeName}')"
wait_for "dart roster full on $node ($expected_members members)" 90 \
dart_roster_ready "$pod" "$expected_members"
done
# The readiness loop applies labels + FirecrackerReady itself.
for node in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do
# The agent's host-readiness recheck cadence is 5 minutes; when a
# first pass fails (e.g. a slow tool install), the labels land on
# the NEXT pass, so the wait must span one full recheck cycle.
wait_for "node $node labeled + FirecrackerReady" 480 \
runtime_node_labeled "$node"
done
pass "firecracker-runtime healthy + DART roster=$expected_members + nodes FirecrackerReady"
}
# --- stage (opensandbox CLI): SandboxTemplate golden image ---------------------
wait_succeeded() { # description attempts probe probe_failed
local description="$1" attempts="$2" probe="$3" probe_failed="$4" attempt=0
while ! "$probe" >/dev/null 2>&1; do
if "$probe_failed" >/dev/null 2>&1; then
failure_dump "template-failed"
fail "$description (template entered Failed)"
fi
attempt=$((attempt + 1))
if [[ "$attempt" -ge "$attempts" ]]; then
failure_dump "template-timeout"
fail "$description (after $attempts attempts)"
fi
sleep 2
done
pass "$description"
}
# Template build via the CLI (server /templates API behind it): the row is
# projected onto a SandboxTemplate CRD; the build runs in fast-sandbox, so
# the builder image must be in the cluster.
TEMPLATE_ID=""
_template_phase() {
osb_api template get "$TEMPLATE_ID" -o json 2>/dev/null | jq -r '.status.phase // empty'
}
template_succeeded() {
[[ "$(_template_phase)" == "Succeeded" ]]
}
template_failed() {
local phase message
phase="$(_template_phase)"
[[ "$phase" == "Failed" ]] || return 1
message="$(osb_api template get "$TEMPLATE_ID" -o json 2>/dev/null | jq -r '.status.message // empty')"
log "template Failed: ${message:-<no message>}"
return 0
}
template_up() {
log "building the sandboxtemplate-builder image"
if ! docker image inspect "$IMG_OCI_TOOLS" >/dev/null 2>&1; then
log "building shared oci build base ($IMG_OCI_TOOLS)"
# shellcheck disable=SC2086
docker build ${DOCKER_BUILD_FLAGS:-} --quiet -t "$IMG_OCI_TOOLS" \
-f "$FSB_DIR/build/Dockerfile.oci-tools" "$FSB_DIR" >/dev/null \
|| die "oci-tools image build failed"
fi
# shellcheck disable=SC2086
docker build ${DOCKER_BUILD_FLAGS:-} --quiet \
--build-arg OCI_TOOLS_IMAGE="$IMG_OCI_TOOLS" -t "$IMG_BUILDER" \
-f "$FSB_DIR/build/Dockerfile.sandboxtemplate-builder" "$FSB_DIR" >/dev/null \
|| die "sandboxtemplate-builder image build failed"
if [[ "$OCI_STACK" == 1 ]]; then
bake_registry_ca "$IMG_BUILDER"
verify_image_ca_bake "$IMG_BUILDER"
probe_registry_tls
fi
kind load docker-image "$IMG_BUILDER" --name "$KIND_CLUSTER" >/dev/null
local created
local -a tmpl_args=(
template create
--image "$SBX_IMAGE"
--publish "s3://$RUSTFS_BUCKET/publish"
--format native
--resource cpu=1 --resource memory=512Mi --resource disk=2Gi
--warmup-seconds 15
--metadata origin=fast-sandbox-env
)
log "verify: creating the golden-image template via the opensandbox CLI (image=$SBX_IMAGE)"
created="$(osb_api "${tmpl_args[@]}" -o json 2>/dev/null)" \
|| fail "template create failed against $SERVER_URL: $(osb_api "${tmpl_args[@]}" 2>&1 >/dev/null | head -c 400)"
TEMPLATE_ID="$(printf '%s' "$created" | jq -r '.template_id')"
printf '%s' "$TEMPLATE_ID" > "$WORK/template-id"
[[ -n "$TEMPLATE_ID" && "$TEMPLATE_ID" != "null" ]] || fail "template create response carried no templateId"
log "template id=$TEMPLATE_ID"
wait_succeeded "template phase=Succeeded" 300 template_succeeded template_failed
local manifest_ref
manifest_ref="$(osb_api template get "$TEMPLATE_ID" -o json | jq -r '.status.manifest_ref // empty')"
[[ -n "$manifest_ref" ]] || fail "template manifestRef is empty"
log "template manifestRef: $manifest_ref"
# OSEP-0022 CA wiring: the server filled spec.egressMitmCertificate from
# template_mitm_ca_secret (first real exercise of the server-side fill),
# and the builder stamped the baked certificate's fingerprint into the
# published manifest lineage.
local cert_ref lineage_fp
cert_ref="$(kubectl -n "$RESOURCE_NS" get sandboxtemplate "$TEMPLATE_ID" \
-o jsonpath='{.spec.egressMitmCertificate.secretRef.name}' 2>/dev/null)"
[[ "$cert_ref" == "$MITM_CA_SECRET" ]] \
|| fail "template CR carries no egressMitmCertificate secretRef (got '$cert_ref', want $MITM_CA_SECRET)"
local cert_key
cert_key="$(kubectl -n "$RESOURCE_NS" get sandboxtemplate "$TEMPLATE_ID" \
-o jsonpath='{.spec.egressMitmCertificate.key}' 2>/dev/null)"
[[ "$cert_key" == "ca.crt" ]] \
|| fail "template CR egressMitmCertificate.key is '$cert_key', want ca.crt (certificate entry only)"
lineage_fp="$(template_mitm_lineage_fingerprint)"
[[ "$lineage_fp" == "$MITM_CA_FINGERPRINT" ]] \
|| fail "template manifest platform.egressMitmCA.certSha256 is '${lineage_fp:-empty}', want pinned $MITM_CA_FINGERPRINT"
log "template baked CA: secret/$cert_ref (ca.crt), lineage fingerprint ${lineage_fp:0:16}…"
pass "SandboxTemplate Succeeded + artifacts published (via opensandbox CLI)"
return 0
}
# An overlaybd template beside the native one: same inputs, published to
# the in-cluster registry. status.manifestRef is the runnable artifact.
OCI_TEMPLATE_ID=""
_overlaybd_template_phase() {
osb_api template get "$OCI_TEMPLATE_ID" -o json 2>/dev/null | jq -r '.status.phase // empty'
}
_overlaybd_template_failed() {
local phase message
phase="$(_overlaybd_template_phase)"
[[ "$phase" == "Failed" ]] || return 1
message="$(osb_api template get "$OCI_TEMPLATE_ID" -o json 2>/dev/null | jq -r '.status.message // empty')"
log "overlaybd template Failed: ${message:-<no message>}"
return 0
}
template_up_overlaybd() {
log "verify: creating the overlaybd template via the opensandbox CLI (publish=oci://$OCI_REGISTRY_HOST/$OCI_REGISTRY_REPO)"
local created create_err="$WORK/overlaybd-create.err"
local -a ovl_args=(
template create
--image "$SBX_IMAGE"
--publish "oci://$OCI_REGISTRY_HOST/$OCI_REGISTRY_REPO"
--format overlaybd
--resource cpu=1 --resource memory=512Mi --resource disk=2Gi
--warmup-seconds 15
--metadata origin=fast-sandbox-env-oci
)
created="$(osb_api "${ovl_args[@]}" -o json 2>"$create_err")" \
|| fail "overlaybd template create failed against $SERVER_URL: $(head -c 400 "$create_err")"
OCI_TEMPLATE_ID="$(printf '%s' "$created" | jq -r '.template_id')"
[[ -n "$OCI_TEMPLATE_ID" && "$OCI_TEMPLATE_ID" != "null" ]] || fail "overlaybd template create carried no templateId"
printf '%s' "$OCI_TEMPLATE_ID" > "$WORK/overlaybd-template-id"
local attempts=0
while ! _overlaybd_template_phase_is_succeeded; do
if _overlaybd_template_failed; then
failure_dump "overlaybd-template-failed"
fail "overlaybd template entered Failed"
fi
attempts=$((attempts + 1))
# OCI builds run the convertor + a Firecracker cold boot.
[[ "$attempts" -ge 450 ]] && { failure_dump "overlaybd-template-timeout"; fail "overlaybd template not Succeeded after 450 attempts"; }
sleep 2
done
local manifest_ref
manifest_ref="$(osb_api template get "$OCI_TEMPLATE_ID" -o json | jq -r '.status.manifest_ref // empty')"
[[ "$manifest_ref" == oci://* ]] || fail "overlaybd template manifestRef is not an oci:// reference: $manifest_ref"
log "overlaybd template manifestRef: $manifest_ref"
pass "overlaybd SandboxTemplate Succeeded (OCI publication)"
}
_overlaybd_template_phase_is_succeeded() {
[[ "$(_overlaybd_template_phase)" == "Succeeded" ]]
}
# --- stage: OverlayBD OCI registry (CA, image CA-bake, in-cluster registry) ---
# bake_registry_ca retags a built image with the registry CA trusted; the
# upstream Dockerfiles take no CA input. Consumers: fastlet (fsb-vm), the
# firecracker-runtime (ublkd), the sandboxtemplate-builder (push).
bake_registry_ca() { # <image>
local image="$1"
docker image inspect "$image" >/dev/null 2>&1 || die "image $image missing for the registry CA bake"
docker build ${DOCKER_BUILD_FLAGS:-} --quiet \
--build-arg BASE_IMAGE="$image" \
-t "$image" -f "$CA_DIR/Dockerfile.ca-bake" "$CA_DIR" >/dev/null \
|| die "registry CA bake failed for $image"
}
registry_ca_up() {
[[ "$OCI_STACK" == 1 ]] || {
log "OCI stack disabled (OCI_STACK=0); skipping the registry CA"
return 0
}
command -v openssl >/dev/null 2>&1 || die "openssl is required for the OCI registry certificates"
mkdir -p "$CA_DIR"
# The CA persists in $WORK; the serving cert is re-issued per run.
# The CA needs a subjectKeyIdentifier (1.1.1 `req -x509` adds none) and
# no duplicate extensions (host v3_ca section + -addext stacks them);
# either defect makes OpenSSL 3 clients fail chain validation
# (cache_extensions reason(158)). Regenerate on either.
if [[ ! -s "$CA_DIR/ca.crt" || ! -s "$CA_DIR/ca.key" ]] \
|| ! openssl x509 -in "$CA_DIR/ca.crt" -noout -text 2>/dev/null | grep -q "Subject Key Identifier" \
|| [[ "$(openssl x509 -in "$CA_DIR/ca.crt" -noout -text 2>/dev/null | grep -c 'X509v3 Basic Constraints')" -gt 1 ]]; then
if [[ -s "$CA_DIR/ca.crt" ]]; then
log "existing CA is missing a subjectKeyIdentifier or carries duplicate extensions; regenerating"
fi
# Fully explicit config: the host openssl.cnf (Al8) defaults `req -x509`
# to its v3_ca section, and combining it with -addext duplicates
# extensions (Basic Constraints twice) — which OpenSSL 3 clients reject
# at ossl_x509v3_cache_extensions:reason(158), failing every TLS verify
# against the registry. A minimal local config keeps the CA deterministic
# across hosts.
cat > "$CA_DIR/ca.cnf" <<'CA_CNF'
[req]
distinguished_name = dn
prompt = no
[dn]
CN = fast-sandbox-env-registry-ca
[x509_ext]
basicConstraints = critical, CA:TRUE
keyUsage = critical, keyCertSign, cRLSign
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always
CA_CNF
openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \
-config "$CA_DIR/ca.cnf" -extensions x509_ext \
-keyout "$CA_DIR/ca.key" -out "$CA_DIR/ca.crt" >/dev/null 2>&1 \
|| die "registry CA generation failed"
fi
openssl req -newkey rsa:4096 -sha256 -nodes \
-keyout "$CA_DIR/server.key" -out "$CA_DIR/server.csr" \
-subj "/CN=$OCI_REGISTRY_HOST" >/dev/null 2>&1 \
|| die "registry server key generation failed"
cat > "$CA_DIR/server.ext" <<EXT
basicConstraints=CA:FALSE
keyUsage=digitalSignature,keyEncipherment
extendedKeyUsage=serverAuth
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
subjectAltName=DNS:registry,DNS:registry.$RESOURCE_NS,DNS:registry.$RESOURCE_NS.svc,DNS:$OCI_REGISTRY_HOST
EXT
openssl x509 -req -sha256 -days 365 \
-in "$CA_DIR/server.csr" -CA "$CA_DIR/ca.crt" -CAkey "$CA_DIR/ca.key" \
-CAcreateserial -out "$CA_DIR/server.crt" \
-extfile "$CA_DIR/server.ext" >/dev/null 2>&1 \
|| die "registry server certificate signing failed"
cat > "$CA_DIR/Dockerfile.ca-bake" <<'EOF_DOCKERFILE'
ARG BASE_IMAGE
FROM ${BASE_IMAGE}
COPY ca.crt /usr/local/share/ca-certificates/fast-sandbox-env-registry.crt
RUN update-ca-certificates
EOF_DOCKERFILE
# The images are retagged in place; build_images/template_up bake right
# after producing them.
pass "registry CA ready ($OCI_REGISTRY_HOST)"
}
registry_up() {
[[ "$OCI_STACK" == 1 ]] || {
log "OCI stack disabled (OCI_STACK=0); skipping the OCI registry"
return 0
}
docker pull -q "$OCI_REGISTRY_IMAGE" >/dev/null || die "pull $OCI_REGISTRY_IMAGE failed"
docker pull -q "$OCI_HTPASSWD_IMAGE" >/dev/null || die "pull $OCI_HTPASSWD_IMAGE failed"
kind load docker-image "$OCI_REGISTRY_IMAGE" --name "$KIND_CLUSTER" >/dev/null
# Namespace is also ensured by credentials_up; keep this stage self-contained.
kubectl create namespace "$RESOURCE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
# One push+pull account: the node ublkd credential context is service-wide.
docker run --rm --entrypoint htpasswd "$OCI_HTPASSWD_IMAGE" \
-Bbn "$OCI_REGISTRY_USER" "$OCI_REGISTRY_PASSWORD" > "$WORK/registry-htpasswd" \
|| die "htpasswd generation failed"
kubectl -n "$RESOURCE_NS" create secret generic registry-htpasswd \
--from-file=htpasswd="$WORK/registry-htpasswd" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
kubectl -n "$RESOURCE_NS" create secret tls registry-tls \
--cert="$CA_DIR/server.crt" --key="$CA_DIR/server.key" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
cat > "$GEN_DIR/oci-registry.yaml" <<YAML
apiVersion: apps/v1
kind: Deployment
metadata:
name: registry
namespace: $RESOURCE_NS
labels: {app: fast-sandbox-env-registry}
spec:
replicas: 1
selector:
matchLabels: {app: fast-sandbox-env-registry}
template:
metadata:
labels: {app: fast-sandbox-env-registry}
spec:
containers:
- name: registry
image: $OCI_REGISTRY_IMAGE
ports:
- containerPort: 5000
env:
- name: REGISTRY_HTTP_ADDR
value: :5000
- name: REGISTRY_HTTP_TLS_CERTIFICATE
value: /tls/tls.crt
- name: REGISTRY_HTTP_TLS_KEY
value: /tls/tls.key
- name: REGISTRY_AUTH
value: htpasswd
- name: REGISTRY_AUTH_HTPASSWD_REALM
value: fast-sandbox-env-registry
- name: REGISTRY_AUTH_HTPASSWD_PATH
value: /auth/htpasswd
readinessProbe:
# /v2/ answers 401 unauthenticated; TLS + auth reachable is the point.
exec:
command: [/bin/sh, -ec, 'nc -z 127.0.0.1 5000']
periodSeconds: 5
volumeMounts:
- name: tls
mountPath: /tls
readOnly: true
- name: auth
mountPath: /auth
readOnly: true
volumes:
- name: tls
secret: {secretName: registry-tls}
- name: auth
secret: {secretName: registry-htpasswd}
---
apiVersion: v1
kind: Service
metadata:
name: registry
namespace: $RESOURCE_NS
labels: {app: fast-sandbox-env-registry}
spec:
ports:
- name: https
port: 443
targetPort: 5000
selector:
app: fast-sandbox-env-registry
YAML
kubectl apply -f "$GEN_DIR/oci-registry.yaml" >/dev/null
kubectl -n "$RESOURCE_NS" rollout status deploy/registry --timeout=180s >/dev/null \
|| { failure_dump "registry-rollout"; fail "the in-cluster OCI registry did not become ready"; }
verify_registry_trust
pass "in-cluster OCI registry up (https://$OCI_REGISTRY_HOST)"
}
# probe_registry_tls observes the registry from inside a pod running the
# builder image: the served certificate and whether the system bundle
# validates it.
probe_registry_tls() {
local out
out="$(kubectl run registry-tls-probe -n "$RESOURCE_NS" \
--image="$(docker image inspect --format '{{.ID}}' "$IMG_BUILDER")" \
--restart=Never --rm -i -q \
--overrides="{\"spec\":{\"containers\":[{\"name\":\"probe\",\"image\":\"$(docker image inspect --format '{{.ID}}' "$IMG_BUILDER")\",\"command\":[\"/bin/sh\",\"-ec\",\"openssl s_client -connect $OCI_REGISTRY_HOST:443 -servername $OCI_REGISTRY_HOST </dev/null 2>/dev/null > /tmp/served.pem; openssl x509 -in /tmp/served.pem -noout -issuer -subject -dates -fingerprint -sha256; echo verify: \$(openssl verify -CAfile /usr/local/share/ca-certificates/fast-sandbox-env-registry.crt /tmp/served.pem 2>&1); echo curl: \$(curl -sS -o /dev/null -w '%{http_code}' https://$OCI_REGISTRY_HOST/v2/ -u $OCI_REGISTRY_USER:$OCI_REGISTRY_PASSWORD 2>&1 | tail -c 300); wc -c < /etc/ssl/certs/ca-certificates.crt\"],\"securityContext\":{\"privileged\":true}}],\"restartPolicy\":\"Never\"}}" 2>&1 | tail -10)" \
|| log "registry TLS probe pod failed: $out"
log "registry TLS probe (in-pod view):"
printf '%s\n' "$out" | sed 's/^/ | /'
}
# cert_fingerprint prints the sha256 fingerprint of a PEM file.
cert_fingerprint() { # <pem>
openssl x509 -in "$1" -noout -fingerprint -sha256 | cut -d= -f2
}
# verify_registry_trust checks the serving Secret carries exactly the
# certificate signed this run (leaf-to-leaf fingerprints).
verify_registry_trust() {
local signed_fp served_fp
signed_fp="$(cert_fingerprint "$CA_DIR/server.crt")"
served_fp="$(kubectl -n "$RESOURCE_NS" get secret registry-tls \
-o jsonpath='{.data.tls\.crt}' | base64 -d | cert_fingerprint /dev/stdin)"
[[ -n "$served_fp" ]] || { failure_dump "registry-trust"; fail "could not read the registry serving certificate"; }
[[ "$signed_fp" == "$served_fp" ]] \
|| { failure_dump "registry-trust"; fail "registry serving cert mismatch: signed=$signed_fp served=$served_fp"; }
log "signed server cert: $(openssl x509 -in "$CA_DIR/server.crt" -noout -fingerprint -sha256 -dates | tr '\n' ' ')"
pass "registry serving cert matches the env-signed certificate ($signed_fp)"
}
# verify_image_ca_bake checks the CA landed in the image and was merged
# into its system bundle.
verify_image_ca_bake() { # <image>
local ca_fp body
ca_fp="$(cert_fingerprint "$CA_DIR/ca.crt")"
local in_img
in_img="$(docker run --rm --entrypoint sh "$1" -c '
openssl x509 -in /usr/local/share/ca-certificates/fast-sandbox-env-registry.crt -noout -fingerprint -sha256
' 2>/dev/null | cut -d= -f2)"
[[ "$in_img" == "$ca_fp" ]] \
|| { failure_dump "ca-bake"; fail "CA bake mismatch for $1: CA=$ca_fp image=$in_img"; }
body="$(grep -v -- '---' "$CA_DIR/ca.crt" | tr -d '\n')"
# The merged bundle keeps the PEM folded at 64 columns, so compare
# against the flattened bundle, not the raw file.
docker run --rm --entrypoint sh "$1" -c "
tr -d '\n' < /etc/ssl/certs/ca-certificates.crt | grep -c '$body'
" >/dev/null \
|| { failure_dump "ca-bake"; fail "CA not merged into the system bundle of $1 (update-ca-certificates)"; }
pass "CA baked + merged in $1 ($ca_fp)"
}
bake_registry_ca_into_images() {
[[ "$OCI_STACK" == 1 ]] || return 0
log "baking the registry CA into the fsb images"
bake_registry_ca "$IMG_FASTLET"
bake_registry_ca "$IMG_RUNTIME"
verify_image_ca_bake "$IMG_FASTLET"
verify_image_ca_bake "$IMG_RUNTIME"
}
# template_mitm_lineage_fingerprint [template-id] -> the certSha256 the
# builder stamped under platform.egressMitmCA in the published artifact-set
# manifest (empty when the manifest or the entry is missing).
template_mitm_lineage_fingerprint() {
local tid manifest_ref manifest_json
tid="${1:-$TEMPLATE_ID}"
manifest_ref="$(osb_api template get "$tid" -o json 2>/dev/null | jq -r '.status.manifest_ref // empty')"
[[ -n "$manifest_ref" ]] || return 0
manifest_json="$(rc cat "chain/$RUSTFS_BUCKET/$(dirname "${manifest_ref#s3://$RUSTFS_BUCKET/}")/manifest.json" 2>/dev/null)" || return 0
printf '%s' "$manifest_json" | jq -r '.platform.egressMitmCA.certificates[0].certSha256 // empty' 2>/dev/null
}
# --- stage: SandboxPool (egress attached, P2P spread) --------------------------------------------
pool_pods() {
kubectl -n "$RESOURCE_NS" get pods \
-l "app=sandbox-fastlet,fast-sandbox.io/pool=$POOL_NAME" \
-o jsonpath='{.items[*].metadata.name}' 2>/dev/null
}
first_pool_pod() {
kubectl -n "$RESOURCE_NS" get pods \
-l "app=sandbox-fastlet,fast-sandbox.io/pool=$POOL_NAME" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null
}
fastlet_pods_ready() {
local ready=0 pod
for pod in $(pool_pods); do
if kubectl -n "$RESOURCE_NS" get pod "$pod" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -q True; then
ready=$((ready + 1))
fi
done
[[ "$ready" -ge "$POOL_MIN" ]]
}
egress_containers_ready() {
local pods pod
pods="$(pool_pods)"
[[ -n "$pods" ]] || return 1
for pod in $pods; do
kubectl -n "$RESOURCE_NS" get pod "$pod" -o jsonpath='{range .status.containerStatuses[*]}{.name}{"="}{.ready}{" "}{end}' 2>/dev/null \
| grep -q 'egress=true' || return 1
done
}
# GET /_fastlet/v1/actions/status must echo the shared apiVersion, ready=true
# and a non-empty instanceId; the handler binds Pod loopback only, so the
# probe runs inside the egress container.
egress_status_ready() {
local pod out
pod="$(first_pool_pod)"
[[ -n "$pod" ]] || return 1
out="$(kubectl -n "$RESOURCE_NS" exec "pod/$pod" -c egress -- \
curl -fsS -m 5 "http://127.0.0.1:18080/_fastlet/v1/actions/status" 2>/dev/null)" || return 1
[[ "$out" == *'"apiVersion":"sandbox.fast.io/actions/v1"'* && "$out" == *'"ready":true'* && "$out" == *'"instanceId":'* ]]
}
pool_status_ready() {
local ready
ready="$(kubectl_get "sandboxpool/$POOL_NAME" '{.status.readyPods}' 2>/dev/null || true)"
[[ "$ready" =~ ^[0-9]+$ ]] && (( ready >= POOL_MIN ))
}
pool_condition_true() { # condition-type
[[ "$(kubectl_get "sandboxpool/$POOL_NAME" "{.status.conditions[?(@.type==\"$1\")].status}" 2>/dev/null)" == "True" ]]
}
warm_images_ready() {
kubectl -n "$RESOURCE_NS" get sandboxpool "$POOL_NAME" -o jsonpath='{.status.warmImages[*].cachedFastlets}' 2>/dev/null | grep -qv '^0*$'
}
# --- stage: ublk on the host and inside the kind nodes (OCI_STACK=1) ------------
# OverlayBD talks to the HOST kernel's ublk module through /dev/ublkc* (the
# control char device) and surfaces devices as /dev/ublkb<N>. The kernel is
# shared with the kind nodes, but the device NODES are not: a kind node's
# /dev is a private tmpfs populated at node creation, while /dev/ublkc*
# appears on the host only at modprobe time — usually after the cluster
# exists. Every ublkd/fsb-vm consumer (the builder Pod, the node ublkd
# sidecar, fastlets) mounts the kind node's /dev, so the nodes need the nodes.
# --- stage: ublk registration check (OCI_STACK=1) -------------------------------
# Informational registration check only. Naming varies by kernel: the char
# (queue) device class is "ublk-char" (Al8 6.6 backport, /dev/ublkc<N>) or
# "ublkc" (mainline), the control node is /dev/ublk-control (misc) or
# /dev/ublkc, and some kernels register NO block major — ublkb nodes use
# per-disk dynamic majors that ublkd mknods itself. The runner may be a
# container whose /dev hides these nodes; /proc/devices always reflects the
# shared kernel.
ublk_module_up() {
[[ "$OCI_STACK" == 1 ]] || {
log "OCI_STACK=0: skipping the ublk check (overlaybd disabled)"
return 0
}
# The overlaybd stack is unusable without the kernel module: the node
# ublkd sidecar crash-loops without /dev/ublk-control and every build
# fails at its probe device, so fail here instead of ~16 minutes later.
if ! grep -qiE 'ublk-char|ublkc' /proc/devices; then
log "ublk not registered; attempting modprobe ublk_drv"
sudo_ modprobe ublk_drv 2>/dev/null || true
fi
grep -qiE 'ublk-char|ublkc' /proc/devices \
|| die "ublk is not registered in /proc/devices after modprobe: the kernel needs CONFIG_BLK_DEV_UBLK (and the runner must be able to load modules); overlaybd requires it when OCI_STACK=1"
log "ublk registered: $(grep -i ublk /proc/devices | tr '\n' ' ')"
pass "ublk module registered"
}
# --- stage: real ublk add/del probe + wedge self-heal (OCI_STACK=1) --------------
# Registration in /proc/devices is necessary but not sufficient: a run
# killed mid-add leaks kernel ublk devices that outlive the kind cluster
# (the module is never reloaded), and one leaked device wedges every later
# `ublkd add` with the daemon's "device failed to start" — surfacing only
# ~16 minutes into the overlaybd template build. This probe drives the same
# daemon -> kernel path the build uses (POST /v1/add, then POST /v1/del on
# the node ublkd's control socket) with a minimal upper-only device, so a
# wedge surfaces in seconds and the heal below clears it automatically.
#
# The probe runs inside each node's overlaybd-ublkd sidecar: that container
# holds both the control socket and the overlaybd tooling, so the device
# created is exactly the kind a template build creates.
ublk_probe_once() { # pod -> prints PROBE_RESULT=<class> first, details after
local pod="$1"
kubectl exec -i -n "$NS" "$pod" -c overlaybd-ublkd -- sh -s <<'PROBE' 2>&1 || true
sock=/run/fast-sandbox/overlaybd/ublkd.sock
tools=/opt/fast-sandbox/oci/current/bin
if ! ping=$(curl -sS -m 5 --noproxy '*' --unix-socket "$sock" http://ublkd/v1/ping 2>&1); then
echo "PROBE_RESULT=ping-failed"
printf '%s\n' "$ping"
exit 0
fi
d=$(mktemp -d /tmp/fsb-ublk-probe.XXXXXX) || { echo "PROBE_RESULT=inconclusive"; echo "mktemp failed"; exit 0; }
trap 'rm -rf "$d"' EXIT
if ! "$tools/overlaybd-create" --hybrid "$d/data" "$d/index" 1 >/dev/null 2>&1; then
echo "PROBE_RESULT=inconclusive"
echo "overlaybd-create --hybrid failed"
exit 0
fi
printf '{"lowers":[],"upper":{"index":"%s","data":"%s","vsize":1},"resultFile":"%s"}' \
"$d/index" "$d/data" "$d/result" > "$d/config.json"
# No --fail: a daemon rejection still carries the JSON error body this
# probe classifies on; no --write-out either (older curl builds reject
# %{httpcode}, which once turned a real wedge into a false
# "inconclusive"). Only a transport failure lands in the if branch.
if ! body=$(curl -sS -m 120 --noproxy '*' --unix-socket "$sock" \
-X POST -H 'Content-Type: application/json' \
-d "{\"config\":\"$d/config.json\"}" http://ublkd/v1/add 2>&1); then
echo "PROBE_RESULT=inconclusive"
printf '%s\n' "$body"
exit 0
fi
case "$body" in
*'"ok":true'*)
# add succeeded: release the probe device before it leaks — a leaked
# probe device would wedge the very next add, so a del that is not
# explicitly acknowledged by the daemon is reported as wedged (the
# heal recycles the module and clears the leak). curl without --fail
# exits 0 on a daemon rejection too, so the response BODY decides.
dev_id=$(printf '%s' "$body" | tr '{,' '\n\n' | grep -o '"dev_id":[0-9]*' | head -1 | tr -cd '0-9')
if [ -z "$dev_id" ]; then
echo "PROBE_RESULT=inconclusive"
echo "add reply carried no dev_id"
printf '%s\n' "$body"
exit 0
fi
if ! del_reply=$(curl -sS -m 30 --noproxy '*' --unix-socket "$sock" \
-X POST -H 'Content-Type: application/json' \
-d "{\"dev_id\":$dev_id}" http://ublkd/v1/del 2>&1); then
del_reply="transport failure: $del_reply"
fi
case "$del_reply" in
*'"ok":true'*)
echo "PROBE_RESULT=ok dev_id=$dev_id"
;;
*)
echo "PROBE_RESULT=wedged"
echo "add succeeded but the probe device release failed (dev_id=$dev_id): $del_reply"
;;
esac
;;
*"device failed to start"*)
echo "PROBE_RESULT=wedged"
printf '%s\n' "$body"
;;
*)
echo "PROBE_RESULT=inconclusive"
printf '%s\n' "$body"
;;
esac
PROBE
}
ublk_heal_wedge() {
log "heal: recycling the host ublk module (leaked devices die with it)"
# The sidecar daemons hold /dev/ublk-control open, which pins the
# module against modprobe -r: drop the runtime pods first. The
# DaemonSet starts replacements right away, but they stay in
# scheduling/init for a few seconds before any daemon re-opens the
# control node — so retry the unload across that window. Success
# proves no process held it; every later open lands on the fresh
# module. A failed recycle is not fatal here: the re-probe decides,
# and its failure carries the explicit reboot guidance.
kubectl -n "$NS" delete pod -l component=firecracker-runtime --now --timeout=90s >/dev/null 2>&1 || true
local attempt unloaded=0
for attempt in 1 2 3 4 5; do
if sudo_ modprobe -r ublk_drv 2>/dev/null; then unloaded=1; break; fi
sleep 2
done
sudo_ modprobe ublk_drv 2>/dev/null || true
grep -qiE 'ublk-char|ublkc' /proc/devices \
|| die "ublk module did not register after the recycle: the runner host needs a reboot to clear the wedged ublk state"
if [[ "$unloaded" == 1 ]]; then
log "heal: ublk module recycled clean on attempt ${attempt}"
else
log "heal: modprobe -r stayed busy (a daemon re-opened the control node first); the re-probe will tell"
fi
kubectl -n "$NS" rollout status daemonset/firecracker-runtime --timeout=300s >/dev/null
log "heal: runtime pods ready"
}
ublk_probe_or_heal() {
[[ "$OCI_STACK" == 1 ]] || { log "OCI_STACK=0: skipping the ublk probe (overlaybd disabled)"; return 0; }
local pods pod result detail healed recheck reok
pods="$(runtime_pods 2>/dev/null || true)"
[[ -n "$pods" ]] || die "no firecracker-runtime pods to probe (run up first)"
healed=0
for pod in $pods; do
detail="$(ublk_probe_once "$pod")"
result="$(printf '%s\n' "$detail" | awk '/^PROBE_RESULT=/ && !found {sub(/^PROBE_RESULT=/, ""); print; found=1}')"
case "$result" in
ok*)
pass "ublk probe ok on ${pod}: node-side add/del through the real daemon -> kernel path"
;;
wedged)
printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
if [[ "$healed" == 1 ]]; then
fail "ublk probe still wedged on ${pod} after the heal: the runner host's ublk state needs a reboot to recover"
fi
log "ublk probe WEDGED on ${pod}: the daemon cannot start devices; healing"
ublk_heal_wedge
healed=1
reok=1
# The heal deleted the pods; the DaemonSet recreated them under
# new names, so re-query before the re-probe — the stale list
# would NotFound on every pod and fail a healthy recovery.
pods="$(runtime_pods 2>/dev/null || true)"
[[ -n "$pods" ]] || fail "no firecracker-runtime pods after the heal"
# The restart reset every daemon, so re-probe all pods.
for recheck in $pods; do
detail="$(ublk_probe_once "$recheck")"
result="$(printf '%s\n' "$detail" | awk '/^PROBE_RESULT=/ && !found {sub(/^PROBE_RESULT=/, ""); print; found=1}')"
if [[ "$result" == ok* ]]; then
pass "ublk probe ok after heal on ${recheck}"
else
reok=0
log "ublk probe after heal on ${recheck}: ${result:-no-result}"
printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
fi
done
[[ "$reok" == 1 ]] || fail "ublk probe still unhealthy after the heal (probe output above)"
;;
ping-failed)
printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
fail "ublk probe: the daemon on ${pod} is unreachable on its control socket (runtime not healthy)"
;;
*)
# An unexpected add response means the probe config hit a
# validation path, not the kernel wedge; never block a
# working stack on it — the build reports the real error.
log "WARNING: ublk probe inconclusive on ${pod} (result: ${result:-none}); continuing"
printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
;;
esac
done
}
render_pool() { # > $GEN_DIR/firecracker-egress-pool.yaml
local src="$MANIFESTS_DIR/pool/firecracker-egress-pool.yaml" out="$GEN_DIR/firecracker-egress-pool.yaml"
mkdir -p "$GEN_DIR"
# Quoted tokens in the template keep rendered scalars' natural types
# (poolMin stays an integer); awk for sed portability.
awk -v fastlet="$IMG_FASTLET" -v egress="$IMG_EGRESS" \
-v pool_min="$POOL_MIN" -v pool_max="$POOL_MAX" \
-v max_per_pod="$MAX_SANDBOXES_PER_POD" \
-v warm="$WARM_IMAGES" -v image="$TEMPLATE_ID" \
-v mitm_ca="$MITM_CA_SECRET" '
{ gsub(/"@FASTLET_IMAGE@"/, fastlet)
gsub(/"@EGRESS_IMAGE@"/, egress)
gsub(/"@POOL_MIN@"/, pool_min)
gsub(/"@POOL_MAX@"/, pool_max)
gsub(/"@MITM_CA_SECRET@"/, mitm_ca)
gsub(/maxSandboxesPerPod: 5/, "maxSandboxesPerPod: " max_per_pod) }
/^# @WARM_IMAGES@$/ {
if (warm == "1") printf " warmImages:\n - %s\n", image
next }
{ print }
' "$src" > "$out"
# OverlayBD fragment: the OCI flag appends to the fastlet's env list and
# the shared node mounts (tools read-only, the ublkd control socket, the
# host /dev so ublk devices reach the VMs) append to its volumeMounts —
# two separate tokens so neither YAML list gains a duplicate key. The
# registry credentials ride the pool controller's compiled
# fast-sandbox-registry projection (env + volume injected by the
# controller), not this manifest.
if [[ "$OCI_STACK" == 1 ]]; then
local oci_env oci_mounts volumes
oci_env="$(cat <<'ENVS'
- name: FAST_SANDBOX_OCI_ENABLED
value: "true"
ENVS
)"
oci_mounts="$(cat <<'MOUNTS'
- name: oci-tools
mountPath: /opt/fast-sandbox/oci
readOnly: true
- name: oci-socket
mountPath: /run/fast-sandbox/overlaybd
- name: host-dev
mountPath: /dev
MOUNTS
)"
volumes="$(cat <<'VOLUMES'
- name: oci-tools
hostPath:
path: /opt/fast-sandbox/oci
type: DirectoryOrCreate
- name: oci-socket
hostPath:
path: /run/fast-sandbox/overlaybd
type: DirectoryOrCreate
- name: host-dev
hostPath:
path: /dev
type: Directory
VOLUMES
)"
# POSIX-safe token substitution: BSD awk rejects newline-bearing -v
# values, so the fragments travel through ENVIRON instead.
oci_env="$oci_env" oci_mounts="$oci_mounts" oci_volumes="$volumes" awk '
/^# @OCI_FASTLET_ENV@$/ { printf "%s\n", ENVIRON["oci_env"]; next }
/^# @OCI_FASTLET_MOUNTS@$/ { printf "%s\n", ENVIRON["oci_mounts"]; next }
/^# @OCI_FASTLET_VOLUMES@$/ { printf "%s\n", ENVIRON["oci_volumes"]; next }
{ print }
' "$out" > "$out.tmp" && mv "$out.tmp" "$out"
else
grep -v '^# @OCI_FASTLET_' "$out" > "$out.tmp" && mv "$out.tmp" "$out"
fi
# Leftover-token guard: match only value positions (key: ...@T@...),
# never the header comments that document the tokens themselves.
if grep -Eq '^[[:space:]]*[A-Za-z][A-Za-z0-9]*:.*@[A-Z_]+@' "$out"; then
die "unrendered token left in $out"
fi
}
pool_up() {
load_mitm_ca_fingerprint
[[ -n "$MITM_CA_FINGERPRINT" ]] || die "no pinned MITM CA fingerprint at $MITM_CA_FINGERPRINT_FILE (run up first; mitm_ca_up stores it there)"
render_pool
log "applying SandboxPool $POOL_NAME (runtime=firecracker, egress attached, MITM CA pinned, poolMin=$POOL_MIN)"
kubectl apply -f "$GEN_DIR/firecracker-egress-pool.yaml" >/dev/null
wait_for "fastlet pods Ready (poolMin=$POOL_MIN)" 300 fastlet_pods_ready
wait_for "egress container ready in every fastlet pod" 180 egress_containers_ready
wait_for "pool condition RuntimeReady=True" 120 pool_condition_true RuntimeReady
wait_for "pool condition InfraReady=True" 120 pool_condition_true InfraReady
wait_for "egress actions endpoint answering (/_fastlet/v1/actions/status)" 60 egress_status_ready
wait_for "egress MITM addon up with the pinned CA fingerprint" 120 egress_mitm_ca_pinned
if [[ "$WARM_IMAGES" == "1" ]]; then
wait_for "pool warmImages Ready" 300 warm_images_ready
p2p_evidence "warm preheat"
pass "fastlet Running + warmImages Ready (P2P evidence captured)"
else
pass "fastlet Running, on-demand pull (default: first sandbox pulls through DART)"
fi
}
# egress_mitm_ca_fingerprint -> the pinned CA fingerprint the egress exposes
# on /healthz (empty while the MITM addon is down/pending or not yet up).
egress_mitm_ca_fingerprint() {
local pod out
pod="$(first_pool_pod)"
[[ -n "$pod" ]] || return 1
out="$(kubectl -n "$RESOURCE_NS" exec "pod/$pod" -c egress -- \
curl -fsS -m 5 "http://127.0.0.1:18080/healthz" 2>/dev/null)" || return 1
printf '%s' "$out" | sed -n 's/.*"mitmCaFingerprint":"\([0-9a-f]*\)".*/\1/p'
}
# Content pinning (OSEP-0022): the egress seed's fingerprint must equal the
# pinned CA's — a same-named Secret copy carrying different (but valid)
# material fails here loudly instead of degrading into opaque guest TLS
# failures.
egress_mitm_ca_pinned() {
local fp
fp="$(egress_mitm_ca_fingerprint)"
[[ -n "$fp" && "$fp" == "$MITM_CA_FINGERPRINT" ]]
}
# Assert the P2P outcome from DART block counters: the artifact set is
# pulled ~once per 4MiB block from the origin cluster-wide, and extra nodes
# must be fed by peers, not the origin.
p2p_evidence() { # description
local description="$1"
local pods pod manifest_ref manifest_key build_dir expected_blocks=0 stat_json
local origin_total=0 peer_total=0 cache_total=0 size source value active_nodes=0 node_total
manifest_ref="$(osb_api template get "$TEMPLATE_ID" -o json 2>/dev/null | jq -r '.status.manifest_ref // empty')"
manifest_key="${manifest_ref#s3://$RUSTFS_BUCKET/}"
build_dir="$(dirname "$manifest_key")"
local object
for object in rootfs.ext4 vmstate.snap memory.snap; do
stat_json="$(rc stat --json "chain/$RUSTFS_BUCKET/$build_dir/$object" 2>&1)" \
|| die "rc stat failed for published $object (publish incomplete?): $stat_json"
size="$(jq -er '.size_bytes | numbers' <<<"$stat_json" 2>/dev/null)" \
|| die "rc stat returned no numeric size_bytes for published $object"
expected_blocks=$((expected_blocks + (size + 4194303) / 4194304))
done
pods="$(runtime_pods)"
for pod in $pods; do
node_total=0
while read -r source value; do
case "$source" in
origin) origin_total=$((origin_total + value)); node_total=$((node_total + value)) ;;
peer) peer_total=$((peer_total + value)); node_total=$((node_total + value)) ;;
cache) cache_total=$((cache_total + value)); node_total=$((node_total + value)) ;;
esac
done < <(dart_source_counters "$pod")
[[ "$node_total" -gt 0 ]] && active_nodes=$((active_nodes + 1))
done
log "p2p evidence ($description): expected origin=$expected_blocks blocks; cluster origin=$origin_total peer=$peer_total cache=$cache_total active-nodes=$active_nodes"
[[ "$origin_total" -ge "$expected_blocks" ]] || fail "cluster origin $origin_total < expected $expected_blocks blocks"
[[ "$origin_total" -le $((expected_blocks + 4)) ]] \
|| fail "origin amplified: $origin_total > $((expected_blocks + 4)): pulls were not deduplicated by DART"
if [[ "$active_nodes" -ge 2 ]]; then
[[ "$peer_total" -gt 0 ]] || fail "no peer traffic across $active_nodes nodes: the second node was not served by the peer"
pass "P2P evidence ($description): origin ~1 fetch per block (cluster=$origin_total/$expected_blocks), peer=$peer_total, nodes=$active_nodes"
else
pass "P2P evidence ($description): origin ~1 fetch per block (cluster=$origin_total/$expected_blocks) on $active_nodes node (no peer needed)"
fi
}
dart_source_counters() { # pod -> "<source> <value>" lines
local pod="$1"
kubectl exec -n "$NS" "$pod" -- sh -c 'curl -fsS --noproxy "*" http://127.0.0.1:8147/metrics' 2>/dev/null \
| awk '/^dart_block_source_total\{source="(cache|peer|origin)"\}/ {
match($0, /source="[^"]+"/); s = substr($0, RSTART + 8, RLENGTH - 9)
match($0, /} [0-9]+$/); print s, substr($0, RSTART + 2)
}'
}
# --- stage: idle-pause Redis (OSEP-0024 profile) --------------------------------
redis_up() {
# The sweeper connects at startup, so Redis must precede the server.
[[ "$IDLE_PAUSE" == "1" ]] || { log "idle-pause profile off (IDLE_PAUSE=0); skipping Redis"; rm -f "$WORK/idle-pause"; return 0; }
touch "$WORK/idle-pause"
mkdir -p "$GEN_DIR"
sed "s|image: redis:7-alpine|image: $REDIS_IMAGE|" \
"$MANIFESTS_DIR/idle-pause/redis.yaml" > "$GEN_DIR/idle-pause-redis.yaml"
kubectl apply -n "$OSB_NS" -f "$GEN_DIR/idle-pause-redis.yaml" >/dev/null
wait_for "idle-pause Redis deployment ready" 120 \
kubectl -n "$OSB_NS" rollout status deploy/idle-pause-redis --timeout=10s
pass "idle-pause Redis up ($REDIS_DSN)"
}
# --- stage: OpenSandbox server + ingress gateway --------------------------------
# Per-workdir key (not per-run): re-applies must keep old routes verifiable.
osb_signing_key() {
if [[ ! -s "$SIGNING_KEY_FILE" ]]; then
openssl rand -base64 32 | tr -d '\n' > "$SIGNING_KEY_FILE"
fi
cat "$SIGNING_KEY_FILE"
}
# Server config.toml (fsb runtime) with workdir tokens; the gateway [ingress]
# block is appended by charts/server from server.gateway.* values.
render_server_config() { # > $GEN_DIR/osb-server-config.toml
mkdir -p "$GEN_DIR"
awk -v api_key="$SERVER_API_KEY" \
-v fastpath="$FASTPATH_ENDPOINT" -v fsb_ns="$RESOURCE_NS" -v pool="$POOL_NAME" \
-v execd="$EXECD" -v execd_init="$EXECD_RUN_AS_INIT" \
-v oci_secret="$OCI_REGISTRY_SECRET" -v oci_stack="$OCI_STACK" \
-v mitm_ca="$MITM_CA_SECRET" '
{ if ($0 == "# @OCI_PUBLISH_SECRET@") {
# emitted in-place so the key stays inside [kubernetes].
if (oci_stack == "1")
printf "template_oci_publish_secret = \"%s\"\n", oci_secret
next }
gsub(/@SERVER_API_KEY@/, api_key)
gsub(/@SIGNING_KEY@/, signing_key)
gsub(/@FASTPATH_ENDPOINT@/, fastpath)
gsub(/@FSB_NAMESPACE@/, fsb_ns)
gsub(/@POOL_NAME@/, pool)
gsub(/@EXECD_IMAGE@/, execd)
gsub(/@EXECD_RUN_AS_INIT@/, execd_init)
gsub(/@MITM_CA_SECRET@/, mitm_ca)
gsub(/@GATEWAY_ADDRESS@/, gateway)
print }
' <<'TOML' > "$GEN_DIR/osb-server-config.toml"
[server]
host = "0.0.0.0"
port = 80
api_key = "@SERVER_API_KEY@"
[log]
level = "INFO"
[runtime]
type = "kubernetes"
execd_image = "@EXECD_IMAGE@"
# OSEP-0018: bake EXECD_INIT=1 into templates; pinned by the init-mode verify.
execd_run_as_init = @EXECD_RUN_AS_INIT@
[kubernetes]
# One block for CR reads + fsb settings; sandboxes live in the pool's
# namespace (poolRef resolves) and get the execd image injected by the server.
namespace = "@FSB_NAMESPACE@"
fastpath_endpoint = "@FASTPATH_ENDPOINT@"
fastpath_resource_pool = "@POOL_NAME@"
fastpath_wait_ready_seconds = 30.0
template_s3_publish_secret = "sandbox-oss-credentials"
# @OCI_PUBLISH_SECRET@
# OSEP-0022 MITM trust anchor: server-created SandboxTemplates bake
# spec.egressMitmCertificate from the pinned CA Secret (certificate entry
# only, never the private key), so the builder installs it into the guest
# trust store at build time.
template_mitm_ca_secret = "@MITM_CA_SECRET@"
template_mitm_ca_secret_key = "ca.crt"
TOML
if [[ "$IDLE_PAUSE" == "1" ]]; then
# Dotted keys nest the redis sub-table.
cat >> "$GEN_DIR/osb-server-config.toml" <<TOML
[automation.idle_pause]
enabled = true
sweep_interval_seconds = 5
activity_ttl_seconds = 1800
redis.dsn = "$REDIS_DSN"
redis.lock_ttl_seconds = 30
TOML
fi
if grep -Eq '@[A-Z_]+@' "$GEN_DIR/osb-server-config.toml"; then
die "unrendered token left in $GEN_DIR/osb-server-config.toml"
fi
}
opensandbox_up() {
kind load docker-image "$IMG_SERVER" --name "$KIND_CLUSTER" >/dev/null
kind load docker-image "$IMG_INGRESS" --name "$KIND_CLUSTER" >/dev/null
render_server_config
apply_ns "$OSB_NS"
# charts/server has the fsb RBAC built in; the config carries the wiring.
helm_render opensandbox-server "$OSB_ROOT/manifests/charts/server" "$OSB_NS" \
"$GEN_DIR/osb-server.yaml" \
--set server.image.repository="$(image_repo "$IMG_SERVER")" \
--set server.image.tag="$(image_tag "$IMG_SERVER")" \
--set server.service.type=NodePort \
--set server.service.nodePort="$SERVER_NODEPORT" \
--set server.resources.requests.cpu=250m \
--set server.resources.requests.memory=512Mi \
--set server.resources.limits.cpu=1 \
--set server.resources.limits.memory=2Gi \
--set-file configToml="$GEN_DIR/osb-server-config.toml" \
--set server.gateway.enabled=true \
--set server.gateway.host="$GATEWAY_ADDRESS" \
--set server.gateway.gatewayRouteMode=header \
--set server.gateway.secureAccess.activeKey=a \
--set "server.gateway.secureAccess.keys[0].key_id=a" \
--set "server.gateway.secureAccess.keys[0].key=$(osb_signing_key)"
kubectl apply -f "$GEN_DIR/osb-server.yaml" >/dev/null
# fast-sandbox provider via FastPath, same signing key as the server.
local gateway_idle_args=()
if [[ "$IDLE_PAUSE" == "1" ]]; then
gateway_idle_args=(
--set gateway.activity.enabled=true
--set gateway.activity.redisDsn="$REDIS_DSN"
--set gateway.activity.minInterval=5s
--set gateway.activity.ttlSeconds=1800
--set gateway.wake.enabled=true
--set gateway.wake.parkBudget="${IDLE_PAUSE_PARK_BUDGET:-5s}"
)
fi
helm_render opensandbox-ingress-gateway "$OSB_ROOT/manifests/charts/ingress-gateway" "$OSB_NS" \
"$GEN_DIR/osb-ingress-gateway.yaml" \
--set gateway.image.repository="$(image_repo "$IMG_INGRESS")" \
--set gateway.image.tag="$(image_tag "$IMG_INGRESS")" \
--set gateway.replicaCount=1 \
--set gateway.providerType=fast-sandbox \
--set gateway.dataplaneNamespace="$NS" \
--set gateway.fastpathEndpoint="$FASTPATH_ENDPOINT" \
--set "gateway.secureAccess.keys[0].key_id=a" \
--set "gateway.secureAccess.keys[0].key=$(osb_signing_key)" \
--set gateway.service.type=NodePort \
--set gateway.service.nodePort="$GATEWAY_NODEPORT" \
--set gateway.resources.requests.cpu=100m \
--set gateway.resources.requests.memory=128Mi \
--set gateway.resources.limits.cpu=1 \
--set gateway.resources.limits.memory=1Gi \
"${gateway_idle_args[@]}"
kubectl apply -f "$GEN_DIR/osb-ingress-gateway.yaml" >/dev/null
wait_for "server deployment ready" 180 \
kubectl -n "$OSB_NS" rollout status deploy/opensandbox-server --timeout=10s
wait_for "ingress gateway deployment ready" 180 \
kubectl -n "$OSB_NS" rollout status deploy/opensandbox-ingress-gateway --timeout=10s
# A ready gateway proves FastPath reachability: it fails startup without
# the gRPC connection.
wait_for "server /health on 127.0.0.1:$SERVER_HOST_PORT" 60 \
curl -fsS -m 5 "$SERVER_URL/health"
wait_for "gateway /status.ok on $GATEWAY_ADDRESS" 60 \
curl -fsS -m 5 --noproxy "*" "$GATEWAY_URL/status.ok"
pass "server + ingress gateway up (fsb runtime, gateway routes signed with key 'a')"
}
# Raw curl, kept for the calls the CLI cannot express (networkpolicy
# replace PUT, metadata merge-patch, snapshot re-entry fence).
server_api() { # method path [json-body]
local method="$1" path="$2" body="${3:-}"
if [[ -n "$body" ]]; then
curl -fsS -m 60 -X "$method" -H "OPEN-SANDBOX-API-KEY: $SERVER_API_KEY" \
-H "Content-Type: application/json" -d "$body" "$SERVER_URL$path"
else
curl -fsS -m 60 -X "$method" -H "OPEN-SANDBOX-API-KEY: $SERVER_API_KEY" "$SERVER_URL$path"
fi
}
# opensandbox CLI pointed at the lifecycle server; --no-color keeps
# -o json parseable.
osb_api() {
"$OSB_BIN" --no-color --domain "127.0.0.1:$SERVER_HOST_PORT" --protocol http \
--api-key "$SERVER_API_KEY" --request-timeout 60 "$@"
}
# The server assigns the sandbox id (CreateSandboxRequest carries none);
# the verify probes share it through VERIFY_ID.
VERIFY_ID=""
# The snapshot verify stage shares the created snapshot ids through
# SNAPSHOT_ID / SNAPSHOT_ID2.
SNAPSHOT_ID=""
SNAPSHOT_ID2=""
# A re-entry snapshot accepted during fence cache lag (202) is tracked here
# so its terminal outcome is asserted and it is cleaned up.
SNAPSHOT_EXTRA=""
verify_sandbox_gone() {
! osb_api sandbox get "$VERIFY_ID" -o json >/dev/null 2>&1
}
verify_policy_enforced() {
local out
out="$(server_api GET "/sandboxes/$VERIFY_ID/networkpolicy" 2>/dev/null)" || return 1
[[ "$(printf '%s' "$out" | jq -r '.mode // empty')" == "enforcing" ]] || return 1
[[ "$(printf '%s' "$out" | jq -r '.policy.egress[0].target // empty')" == "example.com" ]]
}
verify_policy_updated() {
local out
out="$(server_api GET "/sandboxes/$VERIFY_ID/networkpolicy" 2>/dev/null)" || return 1
[[ "$(printf '%s' "$out" | jq -r '.mode // empty')" == "enforcing" ]] || return 1
[[ "$(printf '%s' "$out" | jq -r '.policy.egress[0].target // empty')" == "github.com" ]]
}
# Full wire-up: raw-API create (fsb) -> FastPath -> fastlet -> firecracker
# sandbox -> signed gateway route -> fastlet-proxy -> guest execd /ping ->
# delete. One sandbox per call; availability is measured from the client
# (poll /ping until 200, wall-clock budget — every poll is curl-only, so
# readings resolve ~10ms), not from CR status. The timed windows wrap HTTP
# round trips only: osb invocations would bill ~0.5s of interpreter startup
# to each reading, which is exactly the measurement pollution #2323 called
# out. CLI create coverage lives in the pause-resume/vault/snapshot verifies
# and the lifecycle ops below.
verify_one_sandbox() { # <label> <ping-budget-ms> [template-id]
local label="$1" budget="$2" template_id="${3:-$TEMPLATE_ID}"
local response t0 t1 t2 route code deadline payload
local policy_file="$GEN_DIR/verify-policy.json"
t0="$(now_ms)"
# Egress policy rides on every create (binding -> nft rules), so the
# chain is exercised each time; /ping is inbound and unaffected.
jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "example.com"}, {action: "allow", target: "*.opensandbox.ai"}]}' \
> "$policy_file"
log "verify ($label): creating a sandbox via the raw API (templateId=$template_id; the timed window is curl-only)"
# The timed windows wrap HTTP round trips ONLY: an osb invocation here
# would bill ~0.5s of Python startup to every reading. The CLI surface
# stays covered by the lifecycle ops and the osb create smoke.
local payload
payload="$(jq -n --arg tpl "$template_id" --slurpfile pol "$policy_file" \
'{templateId: $tpl, timeout: 3600, metadata: {origin: "fast-sandbox-env-verify"}, networkPolicy: $pol[0]}')"
response="$(server_api POST "/sandboxes" "$payload")" \
|| fail "POST /sandboxes failed against $SERVER_URL (payload: $(printf '%s' "$payload" | head -c 200))"
VERIFY_ID="$(printf '%s' "$response" | jq -r '.id')"
[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"
t1="$(now_ms)"
# Availability = /ping 200 through the signed gateway route (port 44772),
# measured at ~1ms resolution:
# phase 1 — route publication, curl-only busy poll (the curl spawn
# itself paces ~5ms/attempt; /ping cannot 200 before the route exists,
# so phase-1 cadence delays detection, never pollutes the reading);
# phase 2 — /ping hammered over a bash /dev/tcp socket: no process
# spawn per attempt, EPOCHREALTIME keeps the hot loop fork-free.
local gw gw_host gw_port line deadline_ms
gw="${GATEWAY_URL#*://}"; gw="${gw%%/*}"
gw_host="${gw%%:*}"
gw_port="${gw##*:}"
[[ "$gw_port" == "$gw" ]] && gw_port=80
deadline_ms=$(( t1 / 1000000 + budget ))
while :; do
route="$( { server_api GET "/sandboxes/$VERIFY_ID/endpoints/44772" 2>/dev/null || true; } \
| jq -r '.headers["OpenSandbox-Ingress-To"] // empty')"
[[ -n "$route" ]] && break
if (( $(now_ms) / 1000000 >= deadline_ms )); then
fail "gateway route for $VERIFY_ID did not publish within ${budget}ms"
fi
done
code=""
while :; do
line=""
# NOTE: no 2>/dev/null here — exec-without-command applies
# redirections to the CURRENT shell, so that would permanently
# swallow the script's stderr (every later log/pass goes dark).
# A refused connect prints one honest bash message instead.
if exec 3<>"/dev/tcp/$gw_host/$gw_port"; then
printf 'GET /ping HTTP/1.1\r\nHost: %s\r\nOpenSandbox-Ingress-To: %s\r\nConnection: close\r\n\r\n' \
"$gw" "$route" >&3
IFS= read -r -t 1 line <&3 || true
exec 3>&- 3<&-
else
sleep 0.01
fi
if [[ "$line" == *" 200 "* ]]; then
code=200
break
fi
if [[ -n "${EPOCHREALTIME:-}" ]]; then
now=$((( ${EPOCHREALTIME/./} ) / 1000))
else
now=$(( $(now_ms) / 1000000 ))
fi
if (( now >= deadline_ms )); then
fail "execd /ping did not return 200 within ${budget}ms (last status: ${line:-none})"
fi
done
t2="$(now_ms)"
# jq filter in a variable: literal parens trip older bash's $( ) parser.
local state raw raw_filter
raw_filter='{rt:.status.runtime.state,dp:.status.dataPlane.state,infra:[.status.infraComponents[]?|{n:.name,s:.state}],bind:[.status.actionBindings[]?|{h:.handler,s:.state}],ready:(.status.conditions[]?|select(.type=="Ready")|.status)}'
state="$(osb_api sandbox get "$VERIFY_ID" -o json 2>/dev/null | jq -r '.status.state // empty')"
raw="$(kubectl -n "$RESOURCE_NS" get sandbox "$VERIFY_ID" -o json 2>/dev/null | jq -c "$raw_filter")"
log "verify ($label): $VERIFY_ID access via ingress gateway: curl -H \"OpenSandbox-Ingress-To: $route\" $GATEWAY_URL/ping"
log "verify ($label): $VERIFY_ID create $(( (t1 - t0) / 1000000 ))ms, create->execd /ping 200 $(( (t2 - t1) / 1000000 ))ms, total $(( (t2 - t0) / 1000000 ))ms"
log "verify ($label): CR at ping: server=$state raw=${raw:-unreachable}"
pass "execd /ping 200 through the signed gateway route (44772)"
wait_for "egress policy enforcing (networkPolicy -> egress action binding -> nft)" 120 verify_policy_enforced
# Policy UPDATE path (raw PUT: the CLI only has merge-style patches).
local put_body
put_body="$(jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "github.com"}]}')"
server_api PUT "/sandboxes/$VERIFY_ID/networkpolicy" "$put_body" >/dev/null \
|| fail "PUT networkpolicy failed for $VERIFY_ID: $(printf '%s' "$put_body" | head -c 200)"
wait_for "policy update converged (PUT -> ReplaceActionBindings -> egress)" 120 verify_policy_updated
# Re-sample the CR state: an early Failed observation may converge late.
local state_final
state_final="$(osb_api sandbox get "$VERIFY_ID" -o json 2>/dev/null | jq -r '.status.state // empty')"
log "verify ($label): $VERIFY_ID CR state final: ${state_final:-unknown}"
}
# Remaining lifecycle surface on one sandbox: get, list, metadata merge-patch
# (raw API — no CLI command), renew via the CLI (TTL-based).
verify_lifecycle_ops() { # <sandbox-id>
local id="$1" out expected_expires renewed_expires
out="$(osb_api sandbox get "$id" -o json 2>/dev/null)" || fail "osb sandbox get $id failed"
[[ "$(printf '%s' "$out" | jq -r '.id')" == "$id" ]] || fail "GET returned wrong id: $out"
pass "lifecycle: osb sandbox get"
osb_api sandbox list --page 1 --page-size 50 -o json > "$WORK/last-list.json" 2>"$WORK/last-list.err" \
|| fail "osb sandbox list failed: $(head -c 400 "$WORK/last-list.err" 2>/dev/null)"
out="$(cat "$WORK/last-list.json")"
[[ "$(printf '%s' "$out" | jq -r --arg id "$id" '.items[]?.id | select(. == $id)' | head -1)" == "$id" ]] \
|| fail "list does not contain $id: $(printf '%s' "$out" | jq -c '.pagination')"
pass "lifecycle: osb sandbox list (contains the verify sandbox)"
# JSON Merge Patch (RFC 7396): non-null upserts, null deletes.
server_api PATCH "/sandboxes/$id/metadata" '{"env":"verify","stage":"lifecycle-ops"}' >/dev/null \
|| fail "PATCH metadata upsert failed"
out="$(osb_api sandbox get "$id" -o json 2>/dev/null)"
[[ "$(printf '%s' "$out" | jq -r '.metadata.env')" == "verify" \
&& "$(printf '%s' "$out" | jq -r '.metadata.stage')" == "lifecycle-ops" ]] \
|| fail "metadata upsert not visible: $(printf '%s' "$out" | jq -c '.metadata')"
server_api PATCH "/sandboxes/$id/metadata" '{"stage":null}' >/dev/null \
|| fail "PATCH metadata delete failed"
out="$(osb_api sandbox get "$id" -o json 2>/dev/null)"
[[ "$(printf '%s' "$out" | jq -r '.metadata.stage')" == "null" \
&& "$(printf '%s' "$out" | jq -r '.metadata.env')" == "verify" ]] \
|| fail "metadata delete not visible: $(printf '%s' "$out" | jq -c '.metadata')"
pass "lifecycle: PATCH metadata (upsert + null-delete via JSON Merge Patch)"
# CLI renew takes a TTL: assert the expiration advanced and is visible.
expected_expires="$(osb_api sandbox get "$id" -o json 2>/dev/null | jq -r '.expires_at')"
out="$(osb_api sandbox renew "$id" --timeout 2h -o json 2>"$WORK/last-renew.err")" \
|| fail "osb sandbox renew failed: $(head -c 400 "$WORK/last-renew.err" 2>/dev/null)"
renewed_expires="$(printf '%s' "$out" | jq -r '.expires_at')"
[[ -n "$renewed_expires" && "$renewed_expires" != "null" && "$renewed_expires" != "$expected_expires" ]] \
|| fail "renewed expiresAt did not advance: $expected_expires -> $renewed_expires"
local got_expires
got_expires="$(osb_api sandbox get "$id" -o json 2>/dev/null | jq -r '.expires_at')"
# Same instant, two renderings (renew panel str() vs RFC3339 dump):
# compare epoch seconds, falling back to string equality.
if [[ "$got_expires" != "$renewed_expires" ]] \
&& [[ "$(date -u -d "$renewed_expires" +%s 2>/dev/null || true)" \
!= "$(date -u -d "$got_expires" +%s 2>/dev/null || true)" ]]; then
fail "renewed expiresAt not visible: expected $renewed_expires got $got_expires"
fi
pass "lifecycle: osb sandbox renew 2h ($expected_expires -> $renewed_expires)"
}
# dump_latency_evidence collects the per-create timing instrumentation from
# every component on the create critical path into one greppable file:
# fastlet phase breakdown (fastlet create timing:), fastlet->egress action
# invocations (fastlet action invoke:), egress sidecar SET_BINDING / hook
# steps (timing fields on its subject lines), and fastpath create totals.
# Append-only evidence for create-latency analysis (issue #2323 style).
dump_latency_evidence() {
local out="$LOGS_DIR/latency-evidence-$(date +%s).txt"
{
echo "=== latency evidence $(date -u +%FT%TZ) ==="
echo
echo "--- fastlet: create phase + action-invoke timing ---"
kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet --tail=3000 2>/dev/null \
| grep -E "fastlet create timing|fastlet action invoke" || true
echo
echo "--- egress sidecar: SET_BINDING / hook timing ---"
kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet -c egress --tail=3000 2>/dev/null \
| grep -E "deny-first enforced|SET_BINDING applied|data-plane-ready" || true
echo
echo "--- server: fsb create served (authoritative control-plane timing) ---"
kubectl -n "$OSB_NS" logs deploy/opensandbox-server --tail=3000 2>/dev/null \
| grep "fsb create served" | tail -30 || true
echo
echo "--- fastpath create totals (controller) ---"
kubectl -n "$NS" logs deploy/fast-sandbox-controller --tail=3000 2>/dev/null \
| grep "fastpath sandbox created" | tail -30 || true
} > "$out" 2>&1 || true
log "latency evidence: $out"
# CI runners discard $WORK between runs, so the content must also land
# in the job log to be analyzable.
cat "$out"
}
opensandbox_verify() {
# Optional template override: the overlaybd second pass calls this with
# the OCI template id; the default stays the native template. Cold create
# pulls the golden image through DART (native) or the registry over ublk
# (overlaybd); warm creates: the second may still pull on the other node
# via its DART peer; after both nodes cache the set, creates are
# sub-second (native) / attach cached devices (overlaybd).
local template_id="${1:-$TEMPLATE_ID}"
log "verify (end-to-end): template=$template_id"
local ids=() id label index
verify_one_sandbox "cold #1" 600000 "$template_id"
ids+=("$VERIFY_ID")
verify_one_sandbox "warm #2" 600000 "$template_id"
ids+=("$VERIFY_ID")
for index in 3 4 5 6; do
verify_one_sandbox "warm #$index" 120000 "$template_id"
ids+=("$VERIFY_ID")
done
pass "end-to-end: opensandbox CLI -> server -> FastPath -> fastlet -> sandbox execd -> gateway route OK (6 sandboxes)"
dump_latency_evidence
verify_lifecycle_ops "${ids[0]}"
for id in "${ids[@]}"; do
VERIFY_ID="$id"
osb_api sandbox kill "$id" >/dev/null 2>&1 \
|| log "verify cleanup: kill failed; remove $id manually"
done
for id in "${ids[@]}"; do
VERIFY_ID="$id"
wait_for "verify sandbox $id deleted" 120 verify_sandbox_gone
done
pass "verify sandboxes cleaned up"
}
# --- stage: pause / resume (opensandbox CLI -> FastPath checkpoint) -------------
# Fresh signed route + one GET: doubles as the "runtime actually serving" probe.
execd_ping_ok() {
local route code
route="$(osb_api sandbox endpoint "$VERIFY_ID" --port 44772 -o json 2>/dev/null \
| jq -r '.headers["OpenSandbox-Ingress-To"] // empty')"
[[ -n "$route" ]] || return 1
code="$(curl -sS -m 5 -o /dev/null -w '%{http_code}' \
-H "OpenSandbox-Ingress-To: $route" "$GATEWAY_URL/ping" 2>/dev/null || true)"
[[ "$code" == "200" ]]
}
sandbox_state_is() { # <state>
[[ "$(osb_api sandbox get "$VERIFY_ID" -o json 2>/dev/null | jq -r '.status.state // empty')" == "$1" ]]
}
sandbox_running() { sandbox_state_is Running; }
sandbox_paused() { sandbox_state_is Paused; }
sandbox_state_get() {
osb_api sandbox get "$VERIFY_ID" -o json 2>/dev/null | jq -r '.status.state // empty'
}
pause_resume_verify() {
# Optional template override (the overlaybd second pass); default native.
# Accepted + poll: Paused == checkpoint durable + capacity released;
# resume advances the route generation, so /ping needs a fresh route.
local template_id="${1:-$TEMPLATE_ID}"
local created t0 t1 t2 t3
log "verify (pause): creating a sandbox via the opensandbox CLI (templateId=$template_id)"
t0="$(now_ms)"
created="$(osb_api sandbox create --template "$template_id" --timeout 1h \
--metadata origin=fast-sandbox-env-pause --skip-health-check -o json 2>/dev/null)" \
|| fail "osb sandbox create failed against $SERVER_URL"
VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"
[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"
wait_for "pause target Running" 150 sandbox_running
t1="$(now_ms)"
log "verify (pause): $VERIFY_ID create->Running $(( (t1 - t0) / 1000000 ))ms"
wait_for "pre-pause execd /ping 200 through the gateway" 100 execd_ping_ok
log "verify (pause): osb sandbox pause $VERIFY_ID"
osb_api sandbox pause "$VERIFY_ID" >/dev/null 2>&1 \
|| fail "pause failed for $VERIFY_ID"
# Paused is durable-first: the checkpoint must be complete in the artifact
# store before the state is reported; the dump itself keeps serving.
wait_for "poll GET until Paused (checkpoint durable, capacity released)" 240 sandbox_paused
t2="$(now_ms)"
log "verify (pause): $VERIFY_ID pause->Paused (checkpoint durable) $(( (t2 - t1) / 1000000 ))ms"
if execd_ping_ok; then
fail "paused sandbox still serves /ping through the gateway"
fi
pass "paused: execd /ping no longer served (runtime released, signed route rejected at the gateway)"
log "verify (pause): osb sandbox resume $VERIFY_ID"
t2="$(now_ms)"
osb_api sandbox resume "$VERIFY_ID" --skip-health-check >/dev/null 2>&1 \
|| fail "resume failed for $VERIFY_ID"
wait_for "poll GET until Running (checkpoint restored)" 240 sandbox_running
t3="$(now_ms)"
log "verify (pause): $VERIFY_ID resume->Running (checkpoint restored) $(( (t3 - t2) / 1000000 ))ms"
wait_for "post-resume execd /ping 200 through a fresh route" 100 execd_ping_ok
pass "pause/resume round-trip timings: pause->Paused $(( (t2 - t1) / 1000000 ))ms, resume->Running $(( (t3 - t2) / 1000000 ))ms (opensandbox CLI -> FastPath -> artifact store)"
osb_api sandbox kill "$VERIFY_ID" >/dev/null 2>&1 \
|| log "verify cleanup: kill failed; remove $VERIFY_ID manually"
wait_for "pause/resume sandbox deleted" 120 verify_sandbox_gone
}
# --- stage: execd-as-init in-guest process verify (OSEP-0018) -------------------
# NOTE: verify/init-mode.sh carries the split-layout copy of this stage; keep
# the two in sync until the env.sh split is completed.
# init_mode_entrypoint_probe: guest-side probe printing TAIL_PPID=<ppid> for
# the baked entrypoint workload. Selection is the exact comm name (pgrep -x,
# never -f) plus the exact argv, so the probe's own sh -c shell — whose
# cmdline embeds this very text and whose PPid is execd — can never
# self-match, and a same-name process with different args is skipped; it
# exits nonzero when no matching workload exists.
init_mode_entrypoint_probe='
for p in $(pgrep -x tail); do
argv=$(tr "\0" "\n" </proc/$p/cmdline)
[ "$argv" = "$(printf "tail\n-f\n/dev/null")" ] || continue
echo "TAIL_PPID=$(grep "^PPid:" /proc/$p/status | grep -o "[0-9]*")"
exit 0
done
echo "init-mode probe: no tail -f /dev/null workload process in the guest" >&2
exit 1
'
# guest_out <words...>: run a command in the sandbox via the CLI.
guest_out() {
osb_api command run "$VERIFY_ID" -- "$@" 2>"$LOGS_DIR/init-mode-cmd.err"
}
# guest_value <key> <words...>: extract the guest's KEY=VALUE line. The full
# guest_out output is captured and its exit status checked before parsing, so
# a CLI failure after the target line cannot satisfy the caller's probe.
guest_value() { # <key> <probe-words...>
local key="$1" out status line
shift
out="$(guest_out "$@")"
status=$?
[[ "$status" -eq 0 ]] || return "$status"
while IFS= read -r line || [[ -n "$line" ]]; do
case "$line" in
"$key="*) printf '%s\n' "${line#*=}"; return 0 ;;
esac
done <<<"$out"
return 1
}
init_mode_verify() { # [template-id]
local template_id="${1:-$TEMPLATE_ID}"
local created execd_pid probe
log "verify (init-mode): creating a sandbox via the opensandbox CLI (templateId=$template_id)"
created="$(osb_api sandbox create --template "$template_id" --timeout 1h \
--metadata origin=fast-sandbox-env-init-mode -o json 2>/dev/null)" \
|| fail "osb sandbox create failed against $SERVER_URL"
VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"
[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"
wait_for "init-mode target Running" 600 sandbox_running
# Full process table first: ground truth + failure evidence.
log "verify (init-mode): in-guest process table (pid ppid pgid sid stat args):"
guest_out sh -c 'ps -eo pid,ppid,pgid,sid,stat,args || ps -ef' \
| tee "$LOGS_DIR/init-mode-processes.txt" >&2 \
|| log "verify (init-mode): process table dump failed (continuing; the probes below carry the diagnostics)"
# 1. PID 1 is the guest init.
probe="$(guest_value PID1 sh -c 'echo "PID1=$(tr "\0" " " < /proc/1/cmdline)"')" \
|| fail "init-mode: /proc/1/cmdline probe failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
case "$probe" in
*sandbox-init*) pass "guest PID 1 is the fsb guest init ($probe)" ;;
*) fail "guest PID 1 is not the guest init: /proc/1/cmdline = $probe" ;;
esac
# 2. execd armed with --init.
execd_pid="$(guest_value EXECD_PID sh -c 'echo "EXECD_PID=$(pgrep -x execd | head -1)"')" \
|| fail "init-mode: execd pgrep probe failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
case "$execd_pid" in
''|*[!0-9]*) fail "execd process not found in the guest (EXECD_PID='$execd_pid')" ;;
esac
probe="$(guest_value INIT_FLAG sh -c "tr \"\\0\" \"\\n\" < /proc/$execd_pid/cmdline | grep -qx -- --init && echo INIT_FLAG=1")" \
|| fail "init-mode: execd cmdline probe failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
[[ "$probe" == "1" ]] \
|| fail "execd (pid $execd_pid) does not carry --init (cmdline probe output: $probe)"
pass "execd runs with --init (pid $execd_pid)"
# 3. execd is PID 1's child (bootstrap exec'd away).
probe="$(guest_value EXECD_PPID sh -c "echo \"EXECD_PPID=\$(grep '^PPid:' /proc/$execd_pid/status | grep -o '[0-9]*')\"")" \
|| fail "init-mode: execd PPid probe failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
[[ "$probe" == "1" ]] \
|| fail "execd (pid $execd_pid) is not a child of PID 1: PPid=$probe"
pass "execd is a direct child of the guest init (PPid=1)"
# 4. The entrypoint is execd's child. Exact name + argv selection: a
# fuzzy pgrep -f would match the probe's own sh -c shell (the search
# text sits in its cmdline) and pass vacuously on its execd PPid.
probe="$(guest_value TAIL_PPID sh -c "$init_mode_entrypoint_probe")" \
|| fail "init-mode: entrypoint probe failed (no tail -f /dev/null workload in the guest?): $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
[[ "$probe" == "$execd_pid" ]] \
|| fail "the entrypoint is not a child of execd (pid $execd_pid): entrypoint PPid=$probe"
pass "the baked entrypoint is supervised by execd (entrypoint PPid=$execd_pid)"
# 5. The orphan reparents to execd (subreaper); left to exit for #6.
guest_out sh -c 'sleep 6 >/dev/null 2>&1 & echo $! > /tmp/osb-init-mode-orphan.pid' >/dev/null \
|| fail "init-mode: orphan spawn failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
probe="$(guest_value ORPHAN_PPID sh -c 'p=$(cat /tmp/osb-init-mode-orphan.pid); echo "ORPHAN_PPID=$(grep "^PPid:" /proc/$p/status | grep -o "[0-9]*")"')" \
|| fail "init-mode: orphan PPid probe failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
[[ "$probe" == "$execd_pid" ]] \
|| fail "the orphan did not reparent to execd (pid $execd_pid): orphan PPid=$probe"
pass "orphaned processes reparent to execd (subreaper; orphan PPid=$execd_pid)"
# 6. execd reaps the exited orphan (else it stays a Z child forever).
init_mode_orphan_reaped() {
[[ "$(guest_value REAPED sh -c 'p=$(cat /tmp/osb-init-mode-orphan.pid 2>/dev/null); if [ -n "$p" ] && [ ! -e "/proc/$p" ]; then echo "REAPED=1"; else echo "REAPED=0"; fi')" == "1" ]]
}
wait_for "execd reaps the exited orphan (pid gone from /proc)" 20 init_mode_orphan_reaped
guest_out sh -c 'rm -f /tmp/osb-init-mode-orphan.pid' >/dev/null 2>&1 || true
# 7. No zombies left.
probe="$(guest_value ZOMBIES sh -c 'echo "ZOMBIES=$(ps -eo stat | grep -c "^Z" || true)"')" \
|| fail "init-mode: zombie count probe failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
[[ "$probe" == "0" ]] \
|| fail "the guest process table carries zombies: $probe in Z state"
pass "guest process table is zombie-free after the orphan round trip"
# 8. Report: subreaper tier, no signal shield. Capture the full response
# and check the producer status before scanning for .hardening.
local hardening_line capabilities_out
capabilities_out="$(guest_out curl -fsS http://127.0.0.1:44772/v1/isolated/capabilities 2>/dev/null)" \
|| fail "init-mode: capabilities probe failed: $(head -c 300 "$LOGS_DIR/init-mode-cmd.err" 2>/dev/null)"
hardening_line="$(printf '%s' "$capabilities_out" | while IFS= read -r line || [[ -n "$line" ]]; do
printf '%s' "$line" | jq -ce '.hardening // empty' 2>/dev/null && break
done)" \
|| fail "init-mode: capabilities probe failed (no .hardening in the response)"
log "verify (init-mode): hardening report: $hardening_line"
[[ "$(printf '%s' "$hardening_line" | jq -r '.init_mode')" == "subreaper" ]] \
|| fail "hardening.init_mode is not subreaper: $hardening_line"
[[ "$(printf '%s' "$hardening_line" | jq -r '.signal_shield')" == "false" ]] \
|| fail "hardening.signal_shield is not false (fsb guest init owns PID 1): $hardening_line"
pass "capabilities: hardening.init_mode=subreaper, signal_shield=false (fsb/Pool tier; no isolation TOML, floor layers stay disabled)"
osb_api sandbox kill "$VERIFY_ID" >/dev/null 2>&1 \
|| log "verify cleanup: kill failed; remove $VERIFY_ID manually"
wait_for "init-mode sandbox deleted" 120 verify_sandbox_gone
}
# --- stage: vault lifecycle (OSEP-0022: egress-held vault over the relay) --------
# Status-code-returning variant of server_api for negative-path assertions
# (curl -f hides the status).
server_api_status() { # method path [json-body]
local method="$1" path="$2" body="${3:-}" out
if [[ -n "$body" ]]; then
out="$(curl -sS -m 60 -X "$method" -H "OPEN-SANDBOX-API-KEY: $SERVER_API_KEY" \
-H "Content-Type: application/json" -d "$body" -w '\n%{http_code}' "$SERVER_URL$path" 2>/dev/null)"
else
out="$(curl -sS -m 60 -X "$method" -H "OPEN-SANDBOX-API-KEY: $SERVER_API_KEY" \
-w '\n%{http_code}' "$SERVER_URL$path" 2>/dev/null)"
fi
printf '%s' "${out##*$'\n'}"
}
# The vault stage's sandbox must ride the subject-scoped egress log markers
# (export/import/registration), so every log assertion goes through this.
vault_log_contains() { # <fixed-substring>
kubectl -n "$RESOURCE_NS" logs -l app=sandbox-fastlet -c egress --tail=800 2>/dev/null \
| grep -qF "$1"
}
vault_revision_is() { # <expected-revision>
local out
out="$(server_api GET "/sandboxes/$VERIFY_ID/credential-vault" 2>/dev/null)" || return 1
[[ "$(printf '%s' "$out" | jq -r '.revision // empty')" == "$1" ]]
}
vault_lifecycle_verify() {
local template_id="${1:-$TEMPLATE_ID}"
local created vault_body ack rev sandbox_uid
log "verify (vault): creating a sandbox via the opensandbox CLI (templateId=$template_id)"
# The policy rides the create: a policy-less create produces no egress
# binding at all, so the subject would never register.
jq -n '{defaultAction: "deny", egress: [{action: "allow", target: "ext.test"}]}' \
> "$GEN_DIR/vault-policy.json"
created="$(osb_api sandbox create --template "$template_id" --timeout 1h \
--metadata origin=fast-sandbox-env-vault \
--network-policy-file "$GEN_DIR/vault-policy.json" \
--skip-health-check -o json 2>/dev/null)" \
|| fail "vault verify: sandbox create failed"
VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"
wait_for "vault sandbox Running" 150 sandbox_running
sandbox_uid="$(kubectl -n "$RESOURCE_NS" get sandbox "$VERIFY_ID" -o jsonpath='{.metadata.uid}' 2>/dev/null)"
[[ -n "$sandbox_uid" ]] || fail "vault verify: sandbox CR carried no uid"
# Registered and activated before the vault POST, else the relay 404s.
# (The "(state=active)" re-delivery only follows a policy UPDATE; a
# create-carried policy activates without it.)
wait_for "vault subject registered on egress" 120 \
vault_log_contains "subject s-$sandbox_uid: data-plane-ready, policy active"
# 1) create-only: ack carries the revision; duplicate POST is 409.
vault_body="$(jq -n '{credentials: [{name: "tok", source: {type: "inline", value: "sekrit-vault-e2e"}}], bindings: [{name: "ext", match: {hosts: ["ext.test"]}, auth: {type: "apiKey", name: "X-Api-Key", credential: "tok"}}]}')"
ack="$(server_api POST "/sandboxes/$VERIFY_ID/credential-vault" "$vault_body")" \
|| fail "vault verify: POST credential-vault failed"
rev="$(printf '%s' "$ack" | jq -r '.revision // empty')"
[[ "$rev" =~ ^[0-9]+$ ]] || fail "vault verify: create ack carried no revision: $(printf '%s' "$ack" | head -c 200)"
pass "vault created over the relay (ack revision $rev)"
if server_api POST "/sandboxes/$VERIFY_ID/credential-vault" "$vault_body" >/dev/null 2>&1; then
fail "vault verify: duplicate POST succeeded (create-only must 409)"
fi
[[ "$(server_api_status POST "/sandboxes/$VERIFY_ID/credential-vault" "$vault_body")" == "409" ]] \
|| fail "vault verify: duplicate POST was not 409"
pass "duplicate POST rejected (create-only 409)"
# 2) pause/resume carries the real vault: export -> bundle -> import,
# with the relay GET reporting the same revision.
osb_api sandbox pause "$VERIFY_ID" >/dev/null 2>&1 || fail "vault verify: pause failed"
wait_for "vault sandbox Paused (checkpoint durable)" 240 sandbox_paused
wait_for "vault exported into the checkpoint bundle" 120 \
vault_log_contains "subject s-$sandbox_uid: vault exported to snapshot bundle"
pass "pause exported the vault bundle (subject sealed)"
osb_api sandbox resume "$VERIFY_ID" --skip-health-check >/dev/null 2>&1 || fail "vault verify: resume failed"
wait_for "vault sandbox Running (restored)" 240 sandbox_running
# The pause's REMOVE_BINDING purges the subject, so the restore import
# lands through the unregistered branch and the file reloads at the
# SET_BINDING replay (the reload line carries the restored revision).
wait_for "vault bundle reloaded at re-registration" 120 \
vault_log_contains "subject s-$sandbox_uid: vault reloaded from file"
wait_for "vault revision preserved across pause/resume" 120 vault_revision_is "$rev"
pass "vault survived pause/resume (revision $rev preserved)"
# 3) egress-restart recovery: crash the worker (fresh instanceId ->
# SET_BINDING replay -> vault file reload by incarnation).
local pod restarted=0
for pod in $(pool_pods); do
if kubectl -n "$RESOURCE_NS" exec "$pod" -c egress -- sh -c 'command -v pkill >/dev/null' 2>/dev/null; then
kubectl -n "$RESOURCE_NS" exec "$pod" -c egress -- pkill -f '/opt/opensandbox-egress/egress' >/dev/null 2>&1 || true
restarted=1
else
log "verify (vault): egress restart skipped on ${pod##*/} (pkill unavailable)"
fi
done
if [[ "$restarted" == 1 ]]; then
wait_for "pool egress containers back up" 180 egress_containers_ready
wait_for "vault reloaded after egress restart" 240 vault_revision_is "$rev"
pass "vault reloaded after egress restart (revision $rev preserved)"
fi
# 4) delete + re-create.
server_api DELETE "/sandboxes/$VERIFY_ID/credential-vault" >/dev/null \
|| fail "vault verify: DELETE failed"
[[ "$(server_api_status GET "/sandboxes/$VERIFY_ID/credential-vault")" == "404" ]] \
|| fail "vault verify: GET after DELETE was not 404"
pass "vault deleted (GET 404)"
ack="$(server_api POST "/sandboxes/$VERIFY_ID/credential-vault" "$vault_body")" \
|| fail "vault verify: re-create after DELETE failed"
rev="$(printf '%s' "$ack" | jq -r '.revision // empty')"
[[ "$rev" == "1" ]] || fail "vault verify: fresh vault revision is $rev, expected 1"
pass "vault re-created (fresh revision $rev)"
osb_api sandbox kill "$VERIFY_ID" >/dev/null 2>&1 \
|| log "verify cleanup: kill failed; remove $VERIFY_ID manually"
wait_for "vault sandbox deleted" 120 verify_sandbox_gone
}
# --- stage: MITM CA chain verify (OSEP-0022: baked CA + injection + pinning) ----
# NOTE: verify/vault.sh carries the split-layout copy of this stage (plus its
# helpers); keep the two in sync until the env.sh split is completed and the
# verify scripts become the single home.
# Guest-side HTTPS request to the echo endpoint via `osb command run` (the
# only sandbox-reachable surface). `curl --resolve` pins the SNI to the echo
# domain — the domain is deliberately unresolvable DNS, and the policy
# carries the IP-allow rule the resolved flow needs; the egress DNATs the
# allowed flow to the shared mitmdump, which matches the vault binding by
# SNI. TLS verification runs against the template-baked CA (no -k anywhere).
# The guest-side stderr (curl/CLI error text — the first diagnostic for a
# broken TLS chain) is kept per attempt at $LOGS_DIR/mitm-guest-curl.err for
# the failure dump.
mitm_ca_guest_echo() {
# The CLI joins command-run words with shlex.quote into one shell line —
# the curl invocation must ride as separate words (one quoted string
# would be executed as a single program name and exit 127).
osb_api command run "$VERIFY_ID" -- \
curl -fsS -m 20 --resolve "$MITM_ECHO_DOMAIN:$MITM_ECHO_PORT:$MITM_ECHO_IP" \
"https://$MITM_ECHO_DOMAIN:$MITM_ECHO_PORT/echo" \
2>"$LOGS_DIR/mitm-guest-curl.err"
}
# osb_command_succeeds <command words...>: wait_for probe variant — the
# guest command must exit 0 (a failed execd command makes the CLI exit
# non-zero). stderr lands in the same per-attempt diagnostics file as
# mitm_ca_guest_echo.
osb_command_succeeds() {
osb_api command run "$VERIFY_ID" -- "$@" >/dev/null 2>"$LOGS_DIR/mitm-guest-curl.err"
}
# mitm_ca_guest_header_is <key> <value>: the echo reflected the exact header.
# Scans line-by-line: the CLI stream may carry non-JSON framing.
mitm_ca_guest_header_is() { # <lowercase-key> <expected-value>
local out line
out="$(mitm_ca_guest_echo)" || return 1
while IFS= read -r line; do
[[ -n "$line" ]] || continue
printf '%s' "$line" | jq -e --arg k "$1" --arg v "$2" '.[$k] == $v' >/dev/null 2>&1 && return 0
done <<EOF
$out
EOF
return 1
}
mitm_ca_chain_verify() {
# Optional template override (the overlaybd second pass); default native.
local template_id="${1:-$TEMPLATE_ID}"
# Fingerprint pins live in a file precisely because up and the verify
# actions are separate invocations (same pattern as require_template_id).
load_mitm_ca_fingerprint
[[ -n "$MITM_CA_FINGERPRINT" ]] \
|| die "no pinned MITM CA fingerprint at $MITM_CA_FINGERPRINT_FILE (run up first; mitm_ca_up stores it there)"
load_mitm_echo_endpoint
local created sandbox_uid ack echo_secret
log "verify (mitm-ca): fingerprint pinning (pinned Secret == egress /healthz, plus the manifest lineage on the S3 publication path)"
local healthz_fp lineage_fp manifest_ref
# Guarded so a failed probe lands on the crafted mismatch message (and
# its diagnostics) instead of an ERR-trap abort.
healthz_fp="$(egress_mitm_ca_fingerprint)" || true
[[ "$healthz_fp" == "$MITM_CA_FINGERPRINT" ]] \
|| fail "mitm-ca verify: egress /healthz fingerprint is '${healthz_fp:-empty}', want pinned $MITM_CA_FINGERPRINT"
# The manifest lineage leg is only readable on the S3 publication path:
# an oci:// manifestRef lives in the in-cluster registry, out of rc's
# reach. There the pool-side pinning plus the behavioral chain below
# (the guest accepts the MITM certificate only because it chains to the
# template-baked CA) carry the assertion.
manifest_ref="$(osb_api template get "$template_id" -o json 2>/dev/null | jq -r '.status.manifest_ref // empty')" || true
if [[ "$manifest_ref" == s3://* ]]; then
lineage_fp="$(template_mitm_lineage_fingerprint "$template_id")"
[[ "$lineage_fp" == "$MITM_CA_FINGERPRINT" ]] \
|| fail "mitm-ca verify: template manifest platform.egressMitmCA fingerprint is '${lineage_fp:-empty}', want pinned $MITM_CA_FINGERPRINT"
pass "template↔pool CA pinning: manifest platform.egressMitmCA == egress /healthz == pinned (${MITM_CA_FINGERPRINT:0:16}…)"
else
pass "template↔pool CA pinning: egress /healthz == pinned (${MITM_CA_FINGERPRINT:0:16}…); ${manifest_ref:-no manifest_ref} — guest-side chain proven behaviorally below"
fi
# Sandbox policy: the domain allow covers the vault binding (binding
# validation requires the host be policy-allowed), the IP allow covers
# the --resolve flow at the nft layer (no DNS query ever happens).
jq -n --arg ip "$MITM_ECHO_IP" --arg domain "$MITM_ECHO_DOMAIN" \
'{defaultAction: "deny", egress: [{action: "allow", target: $ip}, {action: "allow", target: $domain}]}' \
> "$GEN_DIR/mitm-ca-policy.json"
log "verify (mitm-ca): creating a sandbox via the opensandbox CLI (templateId=$template_id, echo=$MITM_ECHO_IP)"
created="$(osb_api sandbox create --template "$template_id" --timeout 1h \
--metadata origin=fast-sandbox-env-mitm-ca \
--network-policy-file "$GEN_DIR/mitm-ca-policy.json" \
--skip-health-check -o json 2>/dev/null)" \
|| fail "mitm-ca verify: sandbox create failed"
VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"
[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "mitm-ca verify: create response carried no id"
wait_for "mitm-ca sandbox Running" 300 sandbox_running
sandbox_uid="$(kubectl -n "$RESOURCE_NS" get sandbox "$VERIFY_ID" -o jsonpath='{.metadata.uid}' 2>/dev/null)"
[[ -n "$sandbox_uid" ]] || fail "mitm-ca verify: sandbox CR carried no uid"
wait_for "mitm-ca subject registered on egress" 120 \
vault_log_contains "subject s-$sandbox_uid: data-plane-ready, policy active"
# Negative control BEFORE the vault exists: the flow is allowed and
# passes through the shared mitmdump opaquely (unbound SNI), TLS is
# verified end-to-end against the template-baked CA — and no credential
# header appears. Proves the header in the positive case comes from the
# vault, not from anything ambient.
wait_for "pre-vault guest HTTPS reaches the echo (passthrough, baked-CA verified)" 120 \
osb_command_succeeds curl -fsS -m 20 \
--resolve "$MITM_ECHO_DOMAIN:$MITM_ECHO_PORT:$MITM_ECHO_IP" \
"https://$MITM_ECHO_DOMAIN:$MITM_ECHO_PORT/echo"
local pre_vault_out
pre_vault_out="$(mitm_ca_guest_echo)" \
|| fail "mitm-ca verify: pre-vault guest HTTPS request failed (cannot assert header absence)"
if printf '%s' "$pre_vault_out" | grep -qi 'x-api-key'; then
fail "mitm-ca verify: x-api-key present before any vault binding existed"
fi
pass "pre-vault: guest HTTPS passthrough works, no credential injected"
# Per-subject vault binding: SNI echo.test -> X-Api-Key.
echo_secret="mitm-ca-e2e-$(openssl rand -hex 8)"
local vault_body
vault_body="$(jq -n --arg v "$echo_secret" --arg domain "$MITM_ECHO_DOMAIN" \
'{credentials: [{name: "tok", source: {type: "inline", value: $v}}],
bindings: [{name: "echo", match: {hosts: [$domain]}, auth: {type: "apiKey", name: "X-Api-Key", credential: "tok"}}]}')"
ack="$(server_api POST "/sandboxes/$VERIFY_ID/credential-vault" "$vault_body")" \
|| fail "mitm-ca verify: POST credential-vault failed"
[[ "$(printf '%s' "$ack" | jq -r '.revision // empty')" =~ ^[0-9]+$ ]] \
|| fail "mitm-ca verify: vault create ack carried no revision: $(printf '%s' "$ack" | head -c 200)"
pass "vault bound to SNI $MITM_ECHO_DOMAIN (X-Api-Key) over the relay"
wait_for "guest HTTPS reflects the injected X-Api-Key (MITM decrypt + vault by SNI)" 120 \
mitm_ca_guest_header_is "x-api-key" "$echo_secret"
pass "credential injection observed by the echo endpoint on a guest-originated HTTPS request"
# Pause/resume: trust travels with the snapshot (the baked CA is part of
# the rootfs) and the vault is carried in the bundle — the same request
# must keep working end to end after restore.
osb_api sandbox pause "$VERIFY_ID" >/dev/null 2>&1 || fail "mitm-ca verify: pause failed"
wait_for "mitm-ca sandbox Paused (checkpoint durable)" 240 sandbox_paused
osb_api sandbox resume "$VERIFY_ID" --skip-health-check >/dev/null 2>&1 || fail "mitm-ca verify: resume failed"
wait_for "mitm-ca sandbox Running (restored)" 240 sandbox_running
wait_for "restored subject re-registered on egress" 120 \
vault_log_contains "subject s-$sandbox_uid: vault reloaded from file"
wait_for "post-resume guest HTTPS still MITM-injected (baked CA + vault survive the snapshot)" 180 \
mitm_ca_guest_header_is "x-api-key" "$echo_secret"
pass "paused/restored sandbox keeps passing TLS verification against the baked CA (injection intact)"
osb_api sandbox kill "$VERIFY_ID" >/dev/null 2>&1 \
|| log "verify cleanup: kill failed; remove $VERIFY_ID manually"
wait_for "mitm-ca sandbox deleted" 120 verify_sandbox_gone
}
# --- stage: idle-pause / wake-on-access verify (OSEP-0024 profile) --------------
# Poll the gateway until /ping returns 200 (marked 503s tolerated; the
# resume is never canceled). Fails on budget exhaustion.
idle_wake_wait() { # <label> [sandbox_id]
local label="$1" id="${2:-$VERIFY_ID}" code
local deadline=$(( $(now_ms) + 600000000000 )) wake_seen=0 next_state_log=0
while :; do
code="$(idle_ping_code "$id")"
case "$code" in
200) break ;;
wake-503) wake_seen=1 ;;
esac
if (( $(now_ms) >= next_state_log )); then
log "verify (idle-pause): $label still waiting (last=$code, state=$(sandbox_state_get))"
next_state_log=$(( $(now_ms) + 30000000000 ))
fi
[[ "$(now_ms)" -ge "$deadline" ]] && fail "$label: wake /ping never returned 200 (last=$code)"
sleep 1
done
}
# Gateway answer for one /ping attempt: "200", "wake-503", or "other-<code>".
idle_ping_code() { # [sandbox_id] (default $VERIFY_ID)
local id="${1:-$VERIFY_ID}" route headers code marker
route="$(osb_api sandbox endpoint "$id" --port 44772 -o json 2>/dev/null \
| jq -r '.headers["OpenSandbox-Ingress-To"] // empty')"
[[ -n "$route" ]] || { echo "noroute"; return 0; }
headers="$(mktemp)"
curl -sS -m 10 -D "$headers" -o /dev/null \
-H "OpenSandbox-Ingress-To: $route" "$GATEWAY_URL/ping" >/dev/null 2>&1 || true
code="$(tr -d '\r' < "$headers" | head -1 | awk '{print $2}')" || true
marker="$(tr -d '\r' < "$headers" | grep -i '^OpenSandbox-Wake:' | awk '{print $2}')" || true
rm -f "$headers"
if [[ "$code" == "200" ]]; then
echo "200"
elif [[ "$code" == "503" && "$marker" == "expired" ]]; then
echo "wake-503"
else
echo "other-${code:-none}"
fi
}
idle_pause_verify() {
# Raw curl create: the CLI has no lifecycle.idlePolicy parameter yet.
# Optional template override (the overlaybd second pass); default native.
[[ "$IDLE_PAUSE" == "1" ]] || { log "idle-pause profile off (IDLE_PAUSE=0); skipping verify"; return 0; }
local template_id="${1:-$TEMPLATE_ID}"
log "verify (idle-pause): template=$template_id"
local created t0 t1 code i
local x=30
local rounds="${IDLE_PAUSE_ROUNDS:-3}"
[[ "$rounds" =~ ^[1-9][0-9]*$ ]] || die "IDLE_PAUSE_ROUNDS must be a positive integer (got $rounds)"
log "verify (idle-pause): creating an opted-in sandbox (idleTimeoutSeconds=$x, rounds=$rounds) via the lifecycle API"
t0="$(now_ms)"
created="$(server_api POST /sandboxes "$(jq -n \
--arg template "$template_id" \
--argjson timeout 3600 --argjson x "$x" \
'{templateId: $template, timeout: $timeout,
lifecycle: {idlePolicy: {idleTimeoutSeconds: $x, onIdle: "pause", wakeOnAccess: true}},
metadata: {origin: "fast-sandbox-env-idle-pause"}}')" 2>/dev/null)" \
|| fail "idle-pause create failed against $SERVER_URL"
VERIFY_ID="$(printf '%s' "$created" | jq -r '.id')"
[[ -n "$VERIFY_ID" && "$VERIFY_ID" != "null" ]] || fail "create response carried no id"
wait_for "idle-pause target Running" 300 sandbox_running
log "verify (idle-pause): $VERIFY_ID create->Running $(( ($(now_ms) - t0) / 1000000 ))ms"
wait_for "pre-idle execd /ping 200 through the gateway (records activity)" 100 execd_ping_ok
local -a pause_ms=() wake_ms=() attempts=()
for ((i = 1; i <= rounds; i++)); do
t0="$(now_ms)"
wait_for "round $i/$rounds auto-paused within X + sweep + checkpoint" 240 sandbox_paused
pause_ms+=("$(( ($(now_ms) - t0) / 1000000 ))")
pass "round $i/$rounds: auto-paused in ${pause_ms[-1]}ms"
t1="$(now_ms)"
idle_wake_wait "round $i wake"
wake_ms+=("$(( ($(now_ms) - t1) / 1000000 ))")
attempts+=("$(kubectl -n "$RESOURCE_NS" get sandbox "$VERIFY_ID" \
-o jsonpath='{.status.placement.attempt}' 2>/dev/null || echo '?')")
pass "round $i/$rounds: woke in ${wake_ms[-1]}ms (placement.attempt=${attempts[-1]})"
done
local joined_pause joined_wake joined_att
printf -v joined_pause '%sms/' "${pause_ms[@]}"
printf -v joined_wake '%sms/' "${wake_ms[@]}"
printf -v joined_att '%s/' "${attempts[@]}"
pass "idle-pause ${rounds} rounds: pause=${joined_pause%?} wake=${joined_wake%?} placement.attempt=${joined_att%?}"
local main_id="$VERIFY_ID" created2 id2 payload
log "verify (idle-pause): scenario wakeOnAccess=false"
created2="$(server_api POST /sandboxes "$(jq -n \
--arg template "$template_id" --argjson timeout 3600 --argjson x "$x" \
'{templateId: $template, timeout: $timeout,
lifecycle: {idlePolicy: {idleTimeoutSeconds: $x, onIdle: "pause", wakeOnAccess: false}},
metadata: {origin: "fast-sandbox-env-idle-pause"}}')" 2>/dev/null)" \
|| fail "wakeOnAccess=false create failed"
id2="$(printf '%s' "$created2" | jq -r '.id')"
[[ -n "$id2" && "$id2" != "null" ]] || fail "wakeOnAccess=false create carried no id"
VERIFY_ID="$id2"
wait_for "wakeOnAccess=false target Running" 300 sandbox_running
wait_for "wakeOnAccess=false auto-paused" 240 sandbox_paused
code="$(idle_ping_code "$id2")"
[[ "$code" == "200" || "$code" == "wake-503" ]] \
&& fail "wakeOnAccess=false sandbox was woken by traffic (code=$code)"
sleep 10
sandbox_paused || fail "wakeOnAccess=false sandbox changed state without resume API"
osb_api sandbox resume "$id2" --skip-health-check >/dev/null 2>&1 || fail "manual resume failed"
wait_for "wakeOnAccess=false resumed manually" 240 sandbox_running
pass "wakeOnAccess=false: traffic never woke it, manual resume did"
osb_api sandbox kill "$id2" >/dev/null 2>&1 || true
wait_for "wakeOnAccess=false sandbox deleted" 120 verify_sandbox_gone
VERIFY_ID="$main_id"
log "verify (idle-pause): scenario redis down"
# The previous scenario may have left it Paused.
idle_wake_wait "pre-redis-down wake"
t0="$(now_ms)"
kubectl -n "$OSB_NS" scale deploy/idle-pause-redis --replicas=0 >/dev/null
while sandbox_running; do
(( $(now_ms) - t0 >= 90000000000 )) && break
sleep 5
done
sandbox_running || fail "sandbox paused while Redis was scaled down"
pass "redis down: no pauses while Redis unreachable"
kubectl -n "$OSB_NS" scale deploy/idle-pause-redis --replicas=1 >/dev/null
wait_for "redis back ready" 120 \
kubectl -n "$OSB_NS" rollout status deploy/idle-pause-redis --timeout=10s
wait_for "tombstone pause after redis recovery" 300 sandbox_paused
pass "redis recovery: missing-key tombstone paused after the full idle window"
idle_wake_wait "post-flushall wake"
t0="$(now_ms)"
kubectl -n "$OSB_NS" exec deploy/idle-pause-redis -- redis-cli FLUSHALL >/dev/null 2>&1 \
|| fail "FLUSHALL failed"
while :; do
if sandbox_paused; then
(( $(now_ms) - t0 < (x - 5) * 1000000000 )) \
&& fail "FLUSHALL accelerated the pause ($(( ($(now_ms) - t0) / 1000000 ))ms after flush)"
break
fi
(( $(now_ms) - t0 >= 300000000000 )) && fail "no pause within budget after FLUSHALL"
sleep 2
done
pass "flushall: pause delayed to a full idle window after data loss ($(( ($(now_ms) - t0) / 1000000 ))ms)"
osb_api sandbox kill "$VERIFY_ID" >/dev/null 2>&1 \
|| log "verify cleanup: kill failed; remove $VERIFY_ID manually"
wait_for "idle-pause sandbox deleted" 120 verify_sandbox_gone
}
# --- stage: snapshot (opensandbox CLI -> SandboxSnapshot CR -> restore) ----------
# Ready once the server watcher converges the row from the SandboxSnapshot
# CR (Succeeded + template index published).
snapshot_ready() { # <snapshot-id>
[[ "$(osb_api snapshot get "$1" -o json 2>/dev/null | jq -r '.status.state // empty')" == "Ready" ]]
}
# Terminal (Ready or Failed): the pause-window fence resolves any
# accepted-but-conflicting snapshot one way or the other.
snapshot_terminal() { # <snapshot-id>
local state
state="$(osb_api snapshot get "$1" -o json 2>/dev/null | jq -r '.status.state // empty')"
[[ "$state" == "Ready" || "$state" == "Failed" ]]
}
snapshot_verify() {
# Full round trip on the live stack: snapshot a Running sandbox, watch it
# to Ready, restore a NEW sandbox from the snapshotId and boot it. Also
# covers re-entry fencing, source-sandbox survival, and a repeat snapshot.
# Optional template override (the overlaybd second pass); default native.
local template_id="${1:-$TEMPLATE_ID}"
local created out source_id snapshot_id snapshot_id2 restore_id reentry_out reentry_code
local t0 t1 t2 t3 t4
t0="$(now_ms)"
log "verify (snapshot): creating the source sandbox via the opensandbox CLI (templateId=$template_id)"
created="$(osb_api sandbox create --template "$template_id" --timeout 1h \
--metadata origin=fast-sandbox-env-snapshot --skip-health-check -o json 2>/dev/null)" \
|| fail "osb sandbox create failed against $SERVER_URL"
source_id="$(printf '%s' "$created" | jq -r '.id')"
[[ -n "$source_id" && "$source_id" != "null" ]] || fail "create response carried no id"
VERIFY_ID="$source_id"
wait_for "snapshot source sandbox Running" 300 sandbox_running
wait_for "pre-snapshot execd /ping 200 through the gateway" 100 execd_ping_ok
t1="$(now_ms)"
log "verify (snapshot): source sandbox $source_id create->Running $(( (t1 - t0) / 1000000 ))ms"
# 1. Snapshot create returns Creating; the row converges from the
# SandboxSnapshot CR via the server watcher.
log "verify (snapshot): osb snapshot create $source_id --name env-verify"
t1="$(now_ms)"
out="$(osb_api snapshot create "$source_id" --name env-verify -o json 2>/dev/null)" \
|| fail "snapshot create failed for $source_id"
SNAPSHOT_ID="$(printf '%s' "$out" | jq -r '.id')"
[[ -n "$SNAPSHOT_ID" && "$SNAPSHOT_ID" != "null" ]] || fail "snapshot create carried no id"
[[ "$(printf '%s' "$out" | jq -r '.status')" == "Creating" ]] \
|| fail "snapshot create did not return Creating: $(printf '%s' "$out" | head -c 300)"
# 2. Re-entry during the dump window: FastPath fences with 409 once the
# CR is cache-visible; with cache lag it is accepted (202) and the
# pause window resolves it to a terminal phase later.
reentry_out="$(curl -sS -m 60 -w '\n%{http_code}' -X POST \
-H "OPEN-SANDBOX-API-KEY: $SERVER_API_KEY" -H "Content-Type: application/json" \
-d '{"name":"env-verify-reentry"}' "$SERVER_URL/sandboxes/$source_id/snapshots" 2>/dev/null || true)"
reentry_code="$(printf '%s' "$reentry_out" | tail -n1)"
reentry_out="$(printf '%s' "$reentry_out" | sed '$d')"
case "$reentry_code" in
409)
pass "snapshot: re-entry rejected by the fence (409)"
;;
202)
SNAPSHOT_EXTRA="$(printf '%s' "$reentry_out" | jq -r '.id' 2>/dev/null || true)"
[[ -n "$SNAPSHOT_EXTRA" && "$SNAPSHOT_EXTRA" != "null" ]] \
|| fail "re-entry snapshot POST returned 202 without an id: $(printf '%s' "$reentry_out" | head -c 300)"
log "verify (snapshot): re-entry accepted during fence cache lag ($SNAPSHOT_EXTRA); terminal outcome asserted below"
;;
*)
fail "re-entry snapshot POST returned unexpected HTTP ${reentry_code:-none}: $(printf '%s' "$reentry_out" | head -c 300)"
;;
esac
wait_for "poll osb snapshot get $SNAPSHOT_ID until Ready (watcher -> SandboxSnapshot CR -> store index)" 300 snapshot_ready "$SNAPSHOT_ID"
t2="$(now_ms)"
log "verify (snapshot): $SNAPSHOT_ID snapshot create->Ready $(( (t2 - t1) / 1000000 ))ms"
pass "snapshot: create -> Creating -> watcher -> Ready"
# 3. Source survival: Running + /ping again after the terminal phase.
VERIFY_ID="$source_id"
wait_for "source sandbox Running again after the snapshot" 120 sandbox_running
wait_for "source sandbox execd /ping 200 after the snapshot" 100 execd_ping_ok
pass "snapshot: source sandbox survived (Running + /ping 200)"
# 3b. An accepted re-entry snapshot must reach a terminal phase; a stuck
# one would block all future snapshots via the re-entry fence.
if [[ -n "$SNAPSHOT_EXTRA" ]]; then
wait_for "re-entry snapshot $SNAPSHOT_EXTRA reaches a terminal phase" 150 snapshot_terminal "$SNAPSHOT_EXTRA"
log "verify (snapshot): re-entry snapshot $SNAPSHOT_EXTRA terminal: $(osb_api snapshot get "$SNAPSHOT_EXTRA" -o json 2>/dev/null | jq -r '.status.state')"
pass "snapshot: accepted re-entry snapshot resolved to a terminal phase"
fi
# 4. Second snapshot once the first is terminal: fresh id.
t3="$(now_ms)"
out="$(osb_api snapshot create "$source_id" --name env-verify-2 -o json 2>/dev/null)" \
|| fail "second snapshot create failed for $source_id"
snapshot_id2="$(printf '%s' "$out" | jq -r '.id')"
[[ -n "$snapshot_id2" && "$snapshot_id2" != "null" && "$snapshot_id2" != "$SNAPSHOT_ID" ]] \
|| fail "second snapshot did not produce a distinct id: $snapshot_id2"
SNAPSHOT_ID2="$snapshot_id2"
wait_for "poll osb snapshot get $snapshot_id2 until Ready" 300 snapshot_ready "$snapshot_id2"
t4="$(now_ms)"
log "verify (snapshot): $snapshot_id2 second snapshot create->Ready $(( (t4 - t3) / 1000000 ))ms"
pass "snapshot: repeated snapshot of the same sandbox -> Ready (distinct id)"
# 5. Listing: sandboxId scoping returns both required snapshots, both
# Ready (an accepted re-entry snapshot may add a third, terminal row).
out="$(osb_api snapshot list --sandbox-id "$source_id" --page-size 50 -o json 2>"$WORK/last-snapshot-list.err")" \
|| fail "snapshot list failed for $source_id: $(head -c 400 "$WORK/last-snapshot-list.err" 2>/dev/null)"
[[ "$(printf '%s' "$out" | jq -r --arg id "$SNAPSHOT_ID" --arg id2 "$snapshot_id2" \
'[.items[] | select((.id == $id or .id == $id2) and .status.state == "Ready")] | length')" == "2" ]] \
|| fail "snapshot list does not contain both required snapshots as Ready: $(printf '%s' "$out" | head -c 400)"
if [[ -n "$SNAPSHOT_EXTRA" ]]; then
[[ "$(printf '%s' "$out" | jq -r --arg id "$SNAPSHOT_EXTRA" \
'[.items[] | select(.id == $id)] | length')" == "1" ]] \
|| fail "accepted re-entry snapshot $SNAPSHOT_EXTRA missing from the list"
fi
pass "snapshot: list scoped by sandboxId contains the snapshots (Ready)"
# 6. Restore: the row resolves to the published template index key; the
# resource flags restate the pool profile (firecracker pool).
local restored
log "verify (snapshot): osb sandbox create --snapshot-id $SNAPSHOT_ID (pool profile restated)"
t3="$(now_ms)"
restored="$(osb_api sandbox create --snapshot-id "$SNAPSHOT_ID" --timeout 1h \
--resource cpu=1 --resource memory=512Mi --resource pids=128 \
--skip-health-check -o json 2>/dev/null)" \
|| fail "osb sandbox create (snapshotId=$SNAPSHOT_ID) failed"
restore_id="$(printf '%s' "$restored" | jq -r '.id')"
[[ -n "$restore_id" && "$restore_id" != "null" ]] || fail "restore create carried no id"
VERIFY_ID="$restore_id"
wait_for "restored sandbox Running" 600 sandbox_running
t4="$(now_ms)"
wait_for "restored execd /ping 200 through the gateway" 300 execd_ping_ok
log "verify (snapshot): restored $restore_id create->Running $(( (t4 - t3) / 1000000 ))ms"
pass "snapshot: restore -> sandbox boots the published artifact set -> execd /ping OK"
# 7. Cleanup; the server forwards artifact deletion per snapshot row.
osb_api sandbox kill "$restore_id" >/dev/null 2>&1 \
|| log "verify cleanup: kill $restore_id failed"
wait_for "restored sandbox deleted" 120 verify_sandbox_gone
VERIFY_ID="$source_id"
osb_api sandbox kill "$source_id" >/dev/null 2>&1 \
|| log "verify cleanup: kill $source_id failed"
wait_for "snapshot source sandbox deleted" 120 verify_sandbox_gone
for snapshot_id in "$SNAPSHOT_ID" "$SNAPSHOT_ID2" "$SNAPSHOT_EXTRA"; do
[[ -n "$snapshot_id" ]] || continue
osb_api snapshot delete "$snapshot_id" >/dev/null 2>&1 \
|| log "verify cleanup: snapshot delete $snapshot_id failed"
done
pass "snapshot: cleanup (restored + source sandboxes, both snapshot rows)"
}
# --- stage: python sdk e2e (tests/python/tests/test_fsb_e2e.py) -----------------
sdk_e2e() {
kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null \
|| die "cluster $KIND_CLUSTER is not up (run up first)"
curl -fsS -m 5 "$SERVER_URL/health" >/dev/null 2>&1 \
|| die "server $SERVER_URL is not reachable (run up first)"
command -v uv >/dev/null 2>&1 \
|| die "uv is required on PATH (https://docs.astral.sh/uv/ — curl -LsSf https://astral.sh/uv/install.sh | sh)"
local -a template_ids=() template_origins=()
if [[ "$OCI_STACK" == 1 ]]; then
# Overlaybd only: the suite exercises the SDK contract on the on-demand
# path (the server resolves the OCI template's status.manifestRef on
# template-mode creates). OCI_STACK=0 falls back to the native template.
local oci_template_id
oci_template_id="$(cat "$WORK/overlaybd-template-id" 2>/dev/null || true)"
[[ -n "$oci_template_id" ]] \
|| die "no overlaybd template id at $WORK/overlaybd-template-id (run verify-overlaybd first; template_up_overlaybd writes it)"
template_ids+=("$oci_template_id")
template_origins+=("fast-sandbox-env-oci")
else
local template_id
template_id="$(cat "$WORK/template-id" 2>/dev/null || true)"
[[ -n "$template_id" ]] || die "no template id at $WORK/template-id (run up first; template_up stores it there)"
template_ids+=("$template_id")
template_origins+=("fast-sandbox-env")
fi
export OPENSANDBOX_TEST_DOMAIN="127.0.0.1:$SERVER_HOST_PORT"
export OPENSANDBOX_TEST_PROTOCOL="http"
export OPENSANDBOX_TEST_API_KEY="$SERVER_API_KEY"
cd "$OSB_ROOT/tests/python" || die "tests/python not found under $OSB_ROOT"
# No exec: the ERR trap (and the failure dump) must survive a pytest
# failure, and exec replaces the shell. One pytest run per template so a
# failure names its template.
local index
for index in "${!template_ids[@]}"; do
export OPENSANDBOX_TEST_FSB_TEMPLATE_ID="${template_ids[$index]}"
export OPENSANDBOX_TEST_FSB_TEMPLATE_ORIGIN="${template_origins[$index]}"
log "sdk e2e: template=${template_ids[$index]} (origin=${template_origins[$index]}) server=$SERVER_URL -> tests/python/tests/test_fsb_e2e.py"
uv run pytest tests/test_fsb_e2e.py
done
}
idle_e2e() {
if [[ "$IDLE_PAUSE" != "1" && ! -f "$WORK/idle-pause" ]]; then
die "idle-e2e requires the idle-pause profile (IDLE_PAUSE=1 up)"
fi
kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null \
|| die "cluster $KIND_CLUSTER is not up (run up first)"
curl -fsS -m 5 "$SERVER_URL/health" >/dev/null 2>&1 \
|| die "server $SERVER_URL is not reachable (run up first)"
command -v uv >/dev/null 2>&1 \
|| die "uv is required on PATH (https://docs.astral.sh/uv/ — curl -LsSf https://astral.sh/uv/install.sh | sh)"
local template_id
if [[ "$OCI_STACK" == 1 ]]; then
# Overlaybd only (the on-demand path); OCI_STACK=0 falls back to native.
template_id="$(cat "$WORK/overlaybd-template-id" 2>/dev/null || true)"
[[ -n "$template_id" ]] \
|| die "no overlaybd template id at $WORK/overlaybd-template-id (run verify-overlaybd first; template_up_overlaybd writes it)"
else
template_id="$(cat "$WORK/template-id" 2>/dev/null || true)"
[[ -n "$template_id" ]] || die "no template id at $WORK/template-id (run up first; template_up stores it there)"
fi
export OPENSANDBOX_TEST_FSB_TEMPLATE_ID="$template_id"
export OPENSANDBOX_TEST_DOMAIN="127.0.0.1:$SERVER_HOST_PORT"
export OPENSANDBOX_TEST_PROTOCOL="http"
export OPENSANDBOX_TEST_API_KEY="$SERVER_API_KEY"
export OPENSANDBOX_TEST_IDLE_PAUSE="1"
export OPENSANDBOX_TEST_IDLE_ROUNDS="${IDLE_PAUSE_ROUNDS:-3}"
log "sdk e2e (idle-pause): template=$template_id server=$SERVER_URL rounds=$OPENSANDBOX_TEST_IDLE_ROUNDS -> tests/python/tests/test_fsb_idle_pause_e2e.py"
cd "$OSB_ROOT/tests/python" || die "tests/python not found under $OSB_ROOT"
uv run pytest tests/test_fsb_idle_pause_e2e.py
}
# --- verify actions (native / overlaybd) --------------------------------------
require_live_stack() {
kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null \
|| die "cluster $KIND_CLUSTER is not up (run up first)"
curl -fsS -m 5 "$SERVER_URL/health" >/dev/null 2>&1 \
|| die "server $SERVER_URL is not reachable (run up first)"
[[ -n "$OSB_BIN" ]] || OSB_BIN="$OSB_VENV/bin/osb"
[[ -x "$OSB_BIN" ]] || die "osb not found at $OSB_BIN (run up first, or set OSB_BIN)"
}
verify_native() {
require_live_stack
TEMPLATE_ID="$(cat "$WORK/template-id" 2>/dev/null || true)"
[[ -n "$TEMPLATE_ID" ]] || die "no template id at $WORK/template-id (run up first; template_up stores it there)"
run_stage "end-to-end verify (API create -> gateway -> execd /ping)" opensandbox_verify
run_stage "execd-as-init verify (in-guest process topology)" init_mode_verify
run_stage "pause/resume verify (CLI -> FastPath checkpoint)" pause_resume_verify
run_stage "vault lifecycle verify (CLI -> relay -> egress-held vault)" vault_lifecycle_verify
run_stage "MITM CA chain verify (baked CA -> injection -> pinning)" mitm_ca_chain_verify
run_stage "idle-pause / wake-on-access verify (OSEP-0024 profile)" idle_pause_verify
run_stage "snapshot verify (CLI -> SandboxSnapshot -> restore)" snapshot_verify
}
verify_overlaybd() {
require_live_stack
[[ "$OCI_STACK" == 1 ]] || {
log "OCI_STACK=0: overlaybd disabled; nothing to verify"
return 0
}
# Probe before building: a wedged host ublk table kills the template
# build ~16 minutes in; the probe fails in seconds and heals itself.
run_stage "ublk add/del probe (node daemon -> kernel path)" ublk_probe_or_heal
run_stage "SandboxTemplate build (overlaybd, oci:// publication)" template_up_overlaybd
run_stage "end-to-end verify (overlaybd template)" opensandbox_verify "$OCI_TEMPLATE_ID"
run_stage "execd-as-init verify (overlaybd template)" init_mode_verify "$OCI_TEMPLATE_ID"
run_stage "pause/resume verify (overlaybd template)" pause_resume_verify "$OCI_TEMPLATE_ID"
run_stage "vault lifecycle verify (overlaybd template)" vault_lifecycle_verify "$OCI_TEMPLATE_ID"
run_stage "MITM CA chain verify (overlaybd template)" mitm_ca_chain_verify "$OCI_TEMPLATE_ID"
run_stage "idle-pause / wake-on-access verify (overlaybd template)" idle_pause_verify "$OCI_TEMPLATE_ID"
run_stage "snapshot verify (overlaybd template)" snapshot_verify "$OCI_TEMPLATE_ID"
}
# --- status / summary ---------------------------------------------------------------------
dart_metrics_summary() {
local pods pod node metrics
pods="$(runtime_pods 2>/dev/null || true)"
[[ -n "$pods" ]] || { echo " (no agent pods)"; return 0; }
for pod in $pods; do
node="$(kubectl -n "$NS" get pod "$pod" -o jsonpath='{.spec.nodeName}' 2>/dev/null)"
metrics="$(kubectl exec -n "$NS" "$pod" -- sh -c 'curl -fsS --noproxy "*" http://127.0.0.1:8147/metrics' 2>/dev/null || true)"
echo " $node:"
if [[ -z "$metrics" ]]; then
echo " (DART metrics unreachable)"
continue
fi
printf '%s\n' "$metrics" | grep -E '^dart_block_source_total\{source="(cache|peer|origin)"\}' \
| sed 's/^/ /' || true
done
}
status() {
log "status: RustFS"
docker ps --filter "name=$RUSTFS_CONTAINER" --format '{{.Names}} {{.Status}}' 2>/dev/null || true
echo
log "status: kind cluster / nodes"
if kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null; then
kubectl get nodes -o wide
else
log "kind cluster $KIND_CLUSTER: down"
return 0
fi
echo
log "status: pods ($NS)"
if [[ "$IDLE_PAUSE" == "1" ]]; then
log "status: idle-pause Redis ($OSB_NS)"
kubectl -n "$OSB_NS" get deploy,svc -l app=idle-pause-redis -o wide 2>/dev/null || true
echo
fi
kubectl -n "$NS" get pods -o wide
kubectl -n "$RESOURCE_NS" get pods -o wide
echo
log "status: SandboxPool"
kubectl -n "$RESOURCE_NS" get sandboxpool \
-o custom-columns='NAME:.metadata.name,RUNTIME:.spec.runtime,READY:.status.readyPods,CAPACITY:.spec.capacity.poolMin,WARM_IMAGES:.status.warmImages' 2>/dev/null || true
echo
log "status: fastlet pods (egress sidecar)"
kubectl -n "$RESOURCE_NS" get pods -l app=sandbox-fastlet \
-o custom-columns='NAME:.metadata.name,NODE:.spec.nodeName,CONTAINERS:.status.containerStatuses[*].name,READY:.status.containerStatuses[*].ready' 2>/dev/null || true
echo
log "status: DART P2P (block_source cache/peer/origin per node)"
dart_metrics_summary || true
echo
log "status: OpenSandbox ($OSB_NS)"
if kubectl get namespace "$OSB_NS" >/dev/null 2>&1; then
kubectl -n "$OSB_NS" get pods -o wide
printf ' server health: %s\n' "$(curl -fsS -m 5 "$SERVER_URL/health" >/dev/null 2>&1 && echo OK || echo unreachable)"
printf ' gateway health: %s\n' "$(curl -fsS -m 5 "$GATEWAY_URL/status.ok" >/dev/null 2>&1 && echo OK || echo unreachable)"
printf ' server URL: %s (header OPEN-SANDBOX-API-KEY: %s)\n' "$SERVER_URL" "$SERVER_API_KEY"
printf ' gateway URL: %s (header routing, signed f1.* scopes)\n' "$GATEWAY_URL"
else
echo " (not deployed)"
fi
if [[ "$OCI_STACK" == 1 ]]; then
log "status: OverlayBD OCI"
if kubectl -n "$RESOURCE_NS" get deploy registry >/dev/null 2>&1; then
printf ' registry: %s (%s)\n' "$OCI_REGISTRY_HOST" \
"$(kubectl -n "$RESOURCE_NS" get deploy registry -o jsonpath='{.status.readyReplicas}/{.status.replicas}')"
else
echo " registry: (not deployed)"
fi
local tid ref
for tid in "$(cat "$WORK/template-id" 2>/dev/null || true)" "$(cat "$WORK/overlaybd-template-id" 2>/dev/null || true)"; do
[[ -n "$tid" ]] || continue
ref="$(osb_api template get "$tid" -o json 2>/dev/null | jq -r '[.status.phase, .status.manifest_ref // "-"] | join(" ")')"
printf ' template %s: %s\n' "$tid" "${ref:-unreachable}"
done
fi
}
env_summary() {
highlight "== environment summary =="
printf ' %-22s %s\n' "kind cluster" "$KIND_CLUSTER ($(kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ') nodes)"
printf ' %-22s %s\n' "fast-sandbox" "pinned $(git -C "$FSB_DIR" rev-parse --short HEAD 2>/dev/null || echo "$FSB_COMMIT") ($FSB_DIR)"
printf ' %-22s %s\n' "RustFS endpoint" "$RUSTFS_ENDPOINT"
printf ' %-22s %s\n' "pool" "$POOL_NAME (runtime=firecracker, poolMin=$POOL_MIN, egress=$IMG_EGRESS)"
# The echo endpoint line is conditional: the standalone pool action
# loads the fingerprint pin but never re-derives the echo container IP.
local mitm_echo_at=""
[[ -n "$MITM_ECHO_IP" ]] && mitm_echo_at=", echo=https://$MITM_ECHO_DOMAIN via $MITM_ECHO_IP:$MITM_ECHO_PORT"
printf ' %-22s %s\n' "MITM CA" "$MITM_CA_SECRET (fingerprint ${MITM_CA_FINGERPRINT:0:16}…${mitm_echo_at})"
printf ' %-22s %s\n' "P2P" "DART daemons=$(printf '%s' "$(runtime_pods)" | wc -w | tr -d ' ') (on-demand pulls: cache -> peer -> origin)"
printf ' %-22s %s\n' "template" "${TEMPLATE_ID:-n/a} ($(if [[ -n "$TEMPLATE_ID" ]]; then _template_phase || echo unknown; else echo "not built"; fi))"
printf ' %-22s %s\n' "StateRoot fs" "$(findmnt -no FSTYPE "$XFS_MOUNT_POINT" 2>/dev/null || echo 'plain directory (full copy per sandbox)')"
printf ' %-22s %s\n' "server" "$IMG_SERVER -> $SERVER_URL (fsb runtime)"
printf ' %-22s %s\n' "ingress gateway" "$IMG_INGRESS -> $GATEWAY_URL (fsb provider, header mode)"
printf ' %-22s %s\n' "fastpath" "$FASTPATH_ENDPOINT"
printf ' %-22s %s\n' "logs" "$LOGS_DIR"
}
# --- down ------------------------------------------------------------------------------------
down() {
log "down: teardown"
if kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null; then
kind delete cluster --name "$KIND_CLUSTER" > "$LOGS_DIR/kind-delete.log" 2>&1 || true
fi
[[ -z "$(kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" || true)" ]] \
|| fail "kind cluster $KIND_CLUSTER still exists after delete"
docker rm -f "$RUSTFS_CONTAINER" >/dev/null 2>&1 || true
[[ -z "$(docker ps -a --filter "name=$RUSTFS_CONTAINER" --format '{{.Names}}' || true)" ]] \
|| fail "RustFS container still present"
# TLS echo endpoint + its CA material die with the run (regenerated fresh
# on every up; a stale CA must never resurface).
docker rm -f "$MITM_ECHO_CONTAINER" >/dev/null 2>&1 || true
[[ -z "$(docker ps -a --filter "name=$MITM_ECHO_CONTAINER" --format '{{.Names}}' || true)" ]] \
|| fail "MITM echo container still present"
rm -rf "$MITM_CA_DIR"
# Server/gateway die with the cluster; only the signing key outlives it.
rm -f "$WORK/agent-registry.json" "$SIGNING_KEY_FILE"
rm -rf "$GEN_DIR" "$FSB_GEN_DIR"
# A stale profile marker would let idle-e2e run against a stack whose
# Redis/sweeper are gone.
rm -f "$WORK/idle-pause"
# Same stale-marker risk: sdk-e2e and idle-e2e read
# $WORK/overlaybd-template-id, and their producer (template_up_overlaybd)
# only runs in verify-overlaybd.
rm -f "$WORK/overlaybd-template-id"
# RustFS data is owned by UID/GID 10001; leftovers pollute the host and
# later docker build contexts.
sudo_ rm -rf "$RUSTFS_DATA"
sudo_ rm -rf "$RC_CONFIG_DIR"
sysctl_restore
stateroot_xfs_down
# A committed DART cache is FINAL (never refreshed): purge it, or a
# rebuilt SandboxTemplate stays ignored when the StateRoot survives.
stateroot_scrub
# OCI_STACK=1: the kernel ublk module outlives the cluster, and a run
# killed mid-add leaves devices behind with no daemon to release them.
# On the Al8 6.6 backport ublk registers NO block major — every device
# consumes one of a fixed 64 char minors (host /dev/ublkc0..63, major
# 235, devtmpfs-created) — so the leak surfaces there, and once all
# 64 minors are gone every later `ublkd add` fails with "device
# failed to start" (observed on the nightly runner: a full pool
# accumulated over several days). Nothing holds the module at this
# point, so recycle it whenever leftover nodes exist.
if [[ "$OCI_STACK" == 1 ]]; then
local ublk_leftovers
ublk_leftovers="$(
{
ls -A /dev 2>/dev/null | grep -E '^ublkc[0-9]+$' || true
ls -A /sys/block 2>/dev/null | grep -E '^ublkb[0-9]+$' || true
} | tr '\n' ' ' | tr -s ' ')"
if [[ -n "$ublk_leftovers" ]]; then
log "releasing orphan ublk devices left by this run ($(printf '%s\n' "$ublk_leftovers" | wc -w) node(s)): $ublk_leftovers"
sudo_ modprobe -r ublk_drv 2>/dev/null \
|| log "WARNING: modprobe -r ublk_drv failed (module busy?); the next run's ublk probe will retry the heal"
sudo_ modprobe ublk_drv 2>/dev/null || true
grep -qiE 'ublk-char|ublkc' /proc/devices \
|| log "WARNING: ublk not registered after the recycle; the next run's up will modprobe it"
fi
fi
pass "host cleanup complete"
}
# --- main --------------------------------------------------------------------------------------
usage() {
cat <<'EOF'
usage: integration-env.sh [--auto-clean] {up|verify-native|verify-overlaybd|down|status|pool|sdk-e2e|idle-e2e}
up initialize the environment: fast-sandbox@pinned-commit images,
two-node kind cluster (KVM), RustFS, control plane, firecracker
runtime readiness + DART (P2P), the native SandboxTemplate golden
image, firecracker-egress-pool (egress attached), and the
source-built OpenSandbox server + ingress gateway. With
IDLE_PAUSE=1: Redis + server [automation.idle_pause] + gateway
activity/wake flags.
verify-native run the verify matrix against the native template
(create -> gateway route -> execd /ping, pause/resume,
vault, idle-pause, snapshot); requires `up`
verify-overlaybd build the overlaybd SandboxTemplate (oci:// publication)
and run the same verify matrix against it; requires `up`
(no-op when OCI_STACK=0)
pool re-apply only the SandboxPool (after editing manifests/pool/)
status nodes / pods / pool / DART P2P counters / RustFS / OpenSandbox health
sdk-e2e run the Python SDK e2e suite (tests/python/tests/test_fsb_e2e.py)
against the live stack; requires `up` (template id at
$WORK/template-id) and uv on PATH. Extra pytest args go through
PYTEST_ADDOPTS.
idle-e2e run the OSEP-0024 idle-pause/wake SDK suite
(tests/python/tests/test_fsb_idle_pause_e2e.py); requires an
`up` with IDLE_PAUSE=1 and uv on PATH.
down teardown: kind cluster + RustFS + sysctl + XFS StateRoot + caches
--auto-clean on up failure, run down automatically before dumping logs
The RustFS migration replaces MINIO_* and MC_IMAGE overrides. Before
upgrading an existing environment, run `down` with the previous script.
Notable env overrides: WORK, FSB_DIR, KIND_CLUSTER, KIND_SINGLE,
DOCKER_MIRROR, RUSTFS_*, RC_IMAGE, EGRESS_IMAGE, SERVER_IMAGE, INGRESS_IMAGE,
FSB_GOPROXY (default direct; set e.g. https://mirrors.aliyun.com/goproxy/,direct
when the host cannot reach module VCS hosts directly),
IMAGE_<COMPONENT>, POOL_MIN/POOL_MAX, WARM_IMAGES=1, SBX_IMAGE, EXECD,
XFS_STATEROOT=0, OSB_PACKAGE/OSB_BIN, SKIP_TOOL_INSTALL=1, SKIP_LEFTOVER_CLEAN=1,
IDLE_PAUSE=1 (OSEP-0024 idle-pause/wake profile; REDIS_IMAGE overrides the
Redis image), OCI_STACK=0 (disable the OverlayBD OCI registry + node stack;
default on — native and overlaybd paths run side by side in one cluster).
See the header of this script.
EOF
exit 1
}
# Allow focused tests to load the stage functions without starting the environment.
[[ "${BASH_SOURCE[0]}" == "$0" ]] || return 0
for arg in "$@"; do
case "$arg" in
--auto-clean) AUTO_CLEAN=1 ;;
up|down|status|pool|sdk-e2e|idle-e2e|verify-native|verify-overlaybd) ACTION="$arg" ;;
*) usage ;;
esac
done
[[ -n "$ACTION" ]] || usage
mkdir -p "$WORK" "$LOGS_DIR"
case "$ACTION" in
up)
exec > >(tee -a "$WORK/run.log") 2>&1
log "=== fast-sandbox-env up ($(date -u +%FT%TZ)) ==="
{
echo "environment snapshot ($(date -u +%FT%TZ))"
command -v kind >/dev/null && kind --version
kubectl version --client 2>/dev/null | head -1
go version
docker --version
echo "cluster=$KIND_CLUSTER single=$KIND_SINGLE rustfs=$RUSTFS_IMAGE port=$RUSTFS_PORT bucket=$RUSTFS_BUCKET"
echo "sbxImage=$SBX_IMAGE execd=$EXECD warmImages=$WARM_IMAGES ociStack=$OCI_STACK registry=$([[ "$OCI_STACK" == 1 ]] && echo "$OCI_REGISTRY_HOST" || echo off)"
echo "pool=$POOL_NAME poolMin=$POOL_MIN egress=$IMG_EGRESS"
echo "server=$IMG_SERVER ingress=$IMG_INGRESS fastpath=$FASTPATH_ENDPOINT"
echo "images: controller=$IMG_CONTROLLER runtime=$IMG_RUNTIME"
} > "$LOGS_DIR/environment.txt" 2>&1 || true
if [[ -n "$(kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" || true)" ]] \
|| docker ps -a --format '{{.Names}}' | grep -qx "$RUSTFS_CONTAINER"; then
if [[ "$SKIP_LEFTOVER_CLEAN" == 1 ]]; then
log "leftover resources detected; aborting (SKIP_LEFTOVER_CLEAN=1). Run 'integration-env.sh down' first"
exit 1
fi
log "leftover resources detected; cleaning and rebuilding"
down
fi
trap 'on_error up' ERR
run_stage "preflight + tooling" preflight
run_stage "opensandbox CLI (osb, latest from PyPI)" ensure_osb
run_stage "sysctl (fs.inotify)" sysctl_set
run_stage "fast-sandbox checkout @ pinned commit" ensure_fsb
run_stage "OCI registry CA + serving certificate" registry_ca_up
run_stage "ublk registration check" ublk_module_up
run_stage "build images (fast-sandbox + OpenSandbox)" build_images
run_stage "XFS StateRoot (reflink)" stateroot_xfs_up
run_stage "render kind config" render_kind_config
run_stage "kind cluster (KVM passthrough + labels)" kind_up
run_stage "RustFS + bucket" rustfs_up
run_stage "RustFS endpoint (kind network)" resolve_rustfs_endpoint
run_stage "OCI registry (in-cluster, TLS)" registry_up
run_stage "CRDs + control plane" control_plane_up
run_stage "credentials (publish/pull)" credentials_up
run_stage "MITM CA material (OSEP-0022 trust anchor)" mitm_ca_up
run_stage "firecracker runtime readiness + DART (P2P)" runtime_up
run_stage "idle-pause Redis (OSEP-0024 profile)" redis_up
run_stage "OpenSandbox server + ingress gateway" opensandbox_up
run_stage "TLS echo endpoint (cert signed by the pinned CA)" mitm_echo_up
run_stage "SandboxTemplate build (opensandbox CLI)" template_up
run_stage "SandboxPool $POOL_NAME (egress + P2P)" pool_up
trap - ERR
stage_summary
env_summary
highlight "== up complete: server=$SERVER_URL (header OPEN-SANDBOX-API-KEY: $SERVER_API_KEY), gateway=$GATEWAY_URL =="
highlight "== next: verify-native, verify-overlaybd, sdk-e2e, idle-e2e =="
;;
pool)
exec > >(tee -a "$WORK/run.log") 2>&1
kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null \
|| die "cluster $KIND_CLUSTER is not up (run up first)"
trap 'on_error pool' ERR
run_stage "SandboxPool $POOL_NAME (egress + P2P)" pool_up
trap - ERR
env_summary
;;
status)
status
;;
sdk-e2e)
kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null \
|| die "cluster $KIND_CLUSTER is not up (run up first)"
trap 'on_error sdk-e2e' ERR
sdk_e2e
trap - ERR
;;
idle-e2e)
kind get clusters 2>/dev/null | grep -x "$KIND_CLUSTER" >/dev/null \
|| die "cluster $KIND_CLUSTER is not up (run up first)"
trap 'on_error idle-e2e' ERR
idle_e2e
trap - ERR
;;
verify-native)
exec > >(tee -a "$WORK/run.log") 2>&1
log "=== fast-sandbox-env verify-native ($(date -u +%FT%TZ)) ==="
trap 'on_error verify-native' ERR
verify_native
trap - ERR
stage_summary
;;
verify-overlaybd)
exec > >(tee -a "$WORK/run.log") 2>&1
log "=== fast-sandbox-env verify-overlaybd ($(date -u +%FT%TZ)) ==="
trap 'on_error verify-overlaybd' ERR
verify_overlaybd
trap - ERR
stage_summary
;;
down)
down
;;
esac