Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(fast-sandbox-env): probe the ublk add/del path before the overlaybd build and self-heal wedges

高然b82dc2fa40e0
diff --git a/scripts/fast-sandbox-env/integration-env.sh b/scripts/fast-sandbox-env/integration-env.sh
index 63d504c59..1fe2c8f0f 100755
--- a/scripts/fast-sandbox-env/integration-env.sh
+++ b/scripts/fast-sandbox-env/integration-env.sh
@@ -379,11 +379,26 @@ failure_dump() {
kubectl get pods -n "$NS" -o wide 2>&1 || true
echo "--- controller logs (tail) ---"
kubectl logs -n "$NS" deploy/fast-sandbox-controller --tail=80 2>&1 || true
- echo "--- firecracker-runtime logs (tail) ---"
- kubectl logs -n "$NS" daemonset/firecracker-runtime --all-containers --tail=80 2>&1 || true
- # --all-containers: the label selector alone logs only the first
- # container and drops egress. Deep tail: failure loops flood the
- # window and bury the pre-incident history.
+ echo "--- firecracker-runtime logs (every pod + container, tail) ---"
+ # `kubectl logs daemonset/...` resolves to ONE pod only, so a wedged
+ # ublkd on the other node stays invisible. Loop the pods instead,
+ # and add the ublkd audit log plus the kernel device table: the
+ # daemon reports device-start failures to the audit log, not to
+ # stdout ("device failed to start (see daemon log)").
+ local rpod rcontainer
+ for rpod in $(runtime_pods 2>/dev/null || true); do
+ for rcontainer in $(kubectl -n "$NS" get pod "$rpod" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do
+ echo "### $rpod container=$rcontainer"
+ kubectl logs -n "$NS" "$rpod" -c "$rcontainer" --tail=80 2>&1 || true
+ done
+ echo "### $rpod overlaybd-ublkd audit log + kernel ublk state"
+ kubectl exec -n "$NS" "$rpod" -c overlaybd-ublkd -- sh -c '
+ tail -100 /var/log/overlaybd-audit.log 2>/dev/null
+ echo "-- /dev/ublk*:"
+ ls -la /dev/ublk* 2>&1
+ echo "-- /proc/devices (ublk):"
+ grep -i ublk /proc/devices 2>&1' 2>&1 || true
+ done
echo "--- fastlet pod logs, every container (deep tail) ---"
kubectl logs -n "$RESOURCE_NS" -l app=sandbox-fastlet \
--all-containers --prefix --timestamps --tail=15000 2>&1 || true
@@ -1939,6 +1954,134 @@ ublk_module_up() {
pass "ublk module registered"
}
+# --- stage: real ublk add/del probe + wedge self-heal (OCI_STACK=1) --------------
+
+# Registration in /proc/devices is necessary but not sufficient: a run
+# killed mid-add leaks kernel ublk devices that outlive the kind cluster
+# (the module is never reloaded), and one leaked device wedges every later
+# `ublkd add` with the daemon's "device failed to start" — surfacing only
+# ~16 minutes into the overlaybd template build. This probe drives the same
+# daemon -> kernel path the build uses (POST /v1/add, then POST /v1/del on
+# the node ublkd's control socket) with a minimal upper-only device, so a
+# wedge surfaces in seconds and the heal below clears it automatically.
+#
+# The probe runs inside each node's overlaybd-ublkd sidecar: that container
+# holds both the control socket and the overlaybd tooling, so the device
+# created is exactly the kind a template build creates.
+ublk_probe_once() { # pod -> prints PROBE_RESULT=<class> first, details after
+ local pod="$1"
+ kubectl exec -i -n "$NS" "$pod" -c overlaybd-ublkd -- sh -s <<'PROBE' 2>&1 || true
+sock=/run/fast-sandbox/overlaybd/ublkd.sock
+tools=/opt/fast-sandbox/oci/current/bin
+if ! ping=$(curl -sS -m 5 --noproxy '*' --unix-socket "$sock" http://ublkd/v1/ping 2>&1); then
+ echo "PROBE_RESULT=ping-failed"
+ printf '%s\n' "$ping"
+ exit 0
+fi
+d=$(mktemp -d /tmp/fsb-ublk-probe.XXXXXX) || { echo "PROBE_RESULT=inconclusive"; echo "mktemp failed"; exit 0; }
+trap 'rm -rf "$d"' EXIT
+if ! "$tools/overlaybd-create" --hybrid "$d/data" "$d/index" 1 >/dev/null 2>&1; then
+ echo "PROBE_RESULT=inconclusive"
+ echo "overlaybd-create --hybrid failed"
+ exit 0
+fi
+printf '{"lowers":[],"upper":{"index":"%s","data":"%s","vsize":1},"resultFile":"%s"}' \
+ "$d/index" "$d/data" "$d/result" > "$d/config.json"
+reply=$(curl -sS -m 120 --noproxy '*' --unix-socket "$sock" -w '\n%{httpcode}' \
+ -X POST -H 'Content-Type: application/json' \
+ -d "{\"config\":\"$d/config.json\"}" http://ublkd/v1/add 2>&1)
+code=$(printf '%s\n' "$reply" | tail -n 1)
+body=$(printf '%s\n' "$reply" | sed '$d')
+case "$body" in
+*'"ok":true'*)
+ # add succeeded: release the probe device before it leaks — a leaked
+ # probe device would wedge the very next add, so a failed del is
+ # reported as wedged (the heal recycles the module and clears it).
+ dev_id=$(printf '%s' "$body" | tr '{,' '\n\n' | grep -o '"dev_id":[0-9]*' | head -1 | tr -cd '0-9')
+ if [ -n "$dev_id" ] && curl -sS -m 30 --noproxy '*' --unix-socket "$sock" \
+ -X POST -H 'Content-Type: application/json' \
+ -d "{\"dev_id\":$dev_id}" http://ublkd/v1/del >/dev/null 2>&1; then
+ echo "PROBE_RESULT=ok dev_id=$dev_id"
+ else
+ echo "PROBE_RESULT=wedged"
+ echo "add succeeded but del failed (dev_id=$dev_id)"
+ fi
+ ;;
+*"device failed to start"*)
+ echo "PROBE_RESULT=wedged"
+ printf '%s\n' "$body"
+ ;;
+*)
+ echo "PROBE_RESULT=inconclusive"
+ echo "http=$code"
+ printf '%s\n' "$body"
+ ;;
+esac
+PROBE
+}
+
+ublk_heal_wedge() {
+ log "heal: recycling the host ublk module (nothing holds it while the stack is up; leaked devices die with it)"
+ sudo_ modprobe -r ublk_drv 2>/dev/null || true
+ sudo_ modprobe ublk_drv 2>/dev/null || true
+ grep -qiE 'ublk-char|ublkc' /proc/devices \
+ || die "ublk module did not register after the recycle: the runner host needs a reboot to clear the wedged ublk state"
+ log "heal: restarting daemonset/firecracker-runtime (sidecars must reopen the fresh control node)"
+ kubectl -n "$NS" rollout restart daemonset/firecracker-runtime >/dev/null
+ kubectl -n "$NS" rollout status daemonset/firecracker-runtime --timeout=300s >/dev/null
+}
+
+ublk_probe_or_heal() {
+ [[ "$OCI_STACK" == 1 ]] || { log "OCI_STACK=0: skipping the ublk probe (overlaybd disabled)"; return 0; }
+ local pods pod result detail healed recheck reok
+ pods="$(runtime_pods 2>/dev/null || true)"
+ [[ -n "$pods" ]] || die "no firecracker-runtime pods to probe (run up first)"
+ healed=0
+ for pod in $pods; do
+ detail="$(ublk_probe_once "$pod")"
+ result="$(printf '%s\n' "$detail" | awk '/^PROBE_RESULT=/ && !found {sub(/^PROBE_RESULT=/, ""); print; found=1}')"
+ case "$result" in
+ ok*)
+ pass "ublk probe ok on ${pod}: node-side add/del through the real daemon -> kernel path"
+ ;;
+ wedged)
+ printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
+ if [[ "$healed" == 1 ]]; then
+ fail "ublk probe still wedged on ${pod} after the heal: the runner host's ublk state needs a reboot to recover"
+ fi
+ log "ublk probe WEDGED on ${pod}: the daemon cannot start devices; healing"
+ ublk_heal_wedge
+ healed=1
+ reok=1
+ # The restart reset every daemon, so re-probe all pods.
+ for recheck in $pods; do
+ detail="$(ublk_probe_once "$recheck")"
+ result="$(printf '%s\n' "$detail" | awk '/^PROBE_RESULT=/ && !found {sub(/^PROBE_RESULT=/, ""); print; found=1}')"
+ if [[ "$result" == ok* ]]; then
+ pass "ublk probe ok after heal on ${recheck}"
+ else
+ reok=0
+ log "ublk probe after heal on ${recheck}: ${result:-no-result}"
+ printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
+ fi
+ done
+ [[ "$reok" == 1 ]] || fail "ublk probe still unhealthy after the heal (probe output above)"
+ ;;
+ ping-failed)
+ printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
+ fail "ublk probe: the daemon on ${pod} is unreachable on its control socket (runtime not healthy)"
+ ;;
+ *)
+ # An unexpected add response means the probe config hit a
+ # validation path, not the kernel wedge; never block a
+ # working stack on it — the build reports the real error.
+ log "WARNING: ublk probe inconclusive on ${pod} (result: ${result:-none}); continuing"
+ printf '%s\n' "$detail" | sed 's/^/[fast-sandbox-env] /' >&2
+ ;;
+ esac
+ done
+}
+
render_pool() { # > $GEN_DIR/firecracker-egress-pool.yaml
local src="$MANIFESTS_DIR/pool/firecracker-egress-pool.yaml" out="$GEN_DIR/firecracker-egress-pool.yaml"
mkdir -p "$GEN_DIR"
@@ -3280,6 +3423,9 @@ verify_overlaybd() {
log "OCI_STACK=0: overlaybd disabled; nothing to verify"
return 0
}
+ # Probe before building: a wedged host ublk table kills the template
+ # build ~16 minutes in; the probe fails in seconds and heals itself.
+ run_stage "ublk add/del probe (node daemon -> kernel path)" ublk_probe_or_heal
run_stage "SandboxTemplate build (overlaybd, oci:// publication)" template_up_overlaybd
run_stage "end-to-end verify (overlaybd template)" opensandbox_verify "$OCI_TEMPLATE_ID"
run_stage "pause/resume verify (overlaybd template)" pause_resume_verify "$OCI_TEMPLATE_ID"
@@ -3424,6 +3570,28 @@ down() {
# A committed DART cache is FINAL (never refreshed): purge it, or a
# rebuilt SandboxTemplate stays ignored when the StateRoot survives.
stateroot_scrub
+ # OCI_STACK=1: the kernel ublk module outlives the cluster, and a run
+ # killed mid-add leaves devices behind with no daemon to release them.
+ # One leaked device wedges every later `ublkd add` ("device failed to
+ # start") on every node of the next run. Nothing holds the module at
+ # this point, so recycle it whenever the device table is not empty;
+ # listing the leftovers first keeps the log actionable.
+ if [[ "$OCI_STACK" == 1 ]]; then
+ local ublk_leftovers
+ ublk_leftovers="$(
+ {
+ awk 'NR > 1 && $4 ~ /^ublkb/ {printf "%s ", $4}' /proc/partitions 2>/dev/null || true
+ ls -A /sys/block 2>/dev/null | grep -o '^ublkb[0-9]*' 2>/dev/null | tr '\n' ' ' || true
+ } | tr -s ' ')"
+ if [[ -n "$ublk_leftovers" ]]; then
+ log "releasing orphan ublk devices left by this run: $ublk_leftovers"
+ sudo_ modprobe -r ublk_drv 2>/dev/null \
+ || log "WARNING: modprobe -r ublk_drv failed (module busy?); the next run's ublk probe will retry the heal"
+ sudo_ modprobe ublk_drv 2>/dev/null || true
+ grep -qiE 'ublk-char|ublkc' /proc/devices \
+ || log "WARNING: ublk not registered after the recycle; the next run's up will modprobe it"
+ fi
+ fi
pass "host cleanup complete"
}