Skip to content

The first fruit of the Nüwa tree — an autonomous AI pentest legion (634-team champion, 8h unattended, 1B tokens beat 4.5B). Not just a tool: evolvable by design. Grow your own fruit on the Nüwa Foundation (Create/Language/History).

29.5 KB
Raw Blame
#!/usr/bin/env bash
# watchdog.sh — the dumb guardian loop (commander architecture, SPEC unit 2).
#
# Failure modes handled (single sweep, sleep 15s, checks in SPEC order):
#   1. commander pane dead (DRIVE_PANE_EXITED / window gone / no claude process)
#      -> relaunch claude in window "commander" (own cwd ROOT/commander-home)
#      with cwd-scoped --continue, wait for input box, inject bootstrap prompt
#   2. bootstrap text stuck unsubmitted in the commander input box
#      -> re-Enter (3 tries), then restart the commander window
#   3. trust/permission confirm dialogs in ANY pane -> auto-accept (Down+Enter)
#   4. quota/auth errors in ANY pane -> rotate_key(); TUI workers and the
#      commander restart budget-free; -p workers only logged (the commander
#      notices via board). Rotation never burns restart budget.
#   5. TUI worker DRIVE_PANE_EXITED -> auto-relaunch with cwd-scoped --continue,
#      max 3/pane (restarts map), budget-independent. -p worker exit = normal
#      completion: pane left for the commander (peek + worker.sh kill).
#   6. global deadline (SAPPER_TOTAL_MINUTES from script start) -> C-c all
#      worker panes, finish_flags for EVERY code with a graph, arena close
#      every open instance, log summary, exit 0 (failsafe finish when the
#      commander is dead at the buzzer).
#   7. worker TUI stall -> deliberately NOTHING (nudging is the commander's job).
#
# Explicitly NOT here (zero strategy): challenge selection, spawning new
# challenges, strategy kills, mission text, nudging. Every relaunch reuses the
# DRIVE_PANE_EXITED convention so death stays detectable. Relaunches never
# re-send worker missions — the commander re-briefs amnesiac panes.
set -o pipefail

ROOT="${SAPPER_ROOT:-/work/runs}"
LOG="$ROOT/drive.log"
# Model tiers (SKILL 4.9): the commander runs SMART; each worker keeps the tier
# it was spawned with (persisted by worker.sh in ROOT/pane-env/<pane>.env).
CMD_MODEL="${SAPPER_MODEL:-glm-5.3[1m]}"
FALLBACK_MODEL="${SAPPER_MODEL_FLASH:-glm-5.3-flash[1m]}"
MODEL="$CMD_MODEL"      # legacy var: current relaunch default, updated on rotation
TOTAL_MIN="${SAPPER_TOTAL_MINUTES:-470}"
MAX_WORKERS="${SAPPER_MAX_WORKERS:-8}"
STALL_SECONDS="${SAPPER_STALL_SECONDS:-600}"
SESSION=sapper-cmd
CMD_HOME="$ROOT/commander-home"
PANE_ENV_DIR="$ROOT/pane-env"
SKILL_PATH=/opt/sapper/docker/commander/SKILL.md
MAX_RESTARTS=3          # per TUI worker pane (SPEC unit 2 #5)
mkdir -p "$ROOT" "$CMD_HOME" "$PANE_ENV_DIR"
# stdout only: the entrypoint nohup-redirects us into drive.log (tee here
# double-writes every line — observed in the sim log)
log() { printf '[watch] %s\n' "$*"; }
END_AT=$(( $(date +%s) + TOTAL_MIN * 60 ))
NOW() { date +%s; }

declare -A mode restarts enter_tries mission_restarts last_activity placeholder_since cooldown_log

# ---- arena helper (verbatim from tmux-drive.sh) ------------------------------
arena() {
  python3 - "$@" <<'PYEOF2'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("arena", "/opt/sapper/lib/arena.py")
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
c = m.ArenaClient.load(None)
op = sys.argv[1]
if op == "list":
    print(json.dumps(c.list_challenges()))
elif op == "start":
    print(json.dumps(c.start_challenge(sys.argv[2])))
elif op == "close":
    print(json.dumps(c.close_challenge(sys.argv[2])))
PYEOF2
}

# ---- key-pool rotation ------------------------------------------------------
# SAPPER_KEY_POOL: JSON [{base_url,token,model},...]. On quota/balance/auth
# errors the watchdog rotates its own env; panes relaunched afterwards
# inherit the new key, and `claude --continue` restores the prior session.
# STORM GUARD (live incident 17:36-18:22): with every pool key dead the
# rotation bounced every 16s and relaunched the whole fleet endlessly.
# A 5-minute cooldown between rotations + a 6/hour cap turns that storm
# into a slow drip; transient 429s ride claude's own retry instead.
LAST_ROTATE=0
ROTATIONS_THIS_HOUR=0
HOUR_MARK=$(NOW)
rotation_allowed() {
  local now; now=$(NOW)
  if (( now - HOUR_MARK > 3600 )); then HOUR_MARK=$now; ROTATIONS_THIS_HOUR=0; fi
  if (( ROTATIONS_THIS_HOUR >= 6 )); then
    cooldown_log pool-exhausted "rotation cap (6/h) hit — pool likely fully dead; pausing rotations this hour"
    return 1
  fi
  if (( now - LAST_ROTATE < 300 )); then
    cooldown_log rot-cooldown "rotation cooldown (5min) — transient errors should ride claude's own retry"
    return 1
  fi
  return 0
}
rotate_key() {
  python3 - <<'PYROT'
import json, os
raw = os.environ.get("SAPPER_KEY_POOL", "").strip()
try:
    pool = [p for p in json.loads(raw) if isinstance(p, dict) and p.get("token")]
except Exception:
    pool = []
if not pool:
    raise SystemExit(1)
def fam(e):
    u = (e.get("base_url") or "") + " " + (e.get("model") or "")
    if "bigmodel" in u or "glm" in u: return "glm"
    if "deepseek" in u: return "ds"
    if "moonshot" in u or "kimi" in u: return "kimi"
    return "other"
cur = os.environ.get("ANTHROPIC_AUTH_TOKEN", "")
cur_fam = next((fam(e) for e in pool if e["token"] == cur), None)
if cur_fam is None:  # main key not in pool: classify by its base_url
    b = os.environ.get("ANTHROPIC_BASE_URL", "")
    cur_fam = "glm" if "bigmodel" in b else ("ds" if "deepseek" in b else "other")
# tried-memory (1h window): a family whose every key already failed this hour
# is skipped, so two dead glm keys bounce K1<->K2 forever instead of crossing
# to deepseek. All tried -> quiet stop; the window clears in an hour.
import time
root = os.environ.get("SAPPER_ROOT", "/work/runs")
tried_path = root + "/.rotated-tried.txt"
now = time.time()
tried = []
try:
    for ln in open(tried_path):
        ts, tk = ln.strip().split(" ", 1)
        if now - float(ts) < 3600:
            tried.append(tk)
except Exception:
    pass
same = [e for e in pool if fam(e) == cur_fam]
# same-family FIRST, untried first: a glm pane must never be flung onto
# deepseek while an untried glm key still lives (the mismatch crash-loop)
cands = [e for e in same if e["token"] != cur and e["token"] not in tried]
if not cands:
    cands = [e for e in same if e["token"] != cur]  # family exhausted: retry oldest? no—cross below
    cands = []
if not cands:
    cands = [e for e in pool if e["token"] != cur and e["token"] not in tried]
if not cands:
    raise SystemExit(1)  # everything tried this hour: quiet stop
nxt = cands[0]
model = nxt.get("model") or os.environ.get("SAPPER_MODEL", "")
base = nxt.get("base_url") or os.environ.get("ANTHROPIC_BASE_URL", "")
root = os.environ.get("SAPPER_ROOT", "/work/runs")
with open(root + "/rotated.env", "w") as fh:  # global (commander follows this)
    fh.write(f'export ANTHROPIC_BASE_URL="{base}"\n'
             f'export ANTHROPIC_AUTH_TOKEN="{nxt["token"]}"\n'
             f'export SAPPER_MODEL="{model}"\n')
with open(f"{root}/rotated-{fam(nxt)}.env", "w") as fh:  # per-family current creds
    fh.write(f'ANTHROPIC_BASE_URL={base}\nANTHROPIC_AUTH_TOKEN={nxt["token"]}\nMODEL={model}\n')
if cur and any(e["token"] == cur for e in pool):
    with open(tried_path, "a") as fh:
        fh.write(f"{now} {cur}\n")
raise SystemExit(0)
PYROT
  local rc=$?
  if (( rc == 0 )) && [[ -f "$ROOT/rotated.env" ]]; then
    LAST_ROTATE=$(NOW); ROTATIONS_THIS_HOUR=$((ROTATIONS_THIS_HOUR + 1))
    local old_url="$ANTHROPIC_BASE_URL"
    # shellcheck disable=SC1090
    . "$ROOT/rotated.env"; export ANTHROPIC_BASE_URL ANTHROPIC_AUTH_TOKEN SAPPER_MODEL
    MODEL="${SAPPER_MODEL:-$MODEL}"   # watchdog addition: relaunches must pick up the rotated model
    if [[ -n "$old_url" && "$old_url" != "$ANTHROPIC_BASE_URL" ]]; then
      # cross-family rotation (e.g. glm exhausted -> deepseek). Do NOT rewrite
      # every pane-env: a glm pane flung onto the deepseek endpoint with a glm
      # model id crashes instantly and re-triggers rotation (the 19:04 storm).
      # glm panes keep their identity and simply ride the rotation cooldown;
      # the commander can re-dispatch them on --gw ds if it chooses.
      CMD_MODEL="${SAPPER_MODEL:-$CMD_MODEL}"   # commander relaunches must answer on the new provider too
      log "key rotated cross-family -> $ANTHROPIC_BASE_URL (commander follows; glm panes keep glm creds)"
    else
      log "key rotated -> $ANTHROPIC_BASE_URL (model $SAPPER_MODEL)"
    fi
    return 0
  fi
  return 1
}

QUOTA_RE='insufficient|balance|quota|exceeded|billing|payment|401|[Uu]nauthorized|invalid.{0,12}[Aa][Pp][Ii].[Kk]ey|欠费|余额'
# transient rate limits look quota-ish but mean the key is FINE (live root
# cause of the 17:36 storm: per-minute limits were misread as a dead key)
RATELIMIT_RE='每分钟|分钟级|rate.?limit|too many requests|429|1302|1311|1305' 

# submit every recorded-unsubmitted flag fact in a challenge's graph (verbatim)
finish_flags() {  # finish_flags <code>
  python3 - "$1" <<'PYFIN'
import glob, json, os, sys, importlib.util
code = sys.argv[1]
root = os.environ.get("SAPPER_ROOT", "/work/runs")
spec = importlib.util.spec_from_file_location("arena", "/opt/sapper/lib/arena.py")
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
try:
    client = m.ArenaClient.load(None)
except Exception as exc:
    print(f"[finish] {code}: no arena ({exc})"); sys.exit(0)
for path in sorted(glob.glob(f"{root}/{code}/.sapper/facts/*.json")):
    try:
        fact = json.load(open(path))
    except Exception:
        continue
    if fact.get("kind") != "flag":
        continue
    data = fact.get("data") or {}
    if data.get("status") not in (None, "recorded") or not data.get("value"):
        continue
    try:
        r = client.submit_flag(code, data["value"])
        fact["data"]["status"] = "submitted" if r.get("correct") else "rejected"
        json.dump(fact, open(path, "w"), ensure_ascii=False, indent=2)
        print(f"[finish] {code}: submit {data['value'][:24]}… correct={r.get('correct')}")
    except Exception as exc:
        msg = str(exc)
        if "already" in msg.lower() or "409" in msg:
            # duplicate submit = the flag WAS accepted earlier: absorb it
            fact["data"]["status"] = "submitted"
            json.dump(fact, open(path, "w"), ensure_ascii=False, indent=2)
            print(f"[finish] {code}: already submitted — marked ({data['value'][:20]}…)")
        else:
            print(f"[finish] {code}: submit failed ({exc})")
PYFIN
}

# ---- trust preset (PYTRUST + settings.json, from tmux-drive.sh launch()) ----
preset_trust() {  # preset_trust <dir> — idempotent; runs BEFORE a pane (re)spawn
  local dir="$1"
  mkdir -p "$dir/.claude"
  printf '%s\n' '{"permissions":{"defaultMode":"bypassPermissions"},"autoCompactEnabled":true}' > "$dir/.claude/settings.json"
  python3 - "$dir" <<'PYTRUST'
import json, os, sys
path = os.path.expanduser("~/.claude.json")
try: d = json.load(open(path))
except Exception: d = {}
d.setdefault("projects", {})[sys.argv[1]] = {
    "hasTrustDialogAccepted": True, "bypassPermissionsModeAccepted": True}
json.dump(d, open(path, "w"))
PYTRUST
}

# ---- tmux primitives (verbatim from tmux-drive.sh) ---------------------------
pane_text() { tmux capture-pane -pt "$SESSION:$1" 2>/dev/null; }

send_literal() {  # send_literal <window> <text> [enter]
  tmux send-keys -t "$SESSION:$1" -l -- "$2"
  sleep 0.5
  [[ "${3:-enter}" == "enter" ]] && tmux send-keys -t "$SESSION:$1" Enter
}

accept_dialogs() {  # accept_dialogs <window> <pane> -> 0 if a dialog was dismissed
  local win="$1" pane="$2"
  if printf '%s' "$pane" | grep -qE 'Enter to confirm|Yes, I accept|I accept'; then
    # Aim by the ❯ focus marker: security dialogs default-focus "No, exit",
    # so a bare Enter CHOOSES NO. Only move+confirm when focus is off Yes.
    if printf '%s' "$pane" | grep -qE '❯[^\n]*No, exit'; then
      tmux send-keys -t "$SESSION:$win" Down; sleep 0.4
    fi
    tmux send-keys -t "$SESSION:$win" Enter; sleep 1.5
    return 0
  fi
  return 1
}

# ---- relaunch machinery ------------------------------------------------------
sync_session_env() {  # tmux panes inherit the SESSION environment, NOT this
                      # process's env (only update-environment vars are copied
                      # from clients). After rotate_key() mutates our env, the
                      # rotated key MUST be pushed into the session or every
                      # relaunched pane boots claude on the stale dead key and
                      # re-triggers the quota error forever.
  local k v
  for k in PATH ANTHROPIC_BASE_URL ANTHROPIC_AUTH_TOKEN SAPPER_MODEL \
           SAPPER_MODEL_FLASH BENCHMARK_BASE_URL BENCHMARK_TOKEN; do
    v=${!k:-}
    [[ -n "$v" ]] && tmux set-environment -t "$SESSION" "$k" "$v" >/dev/null 2>&1 || true
  done
}

DS_MODEL="${SAPPER_MODEL_DS:-deepseek-flash}"
DS_BASE_URL="${DEEPSEEK_BASE_URL:-https://api.deepseek.com/anthropic}"
DS_KEY="${DEEPSEEK_API_KEY:-}"

pane_env_get() {  # pane_env_get <win> <KEY> — RAW value from pane-env/<win>.env
  [[ -f "$PANE_ENV_DIR/$1.env" ]] && sed -n "s/^$2=//p" "$PANE_ENV_DIR/$1.env" 2>/dev/null | head -1
}
pane_model() {  # pane_model <win> — the tier this worker was spawned with
  local m; m=$(pane_env_get "$1" MODEL)
  printf '%s' "${m:-$FALLBACK_MODEL}"
}

claude_resume_flag() {  # sets RESUME_FLAG="--continue" iff <cwd> has a prior session
  RESUME_FLAG=""
  # glob is only a has-this-cwd-run heuristic; claude resolves the session by
  # real cwd, so a loose match can never resume the WRONG session
  ls "$HOME/.claude/projects/"*"$(basename "$1")"*/ >/dev/null 2>&1 && RESUME_FLAG="--continue"
  return 0
}

wait_input_box() {  # wait_input_box <win> — 0 when '❯' shows (auto-accepting dialogs)
  local win="$1" waited=0 pane=""
  while (( waited < 75 )); do
    pane=$(pane_text "$win")
    accept_dialogs "$win" "$pane" && { waited=$((waited+3)); continue; }
    printf '%s' "$pane" | grep -q '❯' && return 0
    sleep 1; waited=$((waited+1))
  done
  log "$win: input box never appeared"
  return 1
}

relaunch_pane() {  # relaunch_pane <win> <cwd> <model> — claude TUI, same window, no mission text
  local win="$1" cwd="$2" model="$3" cmd gw engine gw_env
  preset_trust "$cwd"
  claude_resume_flag "$cwd"
  sync_session_env
  # worker panes carry their gateway/engine in pane-env; the commander is always claude+glm-family.
  # Inject THIS pane's FAMILY credentials: same-family rotation keeps a glm pane
  # on glm keys even when the global env (commander) has already crossed to
  # another provider — the mismatch crash-loop fix.
  engine=$(pane_env_get "$win" ENGINE)
  # NO family pinning anywhere (user doctrine: nothing is pinned). Every pane
  # follows the ROTATED GLOBAL credentials via the tmux session env
  # (sync_session_env pushes them after each rotation). Model follows too:
  # commander uses CMD_MODEL, workers use SAPPER_MODEL — both updated on
  # rotation, so a cross-family slide migrates key AND model together.
  # (The old per-family pin once handed the commander a dead glm key while its
  # model had already crossed to deepseek — a 15-min bootstrap boot-lock.)
  gw_env=""
  if [[ "$engine" == codex ]]; then
    local provider=deepseek; [[ "$DS_BASE_URL" == https://api.deepseek.com* ]] && provider=deepseek-local
    cmd="cd '$cwd' && DEEPSEEK_API_KEY='$DS_KEY' codex --skip-git-repo-check -c model_provider=$provider -c model='$model' --dangerously-bypass-approvals-and-sandbox 2>&1; echo DRIVE_PANE_EXITED; exec sleep infinity"
  else
    cmd="cd '$cwd' && $gw_env claude $RESUME_FLAG --model '$model' --effort max --dangerously-skip-permissions 2>&1; echo DRIVE_PANE_EXITED; exec sleep infinity"
  fi
  # archived mission (TUI workers): re-brief after the window comes back —
  # a fresh post-restart session would otherwise sit at an empty prompt
  REBRIEF_TEXT=""
  if [[ "$win" != "commander" ]] && command -v base64 >/dev/null 2>&1; then
    local rb; rb=$(pane_env_get "$win" MISSION_B64)
    [[ -n "$rb" ]] && REBRIEF_TEXT=$(printf '%s' "$rb" | base64 -d 2>/dev/null || true)
  fi
  if tmux list-windows -t "$SESSION" -F '#W' 2>/dev/null | grep -qx -- "$win"; then
    # window still exists: respawn in place (kill-window on a session's last
    # window would destroy the session — and the guardian with it)
    tmux respawn-pane -k -t "$SESSION:$win" "$cmd" 2>>"$LOG" || { log "$win: respawn failed"; return 1; }
  else
    tmux new-window -d -t "$SESSION" -n "$win" "$cmd" 2>>"$LOG" || { log "$win: new-window failed"; return 1; }
  fi
  last_activity[$win]=$(NOW)
  wait_input_box "$win"
  # re-brief: the pane's archived mission (fresh session post-restart would
  # otherwise sit at an empty prompt forever, never calling the LLM)
  if [[ -n "${REBRIEF_TEXT:-}" ]]; then
    tmux send-keys -t "$SESSION:$win" C-u 2>/dev/null; sleep 0.3
    send_literal "$win" "$REBRIEF_TEXT"
    log "$win: archived mission re-sent after relaunch"
    REBRIEF_TEXT=""
  fi
}

relaunch_commander() {  # relaunch_commander <reason> — SPEC unit 2 #1
  local reason="$1" verb
  log "commander: relaunching ($reason, cwd $CMD_HOME)"
  mkdir -p "$CMD_HOME"
  if relaunch_pane commander "$CMD_HOME" "$CMD_MODEL"; then
    verb="start commanding"
    [[ -f "$ROOT/commander-state.md" ]] && verb="resume command"
    # clear any stale text from a previous unsubmitted bootstrap first (else
    # send_literal concatenates onto it — seen live as doubled input text)
    tmux send-keys -t "$SESSION:commander" C-u 2>/dev/null
    sleep 0.3
    send_literal commander \
      "Read $SKILL_PATH and $verb. First read $ROOT/commander-state.md if it exists (your handover from a previous incarnation)."
    last_activity[commander]=$(NOW)
    log "commander: bootstrap prompt sent ($verb)"
  fi
  enter_tries[commander]=0
  mode[commander]=commander
  placeholder_since[commander]=0
}

relaunch_worker() {  # relaunch_worker <win> <reason> — cwd-scoped --continue, no mission
  local win="$1" reason="$2" cwd
  # the pane's `cd` survives death (exec sleep infinity keeps the shell cwd)
  cwd=$(tmux display-message -p -t "$SESSION:$win" '#{pane_current_path}' 2>/dev/null)
  if [[ -z "$cwd" || ! -d "$cwd" ]]; then
    log "$win: cannot determine cwd — leaving dead pane for the commander"
    return 1
  fi
  local m="${SAPPER_MODEL:-$(pane_model "$win")}"
  log "$win: relaunching TUI with --continue ($reason, cwd $cwd, model $m)"
  relaunch_pane "$win" "$cwd" "$m" || return 1
  mode[$win]=tui
  return 0
}

adopt_existing() {  # adopt windows we did not launch ourselves (worker.sh / entrypoint)
  local win cmd
  for win in $(tmux list-windows -t "$SESSION" -F '#W' 2>/dev/null || true); do
    [[ -n "${last_activity[$win]:-}" ]] && continue
    cmd=$(tmux display-message -p -t "$SESSION:$win" '#{pane_start_command}' 2>/dev/null)
    if [[ "$win" == "commander" ]]; then mode[$win]=commander
    elif printf '%s' "$cmd" | grep -q 'sapper auto'; then mode[$win]=p
    else mode[$win]=tui; fi
    last_activity[$win]=$(NOW); restarts[$win]=0
    log "$win: adopted existing window (mode=${mode[$win]})"
  done
}

ensure_session() {
  tmux has-session -t "$SESSION" 2>/dev/null && return 0
  log "tmux session $SESSION missing — recreating"
  tmux new-session -d -s "$SESSION" -n commander 2>>"$LOG" || { log "FATAL: cannot recreate tmux session"; return 1; }
  return 0
}

cooldown_log() {  # cooldown_log <key> <msg> — drive.log is also the operator console: no spam
  local key="$1" msg="$2" now; now=$(NOW)
  if (( now - ${cooldown_log[$key]:-0} > 300 )); then
    cooldown_log[$key]=$now
    log "$msg"
  fi
}

finish_all() {  # SPEC unit 2 #6 — global deadline failsafe
  local win g real code n_codes=0 n_closed=0
  log "global deadline reached — failsafe finish"
  for win in $(tmux list-windows -t "$SESSION" -F '#W' 2>/dev/null || true); do
    [[ "$win" == "commander" ]] && continue
    log "$win: C-c (deadline)"
    tmux send-keys -t "$SESSION:$win" C-c 2>/dev/null
  done
  sleep 3
  # every code with a graph; follow -wN .sapper symlinks back to the primary
  # code so shared graphs are finished exactly once
  local -A seen=()
  for g in "$ROOT"/*/.sapper; do
    [[ -e "$g" ]] || continue
    real=$(realpath "$g" 2>/dev/null) || continue
    code=$(basename "$(dirname "$real")")
    [[ -n "${seen[$code]:-}" ]] && continue
    seen[$code]=1; n_codes=$((n_codes+1))
    log "$code: deadline finish_flags"
    finish_flags "$code" 2>&1 | tee -a "$LOG"
  done
  # close every open platform instance (409/other errors must never crash us)
  for code in $(arena list 2>>"$LOG" | python3 -c '
import json,sys
try: items=json.load(sys.stdin)
except Exception: sys.exit(0)
for c in items:
    if isinstance(c,dict) and not c.get("is_completed"): print(c["unique_code"])' 2>/dev/null); do
    if arena close "$code" >/dev/null 2>&1; then
      log "$code: instance closed"; n_closed=$((n_closed+1))
    else
      log "$code: instance close failed (ignored)"
    fi
  done
  log "watchdog complete: graphs_finished=$n_codes instances_closed=$n_closed model=$MODEL"
  exit 0
}

# ---- startup -----------------------------------------------------------------
keys=$(printf '%s' "${SAPPER_KEY_POOL:-}" | python3 -c '
import json,sys
try: print(len([p for p in json.load(sys.stdin) if isinstance(p,dict) and p.get("token")]))
except Exception: print(0)' 2>/dev/null || echo 0)
log "watchdog start: root=$ROOT session=$SESSION cmd_home=$CMD_HOME"
printf '%s\n' "$END_AT" > "$ROOT/.deadline"   # board reads this: budget_left heartbeat
log "env: model=$MODEL total=${TOTAL_MIN}m max_workers=$MAX_WORKERS stall=${STALL_SECONDS}s(informational) keys=$keys"
log "env: benchmark=${BENCHMARK_BASE_URL:-unset} gateway=${ANTHROPIC_BASE_URL:-unset} deadline=$(date -d "@$END_AT" '+%F %T' 2>/dev/null || echo "in ${TOTAL_MIN}m")"

# family-initial credentials: what relaunch_pane falls back to before any rotation
GLM_BASE_URL_INITIAL="${ANTHROPIC_BASE_URL:-}"
GLM_TOKEN_INITIAL="${ANTHROPIC_AUTH_TOKEN:-}"

fresh_session=0
if ! tmux has-session -t "$SESSION" 2>/dev/null; then
  # entrypoint normally owns first boot; reaching this means it never ran
  # (manual start) or tmux died — the watchdog must be self-sufficient
  tmux new-session -d -s "$SESSION" -n commander 2>>"$LOG" || { log "FATAL: cannot create tmux session $SESSION"; exit 1; }
  fresh_session=1
  log "created tmux session $SESSION (window 0 = commander)"
fi
adopt_existing
if (( fresh_session )); then
  relaunch_commander "fresh session, no commander running"
elif ! tmux list-windows -t "$SESSION" -F '#W' 2>/dev/null | grep -qx commander; then
  relaunch_commander "commander window absent at startup"
fi

# ---- main loop ----------------------------------------------------------------
while (( $(NOW) < END_AT )); do
  ensure_session || { sleep 15; continue; }
  adopt_existing
  wins=$(tmux list-windows -t "$SESSION" -F '#W' 2>/dev/null || true)

  # -- 1) commander pane dead -> relaunch + bootstrap --------------------------
  cmd_handled=0
  if ! printf '%s\n' "$wins" | grep -qx commander; then
    relaunch_commander "commander window gone"
    cmd_handled=1
  else
    cpane=$(pane_text commander)
    if printf '%s' "$cpane" | grep -q 'DRIVE_PANE_EXITED'; then
      relaunch_commander "commander pane exited"
      cmd_handled=1
    fi
  fi

  # -- 1b) DELETED: the pane_current_command whitelist (claude|node) was
  #        unreliable across claude-code builds — it false-positived on the
  #        LIVE commander TUI every ~60s and boot-looped it on the hosted run
  #        (input box existed => TUI alive, yet "no claude process" fired).
  #        Death is now judged ONLY by the iron signals in check 1:
  #        DRIVE_PANE_EXITED on screen, or the window gone.

  # -- 2) unsubmitted bootstrap text in the input box -> re-Enter x3 -> restart
  #       (matches both the watchdog's and the entrypoint's bootstrap text)
  if (( ! cmd_handled )); then
    if printf '%s' "$cpane" | grep -qE "❯.*docker/commander/SKILL\.md"; then
      enter_tries[commander]=$(( ${enter_tries[commander]:-0} + 1 ))
      case $(( ${enter_tries[commander]} )) in
        1|2)
          log "commander: input unsubmitted — re-Enter (${enter_tries[commander]}/3)"
          tmux send-keys -t "$SESSION:commander" Enter
          sleep 2
          ;;
        3)
          # stuck twice: clear the box and retype the whole bootstrap (the
          # text may be mangled mid-line; fresh keystrokes beat more Enters)
          log "commander: input unsubmitted — clearing and resending bootstrap (3/3)"
          tmux send-keys -t "$SESSION:commander" C-u
          sleep 0.5
          tmux send-keys -t "$SESSION:commander" C-u
          sleep 0.5
          tmux send-keys -t "$SESSION:commander" Enter
          sleep 1
          relaunch_commander "bootstrap never submits cleanly"
          ;;
      esac
      last_activity[commander]=$(NOW)
      cmd_handled=1
    else
      enter_tries[commander]=0
    fi
  fi

  # -- 2b) commander hang failsafe: pane content unchanged for 30 min (and
  #        nothing waiting in the input box) -> the TUI is wedged, not thinking.
  #        A dead-silent commander for the rest of the slot is the worst
  #        outcome; a relaunch costs one --continue.
  if (( ! cmd_handled )); then
    csha=$(printf '%s' "$cpane" | sha256sum | cut -c1-12)
    if [[ "${cmd_pane_sha:-}" == "$csha" && -n "$csha" ]]; then
      if (( $(NOW) - ${cmd_pane_since:-$(NOW)} > 1800 )) \
         && ! printf '%s' "$cpane" | grep -qE '❯.+[^ ]'; then
        log "commander: pane frozen 30min with empty input — relaunching (hang failsafe)"
        relaunch_commander "hang failsafe"
        cmd_handled=1
      fi
    else
      cmd_pane_sha="$csha"; cmd_pane_since=$(NOW)
    fi
  fi

  # -- 3+4) commander: dialogs first, then quota -------------------------------
  if (( ! cmd_handled )); then
    if accept_dialogs commander "$cpane"; then
      last_activity[commander]=$(NOW); cmd_handled=1
    elif printf '%s' "$cpane" | grep -qE "$QUOTA_RE"; then
      if printf '%s' "$cpane" | grep -qE "$RATELIMIT_RE"; then
        cooldown_log commander-rl "transient rate limit (not quota) — no rotation; claude retries on its own"
      elif rotation_allowed && rotate_key; then
        cooldown_log commander "quota/auth error in commander — key rotated, relaunching"
        relaunch_commander "quota/auth error"
      else
        cooldown_log commander "quota/auth error in commander and no spare key — waiting"
      fi
      last_activity[commander]=$(NOW); cmd_handled=1
    fi
  fi

  # -- 3,4,5) worker panes ------------------------------------------------------
  for win in $wins; do
    [[ "$win" == "commander" ]] && continue
    pane=$(pane_text "$win")
    # 3) any confirm dialog -> accept
    if accept_dialogs "$win" "$pane"; then
      last_activity[$win]=$(NOW); continue
    fi
    # 4) quota/auth -> rotate; TUI restarts budget-free, -p only logged
    if printf '%s' "$pane" | grep -qE "$QUOTA_RE"; then
      if printf '%s' "$pane" | grep -qE "$RATELIMIT_RE"; then
        cooldown_log "$win-rl" "$win: transient rate limit (not quota) — no rotation; claude retries on its own"
        last_activity[$win]=$(NOW); continue
      fi
      if rotation_allowed && rotate_key; then
        if [[ "${mode[$win]:-tui}" == "p" ]]; then
          cooldown_log "$win" "quota/auth error in -p pane — key rotated; leaving pane (commander will notice via board)"
        else
          cooldown_log "$win" "quota/auth error — key rotated, relaunching TUI with --continue (restart budget untouched)"
          relaunch_worker "$win" "quota/auth error" || true
        fi
      else
        cooldown_log "$win" "quota/auth error and no spare key — leaving pane as-is"
      fi
      last_activity[$win]=$(NOW); continue
    fi
    # 4b) TUI worker mission stuck unsubmitted in the input box -> re-Enter,
    #     then restart the window (regression from tmux-drive.sh #3: the
    #     hosted gateway is slow and paste-loss strands the mission text,
    #     leaving a live-looking pane that NEVER calls the LLM)
    if [[ "${mode[$win]:-tui}" == "tui" ]] \
       && printf '%s' "$pane" | grep -qE '❯.*autonomous pentest operator'; then
      enter_tries[$win]=$(( ${enter_tries[$win]:-0} + 1 ))
      if (( ${enter_tries[$win]} > 3 )); then
        mission_restarts[$win]=$(( ${mission_restarts[$win]:-0} + 1 ))
        if (( ${mission_restarts[$win]} > 3 )); then
          # restart loops help nothing (live: 25min of re-Enter/restart cycling
          # on a wedged TUI) — stop touching it, leave the diagnosis to the
          # commander (peek + kill/park). Cooldown-log to avoid spam.
          cooldown_log "$win-ms" "$win: mission wedged past 3 restarts — leaving pane for the commander"
          last_activity[$win]=$(NOW)
          continue
        fi
        log "$win: mission never submits — restarting window (${mission_restarts[$win]}/3)"
        enter_tries[$win]=0
        relaunch_worker "$win" "mission never submits" || true
      else
        log "$win: mission unsubmitted — re-Enter (${enter_tries[$win]}/3)"
        tmux send-keys -t "$SESSION:$win" Enter
        sleep 2
      fi
      last_activity[$win]=$(NOW)
      continue
    fi
    # 5) TUI worker exited -> relaunch with --continue, max 3/pane; -p = done
    if printf '%s' "$pane" | grep -q 'DRIVE_PANE_EXITED'; then
      if [[ "${mode[$win]:-tui}" == "p" ]]; then
        cooldown_log "$win" "$win: -p run exited — leaving pane for the commander (peek + worker.sh kill)"
        continue
      fi
      restarts[$win]=$(( ${restarts[$win]:-0} + 1 ))
      if (( ${restarts[$win]} > MAX_RESTARTS )); then
        cooldown_log "$win" "$win: TUI exited with no restarts left ($MAX_RESTARTS used) — leaving dead pane for the commander"
        continue
      fi
      relaunch_worker "$win" "TUI exited (${restarts[$win]}/$MAX_RESTARTS)" || true
      continue
    fi
    # 7) stall (no pane change > SAPPER_STALL_SECONDS): deliberately NO nudge,
    #    NO restart — keeping processes alive is the watchdog's whole job
  done

  sleep 15
done

finish_all
exit 0