Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Local Agent Guides CI: judge a finished turn the CLI would not exit from (#10399)
Daniel Handcf12378d68a
diff --git a/.github/scripts/agent-guides-drive.sh b/.github/scripts/agent-guides-drive.shindex 2b96993ff88..114b5a6ef42 100755--- a/.github/scripts/agent-guides-drive.sh+++ b/.github/scripts/agent-guides-drive.sh@@ -167,9 +167,54 @@ assert_reply() {run_timed() { # $1=outfile, rest=commandlocal out="$1"; shiftTIMED_OUT=0+ TURN_DONE=0local t0=$SECONDS- timeout --kill-after=30 "$TIMEOUT" "$@" > "$out" 2>&1- local rc=$?+ local rc+ if [ -z "${TURN_DONE_RE:-}" ]; then+ timeout --kill-after=30 "$TIMEOUT" "$@" > "$out" 2>&1+ rc=$?+ else+ # An agent that prints an end-of-run marker does not have to exit before its+ # turn can be judged. openclaw finishes and then holds its session write lock+ # for the rest of the cap: on 2026-09-06 it answered `pong` and logged+ # `ended with stopReason=stop` 39ms after the model replied, then sat there+ # for the remaining 1200s, costing a 20 minute job and a "never completed a+ # turn" verdict its own transcript contradicted. Give the CLI EXIT_GRACE+ # seconds to leave on its own once the marker lands, then take it down, so+ # the caller judges a finished turn instead of a cap. Only agents with such a+ # marker opt in; for everyone else this is the same blocking call as before.+ : > "$out"+ timeout --kill-after=30 "$TIMEOUT" "$@" > "$out" 2>&1 &+ local tpid=$! seen=""+ while kill -0 "$tpid" 2>/dev/null; do+ if [ -z "$seen" ]; then+ grep -qF -- "$TURN_DONE_RE" "$out" 2>/dev/null && seen=$SECONDS+ elif [ $(( SECONDS - seen )) -ge "${EXIT_GRACE:-30}" ]; then+ TURN_DONE=1+ # The whole group, not timeout(1) and not its direct child. The command+ # is a bash wrapper that runs the agent, so signalling either one leaves+ # the CLI orphaned and unbounded -- the state this is here to end.+ # timeout(1) gives its child a group of its own, so that group is exactly+ # the invocation; refuse to fire if it ever resolves to ours.+ local kid pg+ kid="$(pgrep -P "$tpid" 2>/dev/null | head -1)"+ pg="$(ps -o pgid= -p "${kid:-0}" 2>/dev/null | tr -d ' ')"+ if [ -n "$pg" ] && [ "$pg" != "$(ps -o pgid= -p $$ | tr -d ' ')" ]; then+ kill -TERM "-$pg" 2>/dev/null || true+ sleep 10+ kill -KILL "-$pg" 2>/dev/null || true+ else+ pkill -TERM -P "$tpid" 2>/dev/null || true+ sleep 10+ pkill -KILL -P "$tpid" 2>/dev/null || true+ fi+ break+ fi+ sleep 2+ done+ wait "$tpid"+ rc=$?+ filocal elapsed=$(( SECONDS - t0 ))# Neither status proves expiry on its own. 137 is also an unrelated SIGKILL,# and 124 is also whatever the CLI itself chose to exit with -- timeout(1)@@ -200,6 +245,17 @@ run_timed() { # $1=outfile, rest=command# assertions was wrong for exactly the cases most likely to hit it -- and it# is what a reader sees immediately above the error that contradicts it.echo "::warning::[$AGENT] the CLI printed a transcript but did not exit within ${TIMEOUT}s; whether that is fatal is the caller's call."+ # The marker can land inside the last poll interval, or after a cap reached+ # without the watcher. Either way the turn is over, and the caller may say so.+ if [ -n "${TURN_DONE_RE:-}" ] && grep -qF -- "$TURN_DONE_RE" "$out" 2>/dev/null; then+ TURN_DONE=1+ fi+ fi+ if [ "${TURN_DONE:-0}" = 1 ]; then+ # The fact, not the verdict, for the same reason the cap warning states one:+ # what a finished-but-hung run means is the caller's to say, and a promise+ # made here is read directly above whatever the caller decides.+ echo "::warning::[$AGENT] the CLI logged the end of its run (${TURN_DONE_RE}) and then would not exit."fireturn "$rc"}@@ -465,6 +521,11 @@ case "$MODE" inhermes) patch_hermes_tools noneinvoke_via_connect "$OUT" -z "$PROMPT" ;;openclaw) patch_openclaw_agent notools+ # openclaw logs this once per run, and only when the run is over+ # ("run <uuid> ended with stopReason=stop"). It is the one signal+ # here that a banner cannot forge, so it is what lets a hung exit+ # be told apart from a turn that never came back.+ TURN_DONE_RE='ended with stopReason='CONNECT_CMD_OVERRIDE=openclaw invoke_via_connect "$OUT" agent --local --agent ci \--model "unsloth/${UNSLOTH_MODEL_ID}" --message "$PROMPT" ;;*) invoke_via_connect "$OUT" "$PROMPT" ;;@@ -485,11 +546,20 @@ case "$MODE" in# 600s. Nothing about the documented flow had drifted, and guide_fail said it# had. It is still fatal -- see the note above on why a cap cannot be waived# here -- but it is reported as what it is.- if [ "${TIMED_OUT:-0}" = 1 ]; then+ if [ "${TIMED_OUT:-0}" = 1 ] && [ "${TURN_DONE:-0}" != 1 ]; thenecho "::error::[$AGENT] the documented launch command started but never completed a turn within ${TIMEOUT}s. The recipe in ${CONNECT_REF} is not implicated: the transcript above shows what the CLI was doing when the cap hit. A connection or model-server failure looks like this; so does a headless prompt, which prints nothing at all." >&2exit 1fi- [ "$rc" -eq 0 ] || guide_fail "the documented launch command exited non-zero (rc=$rc) -- see the transcript above"+ # TURN_DONE is not a waiver of the cap, it is the assertion the cap was+ # missing. The reason connection could never rescue a hang is that+ # assert_reply cannot tell a completed reply from a startup banner; an+ # end-of-run line the agent prints only when a run terminates can. A banner+ # still carries no marker and still fails above. The rc check is skipped only+ # here, because the non-zero status is the one run_timed produced itself when+ # it stopped a CLI that had already finished.+ if [ "${TURN_DONE:-0}" != 1 ]; then+ [ "$rc" -eq 0 ] || guide_fail "the documented launch command exited non-zero (rc=$rc) -- see the transcript above"+ fiassert_reply "$OUT"echo "[$AGENT] connection OK";;diff --git a/tests/sh/test_agent_guides_timeout_classification.sh b/tests/sh/test_agent_guides_timeout_classification.shindex 43c0e390c05..b8468d42293 100755--- a/tests/sh/test_agent_guides_timeout_classification.sh+++ b/tests/sh/test_agent_guides_timeout_classification.sh@@ -70,12 +70,15 @@ run_case() { # $1 = TIMEOUT, rest = commandset -uo pipefailAGENT=testagentTIMEOUT=$_t+TURN_DONE_RE='${TURN_DONE_RE:-}'+EXIT_GRACE=${EXIT_GRACE:-30}redact() { :; }guide_fail() { echo "GUIDE_FAIL: \$*"; exit 9; }$(cat "$WORK/${RT:-run_timed}.sh")run_timed "$WORK/out.txt" "\$@"echo "RC=\$?"echo "TIMED_OUT=\${TIMED_OUT:-unset}"+echo "TURN_DONE=\${TURN_DONE:-unset}"EOFbash "$WORK/case.sh" "$@" 2>&1 || true}@@ -89,12 +92,15 @@ run_case_bounded() { # $1 = outer bound, $2 = TIMEOUT, rest = commandset -uo pipefailAGENT=testagentTIMEOUT=$_t+TURN_DONE_RE='${TURN_DONE_RE:-}'+EXIT_GRACE=${EXIT_GRACE:-30}redact() { :; }guide_fail() { echo "GUIDE_FAIL: \$*"; exit 9; }$(cat "$WORK/${RT:-run_timed}.sh")run_timed "$WORK/out.txt" "\$@"echo "RC=\$?"echo "TIMED_OUT=\${TIMED_OUT:-unset}"+echo "TURN_DONE=\${TURN_DONE:-unset}"EOFtimeout --kill-after=5 "$_outer" bash "$WORK/case.sh" "$@" 2>&1 || true}@@ -196,15 +202,32 @@ assert_eq "attribution-ab keeps a hang fatal" \assert_eq "and all four of its invokes go through that guard" \"$(grep -c 'ab_invoke "\$LOGS_DIR/claude-ab-' "$DRIVE_SH" || true)" 4-# The connection guard must stay a bare rc test: assert_reply cannot tell a-# completed reply from a startup banner, so it cannot adjudicate a cap. Read the-# whole statement, not one line of it -- a one-line rewrite is the easy mistake.+# The connection guard still refuses a bare cap. What changed on 2026-09-06 is+# that it no longer has to: openclaw answered `pong` and logged+# `ended with stopReason=stop`, then held its session write lock for the rest of+# the 1200s cap, and the job reported "never completed a turn" over a transcript+# that showed the turn completing. The old reasoning stands -- assert_reply+# cannot tell a completed reply from a startup banner -- so the fix is not to+# waive the cap but to give connection the assertion it was missing: a line the+# agent prints only when a run ends. A banner carries no such line and still+# fails. Read the whole statement, not one line of it.CONN_LINE="$(grep -n 'documented launch command exited non-zero' "$DRIVE_SH" | cut -d: -f1)"CONN_STMT="$(sed -n "$((CONN_LINE - 2)),${CONN_LINE}p" "$DRIVE_SH")"assert_eq "connection guard has no TIMED_OUT escape" \"$(echo "$CONN_STMT" | grep -c 'TIMED_OUT' || true)" 0assert_eq "connection guard still fails on a bare rc" \"$(echo "$CONN_STMT" | grep -c '\[ "\$rc" -eq 0 \] || guide_fail' || true)" 1+assert_eq "a cap with no end-of-run marker is still fatal for connection" \+ "$(grep -c 'TIMED_OUT:-0}" = 1 \] && \[ "${TURN_DONE:-0}" != 1 \]' "$DRIVE_SH" || true)" 1+# TURN_DONE is only ever set where the marker was actually seen, so the guard+# above cannot be satisfied by a hang that printed nothing but a banner.+assert_eq "TURN_DONE is set only behind a marker match" \+ "$(grep -c 'TURN_DONE=1' "$WORK/run_timed.sh")" 2+assert_eq "and both sites grep the transcript for it" \+ "$(grep -c 'grep -qF -- "$TURN_DONE_RE"' "$WORK/run_timed.sh")" 2+# Only openclaw opts in today, and only to its own end-of-run line.+assert_eq "openclaw declares the marker" \+ "$(grep -c "TURN_DONE_RE='ended with stopReason='" "$DRIVE_SH" || true)" 1echo "7. file-edit turn 2 keeps a cap fatal, and asks one thing"# The two-part T2 that would have made a waived turn 2 verifiable degraded the@@ -226,8 +249,11 @@ assert_eq "turn 2 still fails on a bare rc" \echo "8. the cap keeps a finite kill fallback, but expiry comes from the clock"# Cases 3b/3c/3d prove the behaviour; these pin the shape, so a refactor cannot# quietly go back to trusting an exit status.-assert_eq "kill-after restored" \- "$(grep -c 'kill-after=30' "$WORK/run_timed.sh")" 1+# Both invocations carry it -- the plain blocking one and the watched one. A+# marker-watching run that dropped the fallback would leave a TERM-resistant CLI+# unbounded exactly where the watcher is meant to bound it.+assert_eq "kill-after restored on both call sites" \+ "$(grep -c 'kill-after=30' "$WORK/run_timed.sh")" 2assert_eq "neither status alone decides" \"$(grep -c 'rc" -eq 124 \] || \[ "$rc" -eq 137 \]' "$WORK/run_timed.sh")" 1assert_eq "the clock decides, with 1s of slack for truncation" \@@ -235,6 +261,81 @@ assert_eq "the clock decides, with 1s of slack for truncation" \assert_eq "a suffixed cap falls back to 124 alone" \"$(grep -c '\*\[!0-9\]\*) \[ "$rc" -eq 124 \] && expired=1' "$WORK/run_timed.sh")" 1+echo "9. an agent that finishes its run and then will not exit is released early"+# The openclaw case: the marker lands, the CLI keeps running, and without this+# the job burns the whole cap and then calls a completed turn a hang. The stand-in+# ignores TERM so the escalation to KILL is exercised too. Cap 60 with a 2s grace:+# a pass has to come back in seconds, so a regression here shows up as a slow+# test, not a green one.+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=2 \+ run_case_bounded 45 60 bash -c 'trap "" TERM; echo pong; echo run 1 ended with stopReason=stop; while true; do sleep 1; done')"+assert_eq "TURN_DONE set" "$(field "$OUT" TURN_DONE)" 1+assert_eq "not reported as a cap" "$(field "$OUT" TIMED_OUT)" 0+assert_eq "no guide_fail" "$(count "$OUT" 'GUIDE_FAIL')" 0+assert_eq "says the run ended but the CLI would not exit" \+ "$(count "$OUT" 'would not exit')" 1+assert_eq "the transcript survived the kill" \+ "$(grep -c '^pong$' "$WORK/out.txt" || true)" 1++echo "9f. the agent under the wrapper is killed too, not orphaned"+# invoke_via_connect runs the CLI from a generated bash script, so timeout(1)'s+# direct child is that wrapper and the agent is a grandchild. Signalling the+# wrapper alone would leave the CLI running for the rest of the job with the+# transcript's fd still open. The stand-in records the grandchild's pid and+# ignores TERM, so a surviving process is visible after run_timed returns.+rm -f "$WORK/kid.pid"+# Two statements, so bash cannot exec-optimize the wrapper away and the agent is+# a real grandchild -- the shape invoke_via_connect produces. A one-liner wrapper+# collapses into a single process and the case silently stops testing anything.+cat > "$WORK/agent.sh" <<AGENT+trap "" TERM+echo \$\$ > "$WORK/kid.pid"+echo pong+echo run 1 ended with stopReason=stop+while true; do sleep 1; done+AGENT+cat > "$WORK/wrapper.sh" <<WRAP+export STANDIN=1+bash "$WORK/agent.sh"+WRAP+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=2 run_case_bounded 60 90 \+ bash "$WORK/wrapper.sh")"+assert_eq "TURN_DONE set" "$(field "$OUT" TURN_DONE)" 1+KID="$(cat "$WORK/kid.pid" 2>/dev/null || echo 0)"+assert_eq "the grandchild recorded its pid" "$([ "$KID" -gt 0 ] && echo yes || echo no)" yes+sleep 1+assert_eq "and no descendant survived" "$(kill -0 "$KID" 2>/dev/null && echo alive || echo gone)" gone++echo "9b. a banner-then-hang carries no marker and stays a cap"+# The failure the connection guard exists to catch. Same watcher, same grace:+# the only difference is that nothing ever printed the end-of-run line.+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=2 \+ run_case_bounded 45 3 bash -c 'echo Welcome to the agent; sleep 30')"+assert_eq "TURN_DONE stays clear" "$(field "$OUT" TURN_DONE)" 0+assert_eq "still a cap" "$(field "$OUT" TIMED_OUT)" 1+assert_eq "no early-release warning" "$(count "$OUT" 'would not exit')" 0++echo "9c. a marker that lands inside the last poll interval still counts"+# The watcher samples; a run that ends just before the cap can expire before the+# next look. Reading the transcript after the fact keeps the two paths agreeing.+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=600 \+ run_case_bounded 45 3 bash -c 'echo pong; echo run 1 ended with stopReason=stop; sleep 30')"+assert_eq "cap was hit" "$(field "$OUT" TIMED_OUT)" 1+assert_eq "and the finished turn is still recognized" "$(field "$OUT" TURN_DONE)" 1++echo "9d. declaring a marker does not change a CLI that exits on its own"+OUT="$(TURN_DONE_RE='ended with stopReason=' run_case 10 bash -c 'echo pong; echo run 1 ended with stopReason=stop')"+assert_eq "rc 0" "$(field "$OUT" RC)" 0+assert_eq "TIMED_OUT cleared" "$(field "$OUT" TIMED_OUT)" 0+assert_eq "TURN_DONE cleared" "$(field "$OUT" TURN_DONE)" 0+assert_eq "no warning" "$(count "$OUT" '::warning::')" 0++echo "9e. with no marker declared, every path is byte-for-byte the old one"+OUT="$(run_case 1 bash -c 'echo did the work; sleep 5')"+assert_eq "still a cap" "$(field "$OUT" TIMED_OUT)" 1+assert_eq "TURN_DONE never set" "$(field "$OUT" TURN_DONE)" 0+assert_eq "rc is still the timeout" "$(field "$OUT" RC)" 124+echoecho "PASS=$PASS FAIL=$FAIL"[ "$FAIL" -eq 0 ]diff --git a/tests/studio/test_agent_guides_verdicts.py b/tests/studio/test_agent_guides_verdicts.pyindex 00d60eb8176..2b4c0688938 100644--- a/tests/studio/test_agent_guides_verdicts.py+++ b/tests/studio/test_agent_guides_verdicts.py@@ -82,10 +82,10 @@ def test_a_timed_out_connection_is_not_reported_as_recipe_drift() -> None:connection = connection[: connection.index("\n file-edit)")]timed_out = re.search(- r'if \[ "\$\{TIMED_OUT:-0\}" = 1 \]; then(.*?)\n fi', connection, re.DOTALL+ r'if \[ "\$\{TIMED_OUT:-0\}" = 1 \](.*?); then(.*?)\n fi', connection, re.DOTALL)assert timed_out, "the connection case no longer distinguishes a timeout from a non-zero exit"- branch = timed_out.group(1)+ condition, branch = timed_out.group(1), timed_out.group(2)assert "guide_fail" not in branch, ("a timed-out connection still goes through guide_fail, which states that "@@ -98,6 +98,32 @@ def test_a_timed_out_connection_is_not_reported_as_recipe_drift() -> None:"a timed-out connection must stay fatal; assert_reply cannot tell a ""finished reply from a startup banner")+ # The one thing that may narrow it. assert_reply still cannot tell a reply+ # from a banner, so the cap is not waived; what excuses it is a line the+ # agent prints only when a run ends, which a banner cannot produce. Openclaw+ # answered `pong` and logged `ended with stopReason=stop`, then held its+ # session write lock for the remaining 1200s, and this branch called that a+ # turn that never came back.+ assert condition.strip() in ("", '&& [ "${TURN_DONE:-0}" != 1 ]'), (+ "the connection cap may only be narrowed by TURN_DONE, which run_timed "+ f"sets solely on an end-of-run marker; found {condition.strip()!r}"+ )+++def test_only_a_marker_can_excuse_a_connection_cap() -> None:+ """+ TURN_DONE must never be settable without the agent's own end-of-run line,+ or the connection guard is waived by anything that hangs.+ """+ body = _block("run_timed")+ assert body.count("TURN_DONE=1") == body.count(+ 'grep -qF -- "$TURN_DONE_RE"'+ ), "run_timed sets TURN_DONE somewhere that does not first match the marker"+ source = _source()+ declared = re.findall(r"TURN_DONE_RE='([^']*)'", source)+ assert declared == [+ "ended with stopReason="+ ], f"unexpected end-of-run markers declared: {declared}"def test_a_non_zero_exit_is_still_drift() -> None: