Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Local Agent Guides CI: judge a finished turn the CLI would not exit from (#10399)

Daniel Handcf12378d68a
diff --git a/.github/scripts/agent-guides-drive.sh b/.github/scripts/agent-guides-drive.sh
index 2b96993ff88..114b5a6ef42 100755
--- a/.github/scripts/agent-guides-drive.sh
+++ b/.github/scripts/agent-guides-drive.sh
@@ -167,9 +167,54 @@ assert_reply() {
run_timed() { # $1=outfile, rest=command
local out="$1"; shift
TIMED_OUT=0
+ TURN_DONE=0
local t0=$SECONDS
- timeout --kill-after=30 "$TIMEOUT" "$@" > "$out" 2>&1
- local rc=$?
+ local rc
+ if [ -z "${TURN_DONE_RE:-}" ]; then
+ timeout --kill-after=30 "$TIMEOUT" "$@" > "$out" 2>&1
+ rc=$?
+ else
+ # An agent that prints an end-of-run marker does not have to exit before its
+ # turn can be judged. openclaw finishes and then holds its session write lock
+ # for the rest of the cap: on 2026-09-06 it answered `pong` and logged
+ # `ended with stopReason=stop` 39ms after the model replied, then sat there
+ # for the remaining 1200s, costing a 20 minute job and a "never completed a
+ # turn" verdict its own transcript contradicted. Give the CLI EXIT_GRACE
+ # seconds to leave on its own once the marker lands, then take it down, so
+ # the caller judges a finished turn instead of a cap. Only agents with such a
+ # marker opt in; for everyone else this is the same blocking call as before.
+ : > "$out"
+ timeout --kill-after=30 "$TIMEOUT" "$@" > "$out" 2>&1 &
+ local tpid=$! seen=""
+ while kill -0 "$tpid" 2>/dev/null; do
+ if [ -z "$seen" ]; then
+ grep -qF -- "$TURN_DONE_RE" "$out" 2>/dev/null && seen=$SECONDS
+ elif [ $(( SECONDS - seen )) -ge "${EXIT_GRACE:-30}" ]; then
+ TURN_DONE=1
+ # The whole group, not timeout(1) and not its direct child. The command
+ # is a bash wrapper that runs the agent, so signalling either one leaves
+ # the CLI orphaned and unbounded -- the state this is here to end.
+ # timeout(1) gives its child a group of its own, so that group is exactly
+ # the invocation; refuse to fire if it ever resolves to ours.
+ local kid pg
+ kid="$(pgrep -P "$tpid" 2>/dev/null | head -1)"
+ pg="$(ps -o pgid= -p "${kid:-0}" 2>/dev/null | tr -d ' ')"
+ if [ -n "$pg" ] && [ "$pg" != "$(ps -o pgid= -p $$ | tr -d ' ')" ]; then
+ kill -TERM "-$pg" 2>/dev/null || true
+ sleep 10
+ kill -KILL "-$pg" 2>/dev/null || true
+ else
+ pkill -TERM -P "$tpid" 2>/dev/null || true
+ sleep 10
+ pkill -KILL -P "$tpid" 2>/dev/null || true
+ fi
+ break
+ fi
+ sleep 2
+ done
+ wait "$tpid"
+ rc=$?
+ fi
local elapsed=$(( SECONDS - t0 ))
# Neither status proves expiry on its own. 137 is also an unrelated SIGKILL,
# and 124 is also whatever the CLI itself chose to exit with -- timeout(1)
@@ -200,6 +245,17 @@ run_timed() { # $1=outfile, rest=command
# assertions was wrong for exactly the cases most likely to hit it -- and it
# is what a reader sees immediately above the error that contradicts it.
echo "::warning::[$AGENT] the CLI printed a transcript but did not exit within ${TIMEOUT}s; whether that is fatal is the caller's call."
+ # The marker can land inside the last poll interval, or after a cap reached
+ # without the watcher. Either way the turn is over, and the caller may say so.
+ if [ -n "${TURN_DONE_RE:-}" ] && grep -qF -- "$TURN_DONE_RE" "$out" 2>/dev/null; then
+ TURN_DONE=1
+ fi
+ fi
+ if [ "${TURN_DONE:-0}" = 1 ]; then
+ # The fact, not the verdict, for the same reason the cap warning states one:
+ # what a finished-but-hung run means is the caller's to say, and a promise
+ # made here is read directly above whatever the caller decides.
+ echo "::warning::[$AGENT] the CLI logged the end of its run (${TURN_DONE_RE}) and then would not exit."
fi
return "$rc"
}
@@ -465,6 +521,11 @@ case "$MODE" in
hermes) patch_hermes_tools none
invoke_via_connect "$OUT" -z "$PROMPT" ;;
openclaw) patch_openclaw_agent notools
+ # openclaw logs this once per run, and only when the run is over
+ # ("run <uuid> ended with stopReason=stop"). It is the one signal
+ # here that a banner cannot forge, so it is what lets a hung exit
+ # be told apart from a turn that never came back.
+ TURN_DONE_RE='ended with stopReason='
CONNECT_CMD_OVERRIDE=openclaw invoke_via_connect "$OUT" agent --local --agent ci \
--model "unsloth/${UNSLOTH_MODEL_ID}" --message "$PROMPT" ;;
*) invoke_via_connect "$OUT" "$PROMPT" ;;
@@ -485,11 +546,20 @@ case "$MODE" in
# 600s. Nothing about the documented flow had drifted, and guide_fail said it
# had. It is still fatal -- see the note above on why a cap cannot be waived
# here -- but it is reported as what it is.
- if [ "${TIMED_OUT:-0}" = 1 ]; then
+ if [ "${TIMED_OUT:-0}" = 1 ] && [ "${TURN_DONE:-0}" != 1 ]; then
echo "::error::[$AGENT] the documented launch command started but never completed a turn within ${TIMEOUT}s. The recipe in ${CONNECT_REF} is not implicated: the transcript above shows what the CLI was doing when the cap hit. A connection or model-server failure looks like this; so does a headless prompt, which prints nothing at all." >&2
exit 1
fi
- [ "$rc" -eq 0 ] || guide_fail "the documented launch command exited non-zero (rc=$rc) -- see the transcript above"
+ # TURN_DONE is not a waiver of the cap, it is the assertion the cap was
+ # missing. The reason connection could never rescue a hang is that
+ # assert_reply cannot tell a completed reply from a startup banner; an
+ # end-of-run line the agent prints only when a run terminates can. A banner
+ # still carries no marker and still fails above. The rc check is skipped only
+ # here, because the non-zero status is the one run_timed produced itself when
+ # it stopped a CLI that had already finished.
+ if [ "${TURN_DONE:-0}" != 1 ]; then
+ [ "$rc" -eq 0 ] || guide_fail "the documented launch command exited non-zero (rc=$rc) -- see the transcript above"
+ fi
assert_reply "$OUT"
echo "[$AGENT] connection OK"
;;
diff --git a/tests/sh/test_agent_guides_timeout_classification.sh b/tests/sh/test_agent_guides_timeout_classification.sh
index 43c0e390c05..b8468d42293 100755
--- a/tests/sh/test_agent_guides_timeout_classification.sh
+++ b/tests/sh/test_agent_guides_timeout_classification.sh
@@ -70,12 +70,15 @@ run_case() { # $1 = TIMEOUT, rest = command
set -uo pipefail
AGENT=testagent
TIMEOUT=$_t
+TURN_DONE_RE='${TURN_DONE_RE:-}'
+EXIT_GRACE=${EXIT_GRACE:-30}
redact() { :; }
guide_fail() { echo "GUIDE_FAIL: \$*"; exit 9; }
$(cat "$WORK/${RT:-run_timed}.sh")
run_timed "$WORK/out.txt" "\$@"
echo "RC=\$?"
echo "TIMED_OUT=\${TIMED_OUT:-unset}"
+echo "TURN_DONE=\${TURN_DONE:-unset}"
EOF
bash "$WORK/case.sh" "$@" 2>&1 || true
}
@@ -89,12 +92,15 @@ run_case_bounded() { # $1 = outer bound, $2 = TIMEOUT, rest = command
set -uo pipefail
AGENT=testagent
TIMEOUT=$_t
+TURN_DONE_RE='${TURN_DONE_RE:-}'
+EXIT_GRACE=${EXIT_GRACE:-30}
redact() { :; }
guide_fail() { echo "GUIDE_FAIL: \$*"; exit 9; }
$(cat "$WORK/${RT:-run_timed}.sh")
run_timed "$WORK/out.txt" "\$@"
echo "RC=\$?"
echo "TIMED_OUT=\${TIMED_OUT:-unset}"
+echo "TURN_DONE=\${TURN_DONE:-unset}"
EOF
timeout --kill-after=5 "$_outer" bash "$WORK/case.sh" "$@" 2>&1 || true
}
@@ -196,15 +202,32 @@ assert_eq "attribution-ab keeps a hang fatal" \
assert_eq "and all four of its invokes go through that guard" \
"$(grep -c 'ab_invoke "\$LOGS_DIR/claude-ab-' "$DRIVE_SH" || true)" 4
-# The connection guard must stay a bare rc test: assert_reply cannot tell a
-# completed reply from a startup banner, so it cannot adjudicate a cap. Read the
-# whole statement, not one line of it -- a one-line rewrite is the easy mistake.
+# The connection guard still refuses a bare cap. What changed on 2026-09-06 is
+# that it no longer has to: openclaw answered `pong` and logged
+# `ended with stopReason=stop`, then held its session write lock for the rest of
+# the 1200s cap, and the job reported "never completed a turn" over a transcript
+# that showed the turn completing. The old reasoning stands -- assert_reply
+# cannot tell a completed reply from a startup banner -- so the fix is not to
+# waive the cap but to give connection the assertion it was missing: a line the
+# agent prints only when a run ends. A banner carries no such line and still
+# fails. Read the whole statement, not one line of it.
CONN_LINE="$(grep -n 'documented launch command exited non-zero' "$DRIVE_SH" | cut -d: -f1)"
CONN_STMT="$(sed -n "$((CONN_LINE - 2)),${CONN_LINE}p" "$DRIVE_SH")"
assert_eq "connection guard has no TIMED_OUT escape" \
"$(echo "$CONN_STMT" | grep -c 'TIMED_OUT' || true)" 0
assert_eq "connection guard still fails on a bare rc" \
"$(echo "$CONN_STMT" | grep -c '\[ "\$rc" -eq 0 \] || guide_fail' || true)" 1
+assert_eq "a cap with no end-of-run marker is still fatal for connection" \
+ "$(grep -c 'TIMED_OUT:-0}" = 1 \] && \[ "${TURN_DONE:-0}" != 1 \]' "$DRIVE_SH" || true)" 1
+# TURN_DONE is only ever set where the marker was actually seen, so the guard
+# above cannot be satisfied by a hang that printed nothing but a banner.
+assert_eq "TURN_DONE is set only behind a marker match" \
+ "$(grep -c 'TURN_DONE=1' "$WORK/run_timed.sh")" 2
+assert_eq "and both sites grep the transcript for it" \
+ "$(grep -c 'grep -qF -- "$TURN_DONE_RE"' "$WORK/run_timed.sh")" 2
+# Only openclaw opts in today, and only to its own end-of-run line.
+assert_eq "openclaw declares the marker" \
+ "$(grep -c "TURN_DONE_RE='ended with stopReason='" "$DRIVE_SH" || true)" 1
echo "7. file-edit turn 2 keeps a cap fatal, and asks one thing"
# The two-part T2 that would have made a waived turn 2 verifiable degraded the
@@ -226,8 +249,11 @@ assert_eq "turn 2 still fails on a bare rc" \
echo "8. the cap keeps a finite kill fallback, but expiry comes from the clock"
# Cases 3b/3c/3d prove the behaviour; these pin the shape, so a refactor cannot
# quietly go back to trusting an exit status.
-assert_eq "kill-after restored" \
- "$(grep -c 'kill-after=30' "$WORK/run_timed.sh")" 1
+# Both invocations carry it -- the plain blocking one and the watched one. A
+# marker-watching run that dropped the fallback would leave a TERM-resistant CLI
+# unbounded exactly where the watcher is meant to bound it.
+assert_eq "kill-after restored on both call sites" \
+ "$(grep -c 'kill-after=30' "$WORK/run_timed.sh")" 2
assert_eq "neither status alone decides" \
"$(grep -c 'rc" -eq 124 \] || \[ "$rc" -eq 137 \]' "$WORK/run_timed.sh")" 1
assert_eq "the clock decides, with 1s of slack for truncation" \
@@ -235,6 +261,81 @@ assert_eq "the clock decides, with 1s of slack for truncation" \
assert_eq "a suffixed cap falls back to 124 alone" \
"$(grep -c '\*\[!0-9\]\*) \[ "$rc" -eq 124 \] && expired=1' "$WORK/run_timed.sh")" 1
+echo "9. an agent that finishes its run and then will not exit is released early"
+# The openclaw case: the marker lands, the CLI keeps running, and without this
+# the job burns the whole cap and then calls a completed turn a hang. The stand-in
+# ignores TERM so the escalation to KILL is exercised too. Cap 60 with a 2s grace:
+# a pass has to come back in seconds, so a regression here shows up as a slow
+# test, not a green one.
+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=2 \
+ run_case_bounded 45 60 bash -c 'trap "" TERM; echo pong; echo run 1 ended with stopReason=stop; while true; do sleep 1; done')"
+assert_eq "TURN_DONE set" "$(field "$OUT" TURN_DONE)" 1
+assert_eq "not reported as a cap" "$(field "$OUT" TIMED_OUT)" 0
+assert_eq "no guide_fail" "$(count "$OUT" 'GUIDE_FAIL')" 0
+assert_eq "says the run ended but the CLI would not exit" \
+ "$(count "$OUT" 'would not exit')" 1
+assert_eq "the transcript survived the kill" \
+ "$(grep -c '^pong$' "$WORK/out.txt" || true)" 1
+
+echo "9f. the agent under the wrapper is killed too, not orphaned"
+# invoke_via_connect runs the CLI from a generated bash script, so timeout(1)'s
+# direct child is that wrapper and the agent is a grandchild. Signalling the
+# wrapper alone would leave the CLI running for the rest of the job with the
+# transcript's fd still open. The stand-in records the grandchild's pid and
+# ignores TERM, so a surviving process is visible after run_timed returns.
+rm -f "$WORK/kid.pid"
+# Two statements, so bash cannot exec-optimize the wrapper away and the agent is
+# a real grandchild -- the shape invoke_via_connect produces. A one-liner wrapper
+# collapses into a single process and the case silently stops testing anything.
+cat > "$WORK/agent.sh" <<AGENT
+trap "" TERM
+echo \$\$ > "$WORK/kid.pid"
+echo pong
+echo run 1 ended with stopReason=stop
+while true; do sleep 1; done
+AGENT
+cat > "$WORK/wrapper.sh" <<WRAP
+export STANDIN=1
+bash "$WORK/agent.sh"
+WRAP
+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=2 run_case_bounded 60 90 \
+ bash "$WORK/wrapper.sh")"
+assert_eq "TURN_DONE set" "$(field "$OUT" TURN_DONE)" 1
+KID="$(cat "$WORK/kid.pid" 2>/dev/null || echo 0)"
+assert_eq "the grandchild recorded its pid" "$([ "$KID" -gt 0 ] && echo yes || echo no)" yes
+sleep 1
+assert_eq "and no descendant survived" "$(kill -0 "$KID" 2>/dev/null && echo alive || echo gone)" gone
+
+echo "9b. a banner-then-hang carries no marker and stays a cap"
+# The failure the connection guard exists to catch. Same watcher, same grace:
+# the only difference is that nothing ever printed the end-of-run line.
+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=2 \
+ run_case_bounded 45 3 bash -c 'echo Welcome to the agent; sleep 30')"
+assert_eq "TURN_DONE stays clear" "$(field "$OUT" TURN_DONE)" 0
+assert_eq "still a cap" "$(field "$OUT" TIMED_OUT)" 1
+assert_eq "no early-release warning" "$(count "$OUT" 'would not exit')" 0
+
+echo "9c. a marker that lands inside the last poll interval still counts"
+# The watcher samples; a run that ends just before the cap can expire before the
+# next look. Reading the transcript after the fact keeps the two paths agreeing.
+OUT="$(TURN_DONE_RE='ended with stopReason=' EXIT_GRACE=600 \
+ run_case_bounded 45 3 bash -c 'echo pong; echo run 1 ended with stopReason=stop; sleep 30')"
+assert_eq "cap was hit" "$(field "$OUT" TIMED_OUT)" 1
+assert_eq "and the finished turn is still recognized" "$(field "$OUT" TURN_DONE)" 1
+
+echo "9d. declaring a marker does not change a CLI that exits on its own"
+OUT="$(TURN_DONE_RE='ended with stopReason=' run_case 10 bash -c 'echo pong; echo run 1 ended with stopReason=stop')"
+assert_eq "rc 0" "$(field "$OUT" RC)" 0
+assert_eq "TIMED_OUT cleared" "$(field "$OUT" TIMED_OUT)" 0
+assert_eq "TURN_DONE cleared" "$(field "$OUT" TURN_DONE)" 0
+assert_eq "no warning" "$(count "$OUT" '::warning::')" 0
+
+echo "9e. with no marker declared, every path is byte-for-byte the old one"
+OUT="$(run_case 1 bash -c 'echo did the work; sleep 5')"
+assert_eq "still a cap" "$(field "$OUT" TIMED_OUT)" 1
+assert_eq "TURN_DONE never set" "$(field "$OUT" TURN_DONE)" 0
+assert_eq "rc is still the timeout" "$(field "$OUT" RC)" 124
+
echo
echo "PASS=$PASS FAIL=$FAIL"
[ "$FAIL" -eq 0 ]
diff --git a/tests/studio/test_agent_guides_verdicts.py b/tests/studio/test_agent_guides_verdicts.py
index 00d60eb8176..2b4c0688938 100644
--- a/tests/studio/test_agent_guides_verdicts.py
+++ b/tests/studio/test_agent_guides_verdicts.py
@@ -82,10 +82,10 @@ def test_a_timed_out_connection_is_not_reported_as_recipe_drift() -> None:
connection = connection[: connection.index("\n file-edit)")]
timed_out = re.search(
- r'if \[ "\$\{TIMED_OUT:-0\}" = 1 \]; then(.*?)\n fi', connection, re.DOTALL
+ r'if \[ "\$\{TIMED_OUT:-0\}" = 1 \](.*?); then(.*?)\n fi', connection, re.DOTALL
)
assert timed_out, "the connection case no longer distinguishes a timeout from a non-zero exit"
- branch = timed_out.group(1)
+ condition, branch = timed_out.group(1), timed_out.group(2)
assert "guide_fail" not in branch, (
"a timed-out connection still goes through guide_fail, which states that "
@@ -98,6 +98,32 @@ def test_a_timed_out_connection_is_not_reported_as_recipe_drift() -> None:
"a timed-out connection must stay fatal; assert_reply cannot tell a "
"finished reply from a startup banner"
)
+ # The one thing that may narrow it. assert_reply still cannot tell a reply
+ # from a banner, so the cap is not waived; what excuses it is a line the
+ # agent prints only when a run ends, which a banner cannot produce. Openclaw
+ # answered `pong` and logged `ended with stopReason=stop`, then held its
+ # session write lock for the remaining 1200s, and this branch called that a
+ # turn that never came back.
+ assert condition.strip() in ("", '&& [ "${TURN_DONE:-0}" != 1 ]'), (
+ "the connection cap may only be narrowed by TURN_DONE, which run_timed "
+ f"sets solely on an end-of-run marker; found {condition.strip()!r}"
+ )
+
+
+def test_only_a_marker_can_excuse_a_connection_cap() -> None:
+ """
+ TURN_DONE must never be settable without the agent's own end-of-run line,
+ or the connection guard is waived by anything that hangs.
+ """
+ body = _block("run_timed")
+ assert body.count("TURN_DONE=1") == body.count(
+ 'grep -qF -- "$TURN_DONE_RE"'
+ ), "run_timed sets TURN_DONE somewhere that does not first match the marker"
+ source = _source()
+ declared = re.findall(r"TURN_DONE_RE='([^']*)'", source)
+ assert declared == [
+ "ended with stopReason="
+ ], f"unexpected end-of-run markers declared: {declared}"
def test_a_non_zero_exit_is_still_drift() -> None: