Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
"""`TurnGuard`: the loop guard for one prompt-turn, one tool call at a time.
Each tool call arrives as a separate hook process, so the guard's state is on disk between
calls (`to_dict` / `TurnGuard(data=...)`). Per call it folds the result into a `SpiralState`,
checks the cumulative backstop, and asks `rules.assess_abort` (gate -> optional paid judge ->
envelope fallback) whether the turn is stuck; the first stuck verdict is answered with a
redirect, a later one with a stop verdict (which the plugin delivers as a notice to the person,
never as a stop). After a redirect the per-segment state starts afresh, which is the agent's fair
window to recover.
* The unit is a PROMPT-TURN (one user prompt to the agent's stop): in interactive use a person
already splits work into short turns, and a whole session folded as one trajectory looks
stuck merely by being long.
* The redirect is delivered mid-turn, attached to the tool result.
* There is no grader, so the per-repository envelope learns from turns that ENDED GREEN: the
agent stopped on its own, the guard did not stop it, and no verification after its last edit
failed (`resolved_proxy`).
"""
import os
import re
import shlex
import time
from .rules import DEFAULT_THRESHOLDS, assess_abort, is_stuck, nudge_text
from .shell_edits import fold_shell_edits, shell_edit_targets
from .state import ERR_RE, SpiralState, canon_tool
_SLOTS = SpiralState.__slots__
# What counts as running the project's own checks, across ecosystems. Broader than
# `state.TEST_RUN_RE` (the Python vocabulary the guard's thresholds were tuned on); used ONLY
# for the ended-green proxy, never to change what `SpiralState` counts. A runner
# is often named by path (`python tests/runtests.py`, `.venv/bin/pytest`, `./node_modules/.bin/
# jest`): without the optional directories those turns showed no verification at all, so a turn
# whose last test run failed still counted as ended green and fed the envelope.
VERIFY_RE = re.compile(
r"(?i)(?:^|[\s;&|(])(?:~?/)?(?:[\w.\-]+/)*(?:pytest|py\.test|tox|nox|python\d?(?:\.\d+)?\s+-m\s+(?:pytest|unittest)|"
r"manage\.py\s+test|runtests(?:\.py)?|jest|vitest|mocha|ava|playwright\s+test|"
r"(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?test|go\s+test|cargo\s+(?:test|nextest)|"
r"(?:mvn|mvnw|\./mvnw)\s+(?:\S+\s+)*test|(?:gradle|gradlew|\./gradlew)\s+(?:\S+\s+)*test|"
r"make\s+(?:test|check)|ctest|rspec|phpunit|dotnet\s+test|swift\s+test|mix\s+test|"
r"deno\s+test|node\s+--test|bazel\s+test|(?:\./)?bin/test)(?![\w.\-/])")
# A command line's exit status is its LAST command's, so it may not be the test run's: `npm test
# 2>&1 | tail -n 8` exits with `tail`'s status, `pytest | grep FAILED` with `grep`'s, `python -m
# unittest; echo "exit=$?"` and `npm test || true` with 0, and `pytest -q; git diff` with
# `git`'s. For such a test run the runner's own summary line decides (`runner_verdict`). Without
# one, a run whose status was replaced (a pipe, `||`, a report of the status) is read by its
# output's markers, as for an agent that reports no status (Claude Code); a run followed by other
# commands keeps the status the agent reported, since their output is in the result too.
_SHELLS = {"sh", "bash", "zsh", "dash", "ksh"}
_QUOTED_RE = re.compile(r"'[^']*'|\"(?:\\.|[^\"\\])*\"")
_SEP_RE = re.compile(r"(&&|\|\||\|&|(?<!>)\||;|\n)") # `|`, but not the `>|` redirect
# A runner asked for its help, version or test list, not to run the tests.
_NOT_A_RUN_RE = re.compile(r"\s(?:-h|--help|--version|--collect-only|--co|--list\S*)(?!\S)")
# What may precede the command itself: `FOO=1 timeout 60 npm test`.
_PREFIX_RE = re.compile(r"^\s*(?:(?:\w+=\S*|time|nice|nohup|command|env|"
r"timeout(?:\s+(?:-\S+|[A-Z]+|\d\S*))+)\s+)*")
# What only reports on the run before it: `echo "exit=$?"`, `true`, `exit 0`.
_REPORT_RE = re.compile(r"^\s*(?:echo|printf|true|:|exit)(?:\s|$)")
_RANK = {"own": 0, "hidden": 1, "mixed": 2}
def status_of_tests(cmd):
"""What `cmd`'s exit status says of the test runs in it: None when it runs none; "own" when
it is theirs (a run is last, or followed only by `&&`, or piped under `pipefail`); "hidden"
when a pipe, `||` or a mere report of the status replaced it; "mixed" when other commands
ran after, their output in the result too. A command recorded as `bash -lc '<script>'`
(Codex) is judged by its script."""
if not isinstance(cmd, str):
return None
try:
parts = shlex.split(cmd)
except ValueError:
parts = []
if (len(parts) >= 3 and os.path.basename(parts[0]) in _SHELLS
and parts[1].startswith("-") and "c" in parts[1]):
cmd = parts[2]
pieces = _SEP_RE.split(re.sub(r"[(){}]", " ", _QUOTED_RE.sub("Q", cmd)))
while len(pieces) > 1 and not pieces[-1].strip(): # a trailing `;` or newline
del pieces[-2:]
pipefail = "pipefail" in cmd
found = None
for i in range(0, len(pieces), 2):
stage = pieces[i]
if not VERIFY_RE.match(_PREFIX_RE.sub("", stage)) or _NOT_A_RUN_RE.search(stage):
continue
status = "own"
for sep, then in zip(pieces[i + 1::2], pieces[i + 2::2]):
if sep == "&&" or (pipefail and sep in ("|", "|&")):
continue
if sep in ("|", "|&", "||"):
status = "hidden"
break
if not _REPORT_RE.match(then):
status = "mixed"
break
status = "hidden"
if found is None or _RANK[status] > _RANK[found]:
found = status
return found
# A test runner's summary of its run, which holds whatever the command line's exit status.
_RUN_FAILED_RE = re.compile("|".join((
r"\b[1-9]\d* (?:failed|errors?)\b.*\bin \d+(?:\.\d+)?s\b", # pytest
r"^FAILED \((?:failures|errors)=", # unittest, Django
r"^tests finished:.*\b[1-9]\d* (?:failed|exceptions?)\b", # sympy's bin/test
r"^# fail [1-9]", # TAP (`node --test`)
r"^Tests:.*\b[1-9]\d* failed\b", # jest
r"^\s*Tests\s+[1-9]\d* failed\b", # vitest
r"^\s*[1-9]\d* failing\b", # mocha
r"^(?:--- FAIL: |FAIL\t|FAIL$)", # go
r"^test result: FAILED", # cargo
r"\b\d+ examples?, [1-9]\d* failures?\b", # rspec
r"Tests run: \d+, Failures: (?:[1-9]|0, Errors: [1-9])", # JUnit (Maven, Gradle)
r"BUILD FAIL(?:URE|ED)")), re.MULTILINE)
_RUN_PASSED_RE = re.compile("|".join((
r"\b\d+ passed\b.*\bin \d+(?:\.\d+)?s\b",
r"^OK(?: \(.*\))?\s*$",
r"^tests finished: \d+ passed",
r"^# fail 0\b",
r"^Tests:\s+\d+ passed",
r"^\s*Tests\s+\d+ passed",
r"^\s*\d+ passing\b",
r"^ok\s+\S+\s+(?:\d+(?:\.\d+)?s|\(cached\))",
r"^test result: ok\.",
r"\b\d+ examples?, 0 failures\b",
r"BUILD SUCCESS")), re.MULTILINE)
def runner_verdict(text):
"""True when a test runner's summary in `text` says its run failed, False when one says it
passed (and none failed), None when `text` holds no summary."""
if not text:
return None
if _RUN_FAILED_RE.search(text):
return True
return False if _RUN_PASSED_RE.search(text) else None
try:
parts = shlex.split(cmd)
except ValueError:
parts = []
if (len(parts) >= 3 and os.path.basename(parts[0]) in _SHELLS
and parts[1].startswith("-") and "c" in parts[1]):
cmd = parts[2]
pieces = _SEP_RE.split(re.sub(r"[(){}]", " ", _QUOTED_RE.sub("Q", cmd)))
while len(pieces) > 1 and not pieces[-1].strip(): # a trailing `;` or newline
del pieces[-2:]
pipefail = "pipefail" in cmd
for i in range(0, len(pieces) - 1, 2):
stage = pieces[i]
if not VERIFY_RE.match(_PREFIX_RE.sub("", stage)) or _NOT_A_RUN_RE.search(stage):
continue
hidden = False
for sep, then in zip(pieces[i + 1::2], pieces[i + 2::2]):
if sep in ("|", "|&"):
hidden = not pipefail
break
if sep == "||":
hidden = True
break
if sep in (";", "\n"):
if not _REPORT_RE.match(then):
hidden = False
break
hidden = True
if hidden:
return True
return False
_EDIT = {"edit", "create"}
def state_to_dict(st):
return {k: getattr(st, k) for k in _SLOTS}
def state_from_dict(d):
st = SpiralState()
for k in _SLOTS:
if k in (d or {}):
setattr(st, k, d[k])
return st
class TurnGuard:
"""The loop guard for one prompt-turn, persisted between hook calls via `to_dict`."""
JUDGE_BASE_GAP = 5
JUDGE_MAX_GAP = 20
# A background verdict that never came back (its job died) stops blocking the next one
# after this long; a side call is capped at 180 s.
JUDGE_STALE_S = 300
def __init__(self, turn_key="", data=None):
d = data or {}
self.turn_key = d.get("turn_key", turn_key)
self.st = state_from_dict(d.get("st")) # this segment; reset after a nudge
self.whole = state_from_dict(d.get("whole")) # the whole turn; never reset
self.cum_turns = int(d.get("cum_turns", 0))
self.segment = int(d.get("segment", 0))
self.nudges_done = int(d.get("nudges_done", 0))
self.aborted = bool(d.get("aborted", False))
self.abort_delivered = bool(d.get("abort_delivered", False))
self.events = list(d.get("events", []))
self.judge_last = int(d.get("judge_last", -999))
self.judge_gap = int(d.get("judge_gap", self.JUDGE_BASE_GAP))
self.judge_calls = int(d.get("judge_calls", 0))
self.judge_tokens = float(d.get("judge_tokens", 0) or 0)
self.writer_tokens = float(d.get("writer_tokens", 0) or 0)
self.last_edit = int(d.get("last_edit", 0))
self.last_verify = int(d.get("last_verify", 0))
self.last_verify_failed = bool(d.get("last_verify_failed", False))
self.edits = int(d.get("edits", 0))
self.verifications = int(d.get("verifications", 0))
self.judge_pending = d.get("judge_pending") or None # {"id", "since", "segment"}
self.nudge_ready = d.get("nudge_ready") or None # a redirect the judge wrote
def to_dict(self):
return {"turn_key": self.turn_key, "st": state_to_dict(self.st),
"whole": state_to_dict(self.whole),
"cum_turns": self.cum_turns, "segment": self.segment,
"nudges_done": self.nudges_done, "aborted": self.aborted,
"abort_delivered": self.abort_delivered, "events": self.events[-20:],
"judge_last": self.judge_last, "judge_gap": self.judge_gap,
"judge_calls": self.judge_calls, "judge_tokens": self.judge_tokens,
"writer_tokens": self.writer_tokens, "last_edit": self.last_edit,
"last_verify": self.last_verify, "last_verify_failed": self.last_verify_failed,
"edits": self.edits, "verifications": self.verifications,
"judge_pending": self.judge_pending, "nudge_ready": self.nudge_ready}
# ---- outcome tracking for the ended-green proxy ----------------------------------
def _track(self, name, args, result, exit_code, shell_edits=()):
canon = canon_tool(name)
if canon in _EDIT:
self.last_edit = self.cum_turns
self.edits += 1
return
cmd = (args or {}).get("command") if canon == "bash" else None
verifies = isinstance(cmd, str) and bool(VERIFY_RE.search(cmd))
if shell_edits:
# `sed -i ... && pytest`: the edit comes first, so the run verifies it.
self.last_edit = self.cum_turns - (1 if verifies else 0)
self.edits += 1
if verifies:
self.last_verify = self.cum_turns
self.verifications += 1
if exit_code is not None:
self.last_verify_failed = exit_code != 0
else: # Claude Code reports no exit code
self.last_verify_failed = bool(ERR_RE.search(result or ""))
def resolved_proxy(self, stopped_normally=True):
"""Did this turn END GREEN? The stand-in for the grader the envelope cannot have.
Green unless a verification after the last edit failed. Requiring a PASSING check after
the last edit was tried on a continuous Django stream and made the envelope longer, not
shorter: it drops the short turns that edit without running anything, while the long
failing turns end with the agent's own checks passing either way."""
if not stopped_normally or self.aborted or self.cum_turns == 0:
return False
return not (self.last_verify_failed and self.last_verify > self.last_edit)
# ---- what the judge is told -------------------------------------------------------
# The digest says "Tests are still NOT green." whenever it has an envelope: on a benchmark
# the guard only ever watches an unfinished task. Here the guard knows how the
# turn's last verification went, so the judge -- and the redirect it writes -- is told that.
_NOT_GREEN = "Tests are still NOT green."
def verification_state(self):
if not self.verifications:
return "No test or verification run has been seen in this turn."
state = "failed" if self.last_verify_failed else "passed"
since = " There have been edits since." if self.last_edit > self.last_verify else ""
return f"The last verification run {state}.{since}"
def live_digest(self, digest):
return (digest or "").replace(self._NOT_GREEN, self.verification_state())
# ---- the judge, rate-limited with backoff exactly as NudgeGuard._sampled_judge ----
def _backoff(self, v):
if v and v.get("stuck") is False:
self.judge_gap = min(self.JUDGE_MAX_GAP, self.judge_gap + self.JUDGE_BASE_GAP)
else:
self.judge_gap = self.JUDGE_BASE_GAP
def _sampled(self, judge):
def call(digest):
if self.st.turns - self.judge_last < self.judge_gap:
return None
self.judge_last = self.st.turns
self.judge_calls += 1
try:
v = judge(self.live_digest(digest))
except Exception:
v = None
if v:
self.judge_tokens += v.get("_tokens") or 0
self._backoff(v)
return v
return call
def _background(self, judge):
"""`_sampled` for a judge that runs in the background (`plugin.judge.AsyncJudge`):
a verdict is taken whenever one has come back, a new job starts under the same rate
limit, and while one is out the judge answers nothing -- which `assess_abort` reads as
throttled, the same as a call the rate limit skipped."""
def call(digest):
found, v = judge.poll(self)
if found:
self.judge_pending = None
if v:
self.nudge_ready = (v.get("_nudge") or "").strip() or None
self._backoff(v)
return v or None
if self.judge_pending:
if time.time() - float(self.judge_pending.get("since") or 0) < self.JUDGE_STALE_S:
return None
self.judge_pending = None # its job never answered
if self.st.turns - self.judge_last < self.judge_gap:
return None
self.judge_last = self.st.turns
self.judge_calls += 1
job = judge.start(self, self.live_digest(digest))
if job:
self.judge_pending = {"id": job, "since": time.time(), "segment": self.segment}
return None
return call
# ---- one completed tool call ------------------------------------------------------
def fold(self, name, args, result, exit_code=None, *, envelope=None, thr=None,
judge=None, nudge_budget=1):
"""Fold one call; return None, ("nudge", reason) or ("abort", reason).
The body is `NudgeGuard.monitor` for a single completed call."""
thr = thr or DEFAULT_THRESHOLDS
cmd = (args or {}).get("command") if canon_tool(name) == "bash" else None
shell_edits = shell_edit_targets(cmd) if cmd is not None else []
status = status_of_tests(cmd)
if status in ("hidden", "mixed"):
verdict = runner_verdict(result)
if verdict is not None:
exit_code = int(verdict)
elif status == "hidden":
exit_code = None
failed = None if exit_code is None else exit_code != 0
self.st.update(name, args or {}, result or "", failed)
self.whole.update(name, args or {}, result or "", failed)
fold_shell_edits(self.st, shell_edits)
fold_shell_edits(self.whole, shell_edits)
self.cum_turns += 1
self._track(name, args, result, exit_code, shell_edits)
if self.aborted:
return None
backstop = thr.get("backstop_turns")
if backstop is None and envelope is not None:
backstop = envelope * thr["backstop_mult"]
if (backstop is not None and self.cum_turns >= backstop
and is_stuck(self.st, thr)):
# The ladder is nudge, THEN stop. On a benchmark the envelope was ~30 calls, so
# its 2.5x backstop lay well past the stuck rule's 20-call floor and the nudge came
# first. An interactive repository's envelope is ~4-10 calls: its backstop falls
# INSIDE that floor, so the moment a turn first counts as stuck the backstop has
# tripped too, and it would pre-empt every nudge. An unspent nudge goes first;
# after it, a fresh segment must be stuck again before the backstop stops it.
reason = f"cum-backstop:{self.cum_turns}>={backstop:.0f}t"
if self.nudges_done < nudge_budget:
return self._stop("nudge", reason, envelope)
return self._stop("abort", reason, envelope)
wired = None
if judge is not None:
wired = self._background(judge) if hasattr(judge, "poll") else self._sampled(judge)
abort, reason = assess_abort(self.st, envelope=envelope, judge=wired, thr=thr)
if not abort:
if (thr.get("nudge_on_ambiguous") and reason == "judge-ambiguous-spared"
and self.nudges_done < nudge_budget):
return self._stop("nudge", "ambiguous-past-envelope", envelope)
return None
if self.nudges_done < nudge_budget:
return self._stop("nudge", reason, envelope)
return self._stop("abort", reason, envelope)
def _stop(self, action, reason, envelope):
self.events.append({"segment": self.segment, "action": action,
"turn_in_segment": self.st.turns, "cum_turns": self.cum_turns,
"envelope": envelope, "reason": reason,
"snapshot": self.st.snapshot()})
if action == "abort":
self.aborted = True
return action, reason
def nudge_body(self, envelope=None, writer=None):
"""The redirect text: the one a background judge already wrote with its verdict, else
an LLM-written one when a writer is wired, else the count template
(`nudge_text`), which mirrors only the agent's own actions."""
if self.nudge_ready:
text, self.nudge_ready = self.nudge_ready, None
return text
if writer is not None:
try:
out = writer(self.live_digest(self.st.recent_digest(envelope=envelope)))
except Exception:
out = None
if out:
self.writer_tokens += out.get("_tokens") or 0
text = (out.get("text") or "").strip()
if text:
return text
return nudge_text(self.st)
def after_nudge(self):
"""Start the post-nudge segment with a fresh state: `NudgeGuard.continuation`'s reset,
which is the agent's fair re-assessment window."""
self.nudges_done += 1
self.segment += 1
self.st = SpiralState()
self.judge_last = -999
self.judge_gap = self.JUDGE_BASE_GAP
self.judge_pending = None # a verdict on the old segment is about the old one
self.nudge_ready = None
def summary(self, stopped_normally=True):
return {"turn_key": self.turn_key, "tool_calls": self.cum_turns,
"nudges": self.nudges_done, "aborted": self.aborted,
"resolved_proxy": self.resolved_proxy(stopped_normally),
"edits": self.edits, "verifications": self.verifications,
"judge_tokens": self.judge_tokens, "writer_tokens": self.writer_tokens}