Skip to content

HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.

runner: the step budget binds on every base, codex's app-server path included (#495)

Kratos871640d04495
diff --git a/runner/server.py b/runner/server.py
index 9e48f876..3a9a52a6 100644
--- a/runner/server.py
+++ b/runner/server.py
@@ -3018,12 +3018,7 @@ def _dsh_to_claude(obj: dict, state: dict) -> list[dict]:
"model": state.get("model")}]
return []
if m == "__hr_result":
- usage_map = state.get("_dsh_usage") or {}
- usage: dict = {}
- for u in usage_map.values():
- for k, v in u.items():
- usage[k] = usage.get(k, 0) + v
- usage = {k: v for k, v in usage.items() if v}
+ usage = _running_usage(state)
reason = p.get("reason")
final = p.get("final") or state.get("final", "")
state["final"] = final
@@ -3384,8 +3379,7 @@ def _pi_to_claude(obj: dict, state: dict) -> list[dict]:
state["_pi_text"] = ""
return []
err = state.get("_pi_error")
- usage = dict(state.get("_pi_usage") or {})
- usage = {k: v for k, v in usage.items() if v}
+ usage = _running_usage(state)
served = sorted(set(state.get("_served") or []))
requested = str(state.get("model") or "")
other = [m for m in served if requested and not _same_model(requested, m)]
@@ -7021,7 +7015,8 @@ def _build_grok(provider: str, auth: Auth, model: str, prompt: str, cwd: str, en
withheld.append(implied)
cmd += ["--disallowed-tools", ",".join(list(_GROK_ALWAYS_WITHHELD) + withheld)]
if max_turns:
- # Unset is unbounded; reaching it ends the run as `error_max_turns`, which reads as failed.
+ # Unset is unbounded; reaching it ends the run as `error_max_turns` with is_error set, and
+ # the run loop reports it by that subtype: incomplete at the budget, not failed (#471).
cmd += ["--max-turns", str(int(max_turns))]
if partial:
cmd.append("--include-partial-messages")
@@ -11677,9 +11672,45 @@ def _kill_capped(proc: subprocess.Popen, rec: dict) -> None:
_stop_proc(proc, rec)
+# Bases whose runner builders take NO step-budget flag of their own: the generic counter in
+# _run_turn_bg enforces max_turns for these by counting tool calls in the normalised stream and
+# stopping the process at the budget (#471). Bases that pass a native limit flag (claude --max-turns,
+# goose --max-steps, kimi's loop.max_steps, grok, hermes config, minimax, aider, cheetahclaws,
+# systemone, openhands, agentzero, opensquilla, nanobot, zeroclaw) are absent: their budget is the
+# CLI's own. jcode is absent too — its counter in _jcode_to_claude predates this one and emits
+# jcode's own usage, so it stays.
+_GENERIC_STEP_BUDGET = {"codex", "dsh", "pi", "omp", "gajae", "qwen", "gemini", "cline",
+ "opencode", "mimo", "kilo", "qoder"}
+
+
+def _running_usage(state: dict) -> dict:
+ """What a translator has counted of a turn's usage so far: pi's (omp's, gajae's) per message,
+ dsh's per step. It is the turn's usage on their result, and on a budget's stop, which comes
+ before their CLI says anything terminal. {} where nothing is kept as the turn goes."""
+ usage: dict = {}
+ if state.get("_pi_usage"):
+ usage = dict(state["_pi_usage"])
+ for u in (state.get("_dsh_usage") or {}).values():
+ for k, v in u.items():
+ usage[k] = usage.get(k, 0) + v
+ return {k: v for k, v in usage.items() if v}
+
+
+def _count_tool_calls(ev: dict, state: dict) -> int:
+ """The tool calls a turn has made so far, counted by id, so a normaliser that reports one call
+ twice does not spend the budget twice."""
+ seen = state.setdefault("_tool_ids", set())
+ blocks = (ev.get("message") or {}).get("content")
+ for b in blocks if isinstance(blocks, list) else []:
+ if isinstance(b, dict) and b.get("type") == "tool_use":
+ seen.add(str(b.get("id") or f"#{len(seen)}"))
+ return len(seen)
+
+
def _run_turn_bg(turn_id: str, cmd: list[str], env: dict, cwd: str, normalize, model: str,
timeout_seconds: int | None = None, partial: bool = False,
- max_turns: int | None = None, prompt: str = "") -> None:
+ max_turns: int | None = None, prompt: str = "",
+ count_steps: bool = False) -> None:
rec = _turns[turn_id]
rec["turn_id"] = turn_id
state = {"model": model, "final": "", "partial": partial, "cwd": cwd, "started": time.time(),
@@ -11691,7 +11722,31 @@ def _run_turn_bg(turn_id: str, cmd: list[str], env: dict, cwd: str, normalize, m
# (_opensquilla_transcript)
"prompt": prompt}
result_ev = None
+ # The generic step budget (#471): for a base with no limit flag of its own, tool calls are
+ # counted in the normalised stream — every normaliser emits the same Claude-shaped tool_use
+ # blocks — and past the budget the loop below stops the process the way a cancel does, and the
+ # turn ends error_max_turns, which the gateway reports as incomplete.
+ step_cap = int(max_turns) if (count_steps and max_turns) else None
env = {**env, _TURN_MARK: turn_id}
+
+ def _settle(ev: dict) -> None:
+ """An event, before it is visible. A result gets what the relay saw and the budget's word."""
+ ev.setdefault("_ts", time.time())
+ if ev.get("type") != "result":
+ return
+ if state.get("_stop") == "max_turns" and ev.get("subtype") != "error_max_turns":
+ # The turn ended at its budget. What the CLI says while it is stopped (a remote
+ # runtime's cancel, with its charge) is still the turn's account, but not its verdict.
+ ev.update(subtype="error_max_turns", is_error=False)
+ if not ev.get("model"):
+ # The CLI did not say what ran; the relay saw the provider say it. Set before
+ # the event is visible, so the gateway never reads it unlabelled.
+ served = _relay_served_model(env)
+ if served:
+ ev["model"] = served
+ _fill_relay_usage(ev, env)
+ _stamp_thinking(ev, rec)
+
try:
# start_new_session: own process group so cancel/timeout can killpg the CLI AND its
# shell children (see _kill_proc_tree) instead of orphaning a pipe-holding child.
@@ -11739,16 +11794,7 @@ def _run_turn_bg(turn_id: str, cmd: list[str], env: dict, cwd: str, normalize, m
except Exception: # noqa: BLE001
continue
for ev in normalize(obj, state):
- ev.setdefault("_ts", time.time())
- if ev.get("type") == "result" and not ev.get("model"):
- # The CLI did not say what ran; the relay saw the provider say it. Set before
- # the event is visible, so the gateway never reads it unlabelled.
- served = _relay_served_model(env)
- if served:
- ev["model"] = served
- if ev.get("type") == "result":
- _fill_relay_usage(ev, env)
- _stamp_thinking(ev, rec)
+ _settle(ev)
with _turns_lock:
rec["events"].append(ev)
if ev.get("type") == "system" and ev.get("subtype") == "init" and ev.get("session_id"):
@@ -11758,12 +11804,16 @@ def _run_turn_bg(turn_id: str, cmd: list[str], env: dict, cwd: str, normalize, m
# Claude's stream-json carries the final assistant text on the result event;
# the codex normalizer already fills state['final'], so prefer that.
state["final"] = state.get("final") or ev.get("result") or ""
+ if (step_cap and not state.get("_stop") and ev.get("type") == "assistant"
+ and _count_tool_calls(ev, state) > step_cap):
+ state["_stop"] = "max_turns"
if state.get("_stop") and not state.get("_stopped") and proc.poll() is None:
- # A translator ended the turn itself (a CLI with no step limit of its own went past
- # the turn's budget): the process is stopped here, and the result event it emitted
- # says why, so the turn is incomplete at its budget rather than running to the cap.
+ # The turn went past its budget (a CLI with no step limit of its own, counted above
+ # or by its translator): the process is stopped here, the way a cancel stops it, so a
+ # remote runtime is cancelled too, and the turn is incomplete at its budget rather
+ # than running to the cap.
state["_stopped"] = True
- _kill_proc_tree(proc, turn_id)
+ _stop_proc(proc, rec)
rc = proc.wait()
finally:
killer.cancel()
@@ -11777,19 +11827,23 @@ def _run_turn_bg(turn_id: str, cmd: list[str], env: dict, cwd: str, normalize, m
eof = getattr(normalize, "eof", None)
if eof is not None and result_ev is None:
for ev in eof(state, rc):
- ev.setdefault("_ts", time.time())
- if ev.get("type") == "result" and not ev.get("model"):
- served = _relay_served_model(env)
- if served:
- ev["model"] = served
- if ev.get("type") == "result":
- _fill_relay_usage(ev, env)
- _stamp_thinking(ev, rec)
+ _settle(ev)
with _turns_lock:
rec["events"].append(ev)
if ev.get("type") == "result":
result_ev = ev
state["final"] = state.get("final") or ev.get("result") or ""
+ if result_ev is None and state.get("_stop") == "max_turns":
+ # Stopped at its budget before the CLI said anything terminal (codex's turn.completed
+ # never comes once it is killed): the turn's result is the budget's, with what its
+ # translator or the relay counted, and no usage at all where nothing counted it.
+ result_ev = {"type": "result", "subtype": "error_max_turns", "is_error": False,
+ "result": state.get("final", "")}
+ if _running_usage(state):
+ result_ev["usage"] = _running_usage(state) # the translator's count so far (pi, dsh)
+ _settle(result_ev)
+ with _turns_lock:
+ rec["events"].append(result_ev)
if (result_ev is not None and not result_ev.get("is_error")
and _relay_last_finish(env) == "content_filter"):
# The provider declined the last request (finish_reason content_filter, seen by the relay)
@@ -11922,7 +11976,8 @@ def _codex_thread_request(resume_session_id: str | None, cwd: str, model: str) -
def _run_codex_appserver_bg(turn_id: str, cwd: str, env: dict, model: str, prompt: str,
- resume_session_id: str | None, timeout_seconds: int | None) -> None:
+ resume_session_id: str | None, timeout_seconds: int | None,
+ max_turns: int | None = None) -> None:
rec = _turns[turn_id]
state = {"final": ""}
@@ -12043,6 +12098,13 @@ def _run_codex_appserver_bg(turn_id: str, cwd: str, env: dict, model: str, promp
it = p.get("item") or {}
for ev in _codex_tool_item(it): # messages and reasoning yield nothing: already streamed as deltas
append(ev)
+ # The step budget, as _run_turn_bg counts it for `codex exec` (#471): codex has no
+ # limit of its own, and this is the path the image runs codex on by default.
+ if (max_turns and ev.get("type") == "assistant"
+ and _count_tool_calls(ev, state) > int(max_turns)):
+ state["_stop"] = "max_turns"
+ if state.get("_stop"):
+ break # past the budget: the process is stopped below, one turn per process
elif method == "turn/completed":
# Final cumulative usage may also ride the completed turn; read it as a fallback.
turn_obj = p.get("turn") or {}
@@ -12092,18 +12154,20 @@ def _run_codex_appserver_bg(turn_id: str, cwd: str, env: dict, model: str, promp
return (not low) or ("reconnect" in low) or ("stream closed" in low) or low in ("eof", "connection closed")
answered = bool(state["final"].strip())
noise_only = all(_is_reconnect_noise(l) for l in errbuf) if errbuf else True
+ stopped = state.get("_stop") == "max_turns" and not rec.get("cancelled") and not rec.get("capped")
ok = ((turn_status in ("completed", None)) or (answered and noise_only)) \
and not rec.get("cancelled") and not rec.get("capped")
err_txt = ("\n".join(errbuf[-30:]).strip() or f"app-server turn status: {turn_status}")[:2000]
# On a genuine failure, lead with the agent's own last words (often the real reason, e.g.
# "workspace is read-only") and follow with the technical error, so the Result row is never a
# blank "Failed" with no explanation.
- if ok:
+ if ok or stopped:
res_txt = state["final"]
else:
res_txt = "\n\n".join(x for x in (state["final"].strip(), err_txt) if x)[:4000] or err_txt
- ev = {"type": "result", "subtype": "success" if ok else "error", "is_error": not ok,
- "result": res_txt, "usage": usage} # surface the error in the trace
+ # stopped at its budget: incomplete, with the usage the app-server had counted so far
+ ev = {"type": "result", "subtype": "error_max_turns" if stopped else "success" if ok else "error",
+ "is_error": not ok and not stopped, "result": res_txt, "usage": usage} # surface the error in the trace
# What the relay saw the provider say, as on the exec path: the app-server reports no served
# model either.
served = _relay_served_model(env)
@@ -12114,8 +12178,8 @@ def _run_codex_appserver_bg(turn_id: str, cwd: str, env: dict, model: str, promp
append(ev)
rec["result"] = state["final"]
rec["status"] = ("cancelled" if rec.get("cancelled") else "timeout" if rec.get("capped")
- else "done" if ok else "failed")
- if not ok:
+ else "max_turns" if stopped else "done" if ok else "failed")
+ if not ok and not stopped:
rec["error"] = err_txt
rec["tried"] = [{"connection": "codex-app-server", "error": err_txt[:400]}] # -> gateway failure msg
rec["done"] = True
@@ -13540,7 +13604,8 @@ def turn(req: TurnReq, identifier: str = "") -> dict:
_turn_by_key[key] = turn_id
if use_appserver:
threading.Thread(target=_run_codex_appserver_bg,
- args=(turn_id, cwd, env, model, req.prompt, req.resume_session_id, req.timeout_seconds),
+ args=(turn_id, cwd, env, model, req.prompt, req.resume_session_id, req.timeout_seconds,
+ req.max_turns),
daemon=True).start()
elif backend == "hermes":
threading.Thread(target=_run_hermes_bg,
@@ -13551,7 +13616,8 @@ def turn(req: TurnReq, identifier: str = "") -> dict:
threading.Thread(target=_run_turn_bg,
args=(turn_id, cmd, env, cwd, spec["normalize"], model, req.timeout_seconds,
bool(req.partial_messages)), # claude: CLI flag added above
- kwargs={"max_turns": req.max_turns, "prompt": req.prompt or ""},
+ kwargs={"max_turns": req.max_turns, "prompt": req.prompt or "",
+ "count_steps": backend in _GENERIC_STEP_BUDGET},
daemon=True).start()
cap = min(req.timeout_seconds, MAX_TURN_SECONDS) if req.timeout_seconds else MAX_TURN_SECONDS
return {"turn_id": turn_id, "status": "running", "backend": backend, "model": model,
diff --git a/runner/tests/test_generic_step_budget.py b/runner/tests/test_generic_step_budget.py
new file mode 100644
index 00000000..c2d6d6db
--- /dev/null
+++ b/runner/tests/test_generic_step_budget.py
@@ -0,0 +1,240 @@
+"""The generic step budget (#471): a base with no limit flag of its own is bounded by the runner.
+
+_run_turn_bg counts tool calls in the normalised stream (every normaliser emits the same
+Claude-shaped tool_use blocks) and, past the budget, ends the turn error_max_turns — which the
+gateway reports as incomplete — instead of running to the wall-clock cap.
+"""
+import json
+import os
+import pathlib
+import sys
+import time
+import uuid
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
+import server as rs # noqa: E402
+from server import _run_turn_bg, _status_from_result, _turns # noqa: E402
+
+
+def _tool_use(i):
+ return {"type": "assistant", "message": {"content": [
+ {"type": "tool_use", "id": f"t{i}", "name": "Bash", "input": {"cmd": "true"}}]}}
+
+
+def _child_script(n_calls: int = 8, pause: float = 0.05, repeat: bool = False,
+ result: dict | None = None, hang: bool = False) -> str:
+ """A CLI that streams a claude-shaped turn: init, n tool rounds, then a result (or, with `hang`,
+ nothing more until it is killed). `repeat` reports every call twice, the way a normaliser may
+ report a call's start and then its completion."""
+ lines = ["import json,sys,time",
+ "print(json.dumps({\"type\":\"system\",\"subtype\":\"init\",\"session_id\":\"s1\"}), flush=True)"]
+ for i in range(n_calls):
+ tu = json.dumps(_tool_use(i))
+ tr = json.dumps({"type": "user", "message": {"content": [
+ {"type": "tool_result", "tool_use_id": f"t{i}", "content": "ok"}]}})
+ lines.append(f"print({json.dumps(tu)}, flush=True)")
+ if repeat:
+ lines.append(f"print({json.dumps(tu)}, flush=True)")
+ lines.append(f"print({json.dumps(tr)}, flush=True)")
+ if pause:
+ lines.append(f"time.sleep({pause})")
+ done = result or {"type": "result", "subtype": "success", "is_error": False,
+ "result": "done anyway"}
+ lines.append("time.sleep(30)" if hang else f"print({json.dumps(json.dumps(done))}, flush=True)")
+ return "\n".join(lines) + "\n"
+
+
+def _run(tmp_path, script: str, backend: str = "qwen", max_turns: int = 2,
+ count_steps: bool = True) -> dict:
+ turn_id = "gen-budget-" + uuid.uuid4().hex
+ path = tmp_path / "stream.py"
+ path.write_text(script)
+ _turns[turn_id] = {"status": "running", "events": [], "result": "", "done": False,
+ "started": time.time(), "backend": backend, "model": "m"}
+ try:
+ _run_turn_bg(turn_id, [sys.executable, str(path)], os.environ.copy(), str(tmp_path),
+ rs._claude_passthrough, "m", timeout_seconds=30,
+ max_turns=max_turns, count_steps=count_steps)
+ return _turns[turn_id]
+ finally:
+ _turns.pop(turn_id, None)
+
+
+def _results(rec: dict) -> list[dict]:
+ return [ev for ev in rec["events"] if ev.get("type") == "result"]
+
+
+def test_the_generic_budget_ends_the_turn_as_max_turns(tmp_path):
+ rec = _run(tmp_path, _child_script())
+ results = _results(rec)
+ assert results and results[-1]["subtype"] == "error_max_turns"
+ assert results[-1]["is_error"] is False
+ assert rec["status"] == "max_turns" # the gateway reports this as incomplete
+ # the budget stopped the turn: the CLI's own success result never made the record
+ assert not any(ev.get("subtype") == "success" for ev in results)
+ assert rec["done"]
+
+
+def test_no_counting_for_a_base_with_its_own_limit(tmp_path):
+ """The same stream with counting off (a base whose builder passes a native flag) runs whole."""
+ rec = _run(tmp_path, _child_script(), backend="claude", count_steps=False)
+ assert rec["status"] == "done" and rec["result"] == "done anyway"
+
+
+def test_a_result_already_written_when_the_budget_stops_the_turn_is_not_a_success(tmp_path):
+ """The CLI wrote its whole turn before the runner read a line: its success result is still
+ in the pipe after the stop. It keeps its usage, and the turn is still incomplete at its budget."""
+ usage = {"input_tokens": 1200, "output_tokens": 40}
+ rec = _run(tmp_path, _child_script(pause=0, result={
+ "type": "result", "subtype": "success", "is_error": False, "result": "all done",
+ "usage": usage}))
+ assert rec["status"] == "max_turns"
+ final = _results(rec)[-1]
+ assert final["subtype"] == "error_max_turns" and final["is_error"] is False
+ assert final["usage"] == usage
+
+
+def test_a_call_reported_twice_spends_the_budget_once(tmp_path):
+ rec = _run(tmp_path, _child_script(n_calls=2, repeat=True), max_turns=2)
+ assert rec["status"] == "done" and rec["result"] == "done anyway"
+
+
+def test_a_stop_before_any_result_carries_what_the_relay_counted(tmp_path, monkeypatch):
+ """codex never writes turn.completed once it is killed: the budget's result is the turn's, and
+ its usage is the relay's count. Where nothing counted, it carries no usage, never a zero."""
+ hang = _child_script(n_calls=4, hang=True)
+ relay = {"input_tokens": 900, "output_tokens": 12}
+ monkeypatch.setattr(rs, "_relay_usage", lambda env: dict(relay))
+ rec = _run(tmp_path, hang)
+ final = _results(rec)[-1]
+ assert rec["status"] == "max_turns" and final["subtype"] == "error_max_turns"
+ assert final["usage"] == relay
+ monkeypatch.setattr(rs, "_relay_usage", lambda env: {})
+ rec = _run(tmp_path, hang)
+ assert rec["status"] == "max_turns" and "usage" not in _results(rec)[-1]
+
+
+def test_a_remote_base_is_stopped_gracefully_and_its_charge_kept(tmp_path, monkeypatch):
+ """qoder's process is a client of a runtime elsewhere: killing it outright leaves that runtime
+ running and billing. The budget stops it the way a cancel does: SIGTERM first, so the driver
+ cancels the remote turn and reports its charge, which stays on the turn's result."""
+ monkeypatch.setitem(rs.BACKENDS, "remotebase", {"remote": True})
+ cancelled = {"type": "result", "subtype": "error", "is_error": True,
+ "result": "cancel accepted", "charge": {"credits": 3}}
+ script = "\n".join([
+ "import json,signal,sys,time",
+ "def term(*a):",
+ f" print({json.dumps(json.dumps(cancelled))}, flush=True)",
+ " sys.exit(0)",
+ "signal.signal(signal.SIGTERM, term)",
+ ] + [f"print({json.dumps(json.dumps(_tool_use(i)))}, flush=True)" for i in range(4)]
+ + ["time.sleep(30)"]) + "\n"
+ rec = _run(tmp_path, script, backend="remotebase")
+ final = _results(rec)[-1]
+ assert rec["status"] == "max_turns"
+ assert final["subtype"] == "error_max_turns" and final["is_error"] is False
+ assert final["charge"] == {"credits": 3} and final["result"] == "cancel accepted"
+
+
+def test_the_budget_set_covers_exactly_the_unbounded_bases():
+ assert rs._GENERIC_STEP_BUDGET == {"codex", "dsh", "pi", "omp", "gajae", "qwen", "gemini",
+ "cline", "opencode", "mimo", "kilo", "qoder"}
+ # and every entry is a backend the registry offers
+ assert rs._GENERIC_STEP_BUDGET <= set(rs.BACKENDS)
+
+
+def test_the_stop_result_reports_incomplete_at_the_gateway_layer():
+ ev = {"type": "result", "subtype": "error_max_turns", "is_error": False, "result": ""}
+ assert _status_from_result(ev, -9) == "max_turns"
+
+
+_FAKE_APPSERVER = '''
+import json, os, sys, time
+def out(o):
+ print(json.dumps(o), flush=True)
+for line in sys.stdin:
+ m = json.loads(line)
+ meth, mid = m.get("method"), m.get("id")
+ if meth == "initialize":
+ out({"id": mid, "result": {}})
+ elif meth in ("thread/start", "thread/resume"):
+ out({"id": mid, "result": {"thread": {"id": "th1"}}})
+ elif meth == "turn/start":
+ out({"id": mid, "result": {"turn": {"id": "tu1"}}})
+ for i in range(int(os.environ.get("FAKE_CALLS", "4"))):
+ # the model call that asked for the command is counted before the command runs
+ out({"method": "thread/tokenUsage/updated", "params": {"tokenUsage": {"total": {
+ "inputTokens": 100 * (i + 1), "outputTokens": 10 * (i + 1)}}}})
+ out({"method": "item/completed", "params": {"item": {
+ "type": "commandExecution", "id": "c%d" % i, "command": "echo %d" % i,
+ "aggregatedOutput": "ok", "exitCode": 0}}})
+ time.sleep(0.05)
+ out({"method": "item/agentMessage/delta", "params": {"delta": "DONE"}})
+ out({"method": "turn/completed", "params": {"turn": {"status": "completed"}}})
+'''
+
+
+def _appserver_turn(tmp_path, calls: int, max_turns):
+ """codex on the app-server path, which the image runs codex on by default
+ (HARNESS_CODEX_APPSERVER=1 in docker/entrypoint.sh), against a fake app-server."""
+ bindir = tmp_path / "bin"
+ bindir.mkdir()
+ fake = bindir / "codex"
+ fake.write_text(f"#!{sys.executable}\n" + _FAKE_APPSERVER)
+ fake.chmod(0o755)
+ env = {**os.environ, "PATH": f"{bindir}:{os.environ.get('PATH', '')}", "FAKE_CALLS": str(calls)}
+ turn_id = "gen-appserver-" + uuid.uuid4().hex
+ _turns[turn_id] = {"status": "running", "events": [], "result": "", "done": False,
+ "started": time.time(), "backend": "codex", "model": "m"}
+ try:
+ rs._run_codex_appserver_bg(turn_id, str(tmp_path), env, "m", "go", None, 30, max_turns)
+ return _turns[turn_id]
+ finally:
+ _turns.pop(turn_id, None)
+
+
+def test_codex_on_the_app_server_stops_at_its_budget(tmp_path):
+ rec = _appserver_turn(tmp_path, calls=4, max_turns=1)
+ final = _results(rec)[-1]
+ assert rec["status"] == "max_turns" and not rec.get("error")
+ assert final["subtype"] == "error_max_turns" and final["is_error"] is False
+ calls = [b for ev in rec["events"] if ev.get("type") == "assistant"
+ for b in ev["message"]["content"] if b.get("type") == "tool_use"]
+ assert len(calls) == 2 # stopped at the call past the budget
+ assert final["usage"].get("input_tokens") == 200 # what the app-server had counted
+
+
+def test_codex_on_the_app_server_within_its_budget_completes(tmp_path):
+ rec = _appserver_turn(tmp_path, calls=3, max_turns=400)
+ assert rec["status"] == "done" and rec["result"] == "DONE"
+
+
+def test_a_stop_before_any_result_carries_the_translators_running_count(tmp_path, monkeypatch):
+ """pi and dsh count usage as the turn goes (per message, per step): a turn their CLI never got
+ to total still reports what was counted, as their own result would."""
+ monkeypatch.setattr(rs, "_relay_usage", lambda env: {})
+
+ def counting(obj, state):
+ if obj.get("type") == "usage": # stands for pi's message_end usage
+ rs._pi_usage_add(state, obj["usage"])
+ return []
+ return rs._claude_passthrough(obj, state)
+
+ lines = ["import json,time",
+ f"print({json.dumps(json.dumps({'type': 'usage', 'usage': {'input': 50, 'output': 5}}))}, flush=True)"]
+ lines += [f"print({json.dumps(json.dumps(_tool_use(i)))}, flush=True)" for i in range(4)]
+ lines.append("time.sleep(30)")
+ path = tmp_path / "stream.py"
+ path.write_text("\n".join(lines) + "\n")
+ turn_id = "gen-running-" + uuid.uuid4().hex
+ _turns[turn_id] = {"status": "running", "events": [], "result": "", "done": False,
+ "started": time.time(), "backend": "pi", "model": "m"}
+ try:
+ _run_turn_bg(turn_id, [sys.executable, str(path)], os.environ.copy(), str(tmp_path),
+ counting, "m", timeout_seconds=30, max_turns=1, count_steps=True)
+ rec = _turns[turn_id]
+ finally:
+ _turns.pop(turn_id, None)
+ final = _results(rec)[-1]
+ assert rec["status"] == "max_turns" and final["subtype"] == "error_max_turns"
+ assert final["usage"] == {"input_tokens": 50, "output_tokens": 5}
diff --git a/runner/tests/test_relay_usage.py b/runner/tests/test_relay_usage.py
index 86a8134f..b8f1baed 100644
--- a/runner/tests/test_relay_usage.py
+++ b/runner/tests/test_relay_usage.py
@@ -144,5 +144,8 @@ def test_a_backend_that_reports_its_own_usage_keeps_it():
split count (codex, claude) is the record. Pinned on the source of the run loop, which needs
a live process to exercise: both result sites go through the one helper."""
src = pathlib.Path(__file__).resolve().parents[1].joinpath("server.py").read_text()
- assert src.count("_fill_relay_usage(ev, env)") == 4 # the run loop, its eof path, the hermes and codex app-server drivers
+ # the run loop's _settle (its stream, its eof path and a budget's stop), the hermes and codex
+ # app-server drivers
+ assert src.count("_fill_relay_usage(ev, env)") == 3
+ assert src.count("_settle(ev)") == 2 and src.count("_settle(result_ev)") == 1
assert src.count('if ev.get("type") == "result" and not ev.get("usage"):') == 0
diff --git a/ui/src/components/TaskChat.tsx b/ui/src/components/TaskChat.tsx
index 7a879d0d..6e0b473f 100644
--- a/ui/src/components/TaskChat.tsx
+++ b/ui/src/components/TaskChat.tsx
@@ -617,8 +617,10 @@ function Conversation({ harnessId, sessionId, target, models, onModel, onRan, on
<>
{m.status === 'incomplete' && m.incompleteReason && (
<div className="wbx-incomplete-why">
- {m.incompleteReason === 'max_steps' ? 'It hit its step limit.'
- : m.incompleteReason === 'timeout' ? 'It hit its time limit.'
+ {m.incompleteReason === 'max_steps'
+ ? 'It reached its step limit. Raise Max steps under Runtime limits in this harness\u2019s settings to let it go further.'
+ : m.incompleteReason === 'timeout'
+ ? 'It reached its time limit. Raise Timeout under Runtime limits in this harness\u2019s settings to give it longer.'
: m.incompleteReason === 'interrupted'
? 'The turn was interrupted before it finished, for example by a server restart. No limit was hit.'
: null}