Skip to content

Official Codebase of 3DCodeVerse

Python

agents: one classify_end ends every vendor CLI session — a kill with no provider evidence is not transient, a usage limit or 429 is budget

gaoypengdeea64d1c952
diff --git a/harness/codeverse3d/agents/backends.py b/harness/codeverse3d/agents/backends.py
index 3e5c9fdf..f32c65f8 100644
--- a/harness/codeverse3d/agents/backends.py
+++ b/harness/codeverse3d/agents/backends.py
@@ -49,16 +49,16 @@ from typing import Any
from codeverse3d.agents.cli_common import (
CompletedProc,
Session,
+ SessionEnd,
ToolCall,
begin_session,
+ classify_end,
exists_on_path,
failed,
find_json_object,
finish_session,
hardened_env,
invoke,
- is_quota_failure,
- is_rate_limited,
is_transient_failure,
mcp_command_for,
provider_wait,
@@ -469,33 +469,31 @@ class GeminiCliAgent(_CliAgent):
text = out.text.strip()
served = list((stats.get("models") or {}).keys())
if proc.timed_out:
- out.exit_reason = "timeout"
- out.errors.append(watchdog_error(proc))
# the CLI retries 503s itself, with backoff, and never gives up before our wall:
# measured 2026-09-07, 8-17 "Attempt N failed with status 503" per 12-minute
# session, nothing produced. That is a storm, not the task.
- out.transient = not text and bool(retries.rows)
- if out.transient:
+ end = classify_end(proc, ok=False, text=text, provider_retried=bool(retries.rows))
+ out.errors.append(watchdog_error(proc))
+ if end.transient:
out.errors.append(f"{retries.tally()} inside the CLI's own retry loop before the wall; nothing produced")
- return out
- if served and self.model not in served:
+ elif served and self.model not in served:
out.exit_reason = "model_substituted"
out.errors.append(f"requested {self.model!r} but CLI served {served}")
return out
- if parsed is None or proc.rc != 0 or not text:
+ elif parsed is None or proc.rc != 0 or not text:
# the CLI's own give-up object (stderr: {"session_id", "error": {"type", "message", "code"}}),
# else whatever it printed before it died
gave_up = find_json_object(proc.stderr, lambda d: isinstance(d.get("error"), dict))
err = (gave_up or {}).get("error") or {}
said = f"{err.get('code', '')} {err.get('type', '')}: {err.get('message', '')}" if err else proc.stderr + proc.stdout
out.session_id = out.session_id or str((gave_up or {}).get("session_id") or "")
- out.quota = is_quota_failure(said)
- out.transient = is_transient_failure(said) or (parsed is not None and not text)
- out.rate_limited = err.get("code") == 429 or (not err and is_rate_limited(said))
+ end = classify_end(proc, ok=False, text=text, said=(said,), provider_failed=parsed is not None and not text,
+ rate_limited=err.get("code") == 429 if err else None)
out.errors.append(f"rc={proc.rc}; response={'<empty>' if not text else 'ok'}; stderr tail: {proc.stderr[-1500:]}")
- out.exit_reason = "budget" if out.rate_limited or out.quota else "error"
- return out
- out.ok, out.exit_reason = True, "completed"
+ else:
+ end = classify_end(proc, ok=True, text=text)
+ out.ok = True
+ out.exit_reason, out.transient, out.quota, out.rate_limited = end.exit_reason, end.transient, end.quota, end.rate_limited
return out
@@ -741,31 +739,23 @@ class ClaudeCodeAgent(_CliAgent):
text = str((env or {}).get("result") or "")
turns = int((env or {}).get("num_turns") or 0)
errors: list[str] = []
- transient = quota = False
+ sub = str((env or {}).get("subtype", ""))
if proc.timed_out:
- reason, ok = "timeout", False
+ end = classify_end(proc, ok=False, text=text, provider_retried=bool(stream.retries))
errors.append(watchdog_error(proc))
- transient = not text.strip() and (bool(stream.retries) or is_transient_failure(proc.stderr))
elif env is None or proc.rc != 0:
- reason, ok = "error", False
- errors.append(f"rc={proc.rc}; no result envelope; stderr tail: {proc.stderr[-1500:]}")
# a 529 / overloaded exit is the provider's, not the task's (AgentResult.transient)
- quota = is_quota_failure(proc.stderr, proc.stdout)
- transient = is_transient_failure(proc.stderr, proc.stdout)
- if quota or (transient and is_rate_limited(proc.stderr)):
- reason = "budget"
- elif env.get("is_error") or str(env.get("subtype", "")).startswith("error"):
- ok = False
- sub = str(env.get("subtype", ""))
- quota = is_quota_failure(text)
- transient = "max_turns" not in sub and is_transient_failure(text)
- reason = "budget" if "max_turns" in sub or quota else "error"
+ end = classify_end(proc, ok=False, text=text, said=(proc.stderr, proc.stdout))
+ errors.append(f"rc={proc.rc}; no result envelope; stderr tail: {proc.stderr[-1500:]}")
+ elif env.get("is_error") or sub.startswith("error"):
+ # the turn cap is the task's budget, never the provider's
+ end = SessionEnd("budget") if "max_turns" in sub else classify_end(proc, ok=False, text=text, said=(text,))
errors.append(f"claude reported {sub or 'is_error'}: {text[-800:]}")
else:
- reason, ok = "completed", True
+ end = classify_end(proc, ok=True, text=text)
return finish_session(
- s, ok=ok, exit_reason=reason, text=text, usage=usage, tool_calls=n_calls, turns=turns,
- errors=errors, transient=transient, quota=quota,
+ s, ok=end.exit_reason == "completed", exit_reason=end.exit_reason, text=text, usage=usage,
+ tool_calls=n_calls, turns=turns, errors=errors, transient=end.transient, quota=end.quota,
provider_wait_s=provider_wait([(t, b) for t, b, _ in stream.retries], ended, stream.progress),
rc=proc.rc, killed_reason=proc.killed_reason, num_turns=turns,
session_id=(env or {}).get("session_id", ""), subtype=(env or {}).get("subtype", ""),
@@ -954,20 +944,16 @@ class CodexAgent(_CliAgent):
text = "\n\n".join(m for m in events.messages if m.strip())
errors = list(events.errors)
if proc.timed_out:
- ok, reason = False, "timeout"
errors.append(watchdog_error(proc))
elif proc.rc != 0 or events.n_events == 0:
- ok, reason = False, "error"
errors.append(f"rc={proc.rc}; events={events.n_events}; stderr tail: {proc.stderr[-1500:]}")
- elif events.errors and events.turns_completed == 0:
- ok, reason = False, "error"
- else:
- ok, reason = True, "completed"
- said = (*events.errors, proc.stderr) if not ok else ()
+ ok = not proc.timed_out and proc.rc == 0 and events.n_events > 0 and not (events.errors and events.turns_completed == 0)
+ # codex's provider record is its own error events (a dropped stream, a 5xx it gave up on)
+ end = classify_end(proc, ok=ok, text=text, said=(*events.errors, proc.stderr),
+ provider_retried=is_transient_failure(*events.errors))
return finish_session(
- s, ok=ok, exit_reason=reason, text=text, usage=usage, tool_calls=events.tool_calls,
- turns=events.turns_completed, errors=errors,
- transient=is_transient_failure(*said), quota=is_quota_failure(*said),
+ s, ok=ok, exit_reason=end.exit_reason, text=text, usage=usage, tool_calls=events.tool_calls,
+ turns=events.turns_completed, errors=errors, transient=end.transient, quota=end.quota,
rc=proc.rc, killed_reason=proc.killed_reason, thread_id=events.thread_id,
turns_completed=events.turns_completed, usage_raw=events.usage_raw, usage_estimated=estimated,
)
@@ -1233,25 +1219,22 @@ class AntigravityAgent(_CliAgent):
s.notes.append("agy is subscription-billed: cost_usd=0")
died = agy_error(proc.stderr)
said = (json.dumps(died), str((env or {}).get("error") or ""), proc.stderr)
+ ok = False
if proc.timed_out:
- ok, reason = False, "timeout"
errors.append(watchdog_error(proc))
elif proc.rc != 0 or (env is not None and str(env.get("status", "SUCCESS")).upper() not in ("SUCCESS", "OK")):
- ok, reason = False, "error"
errors.append(f"rc={proc.rc}; status={(env or {}).get('status')}; error={(env or {}).get('error', '')}; "
f"stderr tail: {proc.stderr[-1500:]}")
elif not text.strip():
- ok, reason = False, "error"
errors.append("empty response")
else:
- ok, reason = True, "completed"
- quota = not ok and is_quota_failure(*said)
- transient = not ok and (bool(died.get("retryable")) or is_transient_failure(*said)
- or (proc.timed_out and not text.strip() and bool(retries)))
+ ok = True
+ end = classify_end(proc, ok=ok, text=text, said=said, provider_retried=bool(retries) or bool(died.get("retryable")),
+ provider_failed=bool(died.get("retryable")))
usage.tool_calls = n_calls
return finish_session(
- s, ok=ok, exit_reason=reason, text=text, usage=usage, tool_calls=n_calls,
- turns=int((env or {}).get("num_turns") or 0), errors=errors, transient=transient, quota=quota,
+ s, ok=ok, exit_reason=end.exit_reason, text=text, usage=usage, tool_calls=n_calls,
+ turns=int((env or {}).get("num_turns") or 0), errors=errors, transient=end.transient, quota=end.quota,
provider_wait_s=provider_wait([(t, b) for t, b, _ in retries], ended),
rc=proc.rc, killed_reason=proc.killed_reason, agy_error=died,
conversation_id=(env or {}).get("conversation_id", ""), num_turns=(env or {}).get("num_turns", 0),
diff --git a/harness/codeverse3d/agents/cli_common.py b/harness/codeverse3d/agents/cli_common.py
index e2a36f07..d11bb363 100644
--- a/harness/codeverse3d/agents/cli_common.py
+++ b/harness/codeverse3d/agents/cli_common.py
@@ -718,6 +718,43 @@ def is_rate_limited(*texts: str) -> bool:
return any(_RATE_LIMIT_RE.search(t or "") for t in texts)
+@dataclass(frozen=True)
+class SessionEnd:
+ """How a CLI session ended: ``exit_reason`` (completed | timeout | budget | error) and the
+ typed failure class ``AgentResult.transient`` / ``.quota`` carry. ``rate_limited`` is a
+ 429 — gemini-cli cools its key and rotates."""
+
+ exit_reason: str
+ transient: bool = False
+ quota: bool = False
+ rate_limited: bool = False
+
+
+def classify_end(proc: CompletedProc, *, ok: bool, text: str, said: Sequence[str] = (),
+ provider_retried: bool = False, provider_failed: bool = False,
+ rate_limited: bool | None = None) -> SessionEnd:
+ """THE end rule for every CLI backend (N71; D83) — each backend brings only its vendor's
+ evidence: ``said`` (what the CLI printed about the call that ended it), ``provider_retried``
+ (the CLI's OWN record of retried provider calls: gemini-cli's retry lines, claude's
+ ``api_retry`` events, codex's error events, agy's log) and ``provider_failed`` (a typed
+ vendor verdict: agy's ``AGY_ERROR.retryable``, gemini-cli's empty envelope).
+
+ A watchdog kill is ``timeout``, and transient only when nothing was produced AND the CLI
+ recorded provider retries: a kill with no provider evidence is the task's (a retry buys
+ another full window for the same outcome), whatever its stderr says. Any other failure is
+ ``quota`` / ``transient`` by the vocabulary above, and ``budget`` when the usage limit is
+ spent or the call was rate limited, for every vendor."""
+ if ok:
+ return SessionEnd("completed")
+ if proc.timed_out:
+ return SessionEnd("timeout", transient=provider_retried and not text.strip())
+ quota = is_quota_failure(*said)
+ limited = is_rate_limited(*said) if rate_limited is None else rate_limited
+ return SessionEnd("budget" if quota or limited else "error",
+ transient=not quota and (provider_failed or is_transient_failure(*said)),
+ quota=quota, rate_limited=limited)
+
+
# --------------------------------------------------------------------------- provider wait
def provider_wait(failures: Iterable[tuple[float, float]], end: float,
progress: Iterable[float] | None = None) -> float:
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index 90147bec..cd7594a0 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -901,7 +901,9 @@ Pointers: EVAL, PAPER_WRITING = `eval/docs/*.md`; COST, RUNBOOK, ARCHITECTURE, I
over the 11 gemini sessions of the 2026-09-22 live runs, $0.219 booked → $5.11. `provider_wait_s`
is the time a session lost to provider errors (a lower bound for gemini, 0 for codex). Failure
classes travel typed (`AgentResult.transient/quota` → `RoundFailed`); a plain timeout is no longer
- retried as a transport death.
+ retried as a transport death. One `cli_common.classify_end` decides every vendor's end (owner,
+ 2026-09-24, N71): a watchdog kill is transient only when nothing was produced and the CLI's own record
+ shows provider retries, and a spent usage limit or a 429 is `budget` for all four.
* **D84 One minutes, one record of money (owner, 2026-09-22).** Every step is a `StepTime{step,
round, wall_s, lost_s}`; `RunRecord.minutes` = Σ(wall − lost) over run and round steps — lost is a
session's `provider_wait_s` or an API call's time outside its answered round-trips; parallel work is
diff --git a/harness/tests/agents/test_cli_common.py b/harness/tests/agents/test_cli_common.py
index 0cba0e1f..79bd42a1 100644
--- a/harness/tests/agents/test_cli_common.py
+++ b/harness/tests/agents/test_cli_common.py
@@ -184,3 +184,43 @@ def test_mcp_command_resolution(tmp_ws: Workspace):
{"mcpServers": {"3dcode": {"command": "/tmp/evil", "args": ["--pwn"]}}}))
assert mcp_command_for(tmp_ws, job2) == ["python", "-m", "x"]
assert "/tmp/evil" not in mcp_command_for(tmp_ws, job)
+
+
+_ENDINGS = {
+ # the watchdog kills a session that printed a transport line and recorded no provider retry (D83)
+ "killed": ('print(json.dumps({"type": "item.completed", "item": {"type": "agent_message", "text": "working"}}),'
+ ' flush=True)\nprint("Error: read ECONNRESET (mcp child)", file=sys.stderr, flush=True)\ntime.sleep(60)\n',
+ ("timeout", False, False)),
+ "usage_limit": ('print("You\'ve hit your usage limit. Try again later.", file=sys.stderr)\nsys.exit(1)\n',
+ ("budget", False, True)),
+ "rate_limited": ('print("Error 429 Too Many Requests: rate limit", file=sys.stderr)\nsys.exit(1)\n',
+ ("budget", True, False)),
+}
+
+
[email protected]("ending", sorted(_ENDINGS))
[email protected]("vendor", ["gemini-cli", "claude-code", "codex", "agy"])
+def test_every_cli_backend_ends_by_one_rule(tmp_ws: Workspace, fake_bin, monkeypatch, vendor, ending):
+ """N71: one ``classify_end`` — a kill with no provider evidence is not transient, and a spent
+ usage limit or a 429 is ``budget``, whichever vendor's CLI said it."""
+ from codeverse3d.agents.backends import (
+ AntigravityAgent,
+ ClaudeCodeAgent,
+ CodexAgent,
+ GeminiCliAgent,
+ )
+ from codeverse3d.config import get_settings
+
+ monkeypatch.setattr(cli_common, "IDLE_GRACE_S", 0.3)
+ monkeypatch.setattr("codeverse3d.agents.backends.RETRY_KEY_WAIT_S", 0.05)
+ monkeypatch.setattr(get_settings(), "gemini_api_keys", ["n71-key"])
+ body, want = _ENDINGS[ending]
+ binary = fake_bin(vendor, "sys.stdin.read()\n" + body)
+ agent = {"gemini-cli": lambda: GeminiCliAgent("m", binary=binary), "claude-code": lambda: ClaudeCodeAgent("m", binary=binary),
+ "codex": lambda: CodexAgent("m", binary=binary, reasoning_effort=""),
+ "agy": lambda: AntigravityAgent("m", binary=binary)}[vendor]()
+ if vendor == "agy":
+ monkeypatch.setattr(agent, "served_model", lambda: "m")
+ res = agent.run(AgentJob(workspace=str(tmp_ws.root), prompt="p", label=ending, spatial_tools=False,
+ timeout_s=1 if ending == "killed" else 30))
+ assert (res.exit_reason, res.transient, res.quota) == want