Skip to content

Official Codebase of 3DCodeVerse

Python

agents: prompts on stdin; usage, provider wait and failure class from each CLI's own record

gaoypeng6b8be82d3439
diff --git a/harness/codeverse3d/agents/backends.py b/harness/codeverse3d/agents/backends.py
index 3ac9933f..89351750 100644
--- a/harness/codeverse3d/agents/backends.py
+++ b/harness/codeverse3d/agents/backends.py
@@ -1,16 +1,22 @@
"""``gemini-cli:<model>`` — headless Gemini CLI session on a workspace.
-argv: ``gemini -p <prompt> -m <model> --approval-mode yolo --skip-trust --output-format json``
+argv: ``gemini -m <model> --approval-mode yolo --skip-trust --output-format json`` with the
+prompt on stdin (every backend: ``cli_common.invoke``).
* env hardening: host secrets stripped, one pool key as ``GEMINI_API_KEY``,
``GEMINI_CLI_SYSTEM_SETTINGS_PATH`` (per session, in the trajectory dir) forcing
api-key auth + dynamic model configuration (otherwise an unknown model is *silently*
substituted), ``skills.enabled``, the 3dcode MCP server and ``mcp.allowed``, Node heap
cap, no self-relaunch.
-* tool trace: gemini-cli's own chat record of the session (``~/.gemini/tmp/<project>/
- chats/session-*.jsonl``) is folded into the transcript as ``tool_call`` rows
- (``cli_common.record_tool_calls``) — ``--output-format json`` carries only per-tool
- COUNTS, and ``stream-json`` would cost the per-model ``thoughts`` tokens pricing needs.
+* gemini-cli's own chat record of the attempt (``~/.gemini/tmp/<project>/chats/
+ session-*.jsonl``, found by ``gemini_chat_records``) is read once (``read_gemini_chats``):
+ its tool calls become the transcript's ``tool_call`` rows (``cli_common.record_tool_calls``;
+ ``--output-format json`` carries only per-tool COUNTS), and its per-message tokens are the
+ usage of an attempt that printed no envelope — the CLI prints its JSON once, at the end,
+ so a session the watchdog killed printed nothing, and one that gave up on a 503 printed
+ only an error object on stderr with no stats (every gemini-cli session of the 2026-09-22
+ live runs, booked at $0 before this). ``stream-json`` would cost the per-model
+ ``thoughts`` tokens pricing needs.
* JSON envelope ``{session_id, response, stats:{models:{<m>:{tokens:{prompt,
input, candidates, cached, thoughts}}}, tools:{totalCalls}}}`` → Usage (+ cost
via pricing). ``tokens.prompt`` is the TOTAL prompt size and ``tokens.input``
@@ -18,8 +24,10 @@ argv: ``gemini -p <prompt> -m <model> --approval-mode yolo --skip-trust --output
pricing convention (total prompt, cached re-priced). Each served model is
priced at its own rate. Requested model absent from ``stats.models`` →
``model_substituted``.
+* provider wait: the CLI's own stderr line for each API attempt it retries, timed as it
+ arrives, against the chat record's message times (``cli_common.provider_wait``).
* one retry (rotated key when the pool has one, else the same key) on transient
- failures (503 / empty response), two on a quota 429; a throttled/empty pool never raises.
+ failures (503 / empty response), two on a rate-limit 429; a throttled/empty pool never raises.
"""
from __future__ import annotations
@@ -33,6 +41,7 @@ import re
import sqlite3
import time
from collections.abc import Iterable, Mapping
+from datetime import datetime
from functools import lru_cache
from pathlib import Path
from typing import Any
@@ -42,7 +51,6 @@ from codeverse3d.agents.cli_common import (
Session,
ToolCall,
begin_session,
- deliver_prompt,
exists_on_path,
failed,
find_json_object,
@@ -50,8 +58,10 @@ from codeverse3d.agents.cli_common import (
hardened_env,
invoke,
is_quota_failure,
+ is_rate_limited,
is_transient_failure,
mcp_command_for,
+ provider_wait,
record_tool_calls,
release_session,
tail,
@@ -61,7 +71,7 @@ from codeverse3d.agents.materialize import MCP_SERVER_NAME, MCP_TOOL_TIMEOUT_MS,
from codeverse3d.config import get_settings
from codeverse3d.contracts.agent import AgentJob, AgentResult
from codeverse3d.contracts.common import Usage
-from codeverse3d.models.pricing import estimate_cost
+from codeverse3d.models.pricing import cache_write_surcharge, estimate_cost
from codeverse3d.models.retry import KeyPool, KeyPoolExhausted
from codeverse3d.proc import read_jsonl_lenient, run_subprocess
@@ -156,15 +166,43 @@ def gemini_chat_records(ws_root: Path, since: float, env: Mapping[str, str] | No
return []
-def gemini_tool_calls(records: Iterable[Path]) -> list[ToolCall]:
- """Every tool call in those chat records, once per call id (the record is append-only:
- a message is re-written as it progresses, and the last copy wins)."""
+class GeminiChatLog:
+ """What gemini-cli's own chat records of one attempt hold: its tool calls, each served
+ model's tokens (``tokens`` in the envelope's ``stats.models`` shape) and when each
+ message landed (epoch seconds — the session's progress, for ``provider_wait``)."""
+
+ calls: list[ToolCall]
+ tokens: dict[str, dict[str, int]]
+ times: list[float]
+
+ def stats(self) -> dict[str, Any]:
+ return {"models": {m: {"tokens": t} for m, t in self.tokens.items()}}
+
+
+def read_gemini_chats(records: Iterable[Path]) -> GeminiChatLog:
+ """One pass over the chat records. The record is append-only — a message is re-written
+ as it progresses — so every tool call counts once per call id and every model message
+ once per message id, the last copy winning. A model message's ``tokens`` are the API's
+ own usage metadata (``input`` = promptTokenCount, the TOTAL prompt), so their sum is the
+ envelope's ``stats.models`` to the token: checked on two completed sessions of the
+ 2026-09-22 live runs (917 248 / 704 375 cached / 18 767 / 5 306, and 1 056 300 / 907 666
+ / 5 464 / 5 047). What it misses: a utility call outside the main chat (none in either)."""
calls: dict[str, ToolCall] = {}
+ replies: dict[str, dict[str, Any]] = {}
+ times: list[float] = []
for path in records:
for row in read_jsonl_lenient(path, dicts_only=True):
update = row.get("$set") if isinstance(row.get("$set"), dict) else {}
for msg in (row, *(update.get("messages") or [])):
- for tc in (msg.get("toolCalls") if isinstance(msg, dict) else None) or []:
+ if not isinstance(msg, dict):
+ continue
+ if msg.get("timestamp"):
+ with contextlib.suppress(TypeError, ValueError):
+ times.append(datetime.fromisoformat(str(msg["timestamp"]).replace("Z", "+00:00")).timestamp())
+ if msg.get("type") == "gemini" and msg.get("id"):
+ replies[str(msg["id"])] = msg
+ for tc in msg.get("toolCalls") or []:
if not isinstance(tc, dict):
continue
name = str(tc.get("name") or "")
@@ -172,7 +210,47 @@ def gemini_tool_calls(records: Iterable[Path]) -> list[ToolCall]:
calls[str(tc.get("id") or f"{path.name}#{len(calls)}")] = ToolCall(
tool=name, args=args, skill=str(args.get("name") or "") if name == GEMINI_SKILL_TOOL else "",
failed=tc.get("status") in ("error", "cancelled"))
- return list(calls.values())
+ tokens: dict[str, dict[str, int]] = {}
+ for msg in replies.values():
+ tok = msg.get("tokens") if isinstance(msg.get("tokens"), dict) else None
+ if not tok:
+ continue # a reply the session died in the middle of: never counted
+ t = tokens.setdefault(str(msg.get("model") or ""), {"prompt": 0, "cached": 0, "candidates": 0, "thoughts": 0})
+ for ours, theirs in (("prompt", "input"), ("cached", "cached"), ("candidates", "output"), ("thoughts", "thoughts")):
+ t[ours] += int(tok.get(theirs) or 0)
+ return GeminiChatLog(calls=list(calls.values()), tokens=tokens, times=sorted(set(times)))
+
+
+#: what gemini-cli 0.53 prints on stderr for each failed API attempt it is about to retry — with
+#: the HTTP status when it has one ("... with status 503", "... with 5xx error"), with the delay
+#: when the server named one ("Retrying after 2500ms") — and the first line of its give-up
+_GEMINI_RETRY_LINE = re.compile(r"Attempt \d+ failed(?: with status (?P<status>\d{3})| with (?P<kind>429|5xx) error)?"
+ r"(?:.*Retrying after (?P<ms>\d+)ms)?")
+_GEMINI_GIVE_UP = "Error when talking to Gemini API"
+
+
+class GeminiRetries:
+ """gemini-cli's own report of its failed API attempts, timed as its stderr lines arrive:
+ ``rows`` = ``(t, announced back-off s, status)`` — ``"503"`` / ``"429"`` / ``"5xx"``,
+ ``"network"`` for a transport failure ("fetch failed"), ``"gave up"`` for the last line."""
+
+ def __init__(self) -> None:
+ self.rows: list[tuple[float, float, str]] = []
+
+ def feed(self, line: str) -> None:
+ m = _GEMINI_RETRY_LINE.search(line)
+ if m:
+ status = m.group("status") or m.group("kind") or "network"
+ self.rows.append((time.time(), int(m.group("ms") or 0) / 1000.0, status))
+ elif _GEMINI_GIVE_UP in line:
+ self.rows.append((time.time(), 0.0, "gave up"))
+
+ def tally(self) -> str:
+ """``4 x 503, 2 x network`` — what the storm was made of."""
+ counts: dict[str, int] = {}
+ for _, _, status in self.rows:
+ counts[status] = counts.get(status, 0) + 1
+ return ", ".join(f"{n} x {k}" for k, n in counts.items())
def _model_usage(tok: dict[str, Any], name: str) -> Usage:
@@ -259,9 +337,10 @@ class GeminiCliAgent(_CliAgent):
return True, "ok"
# ------------------------------------------------------------------ build
- def build_argv(self, prompt: str) -> list[str]:
- return [self.binary, "-p", prompt, "-m", self.model, "--approval-mode", "yolo",
- "--skip-trust", "--output-format", "json"]
+ def build_argv(self) -> list[str]:
+ """No ``-p``: the prompt comes on stdin, and a piped stdin is what makes 0.53 headless
+ (``isHeadlessMode``); with no ``-p`` the input IS the stdin text, byte for byte."""
+ return [self.binary, "-m", self.model, "--approval-mode", "yolo", "--skip-trust", "--output-format", "json"]
def build_env(self, s: Session, api_key: str) -> dict[str, str]:
env = hardened_env(s.ws, s.job)
@@ -284,7 +363,7 @@ class GeminiCliAgent(_CliAgent):
try:
if not ok:
return failed(s, "error", why)
- prompt = deliver_prompt(s, _compose_prompt(job))
+ prompt = _compose_prompt(job)
pool = _key_pool(get_settings().gemini_api_keys)
try:
key = pool.session_key() # no in-flight slot: the CLI makes its own calls
@@ -295,96 +374,141 @@ class GeminiCliAgent(_CliAgent):
t0 = time.monotonic()
next_soft: float | None = None # attempt 1 uses the job's own window
usage_total = Usage(backend=self.kind, model=self.model)
+ wait_s, retry_from, usage_from = 0.0, 0.0, []
while True:
attempts += 1
used.add(key)
started, env = time.time(), self.build_env(s, key)
- proc = invoke(s, self.build_argv(prompt), env, prompt=prompt, attempt=attempts,
- soft_timeout_s=next_soft, model=self.model, key_tail=key[-4:])
- self._record_trace(s, started, env)
- outcome = self._interpret(s, proc)
- usage_total = usage_total + outcome["usage"]
+ if retry_from: # rotating off a failed key is time the provider cost, too
+ wait_s += max(0.0, started - retry_from)
+ retries = GeminiRetries()
+ proc = invoke(s, self.build_argv(), env, prompt=prompt, attempt=attempts, soft_timeout_s=next_soft,
+ on_stderr=retries.feed, model=self.model, key_tail=key[-4:])
+ ended = time.time()
+ outcome = self._interpret(proc, retries, self._read_record(s, started, env), started, ended)
+ usage_total = usage_total + outcome.usage
+ usage_from.append(outcome.usage_source)
+ wait_s += outcome.wait_s
# the pool is shared with every API call: a hang, a substituted model or the
- # agent's own failure is not the key's fault ("skip" = counters untouched)
- pool.report(key, "429" if outcome["quota"] else "ok" if outcome["ok"]
- else "5xx" if outcome["transient"] else "skip")
+ # agent's own failure is not the key's fault ("skip" = counters untouched).
+ # session_key took no in-flight slot, so there is nothing to release.
+ pool.report(key, "429" if outcome.rate_limited else "ok" if outcome.ok
+ else "5xx" if outcome.transient else "skip")
# a per-minute 429 on the single key a CLI process holds clears within the
# rotated retry's wait; ab_fewer_turns (2026-08-29) lost 4 of 10 cells to
- # two 429s in a row, so a quota failure gets one more rotated attempt
- max_attempts = QUOTA_MAX_ATTEMPTS if outcome["quota"] else 2
- if outcome["ok"] or outcome["exit_reason"] in ("timeout", "model_substituted") or not outcome["transient"] or attempts >= max_attempts:
+ # two 429s in a row, so a rate-limited failure gets one more rotated attempt
+ max_attempts = QUOTA_MAX_ATTEMPTS if outcome.rate_limited else 2
+ if (outcome.ok or outcome.exit_reason in ("timeout", "model_substituted") or not outcome.transient
+ or attempts >= max_attempts):
break
next_soft = retry_window_left(job.timeout_s, time.monotonic() - t0)
if next_soft is None:
- s.notes.append(f"attempt {attempts} transient failure ({outcome['exit_reason']}); "
+ s.notes.append(f"attempt {attempts} transient failure ({outcome.exit_reason}); "
"no session window left — not retrying past the wall")
break
+ retry_from = time.time()
next_key = _retry_key(pool, used)
if next_key is None:
- s.notes.append(f"attempt {attempts} transient failure ({outcome['exit_reason']}); no usable key for a retry")
+ s.notes.append(f"attempt {attempts} transient failure ({outcome.exit_reason}); no usable key for a retry")
break
key = next_key
how = "rotated key" if key not in used else "same key (no alternative available)"
- s.notes.append(f"attempt {attempts} transient failure ({outcome['exit_reason']}); retrying with {how}")
- s.traj.append("retry", attempt=attempts, reason=outcome["errors"][:1])
+ s.notes.append(f"attempt {attempts} transient failure ({outcome.exit_reason}); retrying with {how}")
+ s.traj.append("retry", attempt=attempts, reason=outcome.errors[:1])
usage_total.cost_usd = round(usage_total.cost_usd, 6)
# turns stays 0: gemini-cli's stats carry tools.totalCalls only, no turn count
return finish_session(
- s, ok=outcome["ok"], exit_reason=outcome["exit_reason"], text=outcome["text"],
- usage=usage_total, tool_calls=usage_total.tool_calls, errors=outcome["errors"],
- transient=bool(outcome["transient"]) and not outcome["ok"],
- attempts=attempts, rc=proc.rc, killed_reason=proc.killed_reason, session_id=outcome.get("session_id", ""),
+ s, ok=outcome.ok, exit_reason=outcome.exit_reason, text=outcome.text,
+ usage=usage_total, tool_calls=usage_total.tool_calls, errors=outcome.errors,
+ transient=outcome.transient and not outcome.ok, quota=outcome.quota, provider_wait_s=wait_s,
+ attempts=attempts, rc=proc.rc, killed_reason=proc.killed_reason, session_id=outcome.session_id,
+ usage_from=usage_from, # per attempt: envelope | chat record | none
)
finally:
release_session(s)
- def _record_trace(self, s: Session, started: float, env: Mapping[str, str]) -> None:
- """This attempt's tool calls, from gemini-cli's own chat record — never fatal."""
+ def _read_record(self, s: Session, started: float, env: Mapping[str, str]) -> GeminiChatLog | None:
+ """This attempt's chat record, its tool calls written as the transcript's trace — never fatal."""
try:
records = gemini_chat_records(s.ws.root, started, env)
- if records:
- record_tool_calls(s, gemini_tool_calls(records), source="gemini-cli chat record")
- else:
+ if not records:
s.notes.append("no gemini-cli chat record found for this attempt: tool trace missing")
+ return None
+ chat = read_gemini_chats(records)
+ record_tool_calls(s, chat.calls, source="gemini-cli chat record")
+ return chat
except Exception as e: # noqa: BLE001 — the trace is telemetry; a session must never fail on it
log.debug("gemini-cli chat record unreadable: %s", e)
+ return None
- def _interpret(self, s: Session, proc: CompletedProc) -> dict[str, Any]:
+ def _interpret(self, proc: CompletedProc, retries: GeminiRetries, chat: GeminiChatLog | None,
+ started: float, ended: float) -> _GeminiAttempt:
parsed = parse_gemini_json(proc.stdout)
stats = (parsed or {}).get("stats") or {}
- usage = usage_from_stats(stats, self.model) if stats else Usage(backend=self.kind, model=self.model)
- usage.latency_ms = int(proc.duration_s * 1000)
- text = str((parsed or {}).get("response") or "")
- errors: list[str] = []
+ out = _GeminiAttempt(usage=Usage(backend=self.kind, model=self.model), text=str((parsed or {}).get("response") or ""),
+ session_id=str((parsed or {}).get("session_id") or ""))
+ if stats:
+ out.usage, out.usage_source = usage_from_stats(stats, self.model), "envelope"
+ elif chat is not None and chat.tokens:
+ # no envelope — the CLI prints its JSON once, at the end: a session the watchdog killed
+ # printed none, and one that gave up printed an error object on stderr with no stats.
+ # Its chat record holds every reply's tokens; the envelope, when there is one, wins.
+ out.usage, out.usage_source = usage_from_stats(chat.stats(), self.model), "chat record"
+ out.usage.tool_calls = len(chat.calls)
+ out.usage.latency_ms = int(proc.duration_s * 1000)
+ # without the record's message times there is no telling a storm from the work around
+ # it (every failure would run to the end): unknown, so 0.0
+ out.wait_s = provider_wait([(t, b) for t, b, _ in retries.rows], ended, [started, *chat.times]) if chat else 0.0
+ text = out.text.strip()
served = list((stats.get("models") or {}).keys())
- out: dict[str, Any] = {"usage": usage, "text": text, "ok": False, "exit_reason": "error",
- "transient": False, "quota": False, "errors": errors,
- "session_id": (parsed or {}).get("session_id", "")}
if proc.timed_out:
- out["exit_reason"] = "timeout"
- errors.append(watchdog_error(proc))
+ out.exit_reason = "timeout"
+ out.errors.append(watchdog_error(proc))
# the CLI retries 503s itself, with backoff, and never gives up before our wall:
# measured 2026-09-07, 8-17 "Attempt N failed with status 503" per 12-minute
# session, nothing produced. That is a storm, not the task.
- out["transient"] = not text.strip() and is_transient_failure(proc.stderr)
- if out["transient"]:
- errors.append(f"{proc.stderr.count('status 503')} x 503 inside the CLI's own retry loop before the wall; nothing produced")
+ out.transient = not text and bool(retries.rows)
+ if out.transient:
+ out.errors.append(f"{retries.tally()} inside the CLI's own retry loop before the wall; nothing produced")
return out
if served and self.model not in served:
- out["exit_reason"] = "model_substituted"
- errors.append(f"requested {self.model!r} but CLI served {served}")
+ out.exit_reason = "model_substituted"
+ out.errors.append(f"requested {self.model!r} but CLI served {served}")
return out
- if parsed is None or proc.rc != 0 or not text.strip():
- out["transient"] = is_transient_failure(proc.stderr, proc.stdout) or (parsed is not None and not text.strip())
- out["quota"] = is_quota_failure(proc.stderr, proc.stdout)
- errors.append(f"rc={proc.rc}; response={'<empty>' if not text.strip() else 'ok'}; stderr tail: {tail(proc.stderr, 1500)}")
- out["exit_reason"] = "budget" if out["quota"] else "error"
+ if parsed is None or proc.rc != 0 or not text:
+ # the CLI's own give-up object (stderr: {"session_id", "error": {"type", "message", "code"}}),
+ # else whatever it printed before it died
+ gave_up = find_json_object(proc.stderr, lambda d: isinstance(d.get("error"), dict))
+ err = (gave_up or {}).get("error") or {}
+ said = f"{err.get('code', '')} {err.get('type', '')}: {err.get('message', '')}" if err else proc.stderr + proc.stdout
+ out.session_id = out.session_id or str((gave_up or {}).get("session_id") or "")
+ out.quota = is_quota_failure(said)
+ out.transient = is_transient_failure(said) or (parsed is not None and not text)
+ out.rate_limited = err.get("code") == 429 or (not err and is_rate_limited(said))
+ out.errors.append(f"rc={proc.rc}; response={'<empty>' if not text else 'ok'}; stderr tail: {tail(proc.stderr, 1500)}")
+ out.exit_reason = "budget" if out.rate_limited or out.quota else "error"
return out
- out["ok"] = True
- out["exit_reason"] = "completed"
+ out.ok, out.exit_reason = True, "completed"
return out
+class _GeminiAttempt:
+ """One gemini-cli invocation, read: its envelope when it printed one, else its chat record."""
+
+ usage: Usage
+ text: str = ""
+ session_id: str = ""
+ ok: bool = False
+ exit_reason: str = "error"
+ errors: list[str] = dataclasses.field(default_factory=list)
+ transient: bool = False
+ quota: bool = False
+ rate_limited: bool = False # a 429: the key cools down in the pool, the retry rotates
+ wait_s: float = 0.0
+ usage_source: str = "none" # envelope | chat record | none
+
+
def _compose_prompt(job: AgentJob) -> str:
"""gemini-cli and codex have no system-prompt flag: prepend ``system_append``."""
if not job.system_append:
@@ -417,8 +541,12 @@ def parse_claude_json(stdout: str) -> dict[str, Any] | None:
class ClaudeStream:
- """Folded view over ``--output-format stream-json``: the tool calls the session made,
- and the skill index its ``init`` event reported (the list the model was shown).
+ """Folded view over ``--output-format stream-json``, fed live: the tool calls the
+ session made, the skill index its ``init`` event reported (the list the model was
+ shown), each message's usage block (``messages``: the last copy per message id — what a
+ session killed before its result event billed), the CLI's own ``api_retry`` events
+ (``retries``: ``(t, announced back-off s, HTTP status)``) and when every other event
+ arrived (``progress``) — the two ``cli_common.provider_wait`` reads.
Since 2026-09-22 the backend streams instead of printing one envelope: the envelope
(the last, ``type == "result"`` line) is unchanged, and the lines before it are the
@@ -429,6 +557,10 @@ class ClaudeStream:
self.calls: list[ToolCall] = []
self.skills: list[str] | None = None
self._at: dict[str, int] = {} # tool_use id → index in calls, for its tool_result
+ self.messages: dict[str, dict[str, Any]] = {}
+ self.served = ""
+ self.retries: list[tuple[float, float, int]] = []
+ self.progress: list[float] = []
def feed(self, line: str) -> None:
line = line.strip()
@@ -440,10 +572,19 @@ class ClaudeStream:
return # a >1 MB line arrives chunked (proc._LINE_CAP_BYTES): a Write, never a Skill call
if not isinstance(ev, dict):
return
+ if ev.get("type") == "system" and ev.get("subtype") == "api_retry":
+ # {"attempt", "max_retries", "retry_delay_ms", "error_status": 529, "error": "overloaded"}
+ self.retries.append((time.time(), float(ev.get("retry_delay_ms") or 0) / 1000.0, int(ev.get("error_status") or 0)))
+ return
+ self.progress.append(time.time())
if ev.get("type") == "system" and ev.get("subtype") == "init":
self.skills = [str(x) for x in ev.get("skills") or []]
elif ev.get("type") in ("assistant", "user"):
- for block in (ev.get("message") or {}).get("content") or []:
+ msg = ev.get("message") or {}
+ if ev.get("type") == "assistant" and msg.get("id") and isinstance(msg.get("usage"), dict):
+ self.messages[str(msg["id"])] = msg["usage"]
+ self.served = str(msg.get("model") or self.served)
+ for block in msg.get("content") or []:
if not isinstance(block, dict):
continue
if block.get("type") == "tool_use":
@@ -457,6 +598,19 @@ class ClaudeStream:
if i is not None:
self.calls[i] = dataclasses.replace(self.calls[i], failed=True)
+ def usage(self, model: str) -> Usage:
+ """What a session that never printed its result event billed: every message's own
+ usage block, summed. The input side is exact (a message knows its prompt when it
+ starts); ``output_tokens`` is a FLOOR — stream-json repeats the block each message
+ started with, before its output was counted (a finished 2026-09-22 session: 154 of
+ 30 106). Priced from the table: the ``total_cost_usd`` event never came."""
+ keys = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens")
+ total = {k: sum(int(u.get(k) or 0) for u in self.messages.values()) for k in keys}
+ usage = usage_from_envelope({"usage": total}, self.served or model)
+ usage.cost_usd = (estimate_cost("anthropic", usage.model, usage)
+ + cache_write_surcharge("anthropic", usage.model, total["cache_creation_input_tokens"]))
+ return usage
+
def usage_from_envelope(env: dict[str, Any], model: str) -> Usage:
"""``input_tokens`` is the TOTAL prompt (uncached + cache reads + cache writes), the
@@ -513,11 +667,12 @@ class ClaudeCodeAgent(_CliAgent):
self.model = model
self.binary = binary or get_settings().binaries.claude_cli
- def build_argv(self, s: Session, prompt: str) -> list[str]:
+ def build_argv(self, s: Session) -> list[str]:
+ """``-p`` with no prompt argument: the prompt comes on stdin, byte for byte."""
job = s.job
# stream-json (+ the --verbose it requires in -p mode): same final envelope, plus the
# tool calls the skill read probe reads as ground truth (ClaudeStream)
- argv = [self.binary, "-p", prompt, "--output-format", "stream-json", "--verbose",
+ argv = [self.binary, "-p", "--output-format", "stream-json", "--verbose",
"--dangerously-skip-permissions",
"--max-turns", str(job.max_turns), "--no-session-persistence",
"--allowedTools", ",".join(ALLOWED_TOOLS)]
@@ -549,47 +704,52 @@ class ClaudeCodeAgent(_CliAgent):
try:
if not ok:
return failed(s, "error", why)
- prompt = deliver_prompt(s, job.prompt)
stream = ClaudeStream()
- proc = invoke(s, self.build_argv(s, prompt), self.build_env(s), prompt=prompt, on_stdout=stream.feed)
+ proc = invoke(s, self.build_argv(s), self.build_env(s), prompt=job.prompt, on_stdout=stream.feed)
+ ended = time.time()
record_tool_calls(s, stream.calls, source="claude-code stream-json", skills_index=stream.skills)
env = parse_claude_json(proc.stdout)
- usage = usage_from_envelope(env, self.model) if env else Usage(backend=self.kind, model=self.model)
+ usage = usage_from_envelope(env, self.model) if env else stream.usage(self.model)
usage.latency_ms = usage.latency_ms or int(proc.duration_s * 1000)
text = str((env or {}).get("result") or "")
turns = int((env or {}).get("num_turns") or 0)
errors: list[str] = []
- transient = False
+ transient = quota = False
if proc.timed_out:
reason, ok = "timeout", False
errors.append(watchdog_error(proc))
- transient = not text.strip() and is_transient_failure(proc.stderr)
+ transient = not text.strip() and (bool(stream.retries) or is_transient_failure(proc.stderr))
elif env is None or proc.rc != 0:
reason, ok = "error", False
errors.append(f"rc={proc.rc}; no result envelope; stderr tail: {tail(proc.stderr, 1500)}")
# a 529 / overloaded exit is the provider's, not the task's (AgentResult.transient)
+ quota = is_quota_failure(proc.stderr, proc.stdout)
transient = is_transient_failure(proc.stderr, proc.stdout)
- if transient and is_quota_failure(proc.stderr):
+ if quota or (transient and is_rate_limited(proc.stderr)):
reason = "budget"
elif env.get("is_error") or str(env.get("subtype", "")).startswith("error"):
ok = False
sub = str(env.get("subtype", ""))
- reason = "budget" if "max_turns" in sub else "error"
+ quota = is_quota_failure(text)
+ transient = "max_turns" not in sub and is_transient_failure(text)
+ reason = "budget" if "max_turns" in sub or quota else "error"
errors.append(f"claude reported {sub or 'is_error'}: {tail(text, 800)}")
else:
reason, ok = "completed", True
return finish_session(
s, ok=ok, exit_reason=reason, text=text, usage=usage, tool_calls=max(turns - 1, 0), turns=turns,
- errors=errors, transient=transient, rc=proc.rc, killed_reason=proc.killed_reason, num_turns=turns,
+ errors=errors, transient=transient, quota=quota,
+ provider_wait_s=provider_wait([(t, b) for t, b, _ in stream.retries], ended, stream.progress),
+ rc=proc.rc, killed_reason=proc.killed_reason, num_turns=turns,
session_id=(env or {}).get("session_id", ""), subtype=(env or {}).get("subtype", ""),
model_usage=(env or {}).get("modelUsage", {}),
+ usage_from="envelope" if env else "stream messages" if stream.messages else "none",
)
finally:
release_session(s)
# ===================================================================== codex
-STDIN_PROMPT_BYTES = 100_000
_TOOL_ITEMS = ("command_execution", "file_change", "mcp_tool_call", "web_search", "tool_call")
#: the item fields a tool trace keeps: what was run, never what came back. codex has no
#: skill tool — it lists skills in its prompt and opens SKILL.md with a shell command, so
@@ -699,8 +859,8 @@ class CodexAgent(_CliAgent):
self.model, self.reasoning_effort = split_model_effort(model, reasoning_effort)
self.binary = binary or get_settings().binaries.codex_cli
- def build_argv(self, s: Session, prompt: str | None) -> list[str]:
- """``prompt=None`` means 'read it from stdin' (``-`` positional)."""
+ def build_argv(self, s: Session) -> list[str]:
+ """``-`` as the prompt: codex exec reads it from stdin."""
job = s.job
argv = [self.binary, "exec", "--json", "-C", str(s.ws.root), "--sandbox", "workspace-write",
"--skip-git-repo-check", "--ephemeral", "--color", "never"]
@@ -709,24 +869,25 @@ class CodexAgent(_CliAgent):
argv += effort_overrides(self.reasoning_effort)
if job.spatial_tools:
argv += codex_mcp_overrides(mcp_command_for(s.ws, job))
- argv.append(prompt if prompt is not None else "-")
+ argv.append("-")
return argv
def build_env(self, s: Session) -> dict[str, str]:
return hardened_env(s.ws, s.job, keep={"OPENAI_API_KEY", "CODEX_API_KEY"})
def run(self, job: AgentJob) -> AgentResult:
+ """``usage`` arrives per TURN (``turn.completed``): a session killed mid-turn books
+ nothing, and ``--ephemeral`` leaves no rollout to recover it from — 1 of 357 recorded
+ codex sessions. ``provider_wait_s`` stays 0.0: codex retries a 5xx inside its HTTP
+ client without an event (two fake 503s, 2026-09-22: nothing on the stream)."""
ok, why = self.available()
s = begin_session(job, self.kind)
try:
if not ok:
return failed(s, "error", why)
- prompt = _compose_prompt(job)
- via_stdin = len(prompt.encode("utf-8")) > STDIN_PROMPT_BYTES
- argv = self.build_argv(s, None if via_stdin else prompt)
events = CodexEvents()
- proc = invoke(s, argv, self.build_env(s), prompt=prompt, stdout_name="stdout.jsonl",
- on_stdout=events.feed, stdin=prompt if via_stdin else None)
+ proc = invoke(s, self.build_argv(s), self.build_env(s), prompt=_compose_prompt(job),
+ stdout_name="stdout.jsonl", on_stdout=events.feed)
record_tool_calls(s, events.calls, source="codex exec --json")
usage = events.usage(self.model)
usage.latency_ms = int(proc.duration_s * 1000)
@@ -742,9 +903,11 @@ class CodexAgent(_CliAgent):
ok, reason = False, "error"
else:
ok, reason = True, "completed"
+ said = (*events.errors, proc.stderr) if not ok else ()
return finish_session(
s, ok=ok, exit_reason=reason, text=text, usage=usage, tool_calls=events.tool_calls,
turns=events.turns_completed, errors=errors,
+ transient=is_transient_failure(*said), quota=is_quota_failure(*said),
rc=proc.rc, killed_reason=proc.killed_reason, thread_id=events.thread_id,
turns_completed=events.turns_completed, usage_raw=events.usage_raw,
)
@@ -811,6 +974,8 @@ def parse_agy_json(stdout: str) -> dict[str, Any] | None:
return find_json_object(stdout, lambda d: "response" in d or "status" in d)
+#: the per-session CLI log agy writes into the trajectory dir (``--log-file``)
+AGY_LOG = "agy.log"
#: where agy 1.2.2 keeps each conversation: one SQLite file named by the envelope's
#: ``conversation_id``, a ``steps`` table whose ``step_payload`` is protobuf
AGY_CONVERSATIONS = Path.home() / ".gemini" / "antigravity-cli" / "conversations"
@@ -894,6 +1059,46 @@ def agy_tool_calls(db: Path) -> list[ToolCall]:
return list(calls.values())
+#: agy 1.2.8's CLI log (``--log-file``; glog format, LOCAL time, no year) reports every failed
+#: model call it is about to retry: ``I0922 14:53:40.358454 284 run.go:395] Run: attempt 1 failed
+#: (Error 503, Message: ..., Status: UNAVAILABLE, Details: []), retrying in 4s``
+_AGY_RETRY = re.compile(r"^[IWEF](?P<md>\d{4}) (?P<hms>\d\d:\d\d:\d\d)(?P<frac>\.\d+)?\s.*\battempt \d+ failed "
+ r"\((?P<why>.*)\), retrying in (?P<wait>[0-9.hmsuµn]+)\s*$")
+_GO_DURATION = re.compile(r"([0-9.]+)(h|ms|m|s|us|µs|ns)")
+_GO_UNIT = {"h": 3600.0, "m": 60.0, "s": 1.0, "ms": 1e-3, "us": 1e-6, "µs": 1e-6, "ns": 1e-9}
+
+
+def agy_retries(log_path: Path) -> list[tuple[float, float, str]]:
+ """``(t, announced back-off s, why)`` for every retried model call in one agy CLI log."""
+ now = time.time()
+ year = time.localtime(now).tm_year
+ out: list[tuple[float, float, str]] = []
+ try:
+ lines = log_path.read_text(errors="replace").splitlines()
+ except OSError:
+ return out
+ for line in lines:
+ m = _AGY_RETRY.match(line)
+ if not m:
+ continue
+ t = time.mktime(time.strptime(f"{year}{m['md']} {m['hms']}", "%Y%m%d %H:%M:%S")) + float(m["frac"] or 0.0)
+ if t > now + 86_400: # a log written last December, read on New Year's Day
+ t = time.mktime(time.strptime(f"{year - 1}{m['md']} {m['hms']}", "%Y%m%d %H:%M:%S")) + float(m["frac"] or 0.0)
+ out.append((t, sum(float(n) * _GO_UNIT[u] for n, u in _GO_DURATION.findall(m["wait"])), m["why"]))
+ return out
+
+
+def agy_error(stderr: str) -> dict[str, Any]:
+ """agy's own verdict on a turn that died (1.2.x, exit code 3): the ``AGY_ERROR: {...}`` line —
+ ``status``, ``error_code``, ``retryable`` — typed, where the envelope has only prose."""
+ for line in reversed(stderr.splitlines()):
+ if line.startswith("AGY_ERROR:"):
+ with contextlib.suppress(json.JSONDecodeError):
+ got = json.loads(line[len("AGY_ERROR:"):])
+ return got if isinstance(got, dict) else {}
+ return {}
+
+
def usage_from_agy(env: dict[str, Any], model: str) -> Usage:
u = env.get("usage") or {}
return Usage(
@@ -919,11 +1124,14 @@ class AntigravityAgent(_CliAgent):
"""The id actually sent to agy (a bare one gains its effort suffix)."""
return resolve_model(self.model, self.binary)
- def build_argv(self, s: Session, prompt: str) -> list[str]:
+ def build_argv(self, s: Session) -> list[str]:
+ """No ``--print``: agy 1.2.8 reads the prompt from a piped stdin, byte for byte, and
+ still answers as print mode (``--print -`` would send the prompt "-"). ``--log-file``
+ keeps its CLI log with the session — the record of the retries it made."""
minutes = max(1, int(s.job.timeout_s // 60) + 1)
- argv = [self.binary, "--print", prompt, "--dangerously-skip-permissions",
+ argv = [self.binary, "--dangerously-skip-permissions",
"--print-timeout", f"{minutes}m", "--output-format", "json",
- "--add-dir", str(s.ws.root), "--disable-slash-commands"]
+ "--add-dir", str(s.ws.root), "--disable-slash-commands", "--log-file", str(s.traj.dir / AGY_LOG)]
if self.model:
argv += ["--model", self.served_model()]
return argv
@@ -945,15 +1153,18 @@ class AntigravityAgent(_CliAgent):
return 0
def run(self, job: AgentJob) -> AgentResult:
+ """``usage`` comes from agy's envelope only: a killed session books nothing (its
+ conversation database is named by the envelope's id). ``provider_wait_s`` from its log."""
ok, why = self.available()
s = begin_session(job, self.kind)
try:
if not ok:
return failed(s, "error", why)
- prompt = deliver_prompt(s, _compose_prompt_agy(s))
- proc = invoke(s, self.build_argv(s, prompt), self.build_env(s), prompt=prompt)
+ proc = invoke(s, self.build_argv(s), self.build_env(s), prompt=_compose_prompt_agy(s))
+ ended = time.time()
env = parse_agy_json(proc.stdout)
n_calls = self._record_trace(s, str((env or {}).get("conversation_id") or ""))
+ retries = agy_retries(s.traj.dir / AGY_LOG)
served = self.served_model()
usage = usage_from_agy(env, served) if env else Usage(backend=self.kind, model=served)
usage.latency_ms = usage.latency_ms or int(proc.duration_s * 1000)
@@ -962,6 +1173,8 @@ class AntigravityAgent(_CliAgent):
if env is None:
s.notes.append("no JSON envelope from agy; usage unknown (zeros), response taken from plain stdout")
s.notes.append("agy is subscription-billed: cost_usd=0")
+ died = agy_error(proc.stderr)
+ said = (json.dumps(died), str((env or {}).get("error") or ""), proc.stderr)
if proc.timed_out:
ok, reason = False, "timeout"
errors.append(watchdog_error(proc))
@@ -974,11 +1187,15 @@ class AntigravityAgent(_CliAgent):
errors.append("empty response")
else:
ok, reason = True, "completed"
+ quota = not ok and is_quota_failure(*said)
+ transient = not ok and (bool(died.get("retryable")) or is_transient_failure(*said)
+ or (proc.timed_out and not text.strip() and bool(retries)))
usage.tool_calls = n_calls
return finish_session(
s, ok=ok, exit_reason=reason, text=text, usage=usage, tool_calls=n_calls,
- turns=int((env or {}).get("num_turns") or 0), errors=errors,
- rc=proc.rc, killed_reason=proc.killed_reason,
+ turns=int((env or {}).get("num_turns") or 0), errors=errors, transient=transient, quota=quota,
+ provider_wait_s=provider_wait([(t, b) for t, b, _ in retries], ended),
+ rc=proc.rc, killed_reason=proc.killed_reason, agy_error=died,
conversation_id=(env or {}).get("conversation_id", ""), num_turns=(env or {}).get("num_turns", 0),
)
finally:
diff --git a/harness/codeverse3d/agents/cli_common.py b/harness/codeverse3d/agents/cli_common.py
index 0169c797..1898b295 100644
--- a/harness/codeverse3d/agents/cli_common.py
+++ b/harness/codeverse3d/agents/cli_common.py
@@ -178,8 +178,8 @@ def run_with_watchdog(
Fixed by that move: a KeyboardInterrupt anywhere in the poll loop kills the
group before propagating (it used to orphan the whole node → chrome tree —
SIGINT never reaches a ``start_new_session`` child); stdin is written from a
- helper thread, so a child that never reads a >64 KiB prompt (codex pipes
- prompts >100 kB via stdin) can no longer wedge the main thread before the
+ helper thread, so a child that never reads a >64 KiB prompt (every agent
+ CLI gets its prompt on stdin) can no longer wedge the main thread before the
clocks start; captured output is bounded per stream (head+tail + marker).
"""
if hard_timeout_s is None:
@@ -247,8 +247,6 @@ def _reject_control_chars(cmd: Sequence[str], cwd: object) -> None:
# ===================================================================== cli_common
log = logging.getLogger(__name__)
-#: argv prompts above this many bytes are written to a file instead.
-MAX_ARGV_PROMPT_BYTES = 100_000
def is_secret_env(name: str) -> bool:
"""True for env vars that look like credentials (stripped from agent children).
@@ -494,11 +492,15 @@ def finish_session(
turns: int = 0,
errors: list[str] | None = None,
transient: bool = False,
+ quota: bool = False,
+ provider_wait_s: float = 0.0,
**extra: Any,
) -> AgentResult:
"""Commit the agent's work, compute ``files_changed`` via git (attributed to this
session — see :func:`attribute_changes`), write result.json. ``turns`` is the
- backend's own count (``AgentResult.turns``); ``extra`` lands in result.json only.
+ backend's own count (``AgentResult.turns``); ``transient`` / ``quota`` its typed
+ failure class and ``provider_wait_s`` the time its CLI lost to provider errors
+ (:func:`provider_wait`); ``extra`` lands in result.json only.
Write scope is enforced here for the CLI backends (they have no write-time gate):
writes outside ``write_roots`` (and, for ``edit_only``, outside ``files_hint``) are
@@ -517,11 +519,12 @@ def finish_session(
s.notes.append(msg)
s.ws.commit(f"agent:{s.label}")
files = attribute_changes(s.ws.changed_files(s.head_before), write_roots=s.job.write_roots)
+ duration = round(time.monotonic() - s.t0, 3)
res = AgentResult(
ok=ok, exit_reason=exit_reason, text=text, files_changed=files,
transcript_path=str(s.traj.transcript_path if s.traj.transcript_path.exists() else s.traj.dir),
- usage=usage, duration_s=round(time.monotonic() - s.t0, 3), tool_calls=tool_calls,
- turns=turns, errors=errors, transient=transient,
+ usage=usage, duration_s=duration, provider_wait_s=round(min(max(provider_wait_s, 0.0), duration), 3),
+ tool_calls=tool_calls, turns=turns, errors=errors, transient=transient and not quota, quota=quota,
)
s.traj.write_result(res, kind=s.kind, label=s.label, round=s.round_index, attempt=s.attempt, job_label=s.job.label,
head_before=s.head_before, head_after=s.ws.head(), notes=s.notes, **extra)
@@ -535,21 +538,6 @@ def failed(s: Session, reason: str, message: str, usage: Usage | None = None) ->
return finish_session(s, ok=False, exit_reason=reason, text="", usage=usage or Usage(), errors=[message])
-# --------------------------------------------------------------------------- prompts
-def deliver_prompt(s: Session, prompt: str, *, max_bytes: int = MAX_ARGV_PROMPT_BYTES) -> str:
- """Return the text to pass on argv: the prompt itself, or a stub pointing at a file."""
- if len(prompt.encode("utf-8")) <= max_bytes:
- return prompt
- p = s.traj.write_text("task_prompt.md", prompt)
- s.notes.append(f"prompt ({len(prompt)} chars) delivered via file {p}")
- rel = os.path.relpath(p, s.ws.root)
- return (
- f"Your full task is in the file `{rel}` (relative to the current working directory). "
- "Read that file completely with your file-reading tool FIRST, then carry out every "
- "instruction in it. Do not ask for confirmation."
- )
-
-
# --------------------------------------------------------------------------- invoke
#: seconds without output or workspace activity before the watchdog kills a CLI agent;
#: read at call time by :func:`invoke` so a test can monkeypatch it here.
@@ -570,34 +558,40 @@ def invoke(
prompt: str,
stdout_name: str = "stdout.json",
on_stdout: Callable[[str], None] | None = None,
- stdin: str | None = None,
+ on_stderr: Callable[[str], None] | None = None,
attempt: int = 1,
soft_timeout_s: float | None = None,
**invoke_extra: Any,
) -> CompletedProc:
"""Run one CLI agent process under the watchdog and record it in the trajectory.
- Replaces the block ``claude_code.ClaudeCodeAgent.run``, ``codex.CodexAgent.run``,
- ``antigravity.AntigravityAgent.run`` and ``gemini_cli.GeminiCliAgent._invoke`` each
- carried verbatim: the ``invoke`` transcript line with the prompt masked out of argv,
- ``run_with_watchdog`` with identical kwargs (workspace cwd, the job's soft timeout,
- ``IDLE_GRACE_S``, every output line into the transcript, ``src/`` + ``public/`` as
- activity dirs) and the stdout/stderr captures next to the transcript.
-
- ``stdout_name`` sets the capture's name (codex streams JSONL → ``stdout.jsonl``);
- ``on_stdout`` also receives each stdout line (codex folds events live); ``stdin``
- feeds the prompt through stdin instead of argv; ``attempt`` > 1 writes
- ``stdout.<n>.json`` / ``stderr.<n>.log`` so a retry keeps attempt 1's captures.
- The ``invoke`` line records ``attempt`` and whether stdin was used, plus
- ``invoke_extra`` (gemini-cli adds ``model`` and ``key_tail``).
+ The prompt goes to the CLI on STDIN, always, for every backend — never on argv and never
+ through a file the agent must read. argv caps one argument at 128 KiB (``MAX_ARG_STRLEN``),
+ and the file stub this replaced (``task_prompt.md`` above 100 kB) handed the prompt to the
+ agent's own file tool: gemini-cli's ``read_file`` stops at 2 000 lines, so the last zones of a
+ 5-zone scene prompt were silently dropped. Each CLI reads its whole stdin as the prompt,
+ byte for byte (checked 2026-09-22 against gemini-cli 0.53, claude-code 2.1.280, codex
+ 0.155.1 and agy 1.2.8 with a 185 000-character, 2 502-line prompt; agy alone truncates a
+ request past ~175 kB, whatever carries it).
+
+ One block every backend shares: the ``invoke`` transcript line, ``run_with_watchdog``
+ (workspace cwd, the job's soft timeout, ``IDLE_GRACE_S``, every output line into the
+ transcript, ``src/`` + ``public/`` as activity dirs) and the stdout/stderr captures next to
+ the transcript. ``stdout_name`` sets the capture's name (codex streams JSONL →
+ ``stdout.jsonl``); ``on_stdout`` / ``on_stderr`` also receive each line as it arrives (the
+ backends fold their event streams and retry reports live); ``attempt`` > 1 writes
+ ``stdout.<n>.json`` / ``stderr.<n>.log`` so a retry keeps attempt 1's captures. The
+ ``invoke`` line records ``attempt`` and the prompt's size, plus ``invoke_extra`` (gemini-cli
+ adds ``model`` and ``key_tail``).
"""
- masked = [a if a != prompt else f"<prompt {len(prompt)} chars>" for a in argv]
- s.traj.append("invoke", argv=masked, attempt=attempt, stdin=stdin is not None, **invoke_extra)
+ s.traj.append("invoke", argv=list(argv), attempt=attempt, stdin_bytes=len(prompt.encode("utf-8")),
+ **invoke_extra)
def on_line(stream: str, line: str) -> None:
s.traj.append("line", stream=stream, text=line[:4000])
- if on_stdout is not None and stream == "stdout":
- on_stdout(line)
+ observer = on_stdout if stream == "stdout" else on_stderr
+ if observer is not None:
+ observer(line)
# A streaming session must not outlive its window by half of it again: the watchdog's
# default hard kill is max(1.5x soft, soft+600), and chair_bl (loop_w1, 2026-08-28)
@@ -607,7 +601,7 @@ def invoke(
proc = run_with_watchdog(
argv, cwd=s.ws.root, env=env, soft_timeout_s=soft, hard_timeout_s=soft + 300.0,
idle_grace_s=IDLE_GRACE_S, poll_s=POLL_S,
- on_line=on_line, stdin=stdin, activity_dirs=[s.ws.src, s.ws.public],
+ on_line=on_line, stdin=prompt, activity_dirs=[s.ws.src, s.ws.public],
)
suffix = "" if attempt == 1 else f".{attempt}"
out = Path(stdout_name)
@@ -679,23 +673,81 @@ def record_tool_calls(s: Session, calls: Iterable[ToolCall], *, source: str,
# --------------------------------------------------------------------------- failures
+#: THE vocabulary for what a CLI says about the call that ended its session — one place for
+#: every backend. The backends turn it into typed flags (``AgentResult.transient`` / ``.quota``)
+#: that ride ``GenerationResult`` into ``RoundFailed``; the round loop never reads a message.
+#: Transient: a provider failure a retry may get through — capacity (5xx, UNAVAILABLE,
+#: overloaded), rate limits (429, RESOURCE_EXHAUSTED) and the transport (a dropped, refused or
+#: reset connection, an empty or broken stream). The gRPC status names match case-sensitively:
+#: lower-case "internal" is in every node stack trace (``node:internal/process/...``), which made
+#: any crashed CLI look transient. A plain "timed out" is NOT here: the watchdog's own kill, a
+#: slow MCP tool and a provider timeout all say it, and a retry of the first two buys another
+#: full window for the same outcome (provider timeouts arrive typed: gemini-cli's retry lines,
+#: ``ModelError(status=408, retryable=True)``).
_TRANSIENT_RE = re.compile(
- r"(\b429\b|RESOURCE_EXHAUSTED|rate.?limit|quota|\b503\b|UNAVAILABLE|overloaded|"
- r"\b502\b|\b500\b|INTERNAL|empty response|Invalid stream|ECONNRESET|ETIMEDOUT|socket hang up)",
+ r"(?-i:\b(?:UNAVAILABLE|INTERNAL|RESOURCE_EXHAUSTED|DEADLINE_EXCEEDED)\b)|\b(?:429|50[0234]|529)\b|overloaded"
+ r"|rate.?limit|empty response|Invalid stream|fetch failed|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN"
+ r"|socket hang up|connection reset|stream disconnected",
re.IGNORECASE,
)
-_QUOTA_RE = re.compile(r"(\b429\b|RESOURCE_EXHAUSTED|quota|rate.?limit)", re.IGNORECASE)
-
-
-def is_transient_failure(*texts: str) -> bool:
- """Heuristic: did the CLI die of a rate-limit / 5xx / empty-stream glitch?"""
- return any(_TRANSIENT_RE.search(t or "") for t in texts)
+#: Quota: the vendor's usage limit or credits are spent and nothing gets through until they reset
+#: (codex "You've hit your usage limit", claude "usage limit reached" / "You've hit your limit" /
+#: "out of usage credits", OpenAI ``insufficient_quota``). Not a rate limit: gemini says "Quota
+#: exceeded ... per minute" and "You exceeded your current quota" of a per-key 429 the key pool
+#: rotates past, so neither phrase is here.
+_QUOTA_RE = re.compile(
+ r"usage limit|hit your limit|out of (?:extra )?usage|usage credits|credit balance (?:is )?too low"
+ r"|purchase more credits|insufficient[_ ]quota|insufficient credits|billing hard limit",
+ re.IGNORECASE,
+)
+_RATE_LIMIT_RE = re.compile(r"\b429\b|(?-i:\bRESOURCE_EXHAUSTED\b)|rate.?limit", re.IGNORECASE)
def is_quota_failure(*texts: str) -> bool:
+ """The vendor's usage limit is spent (``AgentResult.quota``): stop, do not retry."""
return any(_QUOTA_RE.search(t or "") for t in texts)
+def is_transient_failure(*texts: str) -> bool:
+ """The provider, not the task, ended the session and a retry may get through
+ (``AgentResult.transient``); a spent usage limit never is."""
+ return any(_TRANSIENT_RE.search(t or "") for t in texts) and not is_quota_failure(*texts)
+
+
+def is_rate_limited(*texts: str) -> bool:
+ """A 429: transient, and the key that got it should cool down (``exit_reason="budget"``)."""
+ return any(_RATE_LIMIT_RE.search(t or "") for t in texts)
+
+
+# --------------------------------------------------------------------------- provider wait
+def provider_wait(failures: Iterable[tuple[float, float]], end: float,
+ progress: Iterable[float] | None = None) -> float:
+ """Seconds a session lost to provider errors, from the CLI's own record of them.
+
+ ``failures`` are ``(t, backoff_s)``: when the CLI reported a failed API attempt, and the
+ back-off it announced before the next one (0 when it does not say — gemini-cli's 5xx
+ lines). ``progress`` are the times the session demonstrably moved on (a model message, a
+ tool result); ``None`` when the CLI records none (agy's log). Failures with no progress
+ between them form one burst, which costs from the last progress before it — when the
+ request that failed went out; its first failure when nothing precedes it — until the
+ CLI tried again after the burst's last failure (``t + backoff``). A burst no progress
+ ever followed costs until ``end``: the CLI gave up, or the watchdog killed it mid-storm.
+ A lower bound wherever the back-off is unannounced (≤ 30 s per burst for gemini-cli)."""
+ fails = sorted(failures)
+ prog = sorted(progress) if progress is not None else []
+ total, i = 0.0, 0
+ while i < len(fails):
+ j = i
+ while j + 1 < len(fails) and not any(fails[j][0] < p < fails[j + 1][0] for p in prog):
+ j += 1
+ first, (last, backoff) = fails[i][0], fails[j]
+ start = max((p for p in prog if p <= first), default=first)
+ resumed = progress is None or any(p > last for p in prog)
+ total += max(0.0, (min(last + backoff, end) if resumed else end) - start)
+ i = j + 1
+ return round(total, 3)
+
+
def tail(text: str, n: int = 2000) -> str:
return text if len(text) <= n else text[-n:]
diff --git a/harness/codeverse3d/agents/materialize.py b/harness/codeverse3d/agents/materialize.py
index b8b06aaa..9d0d7a21 100644
--- a/harness/codeverse3d/agents/materialize.py
+++ b/harness/codeverse3d/agents/materialize.py
@@ -27,8 +27,8 @@ roots in an untrusted folder), and each backend records what its CLI activated
Ignore files: ``.geminiignore`` / ``.aiexclude`` hide only noise (:data:`IGNORE_LINES`);
``.gemini/settings.json`` gets ``context.fileFiltering.respectGitIgnore=false`` because the
-workspace ``.gitignore`` hides ``artifacts/`` + ``trajectories/`` from git and gemini-cli
-would otherwise refuse to read the build census / the long-prompt file there.
+workspace ``.gitignore`` hides ``artifacts/`` from git and gemini-cli would otherwise refuse
+to read the build census there.
"""
from __future__ import annotations
@@ -49,11 +49,11 @@ MCP_SERVER_NAME = "3dcode"
MCP_TOOL_TIMEOUT_MS = 600_000
#: ``.geminiignore`` / ``.aiexclude`` — gemini-cli's read_file/glob REFUSE ignored paths, so the
#: agent-facing artefacts must stay readable: ``artifacts/*.json`` (the build tool advertises
-#: ``census.json`` etc.), ``artifacts/tool_renders/`` and ``trajectories/<label>_rNN/task_prompt.md``
-#: (the long-prompt fallback). Only noise / harness-private output is hidden.
+#: ``census.json`` etc.) and ``artifacts/tool_renders/``. Only noise / harness-private output is
+#: hidden — ``trajectories/`` whole since the prompt reaches every CLI on stdin (it was the
+#: long-prompt file's home until 2026-09-22).
IGNORE_LINES = (
- "artifacts/renders/", "artifacts/judge/",
- "trajectories/*/stdout*", "trajectories/*/stderr*", "trajectories/*/transcript.jsonl",
+ "artifacts/renders/", "artifacts/judge/", "trajectories/",
"stages/", "rounds/", "_cand/", "_assets/",
".git/", "node_modules/", "__pycache__/", ".gemini/tmp/",
)
diff --git a/harness/codeverse3d/proc.py b/harness/codeverse3d/proc.py
index 004312c3..09ef6bad 100644
--- a/harness/codeverse3d/proc.py
+++ b/harness/codeverse3d/proc.py
@@ -177,8 +177,8 @@ class ManagedProcess:
its trailing newline stripped; a line longer than :data:`_LINE_CAP_BYTES`
arrives as several chunk-sized calls; observer exceptions are suppressed);
* a stdin WRITER THREAD — a child that never reads a >64 KiB pipe-buffer of
- stdin can no longer wedge the caller's main thread in ``stdin.write`` (codex
- really does deliver prompts via stdin once they exceed 100 kB);
+ stdin can no longer wedge the caller's main thread in ``stdin.write`` (every
+ agent CLI gets its prompt on stdin, up to hundreds of kB);
* ``__exit__``: on ANY exception — KeyboardInterrupt above all, because
``start_new_session`` children never receive the terminal's Ctrl-C SIGINT —
or with the child still running at scope exit, the whole group gets
diff --git a/harness/tests/agents/test_antigravity.py b/harness/tests/agents/test_antigravity.py
index 3b3810ef..d8f63913 100644
--- a/harness/tests/agents/test_antigravity.py
+++ b/harness/tests/agents/test_antigravity.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import shutil
+from pathlib import Path
import pytest
@@ -23,12 +24,30 @@ ENVELOPE = {"conversation_id": "c1", "status": "SUCCESS", "response": "DONE\n",
FAKE_AGY = r'''
args = sys.argv[1:]
-prompt = args[args.index("--print") + 1]
+assert "--print" not in args and "-p" not in args, "the prompt comes on stdin (--print - would send '-')"
+prompt = sys.stdin.read()
assert "--dangerously-skip-permissions" in args and "--add-dir" in args
assert "FAKE_SERVICE_API_KEY" not in os.environ
+log = args[args.index("--log-file") + 1]
os.makedirs("src", exist_ok=True)
open("src/hello.txt", "w").write("hi")
+open("src/prompt.txt", "w").write(prompt)
mode = os.environ.get("FAKE_MODE", "ok")
+if mode == "retried": # agy 1.2.8 retried two 503s and a 429 inside the turn, then answered
+ from datetime import datetime, timedelta
+ t0 = datetime.now() - timedelta(seconds=21)
+ with open(log, "w") as f:
+ f.write(t0.strftime("I%m%d %H:%M:%S.%f") + " 66 server.go:609] Language server will attempt to listen\n")
+ for i, (dt, why, wait) in enumerate([(0, "Error 503, Message: busy, Status: UNAVAILABLE", "4s"),
+ (4, "Error 503, Message: busy, Status: UNAVAILABLE", "6.069232705s"),
+ (10, "Error 429, Message: slow down, Status: RESOURCE_EXHAUSTED", "10.690413026s")]):
+ f.write((t0 + timedelta(seconds=dt)).strftime("I%m%d %H:%M:%S.%f")
+ + f" 284 run.go:395] Run: attempt {i + 1} failed ({why}, Details: []), retrying in {wait}\n")
+if mode == "died": # a turn that ended on the API: AGY_ERROR on stderr, exit code 3
+ print(json.dumps({"conversation_id": "c9", "status": "ERROR", "response": "", "error": "generating: Error 503"}))
+ print('AGY_ERROR: {"short_error":"generating: Error 503","status":"UNAVAILABLE","error_code":503,'
+ '"code_kind":"http","retryable":true,"error_id":"x"}', file=sys.stderr)
+ sys.exit(3)
if mode == "text":
print("plain text answer")
else:
@@ -72,8 +91,9 @@ def test_parse_and_usage():
def test_argv(tmp_ws: Workspace):
a = AntigravityAgent("gemini-3.6-flash-high", binary="agy")
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", timeout_s=1800), "agy")
- argv = a.build_argv(s, "p")
- assert argv[:3] == ["agy", "--print", "p"] and argv[argv.index("--print-timeout") + 1] == "31m"
+ argv = a.build_argv(s)
+ assert "--print" not in argv and "p" not in argv and argv[argv.index("--print-timeout") + 1] == "31m"
+ assert argv[argv.index("--log-file") + 1] == str(s.traj.dir / "agy.log") # its retries, kept with the session
assert argv[argv.index("--model") + 1] == "gemini-3.6-flash-high" and argv[argv.index("--add-dir") + 1] == str(tmp_ws.root)
assert argv[argv.index("--output-format") + 1] == "json"
@@ -82,12 +102,42 @@ def test_fake_run_json_and_text(tmp_ws: Workspace, fake_bin, monkeypatch):
a = AntigravityAgent("gemini-3.6-flash-high", binary=fake_bin("agy", FAKE_AGY))
res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="a", timeout_s=30))
assert res.ok and res.text.strip() == "DONE" and res.usage.input_tokens == 18589, res.errors
- assert [f.path for f in res.files_changed] == ["src/hello.txt"]
+ assert [f.path for f in res.files_changed] == ["src/hello.txt", "src/prompt.txt"]
+ assert (tmp_ws.src / "prompt.txt").read_text().endswith("\n\nhello") # on stdin, after the workspace header
monkeypatch.setenv("FAKE_MODE", "text")
res2 = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="b", timeout_s=30))
assert res2.ok and res2.text == "plain text answer" and res2.usage.input_tokens == 0
+def test_the_retries_in_agys_own_log_are_its_provider_wait(tmp_ws: Workspace, fake_bin, monkeypatch):
+ """agy 1.2.8 logs every model call it retries, with the back-off it announces (fake API,
+ 2026-09-22: 503, 503, 429, then an answer 20.8 s later) — the burst runs from the first
+ failure to the last one plus its back-off."""
+ from codeverse3d.agents.backends import agy_retries
+ from codeverse3d.agents.cli_common import provider_wait
+
+ a = AntigravityAgent("gemini-3.6-flash-high", binary=fake_bin("agy", FAKE_AGY))
+ monkeypatch.setenv("FAKE_MODE", "retried")
+ res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="r", timeout_s=30))
+ assert res.ok, res.errors
+ rows = agy_retries(Path(res.transcript_path).parent / "agy.log")
+ assert [(round(b, 3), why.split(",")[0]) for _, b, why in rows] == [
+ (4.0, "Error 503"), (6.069, "Error 503"), (10.69, "Error 429")]
+ assert [round(t - rows[0][0], 3) for t, _, _ in rows] == [0.0, 4.0, 10.0]
+ assert provider_wait([(t, b) for t, b, _ in rows], end=rows[0][0] + 60) == pytest.approx(20.690413, abs=1e-3)
+ # the fake wrote a log 21 s in the past: the session's own wait is capped at its length
+ assert 0.0 < res.provider_wait_s <= res.duration_s
+
+
+def test_agy_says_itself_whether_a_dead_turn_is_worth_a_retry(tmp_ws: Workspace, fake_bin, monkeypatch):
+ a = AntigravityAgent("gemini-3.6-flash-high", binary=fake_bin("agy", FAKE_AGY))
+ monkeypatch.setenv("FAKE_MODE", "died")
+ res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="d", timeout_s=30))
+ assert not res.ok and res.exit_reason == "error" and res.transient and not res.quota
+ rec = json.loads((Path(res.transcript_path).parent / "result.json").read_text())
+ assert rec["agy_error"]["retryable"] is True and rec["agy_error"]["error_code"] == 503
+
+
@pytest.mark.live
def test_live_agy_tiny(tmp_ws: Workspace):
if not shutil.which("agy"):
@@ -109,7 +159,7 @@ def test_a_bare_model_id_gains_the_effort_agy_requires(tmp_ws: Workspace, fake_b
a = AntigravityAgent("gemini-3.7-flash", binary=fake_bin("agy", FAKE_AGY_MODELS))
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", timeout_s=60), "agy")
- argv = a.build_argv(s, "p")
+ argv = a.build_argv(s)
assert argv[argv.index("--model") + 1] == "gemini-3.7-flash-medium"
assert "--effort" not in argv
diff --git a/harness/tests/agents/test_claude_code.py b/harness/tests/agents/test_claude_code.py
index 5583861a..9d4b6aa5 100644
--- a/harness/tests/agents/test_claude_code.py
+++ b/harness/tests/agents/test_claude_code.py
@@ -42,11 +42,16 @@ TWO_MODEL_ENVELOPE = {
FAKE_CLAUDE = r'''
args = sys.argv[1:]
-prompt = args[args.index("-p") + 1]
+assert args[args.index("-p") + 1].startswith("--"), "-p takes no prompt argument: the prompt comes on stdin"
+prompt = sys.stdin.read()
assert "--dangerously-skip-permissions" in args and "--output-format" in args
assert "--strict-mcp-config" in args
assert "FAKE_SERVICE_API_KEY" not in os.environ
mode = os.environ.get("FAKE_MODE", "ok")
+if mode == "usage_limit": # the subscription is spent: a result event, is_error, and claude's own words
+ print(json.dumps({"type": "result", "subtype": "success", "is_error": True, "num_turns": 1,
+ "result": "Claude AI usage limit reached|1790000000", "session_id": "q"}))
+ sys.exit(1)
if mode == "overloaded": # what `claude -p` exits with when the API is out of capacity
print('API Error: 529 {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', file=sys.stderr)
sys.exit(1)
@@ -55,6 +60,7 @@ if mode == "rate_limited":
sys.exit(1)
os.makedirs("src", exist_ok=True)
open("src/hello.txt", "w").write("hi")
+open("src/prompt.txt", "w").write(prompt)
env = json.loads(''' + repr(json.dumps(ENVELOPE)) + r''')
if mode == "error":
env["is_error"] = True; env["subtype"] = "error_max_turns"
@@ -84,8 +90,8 @@ def test_argv_includes_mcp_when_materialized(tmp_ws: Workspace):
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", system_append="S", max_turns=7,
mcp_command=["python", "-m", "codeverse3d.spatial.mcp_server", "--workspace", str(tmp_ws.root)]),
"claude-code")
- argv = a.build_argv(s, "p")
- assert argv[:3] == ["claude", "-p", "p"]
+ argv = a.build_argv(s)
+ assert argv[:3] == ["claude", "-p", "--output-format"] and "p" not in argv # the prompt goes on stdin
# the config is written fresh into THIS session's trajectory dir from the typed job,
# never the workspace .mcp.json the agent can rewrite between rounds
cfg = Path(argv[argv.index("--mcp-config") + 1])
@@ -94,14 +100,21 @@ def test_argv_includes_mcp_when_materialized(tmp_ws: Workspace):
assert server["command"] == "python" and "--workspace" in server["args"] # the TYPED job command, verbatim
(tmp_ws.root / ".mcp.json").write_text(json.dumps(
{"mcpServers": {"3dcode": {"command": "/tmp/evil", "args": []}}}))
- argv2 = a.build_argv(s, "p") # a tampered workspace file changes nothing
+ argv2 = a.build_argv(s) # a tampered workspace file changes nothing
assert json.loads(Path(argv2[argv2.index("--mcp-config") + 1]).read_text(
))["mcpServers"]["3dcode"]["command"].endswith("python")
assert "--strict-mcp-config" in argv and argv[argv.index("--max-turns") + 1] == "7"
assert argv[argv.index("--append-system-prompt") + 1] == "S" and argv[argv.index("--model") + 1] == "sonnet"
assert "mcp__3dcode__*" in argv[argv.index("--allowedTools") + 1]
s2 = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "claude-code")
- assert "--mcp-config" not in a.build_argv(s2, "p")
+ assert "--mcp-config" not in a.build_argv(s2)
+
+
+def test_the_usage_limit_is_quota_not_a_transient_death(tmp_ws: Workspace, fake_bin, monkeypatch):
+ a = ClaudeCodeAgent("sonnet", binary=fake_bin("claude", FAKE_CLAUDE))
+ monkeypatch.setenv("FAKE_MODE", "usage_limit")
+ res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="u", timeout_s=30))
+ assert not res.ok and res.quota and not res.transient and res.exit_reason == "budget"
def test_fake_run_success_and_error(tmp_ws: Workspace, fake_bin, monkeypatch):
@@ -110,7 +123,8 @@ def test_fake_run_success_and_error(tmp_ws: Workspace, fake_bin, monkeypatch):
assert res.ok and res.exit_reason == "completed", res.errors
assert res.text.endswith("sys=SYS") and res.usage.cost_usd == 0.0123 and res.tool_calls == 2
assert res.turns == 3 # the envelope's num_turns, in the typed result
- assert [f.path for f in res.files_changed] == ["src/hello.txt"]
+ assert [f.path for f in res.files_changed] == ["src/hello.txt", "src/prompt.txt"]
+ assert (tmp_ws.src / "prompt.txt").read_text() == "hello" # on stdin, byte for byte
rec = json.loads((Path(res.transcript_path).parent / "result.json").read_text())
assert rec["num_turns"] == 3 and rec["session_id"] == "abc"
monkeypatch.setenv("FAKE_MODE", "error")
@@ -125,10 +139,10 @@ def test_an_overloaded_exit_is_transient(tmp_ws: Workspace, fake_bin, monkeypatc
a = ClaudeCodeAgent("sonnet", binary=fake_bin("claude", FAKE_CLAUDE))
monkeypatch.setenv("FAKE_MODE", "overloaded")
res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="o", timeout_s=30))
- assert not res.ok and res.exit_reason == "error" and res.transient is True
+ assert not res.ok and res.exit_reason == "error" and res.transient is True and not res.quota
monkeypatch.setenv("FAKE_MODE", "rate_limited")
res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="q", timeout_s=30))
- assert not res.ok and res.exit_reason == "budget" and res.transient is True
+ assert not res.ok and res.exit_reason == "budget" and res.transient is True and not res.quota
@pytest.mark.live
@@ -172,3 +186,47 @@ def test_the_dearest_entry_wins_when_the_token_counts_do_not_match():
def test_no_model_usage_block_keeps_what_we_asked_for():
assert primary_served_model({"usage": {"input_tokens": 1}}, "sonnet") == "sonnet"
assert usage_from_envelope({"usage": {"input_tokens": 1}}, "sonnet").model == "sonnet"
+
+
+# --------------------------------------------------------------------------- killed sessions
+FAKE_KILLED_CLAUDE = r'''
+for ev in json.loads(os.environ["FAKE_EVENTS"]):
+ print(json.dumps(ev), flush=True)
+ time.sleep(0.05)
+time.sleep(60) # an overloaded provider: claude keeps retrying until the watchdog kills it
+'''
+_MSG_USAGE = {"input_tokens": 2, "cache_creation_input_tokens": 19190, "cache_read_input_tokens": 20354, "output_tokens": 5}
+KILLED_STREAM = [
+ {"type": "system", "subtype": "init", "session_id": "k", "model": "claude-sonnet-5", "skills": []},
+ {"type": "assistant", "message": {"id": "msg_1", "model": "claude-sonnet-5", "usage": _MSG_USAGE,
+ "content": [{"type": "text", "text": "Reading the plan"}]}},
+ {"type": "assistant", "message": {"id": "msg_1", "model": "claude-sonnet-5", "usage": _MSG_USAGE, # same message
+ "content": [{"type": "tool_use", "id": "t1", "name": "Read", "input": {}}]}},
+ {"type": "user", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1"}]}},
+ {"type": "system", "subtype": "api_retry", "attempt": 1, "max_retries": 10, "retry_delay_ms": 500,
+ "error_status": 529, "error": "overloaded"},
+ {"type": "system", "subtype": "api_retry", "attempt": 2, "max_retries": 10, "retry_delay_ms": 1000,
+ "error_status": 529, "error": "overloaded"},
+]
+
+
+def test_a_killed_session_is_booked_from_its_messages_and_its_retries_are_provider_wait(tmp_ws: Workspace,
+ fake_bin, monkeypatch):
+ """claude prints no result event when it is killed (checked 2026-09-22: SIGTERM mid-request →
+ nothing), so the envelope's usage never comes. Each message's own usage block does: the input
+ side exact, the output a floor. Its api_retry events (claude's own, with the announced
+ back-off) are the time the provider cost — here until the kill, since nothing followed."""
+ from codeverse3d.models.pricing import cache_write_surcharge, estimate_cost
+
+ monkeypatch.setenv("FAKE_EVENTS", json.dumps(KILLED_STREAM))
+ monkeypatch.setattr("codeverse3d.agents.cli_common.IDLE_GRACE_S", 1.0)
+ a = ClaudeCodeAgent("sonnet", binary=fake_bin("claude", FAKE_KILLED_CLAUDE))
+ res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="k", timeout_s=1, spatial_tools=False))
+ assert not res.ok and res.exit_reason == "timeout" and res.transient and not res.quota
+ u = res.usage
+ assert (u.input_tokens, u.cached_tokens, u.output_tokens, u.model) == (2 + 19190 + 20354, 20354, 5, "claude-sonnet-5")
+ assert u.cost_usd == pytest.approx(estimate_cost("anthropic", "claude-sonnet-5", u)
+ + cache_write_surcharge("anthropic", "claude-sonnet-5", 19190)) and u.cost_usd > 0
+ assert 0.0 < res.provider_wait_s <= res.duration_s
+ rec = json.loads((Path(res.transcript_path).parent / "result.json").read_text())
+ assert rec["usage_from"] == "stream messages"
diff --git a/harness/tests/agents/test_cli_common.py b/harness/tests/agents/test_cli_common.py
index a777cd3c..e0e6433f 100644
--- a/harness/tests/agents/test_cli_common.py
+++ b/harness/tests/agents/test_cli_common.py
@@ -9,12 +9,14 @@ from codeverse3d.agents.cli_common import (
Trajectory,
attribute_changes,
begin_session,
- deliver_prompt,
finish_session,
hardened_env,
+ invoke,
is_quota_failure,
+ is_rate_limited,
is_secret_env,
is_transient_failure,
+ provider_wait,
)
from codeverse3d.contracts.agent import AgentJob, FileChange
from codeverse3d.contracts.common import Usage
@@ -79,22 +81,64 @@ def test_attribute_changes_pure():
assert [f.path for f in attribute_changes(files, write_roots=["public"])] == ["public/assets/y.glb"]
-def test_deliver_prompt_uses_file_when_long(tmp_ws: Workspace):
- job = AgentJob(workspace=str(tmp_ws.root), prompt="x")
- s = begin_session(job, "fake")
- short = deliver_prompt(s, "hello")
- assert short == "hello"
- long = "y" * 200_001
- stub = deliver_prompt(s, long)
- assert "task_prompt.md" in stub and (s.traj.dir / "task_prompt.md").read_text() == long
- assert os.path.isabs(s.traj.dir.as_posix())
-
-
-def test_transient_markers():
- assert is_transient_failure("Error: 429 RESOURCE_EXHAUSTED")
- assert is_transient_failure("", "got an empty response from the model")
- assert is_quota_failure("quota exceeded") and not is_quota_failure("503 UNAVAILABLE")
- assert not is_transient_failure("SyntaxError: bad code")
+def test_the_prompt_goes_on_stdin_byte_for_byte_and_never_on_argv(tmp_ws: Workspace):
+ """A 2 500-line, 200 kB prompt — past argv's 128 KiB per-argument cap and past gemini-cli's
+ 2 000-line read_file window, which is where the old file stub dropped a scene's last zones —
+ reaches the CLI whole, and the transcript records its size, not its text."""
+ import sys
+
+ s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="x", label="p"), "fake")
+ prompt = " leading spaces\n" + "\n".join(f"line {i}: ünïcode ✓" for i in range(2500)) + "\nSENTINEL\n\n"
+ got = tmp_ws.root / "artifacts" / "stdin.bin"
+ argv = [sys.executable, "-c", f"import sys; open({str(got)!r}, 'wb').write(sys.stdin.buffer.read())"]
+ proc = invoke(s, argv, dict(os.environ), prompt=prompt)
+ assert proc.rc == 0 and got.read_bytes() == prompt.encode("utf-8")
+ row = next(r for r in s.traj.read_transcript() if r["kind"] == "invoke")
+ assert row["argv"] == argv and row["stdin_bytes"] == len(prompt.encode("utf-8"))
+ assert not (s.traj.dir / "task_prompt.md").exists()
+
+
+def test_one_failure_vocabulary():
+ """The backends classify what the CLI said about the call that ended its session; the round
+ loop reads only the typed flags. Provider failures a retry may get through are transient; a
+ spent usage limit is quota and never transient; the task's own failure is neither."""
+ for text in ("Attempt 3 failed with status 503", "API Error: 529 overloaded_error", "got status: UNAVAILABLE",
+ "Error: 429 RESOURCE_EXHAUSTED", "TypeError: fetch failed sending request", "read ECONNRESET",
+ "stream disconnected before completion", "got an empty response from the model"):
+ assert is_transient_failure("x", text), text
+ for text in ("SyntaxError: bad code", "at process.processTicksAndRejections (node:internal/process/task_queues:95)",
+ "killed by watchdog (idle) after 305s", "mcp tool build timed out", "the tool is unavailable", ""):
+ assert not is_transient_failure(text), text # a node stack trace says "internal" in lower case
+ for text in ("You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits",
+ "Claude AI usage limit reached|1790000000", "You've hit your limit · resets 3pm",
+ "insufficient_quota: You exceeded your current quota", "You're out of extra usage"):
+ assert is_quota_failure(text) and not is_transient_failure(text, "503"), text
+ # gemini's per-key rate limits say "quota" too, and the key pool rotates past them
+ gemini_429 = "Quota exceeded for quota metric 'Generate Content API requests per minute'"
+ assert not is_quota_failure(gemini_429) and not is_quota_failure("You exceeded your current quota")
+ assert is_rate_limited("status 429") and is_rate_limited("RESOURCE_EXHAUSTED") and not is_rate_limited("503")
+
+
+def test_provider_wait_counts_each_burst_from_the_request_that_failed():
+ # progress at 0 and 100; a burst of three failures, back-off announced, then progress again
+ assert provider_wait([(10.0, 5.0), (20.0, 10.0), (35.0, 20.0)], end=500.0, progress=[0.0, 100.0]) == 55.0
+ # unannounced back-off (gemini-cli 5xx lines): a lower bound, the failed request's own time
+ assert provider_wait([(40.0, 0.0), (80.0, 0.0)], end=500.0, progress=[30.0, 120.0]) == 50.0
+ # a burst nothing followed — the CLI gave up, or the watchdog killed it mid-storm — runs to the end
+ assert provider_wait([(40.0, 0.0), (50.0, 0.0)], end=300.0, progress=[0.0, 30.0]) == 270.0
+ # progress between two failures splits them into two bursts
+ assert provider_wait([(10.0, 1.0), (30.0, 1.0)], end=100.0, progress=[0.0, 20.0, 50.0]) == 22.0
+ # a CLI that records no progress (agy's log): each burst from its first failure to the retry
+ assert provider_wait([(10.0, 4.0), (14.0, 6.0), (20.0, 10.0)], end=31.0, progress=None) == 20.0
+ assert provider_wait([], end=10.0, progress=[1.0]) == 0.0
+
+
+def test_finish_session_carries_the_typed_failure_and_the_wait(tmp_ws: Workspace):
+ s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", label="w"), "fake")
+ res = finish_session(s, ok=False, exit_reason="error", text="", usage=Usage(), transient=True, quota=True,
+ provider_wait_s=1e9)
+ assert res.quota and not res.transient # a spent usage limit is never "retry me"
+ assert 0.0 <= res.provider_wait_s <= res.duration_s # never more than the session lasted
def test_trajectory_jsonl_and_result(tmp_path):
diff --git a/harness/tests/agents/test_codex.py b/harness/tests/agents/test_codex.py
index 51e412a1..5711cab8 100644
--- a/harness/tests/agents/test_codex.py
+++ b/harness/tests/agents/test_codex.py
@@ -27,9 +27,8 @@ FAKE_CODEX = r'''
args = sys.argv[1:]
assert args[0] == "exec" and "--json" in args and "-C" in args
ws = args[args.index("-C") + 1]
-prompt = args[-1]
-if prompt == "-":
- prompt = sys.stdin.read()
+assert args[-1] == "-", "the prompt comes on stdin"
+prompt = sys.stdin.read()
assert "FAKE_SERVICE_API_KEY" not in os.environ
os.makedirs(os.path.join(ws, "src"), exist_ok=True)
open(os.path.join(ws, "src", "hello.txt"), "w").write(prompt[:5])
@@ -37,6 +36,13 @@ mode = os.environ.get("FAKE_MODE", "ok")
events = json.loads(''' + repr(json.dumps(EVENTS)) + r''')
if mode == "fail":
events = events[:2] + [{"type": "turn.failed", "error": {"message": "boom"}}]
+if mode == "usage_limit":
+ events = events[:2] + [{"type": "error", "message": "You've hit your usage limit. Visit https://chatgpt.com/codex/"
+ "settings/usage to purchase more credits or try again at Sep 14th, 2026 6:25 PM."},
+ {"type": "turn.failed", "error": {"message": "You've hit your usage limit."}}]
+if mode == "storm":
+ events = events[:2] + [{"type": "error", "message": "stream disconnected before completion: 503 Service Unavailable"},
+ {"type": "turn.failed", "error": {"message": "exceeded retry limit, last status: 503"}}]
for e in events:
print(json.dumps(e)); sys.stdout.flush()
'''
@@ -63,28 +69,28 @@ def test_reasoning_tokens_are_not_billed_twice():
def test_argv_with_mcp_overrides(tmp_ws: Workspace):
a = CodexAgent("gpt-5.6-sol", binary="codex")
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p"), "codex")
- argv = a.build_argv(s, "p")
+ argv = a.build_argv(s)
assert argv[:3] == ["codex", "exec", "--json"] and argv[argv.index("-C") + 1] == str(tmp_ws.root)
assert argv[argv.index("--sandbox") + 1] == "workspace-write" and "--skip-git-repo-check" in argv
joined = " ".join(argv)
- assert 'mcp_servers.3dcode.command="' in joined and "mcp_servers.3dcode.args=[" in joined and argv[-1] == "p"
+ assert 'mcp_servers.3dcode.command="' in joined and "mcp_servers.3dcode.args=[" in joined and argv[-1] == "-"
# codex exec has nobody to answer the per-tool approval elicitation → every MCP call would be cancelled
assert 'mcp_servers.3dcode.default_tools_approval_mode="approve"' in argv
assert argv[argv.index("--model") + 1] == "gpt-5.6-sol"
s2 = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "codex")
- assert "mcp_servers" not in " ".join(a.build_argv(s2, None)) and a.build_argv(s2, None)[-1] == "-"
+ assert "mcp_servers" not in " ".join(a.build_argv(s2)) and a.build_argv(s2)[-1] == "-"
def test_reasoning_effort_is_explicit(tmp_ws: Workspace, monkeypatch):
"""`codex exec` defaults to medium; every harness call states the effort instead."""
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "codex")
- argv = CodexAgent("gpt-5.6-sol", binary="codex").build_argv(s, "p")
+ argv = CodexAgent("gpt-5.6-sol", binary="codex").build_argv(s)
assert argv[argv.index("-c") + 1] == "model_reasoning_effort=high"
a = CodexAgent("gpt-5.6-terra@medium", binary="codex")
assert (a.model, a.reasoning_effort, a.id) == ("gpt-5.6-terra", "medium", "codex:gpt-5.6-terra")
- assert "model_reasoning_effort=medium" in a.build_argv(s, "p")
+ assert "model_reasoning_effort=medium" in a.build_argv(s)
assert "model_reasoning_effort" not in " ".join(CodexAgent("gpt-5.6-sol", binary="codex",
- reasoning_effort="").build_argv(s, "p"))
+ reasoning_effort="").build_argv(s))
assert split_model_effort("gpt-5.6-luna", "high") == ("gpt-5.6-luna", "high")
with pytest.raises(ValueError, match="reasoning effort"):
CodexAgent("gpt-5.6-sol", binary="codex", reasoning_effort="ludicrous")
@@ -102,12 +108,24 @@ def test_fake_run(tmp_ws: Workspace, fake_bin, monkeypatch):
monkeypatch.setenv("FAKE_MODE", "fail")
res2 = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="y", timeout_s=30))
assert not res2.ok and res2.exit_reason == "error" and "boom" in res2.errors[0]
+ assert not res2.transient and not res2.quota # the task's own failure
-def test_long_prompt_goes_via_stdin(tmp_ws: Workspace, fake_bin, monkeypatch):
- monkeypatch.setattr("codeverse3d.agents.backends.STDIN_PROMPT_BYTES", 10)
+def test_a_codex_failure_is_typed_quota_or_transient(tmp_ws: Workspace, fake_bin, monkeypatch):
+ """codex had no classification at all: its usage limit reached the round loop only because
+ the loop's own list said "usage limit", and a 503 death was never transient."""
a = CodexAgent("gpt-5.6-sol", binary=fake_bin("codex", FAKE_CODEX))
- res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello world this is long", label="z", timeout_s=30))
+ monkeypatch.setenv("FAKE_MODE", "usage_limit")
+ res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="q", timeout_s=30))
+ assert not res.ok and res.quota and not res.transient
+ monkeypatch.setenv("FAKE_MODE", "storm")
+ res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="t", timeout_s=30))
+ assert not res.ok and res.transient and not res.quota and res.provider_wait_s == 0.0 # codex does not say
+
+
+def test_every_prompt_goes_via_stdin(tmp_ws: Workspace, fake_bin):
+ a = CodexAgent("gpt-5.6-sol", binary=fake_bin("codex", FAKE_CODEX))
+ res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello world", label="z", timeout_s=30))
assert res.ok and (tmp_ws.src / "hello.txt").read_text() == "hello"
diff --git a/harness/tests/agents/test_gemini_cli.py b/harness/tests/agents/test_gemini_cli.py
index 7c93d166..7708171e 100644
--- a/harness/tests/agents/test_gemini_cli.py
+++ b/harness/tests/agents/test_gemini_cli.py
@@ -13,10 +13,33 @@ from codeverse3d.contracts.agent import AgentJob
from codeverse3d.workspace import Workspace
FAKE_GEMINI = r'''
+from datetime import datetime, timezone
args = sys.argv[1:]
model = args[args.index("-m") + 1]
-prompt = args[args.index("-p") + 1]
+assert "-p" not in args and "--prompt" not in args, "the prompt goes on stdin"
+prompt = sys.stdin.read()
+open("artifacts/stdin.txt", "w").write(prompt)
mode = os.environ.get("FAKE_MODE", "ok")
+
+def chat_record():
+ """What gemini-cli 0.53 appends to ~/.gemini/tmp/<project>/chats/ as the session runs."""
+ proj = os.path.join(os.environ["GEMINI_CLI_HOME"], ".gemini", "tmp", "fake-project")
+ os.makedirs(os.path.join(proj, "chats"), exist_ok=True)
+ open(os.path.join(proj, ".project_root"), "w").write(os.getcwd())
+ now = lambda: datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+ rows = [{"sessionId": "fake", "kind": "main"},
+ {"id": "u1", "type": "user", "timestamp": now(), "content": [{"text": prompt[:20]}]},
+ {"id": "g1", "type": "gemini", "timestamp": now(), "model": model, "content": "",
+ "toolCalls": [{"id": "write_file__1", "name": "write_file", "args": {"file_path": "src/a.js"}, "status": "success"}],
+ "tokens": {"input": 1000, "output": 20, "cached": 400, "thoughts": 7, "tool": 0, "total": 1027}},
+ {"id": "g1", "type": "gemini", "timestamp": now(), "model": model, "content": "", # re-written: once
+ "toolCalls": [{"id": "write_file__1", "name": "write_file", "args": {"file_path": "src/a.js"}, "status": "success"}],
+ "tokens": {"input": 1000, "output": 20, "cached": 400, "thoughts": 7, "tool": 0, "total": 1027}},
+ {"id": "u2", "type": "user", "timestamp": now(), "content": [{"functionResponse": {"id": "write_file__1"}}]},
+ {"id": "g2", "type": "gemini", "timestamp": now(), "model": model, "content": "",
+ "tokens": {"input": 3000, "output": 50, "cached": 2000, "thoughts": 13, "tool": 0, "total": 3063}}]
+ with open(os.path.join(proj, "chats", "session-2026-09-22T21-00-fake0001.jsonl"), "a") as f:
+ f.write("".join(json.dumps(r) + "\n" for r in rows))
assert os.environ.get("GEMINI_API_KEY"), "no key in env"
assert "GEMINI_API_KEYS" not in os.environ
assert "FAKE_SERVICE_API_KEY" not in os.environ, "secret leaked"
@@ -37,9 +60,19 @@ if mode == "crash": # the agent's own failure: nothing transient about it
print("TypeError: Cannot read properties of undefined (reading 'parts')", file=sys.stderr)
sys.exit(2)
if mode == "storm": # what gemini-cli 0.53 prints, 8-17 times per session, in a 503 storm
+ chat_record()
for i in range(4):
print(f"Attempt {i + 1} failed with status 503. Retrying with backoff... _ApiError: UNAVAILABLE", file=sys.stderr)
+ sys.stderr.flush()
+ time.sleep(0.05)
time.sleep(60)
+if mode == "gave_up": # 0.53 after its own retries: an error object on STDERR, no stats, exit(503) = 247
+ chat_record()
+ print("Attempt 1 failed with status 503. Retrying with backoff... _ApiError: UNAVAILABLE", file=sys.stderr)
+ print("Error when talking to Gemini API Full report available at: /tmp/x.json", file=sys.stderr)
+ print(json.dumps({"session_id": "s9", "error": {"type": "Error", "message": "got status: UNAVAILABLE.", "code": 503}},
+ indent=2), file=sys.stderr)
+ sys.exit(247)
served = "gemini-9-pro" if mode == "substitute" else model
os.makedirs("src", exist_ok=True)
open("src/hello.txt", "w").write(prompt[:20])
@@ -51,8 +84,9 @@ print(json.dumps(out, indent=2))
@pytest.fixture
-def agent(fake_bin, monkeypatch):
+def agent(fake_bin, monkeypatch, tmp_path):
binary = fake_bin("gemini", FAKE_GEMINI)
+ monkeypatch.setenv("GEMINI_CLI_HOME", str(tmp_path / "gemini_home")) # the fake writes its chat record here
s = get_settings()
monkeypatch.setattr(s, "gemini_api_keys", ["k1", "k2"])
monkeypatch.setattr(s, "cache_dir", Path(binary).parent / "cache")
@@ -83,6 +117,53 @@ def _job(ws: Workspace, **kw) -> AgentJob:
return AgentJob(**base)
+def test_the_prompt_arrives_on_stdin_whole(tmp_ws: Workspace, agent: GeminiCliAgent):
+ """No -p: a piped stdin makes gemini-cli 0.53 headless and IS the input. A 5-zone scene
+ prompt runs past 2 000 lines — the window of the read_file tool the old >100 kB file stub
+ relied on — and its last zone must still arrive (checked live 2026-09-22, docs/SKILLS.md)."""
+ prompt = "\n".join(f"zone {i % 5} requirement line {i}" for i in range(2600)) + "\nLAST-ZONE-SENTINEL"
+ res = agent.run(_job(tmp_ws, prompt=prompt))
+ assert res.ok, res.errors
+ assert (tmp_ws.artifacts / "stdin.txt").read_text() == prompt
+
+
+def test_a_session_killed_in_a_storm_is_booked_from_its_chat_record(tmp_ws: Workspace, agent: GeminiCliAgent,
+ monkeypatch):
+ """The watchdog killed it, so the CLI printed no envelope — and until 2026-09-22 the session
+ was booked at $0 while its chat record held every reply's tokens (live: 6.47 M prompt tokens,
+ ≈ $1.2, in one baseline). The record is the usage now, and its tool calls the tool count."""
+ from codeverse3d.models.pricing import estimate_cost
+
+ monkeypatch.setenv("FAKE_MODE", "storm")
+ monkeypatch.setattr("codeverse3d.agents.cli_common.IDLE_GRACE_S", 1.0)
+ res = agent.run(_job(tmp_ws, timeout_s=1))
+ assert not res.ok and res.exit_reason == "timeout" and res.transient
+ u = res.usage
+ assert (u.input_tokens, u.cached_tokens, u.output_tokens, u.thoughts_tokens) == (4000, 2400, 70, 20) # g1 once
+ assert u.cost_usd == pytest.approx(estimate_cost("gemini", "gemini-3.7-flash", u), abs=1e-6) and u.cost_usd > 0
+ assert res.tool_calls == u.tool_calls == 1
+ # the storm ran from the last reply until the kill: all of it lost to the provider
+ assert 0.0 < res.provider_wait_s <= res.duration_s
+ rec = json.loads((Path(res.transcript_path).parent / "result.json").read_text())
+ assert rec["usage_from"] == ["chat record"]
+
+
+def test_a_session_that_gave_up_is_booked_from_its_record_and_is_typed_transient(tmp_ws: Workspace,
+ agent: GeminiCliAgent, monkeypatch):
+ """0.53 writes its give-up object to STDERR with no stats and exits 503 & 255 = 247: every
+ gemini-cli session of the 2026-09-22 live runs that "completed a round" ended this way."""
+ monkeypatch.setattr(get_settings(), "gemini_api_keys", ["only"])
+ from codeverse3d.models import gemini as gm
+ for sig in [x for x in list(gm._pools) if x and x[0] == "only"]: # noqa: SLF001
+ gm._pools.pop(sig, None) # noqa: SLF001
+ monkeypatch.setenv("FAKE_MODE", "gave_up")
+ res = agent.run(_job(tmp_ws))
+ assert not res.ok and res.exit_reason == "error" and res.transient and not res.quota
+ assert res.usage.input_tokens == 2 * 4000 # both attempts, each from its own record
+ rec = json.loads((Path(res.transcript_path).parent / "result.json").read_text())
+ assert rec["attempts"] == 2 and rec["usage_from"] == ["chat record", "chat record"] and rec["session_id"] == "s9"
+
+
def test_success_path(tmp_ws: Workspace, agent: GeminiCliAgent):
res = agent.run(_job(tmp_ws))
assert res.ok and res.exit_reason == "completed", res.errors
@@ -93,7 +174,8 @@ def test_success_path(tmp_ws: Workspace, agent: GeminiCliAgent):
traj = Path(res.transcript_path).parent
assert (traj / "stdout.json").exists() and (traj / "prompt.md").exists() and (traj / "result.json").exists()
rec = json.loads((traj / "result.json").read_text())
- assert rec["attempts"] == 1 and rec["session_id"] == "s1"
+ assert rec["attempts"] == 1 and rec["session_id"] == "s1" and rec["usage_from"] == ["envelope"]
+ assert res.provider_wait_s == 0.0
# commits: pre + agent
assert "agent:t" in tmp_ws._git("log", "--oneline").stdout
@@ -233,6 +315,7 @@ def test_a_timeout_after_a_503_streak_is_marked_transient(tmp_ws: Workspace, age
res = agent.run(_job(tmp_ws, timeout_s=1))
assert not res.ok and res.exit_reason == "timeout" and res.transient is True
assert any("4 x 503" in e for e in res.errors), res.errors
+ assert res.provider_wait_s > 0.0
def test_the_key_pool_hears_a_5xx_only_when_the_failure_was_transient(tmp_ws: Workspace, agent: GeminiCliAgent,
diff --git a/harness/tests/agents/test_materialize.py b/harness/tests/agents/test_materialize.py
index 1a14e700..e6c7f540 100644
--- a/harness/tests/agents/test_materialize.py
+++ b/harness/tests/agents/test_materialize.py
@@ -66,8 +66,9 @@ def test_no_mcp_server_is_written_into_the_workspace(tmp_ws: Workspace):
def test_ignore_files_keep_agent_facing_paths_readable(tmp_ws: Workspace):
- """gemini-cli refuses read_file on ignored paths: the build census, tool renders and the
- long-prompt file (trajectories/<label>_rNN/task_prompt.md) must NOT be ignored."""
+ """gemini-cli refuses read_file on ignored paths: the build census and tool renders must NOT
+ be ignored. trajectories/ is, whole: the prompt reaches every CLI on stdin since 2026-09-22,
+ so nothing there is the agent's to read."""
import fnmatch
from codeverse3d.agents.materialize import IGNORE_LINES
@@ -86,12 +87,13 @@ def test_ignore_files_keep_agent_facing_paths_readable(tmp_ws: Workspace):
for f in (".geminiignore", ".aiexclude"):
lines = tuple(ln for ln in (tmp_ws.root / f).read_text().splitlines() if ln.strip())
assert lines == IGNORE_LINES
- assert "artifacts/" not in lines and "trajectories/" not in lines
+ assert "artifacts/" not in lines and "trajectories/" in lines
for readable in ("artifacts/census.json", "artifacts/build.json", "artifacts/measurement.json",
"artifacts/gates/r00/contract_tool.json", "artifacts/tool_renders/r00_ab/sheet.png",
- "trajectories/baseline_r00/task_prompt.md", "src/model.py", ".3dcode/cookbook.md"):
+ "src/model.py", ".3dcode/cookbook.md"):
assert not ignored(readable, lines), readable
for hidden in ("artifacts/renders/r00/sheet.png", "trajectories/baseline_r00/stdout.json",
+ "trajectories/baseline_r00/prompt.md", "trajectories/baseline_r00/gemini_settings.json",
"trajectories/baseline_r00/stderr.log", "node_modules/three/x.js", ".git/HEAD"):
assert ignored(hidden, lines), hidden
diff --git a/harness/tests/agents/test_tool_trace.py b/harness/tests/agents/test_tool_trace.py
index 9d142ea1..f586908d 100644
--- a/harness/tests/agents/test_tool_trace.py
+++ b/harness/tests/agents/test_tool_trace.py
@@ -22,8 +22,8 @@ from codeverse3d.agents.backends import (
CodexEvents,
agy_tool_calls,
gemini_chat_records,
- gemini_tool_calls,
parse_claude_json,
+ read_gemini_chats,
write_system_settings,
)
from codeverse3d.agents.cli_common import (
@@ -95,7 +95,7 @@ def test_the_stream_json_envelope_is_the_result_line_and_an_unfinished_stream_ha
def test_claude_is_launched_streaming(tmp_ws: Workspace):
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "claude-code")
- argv = ClaudeCodeAgent("sonnet", binary="claude").build_argv(s, "p")
+ argv = ClaudeCodeAgent("sonnet", binary="claude").build_argv(s)
assert argv[argv.index("--output-format") + 1] == "stream-json" and "--verbose" in argv
assert "Skill" in argv[argv.index("--allowedTools") + 1].split(",")
assert "--disable-slash-commands" not in argv # claude-code: "Disable all skills"
@@ -170,7 +170,7 @@ def test_gemini_chat_record_is_found_by_workspace_and_time_and_folded_by_call_id
home = _gemini_home(tmp_path, ws)
recs = gemini_chat_records(ws, time.time() - 60, {"GEMINI_CLI_HOME": str(home)})
assert [p.name for p in recs] == ["session-2026-09-22T19-31-da5c91db.jsonl"]
- calls = gemini_tool_calls(recs)
+ calls = read_gemini_chats(recs).calls
assert [(c.tool, c.skill, c.failed) for c in calls] == [
("activate_skill", "c3d-blender-forms", False), ("activate_skill", "c3d-part-contact", False),
("read_file", "", False), ("read_file", "", True)]
@@ -184,10 +184,31 @@ def test_gemini_system_settings_pin_skills_on_and_keep_the_workspace_trusted(tmp
``skills.enabled: false`` cannot hide the bundles; ``folderTrust`` off is what makes
``isTrustedFolder()`` true, without which ``discoverSkills`` skips both workspace roots."""
data = json.loads(write_system_settings(tmp_path / "s.json").read_text())
- assert data["skills"] == {"enabled": True} == SYSTEM_SETTINGS["skills"]
+ assert data["skills"]["enabled"] is True and data["skills"] == SYSTEM_SETTINGS["skills"]
assert data["security"]["folderTrust"] == {"enabled": False}
+def test_the_chat_record_is_the_usage_of_a_session_that_printed_no_envelope(tmp_path):
+ """Every reply once (the record re-writes a message as it progresses), its tokens in the
+ envelope's ``stats.models`` shape — the sum reproduced two live envelopes to the token."""
+ ws = tmp_path / "gemini1"
+ ws.mkdir()
+ home = _gemini_home(tmp_path, ws)
+ rec = next((home / ".gemini" / "tmp" / ws.name / "chats").glob("session-*"))
+ tok = {"input": 5000, "output": 40, "cached": 3000, "thoughts": 9, "tool": 0, "total": 5049}
+ with rec.open("a") as f:
+ for row in ({"id": "fb86", "type": "gemini", "model": "gemini-3.7-flash", "timestamp": "2026-09-22T19:31:40.000Z",
+ "content": "", "tokens": tok},
+ {"id": "fb86", "type": "gemini", "model": "gemini-3.7-flash", "timestamp": "2026-09-22T19:31:41.000Z",
+ "content": "", "tokens": tok}, # the same reply, re-written
+ {"id": "fb99", "type": "gemini", "model": "gemini-3.7-flash", "content": ""}): # died mid-reply
+ f.write(json.dumps(row) + "\n")
+ chat = read_gemini_chats(gemini_chat_records(ws, time.time() - 60, {"GEMINI_CLI_HOME": str(home)}))
+ assert chat.stats() == {"models": {"gemini-3.7-flash": {"tokens": {"prompt": 5000, "cached": 3000,
+ "candidates": 40, "thoughts": 9}}}}
+ assert len(chat.times) == 2 and chat.times[1] - chat.times[0] == 1.0
+
+
# --------------------------------------------------------------------------- codex
CODEX_EVENTS = [
{"type": "thread.started", "thread_id": "01a0ca9a"},