Skip to content

Official Codebase of 3DCodeVerse

Python

proc/cost: kill by group membership, and bill every paid round-trip

gaoypeng8432fa2c63a2
diff --git a/harness/codeverse/agents/api_agent.py b/harness/codeverse/agents/api_agent.py
index 23325beb..99befead 100644
--- a/harness/codeverse/agents/api_agent.py
+++ b/harness/codeverse/agents/api_agent.py
@@ -243,6 +243,7 @@ class _Loop:
self.usage = self.usage + resp.usage
return resp
except Exception as e: # ModelError or provider glitch
+ self.usage = self.usage + (getattr(e, "usage", None) or Usage()) # billed anyway
retryable = bool(getattr(e, "retryable", False))
# the session deadline used to be checked only between turns, so a turn that
# met it inside this loop still slept and re-entered the model's whole
diff --git a/harness/codeverse/agents/transcript.py b/harness/codeverse/agents/transcript.py
index d1b92277..ca685216 100644
--- a/harness/codeverse/agents/transcript.py
+++ b/harness/codeverse/agents/transcript.py
@@ -17,6 +17,12 @@ from pydantic import BaseModel
from codeverse.proc import append_jsonl_line, read_jsonl_lenient
+#: per-session transcript caps. Rows are byte-capped individually but nothing capped
+#: the file: a child emitting 100k lines wrote 27 MB, ~406 MB at the per-row cap. The
+#: unabridged (bounded) stream still lands in stdout.json.
+LINE_BUDGET_BYTES = 64 * 1024 * 1024
+LINE_BUDGET_ROWS = 200_000
+
class Trajectory:
"""Append-only writer for one agent session's files."""
@@ -25,6 +31,9 @@ class Trajectory:
self.dir = Path(directory)
self.dir.mkdir(parents=True, exist_ok=True)
self._lock = threading.Lock()
+ self._budget = threading.Lock() # guards the counters below, never nested in _lock
+ self._rows = self._bytes = 0
+ self._spent = False
# ------------------------------------------------------------------ paths
@property
@@ -59,8 +68,19 @@ class Trajectory:
return p
def append(self, kind: str, **data: Any) -> None:
- """Append one JSONL turn: ``{"t": epoch, "kind": kind, **data}``."""
- append_jsonl_line(self.transcript_path, {"t": round(time.time(), 3), "kind": kind, **data}, self._lock)
+ """Append one JSONL turn: ``{"t": epoch, "kind": kind, **data}``, until the
+ session's byte/row budget trips — then one marker row and nothing more."""
+ rec: dict[str, Any] = {"t": round(time.time(), 3), "kind": kind, **data}
+ with self._budget:
+ if self._spent:
+ return
+ self._rows += 1
+ self._bytes += len(json.dumps(rec, ensure_ascii=False, default=str))
+ self._spent = self._rows > LINE_BUDGET_ROWS or self._bytes > LINE_BUDGET_BYTES
+ if self._spent:
+ rec = {"t": rec["t"], "kind": "line_budget_exhausted",
+ "rows": self._rows - 1, "bytes": self._bytes}
+ append_jsonl_line(self.transcript_path, rec, self._lock)
def write_result(self, result: BaseModel, **extra: Any) -> Path:
"""Write ``result.json`` = AgentResult fields + any extra diagnostics."""
diff --git a/harness/codeverse/cost/instrument.py b/harness/codeverse/cost/instrument.py
index 14608405..a9e8f320 100644
--- a/harness/codeverse/cost/instrument.py
+++ b/harness/codeverse/cost/instrument.py
@@ -142,21 +142,25 @@ class MeteredChatModel:
# captured NOW: a hedge loser lands after this call returned, in a thread
# with no context, and its row must still reach THIS run's ledger
ledger = default_ledger()
+ booked: list[int] = [] # round-trips that already billed themselves as "extra"
def attempt_row(rec: AttemptRecord) -> None:
- self._record_attempt(request, call_id, rec, ledger)
+ if self._record_attempt(request, call_id, rec, ledger):
+ booked.append(rec.attempt)
t0 = time.perf_counter()
try:
with attempt_recording(attempt_row):
resp = self._inner.generate(request)
except BaseException as exc: # noqa: BLE001 - record the attempt, then re-raise
- # a failed call may still have been billed: ModelError.usage carries what
- # the provider charged (a bad-JSON reply costs like a good one)
+ # a failed call may still have been billed: ModelError.usage carries what the
+ # provider charged (a bad-JSON reply costs like a good one) — but only when
+ # the round-trips did not already book it, or the last one counts twice
usage = getattr(exc, "usage", None)
- self._record(usage if isinstance(usage, Usage)
- else Usage(backend=self.provider, model=self.model),
- request, outcome=_outcome(exc),
+ usage = usage if isinstance(usage, Usage) else Usage(backend=self.provider, model=self.model)
+ if booked:
+ usage = Usage(backend=usage.backend, model=usage.model, latency_ms=usage.latency_ms)
+ self._record(usage, request, outcome=_outcome(exc),
ms=int((time.perf_counter() - t0) * 1000),
attempts=getattr(exc, "attempts", 0), call_id=call_id)
raise
@@ -180,21 +184,26 @@ class MeteredChatModel:
log.debug("cost: could not record %s: %s", request.label, e)
def _record_attempt(self, request: ChatRequest, call_id: str, rec: AttemptRecord,
- ledger: CostLedger | None) -> None:
- """One ``source="attempt"`` row per round-trip. Excluded from every total
- (``ledger.load_ledger`` / ``summarise``); they exist so a hedge loser's or
- a charged-but-invalid reply's tokens are visible at all (audit 2026-08-27)."""
+ ledger: CostLedger | None) -> bool:
+ """One row per round-trip. A round-trip that was DISCARDED and still cost money
+ (a billed-but-invalid reply, a hedge loser that landed) is money nothing else
+ records, so it goes in as ``source="extra"`` and counts in every total; the rest
+ are ``source="attempt"`` forensics, excluded because the call's logical row
+ already carries their tokens. Returns True when it billed an ``extra``."""
+ extra = bool(rec.discarded and rec.usage.cost_usd)
try:
record_call(rec.usage, label=request.label,
backend=rec.usage.backend or self.provider,
model=rec.usage.model or self.model,
outcome="discarded" if rec.discarded else "ok",
latency_ms=rec.usage.latency_ms,
- source="attempt", key=_key_suffix(rec.key),
+ source="extra" if extra else "attempt", key=_key_suffix(rec.key),
call_id=call_id, attempt=rec.attempt, discarded=rec.discarded,
ledger=ledger)
except Exception as e: # pragma: no cover - accounting must never break a call
log.debug("cost: could not record attempt %s#%d: %s", request.label, rec.attempt, e)
+ return False
+ return extra
def _key_suffix(key: object) -> str:
diff --git a/harness/codeverse/cost/ledger.py b/harness/codeverse/cost/ledger.py
index 22fec073..08c291a1 100644
--- a/harness/codeverse/cost/ledger.py
+++ b/harness/codeverse/cost/ledger.py
@@ -349,9 +349,12 @@ def load_ledger(path: str | Path, *, include_attempts: bool = False) -> list[Cal
debug log — a truncated last line never loses the rest of the file.
``source="attempt"`` rows (one per round-trip, ``instrument.MeteredChatModel``)
- are left out unless ``include_attempts=True``: the winning round-trip's tokens
- are already on the call's logical row, so every aggregate built on this reader
- (``summarise``, ``reconstruct``, the CLI) keeps counting each call exactly once."""
+ are left out unless ``include_attempts=True``: their tokens are already on the
+ call's logical row, so every aggregate built on this reader (``summarise``,
+ ``reconstruct``, the CLI) keeps counting each call exactly once. This is the ONLY
+ place that filter lives — ``summarise`` used to repeat it, which made
+ ``include_attempts=True`` summarise to $0. Paid-but-discarded round-trips are
+ ``source="extra"`` instead and always count: nothing else records them."""
p = Path(path)
if p.is_dir():
found = existing_ledger_path(p)
@@ -383,8 +386,6 @@ def summarise(rows: Iterable[CallCost], *, dimensions: Sequence[str] = DIMENSION
of :class:`CallCost` (defaults to :data:`DIMENSIONS`)."""
out = Summary()
for row in rows:
- if row.source == "attempt":
- continue # per-round-trip forensic rows; their call is already a row
out.total.add(row)
for dim in dimensions:
bucket = out.by.setdefault(dim, {}).setdefault(_key(row, dim), CostBucket(key=_key(row, dim)))
diff --git a/harness/codeverse/cost/types.py b/harness/codeverse/cost/types.py
index d79414e6..ac95c4e0 100644
--- a/harness/codeverse/cost/types.py
+++ b/harness/codeverse/cost/types.py
@@ -184,7 +184,9 @@ class CallCost(BaseModel):
cache_hit: bool = False
outcome: str = "ok" # ok | error | timeout | budget | degraded | discarded
n_calls: int = 1 # >1 when a row aggregates a whole agent session
- source: str = "live" # live | record | events | transcript | stdout | residual | session | attempt
+ #: extra = a round-trip that was discarded yet billed (hedge loser, charged-but-invalid
+ #: reply); nothing else records it, so unlike ``attempt`` it counts in every total
+ source: str = "live" # live | record | events | transcript | stdout | residual | session | attempt | extra
#: which API key served the call, as its last 4 chars ("…ab12") — never the key
#: itself; "" for a failed call, a session row, or a row older than 2026-08-26
key: str = ""
@@ -198,8 +200,8 @@ class CallCost(BaseModel):
#: 1-based round-trip number within its logical call (``source="attempt"`` rows)
attempt: int = 0
#: True when this round-trip's response was thrown away: a hedge loser, or a
- #: failed try of a call that went on to retry. Real money, but never part of a
- #: total — ``ledger.load_ledger`` / ``summarise`` exclude ``source="attempt"``
+ #: failed try of a call that went on to retry. A paid one is ``source="extra"``
+ #: and counts; a free one is ``source="attempt"`` and is filtered out
discarded: bool = False
@property
diff --git a/harness/codeverse/judges/pairwise.py b/harness/codeverse/judges/pairwise.py
index 1475b04f..ea4cf11e 100644
--- a/harness/codeverse/judges/pairwise.py
+++ b/harness/codeverse/judges/pairwise.py
@@ -153,7 +153,7 @@ class PairwiseJudge:
try:
resp = self.model.generate(req)
except ModelError as e:
- return Usage(), None, None, f"ModelError: {e}"
+ return getattr(e, "usage", None) or Usage(), None, None, f"ModelError: {e}"
try:
payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
reply = PairwiseReply.model_validate(payload)
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index 6180e43e..b2c6f233 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -216,6 +216,7 @@ class VlmJudge:
try:
resp: ChatResponse = self.model.generate(req)
except ModelError as e:
+ usage = usage + (getattr(e, "usage", None) or Usage()) # the provider billed it
last = f"ModelError(attempt {attempt}): {e}"
log.warning("judge %s: %s", req.label, last)
if not e.retryable and attempt >= 2:
diff --git a/harness/codeverse/models/anthropic.py b/harness/codeverse/models/anthropic.py
index 7a8b2304..6e2090ea 100644
--- a/harness/codeverse/models/anthropic.py
+++ b/harness/codeverse/models/anthropic.py
@@ -144,7 +144,7 @@ class AnthropicModel:
if stop == "refusal":
raise ModelError(
f"Anthropic refused the request: {getattr(msg, 'stop_details', None)}",
- retryable=False,
+ retryable=False, usage=usage, # a refusal is billed like any other reply
)
parsed: Any = None
if request.response_schema is not None:
@@ -164,10 +164,11 @@ class AnthropicModel:
except JsonParseError as exc:
raise ModelError(
f"structured output missing (stop_reason={stop}): {exc}",
- retryable=stop != "max_tokens",
+ retryable=stop != "max_tokens", usage=usage,
) from exc
if not text and not calls and parsed is None:
- raise ModelError(f"Anthropic returned no content (stop_reason={stop})", retryable=True)
+ raise ModelError(f"Anthropic returned no content (stop_reason={stop})",
+ retryable=True, usage=usage)
raw: dict[str, Any] = {"stop_reason": stop, "id": msg.id, "model": msg.model}
return ChatResponse(
text=text, parsed=parsed, tool_calls=calls, finish_reason=stop, usage=usage, raw=raw
diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.py
index 63a3604f..554a2287 100644
--- a/harness/codeverse/models/gemini.py
+++ b/harness/codeverse/models/gemini.py
@@ -264,6 +264,7 @@ class GeminiModel:
budget = RETRY_DEADLINE_S if request.max_wait_s is None else min(RETRY_DEADLINE_S, float(request.max_wait_s))
deadline = time.monotonic() + budget # mirrors rotate_with_retries' own deadline
stats: dict[str, Any] = {}
+ wasted: list[Usage] = [] # discarded round-trips the provider still billed
try:
resp = rotate_with_retries(
self.pool,
@@ -286,7 +287,7 @@ class GeminiModel:
storm_gate=self.storm_gate,
label=f"gemini {self.model}",
stats=stats,
- on_attempt=self._attempt_hook(),
+ on_attempt=self._attempt_hook(wasted),
**({} if self.storm_attempts is None else {"storm_attempts": self.storm_attempts}),
)
except ModelError as err:
@@ -295,6 +296,10 @@ class GeminiModel:
# how hard the call was, next to which key served it (``_once``): ledger fields
resp.raw["attempts"] = int(stats.get("attempts", 0))
resp.raw["hedged"] = int(stats.get("hedged", 0))
+ if wasted:
+ # money the winner's usage does not include; a hedge loser that lands AFTER
+ # this returns reaches the ledger through the sink, never this field
+ resp.raw["wasted_usage"] = sum(wasted[1:], wasted[0])
return resp
def _config(self, request: ChatRequest, warnings: list[str]) -> types.GenerateContentConfig:
@@ -316,19 +321,20 @@ class GeminiModel:
want_s = min(self.timeout_s, max(HTTP_TIMEOUT_FLOOR_S, remaining))
return clip_timeout(config, int(want_s * 1000))
- def _attempt_hook(self) -> OnAttempt | None:
- """The per-attempt ledger recorder, when the metering layer installed one
- (``cost.instrument.MeteredChatModel``). Captured once per logical call so
- a hedge loser landing later, in its own thread, still reports through it.
- Imported lazily: the models package must not import the cost package at
- module level (``cost.ledger`` imports ``models.pricing``)."""
+ def _attempt_hook(self, wasted: list[Usage]) -> OnAttempt:
+ """Per-round-trip observer. It collects every DISCARDED round-trip that still
+ cost money into ``wasted`` (published on ``resp.raw["wasted_usage"]`` — the only
+ place a budget guard can see that money) and feeds the ledger's per-attempt sink
+ when the metering layer installed one (``cost.instrument.MeteredChatModel``).
+ Captured once per logical call so a hedge loser landing later, in its own thread,
+ still reports through it. Imported lazily: the models package must not import
+ the cost package at module level (``cost.ledger`` imports ``models.pricing``)."""
try:
from codeverse.cost.context import AttemptRecord, attempt_sink
+
+ sink = attempt_sink()
except Exception: # pragma: no cover - accounting must never break a call
- return None
- sink = attempt_sink()
- if sink is None:
- return None
+ AttemptRecord = sink = None
def on_attempt(t: _Try, no: int, discarded: bool) -> None:
if t.err is None:
@@ -337,8 +343,11 @@ class GeminiModel:
else:
usage = getattr(t.err, "usage", None) or Usage()
outcome, error = t.outcome, str(t.err)
- sink(AttemptRecord(attempt=no, key=t.key, outcome=outcome,
- discarded=discarded, usage=usage, error=error))
+ if discarded and usage.cost_usd:
+ wasted.append(usage)
+ if sink is not None:
+ sink(AttemptRecord(attempt=no, key=t.key, outcome=outcome,
+ discarded=discarded, usage=usage, error=error))
return on_attempt
@@ -355,11 +364,11 @@ class GeminiModel:
resp = client.models.generate_content(
model=self.model, contents=contents, config=config
)
- text, calls, finish = extract_candidate(resp)
- usage = parse_usage(resp, self.model)
+ usage = parse_usage(resp, self.model) # BEFORE extract_candidate: its raises carry it
usage.latency_ms = sw.ms
- usage.tool_calls = len(calls)
usage.cost_usd = estimate_cost("gemini", self.model, usage)
+ text, calls, finish = extract_candidate(resp, usage)
+ usage.tool_calls = len(calls)
parsed: Any = None
if request.response_schema is not None and not calls:
try:
diff --git a/harness/codeverse/models/gemini_convert.py b/harness/codeverse/models/gemini_convert.py
index dd547234..4c2075c2 100644
--- a/harness/codeverse/models/gemini_convert.py
+++ b/harness/codeverse/models/gemini_convert.py
@@ -205,17 +205,21 @@ def parse_usage(resp: types.GenerateContentResponse, model: str) -> Usage:
)
-def extract_candidate(resp: types.GenerateContentResponse) -> tuple[str, list[ToolCallPart], str]:
+def extract_candidate(
+ resp: types.GenerateContentResponse, usage: Usage | None = None
+) -> tuple[str, list[ToolCallPart], str]:
"""→ ``(text, tool_calls, finish_reason)``. Raises ``ModelError`` on blocked
- prompts (non-retryable) or empty candidates (retryable)."""
+ prompts (non-retryable) or empty candidates (retryable); ``usage`` (what the caller
+ already parsed off ``resp``) rides on the error, because every one of these
+ failures was billed."""
pf = resp.prompt_feedback
if pf is not None and pf.block_reason:
raise ModelError(
f"prompt blocked by Gemini: {pf.block_reason} {pf.block_reason_message or ''}".strip(),
- retryable=False,
+ retryable=False, usage=usage,
)
if not resp.candidates:
- raise ModelError("Gemini returned no candidates", retryable=True)
+ raise ModelError("Gemini returned no candidates", retryable=True, usage=usage)
cand = resp.candidates[0]
finish = str(cand.finish_reason.value if cand.finish_reason else "") or "UNKNOWN"
texts: list[str] = []
@@ -236,16 +240,18 @@ def extract_candidate(resp: types.GenerateContentResponse) -> tuple[str, list[To
if not text and not calls:
if finish in FATAL_FINISH:
raise ModelError(
- f"Gemini produced no content (finish_reason={finish})", retryable=False
+ f"Gemini produced no content (finish_reason={finish})", retryable=False, usage=usage
)
if finish == "MAX_TOKENS":
# Gemini 3 models always think (budget 0 is accepted but ignored); a tiny
- # max_output_tokens is eaten by thoughts. Retrying cannot help.
+ # max_output_tokens is eaten by thoughts. Retrying cannot help — and this is
+ # the most expensive failure on Gemini 3: every thought token was billed.
thoughts = resp.usage_metadata.thoughts_token_count if resp.usage_metadata else None
raise ModelError(
f"Gemini produced no content: max_output_tokens exhausted by thinking "
f"(thoughts_tokens={thoughts}); raise max_output_tokens",
- retryable=False,
+ retryable=False, usage=usage,
)
- raise ModelError(f"Gemini produced no content (finish_reason={finish})", retryable=True)
+ raise ModelError(f"Gemini produced no content (finish_reason={finish})",
+ retryable=True, usage=usage)
return text, calls, finish
diff --git a/harness/codeverse/models/gemini_image.py b/harness/codeverse/models/gemini_image.py
index 49c747d8..2486556f 100644
--- a/harness/codeverse/models/gemini_image.py
+++ b/harness/codeverse/models/gemini_image.py
@@ -304,11 +304,14 @@ class GeminiImageModel:
client = self._clients._client(key)
with Stopwatch() as sw:
resp = client.models.generate_content(model=model, contents=contents, config=config)
+ usage = parse_usage(resp, model) # parsed first: the raises below were billed too
+ usage.backend = "gemini-image"
+ usage.latency_ms = sw.ms
pf = resp.prompt_feedback
if pf is not None and pf.block_reason:
- raise ModelError(f"image prompt blocked: {pf.block_reason}", retryable=False)
+ raise ModelError(f"image prompt blocked: {pf.block_reason}", retryable=False, usage=usage)
if not resp.candidates:
- raise ModelError("image model returned no candidates", retryable=True)
+ raise ModelError("image model returned no candidates", retryable=True, usage=usage)
cand = resp.candidates[0]
finish = str(cand.finish_reason.name if cand.finish_reason is not None else "")
images: list[Image.Image] = []
@@ -320,11 +323,10 @@ class GeminiImageModel:
images.append(img.convert("RGB"))
if not images:
if finish in FATAL_FINISH:
- raise ModelError(f"image generation refused (finish_reason={finish})", retryable=False)
- raise ModelError(f"image model returned no image (finish_reason={finish or 'unknown'})", retryable=True)
- usage = parse_usage(resp, model)
- usage.backend = "gemini-image"
- usage.latency_ms = sw.ms
+ raise ModelError(f"image generation refused (finish_reason={finish})",
+ retryable=False, usage=usage)
+ raise ModelError(f"image model returned no image (finish_reason={finish or 'unknown'})",
+ retryable=True, usage=usage)
usage.cost_usd = image_cost(model, usage, len(images))
if int(size) not in (0, 1024) and images[0].size != (size, size):
images = [im.resize((int(size), int(size)), Image.LANCZOS) for im in images]
diff --git a/harness/codeverse/models/openai.py b/harness/codeverse/models/openai.py
index 8d609857..bf1ebe16 100644
--- a/harness/codeverse/models/openai.py
+++ b/harness/codeverse/models/openai.py
@@ -169,12 +169,12 @@ class OpenAIModel:
except JsonParseError as exc:
raise ModelError(
f"structured output is not valid JSON (finish_reason={finish}): {exc}",
- retryable=finish != "length",
+ retryable=finish != "length", usage=usage, # billed like a good reply
) from exc
if not text and not calls:
raise ModelError(
f"OpenAI returned no content (finish_reason={finish})",
- retryable=finish != "content_filter",
+ retryable=finish != "content_filter", usage=usage,
)
raw: dict[str, Any] = {
"finish_reason": finish,
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index 6a74bd49..f111c568 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -300,6 +300,10 @@ def rotate_with_retries[T](
def out_of_time() -> bool:
return monotonic() >= deadline
+ def clip(delay: float) -> float:
+ """A sleep never outlives the budget: waking only to give up helps nobody."""
+ return delay if deadline == float("inf") else min(delay, max(0.0, deadline - monotonic()))
+
def is_storm(err: ModelError) -> bool:
return bool(err.retryable and err.status in (503, 529))
@@ -317,6 +321,13 @@ def rotate_with_retries[T](
exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())
if storm_gate is not None:
storm_gate.enter(None if deadline == float("inf") else deadline)
+ if out_of_time():
+ # the gate held us past the budget. The check at the top of the loop
+ # cannot cover this: a worker parked at a gate ANOTHER thread closed
+ # has no last_err, and would go on to spend a full round-trip.
+ log.warning("%s budget of %.0f s spent waiting at the storm gate", label, max_total_s)
+ raise (last_err or classify(TimeoutError(
+ f"{label}: retry budget spent waiting for capacity"))) from last_exc
budget_left = None if deadline == float("inf") else max(0.0, deadline - monotonic())
try:
# the key/slot wait must fit the remaining budget, never outlive it
@@ -394,11 +405,7 @@ def rotate_with_retries[T](
# jitter FIRST, cap LAST: capping before a >1 jitter factor let a
# "<=5 s" wait land at 6.25 s in the wild (observed 2026-08-24).
raw = base_delay * (2 ** min(storm, 8)) * (0.75 + 0.5 * random.random())
- delay = min(storm_max_delay, raw)
- if deadline != float("inf"):
- # clip the storm sleep to the remainder: sleeping past the
- # deadline only to give up on waking helps nobody
- delay = min(delay, max(0.0, deadline - monotonic()))
+ delay = clip(min(storm_max_delay, raw))
if storm_gate is not None:
# tell every other worker as well: the next one to arrive parks at
# the gate instead of spending its own round-trip to find the storm
@@ -424,7 +431,7 @@ def rotate_with_retries[T](
log.warning(
"%s key …%s throttled (%s); rotating to a fresh key", label, key[-4:], err
)
- sleep(0.5)
+ sleep(clip(0.5))
continue
if not err.retryable or attempt >= max_attempts:
bench()
@@ -437,7 +444,7 @@ def rotate_with_retries[T](
"%s attempt %d/%d failed (%s); retrying in %.1fs",
label, attempt, max_attempts, err, delay,
)
- sleep(delay)
+ sleep(clip(delay))
assert last_err is not None
bench()
raise last_err
diff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.py
index ffc014e7..3020bdee 100644
--- a/harness/codeverse/proc.py
+++ b/harness/codeverse/proc.py
@@ -90,6 +90,17 @@ def run_subprocess(
#: for every build/render path that funnels through here.
DRAIN_TIMEOUT_S = 10.0
_ABANDONED = "(output truncated: a detached descendant still held the pipe past the drain window)"
+_STILL_HELD = "(output truncated: a process in the group still held the pipe past the drain window)"
+
+
+def _group_alive(pgid: int) -> bool:
+ """True while ANY member of process group ``pgid`` exists (the leader may be gone)."""
+ try:
+ os.killpg(pgid, 0)
+ except (ProcessLookupError, PermissionError):
+ return False
+ return True
+
#: per-stream cap on captured output: the first ``budget // 2`` bytes and the last
#: ``budget - budget // 2`` bytes are kept, with a truncation marker in between.
@@ -192,7 +203,7 @@ class ManagedProcess:
self._on_line = on_line
self._grace_s = grace_s
self._stop = threading.Event() # tells an abandoned pump to quit feeding
- self._abandoned = False
+ self._abandoned = "" # the truncation note to append, once we know which one
self._sinks = {"stdout": _BoundedSink(stream_budget), "stderr": _BoundedSink(stream_budget)}
self.proc: subprocess.Popen[bytes] = subprocess.Popen(
list(cmd),
@@ -269,7 +280,7 @@ class ManagedProcess:
"""Decoded, bounded stderr (+ a note when an escaped descendant held the pipe)."""
text = self._sinks["stderr"].text()
if self._abandoned:
- return f"{text}\n{_ABANDONED}" if text else _ABANDONED
+ return f"{text}\n{self._abandoned}" if text else self._abandoned
return text
# ------------------------------------------------------------- kills and teardown
@@ -285,30 +296,36 @@ class ManagedProcess:
"""SIGTERM the group, escalate to SIGKILL after ``grace_s`` (default: the
constructor's ``grace_s`` — the old ``watchdog.kill_process_group`` contract)."""
grace = self._grace_s if grace_s is None else grace_s
- if self.proc.poll() is not None:
- return
try:
+ # killpg after the leader was reaped is a microsecond pid-reuse race the
+ # code has always taken; a live grandchild is the certain cost of not doing it
os.killpg(self.proc.pid, signal.SIGTERM)
except ProcessLookupError:
return
except PermissionError: # cannot signal the group — go for the child itself
self.proc.terminate()
- try:
- self.proc.wait(timeout=grace)
- except subprocess.TimeoutExpired:
- self.kill()
- with contextlib.suppress(subprocess.TimeoutExpired):
- self.proc.wait(timeout=self._grace_s)
+ deadline = time.monotonic() + grace
+ while time.monotonic() < deadline:
+ self.proc.poll() # reap first: an unreaped zombie leader keeps the group "alive"
+ if not _group_alive(self.proc.pid):
+ return
+ time.sleep(0.02)
+ self.kill()
+ with contextlib.suppress(subprocess.TimeoutExpired):
+ self.proc.wait(timeout=self._grace_s)
def __enter__(self) -> ManagedProcess:
return self
def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: object) -> None:
- # Any in-flight exception (KeyboardInterrupt included) or a child still
- # running at scope exit: the group dies HERE. Ctrl-C's SIGINT never reaches
- # a start_new_session child, so this is the only thing standing between an
- # interrupt and an orphaned blender/node/chrome tree. Never swallows exc.
- if self.proc.poll() is None:
+ # Any in-flight exception (KeyboardInterrupt included) or a group still alive at
+ # scope exit: the group dies HERE. Ctrl-C's SIGINT never reaches a
+ # start_new_session child, so this is all that stands between an interrupt and an
+ # orphaned blender/node/chrome tree. Never swallows exc. LEADER liveness is the
+ # wrong question — one that exits 0 leaves its same-group grandchildren running;
+ # reap it (a zombie still counts as a member), then ask if the GROUP exists.
+ self.proc.poll()
+ if _group_alive(self.proc.pid):
self.terminate()
self._drain_and_reap()
@@ -318,11 +335,11 @@ class ManagedProcess:
self._writer.join(timeout=max(0.0, deadline - time.monotonic()))
for t in self._pumps:
t.join(timeout=max(0.0, deadline - time.monotonic()))
- if any(t.is_alive() for t in self._pumps):
- self._stop.set() # the parked daemon pump self-cleans when its read returns
- self._abandoned = True
with contextlib.suppress(subprocess.TimeoutExpired):
self.proc.wait(timeout=DRAIN_TIMEOUT_S) # reap — a zombie survives killpg
+ if any(t.is_alive() for t in self._pumps):
+ self._stop.set() # the parked daemon pump self-cleans when its read returns
+ self._abandoned = _STILL_HELD if _group_alive(self.proc.pid) else _ABANDONED
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index 06cdcb1e..9ef53dc8 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -216,7 +216,10 @@ def generate_files(
label=f"{task.label}:r{task.round:02d}",
)
resp = model.generate(req)
- usage = resp.usage
+ # + every paid round-trip the winner does not represent (a billed-but-invalid
+ # response, a late hedge loser): the ledger records those as source="extra";
+ # without this the guard never sees them (measured $0.18 on one call).
+ usage = resp.usage + (resp.raw.get("wasted_usage") or Usage())
# book the money WITHOUT enforcing: the response is already paid for, and raising
# here would discard it before transcript/parse/write_files. The ceiling is
# enforced at the round's phase boundary instead (steps._run_phase).
diff --git a/harness/tests/agents/test_transcript.py b/harness/tests/agents/test_transcript.py
new file mode 100644
index 00000000..d0cc4185
--- /dev/null
+++ b/harness/tests/agents/test_transcript.py
@@ -0,0 +1,27 @@
+"""codeverse/agents/transcript.py — the per-session transcript budget."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from codeverse.agents import transcript as tr
+from codeverse.agents.transcript import Trajectory
+
+
+def test_the_transcript_stops_at_its_budget_with_a_marker_row(tmp_path: Path, monkeypatch):
+ """Rows were byte-capped one by one but the FILE was not: a child emitting 100k
+ lines wrote a 27 MB transcript.jsonl (~406 MB at the 4000-char row cap)."""
+ monkeypatch.setattr(tr, "LINE_BUDGET_ROWS", 5)
+ t = Trajectory(tmp_path / "traj")
+ for i in range(20):
+ t.append("stdout", line=f"line {i}")
+ rows = t.read_transcript()
+ assert [r["kind"] for r in rows] == ["stdout"] * 5 + ["line_budget_exhausted"]
+ assert rows[-1]["rows"] == 5
+
+ monkeypatch.setattr(tr, "LINE_BUDGET_BYTES", 400)
+ t2 = Trajectory(tmp_path / "traj2")
+ for _ in range(50):
+ t2.append("stdout", line="x" * 100)
+ rows2 = t2.read_transcript()
+ assert len(rows2) < 6 and rows2[-1]["kind"] == "line_budget_exhausted"
diff --git a/harness/tests/core/test_proc.py b/harness/tests/core/test_proc.py
index efd2b395..57d954ff 100644
--- a/harness/tests/core/test_proc.py
+++ b/harness/tests/core/test_proc.py
@@ -43,20 +43,51 @@ def test_run_subprocess_stdin(tmp_path: Path):
assert r.stdout == "hello"
+def _assert_pid_gone(pid: int, timeout_s: float = 5.0) -> None:
+ deadline = time.monotonic() + timeout_s
+ while time.monotonic() < deadline:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return # gone — the group kill reached it
+ time.sleep(0.05)
+ raise AssertionError(f"pid {pid} survived the group kill")
+
+
def test_timeout_kills_the_whole_process_group(tmp_path: Path):
"""A timed-out child AND its grandchildren die (killpg on the new session)."""
r = run_subprocess(["bash", "-c", "sleep 30 & echo $!; wait"], cwd=tmp_path, timeout_s=0.4)
assert r.timed_out
assert r.returncode != 0
- grandchild = int(r.stdout.strip().splitlines()[0])
- deadline = time.monotonic() + 3.0
- while time.monotonic() < deadline:
- try:
- os.kill(grandchild, 0)
- except ProcessLookupError:
- return # gone — group kill worked
- time.sleep(0.05)
- raise AssertionError(f"grandchild sleep (pid {grandchild}) survived the group kill")
+ _assert_pid_gone(int(r.stdout.strip().splitlines()[0]))
+
+
+def test_a_leader_that_exits_0_still_takes_its_grandchildren_down(tmp_path: Path):
+ """``__exit__`` asked "is the LEADER still running?", so a leader that exited 0 left
+ its same-group grandchildren (pgid == leader pid) alive — the exact incident this
+ module exists to end. Detached-stdio shape: nothing holds the pipes."""
+ pid_file = tmp_path / "pid"
+ r = run_subprocess(["bash", "-c", f"sleep 300 >/dev/null 2>&1 & echo $! > {pid_file}; exit 0"],
+ cwd=tmp_path, timeout_s=10)
+ assert r.returncode == 0 and not r.timed_out
+ _assert_pid_gone(int(pid_file.read_text()))
+
+
+def test_a_leader_that_exits_0_does_not_burn_the_drain_window(tmp_path: Path, monkeypatch):
+ """Same shape with the grandchild holding the INHERITED pipes: the drain used to
+ wait out DRAIN_TIMEOUT_S for a process nobody had signalled, then blame a
+ "detached descendant" that had never left the group."""
+ import codeverse.proc as proc_mod
+
+ monkeypatch.setattr(proc_mod, "DRAIN_TIMEOUT_S", 3.0)
+ pid_file = tmp_path / "pid"
+ t0 = time.monotonic()
+ r = run_subprocess(["bash", "-c", f"sleep 300 & echo $! > {pid_file}; echo done"],
+ cwd=tmp_path, timeout_s=10)
+ assert r.returncode == 0 and "done" in r.stdout
+ assert time.monotonic() - t0 < 2.5, "the drain window was burned on a live group member"
+ assert "detached descendant" not in r.stderr
+ _assert_pid_gone(int(pid_file.read_text()))
def test_a_detached_descendant_holding_the_pipes_cannot_extend_the_timeout(tmp_path: Path, monkeypatch):
diff --git a/harness/tests/cost/test_instrument.py b/harness/tests/cost/test_instrument.py
index 7690f7ff..387c7931 100644
--- a/harness/tests/cost/test_instrument.py
+++ b/harness/tests/cost/test_instrument.py
@@ -358,10 +358,10 @@ def test_per_key_buckets_and_tries_per_call(tmp_path: Path):
table = bucket_rows(keyed_buckets(audit), total=1.0)
assert BUCKET_HEADERS[-1] == "tries/call" and table[0][-1] == "2.00" and table[1][-1] == "1.00"
# ------------------------------------------------------- per-attempt rows (audit 2026-08-27)
-def test_a_bad_json_reply_reaches_the_ledger_as_a_discarded_attempt_row(tmp_path: Path):
+def test_a_billed_but_invalid_attempt_is_in_the_total_exactly_once(tmp_path: Path):
"""A charged-but-invalid reply used to vanish: the logical row carries only the
- winner's usage. Its tokens now land as a source="attempt" row that joins the
- logical row by call_id — and never double-counts in any total."""
+ winner's usage and its own row was ``source="attempt"``, which every total
+ filtered out. It is now ``source="extra"`` — counted once, joined by call_id."""
from codeverse.cost.ledger import summarise
from tests.models.test_gemini import make_model, text_response
@@ -371,23 +371,24 @@ def test_a_bad_json_reply_reaches_the_ledger_as_a_discarded_attempt_row(tmp_path
ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"},
label="planner"))
assert resp.parsed == {"ok": True}
- (logical,) = load_ledger(tmp_path) # attempt rows are filtered by default
+ rows = load_ledger(tmp_path) # "attempt" forensics are filtered here; "extra" is money
+ bad, = [r for r in rows if r.source == "extra"]
+ logical, = [r for r in rows if r.source == "live"]
assert logical.outcome == "ok" and logical.attempts == 2 and logical.call_id
- rows = load_ledger(tmp_path, include_attempts=True)
- attempts = sorted((r for r in rows if r.source == "attempt"), key=lambda r: r.attempt)
- assert [r.attempt for r in attempts] == [1, 2]
- bad, win = attempts
assert bad.discarded and bad.outcome == "discarded" and bad.input_tokens == 100
- assert not win.discarded and win.outcome == "ok"
- assert bad.call_id == win.call_id == logical.call_id
- assert summarise(rows).total.input_tokens == logical.input_tokens, \
- "attempt rows never double-count"
+ assert bad.call_id == logical.call_id and bad.cost_usd > 0
+ total = summarise(rows).total
+ assert total.input_tokens == logical.input_tokens + bad.input_tokens
+ assert summarise(load_ledger(tmp_path, include_attempts=True)).total.input_tokens \
+ == total.input_tokens + 100, "include_attempts adds the winner's forensic row, once"
+ win, = [r for r in load_ledger(tmp_path, include_attempts=True) if r.source == "attempt"]
+ assert not win.discarded and win.attempt == 2
def test_a_hedge_losers_tokens_reach_the_ledger_when_it_lands(tmp_path: Path):
"""The loser of a raced 503 retry keeps running after the winner returned; when
- it lands, its billed tokens must appear as a discarded attempt row — before,
- that money left no trace anywhere."""
+ it lands, its billed tokens must appear in the ledger TOTAL — before, that money
+ left no trace anywhere, and then no trace in any total."""
import threading
import time as _time
@@ -420,16 +421,19 @@ def test_a_hedge_losers_tokens_reach_the_ledger_when_it_lands(tmp_path: Path):
deadline = _time.monotonic() + 5.0
loser = None
while _time.monotonic() < deadline:
- rows = load_ledger(tmp_path, include_attempts=True)
- loser = next((r for r in rows if r.source == "attempt" and r.key == "…k2"), None)
+ rows = load_ledger(tmp_path)
+ loser = next((r for r in rows if r.source == "extra" and r.key == "…k2"), None)
if loser is not None:
break
_time.sleep(0.01)
assert loser is not None, "the loser's row never arrived"
assert loser.discarded and loser.input_tokens == 100, "the loser's paid tokens are on the ledger"
- (logical,) = load_ledger(tmp_path)
+ logical, = [r for r in load_ledger(tmp_path) if r.source == "live"]
# endswith, not ==: the 2-char fake key is shorter than the …last-4 redaction
assert logical.key.endswith("k3") and logical.call_id == loser.call_id
+ from codeverse.cost.ledger import summarise
+ assert summarise(load_ledger(tmp_path)).total.cost_usd == pytest.approx(
+ logical.cost_usd + loser.cost_usd), "the loser is in the total exactly once"
def test_a_failed_calls_error_row_carries_what_was_billed(tmp_path: Path):
diff --git a/harness/tests/models/test_anthropic.py b/harness/tests/models/test_anthropic.py
index 086471d3..96ffbf36 100644
--- a/harness/tests/models/test_anthropic.py
+++ b/harness/tests/models/test_anthropic.py
@@ -299,3 +299,19 @@ def test_tool_call_part_in_user_message_ignored_but_assistant_first_gets_user_pr
[ChatMessage(role="assistant", parts=[ToolCallPart(id="t", name="n", arguments={})])]
)
assert msgs[0]["role"] == "user" and msgs[1]["role"] == "assistant"
+
+
+def test_a_failed_reply_carries_what_it_was_billed():
+ """Three raise sites built a real Usage from ``msg.usage`` and threw it away, so a
+ refusal / bad-JSON / empty reply looked FREE to the ledger and to the key pool."""
+ for script, req in (
+ ([msg([], stop="refusal")], ChatRequest(messages=[ChatMessage.user("x")])),
+ ([msg([text("not json")], stop="max_tokens")],
+ ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})),
+ ([msg([], stop="end_turn")] * 6, ChatRequest(messages=[ChatMessage.user("x")])),
+ ):
+ m, _ = make(script)
+ with pytest.raises(ModelError) as e:
+ m.generate(req)
+ assert e.value.usage.input_tokens == 100 and e.value.usage.output_tokens == 20
+
diff --git a/harness/tests/models/test_gemini.py b/harness/tests/models/test_gemini.py
index a616265e..81c4193a 100644
--- a/harness/tests/models/test_gemini.py
+++ b/harness/tests/models/test_gemini.py
@@ -639,3 +639,22 @@ def test_two_parallel_same_name_calls_keep_their_provider_ids():
("id-b", "leg"),
("id-a", "seat"),
]
+
+
+def test_max_output_tokens_eaten_by_thinking_carries_the_thought_tokens():
+ """The most expensive failure on Gemini 3: the reply is empty because thinking ate
+ the whole budget, and every thought token was billed. ``extract_candidate`` read
+ ``thoughts_token_count`` only to put it in the message and dropped the usage."""
+ resp = types.GenerateContentResponse(
+ model_version="gemini-3.7-flash",
+ candidates=[types.Candidate(content=types.Content(role="model", parts=[]),
+ finish_reason="MAX_TOKENS")],
+ usage_metadata=types.GenerateContentResponseUsageMetadata(
+ prompt_token_count=1000, candidates_token_count=0, thoughts_token_count=8000),
+ )
+ m, _log, _ = make_model([resp])
+ with pytest.raises(ModelError) as e:
+ m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
+ assert "exhausted by thinking" in str(e.value)
+ assert e.value.usage.thoughts_tokens == 8000 and e.value.usage.cost_usd > 0
+
diff --git a/harness/tests/models/test_openai.py b/harness/tests/models/test_openai.py
index 3ebe905d..2c84e3ce 100644
--- a/harness/tests/models/test_openai.py
+++ b/harness/tests/models/test_openai.py
@@ -273,3 +273,19 @@ def test_base_url_from_settings(monkeypatch):
assert str(c.base_url).startswith("http://localhost:8000/v1")
finally:
get_settings.cache_clear()
+
+
+def test_a_failed_reply_carries_what_it_was_billed():
+ """Both raise sites had ``usage`` in hand (built two lines above) and dropped it:
+ a bad-JSON or empty completion is billed exactly like a good one."""
+ for script, req in (
+ ([completion("not json", finish="length")],
+ ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})),
+ ([completion(None, finish="content_filter")], ChatRequest(messages=[ChatMessage.user("x")])),
+ ):
+ m, _ = make(script)
+ with pytest.raises(ModelError) as e:
+ m.generate(req)
+ u = e.value.usage
+ assert u.input_tokens == 100 and u.output_tokens == 20 and u.thoughts_tokens == 10
+
diff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.py
index 0ca35158..c200daa7 100644
--- a/harness/tests/models/test_retry.py
+++ b/harness/tests/models/test_retry.py
@@ -760,3 +760,29 @@ def test_a_broken_on_attempt_hook_never_breaks_the_call():
raise RuntimeError("accounting on fire")
assert _rotate(pool, lambda key: "ok", on_attempt=hook) == "ok"
+
+
+def test_a_budget_spent_at_the_storm_gate_issues_no_round_trip():
+ """A worker parked at a gate ANOTHER thread closed carries no ``last_err``, so the
+ guard at the top of the loop (which requires one) let it through: it woke past its
+ deadline and still spent a full round-trip (measured: 25 s on a 5 s budget)."""
+ now = {"t": 1000.0}
+ calls: list[str] = []
+
+ class Gate: # parks the caller for longer than the whole budget
+ def enter(self, deadline=None):
+ now["t"] += 30.0
+ return 30.0
+
+ def hit(self, retry_after=None):
+ return 0.0
+
+ def ok(self):
+ pass
+
+ with pytest.raises(ModelError) as e:
+ _rotate(KeyPool(["k1"], rpm_per_key=10_000), lambda key: calls.append(key),
+ max_total_s=5.0, monotonic=lambda: now["t"], storm_gate=Gate(), label="t")
+ assert calls == [], "a round-trip was issued after the budget was already spent"
+ assert "capacity" in str(e.value)
+