Official Codebase of 3DCodeVerse
Python
proc/cost: kill by group membership, and bill every paid round-trip
gaoypeng8432fa2c63a2
diff --git a/harness/codeverse/agents/api_agent.py b/harness/codeverse/agents/api_agent.pyindex 23325beb..99befead 100644--- a/harness/codeverse/agents/api_agent.py+++ b/harness/codeverse/agents/api_agent.py@@ -243,6 +243,7 @@ class _Loop:self.usage = self.usage + resp.usagereturn respexcept Exception as e: # ModelError or provider glitch+ self.usage = self.usage + (getattr(e, "usage", None) or Usage()) # billed anywayretryable = bool(getattr(e, "retryable", False))# the session deadline used to be checked only between turns, so a turn that# met it inside this loop still slept and re-entered the model's wholediff --git a/harness/codeverse/agents/transcript.py b/harness/codeverse/agents/transcript.pyindex d1b92277..ca685216 100644--- a/harness/codeverse/agents/transcript.py+++ b/harness/codeverse/agents/transcript.py@@ -17,6 +17,12 @@ from pydantic import BaseModelfrom codeverse.proc import append_jsonl_line, read_jsonl_lenient+#: per-session transcript caps. Rows are byte-capped individually but nothing capped+#: the file: a child emitting 100k lines wrote 27 MB, ~406 MB at the per-row cap. The+#: unabridged (bounded) stream still lands in stdout.json.+LINE_BUDGET_BYTES = 64 * 1024 * 1024+LINE_BUDGET_ROWS = 200_000+class Trajectory:"""Append-only writer for one agent session's files."""@@ -25,6 +31,9 @@ class Trajectory:self.dir = Path(directory)self.dir.mkdir(parents=True, exist_ok=True)self._lock = threading.Lock()+ self._budget = threading.Lock() # guards the counters below, never nested in _lock+ self._rows = self._bytes = 0+ self._spent = False# ------------------------------------------------------------------ paths@property@@ -59,8 +68,19 @@ class Trajectory:return pdef append(self, kind: str, **data: Any) -> None:- """Append one JSONL turn: ``{"t": epoch, "kind": kind, **data}``."""- append_jsonl_line(self.transcript_path, {"t": round(time.time(), 3), "kind": kind, **data}, self._lock)+ """Append one JSONL turn: ``{"t": epoch, "kind": kind, **data}``, until the+ session's byte/row budget trips — then one marker row and nothing more."""+ rec: dict[str, Any] = {"t": round(time.time(), 3), "kind": kind, **data}+ with self._budget:+ if self._spent:+ return+ self._rows += 1+ self._bytes += len(json.dumps(rec, ensure_ascii=False, default=str))+ self._spent = self._rows > LINE_BUDGET_ROWS or self._bytes > LINE_BUDGET_BYTES+ if self._spent:+ rec = {"t": rec["t"], "kind": "line_budget_exhausted",+ "rows": self._rows - 1, "bytes": self._bytes}+ append_jsonl_line(self.transcript_path, rec, self._lock)def write_result(self, result: BaseModel, **extra: Any) -> Path:"""Write ``result.json`` = AgentResult fields + any extra diagnostics."""diff --git a/harness/codeverse/cost/instrument.py b/harness/codeverse/cost/instrument.pyindex 14608405..a9e8f320 100644--- a/harness/codeverse/cost/instrument.py+++ b/harness/codeverse/cost/instrument.py@@ -142,21 +142,25 @@ class MeteredChatModel:# captured NOW: a hedge loser lands after this call returned, in a thread# with no context, and its row must still reach THIS run's ledgerledger = default_ledger()+ booked: list[int] = [] # round-trips that already billed themselves as "extra"def attempt_row(rec: AttemptRecord) -> None:- self._record_attempt(request, call_id, rec, ledger)+ if self._record_attempt(request, call_id, rec, ledger):+ booked.append(rec.attempt)t0 = time.perf_counter()try:with attempt_recording(attempt_row):resp = self._inner.generate(request)except BaseException as exc: # noqa: BLE001 - record the attempt, then re-raise- # a failed call may still have been billed: ModelError.usage carries what- # the provider charged (a bad-JSON reply costs like a good one)+ # a failed call may still have been billed: ModelError.usage carries what the+ # provider charged (a bad-JSON reply costs like a good one) — but only when+ # the round-trips did not already book it, or the last one counts twiceusage = getattr(exc, "usage", None)- self._record(usage if isinstance(usage, Usage)- else Usage(backend=self.provider, model=self.model),- request, outcome=_outcome(exc),+ usage = usage if isinstance(usage, Usage) else Usage(backend=self.provider, model=self.model)+ if booked:+ usage = Usage(backend=usage.backend, model=usage.model, latency_ms=usage.latency_ms)+ self._record(usage, request, outcome=_outcome(exc),ms=int((time.perf_counter() - t0) * 1000),attempts=getattr(exc, "attempts", 0), call_id=call_id)raise@@ -180,21 +184,26 @@ class MeteredChatModel:log.debug("cost: could not record %s: %s", request.label, e)def _record_attempt(self, request: ChatRequest, call_id: str, rec: AttemptRecord,- ledger: CostLedger | None) -> None:- """One ``source="attempt"`` row per round-trip. Excluded from every total- (``ledger.load_ledger`` / ``summarise``); they exist so a hedge loser's or- a charged-but-invalid reply's tokens are visible at all (audit 2026-08-27)."""+ ledger: CostLedger | None) -> bool:+ """One row per round-trip. A round-trip that was DISCARDED and still cost money+ (a billed-but-invalid reply, a hedge loser that landed) is money nothing else+ records, so it goes in as ``source="extra"`` and counts in every total; the rest+ are ``source="attempt"`` forensics, excluded because the call's logical row+ already carries their tokens. Returns True when it billed an ``extra``."""+ extra = bool(rec.discarded and rec.usage.cost_usd)try:record_call(rec.usage, label=request.label,backend=rec.usage.backend or self.provider,model=rec.usage.model or self.model,outcome="discarded" if rec.discarded else "ok",latency_ms=rec.usage.latency_ms,- source="attempt", key=_key_suffix(rec.key),+ source="extra" if extra else "attempt", key=_key_suffix(rec.key),call_id=call_id, attempt=rec.attempt, discarded=rec.discarded,ledger=ledger)except Exception as e: # pragma: no cover - accounting must never break a calllog.debug("cost: could not record attempt %s#%d: %s", request.label, rec.attempt, e)+ return False+ return extradef _key_suffix(key: object) -> str:diff --git a/harness/codeverse/cost/ledger.py b/harness/codeverse/cost/ledger.pyindex 22fec073..08c291a1 100644--- a/harness/codeverse/cost/ledger.py+++ b/harness/codeverse/cost/ledger.py@@ -349,9 +349,12 @@ def load_ledger(path: str | Path, *, include_attempts: bool = False) -> list[Caldebug log — a truncated last line never loses the rest of the file.``source="attempt"`` rows (one per round-trip, ``instrument.MeteredChatModel``)- are left out unless ``include_attempts=True``: the winning round-trip's tokens- are already on the call's logical row, so every aggregate built on this reader- (``summarise``, ``reconstruct``, the CLI) keeps counting each call exactly once."""+ are left out unless ``include_attempts=True``: their tokens are already on the+ call's logical row, so every aggregate built on this reader (``summarise``,+ ``reconstruct``, the CLI) keeps counting each call exactly once. This is the ONLY+ place that filter lives — ``summarise`` used to repeat it, which made+ ``include_attempts=True`` summarise to $0. Paid-but-discarded round-trips are+ ``source="extra"`` instead and always count: nothing else records them."""p = Path(path)if p.is_dir():found = existing_ledger_path(p)@@ -383,8 +386,6 @@ def summarise(rows: Iterable[CallCost], *, dimensions: Sequence[str] = DIMENSIONof :class:`CallCost` (defaults to :data:`DIMENSIONS`)."""out = Summary()for row in rows:- if row.source == "attempt":- continue # per-round-trip forensic rows; their call is already a rowout.total.add(row)for dim in dimensions:bucket = out.by.setdefault(dim, {}).setdefault(_key(row, dim), CostBucket(key=_key(row, dim)))diff --git a/harness/codeverse/cost/types.py b/harness/codeverse/cost/types.pyindex d79414e6..ac95c4e0 100644--- a/harness/codeverse/cost/types.py+++ b/harness/codeverse/cost/types.py@@ -184,7 +184,9 @@ class CallCost(BaseModel):cache_hit: bool = Falseoutcome: str = "ok" # ok | error | timeout | budget | degraded | discardedn_calls: int = 1 # >1 when a row aggregates a whole agent session- source: str = "live" # live | record | events | transcript | stdout | residual | session | attempt+ #: extra = a round-trip that was discarded yet billed (hedge loser, charged-but-invalid+ #: reply); nothing else records it, so unlike ``attempt`` it counts in every total+ source: str = "live" # live | record | events | transcript | stdout | residual | session | attempt | extra#: which API key served the call, as its last 4 chars ("…ab12") — never the key#: itself; "" for a failed call, a session row, or a row older than 2026-08-26key: str = ""@@ -198,8 +200,8 @@ class CallCost(BaseModel):#: 1-based round-trip number within its logical call (``source="attempt"`` rows)attempt: int = 0#: True when this round-trip's response was thrown away: a hedge loser, or a- #: failed try of a call that went on to retry. Real money, but never part of a- #: total — ``ledger.load_ledger`` / ``summarise`` exclude ``source="attempt"``+ #: failed try of a call that went on to retry. A paid one is ``source="extra"``+ #: and counts; a free one is ``source="attempt"`` and is filtered outdiscarded: bool = False@propertydiff --git a/harness/codeverse/judges/pairwise.py b/harness/codeverse/judges/pairwise.pyindex 1475b04f..ea4cf11e 100644--- a/harness/codeverse/judges/pairwise.py+++ b/harness/codeverse/judges/pairwise.py@@ -153,7 +153,7 @@ class PairwiseJudge:try:resp = self.model.generate(req)except ModelError as e:- return Usage(), None, None, f"ModelError: {e}"+ return getattr(e, "usage", None) or Usage(), None, None, f"ModelError: {e}"try:payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)reply = PairwiseReply.model_validate(payload)diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.pyindex 6180e43e..b2c6f233 100644--- a/harness/codeverse/judges/vlm_judge.py+++ b/harness/codeverse/judges/vlm_judge.py@@ -216,6 +216,7 @@ class VlmJudge:try:resp: ChatResponse = self.model.generate(req)except ModelError as e:+ usage = usage + (getattr(e, "usage", None) or Usage()) # the provider billed itlast = f"ModelError(attempt {attempt}): {e}"log.warning("judge %s: %s", req.label, last)if not e.retryable and attempt >= 2:diff --git a/harness/codeverse/models/anthropic.py b/harness/codeverse/models/anthropic.pyindex 7a8b2304..6e2090ea 100644--- a/harness/codeverse/models/anthropic.py+++ b/harness/codeverse/models/anthropic.py@@ -144,7 +144,7 @@ class AnthropicModel:if stop == "refusal":raise ModelError(f"Anthropic refused the request: {getattr(msg, 'stop_details', None)}",- retryable=False,+ retryable=False, usage=usage, # a refusal is billed like any other reply)parsed: Any = Noneif request.response_schema is not None:@@ -164,10 +164,11 @@ class AnthropicModel:except JsonParseError as exc:raise ModelError(f"structured output missing (stop_reason={stop}): {exc}",- retryable=stop != "max_tokens",+ retryable=stop != "max_tokens", usage=usage,) from excif not text and not calls and parsed is None:- raise ModelError(f"Anthropic returned no content (stop_reason={stop})", retryable=True)+ raise ModelError(f"Anthropic returned no content (stop_reason={stop})",+ retryable=True, usage=usage)raw: dict[str, Any] = {"stop_reason": stop, "id": msg.id, "model": msg.model}return ChatResponse(text=text, parsed=parsed, tool_calls=calls, finish_reason=stop, usage=usage, raw=rawdiff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.pyindex 63a3604f..554a2287 100644--- a/harness/codeverse/models/gemini.py+++ b/harness/codeverse/models/gemini.py@@ -264,6 +264,7 @@ class GeminiModel:budget = RETRY_DEADLINE_S if request.max_wait_s is None else min(RETRY_DEADLINE_S, float(request.max_wait_s))deadline = time.monotonic() + budget # mirrors rotate_with_retries' own deadlinestats: dict[str, Any] = {}+ wasted: list[Usage] = [] # discarded round-trips the provider still billedtry:resp = rotate_with_retries(self.pool,@@ -286,7 +287,7 @@ class GeminiModel:storm_gate=self.storm_gate,label=f"gemini {self.model}",stats=stats,- on_attempt=self._attempt_hook(),+ on_attempt=self._attempt_hook(wasted),**({} if self.storm_attempts is None else {"storm_attempts": self.storm_attempts}),)except ModelError as err:@@ -295,6 +296,10 @@ class GeminiModel:# how hard the call was, next to which key served it (``_once``): ledger fieldsresp.raw["attempts"] = int(stats.get("attempts", 0))resp.raw["hedged"] = int(stats.get("hedged", 0))+ if wasted:+ # money the winner's usage does not include; a hedge loser that lands AFTER+ # this returns reaches the ledger through the sink, never this field+ resp.raw["wasted_usage"] = sum(wasted[1:], wasted[0])return respdef _config(self, request: ChatRequest, warnings: list[str]) -> types.GenerateContentConfig:@@ -316,19 +321,20 @@ class GeminiModel:want_s = min(self.timeout_s, max(HTTP_TIMEOUT_FLOOR_S, remaining))return clip_timeout(config, int(want_s * 1000))- def _attempt_hook(self) -> OnAttempt | None:- """The per-attempt ledger recorder, when the metering layer installed one- (``cost.instrument.MeteredChatModel``). Captured once per logical call so- a hedge loser landing later, in its own thread, still reports through it.- Imported lazily: the models package must not import the cost package at- module level (``cost.ledger`` imports ``models.pricing``)."""+ def _attempt_hook(self, wasted: list[Usage]) -> OnAttempt:+ """Per-round-trip observer. It collects every DISCARDED round-trip that still+ cost money into ``wasted`` (published on ``resp.raw["wasted_usage"]`` — the only+ place a budget guard can see that money) and feeds the ledger's per-attempt sink+ when the metering layer installed one (``cost.instrument.MeteredChatModel``).+ Captured once per logical call so a hedge loser landing later, in its own thread,+ still reports through it. Imported lazily: the models package must not import+ the cost package at module level (``cost.ledger`` imports ``models.pricing``)."""try:from codeverse.cost.context import AttemptRecord, attempt_sink++ sink = attempt_sink()except Exception: # pragma: no cover - accounting must never break a call- return None- sink = attempt_sink()- if sink is None:- return None+ AttemptRecord = sink = Nonedef on_attempt(t: _Try, no: int, discarded: bool) -> None:if t.err is None:@@ -337,8 +343,11 @@ class GeminiModel:else:usage = getattr(t.err, "usage", None) or Usage()outcome, error = t.outcome, str(t.err)- sink(AttemptRecord(attempt=no, key=t.key, outcome=outcome,- discarded=discarded, usage=usage, error=error))+ if discarded and usage.cost_usd:+ wasted.append(usage)+ if sink is not None:+ sink(AttemptRecord(attempt=no, key=t.key, outcome=outcome,+ discarded=discarded, usage=usage, error=error))return on_attempt@@ -355,11 +364,11 @@ class GeminiModel:resp = client.models.generate_content(model=self.model, contents=contents, config=config)- text, calls, finish = extract_candidate(resp)- usage = parse_usage(resp, self.model)+ usage = parse_usage(resp, self.model) # BEFORE extract_candidate: its raises carry itusage.latency_ms = sw.ms- usage.tool_calls = len(calls)usage.cost_usd = estimate_cost("gemini", self.model, usage)+ text, calls, finish = extract_candidate(resp, usage)+ usage.tool_calls = len(calls)parsed: Any = Noneif request.response_schema is not None and not calls:try:diff --git a/harness/codeverse/models/gemini_convert.py b/harness/codeverse/models/gemini_convert.pyindex dd547234..4c2075c2 100644--- a/harness/codeverse/models/gemini_convert.py+++ b/harness/codeverse/models/gemini_convert.py@@ -205,17 +205,21 @@ def parse_usage(resp: types.GenerateContentResponse, model: str) -> Usage:)-def extract_candidate(resp: types.GenerateContentResponse) -> tuple[str, list[ToolCallPart], str]:+def extract_candidate(+ resp: types.GenerateContentResponse, usage: Usage | None = None+) -> tuple[str, list[ToolCallPart], str]:"""→ ``(text, tool_calls, finish_reason)``. Raises ``ModelError`` on blocked- prompts (non-retryable) or empty candidates (retryable)."""+ prompts (non-retryable) or empty candidates (retryable); ``usage`` (what the caller+ already parsed off ``resp``) rides on the error, because every one of these+ failures was billed."""pf = resp.prompt_feedbackif pf is not None and pf.block_reason:raise ModelError(f"prompt blocked by Gemini: {pf.block_reason} {pf.block_reason_message or ''}".strip(),- retryable=False,+ retryable=False, usage=usage,)if not resp.candidates:- raise ModelError("Gemini returned no candidates", retryable=True)+ raise ModelError("Gemini returned no candidates", retryable=True, usage=usage)cand = resp.candidates[0]finish = str(cand.finish_reason.value if cand.finish_reason else "") or "UNKNOWN"texts: list[str] = []@@ -236,16 +240,18 @@ def extract_candidate(resp: types.GenerateContentResponse) -> tuple[str, list[Toif not text and not calls:if finish in FATAL_FINISH:raise ModelError(- f"Gemini produced no content (finish_reason={finish})", retryable=False+ f"Gemini produced no content (finish_reason={finish})", retryable=False, usage=usage)if finish == "MAX_TOKENS":# Gemini 3 models always think (budget 0 is accepted but ignored); a tiny- # max_output_tokens is eaten by thoughts. Retrying cannot help.+ # max_output_tokens is eaten by thoughts. Retrying cannot help — and this is+ # the most expensive failure on Gemini 3: every thought token was billed.thoughts = resp.usage_metadata.thoughts_token_count if resp.usage_metadata else Noneraise ModelError(f"Gemini produced no content: max_output_tokens exhausted by thinking "f"(thoughts_tokens={thoughts}); raise max_output_tokens",- retryable=False,+ retryable=False, usage=usage,)- raise ModelError(f"Gemini produced no content (finish_reason={finish})", retryable=True)+ raise ModelError(f"Gemini produced no content (finish_reason={finish})",+ retryable=True, usage=usage)return text, calls, finishdiff --git a/harness/codeverse/models/gemini_image.py b/harness/codeverse/models/gemini_image.pyindex 49c747d8..2486556f 100644--- a/harness/codeverse/models/gemini_image.py+++ b/harness/codeverse/models/gemini_image.py@@ -304,11 +304,14 @@ class GeminiImageModel:client = self._clients._client(key)with Stopwatch() as sw:resp = client.models.generate_content(model=model, contents=contents, config=config)+ usage = parse_usage(resp, model) # parsed first: the raises below were billed too+ usage.backend = "gemini-image"+ usage.latency_ms = sw.mspf = resp.prompt_feedbackif pf is not None and pf.block_reason:- raise ModelError(f"image prompt blocked: {pf.block_reason}", retryable=False)+ raise ModelError(f"image prompt blocked: {pf.block_reason}", retryable=False, usage=usage)if not resp.candidates:- raise ModelError("image model returned no candidates", retryable=True)+ raise ModelError("image model returned no candidates", retryable=True, usage=usage)cand = resp.candidates[0]finish = str(cand.finish_reason.name if cand.finish_reason is not None else "")images: list[Image.Image] = []@@ -320,11 +323,10 @@ class GeminiImageModel:images.append(img.convert("RGB"))if not images:if finish in FATAL_FINISH:- raise ModelError(f"image generation refused (finish_reason={finish})", retryable=False)- raise ModelError(f"image model returned no image (finish_reason={finish or 'unknown'})", retryable=True)- usage = parse_usage(resp, model)- usage.backend = "gemini-image"- usage.latency_ms = sw.ms+ raise ModelError(f"image generation refused (finish_reason={finish})",+ retryable=False, usage=usage)+ raise ModelError(f"image model returned no image (finish_reason={finish or 'unknown'})",+ retryable=True, usage=usage)usage.cost_usd = image_cost(model, usage, len(images))if int(size) not in (0, 1024) and images[0].size != (size, size):images = [im.resize((int(size), int(size)), Image.LANCZOS) for im in images]diff --git a/harness/codeverse/models/openai.py b/harness/codeverse/models/openai.pyindex 8d609857..bf1ebe16 100644--- a/harness/codeverse/models/openai.py+++ b/harness/codeverse/models/openai.py@@ -169,12 +169,12 @@ class OpenAIModel:except JsonParseError as exc:raise ModelError(f"structured output is not valid JSON (finish_reason={finish}): {exc}",- retryable=finish != "length",+ retryable=finish != "length", usage=usage, # billed like a good reply) from excif not text and not calls:raise ModelError(f"OpenAI returned no content (finish_reason={finish})",- retryable=finish != "content_filter",+ retryable=finish != "content_filter", usage=usage,)raw: dict[str, Any] = {"finish_reason": finish,diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.pyindex 6a74bd49..f111c568 100644--- a/harness/codeverse/models/retry.py+++ b/harness/codeverse/models/retry.py@@ -300,6 +300,10 @@ def rotate_with_retries[T](def out_of_time() -> bool:return monotonic() >= deadline+ def clip(delay: float) -> float:+ """A sleep never outlives the budget: waking only to give up helps nobody."""+ return delay if deadline == float("inf") else min(delay, max(0.0, deadline - monotonic()))+def is_storm(err: ModelError) -> bool:return bool(err.retryable and err.status in (503, 529))@@ -317,6 +321,13 @@ def rotate_with_retries[T](exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())if storm_gate is not None:storm_gate.enter(None if deadline == float("inf") else deadline)+ if out_of_time():+ # the gate held us past the budget. The check at the top of the loop+ # cannot cover this: a worker parked at a gate ANOTHER thread closed+ # has no last_err, and would go on to spend a full round-trip.+ log.warning("%s budget of %.0f s spent waiting at the storm gate", label, max_total_s)+ raise (last_err or classify(TimeoutError(+ f"{label}: retry budget spent waiting for capacity"))) from last_excbudget_left = None if deadline == float("inf") else max(0.0, deadline - monotonic())try:# the key/slot wait must fit the remaining budget, never outlive it@@ -394,11 +405,7 @@ def rotate_with_retries[T](# jitter FIRST, cap LAST: capping before a >1 jitter factor let a# "<=5 s" wait land at 6.25 s in the wild (observed 2026-08-24).raw = base_delay * (2 ** min(storm, 8)) * (0.75 + 0.5 * random.random())- delay = min(storm_max_delay, raw)- if deadline != float("inf"):- # clip the storm sleep to the remainder: sleeping past the- # deadline only to give up on waking helps nobody- delay = min(delay, max(0.0, deadline - monotonic()))+ delay = clip(min(storm_max_delay, raw))if storm_gate is not None:# tell every other worker as well: the next one to arrive parks at# the gate instead of spending its own round-trip to find the storm@@ -424,7 +431,7 @@ def rotate_with_retries[T](log.warning("%s key …%s throttled (%s); rotating to a fresh key", label, key[-4:], err)- sleep(0.5)+ sleep(clip(0.5))continueif not err.retryable or attempt >= max_attempts:bench()@@ -437,7 +444,7 @@ def rotate_with_retries[T]("%s attempt %d/%d failed (%s); retrying in %.1fs",label, attempt, max_attempts, err, delay,)- sleep(delay)+ sleep(clip(delay))assert last_err is not Nonebench()raise last_errdiff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.pyindex ffc014e7..3020bdee 100644--- a/harness/codeverse/proc.py+++ b/harness/codeverse/proc.py@@ -90,6 +90,17 @@ def run_subprocess(#: for every build/render path that funnels through here.DRAIN_TIMEOUT_S = 10.0_ABANDONED = "(output truncated: a detached descendant still held the pipe past the drain window)"+_STILL_HELD = "(output truncated: a process in the group still held the pipe past the drain window)"+++def _group_alive(pgid: int) -> bool:+ """True while ANY member of process group ``pgid`` exists (the leader may be gone)."""+ try:+ os.killpg(pgid, 0)+ except (ProcessLookupError, PermissionError):+ return False+ return True+#: per-stream cap on captured output: the first ``budget // 2`` bytes and the last#: ``budget - budget // 2`` bytes are kept, with a truncation marker in between.@@ -192,7 +203,7 @@ class ManagedProcess:self._on_line = on_lineself._grace_s = grace_sself._stop = threading.Event() # tells an abandoned pump to quit feeding- self._abandoned = False+ self._abandoned = "" # the truncation note to append, once we know which oneself._sinks = {"stdout": _BoundedSink(stream_budget), "stderr": _BoundedSink(stream_budget)}self.proc: subprocess.Popen[bytes] = subprocess.Popen(list(cmd),@@ -269,7 +280,7 @@ class ManagedProcess:"""Decoded, bounded stderr (+ a note when an escaped descendant held the pipe)."""text = self._sinks["stderr"].text()if self._abandoned:- return f"{text}\n{_ABANDONED}" if text else _ABANDONED+ return f"{text}\n{self._abandoned}" if text else self._abandonedreturn text# ------------------------------------------------------------- kills and teardown@@ -285,30 +296,36 @@ class ManagedProcess:"""SIGTERM the group, escalate to SIGKILL after ``grace_s`` (default: theconstructor's ``grace_s`` — the old ``watchdog.kill_process_group`` contract)."""grace = self._grace_s if grace_s is None else grace_s- if self.proc.poll() is not None:- returntry:+ # killpg after the leader was reaped is a microsecond pid-reuse race the+ # code has always taken; a live grandchild is the certain cost of not doing itos.killpg(self.proc.pid, signal.SIGTERM)except ProcessLookupError:returnexcept PermissionError: # cannot signal the group — go for the child itselfself.proc.terminate()- try:- self.proc.wait(timeout=grace)- except subprocess.TimeoutExpired:- self.kill()- with contextlib.suppress(subprocess.TimeoutExpired):- self.proc.wait(timeout=self._grace_s)+ deadline = time.monotonic() + grace+ while time.monotonic() < deadline:+ self.proc.poll() # reap first: an unreaped zombie leader keeps the group "alive"+ if not _group_alive(self.proc.pid):+ return+ time.sleep(0.02)+ self.kill()+ with contextlib.suppress(subprocess.TimeoutExpired):+ self.proc.wait(timeout=self._grace_s)def __enter__(self) -> ManagedProcess:return selfdef __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: object) -> None:- # Any in-flight exception (KeyboardInterrupt included) or a child still- # running at scope exit: the group dies HERE. Ctrl-C's SIGINT never reaches- # a start_new_session child, so this is the only thing standing between an- # interrupt and an orphaned blender/node/chrome tree. Never swallows exc.- if self.proc.poll() is None:+ # Any in-flight exception (KeyboardInterrupt included) or a group still alive at+ # scope exit: the group dies HERE. Ctrl-C's SIGINT never reaches a+ # start_new_session child, so this is all that stands between an interrupt and an+ # orphaned blender/node/chrome tree. Never swallows exc. LEADER liveness is the+ # wrong question — one that exits 0 leaves its same-group grandchildren running;+ # reap it (a zombie still counts as a member), then ask if the GROUP exists.+ self.proc.poll()+ if _group_alive(self.proc.pid):self.terminate()self._drain_and_reap()@@ -318,11 +335,11 @@ class ManagedProcess:self._writer.join(timeout=max(0.0, deadline - time.monotonic()))for t in self._pumps:t.join(timeout=max(0.0, deadline - time.monotonic()))- if any(t.is_alive() for t in self._pumps):- self._stop.set() # the parked daemon pump self-cleans when its read returns- self._abandoned = Truewith contextlib.suppress(subprocess.TimeoutExpired):self.proc.wait(timeout=DRAIN_TIMEOUT_S) # reap — a zombie survives killpg+ if any(t.is_alive() for t in self._pumps):+ self._stop.set() # the parked daemon pump self-cleans when its read returns+ self._abandoned = _STILL_HELD if _group_alive(self.proc.pid) else _ABANDONEDdiff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.pyindex 06cdcb1e..9ef53dc8 100644--- a/harness/codeverse/tracks/generation.py+++ b/harness/codeverse/tracks/generation.py@@ -216,7 +216,10 @@ def generate_files(label=f"{task.label}:r{task.round:02d}",)resp = model.generate(req)- usage = resp.usage+ # + every paid round-trip the winner does not represent (a billed-but-invalid+ # response, a late hedge loser): the ledger records those as source="extra";+ # without this the guard never sees them (measured $0.18 on one call).+ usage = resp.usage + (resp.raw.get("wasted_usage") or Usage())# book the money WITHOUT enforcing: the response is already paid for, and raising# here would discard it before transcript/parse/write_files. The ceiling is# enforced at the round's phase boundary instead (steps._run_phase).diff --git a/harness/tests/agents/test_transcript.py b/harness/tests/agents/test_transcript.pynew file mode 100644index 00000000..d0cc4185--- /dev/null+++ b/harness/tests/agents/test_transcript.py@@ -0,0 +1,27 @@+"""codeverse/agents/transcript.py — the per-session transcript budget."""++from __future__ import annotations++from pathlib import Path++from codeverse.agents import transcript as tr+from codeverse.agents.transcript import Trajectory+++def test_the_transcript_stops_at_its_budget_with_a_marker_row(tmp_path: Path, monkeypatch):+ """Rows were byte-capped one by one but the FILE was not: a child emitting 100k+ lines wrote a 27 MB transcript.jsonl (~406 MB at the 4000-char row cap)."""+ monkeypatch.setattr(tr, "LINE_BUDGET_ROWS", 5)+ t = Trajectory(tmp_path / "traj")+ for i in range(20):+ t.append("stdout", line=f"line {i}")+ rows = t.read_transcript()+ assert [r["kind"] for r in rows] == ["stdout"] * 5 + ["line_budget_exhausted"]+ assert rows[-1]["rows"] == 5++ monkeypatch.setattr(tr, "LINE_BUDGET_BYTES", 400)+ t2 = Trajectory(tmp_path / "traj2")+ for _ in range(50):+ t2.append("stdout", line="x" * 100)+ rows2 = t2.read_transcript()+ assert len(rows2) < 6 and rows2[-1]["kind"] == "line_budget_exhausted"diff --git a/harness/tests/core/test_proc.py b/harness/tests/core/test_proc.pyindex efd2b395..57d954ff 100644--- a/harness/tests/core/test_proc.py+++ b/harness/tests/core/test_proc.py@@ -43,20 +43,51 @@ def test_run_subprocess_stdin(tmp_path: Path):assert r.stdout == "hello"+def _assert_pid_gone(pid: int, timeout_s: float = 5.0) -> None:+ deadline = time.monotonic() + timeout_s+ while time.monotonic() < deadline:+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return # gone — the group kill reached it+ time.sleep(0.05)+ raise AssertionError(f"pid {pid} survived the group kill")++def test_timeout_kills_the_whole_process_group(tmp_path: Path):"""A timed-out child AND its grandchildren die (killpg on the new session)."""r = run_subprocess(["bash", "-c", "sleep 30 & echo $!; wait"], cwd=tmp_path, timeout_s=0.4)assert r.timed_outassert r.returncode != 0- grandchild = int(r.stdout.strip().splitlines()[0])- deadline = time.monotonic() + 3.0- while time.monotonic() < deadline:- try:- os.kill(grandchild, 0)- except ProcessLookupError:- return # gone — group kill worked- time.sleep(0.05)- raise AssertionError(f"grandchild sleep (pid {grandchild}) survived the group kill")+ _assert_pid_gone(int(r.stdout.strip().splitlines()[0]))+++def test_a_leader_that_exits_0_still_takes_its_grandchildren_down(tmp_path: Path):+ """``__exit__`` asked "is the LEADER still running?", so a leader that exited 0 left+ its same-group grandchildren (pgid == leader pid) alive — the exact incident this+ module exists to end. Detached-stdio shape: nothing holds the pipes."""+ pid_file = tmp_path / "pid"+ r = run_subprocess(["bash", "-c", f"sleep 300 >/dev/null 2>&1 & echo $! > {pid_file}; exit 0"],+ cwd=tmp_path, timeout_s=10)+ assert r.returncode == 0 and not r.timed_out+ _assert_pid_gone(int(pid_file.read_text()))+++def test_a_leader_that_exits_0_does_not_burn_the_drain_window(tmp_path: Path, monkeypatch):+ """Same shape with the grandchild holding the INHERITED pipes: the drain used to+ wait out DRAIN_TIMEOUT_S for a process nobody had signalled, then blame a+ "detached descendant" that had never left the group."""+ import codeverse.proc as proc_mod++ monkeypatch.setattr(proc_mod, "DRAIN_TIMEOUT_S", 3.0)+ pid_file = tmp_path / "pid"+ t0 = time.monotonic()+ r = run_subprocess(["bash", "-c", f"sleep 300 & echo $! > {pid_file}; echo done"],+ cwd=tmp_path, timeout_s=10)+ assert r.returncode == 0 and "done" in r.stdout+ assert time.monotonic() - t0 < 2.5, "the drain window was burned on a live group member"+ assert "detached descendant" not in r.stderr+ _assert_pid_gone(int(pid_file.read_text()))def test_a_detached_descendant_holding_the_pipes_cannot_extend_the_timeout(tmp_path: Path, monkeypatch):diff --git a/harness/tests/cost/test_instrument.py b/harness/tests/cost/test_instrument.pyindex 7690f7ff..387c7931 100644--- a/harness/tests/cost/test_instrument.py+++ b/harness/tests/cost/test_instrument.py@@ -358,10 +358,10 @@ def test_per_key_buckets_and_tries_per_call(tmp_path: Path):table = bucket_rows(keyed_buckets(audit), total=1.0)assert BUCKET_HEADERS[-1] == "tries/call" and table[0][-1] == "2.00" and table[1][-1] == "1.00"# ------------------------------------------------------- per-attempt rows (audit 2026-08-27)-def test_a_bad_json_reply_reaches_the_ledger_as_a_discarded_attempt_row(tmp_path: Path):+def test_a_billed_but_invalid_attempt_is_in_the_total_exactly_once(tmp_path: Path):"""A charged-but-invalid reply used to vanish: the logical row carries only the- winner's usage. Its tokens now land as a source="attempt" row that joins the- logical row by call_id — and never double-counts in any total."""+ winner's usage and its own row was ``source="attempt"``, which every total+ filtered out. It is now ``source="extra"`` — counted once, joined by call_id."""from codeverse.cost.ledger import summarisefrom tests.models.test_gemini import make_model, text_response@@ -371,23 +371,24 @@ def test_a_bad_json_reply_reaches_the_ledger_as_a_discarded_attempt_row(tmp_pathChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"},label="planner"))assert resp.parsed == {"ok": True}- (logical,) = load_ledger(tmp_path) # attempt rows are filtered by default+ rows = load_ledger(tmp_path) # "attempt" forensics are filtered here; "extra" is money+ bad, = [r for r in rows if r.source == "extra"]+ logical, = [r for r in rows if r.source == "live"]assert logical.outcome == "ok" and logical.attempts == 2 and logical.call_id- rows = load_ledger(tmp_path, include_attempts=True)- attempts = sorted((r for r in rows if r.source == "attempt"), key=lambda r: r.attempt)- assert [r.attempt for r in attempts] == [1, 2]- bad, win = attemptsassert bad.discarded and bad.outcome == "discarded" and bad.input_tokens == 100- assert not win.discarded and win.outcome == "ok"- assert bad.call_id == win.call_id == logical.call_id- assert summarise(rows).total.input_tokens == logical.input_tokens, \- "attempt rows never double-count"+ assert bad.call_id == logical.call_id and bad.cost_usd > 0+ total = summarise(rows).total+ assert total.input_tokens == logical.input_tokens + bad.input_tokens+ assert summarise(load_ledger(tmp_path, include_attempts=True)).total.input_tokens \+ == total.input_tokens + 100, "include_attempts adds the winner's forensic row, once"+ win, = [r for r in load_ledger(tmp_path, include_attempts=True) if r.source == "attempt"]+ assert not win.discarded and win.attempt == 2def test_a_hedge_losers_tokens_reach_the_ledger_when_it_lands(tmp_path: Path):"""The loser of a raced 503 retry keeps running after the winner returned; when- it lands, its billed tokens must appear as a discarded attempt row — before,- that money left no trace anywhere."""+ it lands, its billed tokens must appear in the ledger TOTAL — before, that money+ left no trace anywhere, and then no trace in any total."""import threadingimport time as _time@@ -420,16 +421,19 @@ def test_a_hedge_losers_tokens_reach_the_ledger_when_it_lands(tmp_path: Path):deadline = _time.monotonic() + 5.0loser = Nonewhile _time.monotonic() < deadline:- rows = load_ledger(tmp_path, include_attempts=True)- loser = next((r for r in rows if r.source == "attempt" and r.key == "…k2"), None)+ rows = load_ledger(tmp_path)+ loser = next((r for r in rows if r.source == "extra" and r.key == "…k2"), None)if loser is not None:break_time.sleep(0.01)assert loser is not None, "the loser's row never arrived"assert loser.discarded and loser.input_tokens == 100, "the loser's paid tokens are on the ledger"- (logical,) = load_ledger(tmp_path)+ logical, = [r for r in load_ledger(tmp_path) if r.source == "live"]# endswith, not ==: the 2-char fake key is shorter than the …last-4 redactionassert logical.key.endswith("k3") and logical.call_id == loser.call_id+ from codeverse.cost.ledger import summarise+ assert summarise(load_ledger(tmp_path)).total.cost_usd == pytest.approx(+ logical.cost_usd + loser.cost_usd), "the loser is in the total exactly once"def test_a_failed_calls_error_row_carries_what_was_billed(tmp_path: Path):diff --git a/harness/tests/models/test_anthropic.py b/harness/tests/models/test_anthropic.pyindex 086471d3..96ffbf36 100644--- a/harness/tests/models/test_anthropic.py+++ b/harness/tests/models/test_anthropic.py@@ -299,3 +299,19 @@ def test_tool_call_part_in_user_message_ignored_but_assistant_first_gets_user_pr[ChatMessage(role="assistant", parts=[ToolCallPart(id="t", name="n", arguments={})])])assert msgs[0]["role"] == "user" and msgs[1]["role"] == "assistant"+++def test_a_failed_reply_carries_what_it_was_billed():+ """Three raise sites built a real Usage from ``msg.usage`` and threw it away, so a+ refusal / bad-JSON / empty reply looked FREE to the ledger and to the key pool."""+ for script, req in (+ ([msg([], stop="refusal")], ChatRequest(messages=[ChatMessage.user("x")])),+ ([msg([text("not json")], stop="max_tokens")],+ ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})),+ ([msg([], stop="end_turn")] * 6, ChatRequest(messages=[ChatMessage.user("x")])),+ ):+ m, _ = make(script)+ with pytest.raises(ModelError) as e:+ m.generate(req)+ assert e.value.usage.input_tokens == 100 and e.value.usage.output_tokens == 20+diff --git a/harness/tests/models/test_gemini.py b/harness/tests/models/test_gemini.pyindex a616265e..81c4193a 100644--- a/harness/tests/models/test_gemini.py+++ b/harness/tests/models/test_gemini.py@@ -639,3 +639,22 @@ def test_two_parallel_same_name_calls_keep_their_provider_ids():("id-b", "leg"),("id-a", "seat"),]+++def test_max_output_tokens_eaten_by_thinking_carries_the_thought_tokens():+ """The most expensive failure on Gemini 3: the reply is empty because thinking ate+ the whole budget, and every thought token was billed. ``extract_candidate`` read+ ``thoughts_token_count`` only to put it in the message and dropped the usage."""+ resp = types.GenerateContentResponse(+ model_version="gemini-3.7-flash",+ candidates=[types.Candidate(content=types.Content(role="model", parts=[]),+ finish_reason="MAX_TOKENS")],+ usage_metadata=types.GenerateContentResponseUsageMetadata(+ prompt_token_count=1000, candidates_token_count=0, thoughts_token_count=8000),+ )+ m, _log, _ = make_model([resp])+ with pytest.raises(ModelError) as e:+ m.generate(ChatRequest(messages=[ChatMessage.user("x")]))+ assert "exhausted by thinking" in str(e.value)+ assert e.value.usage.thoughts_tokens == 8000 and e.value.usage.cost_usd > 0+diff --git a/harness/tests/models/test_openai.py b/harness/tests/models/test_openai.pyindex 3ebe905d..2c84e3ce 100644--- a/harness/tests/models/test_openai.py+++ b/harness/tests/models/test_openai.py@@ -273,3 +273,19 @@ def test_base_url_from_settings(monkeypatch):assert str(c.base_url).startswith("http://localhost:8000/v1")finally:get_settings.cache_clear()+++def test_a_failed_reply_carries_what_it_was_billed():+ """Both raise sites had ``usage`` in hand (built two lines above) and dropped it:+ a bad-JSON or empty completion is billed exactly like a good one."""+ for script, req in (+ ([completion("not json", finish="length")],+ ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})),+ ([completion(None, finish="content_filter")], ChatRequest(messages=[ChatMessage.user("x")])),+ ):+ m, _ = make(script)+ with pytest.raises(ModelError) as e:+ m.generate(req)+ u = e.value.usage+ assert u.input_tokens == 100 and u.output_tokens == 20 and u.thoughts_tokens == 10+diff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.pyindex 0ca35158..c200daa7 100644--- a/harness/tests/models/test_retry.py+++ b/harness/tests/models/test_retry.py@@ -760,3 +760,29 @@ def test_a_broken_on_attempt_hook_never_breaks_the_call():raise RuntimeError("accounting on fire")assert _rotate(pool, lambda key: "ok", on_attempt=hook) == "ok"+++def test_a_budget_spent_at_the_storm_gate_issues_no_round_trip():+ """A worker parked at a gate ANOTHER thread closed carries no ``last_err``, so the+ guard at the top of the loop (which requires one) let it through: it woke past its+ deadline and still spent a full round-trip (measured: 25 s on a 5 s budget)."""+ now = {"t": 1000.0}+ calls: list[str] = []++ class Gate: # parks the caller for longer than the whole budget+ def enter(self, deadline=None):+ now["t"] += 30.0+ return 30.0++ def hit(self, retry_after=None):+ return 0.0++ def ok(self):+ pass++ with pytest.raises(ModelError) as e:+ _rotate(KeyPool(["k1"], rpm_per_key=10_000), lambda key: calls.append(key),+ max_total_s=5.0, monotonic=lambda: now["t"], storm_gate=Gate(), label="t")+ assert calls == [], "a round-trip was issued after the budget was already spent"+ assert "capacity" in str(e.value)+