Official Codebase of 3DCodeVerse
Python
docs: the time audit — 51 storm-day runs vs 52 baseline, every stage and call; waiting is the held 503 round-trip, not the sleep
gaoypeng399894ffb898
diff --git a/harness/bench/time_audit/calls.py b/harness/bench/time_audit/calls.pynew file mode 100644index 00000000..fb4266e3--- /dev/null+++ b/harness/bench/time_audit/calls.py@@ -0,0 +1,129 @@+"""Task 2: per-model-call latency distributions (baseline vs storm day), calls per+run, calls with >= 1 retry, and total retry-wait per run; plus the worker.log storm+lines for the fancy_v1 cells (the only corpus whose driver keeps a per-cell log)."""+from __future__ import annotations++import re+import sys+from collections import Counter, defaultdict+from pathlib import Path+from typing import Any++sys.path.insert(0, str(Path(__file__).parent))+from corpus import RunRef, cost_rows, discover, fmt, med_p90, trajectories # noqa: E402+from turns import parse_turns # noqa: E402++RETRY_THRESHOLD_S = 1.0 # min storm sleep is 1.5 s (2^1 * 0.75); 429 rotation pause 0.5 s+STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")+FAIL_RE = re.compile(r"gemini (\S+) attempt (\d)/6 failed \((.*?)\); retrying in ([\d.]+)s")+GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")+++def pct(xs: list[float]) -> str:+ if not xs:+ return "-"+ s = sorted(xs)+ q = lambda p: s[min(len(s) - 1, int(round(p * (len(s) - 1))))] # noqa: E731+ return f"n={len(s)} p50={q(.5):.1f} p90={q(.9):.1f} p99={q(.99):.1f} max={q(1):.0f}"+++def latency_table(refs: list[RunRef]) -> None:+ lat: dict[tuple[str, str, str], list[float]] = defaultdict(list)+ per_run: dict[tuple[str, str], list[int]] = defaultdict(list)+ err: Counter[tuple[str, str]] = Counter()+ for ref in refs:+ n: Counter[str] = Counter()+ for r in cost_rows(ref):+ key = (ref.bucket, r["model"], r["role"])+ if r["outcome"] == "ok":+ lat[key].append(r["latency_ms"] / 1000)+ n[r["model"]] += 1+ else:+ err[(ref.bucket, r["model"])] += 1+ for m, c in n.items():+ per_run[(ref.bucket, m)].append(c)+ print("## successful-call latency (s) by corpus x model x role")+ for k in sorted(lat):+ print(f" {k[0]:14s} {k[1]:24s} {k[2]:9s} {pct(lat[k])}")+ print("## ok calls per run (median / p90) and error rows")+ for k in sorted(per_run):+ m, p, cnt = med_p90([float(x) for x in per_run[k]])+ print(f" {k[0]:14s} {k[1]:24s} runs={cnt} calls/run {fmt(m)}/{fmt(p)} error rows={err[k]}")+++def retry_table(refs: list[RunRef]) -> None:+ print("## generator turns (transcripts): waiting = wall - recorded latency")+ agg: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))+ for ref in refs:+ turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]+ if not turns:+ continue+ real = [t for t in turns if t.latency_s > 0]+ retried = [t for t in real if t.wait_s > RETRY_THRESHOLD_S]+ a = agg[(ref.bucket, ref.track)]+ a["calls"].append(len(real))+ a["retried"].append(len(retried))+ a["retried_pct"].append(100.0 * len(retried) / max(1, len(real)))+ a["wait_s"].append(sum(t.wait_s for t in real) + sum(t.error_s for t in turns))+ a["wait_per_retried"].extend(t.wait_s for t in retried)+ a["errors"].append(sum(1 for t in turns if t.error_s > 0))+ for k in sorted(agg):+ a = agg[k]+ c, cp, n = med_p90(a["calls"])+ r, rp, _ = med_p90(a["retried"])+ pc, _, _ = med_p90(a["retried_pct"])+ w, wp, _ = med_p90(a["wait_s"])+ wr, wrp, _ = med_p90(a["wait_per_retried"])+ e = sum(a["errors"])+ print(f" {k[0]:14s} {k[1]:16s} runs={n:2d} calls/run {fmt(c)}/{fmt(cp)} retried/run {fmt(r)}/{fmt(rp)} "+ f"({fmt(pc)}%) wait/run {fmt(w)}/{fmt(wp)} s wait/retried-call {fmt(wr, 1)}/{fmt(wrp, 1)} s model_error rows={e}")+++def worker_logs(refs: list[RunRef]) -> None:+ print("## fancy_v1 worker.log (per cell; sleeps only — the failed round-trips are not logged)")+ per: dict[str, list[float]] = defaultdict(list)+ tot: Counter[str] = Counter()+ sleep: Counter[str] = Counter()+ depth: Counter[int] = Counter()+ for ref in refs:+ cd = ref.cell_dir+ if cd is None:+ continue+ s_flash = s_pro = 0.0+ for line in (cd / "worker.log").read_text(errors="replace").splitlines():+ m = STORM_RE.match(line)+ if m:+ tot[f"storm:{m.group(1)}"] += 1+ sleep[m.group(1)] += int(m.group(3))+ depth[int(m.group(2))] += 1+ if "flash" in m.group(1):+ s_flash += int(m.group(3))+ else:+ s_pro += int(m.group(3))+ continue+ m = FAIL_RE.match(line)+ if m:+ tot[f"fail:{m.group(1)}:{m.group(3)[:28]}"] += 1+ continue+ m = GIVEUP_RE.match(line)+ if m:+ tot[f"giveup:{m.group(1)}"] += 1+ per["flash_sleep_s"].append(s_flash)+ per["pro_sleep_s"].append(s_pro)+ for k, v in sorted(tot.items()):+ print(f" {v:5d} {k}")+ for k, v in sleep.items():+ print(f" total logged sleep {k}: {v} s over {len(per['flash_sleep_s'])} cells")+ for k in ("flash_sleep_s", "pro_sleep_s"):+ m, p, n = med_p90(per[k])+ print(f" {k} per cell med/p90 = {fmt(m)}/{fmt(p)} (n={n})")+ mx = max(depth) if depth else 0+ print(f" storm streak depth: max {mx}; streaks >= 10: {sum(v for d, v in depth.items() if d >= 10)} lines; "+ f"streak 1: {depth[1]} (= distinct storm episodes)")+++if __name__ == "__main__":+ refs = discover()+ latency_table(refs)+ retry_table(refs)+ worker_logs(refs)diff --git a/harness/bench/time_audit/corpus.py b/harness/bench/time_audit/corpus.pynew file mode 100644index 00000000..dd1c4908--- /dev/null+++ b/harness/bench/time_audit/corpus.py@@ -0,0 +1,126 @@+"""Read-only corpus discovery + loaders for the time audit (no side effects)."""+from __future__ import annotations++import glob+import json+import statistics+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any++OUT = Path("/home/yipeng/3dcodeverse/harness/bench/out")+STORM_DAY = "storm(08-26)"+BASELINE = "base(08-23/25)"++# corpus name -> (glob under bench/out, corpus bucket)+CORPORA: dict[str, tuple[str, str]] = {+ "fancy_v1": ("fancy_v1/*/arms/*/cells/*/harness_api-agent_gemini_gemini-3.7-flash/run", STORM_DAY),+ "h2h_brilliana_v1": ("h2h_brilliana_v1/*/runs/*", STORM_DAY),+ "h2h_scene_v1": ("h2h_scene_v1/runs/*", STORM_DAY),+ "refs_v1_graphics": ("refs_v1_graphics/runs/*", STORM_DAY),+ "teaser": ("teaser/runs/*", BASELINE),+ "static_v2_flash": ("static_v2_flash/runs/*", BASELINE),+ "scenes_v1_flash": ("scenes_v1_flash/runs/*", BASELINE),+}+++@dataclass+class RunRef:+ corpus: str+ bucket: str+ run: Path+ track: str = ""+ events: list[dict[str, Any]] = field(default_factory=list)++ @property+ def cell_dir(self) -> Path | None:+ return self.run.parent if (self.run.parent / "worker.log").is_file() else None++ @property+ def name(self) -> str:+ return self.run.parent.parent.name if self.cell_dir else self.run.name+++def read_jsonl(path: Path) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ if not path.is_file():+ return rows+ with path.open() as fh:+ for line in fh:+ line = line.strip()+ if not line:+ continue+ try:+ rows.append(json.loads(line))+ except json.JSONDecodeError:+ continue+ return rows+++def track_of(events: list[dict[str, Any]]) -> str:+ start = next((e for e in events if e.get("event") == "run.start"), None)+ if start is None:+ return "?"+ tr, lang = start.get("track", "?"), start.get("language", "?")+ if tr == "static_object":+ return f"static/{lang}"+ if tr == "articulated_object":+ return "articulated"+ return tr+++def discover(corpora: dict[str, tuple[str, str]] = CORPORA) -> list[RunRef]:+ refs: list[RunRef] = []+ for corpus, (pat, bucket) in corpora.items():+ for d in sorted(glob.glob(str(OUT / pat))):+ run = Path(d)+ ev = read_jsonl(run / "events.jsonl")+ if not ev:+ continue+ refs.append(RunRef(corpus=corpus, bucket=bucket, run=run, track=track_of(ev), events=ev))+ return refs+++def cost_rows(ref: RunRef) -> list[dict[str, Any]]:+ return read_jsonl(ref.run / "telemetry" / "cost.jsonl")+++def trajectories(ref: RunRef) -> list[tuple[str, dict[str, Any], list[dict[str, Any]]]]:+ """(session label, result.json, transcript rows) for every trajectory dir."""+ out: list[tuple[str, dict[str, Any], list[dict[str, Any]]]] = []+ tdir = ref.run / "trajectories"+ if not tdir.is_dir():+ return out+ for sd in sorted(p for p in tdir.iterdir() if p.is_dir()):+ res: dict[str, Any] = {}+ rp = sd / "result.json"+ if rp.is_file():+ try:+ res = json.loads(rp.read_text())+ except json.JSONDecodeError:+ res = {}+ out.append((sd.name, res, read_jsonl(sd / "transcript.jsonl")))+ return out+++def rounds(ref: RunRef) -> list[dict[str, Any]]:+ out: list[dict[str, Any]] = []+ for p in sorted((ref.run / "rounds").glob("r*.json")):+ try:+ out.append(json.loads(p.read_text()))+ except (OSError, json.JSONDecodeError):+ continue+ return out+++def med_p90(xs: list[float]) -> tuple[float, float, int]:+ """(median, p90, n); p90 by nearest-rank."""+ if not xs:+ return (float("nan"), float("nan"), 0)+ s = sorted(xs)+ p90 = s[min(len(s) - 1, int(round(0.9 * (len(s) - 1))))]+ return (statistics.median(s), p90, len(s))+++def fmt(x: float, nd: int = 0) -> str:+ return "-" if x != x else f"{x:.{nd}f}"diff --git a/harness/bench/time_audit/deadline.py b/harness/bench/time_audit/deadline.pynew file mode 100644index 00000000..869c9227--- /dev/null+++ b/harness/bench/time_audit/deadline.py@@ -0,0 +1,59 @@+"""Deadline overshoots: sessions past their 1800-s job timeout, runs killed by the+bench max_minutes with no round completed, and the storm-streak depth at each+'giving up after 900 s' (=> seconds per failed 503 attempt, independent of transcripts)."""+from __future__ import annotations++import json+import re+import sys+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent))+from corpus import discover, fmt, med_p90, trajectories # noqa: E402++STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 ")+GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")+BUDGET_RE = re.compile(r"elapsed ([\d.]+) min exceeds max_minutes ([\d.]+)")+SESSION_TIMEOUT_S = 1800.0+++def main() -> None:+ refs = discover()+ over, killed, zero_rounds, depths = [], [], 0, []+ for ref in refs:+ if ref.bucket != "storm(08-26)":+ continue+ for _, res, _ in trajectories(ref):+ d = float(res.get("duration_s") or 0)+ if d > SESSION_TIMEOUT_S and res.get("exit_reason") in ("timeout", "error"):+ over.append(d - SESSION_TIMEOUT_S)+ rounds_done = sum(1 for e in ref.events if e["event"] == "round.done")+ for e in ref.events:+ if e["event"] == "budget.exceeded":+ m = BUDGET_RE.search(str(e.get("reason", "")))+ if m:+ killed.append((float(m.group(1)) - float(m.group(2))) * 60)+ zero_rounds += rounds_done == 0+ cd = ref.cell_dir+ if cd is None:+ continue+ last = 0+ for line in (cd / "worker.log").read_text(errors="replace").splitlines():+ m = STORM_RE.match(line)+ if m and "flash" in m.group(1):+ last = int(m.group(2))+ elif GIVEUP_RE.match(line) and "flash" in line:+ depths.append(last)+ m, p, n = med_p90(over)+ print(f"sessions past the {SESSION_TIMEOUT_S:.0f}-s job timeout (exit timeout/error): n={n} overshoot med/p90 = {fmt(m)}/{fmt(p)} s, sum={sum(over):.0f} s")+ m, p, n = med_p90(killed)+ print(f"runs stopped by bench max_minutes: n={n}, overshoot past the ceiling med/p90 = {fmt(m)}/{fmt(p)} s; with ZERO completed rounds: {zero_rounds}")+ m, p, n = med_p90([float(d) for d in depths])+ if n:+ print(f"flash give-ups in fancy_v1 logs: n={n}; storm-streak depth at give-up med/p90 = {fmt(m)}/{fmt(p)} "+ f"=> 900 s / depth = {fmt(900 / m, 1)} s per failed attempt (sleep included, <= 5 s of it)")+ print(json.dumps({"overshoot_sum_s": round(sum(over)), "killed": len(killed), "zero_rounds": zero_rounds}))+++if __name__ == "__main__":+ main()diff --git a/harness/bench/time_audit/judge_plan.py b/harness/bench/time_audit/judge_plan.pynew file mode 100644index 00000000..2d417d1a--- /dev/null+++ b/harness/bench/time_audit/judge_plan.py@@ -0,0 +1,55 @@+"""Judge + planner stages: stage span vs recorded model latency (the difference is+retry/timeout waiting), per corpus; judge samples that errored and what they cost."""+from __future__ import annotations++import sys+from collections import defaultdict+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent))+from corpus import cost_rows, discover, fmt, med_p90 # noqa: E402+++def main() -> None:+ J: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))+ for ref in discover():+ rows = cost_rows(ref)+ by_round: dict[int, list[dict]] = defaultdict(list)+ for r in rows:+ if r["role"] == "judge":+ by_round[int(r.get("round") or 0)].append(r)+ b = J[ref.bucket]+ for e in ref.events:+ if e["event"] == "judge.done":+ d = float(e.get("duration_s") or 0)+ rs = by_round.get(int(e.get("round") or 0), [])+ ok = sum(r["latency_ms"] / 1000 for r in rs if r["outcome"] == "ok")+ err = [r["latency_ms"] / 1000 for r in rs if r["outcome"] != "ok"]+ b["judge_s"].append(d)+ b["judge_ok_lat"].append(ok)+ b["judge_wait"].append(max(0.0, d - ok))+ b["judge_n_ok"].append(float(sum(1 for r in rs if r["outcome"] == "ok")))+ if err:+ b["judge_err_rows"].extend(err)+ b["judge_rounds_with_err"].append(d)+ elif e["event"] == "stage.done" and e.get("stage") == "plan":+ d = float(e.get("duration_s") or 0)+ pl = [r for r in rows if r["role"] == "planner"]+ ok = sum(r["latency_ms"] / 1000 for r in pl if r["outcome"] == "ok")+ b["plan_s"].append(d)+ b["plan_ok_lat"].append(ok)+ b["plan_calls"].append(float(len(pl)))+ b["plan_wait"].append(max(0.0, d - ok))+ for bucket, b in sorted(J.items()):+ print(f"## {bucket}")+ for k in ("judge_s", "judge_ok_lat", "judge_wait", "judge_n_ok", "plan_s", "plan_ok_lat", "plan_calls", "plan_wait"):+ m, p, n = med_p90(b[k])+ print(f" {k:14s} n={n:3d} med={fmt(m, 1):>7s} p90={fmt(p, 1):>7s} sum={sum(b[k]):.0f}")+ er = b["judge_err_rows"]+ print(f" judge error rows: n={len(er)} sum={sum(er):.0f} s each: {[round(x) for x in sorted(er, reverse=True)[:8]]}")+ rw = b["judge_rounds_with_err"]+ print(f" judge rounds with an errored sample: {len(rw)} / {len(b['judge_s'])}; their judge_s: {[round(x) for x in sorted(rw, reverse=True)[:8]]}")+++if __name__ == "__main__":+ main()diff --git a/harness/bench/time_audit/keys.py b/harness/bench/time_audit/keys.pynew file mode 100644index 00000000..6600a19e--- /dev/null+++ b/harness/bench/time_audit/keys.py@@ -0,0 +1,60 @@+"""Task 4: does any persisted telemetry row say WHICH Gemini key served a call?+Scans every field name in cost.jsonl / transcript.jsonl / result.json / rounds /+record.json for something key-like, and greps the raw text for the '…xxxx' redaction+that gemini.py puts into ChatResponse.raw."""+from __future__ import annotations++import json+import re+import sys+from collections import Counter+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent))+from corpus import discover, read_jsonl # noqa: E402++KEYLIKE = re.compile(r"key|api_key|credential", re.I)+REDACTED = re.compile(r'"key":\s*"…[^"]{0,8}"')+++def walk(obj: object, prefix: str, found: Counter[str]) -> None:+ if isinstance(obj, dict):+ for k, v in obj.items():+ if KEYLIKE.search(str(k)):+ found[f"{prefix}.{k}"] += 1+ walk(v, f"{prefix}.{k}", found)+ elif isinstance(obj, list):+ for v in obj[:50]:+ walk(v, prefix + "[]", found)+++def main() -> None:+ found: Counter[str] = Counter()+ redacted = 0+ files = 0+ for ref in discover():+ for p in [ref.run / "telemetry" / "cost.jsonl", ref.run / "record.json", *(ref.run / "rounds").glob("r*.json"),+ *(ref.run / "trajectories").glob("*/result.json"), *(ref.run / "trajectories").glob("*/transcript.jsonl")]:+ if not p.is_file():+ continue+ files += 1+ text = p.read_text(errors="replace")+ redacted += len(REDACTED.findall(text))+ if p.suffix == ".jsonl":+ for row in read_jsonl(p):+ walk(row, p.name, found)+ else:+ try:+ walk(json.loads(text), p.name, found)+ except json.JSONDecodeError:+ pass+ print(f"scanned {files} files; '\"key\": \"…xxxx\"' occurrences: {redacted}")+ print("key-like field names found (name -> count):")+ for k, v in found.most_common(20):+ print(f" {v:7d} {k}")+ if not any("api" in k.lower() or k.endswith(".key") for k in found):+ print("=> no per-call key identity is persisted anywhere in the run telemetry.")+++if __name__ == "__main__":+ main()diff --git a/harness/bench/time_audit/sessions.py b/harness/bench/time_audit/sessions.pynew file mode 100644index 00000000..55ffd861--- /dev/null+++ b/harness/bench/time_audit/sessions.py@@ -0,0 +1,58 @@+"""Task 3: per agent session (api-agent transcripts) — turns, tool calls by name+with durations, and the model-thinking / tool-execution / waiting split, one table+per track x corpus."""+from __future__ import annotations++import sys+from collections import defaultdict+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent))+from corpus import discover, fmt, med_p90, trajectories # noqa: E402+from turns import parse_turns, split # noqa: E402+++def main() -> None:+ sess: dict[tuple[str, str], list[tuple[int, float, float, float, float]]] = defaultdict(list)+ tools: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))+ for ref in discover():+ k = (ref.bucket, ref.track)+ for _, res, tr in trajectories(ref):+ turns = parse_turns(tr)+ if not turns:+ continue+ think, tool, wait = split(turns)+ sess[k].append((len([t for t in turns if t.latency_s > 0]), float(res.get("duration_s") or 0), think, tool, wait))+ for t in turns:+ for name, d in t.tools:+ tools[k][name].append(d)+ print("## per session: turns, duration, and the split (medians; shares are of summed session time)")+ print("| corpus | track | sessions | turns med/p90 | dur s med/p90 | think s | tool s | wait s | think% | tool% | wait% |")+ print("|---|---|---|---|---|---|---|---|---|---|---|")+ for k in sorted(sess):+ rows = sess[k]+ tm, tp, n = med_p90([float(r[0]) for r in rows])+ dm, dp, _ = med_p90([r[1] for r in rows])+ th, _, _ = med_p90([r[2] for r in rows])+ to, _, _ = med_p90([r[3] for r in rows])+ wa, _, _ = med_p90([r[4] for r in rows])+ T = sum(r[2] + r[3] + r[4] for r in rows) or 1.0+ print(f"| {k[0]} | {k[1]} | {n} | {fmt(tm)}/{fmt(tp)} | {fmt(dm)}/{fmt(dp)} | {fmt(th)} | {fmt(to)} | {fmt(wa)} | "+ f"{100 * sum(r[2] for r in rows) / T:.0f} | {100 * sum(r[3] for r in rows) / T:.0f} | {100 * sum(r[4] for r in rows) / T:.0f} |")+ print("\n## tool calls by name: count, median s, p90 s, total s per session (top tools per track; storm-day + baseline pooled by track)")+ pooled: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))+ nsess: dict[str, int] = defaultdict(int)+ for k, d in tools.items():+ for name, xs in d.items():+ pooled[k[1]][name].extend(xs)+ nsess[k[1]] += len(sess[k])+ for track in sorted(pooled):+ items = sorted(pooled[track].items(), key=lambda kv: -sum(kv[1]))+ print(f" {track} ({nsess[track]} sessions):")+ for name, xs in items[:12]:+ m, p, n = med_p90(xs)+ print(f" {name:18s} n={n:5d} med={fmt(m, 2):>6s} p90={fmt(p, 1):>6s} total/session={sum(xs) / max(1, nsess[track]):6.1f} s")+++if __name__ == "__main__":+ main()diff --git a/harness/bench/time_audit/stage_times.py b/harness/bench/time_audit/stage_times.pynew file mode 100644index 00000000..5349c1b1--- /dev/null+++ b/harness/bench/time_audit/stage_times.py@@ -0,0 +1,156 @@+"""Task 1: per track x corpus, median / p90 of run wall and every stage, plus+"waiting on the provider" (retry sleeps + failed attempts), derived from transcript+timestamps (agent sessions) and from judge/planner stage span minus recorded latency.+Writes stage_runs.jsonl (one row per run) and prints the table."""+from __future__ import annotations++import json+import sys+from collections import defaultdict+from pathlib import Path+from typing import Any++sys.path.insert(0, str(Path(__file__).parent))+from corpus import RunRef, cost_rows, discover, fmt, med_p90, rounds, trajectories # noqa: E402+from turns import parse_turns, split # noqa: E402++HERE = Path(__file__).parent+++def round_segments(ev: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """One dict per round.start..round.done segment (an abandoned segment ends at the+ next round.start / run.start and is flagged)."""+ segs: list[dict[str, Any]] = []+ cur: dict[str, Any] | None = None+ cand: dict[Any, float] = {} # best-of-N candidates are generated BEFORE round.start+ for e in ev:+ k = e["event"]+ if k == "candidates.start":+ cand[e.get("round")] = e["t"]+ if k in ("round.start", "run.start"):+ if cur is not None:+ cur["abandoned"] = True+ cur["end"] = e["t"]+ segs.append(cur)+ cur = None+ if k == "round.start":+ cur = {"round": e.get("round"), "start": min(e["t"], cand.pop(e.get("round"), e["t"])), "abandoned": False}+ continue+ if cur is None:+ continue+ if k == "build.done":+ cur.setdefault("first_build", e["t"])+ cur["last_build"] = e["t"]+ cur["build_ms"] = cur.get("build_ms", 0) + int(e.get("duration_ms") or 0)+ cur["n_builds"] = cur.get("n_builds", 0) + 1+ elif k == "gates.done":+ cur["gates_t"] = e["t"]+ elif k == "judge.done":+ cur["judge_s"] = float(e.get("duration_s") or 0)+ cur["judge_t"] = e["t"]+ elif k == "round.done":+ cur["end"] = e["t"]+ cur["duration_s"] = float(e.get("duration_s") or (e["t"] - cur["start"]))+ segs.append(cur)+ cur = None+ if cur is not None:+ cur["abandoned"] = True+ cur["end"] = ev[-1]["t"]+ segs.append(cur)+ return segs+++def analyse(ref: RunRef) -> dict[str, Any]:+ ev = ref.events+ done = [e for e in ev if e["event"] == "run.done"]+ row: dict[str, Any] = {+ "corpus": ref.corpus, "bucket": ref.bucket, "track": ref.track, "run": ref.name,+ "finished": bool(done), "wall_s": ev[-1]["t"] - ev[0]["t"],+ "status": done[-1].get("status") if done else "RUNNING",+ }+ plan_s = sum(float(e.get("duration_s") or 0) for e in ev if e["event"] == "stage.done" and e.get("stage") == "plan")+ row["plan_s"] = plan_s+ scene_gen = sum(float(e.get("duration_s") or 0) for e in ev+ if e["event"] == "stage.done" and e.get("stage") in ("assets", "env", "zones", "assemble"))+ row["plan_cached"] = any(e["event"] == "stage.cached" and e.get("stage") == "plan" for e in ev)+ rows = cost_rows(ref)+ lat = lambda role, ok=True: sum( # noqa: E731+ r["latency_ms"] / 1000 for r in rows if r["role"] == role and (r["outcome"] == "ok") == ok)+ row["plan_model_s"] = lat("planner")+ row["plan_wait_s"] = max(0.0, plan_s - lat("planner")) if plan_s else 0.0+ row["judge_err_s"] = lat("judge", ok=False) # error rows carry the whole failed span+ segs = [s for s in round_segments(ev) if not s["abandoned"]]+ row["abandoned_s"] = sum(s["end"] - s["start"] for s in round_segments(ev) if s["abandoned"])+ rd = {r.get("index"): r for r in rounds(ref)}+ gen, build, gates, render, judge, other = [], [], [], [], [], []+ for s in segs:+ if "gates_t" not in s or "last_build" not in s:+ continue+ b = s["build_ms"] / 1000+ gen.append(s["last_build"] - s["start"] - b)+ build.append(b)+ gates.append(s["gates_t"] - s["last_build"])+ r = rd.get(s["round"], {})+ rnd = float(((r.get("renders") or {}).get("duration_ms") or 0)) / 1000+ render.append(rnd)+ j = s.get("judge_s", 0.0)+ judge.append(j)+ if "judge_t" in s:+ other.append(max(0.0, s["judge_t"] - s["gates_t"] - j - rnd))+ judge_ok = lat("judge")+ row.update(gen_s=sum(gen) + scene_gen, build_s=sum(build), gates_s=sum(gates), render_s=sum(render),+ judge_s=sum(judge), other_s=sum(other), n_rounds=len(gen),+ judge_model_s=judge_ok, judge_wait_s=max(0.0, sum(judge) - judge_ok))+ think = tools = wait = sess = 0.0+ n_sess = 0+ for _, res, tr in trajectories(ref):+ a, b2, c = split(parse_turns(tr))+ think, tools, wait, sess, n_sess = think + a, tools + b2, wait + c, sess + float(res.get("duration_s") or 0), n_sess + 1+ row.update(sess_think_s=think, sess_tool_s=tools, sess_wait_s=wait, sess_total_s=sess, n_sessions=n_sess)+ row["wait_total_s"] = wait + row["plan_wait_s"] + row["judge_wait_s"]+ return row+++def main() -> None:+ refs = discover()+ out = [analyse(r) for r in refs]+ (HERE / "stage_runs.jsonl").write_text("".join(json.dumps(r) + "\n" for r in out))+ cols = ["wall_s", "plan_s", "gen_s", "build_s", "gates_s", "render_s", "judge_s", "wait_total_s", "sess_wait_s", "judge_wait_s", "judge_err_s", "abandoned_s"]+ groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)+ for r in out:+ groups[(r["bucket"], r["track"])].append(r)+ print("| corpus | track | n(fin) | " + " | ".join(f"{c} med/p90" for c in cols) + " |")+ print("|" + "---|" * (len(cols) + 3))+ for (bucket, track), rs in sorted(groups.items()):+ fin = [r for r in rs if r["finished"]]+ cells = []+ for c in cols:+ src = fin if c == "wall_s" else rs+ m, p, _ = med_p90([float(r[c]) for r in src])+ cells.append(f"{fmt(m)}/{fmt(p)}")+ print(f"| {bucket} | {track} | {len(rs)}({len(fin)}) | " + " | ".join(cells) + " |")+ print("\nshare of run wall: waiting / thinking / tools (medians of per-run ratios, finished runs)")+ for (bucket, track), rs in sorted(groups.items()):+ fin = [r for r in rs if r["finished"] and r["wall_s"] > 0]+ if not fin:+ continue+ w = med_p90([100 * r["wait_total_s"] / r["wall_s"] for r in fin])[0]+ th = med_p90([100 * r["sess_think_s"] / r["wall_s"] for r in fin])[0]+ to = med_p90([100 * (r["sess_tool_s"] + r["build_s"] + r["gates_s"] + r["render_s"]) / r["wall_s"] for r in fin])[0]+ jm = med_p90([100 * r["judge_model_s"] / r["wall_s"] for r in fin])[0]+ print(f" {bucket:14s} {track:16s} wait {fmt(w)}% think {fmt(th)}% tools+build+gates+render {fmt(to)}% judge model {fmt(jm)}%")+ # per-round view (generate + judge per round) for the storm/baseline contrast+ print("\nper-round: gen_s and judge_s med/p90 (rounds with a build+gates)")+ per: dict[tuple[str, str], tuple[list[float], list[float]]] = defaultdict(lambda: ([], []))+ for r in out:+ n = max(1, r["n_rounds"])+ per[(r["bucket"], r["track"])][0].append(r["gen_s"] / n)+ per[(r["bucket"], r["track"])][1].append(r["judge_s"] / n)+ for k, (g, j) in sorted(per.items()):+ gm, gp, _ = med_p90(g)+ jm, jp, _ = med_p90(j)+ print(f" {k[0]:14s} {k[1]:16s} gen {fmt(gm)}/{fmt(gp)} judge {fmt(jm)}/{fmt(jp)}")+++if __name__ == "__main__":+ main()diff --git a/harness/bench/time_audit/storm_attempts.py b/harness/bench/time_audit/storm_attempts.pynew file mode 100644index 00000000..a8c1158b--- /dev/null+++ b/harness/bench/time_audit/storm_attempts.py@@ -0,0 +1,78 @@+"""Price the pieces of "waiting": backoff sleeps (worker.log) vs failed 503 round+trips (transcript wait minus sleep, per storm line) vs 900-s give-ups (model_error+rows / turns with wait >= 850 s), plus cell wall vs run span for the fancy_v1 cells."""+from __future__ import annotations++import json+import re+import sys+from collections import Counter+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent))+from corpus import RunRef, discover, fmt, med_p90, trajectories # noqa: E402+from turns import parse_turns # noqa: E402++STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")+GIVEUP_S = 850.0+++def hist(xs: list[float], edges: list[float]) -> str:+ c = Counter()+ for x in xs:+ c[next((e for e in edges if x < e), float("inf"))] += 1+ tot = max(1, len(xs))+ return " ".join(f"<{e:g}s:{100 * c[e] / tot:.0f}%" for e in [*edges, float("inf")])+++def main() -> None:+ refs = discover()+ for bucket in ("base(08-23/25)", "storm(08-26)"):+ waits, big, errs = [], [], []+ for ref in [r for r in refs if r.bucket == bucket]:+ for _, _, tr in trajectories(ref):+ for t in parse_turns(tr):+ if t.latency_s > 0:+ waits.append(t.wait_s)+ if t.error_s > 0:+ errs.append(t.error_s)+ tot = sum(waits) + sum(errs)+ big = [w for w in waits if w >= GIVEUP_S]+ print(f"## {bucket}: {len(waits)} generator turns; wait histogram: {hist(waits, [1, 5, 30, 120, 850])}")+ print(f" total wait {tot:.0f} s; turns with wait>=850 s: {len(big)} = {100 * sum(big) / max(1, tot):.0f}% of all wait; "+ f"model_error spans: n={len(errs)} sum={sum(errs):.0f} s ({100 * sum(errs) / max(1, tot):.0f}%) "+ f"med/p90={fmt(med_p90(errs)[0])}/{fmt(med_p90(errs)[1])} s")+ print(f" wait>=850s turn: {hist(big, [900, 1800, 2700])}")+ print("## fancy_v1 cells: transcript wait vs logged sleep -> cost of one failed 503 round trip")+ rt, per_cell, outside = [], [], []+ for ref in refs:+ cd = ref.cell_dir+ if cd is None:+ continue+ lines = (cd / "worker.log").read_text(errors="replace").splitlines()+ n = sleep = 0+ for m in filter(None, map(STORM_RE.match, lines)):+ if "flash" in m.group(1):+ n += 1+ sleep += int(m.group(3))+ turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]+ w = sum(t.wait_s for t in turns) + sum(t.error_s for t in turns)+ w_nogiveup = sum(min(t.wait_s, 0) if t.wait_s >= GIVEUP_S else t.wait_s for t in turns)+ if n:+ rt.append((w_nogiveup - sleep) / n)+ per_cell.append((ref.name, n, sleep, round(w), round(w_nogiveup)))+ try:+ cell = json.loads((cd / "cell.json").read_text())+ outside.append(float(cell.get("wall_s") or 0) - (ref.events[-1]["t"] - ref.events[0]["t"]))+ except (OSError, json.JSONDecodeError):+ pass+ m, p, k = med_p90(rt)+ print(f" per-cell (wait_excluding_giveups - sleep) / storm_lines: med {fmt(m, 1)} s p90 {fmt(p, 1)} s (n={k} cells)")+ print(f" cell wall_s - run event span (pairwise + final pro scoring outside the run): med/p90 = "+ f"{fmt(med_p90(outside)[0])}/{fmt(med_p90(outside)[1])} s")+ for row in sorted(per_cell, key=lambda r: -r[3])[:5]:+ print(" worst:", row)+++if __name__ == "__main__":+ main()diff --git a/harness/bench/time_audit/turns.py b/harness/bench/time_audit/turns.pynew file mode 100644index 00000000..e831f541--- /dev/null+++ b/harness/bench/time_audit/turns.py@@ -0,0 +1,67 @@+"""Per-turn reconstruction of an api-agent transcript.++The ledger's ``latency_ms`` is the stopwatch of the SUCCESSFUL attempt only+(``gemini.py:_once``); backoff sleeps, failed attempts and KeyPool waits are not+recorded anywhere. Transcript rows are stamped when the model returns+(``assistant``) and when a tool finishes (``tool_result``), so+``model_wall = t(assistant) - t(previous row)`` and ``wait = model_wall - latency``.+"""+from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any+++@dataclass+class Turn:+ turn: int+ model_wall_s: float+ latency_s: float+ wait_s: float+ tools: list[tuple[str, float]] = field(default_factory=list)+ error_s: float = 0.0 # time spent in a call that ended in a model_error row+++def is_gemini_api_session(rows: list[dict[str, Any]]) -> bool:+ """Only in-process api-agent sessions on Gemini stamp a per-turn latency."""+ return any(r.get("kind") == "assistant" and (r.get("usage") or {}).get("backend") == "gemini" for r in rows)+++def parse_turns(rows: list[dict[str, Any]]) -> list[Turn]:+ turns: list[Turn] = []+ if not rows or not is_gemini_api_session(rows):+ return turns+ prev_end = float(rows[0]["t"])+ cur: Turn | None = None+ tool_start = prev_end+ pending_err = 0.0+ for r in rows:+ t = float(r["t"])+ kind = r.get("kind")+ if kind == "assistant":+ wall = max(0.0, t - prev_end)+ lat = float((r.get("usage") or {}).get("latency_ms") or 0) / 1000.0+ cur = Turn(int(r.get("turn", len(turns))), wall, lat, max(0.0, wall - lat), error_s=pending_err)+ pending_err = 0.0+ turns.append(cur)+ prev_end = tool_start = t+ elif kind == "tool_result":+ if cur is not None:+ cur.tools.append((str(r.get("name", "?")), max(0.0, t - tool_start)))+ tool_start = prev_end = t+ elif kind == "model_error":+ pending_err += max(0.0, t - prev_end)+ prev_end = t+ elif kind in ("nudge", "compact", "system"):+ prev_end = t+ if pending_err > 0: # session died in a model_error after the last good turn+ turns.append(Turn(len(turns), 0.0, 0.0, 0.0, error_s=pending_err))+ return turns+++def split(turns: list[Turn]) -> tuple[float, float, float]:+ """(model thinking s, tool execution s, waiting s) over a session."""+ think = sum(t.latency_s for t in turns)+ tools = sum(d for t in turns for _, d in t.tools)+ wait = sum(t.wait_s + t.error_s for t in turns)+ return think, tools, waitdiff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 72748235..23b6a138 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1220,3 +1220,35 @@ streak, up to 900 s); what it costs is one more round-trip on a fresh key. Follmeasuring: rank keys by recent latency (the 30 s keys are consistent), and record the keyindex in `telemetry/usage.jsonl` so the distribution of calls per key can be read instead ofprobed.+++## 25. Where the time goes — the 2026-08-26 audit (`docs/TIME_AUDIT_2026-08-26.md`)++51 storm-day runs against 52 baseline runs, every stage and every model call, scripts in+`bench/time_audit/` (read-only over `bench/out`). The numbers that decide what to build next:++* A blender object run is **1 811 s** median on a healthy provider and **4 726 s** under the storm;+ build + gates + render together are 3–19 % of a baseline run and 1–4 % of a storm run. The+ agent session is where the time is: baseline 57 % model thinking / 11 % tools / 32 % waiting;+ storm day **22 % / 2 % / 77 %**.+* Waiting is not the backoff sleep. Logged sleeps are ~13 % of the wait; a failed 503 costs a+ **21–50 s held round-trip** before the provider rejects it, and streaks average 4.7 attempts.+ **30 % of all storm-day waiting (72 921 s) is 66 give-up spans** — one call retrying to the+ 900 s `RETRY_DEADLINE_S`, up to three times per turn through `api_agent.MODEL_RETRIES`,+ never clipped to the session's or the round's remaining budget. 18 of 51 storm runs ended+ with zero rounds; 14 of them overshot the bench ceiling by 837 s median because the ceiling is+ only checked between stages.+* Successful calls are also slower under the storm: flash p50 3.3 → 8.3 s, p90 17 → 31 s; the+ judge (pro) p50 42 → 50 s. The judge is not the bottleneck at the median; its tail is+ (2 rounds lost 1 100 s each to three 300 s read timeouts). The planner waits 492 s median+ per storm-day run for 39 s of model time.+* No telemetry row records which key served a call, so per-key distribution was unanswerable+ from the corpus (the §24 probe answered it directly).++Ranked by measured seconds per storm-day run: (1) clip the per-call retry budget to the+remaining session / round budget (~1 430 s/run) — (2) hedge a 503 retry across 2–3 keys+(~900–1 400 s/run) — (3) hedge / cap the planner and judge calls (~400 s/run + the judge tail)+— (4) plan cache on re-runs (548 s) — (5) no sleep on 503 (≤ 13 %; landed with §24) — (6) enforce+the ceiling inside a round (837 s sooner on killed runs) — (7) flash as the loop judge (~60 s+baseline) — (8) 30 s render cap (≤ 100 s). (1)–(3) and the key/attempt ledger fields are in+progress on branch `retry-budget`.diff --git a/harness/docs/TIME_AUDIT_2026-08-26.md b/harness/docs/TIME_AUDIT_2026-08-26.mdnew file mode 100644index 00000000..26cce8b1--- /dev/null+++ b/harness/docs/TIME_AUDIT_2026-08-26.md@@ -0,0 +1,163 @@+# Where the time goes — 3dcodeverse harness, storm day (2026-08-26) vs baseline (08-23/25)++Corpus: 51 storm-day runs (fancy_v1 28 cells, h2h_brilliana 12, h2h_scene 5, refs_v1_graphics 6; 45 finished) and+52 baseline runs (static_v2_flash 20, scenes_v1_flash 4, teaser 28). All api-agent gemini-3.7-flash generator,+gemini-3.1-pro-preview judge. Scripts (read-only, in this directory): `corpus.py` `turns.py` (shared),+`stage_times.py` (§1), `calls.py` (§2), `sessions.py` (§3), `keys.py` (§4), `judge_plan.py`, `storm_attempts.py`,+`deadline.py` (§5 inputs); their raw output is in `out_*.txt`, per-run rows in `stage_runs.jsonl`.++**How "waiting on the provider" was measured — and what is NOT recorded.** `telemetry/cost.jsonl.latency_ms` is the+stopwatch of the *successful* attempt only (`gemini.py:_once`; `cost/instrument.py` uses `usage.latency_ms or ms`), so+backoff sleeps, failed 503 round-trips and KeyPool waits leave no row; only a call that fails outright gets an `error`+row whose latency is the whole wrapper span. There is no `attempts`/`key` field. `worker.log` (fancy cells only) has+the storm lines but no timestamps. So: generator waiting = `t(assistant row) − t(previous transcript row) − latency`+per turn (`turns.py`; 88 % of baseline turns give < 1 s, i.e. harness overhead), a `model_error` row's span counts+fully; judge/planner waiting = stage span − Σ ok-latency. "wait" below is thread-time (scoped rounds run 3–4+sessions in parallel), which is why it can exceed wall.++## 1. Run wall and stages — median / p90 seconds (n = runs (finished))++| corpus | track | n | wall | plan | generate | build | gates | render | judge | wait (thread) | session wait | judge wait | killed-run span |+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|+| base | static/blender | 23 | 1811/3073 | 43/136 | 900/2102 | 0/1 | 4/69 | 3/68 | 116/448 | 321/1719 | 248/1712 | 1/275 | 0/654 |+| base | static/cadquery | 4 | 3288/5782 | 307/429 | 1873/3135 | 14/20 | 79/89 | 74/86 | 331/897 | 521/2054 | 285/1675 | 0/0 | 0/1724 |+| base | static/threejs | 3 | 2403/5152 | 459/584 | 1823/3118 | 0/1 | 66/117 | 66/107 | 123/1366 | 2118/3956 | 1905/3092 | 0/466 | 0/0 |+| base | graphics | 9 | 926/2003 | 77/122 | 663/1714 | 5/13 | 0/0 | 5/12 | 199/372 | 421/1090 | 275/1019 | 0/132 | 0/0 |+| base | scene | 10 | 4118/5097 | 50/120 | 2859/4122 | 8/23 | 22/83 | 21/81 | 284/490 | 1293/4214 | 1292/4096 | 0/103 | 0/296 |+| base | articulated | 3 | 5869/6376 | 325/417 | 3430/3876 | 1/2 | 177/195 | 2/41 | 98/1005 | 1191/1886 | 925/1145 | 0/573 | 0/2195 |+| storm | static/blender | 23(20) | 4726/5536 | 0*/577 | 1989/3127 | 0/0 | 1/18 | 1/16 | 38/116 | 5356/10333 | 5030/10333 | 0/4 | 2409/5161 |+| storm | static/cadquery | 8(7) | 6649/8344 | 0*/749 | 3159/5401 | 4/14 | 3/95 | 2/94 | 176/666 | 4116/5222 | 3002/4807 | 43/415 | 902/2265 |+| storm | static/threejs | 9 | 5328/7802 | 0*/548 | 3595/5501 | 0/0 | 3/5 | 1/3 | 94/430 | 6747/10174 | 6579/8401 | 27/370 | 1606/3648 |+| storm | graphics | 6 | 6003/8818 | 380/628 | 2971/3620 | 12/13 | 0/0 | 12/13 | 228/574 | 3605/6488 | 3223/5897 | 5/55 | 1001/2330 |+| storm | scene | 5(3) | 4404/4711 | 576/646 | 1406/2960 | 1/3 | 4/8 | 3/7 | 0/107 | 1218/2823 | 1026/2823 | 0/76 | 0/583 |++\* fancy_v1 cells get a pre-computed plan (`stage.cached`). "killed-run span" = time inside a round that never reached+`round.done`: 18 of 51 storm runs ended with **zero rounds** (11 of 23 blender cells, median 4 523 s each), 14 of them+stopped by the bench `max_minutes 60` ceiling, which is only checked between stages, so they overshot it by 837 s+median / 1 560 s p90. Per round: generate 314→1 008 s (blender), 989→2 049 s (threejs); judge 48→38 s (blender),+110→76 s (cadquery) — the judge stage is **not** slower on storm day at the median; its tail is (§2). Build 0.1–4 s,+gates 1–80 s, render 1–90 s: together 1–4 % of storm wall, 3–19 % of baseline wall.++## 2. Per model call++Successful-call latency (s), from the ledger:++| model / role | corpus | n | p50 | p90 | p99 | max |+|---|---|---|---|---|---|---|+| gemini-3.7-flash generator | base | 8097 | 3.3 | 17.2 | 49.6 | 172 |+| gemini-3.7-flash generator | storm | 5867 | **8.3** | **31.1** | 74.9 | 239 |+| gemini-3.7-flash planner | base / storm | 60 / 36 | 13.7 / 23.4 | 32 / 43 | 53 / 68 | 76 / 68 |+| gemini-3.1-pro-preview judge | base / storm | 328 / 67 | 42.2 / 50.5 | 73 / 103 | 129 / 165 | 209 / 171 |++Even the calls that succeed are **2.5x slower** under the storm (think time per session 106 → 283 s, §3).+Calls per run (ok, median/p90): flash 176/292 base vs 102/213 storm; pro 5/15 vs 1/5. Error rows: flash 41 → 133,+pro 0 → 5. Retries from transcripts (a turn whose wall exceeds its latency by > 1 s):++| corpus | track | runs | calls/run | calls with ≥1 retry /run (share) | wait per retried call med/p90 | retry-wait per run med/p90 | 900-s give-up rows |+|---|---|---|---|---|---|---|---|+| base | static/blender | 22 | 186/302 | 10/31 (6 %) | 10.9/63.8 s | 263/1712 s | 1 |+| base | scene | 8 | 216/265 | 28/43 (13 %) | 19.8/116 s | 2120/4096 s | 4 |+| base | graphics | 9 | 25/28 | 3/8 (20 %) | 32/158 s | 275/1019 s | 17 |+| storm | static/blender | 23 | 125/257 | 33/65 (24 %) | **47/333 s** | **5030/10333 s** | 30 |+| storm | static/cadquery | 7 | 87/148 | 18/30 (21 %) | 55/389 s | 3103/4807 s | 3 |+| storm | static/threejs | 9 | 160/275 | 59/71 (32 %) | 40/198 s | 6579/8401 s | 8 |+| storm | graphics | 6 | 44/49 | 10/14 (33 %) | 76/387 s | 3223/5897 s | 24 |+| storm | scene | 3 | 36/243 | 4/49 (20 %) | 12/70 s | 1130/2823 s | 1 |++Where the storm-day wait goes (246 297 s over 5 799 turns): **30 % (72 921 s) is 66 `model_error` spans** — a call that+retried until the 900-s `RETRY_DEADLINE_S` (median span 923 s, p90 2 743 s = three consecutive give-ups on one turn+via `api_agent.MODEL_RETRIES=3`); 45 sessions were hit, ~1 430 s per storm run on average. The other 70 % is ordinary+storm streaks. From the 28 fancy worker.logs: 5 143 flash storm lines + 158 pro, 46 flash + 10 pro give-ups, 44 pro+read-timeouts (300 s each). Logged **sleep is only 645 s/cell median (1 525 p90), 21 943 s in total = ~13 % of the+wait**; `(wait − sleep) / storm lines` = **21.5 s per failed attempt** (p90 28.5), and the streak depth at give-up+(median 18 of 60) says 900/18 = ~50 s per attempt late in a storm. The cost of a 503 is the round-trip the provider+holds before rejecting, not the ≤ 5 s backoff. Judge: 2 of 56 storm rounds lost 1 162 s and 927 s (3 × 300-s+timeouts before the deadline) then a second sample answered in 128 s → `judge_s` 1 818 / 1 593 s; judge wait sum+6 847 s / 56 rounds (median 1.8 s, p90 370 s). Planner: storm plan stage 548 s median (p90 995) for 39 s of model+time → **492 s of waiting per run that plans**; baseline 69 s (17 s model). Outside the run, the fancy cells spend a+further 252 s median / 935 s p90 in the driver's pairwise + final pro scoring (`cell.wall_s` − run span).++## 3. Per agent session (api-agent transcripts): thinking vs tools vs waiting++| corpus | track | sessions | turns med/p90 | duration med/p90 | think s | tool s | wait s | think % | tool % | wait % |+|---|---|---|---|---|---|---|---|---|---|---|+| base | static/blender | 155 | 25/60 | 158/446 | 106 | 6 | 12 | 57 | 11 | 32 |+| base | static/cadquery | 10 | 30/60 | 511/1659 | 277 | 23 | 129 | 56 | 9 | 34 |+| base | static/threejs | 32 | 19/38 | 298/715 | 137 | 25 | 117 | 43 | 12 | 45 |+| base | graphics | 24 | 8/17 | 141/754 | 87 | 5 | 31 | 42 | 2 | 56 |+| base | scene | 82 | 20/30 | 336/933 | 128 | 32 | 90 | 35 | 14 | 52 |+| base | articulated | 6 | 34/38 | 1259/1825 | 602 | 213 | 361 | 45 | 23 | 32 |+| storm | static/blender | 130 | 20/45 | **1528**/2408 | 283 | 4 | **976** | 22 | 2 | **77** |+| storm | static/cadquery | 22 | 26/49 | 1802/2403 | 417 | 28 | 1025 | 30 | 2 | 68 |+| storm | static/threejs | 57 | 23/46 | 1288/2181 | 340 | 10 | 707 | 29 | 3 | 69 |+| storm | graphics | 23 | 9/17 | 841/2932 | 179 | 5 | 612 | 17 | 1 | 83 |+| storm | scene | 11 | 19/57 | 581/890 | 194 | 15 | 207 | 29 | 4 | 67 |++Tool execution (pooled, median / p90 / seconds per session): blender `render_views` 0.8/61 → 9.6 s, `render_sheet`+0.9/61 → 7.4, `isolate` 0.7/61 → 3.9, `build` 0.5/0.7 → 2.3, `check_connectivity` 0.13/0.5 → 0.6; cadquery `build`+3.3/5.9 → 17 s; threejs `render_views` 2.5/87 → 16 s, `build` 1.0/1.7 → 5; graphics `gl_probe` 1.0/1.3 → 2.2,+`build` 4.8/6 → 2.0, `gl_frames` 2.8/3.2 → 1.9; scene `scene_views` 6.1/16 → 25 s, `scene_probe` 2.4/16 → 17,+`build` 3.1/8.3 → 15; articulated `joint_sweep` 50/93 → 259 s. read/write/edit/list/check_contract/measure ≈ 0.+Renders have a 60-s-cap tail (p90 = 61 s) but total tool time is ≤ 30 s per session everywhere except `joint_sweep`.+Sessions past their 1 800-s job timeout (exit timeout/error): 97 on storm day, overshoot 182 s med / 1 154 s p90,+41 355 s in total — the deadline is only checked at the top of a turn, never inside the retry loop.++## 4. Key rotation — evidence and mechanism++**No telemetry row records which key served a call.** `keys.py` scanned 1 542 files (cost.jsonl, transcripts,+result.json, rounds, record.json): zero occurrences of the `"key": "…xxxx"` that `gemini.py:_once` puts into+`ChatResponse.raw`, and no key-like field anywhere (only `settings.key_pool_size`). Per-key distribution and+"is one key hammered" are therefore unanswerable from the corpus — that is the finding; `KeyPool.stats()` exists but is+only surfaced by `3dcv doctor --live`, never persisted. How rotation works today (`keypool.py`, `retry.py`,+`storm.py`, `gemini.py`, `agents/api_agent.py`, `judges/vlm_judge.py`):+1. `KeyPool.acquire` is **round-robin per call**: the cursor advances past the chosen key, so consecutive calls (and+ consecutive retries of one call) land on different keys by construction, subject to per-key RPM (1 000/min) and TPM+ (1 M/min, reserved from the estimated prompt tokens, reconciled on report), a ≤ 5 s 429 cooldown, health ≥ 0.3,+ a 1-h bench for dead keys, and a process-wide `max_in_flight = 64` semaphore; it blocks ≤ 120 s if nothing is ready.+2. Key-scoped errors (401/403, invalid key) rotate at once for free; 429 rotates for free with a 0.5 s pause while an+ untried key remains, then counts against `max_attempts = 6` with backoff 1·2^n capped at `MAX_WAIT_S = 5`.+3. **503/529 = "storm" branch**: sleep `min(5, 2^storm × [0.75, 1.25))` s, re-acquire (a new key), up to 60 storm+ attempts that do NOT consume `max_attempts`, bounded only by `RETRY_DEADLINE_S = 900` — checked **after** a failed+ attempt returns, so a 300-s read timeout (`model_timeout_s`) can push one call to ~1 200 s. Other retryables+ (timeouts, bad JSON, empty candidates) get the 6-attempt/900-s budget. `StormGate` exists but is OFF (lost its A/B).+4. Above that, `api_agent._generate` retries a failed *turn* 3x (sleep 2, 4 s) and `VlmJudge._sample` retries a+ sample 3x, each retry re-entering the full 900-s machine: one turn can burn 2 700 s; a session's `timeout_s` and the+ run's `max_minutes` are only checked between turns / stages. Each `ab_plan` cell is its own process (own pool).+What costs wall clock under a 503 storm: (a) every attempt is **sequential and single-flight** — one request in the+air per logical call, and a failed 503 costs 20–50 s of held round-trip before the ≤ 5 s sleep even starts; (b) no+hedging across keys, though each attempt already lands on a fresh key; (c) the 900-s budget is per *call*, multiplied+by the outer 3x loops, and never clipped to the remaining session/round/run budget; (d) the judge's 200 k-token call+gets the same 300-s read timeout x 3 before its deadline; (e) a scoped baseline round waits for the slowest of 3–4+sessions, so one session's give-up chain stalls the round and the run runs into the 60-min ceiling with nothing built.++## 5. Accelerations, ranked by measured seconds per storm-day run (base-day gain in brackets)++1. **Clip the per-call retry budget to the remaining session/round/run budget and cap it (~120 s inside an agent+ turn; a turn that still fails ends the session as `error` and the round salvages what exists).** Removes the 66+ give-up spans: 72 921 s / 51 runs = **~1 430 s/run mean** (~800 s per span at a 120-s cap), plus the 837 s median+ overshoot on the 32 ceiling-killed runs and the 41 355 s of session overshoot. Code: `retry.py` (`max_total_s`+ from caller), `api_agent._generate` (check `t_deadline` inside `MODEL_RETRIES`), `vlm_judge._sample`,+ `tracks/generation.py` (`timeout_s = min(agent_timeout_s, budget remaining)`). [base: 7 137 s / 52 runs ≈ 140 s]+2. **Hedge the retry of a 503: after the first 503, issue the next attempt on 2–3 keys concurrently, take the first+ success (a 503 bills nothing, so the hedge is free while storming).** Storm streaks average 4.7 attempts (1 101+ episodes / 5 143 lines) at 21–50 s each; a 3-way hedge cuts expected rounds to ~1.7, i.e. ~60 % of the retry+ wait: 33 retried calls × 47 s × 0.6 ≈ **930 s/run blender**, 59 × 40 × 0.6 ≈ **1 400 s threejs**, ~600 s cadquery,+ ~450 s graphics. Code: `retry.py` + `gemini.py` (a small thread fan-out), validate with `concurrency_probe.py`.+3. **Planner and judge: hedge on a second key after ~1.5 × p90 (60 s planner, 150 s judge) and cap the sample at+ ~200 s instead of 3 × 300 s.** Planner: 492 s median waiting per run that plans (h2h/refs/scene; **~400 s/run**);+ judge: 3 150 s over the 2 timed-out rounds = **~56 s/round mean**, `judge_s` p90 451 → ~170 s. Code+ (`vlm_judge._sample`, planner call site) or a role-specific `model_timeout_s` (today one global 300 s).+4. **Pre-compute / cache the plan per prompt** (settings — `ab_plan` already does it: fancy plan stage = 0 s):+ **548 s/run** saved on any re-run or second arm of the same prompt on storm day; 69 s baseline.+5. **Rotate to the next key with no (or 0.5 s) sleep on 503** (settings: `GeminiModel(base_delay, storm_max_delay)`):+ the sleeps are 645 s/cell median, 1 525 p90 — **≤ 13 % of the wait**; only worth it together with 2.+6. **Enforce the bench ceiling inside a round** (code, `lifecycle` budget check in the session fan-out): the 14+ zero-round killed runs would fail **837 s (p90 1 560 s) sooner**; they produce nothing either way unless salvage+ builds the parts that did finish (2 of 4 sessions typically complete at ~1 500–1 900 s).+7. **Cheaper loop judge** (setting `--loop-judge gemini:gemini-3.7-flash`, pro for the final): judge model time is+ 12 % of baseline blender wall (116 s/run); flash p50 21 s vs pro 42 s → **~60 s/run base, ~20 s storm**.+8. **Shorter render tools** (code: 30-s render cap): render/isolate p90 = 61–91 s tail, total ≤ 30 s/session →+ **≤ ~100 s/run**; `joint_sweep` (259 s/session, articulated) is the only tool worth batching.+9. Parallel judge samples — already parallel (`fan_out`; baseline `judge_s` 52 s vs Σ samples 100 s): **0 s**.+Not fixable from here: successful flash calls are 2.5x slower during the storm (p50 8.3 s), which alone adds ~180 s+of think time per session; the only lever is fewer turns.