Skip to content

Official Codebase of 3DCodeVerse

Python

docs: the time audit — 51 storm-day runs vs 52 baseline, every stage and call; waiting is the held 503 round-trip, not the sleep

gaoypeng399894ffb898
diff --git a/harness/bench/time_audit/calls.py b/harness/bench/time_audit/calls.py
new file mode 100644
index 00000000..fb4266e3
--- /dev/null
+++ b/harness/bench/time_audit/calls.py
@@ -0,0 +1,129 @@
+"""Task 2: per-model-call latency distributions (baseline vs storm day), calls per
+run, calls with >= 1 retry, and total retry-wait per run; plus the worker.log storm
+lines for the fancy_v1 cells (the only corpus whose driver keeps a per-cell log)."""
+from __future__ import annotations
+
+import re
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).parent))
+from corpus import RunRef, cost_rows, discover, fmt, med_p90, trajectories # noqa: E402
+from turns import parse_turns # noqa: E402
+
+RETRY_THRESHOLD_S = 1.0 # min storm sleep is 1.5 s (2^1 * 0.75); 429 rotation pause 0.5 s
+STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")
+FAIL_RE = re.compile(r"gemini (\S+) attempt (\d)/6 failed \((.*?)\); retrying in ([\d.]+)s")
+GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")
+
+
+def pct(xs: list[float]) -> str:
+ if not xs:
+ return "-"
+ s = sorted(xs)
+ q = lambda p: s[min(len(s) - 1, int(round(p * (len(s) - 1))))] # noqa: E731
+ return f"n={len(s)} p50={q(.5):.1f} p90={q(.9):.1f} p99={q(.99):.1f} max={q(1):.0f}"
+
+
+def latency_table(refs: list[RunRef]) -> None:
+ lat: dict[tuple[str, str, str], list[float]] = defaultdict(list)
+ per_run: dict[tuple[str, str], list[int]] = defaultdict(list)
+ err: Counter[tuple[str, str]] = Counter()
+ for ref in refs:
+ n: Counter[str] = Counter()
+ for r in cost_rows(ref):
+ key = (ref.bucket, r["model"], r["role"])
+ if r["outcome"] == "ok":
+ lat[key].append(r["latency_ms"] / 1000)
+ n[r["model"]] += 1
+ else:
+ err[(ref.bucket, r["model"])] += 1
+ for m, c in n.items():
+ per_run[(ref.bucket, m)].append(c)
+ print("## successful-call latency (s) by corpus x model x role")
+ for k in sorted(lat):
+ print(f" {k[0]:14s} {k[1]:24s} {k[2]:9s} {pct(lat[k])}")
+ print("## ok calls per run (median / p90) and error rows")
+ for k in sorted(per_run):
+ m, p, cnt = med_p90([float(x) for x in per_run[k]])
+ print(f" {k[0]:14s} {k[1]:24s} runs={cnt} calls/run {fmt(m)}/{fmt(p)} error rows={err[k]}")
+
+
+def retry_table(refs: list[RunRef]) -> None:
+ print("## generator turns (transcripts): waiting = wall - recorded latency")
+ agg: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+ for ref in refs:
+ turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]
+ if not turns:
+ continue
+ real = [t for t in turns if t.latency_s > 0]
+ retried = [t for t in real if t.wait_s > RETRY_THRESHOLD_S]
+ a = agg[(ref.bucket, ref.track)]
+ a["calls"].append(len(real))
+ a["retried"].append(len(retried))
+ a["retried_pct"].append(100.0 * len(retried) / max(1, len(real)))
+ a["wait_s"].append(sum(t.wait_s for t in real) + sum(t.error_s for t in turns))
+ a["wait_per_retried"].extend(t.wait_s for t in retried)
+ a["errors"].append(sum(1 for t in turns if t.error_s > 0))
+ for k in sorted(agg):
+ a = agg[k]
+ c, cp, n = med_p90(a["calls"])
+ r, rp, _ = med_p90(a["retried"])
+ pc, _, _ = med_p90(a["retried_pct"])
+ w, wp, _ = med_p90(a["wait_s"])
+ wr, wrp, _ = med_p90(a["wait_per_retried"])
+ e = sum(a["errors"])
+ print(f" {k[0]:14s} {k[1]:16s} runs={n:2d} calls/run {fmt(c)}/{fmt(cp)} retried/run {fmt(r)}/{fmt(rp)} "
+ f"({fmt(pc)}%) wait/run {fmt(w)}/{fmt(wp)} s wait/retried-call {fmt(wr, 1)}/{fmt(wrp, 1)} s model_error rows={e}")
+
+
+def worker_logs(refs: list[RunRef]) -> None:
+ print("## fancy_v1 worker.log (per cell; sleeps only — the failed round-trips are not logged)")
+ per: dict[str, list[float]] = defaultdict(list)
+ tot: Counter[str] = Counter()
+ sleep: Counter[str] = Counter()
+ depth: Counter[int] = Counter()
+ for ref in refs:
+ cd = ref.cell_dir
+ if cd is None:
+ continue
+ s_flash = s_pro = 0.0
+ for line in (cd / "worker.log").read_text(errors="replace").splitlines():
+ m = STORM_RE.match(line)
+ if m:
+ tot[f"storm:{m.group(1)}"] += 1
+ sleep[m.group(1)] += int(m.group(3))
+ depth[int(m.group(2))] += 1
+ if "flash" in m.group(1):
+ s_flash += int(m.group(3))
+ else:
+ s_pro += int(m.group(3))
+ continue
+ m = FAIL_RE.match(line)
+ if m:
+ tot[f"fail:{m.group(1)}:{m.group(3)[:28]}"] += 1
+ continue
+ m = GIVEUP_RE.match(line)
+ if m:
+ tot[f"giveup:{m.group(1)}"] += 1
+ per["flash_sleep_s"].append(s_flash)
+ per["pro_sleep_s"].append(s_pro)
+ for k, v in sorted(tot.items()):
+ print(f" {v:5d} {k}")
+ for k, v in sleep.items():
+ print(f" total logged sleep {k}: {v} s over {len(per['flash_sleep_s'])} cells")
+ for k in ("flash_sleep_s", "pro_sleep_s"):
+ m, p, n = med_p90(per[k])
+ print(f" {k} per cell med/p90 = {fmt(m)}/{fmt(p)} (n={n})")
+ mx = max(depth) if depth else 0
+ print(f" storm streak depth: max {mx}; streaks >= 10: {sum(v for d, v in depth.items() if d >= 10)} lines; "
+ f"streak 1: {depth[1]} (= distinct storm episodes)")
+
+
+if __name__ == "__main__":
+ refs = discover()
+ latency_table(refs)
+ retry_table(refs)
+ worker_logs(refs)
diff --git a/harness/bench/time_audit/corpus.py b/harness/bench/time_audit/corpus.py
new file mode 100644
index 00000000..dd1c4908
--- /dev/null
+++ b/harness/bench/time_audit/corpus.py
@@ -0,0 +1,126 @@
+"""Read-only corpus discovery + loaders for the time audit (no side effects)."""
+from __future__ import annotations
+
+import glob
+import json
+import statistics
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+OUT = Path("/home/yipeng/3dcodeverse/harness/bench/out")
+STORM_DAY = "storm(08-26)"
+BASELINE = "base(08-23/25)"
+
+# corpus name -> (glob under bench/out, corpus bucket)
+CORPORA: dict[str, tuple[str, str]] = {
+ "fancy_v1": ("fancy_v1/*/arms/*/cells/*/harness_api-agent_gemini_gemini-3.7-flash/run", STORM_DAY),
+ "h2h_brilliana_v1": ("h2h_brilliana_v1/*/runs/*", STORM_DAY),
+ "h2h_scene_v1": ("h2h_scene_v1/runs/*", STORM_DAY),
+ "refs_v1_graphics": ("refs_v1_graphics/runs/*", STORM_DAY),
+ "teaser": ("teaser/runs/*", BASELINE),
+ "static_v2_flash": ("static_v2_flash/runs/*", BASELINE),
+ "scenes_v1_flash": ("scenes_v1_flash/runs/*", BASELINE),
+}
+
+
+@dataclass
+class RunRef:
+ corpus: str
+ bucket: str
+ run: Path
+ track: str = ""
+ events: list[dict[str, Any]] = field(default_factory=list)
+
+ @property
+ def cell_dir(self) -> Path | None:
+ return self.run.parent if (self.run.parent / "worker.log").is_file() else None
+
+ @property
+ def name(self) -> str:
+ return self.run.parent.parent.name if self.cell_dir else self.run.name
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ if not path.is_file():
+ return rows
+ with path.open() as fh:
+ for line in fh:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ rows.append(json.loads(line))
+ except json.JSONDecodeError:
+ continue
+ return rows
+
+
+def track_of(events: list[dict[str, Any]]) -> str:
+ start = next((e for e in events if e.get("event") == "run.start"), None)
+ if start is None:
+ return "?"
+ tr, lang = start.get("track", "?"), start.get("language", "?")
+ if tr == "static_object":
+ return f"static/{lang}"
+ if tr == "articulated_object":
+ return "articulated"
+ return tr
+
+
+def discover(corpora: dict[str, tuple[str, str]] = CORPORA) -> list[RunRef]:
+ refs: list[RunRef] = []
+ for corpus, (pat, bucket) in corpora.items():
+ for d in sorted(glob.glob(str(OUT / pat))):
+ run = Path(d)
+ ev = read_jsonl(run / "events.jsonl")
+ if not ev:
+ continue
+ refs.append(RunRef(corpus=corpus, bucket=bucket, run=run, track=track_of(ev), events=ev))
+ return refs
+
+
+def cost_rows(ref: RunRef) -> list[dict[str, Any]]:
+ return read_jsonl(ref.run / "telemetry" / "cost.jsonl")
+
+
+def trajectories(ref: RunRef) -> list[tuple[str, dict[str, Any], list[dict[str, Any]]]]:
+ """(session label, result.json, transcript rows) for every trajectory dir."""
+ out: list[tuple[str, dict[str, Any], list[dict[str, Any]]]] = []
+ tdir = ref.run / "trajectories"
+ if not tdir.is_dir():
+ return out
+ for sd in sorted(p for p in tdir.iterdir() if p.is_dir()):
+ res: dict[str, Any] = {}
+ rp = sd / "result.json"
+ if rp.is_file():
+ try:
+ res = json.loads(rp.read_text())
+ except json.JSONDecodeError:
+ res = {}
+ out.append((sd.name, res, read_jsonl(sd / "transcript.jsonl")))
+ return out
+
+
+def rounds(ref: RunRef) -> list[dict[str, Any]]:
+ out: list[dict[str, Any]] = []
+ for p in sorted((ref.run / "rounds").glob("r*.json")):
+ try:
+ out.append(json.loads(p.read_text()))
+ except (OSError, json.JSONDecodeError):
+ continue
+ return out
+
+
+def med_p90(xs: list[float]) -> tuple[float, float, int]:
+ """(median, p90, n); p90 by nearest-rank."""
+ if not xs:
+ return (float("nan"), float("nan"), 0)
+ s = sorted(xs)
+ p90 = s[min(len(s) - 1, int(round(0.9 * (len(s) - 1))))]
+ return (statistics.median(s), p90, len(s))
+
+
+def fmt(x: float, nd: int = 0) -> str:
+ return "-" if x != x else f"{x:.{nd}f}"
diff --git a/harness/bench/time_audit/deadline.py b/harness/bench/time_audit/deadline.py
new file mode 100644
index 00000000..869c9227
--- /dev/null
+++ b/harness/bench/time_audit/deadline.py
@@ -0,0 +1,59 @@
+"""Deadline overshoots: sessions past their 1800-s job timeout, runs killed by the
+bench max_minutes with no round completed, and the storm-streak depth at each
+'giving up after 900 s' (=> seconds per failed 503 attempt, independent of transcripts)."""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from corpus import discover, fmt, med_p90, trajectories # noqa: E402
+
+STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 ")
+GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")
+BUDGET_RE = re.compile(r"elapsed ([\d.]+) min exceeds max_minutes ([\d.]+)")
+SESSION_TIMEOUT_S = 1800.0
+
+
+def main() -> None:
+ refs = discover()
+ over, killed, zero_rounds, depths = [], [], 0, []
+ for ref in refs:
+ if ref.bucket != "storm(08-26)":
+ continue
+ for _, res, _ in trajectories(ref):
+ d = float(res.get("duration_s") or 0)
+ if d > SESSION_TIMEOUT_S and res.get("exit_reason") in ("timeout", "error"):
+ over.append(d - SESSION_TIMEOUT_S)
+ rounds_done = sum(1 for e in ref.events if e["event"] == "round.done")
+ for e in ref.events:
+ if e["event"] == "budget.exceeded":
+ m = BUDGET_RE.search(str(e.get("reason", "")))
+ if m:
+ killed.append((float(m.group(1)) - float(m.group(2))) * 60)
+ zero_rounds += rounds_done == 0
+ cd = ref.cell_dir
+ if cd is None:
+ continue
+ last = 0
+ for line in (cd / "worker.log").read_text(errors="replace").splitlines():
+ m = STORM_RE.match(line)
+ if m and "flash" in m.group(1):
+ last = int(m.group(2))
+ elif GIVEUP_RE.match(line) and "flash" in line:
+ depths.append(last)
+ m, p, n = med_p90(over)
+ print(f"sessions past the {SESSION_TIMEOUT_S:.0f}-s job timeout (exit timeout/error): n={n} overshoot med/p90 = {fmt(m)}/{fmt(p)} s, sum={sum(over):.0f} s")
+ m, p, n = med_p90(killed)
+ print(f"runs stopped by bench max_minutes: n={n}, overshoot past the ceiling med/p90 = {fmt(m)}/{fmt(p)} s; with ZERO completed rounds: {zero_rounds}")
+ m, p, n = med_p90([float(d) for d in depths])
+ if n:
+ print(f"flash give-ups in fancy_v1 logs: n={n}; storm-streak depth at give-up med/p90 = {fmt(m)}/{fmt(p)} "
+ f"=> 900 s / depth = {fmt(900 / m, 1)} s per failed attempt (sleep included, <= 5 s of it)")
+ print(json.dumps({"overshoot_sum_s": round(sum(over)), "killed": len(killed), "zero_rounds": zero_rounds}))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/harness/bench/time_audit/judge_plan.py b/harness/bench/time_audit/judge_plan.py
new file mode 100644
index 00000000..2d417d1a
--- /dev/null
+++ b/harness/bench/time_audit/judge_plan.py
@@ -0,0 +1,55 @@
+"""Judge + planner stages: stage span vs recorded model latency (the difference is
+retry/timeout waiting), per corpus; judge samples that errored and what they cost."""
+from __future__ import annotations
+
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from corpus import cost_rows, discover, fmt, med_p90 # noqa: E402
+
+
+def main() -> None:
+ J: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+ for ref in discover():
+ rows = cost_rows(ref)
+ by_round: dict[int, list[dict]] = defaultdict(list)
+ for r in rows:
+ if r["role"] == "judge":
+ by_round[int(r.get("round") or 0)].append(r)
+ b = J[ref.bucket]
+ for e in ref.events:
+ if e["event"] == "judge.done":
+ d = float(e.get("duration_s") or 0)
+ rs = by_round.get(int(e.get("round") or 0), [])
+ ok = sum(r["latency_ms"] / 1000 for r in rs if r["outcome"] == "ok")
+ err = [r["latency_ms"] / 1000 for r in rs if r["outcome"] != "ok"]
+ b["judge_s"].append(d)
+ b["judge_ok_lat"].append(ok)
+ b["judge_wait"].append(max(0.0, d - ok))
+ b["judge_n_ok"].append(float(sum(1 for r in rs if r["outcome"] == "ok")))
+ if err:
+ b["judge_err_rows"].extend(err)
+ b["judge_rounds_with_err"].append(d)
+ elif e["event"] == "stage.done" and e.get("stage") == "plan":
+ d = float(e.get("duration_s") or 0)
+ pl = [r for r in rows if r["role"] == "planner"]
+ ok = sum(r["latency_ms"] / 1000 for r in pl if r["outcome"] == "ok")
+ b["plan_s"].append(d)
+ b["plan_ok_lat"].append(ok)
+ b["plan_calls"].append(float(len(pl)))
+ b["plan_wait"].append(max(0.0, d - ok))
+ for bucket, b in sorted(J.items()):
+ print(f"## {bucket}")
+ for k in ("judge_s", "judge_ok_lat", "judge_wait", "judge_n_ok", "plan_s", "plan_ok_lat", "plan_calls", "plan_wait"):
+ m, p, n = med_p90(b[k])
+ print(f" {k:14s} n={n:3d} med={fmt(m, 1):>7s} p90={fmt(p, 1):>7s} sum={sum(b[k]):.0f}")
+ er = b["judge_err_rows"]
+ print(f" judge error rows: n={len(er)} sum={sum(er):.0f} s each: {[round(x) for x in sorted(er, reverse=True)[:8]]}")
+ rw = b["judge_rounds_with_err"]
+ print(f" judge rounds with an errored sample: {len(rw)} / {len(b['judge_s'])}; their judge_s: {[round(x) for x in sorted(rw, reverse=True)[:8]]}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/harness/bench/time_audit/keys.py b/harness/bench/time_audit/keys.py
new file mode 100644
index 00000000..6600a19e
--- /dev/null
+++ b/harness/bench/time_audit/keys.py
@@ -0,0 +1,60 @@
+"""Task 4: does any persisted telemetry row say WHICH Gemini key served a call?
+Scans every field name in cost.jsonl / transcript.jsonl / result.json / rounds /
+record.json for something key-like, and greps the raw text for the '…xxxx' redaction
+that gemini.py puts into ChatResponse.raw."""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from corpus import discover, read_jsonl # noqa: E402
+
+KEYLIKE = re.compile(r"key|api_key|credential", re.I)
+REDACTED = re.compile(r'"key":\s*"…[^"]{0,8}"')
+
+
+def walk(obj: object, prefix: str, found: Counter[str]) -> None:
+ if isinstance(obj, dict):
+ for k, v in obj.items():
+ if KEYLIKE.search(str(k)):
+ found[f"{prefix}.{k}"] += 1
+ walk(v, f"{prefix}.{k}", found)
+ elif isinstance(obj, list):
+ for v in obj[:50]:
+ walk(v, prefix + "[]", found)
+
+
+def main() -> None:
+ found: Counter[str] = Counter()
+ redacted = 0
+ files = 0
+ for ref in discover():
+ for p in [ref.run / "telemetry" / "cost.jsonl", ref.run / "record.json", *(ref.run / "rounds").glob("r*.json"),
+ *(ref.run / "trajectories").glob("*/result.json"), *(ref.run / "trajectories").glob("*/transcript.jsonl")]:
+ if not p.is_file():
+ continue
+ files += 1
+ text = p.read_text(errors="replace")
+ redacted += len(REDACTED.findall(text))
+ if p.suffix == ".jsonl":
+ for row in read_jsonl(p):
+ walk(row, p.name, found)
+ else:
+ try:
+ walk(json.loads(text), p.name, found)
+ except json.JSONDecodeError:
+ pass
+ print(f"scanned {files} files; '\"key\": \"…xxxx\"' occurrences: {redacted}")
+ print("key-like field names found (name -> count):")
+ for k, v in found.most_common(20):
+ print(f" {v:7d} {k}")
+ if not any("api" in k.lower() or k.endswith(".key") for k in found):
+ print("=> no per-call key identity is persisted anywhere in the run telemetry.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/harness/bench/time_audit/sessions.py b/harness/bench/time_audit/sessions.py
new file mode 100644
index 00000000..55ffd861
--- /dev/null
+++ b/harness/bench/time_audit/sessions.py
@@ -0,0 +1,58 @@
+"""Task 3: per agent session (api-agent transcripts) — turns, tool calls by name
+with durations, and the model-thinking / tool-execution / waiting split, one table
+per track x corpus."""
+from __future__ import annotations
+
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from corpus import discover, fmt, med_p90, trajectories # noqa: E402
+from turns import parse_turns, split # noqa: E402
+
+
+def main() -> None:
+ sess: dict[tuple[str, str], list[tuple[int, float, float, float, float]]] = defaultdict(list)
+ tools: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+ for ref in discover():
+ k = (ref.bucket, ref.track)
+ for _, res, tr in trajectories(ref):
+ turns = parse_turns(tr)
+ if not turns:
+ continue
+ think, tool, wait = split(turns)
+ sess[k].append((len([t for t in turns if t.latency_s > 0]), float(res.get("duration_s") or 0), think, tool, wait))
+ for t in turns:
+ for name, d in t.tools:
+ tools[k][name].append(d)
+ print("## per session: turns, duration, and the split (medians; shares are of summed session time)")
+ print("| corpus | track | sessions | turns med/p90 | dur s med/p90 | think s | tool s | wait s | think% | tool% | wait% |")
+ print("|---|---|---|---|---|---|---|---|---|---|---|")
+ for k in sorted(sess):
+ rows = sess[k]
+ tm, tp, n = med_p90([float(r[0]) for r in rows])
+ dm, dp, _ = med_p90([r[1] for r in rows])
+ th, _, _ = med_p90([r[2] for r in rows])
+ to, _, _ = med_p90([r[3] for r in rows])
+ wa, _, _ = med_p90([r[4] for r in rows])
+ T = sum(r[2] + r[3] + r[4] for r in rows) or 1.0
+ print(f"| {k[0]} | {k[1]} | {n} | {fmt(tm)}/{fmt(tp)} | {fmt(dm)}/{fmt(dp)} | {fmt(th)} | {fmt(to)} | {fmt(wa)} | "
+ f"{100 * sum(r[2] for r in rows) / T:.0f} | {100 * sum(r[3] for r in rows) / T:.0f} | {100 * sum(r[4] for r in rows) / T:.0f} |")
+ print("\n## tool calls by name: count, median s, p90 s, total s per session (top tools per track; storm-day + baseline pooled by track)")
+ pooled: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+ nsess: dict[str, int] = defaultdict(int)
+ for k, d in tools.items():
+ for name, xs in d.items():
+ pooled[k[1]][name].extend(xs)
+ nsess[k[1]] += len(sess[k])
+ for track in sorted(pooled):
+ items = sorted(pooled[track].items(), key=lambda kv: -sum(kv[1]))
+ print(f" {track} ({nsess[track]} sessions):")
+ for name, xs in items[:12]:
+ m, p, n = med_p90(xs)
+ print(f" {name:18s} n={n:5d} med={fmt(m, 2):>6s} p90={fmt(p, 1):>6s} total/session={sum(xs) / max(1, nsess[track]):6.1f} s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/harness/bench/time_audit/stage_times.py b/harness/bench/time_audit/stage_times.py
new file mode 100644
index 00000000..5349c1b1
--- /dev/null
+++ b/harness/bench/time_audit/stage_times.py
@@ -0,0 +1,156 @@
+"""Task 1: per track x corpus, median / p90 of run wall and every stage, plus
+"waiting on the provider" (retry sleeps + failed attempts), derived from transcript
+timestamps (agent sessions) and from judge/planner stage span minus recorded latency.
+Writes stage_runs.jsonl (one row per run) and prints the table."""
+from __future__ import annotations
+
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).parent))
+from corpus import RunRef, cost_rows, discover, fmt, med_p90, rounds, trajectories # noqa: E402
+from turns import parse_turns, split # noqa: E402
+
+HERE = Path(__file__).parent
+
+
+def round_segments(ev: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """One dict per round.start..round.done segment (an abandoned segment ends at the
+ next round.start / run.start and is flagged)."""
+ segs: list[dict[str, Any]] = []
+ cur: dict[str, Any] | None = None
+ cand: dict[Any, float] = {} # best-of-N candidates are generated BEFORE round.start
+ for e in ev:
+ k = e["event"]
+ if k == "candidates.start":
+ cand[e.get("round")] = e["t"]
+ if k in ("round.start", "run.start"):
+ if cur is not None:
+ cur["abandoned"] = True
+ cur["end"] = e["t"]
+ segs.append(cur)
+ cur = None
+ if k == "round.start":
+ cur = {"round": e.get("round"), "start": min(e["t"], cand.pop(e.get("round"), e["t"])), "abandoned": False}
+ continue
+ if cur is None:
+ continue
+ if k == "build.done":
+ cur.setdefault("first_build", e["t"])
+ cur["last_build"] = e["t"]
+ cur["build_ms"] = cur.get("build_ms", 0) + int(e.get("duration_ms") or 0)
+ cur["n_builds"] = cur.get("n_builds", 0) + 1
+ elif k == "gates.done":
+ cur["gates_t"] = e["t"]
+ elif k == "judge.done":
+ cur["judge_s"] = float(e.get("duration_s") or 0)
+ cur["judge_t"] = e["t"]
+ elif k == "round.done":
+ cur["end"] = e["t"]
+ cur["duration_s"] = float(e.get("duration_s") or (e["t"] - cur["start"]))
+ segs.append(cur)
+ cur = None
+ if cur is not None:
+ cur["abandoned"] = True
+ cur["end"] = ev[-1]["t"]
+ segs.append(cur)
+ return segs
+
+
+def analyse(ref: RunRef) -> dict[str, Any]:
+ ev = ref.events
+ done = [e for e in ev if e["event"] == "run.done"]
+ row: dict[str, Any] = {
+ "corpus": ref.corpus, "bucket": ref.bucket, "track": ref.track, "run": ref.name,
+ "finished": bool(done), "wall_s": ev[-1]["t"] - ev[0]["t"],
+ "status": done[-1].get("status") if done else "RUNNING",
+ }
+ plan_s = sum(float(e.get("duration_s") or 0) for e in ev if e["event"] == "stage.done" and e.get("stage") == "plan")
+ row["plan_s"] = plan_s
+ scene_gen = sum(float(e.get("duration_s") or 0) for e in ev
+ if e["event"] == "stage.done" and e.get("stage") in ("assets", "env", "zones", "assemble"))
+ row["plan_cached"] = any(e["event"] == "stage.cached" and e.get("stage") == "plan" for e in ev)
+ rows = cost_rows(ref)
+ lat = lambda role, ok=True: sum( # noqa: E731
+ r["latency_ms"] / 1000 for r in rows if r["role"] == role and (r["outcome"] == "ok") == ok)
+ row["plan_model_s"] = lat("planner")
+ row["plan_wait_s"] = max(0.0, plan_s - lat("planner")) if plan_s else 0.0
+ row["judge_err_s"] = lat("judge", ok=False) # error rows carry the whole failed span
+ segs = [s for s in round_segments(ev) if not s["abandoned"]]
+ row["abandoned_s"] = sum(s["end"] - s["start"] for s in round_segments(ev) if s["abandoned"])
+ rd = {r.get("index"): r for r in rounds(ref)}
+ gen, build, gates, render, judge, other = [], [], [], [], [], []
+ for s in segs:
+ if "gates_t" not in s or "last_build" not in s:
+ continue
+ b = s["build_ms"] / 1000
+ gen.append(s["last_build"] - s["start"] - b)
+ build.append(b)
+ gates.append(s["gates_t"] - s["last_build"])
+ r = rd.get(s["round"], {})
+ rnd = float(((r.get("renders") or {}).get("duration_ms") or 0)) / 1000
+ render.append(rnd)
+ j = s.get("judge_s", 0.0)
+ judge.append(j)
+ if "judge_t" in s:
+ other.append(max(0.0, s["judge_t"] - s["gates_t"] - j - rnd))
+ judge_ok = lat("judge")
+ row.update(gen_s=sum(gen) + scene_gen, build_s=sum(build), gates_s=sum(gates), render_s=sum(render),
+ judge_s=sum(judge), other_s=sum(other), n_rounds=len(gen),
+ judge_model_s=judge_ok, judge_wait_s=max(0.0, sum(judge) - judge_ok))
+ think = tools = wait = sess = 0.0
+ n_sess = 0
+ for _, res, tr in trajectories(ref):
+ a, b2, c = split(parse_turns(tr))
+ think, tools, wait, sess, n_sess = think + a, tools + b2, wait + c, sess + float(res.get("duration_s") or 0), n_sess + 1
+ row.update(sess_think_s=think, sess_tool_s=tools, sess_wait_s=wait, sess_total_s=sess, n_sessions=n_sess)
+ row["wait_total_s"] = wait + row["plan_wait_s"] + row["judge_wait_s"]
+ return row
+
+
+def main() -> None:
+ refs = discover()
+ out = [analyse(r) for r in refs]
+ (HERE / "stage_runs.jsonl").write_text("".join(json.dumps(r) + "\n" for r in out))
+ cols = ["wall_s", "plan_s", "gen_s", "build_s", "gates_s", "render_s", "judge_s", "wait_total_s", "sess_wait_s", "judge_wait_s", "judge_err_s", "abandoned_s"]
+ groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
+ for r in out:
+ groups[(r["bucket"], r["track"])].append(r)
+ print("| corpus | track | n(fin) | " + " | ".join(f"{c} med/p90" for c in cols) + " |")
+ print("|" + "---|" * (len(cols) + 3))
+ for (bucket, track), rs in sorted(groups.items()):
+ fin = [r for r in rs if r["finished"]]
+ cells = []
+ for c in cols:
+ src = fin if c == "wall_s" else rs
+ m, p, _ = med_p90([float(r[c]) for r in src])
+ cells.append(f"{fmt(m)}/{fmt(p)}")
+ print(f"| {bucket} | {track} | {len(rs)}({len(fin)}) | " + " | ".join(cells) + " |")
+ print("\nshare of run wall: waiting / thinking / tools (medians of per-run ratios, finished runs)")
+ for (bucket, track), rs in sorted(groups.items()):
+ fin = [r for r in rs if r["finished"] and r["wall_s"] > 0]
+ if not fin:
+ continue
+ w = med_p90([100 * r["wait_total_s"] / r["wall_s"] for r in fin])[0]
+ th = med_p90([100 * r["sess_think_s"] / r["wall_s"] for r in fin])[0]
+ to = med_p90([100 * (r["sess_tool_s"] + r["build_s"] + r["gates_s"] + r["render_s"]) / r["wall_s"] for r in fin])[0]
+ jm = med_p90([100 * r["judge_model_s"] / r["wall_s"] for r in fin])[0]
+ print(f" {bucket:14s} {track:16s} wait {fmt(w)}% think {fmt(th)}% tools+build+gates+render {fmt(to)}% judge model {fmt(jm)}%")
+ # per-round view (generate + judge per round) for the storm/baseline contrast
+ print("\nper-round: gen_s and judge_s med/p90 (rounds with a build+gates)")
+ per: dict[tuple[str, str], tuple[list[float], list[float]]] = defaultdict(lambda: ([], []))
+ for r in out:
+ n = max(1, r["n_rounds"])
+ per[(r["bucket"], r["track"])][0].append(r["gen_s"] / n)
+ per[(r["bucket"], r["track"])][1].append(r["judge_s"] / n)
+ for k, (g, j) in sorted(per.items()):
+ gm, gp, _ = med_p90(g)
+ jm, jp, _ = med_p90(j)
+ print(f" {k[0]:14s} {k[1]:16s} gen {fmt(gm)}/{fmt(gp)} judge {fmt(jm)}/{fmt(jp)}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/harness/bench/time_audit/storm_attempts.py b/harness/bench/time_audit/storm_attempts.py
new file mode 100644
index 00000000..a8c1158b
--- /dev/null
+++ b/harness/bench/time_audit/storm_attempts.py
@@ -0,0 +1,78 @@
+"""Price the pieces of "waiting": backoff sleeps (worker.log) vs failed 503 round
+trips (transcript wait minus sleep, per storm line) vs 900-s give-ups (model_error
+rows / turns with wait >= 850 s), plus cell wall vs run span for the fancy_v1 cells."""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from corpus import RunRef, discover, fmt, med_p90, trajectories # noqa: E402
+from turns import parse_turns # noqa: E402
+
+STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")
+GIVEUP_S = 850.0
+
+
+def hist(xs: list[float], edges: list[float]) -> str:
+ c = Counter()
+ for x in xs:
+ c[next((e for e in edges if x < e), float("inf"))] += 1
+ tot = max(1, len(xs))
+ return " ".join(f"<{e:g}s:{100 * c[e] / tot:.0f}%" for e in [*edges, float("inf")])
+
+
+def main() -> None:
+ refs = discover()
+ for bucket in ("base(08-23/25)", "storm(08-26)"):
+ waits, big, errs = [], [], []
+ for ref in [r for r in refs if r.bucket == bucket]:
+ for _, _, tr in trajectories(ref):
+ for t in parse_turns(tr):
+ if t.latency_s > 0:
+ waits.append(t.wait_s)
+ if t.error_s > 0:
+ errs.append(t.error_s)
+ tot = sum(waits) + sum(errs)
+ big = [w for w in waits if w >= GIVEUP_S]
+ print(f"## {bucket}: {len(waits)} generator turns; wait histogram: {hist(waits, [1, 5, 30, 120, 850])}")
+ print(f" total wait {tot:.0f} s; turns with wait>=850 s: {len(big)} = {100 * sum(big) / max(1, tot):.0f}% of all wait; "
+ f"model_error spans: n={len(errs)} sum={sum(errs):.0f} s ({100 * sum(errs) / max(1, tot):.0f}%) "
+ f"med/p90={fmt(med_p90(errs)[0])}/{fmt(med_p90(errs)[1])} s")
+ print(f" wait>=850s turn: {hist(big, [900, 1800, 2700])}")
+ print("## fancy_v1 cells: transcript wait vs logged sleep -> cost of one failed 503 round trip")
+ rt, per_cell, outside = [], [], []
+ for ref in refs:
+ cd = ref.cell_dir
+ if cd is None:
+ continue
+ lines = (cd / "worker.log").read_text(errors="replace").splitlines()
+ n = sleep = 0
+ for m in filter(None, map(STORM_RE.match, lines)):
+ if "flash" in m.group(1):
+ n += 1
+ sleep += int(m.group(3))
+ turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]
+ w = sum(t.wait_s for t in turns) + sum(t.error_s for t in turns)
+ w_nogiveup = sum(min(t.wait_s, 0) if t.wait_s >= GIVEUP_S else t.wait_s for t in turns)
+ if n:
+ rt.append((w_nogiveup - sleep) / n)
+ per_cell.append((ref.name, n, sleep, round(w), round(w_nogiveup)))
+ try:
+ cell = json.loads((cd / "cell.json").read_text())
+ outside.append(float(cell.get("wall_s") or 0) - (ref.events[-1]["t"] - ref.events[0]["t"]))
+ except (OSError, json.JSONDecodeError):
+ pass
+ m, p, k = med_p90(rt)
+ print(f" per-cell (wait_excluding_giveups - sleep) / storm_lines: med {fmt(m, 1)} s p90 {fmt(p, 1)} s (n={k} cells)")
+ print(f" cell wall_s - run event span (pairwise + final pro scoring outside the run): med/p90 = "
+ f"{fmt(med_p90(outside)[0])}/{fmt(med_p90(outside)[1])} s")
+ for row in sorted(per_cell, key=lambda r: -r[3])[:5]:
+ print(" worst:", row)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/harness/bench/time_audit/turns.py b/harness/bench/time_audit/turns.py
new file mode 100644
index 00000000..e831f541
--- /dev/null
+++ b/harness/bench/time_audit/turns.py
@@ -0,0 +1,67 @@
+"""Per-turn reconstruction of an api-agent transcript.
+
+The ledger's ``latency_ms`` is the stopwatch of the SUCCESSFUL attempt only
+(``gemini.py:_once``); backoff sleeps, failed attempts and KeyPool waits are not
+recorded anywhere. Transcript rows are stamped when the model returns
+(``assistant``) and when a tool finishes (``tool_result``), so
+``model_wall = t(assistant) - t(previous row)`` and ``wait = model_wall - latency``.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Turn:
+ turn: int
+ model_wall_s: float
+ latency_s: float
+ wait_s: float
+ tools: list[tuple[str, float]] = field(default_factory=list)
+ error_s: float = 0.0 # time spent in a call that ended in a model_error row
+
+
+def is_gemini_api_session(rows: list[dict[str, Any]]) -> bool:
+ """Only in-process api-agent sessions on Gemini stamp a per-turn latency."""
+ return any(r.get("kind") == "assistant" and (r.get("usage") or {}).get("backend") == "gemini" for r in rows)
+
+
+def parse_turns(rows: list[dict[str, Any]]) -> list[Turn]:
+ turns: list[Turn] = []
+ if not rows or not is_gemini_api_session(rows):
+ return turns
+ prev_end = float(rows[0]["t"])
+ cur: Turn | None = None
+ tool_start = prev_end
+ pending_err = 0.0
+ for r in rows:
+ t = float(r["t"])
+ kind = r.get("kind")
+ if kind == "assistant":
+ wall = max(0.0, t - prev_end)
+ lat = float((r.get("usage") or {}).get("latency_ms") or 0) / 1000.0
+ cur = Turn(int(r.get("turn", len(turns))), wall, lat, max(0.0, wall - lat), error_s=pending_err)
+ pending_err = 0.0
+ turns.append(cur)
+ prev_end = tool_start = t
+ elif kind == "tool_result":
+ if cur is not None:
+ cur.tools.append((str(r.get("name", "?")), max(0.0, t - tool_start)))
+ tool_start = prev_end = t
+ elif kind == "model_error":
+ pending_err += max(0.0, t - prev_end)
+ prev_end = t
+ elif kind in ("nudge", "compact", "system"):
+ prev_end = t
+ if pending_err > 0: # session died in a model_error after the last good turn
+ turns.append(Turn(len(turns), 0.0, 0.0, 0.0, error_s=pending_err))
+ return turns
+
+
+def split(turns: list[Turn]) -> tuple[float, float, float]:
+ """(model thinking s, tool execution s, waiting s) over a session."""
+ think = sum(t.latency_s for t in turns)
+ tools = sum(d for t in turns for _, d in t.tools)
+ wait = sum(t.wait_s + t.error_s for t in turns)
+ return think, tools, wait
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 72748235..23b6a138 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1220,3 +1220,35 @@ streak, up to 900 s); what it costs is one more round-trip on a fresh key. Foll
measuring: rank keys by recent latency (the 30 s keys are consistent), and record the key
index in `telemetry/usage.jsonl` so the distribution of calls per key can be read instead of
probed.
+
+
+## 25. Where the time goes — the 2026-08-26 audit (`docs/TIME_AUDIT_2026-08-26.md`)
+
+51 storm-day runs against 52 baseline runs, every stage and every model call, scripts in
+`bench/time_audit/` (read-only over `bench/out`). The numbers that decide what to build next:
+
+* A blender object run is **1 811 s** median on a healthy provider and **4 726 s** under the storm;
+ build + gates + render together are 3–19 % of a baseline run and 1–4 % of a storm run. The
+ agent session is where the time is: baseline 57 % model thinking / 11 % tools / 32 % waiting;
+ storm day **22 % / 2 % / 77 %**.
+* Waiting is not the backoff sleep. Logged sleeps are ~13 % of the wait; a failed 503 costs a
+ **21–50 s held round-trip** before the provider rejects it, and streaks average 4.7 attempts.
+ **30 % of all storm-day waiting (72 921 s) is 66 give-up spans** — one call retrying to the
+ 900 s `RETRY_DEADLINE_S`, up to three times per turn through `api_agent.MODEL_RETRIES`,
+ never clipped to the session's or the round's remaining budget. 18 of 51 storm runs ended
+ with zero rounds; 14 of them overshot the bench ceiling by 837 s median because the ceiling is
+ only checked between stages.
+* Successful calls are also slower under the storm: flash p50 3.3 → 8.3 s, p90 17 → 31 s; the
+ judge (pro) p50 42 → 50 s. The judge is not the bottleneck at the median; its tail is
+ (2 rounds lost 1 100 s each to three 300 s read timeouts). The planner waits 492 s median
+ per storm-day run for 39 s of model time.
+* No telemetry row records which key served a call, so per-key distribution was unanswerable
+ from the corpus (the §24 probe answered it directly).
+
+Ranked by measured seconds per storm-day run: (1) clip the per-call retry budget to the
+remaining session / round budget (~1 430 s/run) — (2) hedge a 503 retry across 2–3 keys
+(~900–1 400 s/run) — (3) hedge / cap the planner and judge calls (~400 s/run + the judge tail)
+— (4) plan cache on re-runs (548 s) — (5) no sleep on 503 (≤ 13 %; landed with §24) — (6) enforce
+the ceiling inside a round (837 s sooner on killed runs) — (7) flash as the loop judge (~60 s
+baseline) — (8) 30 s render cap (≤ 100 s). (1)–(3) and the key/attempt ledger fields are in
+progress on branch `retry-budget`.
diff --git a/harness/docs/TIME_AUDIT_2026-08-26.md b/harness/docs/TIME_AUDIT_2026-08-26.md
new file mode 100644
index 00000000..26cce8b1
--- /dev/null
+++ b/harness/docs/TIME_AUDIT_2026-08-26.md
@@ -0,0 +1,163 @@
+# Where the time goes — 3dcodeverse harness, storm day (2026-08-26) vs baseline (08-23/25)
+
+Corpus: 51 storm-day runs (fancy_v1 28 cells, h2h_brilliana 12, h2h_scene 5, refs_v1_graphics 6; 45 finished) and
+52 baseline runs (static_v2_flash 20, scenes_v1_flash 4, teaser 28). All api-agent gemini-3.7-flash generator,
+gemini-3.1-pro-preview judge. Scripts (read-only, in this directory): `corpus.py` `turns.py` (shared),
+`stage_times.py` (§1), `calls.py` (§2), `sessions.py` (§3), `keys.py` (§4), `judge_plan.py`, `storm_attempts.py`,
+`deadline.py` (§5 inputs); their raw output is in `out_*.txt`, per-run rows in `stage_runs.jsonl`.
+
+**How "waiting on the provider" was measured — and what is NOT recorded.** `telemetry/cost.jsonl.latency_ms` is the
+stopwatch of the *successful* attempt only (`gemini.py:_once`; `cost/instrument.py` uses `usage.latency_ms or ms`), so
+backoff sleeps, failed 503 round-trips and KeyPool waits leave no row; only a call that fails outright gets an `error`
+row whose latency is the whole wrapper span. There is no `attempts`/`key` field. `worker.log` (fancy cells only) has
+the storm lines but no timestamps. So: generator waiting = `t(assistant row) − t(previous transcript row) − latency`
+per turn (`turns.py`; 88 % of baseline turns give < 1 s, i.e. harness overhead), a `model_error` row's span counts
+fully; judge/planner waiting = stage span − Σ ok-latency. "wait" below is thread-time (scoped rounds run 3–4
+sessions in parallel), which is why it can exceed wall.
+
+## 1. Run wall and stages — median / p90 seconds (n = runs (finished))
+
+| corpus | track | n | wall | plan | generate | build | gates | render | judge | wait (thread) | session wait | judge wait | killed-run span |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| base | static/blender | 23 | 1811/3073 | 43/136 | 900/2102 | 0/1 | 4/69 | 3/68 | 116/448 | 321/1719 | 248/1712 | 1/275 | 0/654 |
+| base | static/cadquery | 4 | 3288/5782 | 307/429 | 1873/3135 | 14/20 | 79/89 | 74/86 | 331/897 | 521/2054 | 285/1675 | 0/0 | 0/1724 |
+| base | static/threejs | 3 | 2403/5152 | 459/584 | 1823/3118 | 0/1 | 66/117 | 66/107 | 123/1366 | 2118/3956 | 1905/3092 | 0/466 | 0/0 |
+| base | graphics | 9 | 926/2003 | 77/122 | 663/1714 | 5/13 | 0/0 | 5/12 | 199/372 | 421/1090 | 275/1019 | 0/132 | 0/0 |
+| base | scene | 10 | 4118/5097 | 50/120 | 2859/4122 | 8/23 | 22/83 | 21/81 | 284/490 | 1293/4214 | 1292/4096 | 0/103 | 0/296 |
+| base | articulated | 3 | 5869/6376 | 325/417 | 3430/3876 | 1/2 | 177/195 | 2/41 | 98/1005 | 1191/1886 | 925/1145 | 0/573 | 0/2195 |
+| storm | static/blender | 23(20) | 4726/5536 | 0*/577 | 1989/3127 | 0/0 | 1/18 | 1/16 | 38/116 | 5356/10333 | 5030/10333 | 0/4 | 2409/5161 |
+| storm | static/cadquery | 8(7) | 6649/8344 | 0*/749 | 3159/5401 | 4/14 | 3/95 | 2/94 | 176/666 | 4116/5222 | 3002/4807 | 43/415 | 902/2265 |
+| storm | static/threejs | 9 | 5328/7802 | 0*/548 | 3595/5501 | 0/0 | 3/5 | 1/3 | 94/430 | 6747/10174 | 6579/8401 | 27/370 | 1606/3648 |
+| storm | graphics | 6 | 6003/8818 | 380/628 | 2971/3620 | 12/13 | 0/0 | 12/13 | 228/574 | 3605/6488 | 3223/5897 | 5/55 | 1001/2330 |
+| storm | scene | 5(3) | 4404/4711 | 576/646 | 1406/2960 | 1/3 | 4/8 | 3/7 | 0/107 | 1218/2823 | 1026/2823 | 0/76 | 0/583 |
+
+\* fancy_v1 cells get a pre-computed plan (`stage.cached`). "killed-run span" = time inside a round that never reached
+`round.done`: 18 of 51 storm runs ended with **zero rounds** (11 of 23 blender cells, median 4 523 s each), 14 of them
+stopped by the bench `max_minutes 60` ceiling, which is only checked between stages, so they overshot it by 837 s
+median / 1 560 s p90. Per round: generate 314→1 008 s (blender), 989→2 049 s (threejs); judge 48→38 s (blender),
+110→76 s (cadquery) — the judge stage is **not** slower on storm day at the median; its tail is (§2). Build 0.1–4 s,
+gates 1–80 s, render 1–90 s: together 1–4 % of storm wall, 3–19 % of baseline wall.
+
+## 2. Per model call
+
+Successful-call latency (s), from the ledger:
+
+| model / role | corpus | n | p50 | p90 | p99 | max |
+|---|---|---|---|---|---|---|
+| gemini-3.7-flash generator | base | 8097 | 3.3 | 17.2 | 49.6 | 172 |
+| gemini-3.7-flash generator | storm | 5867 | **8.3** | **31.1** | 74.9 | 239 |
+| gemini-3.7-flash planner | base / storm | 60 / 36 | 13.7 / 23.4 | 32 / 43 | 53 / 68 | 76 / 68 |
+| gemini-3.1-pro-preview judge | base / storm | 328 / 67 | 42.2 / 50.5 | 73 / 103 | 129 / 165 | 209 / 171 |
+
+Even the calls that succeed are **2.5x slower** under the storm (think time per session 106 → 283 s, §3).
+Calls per run (ok, median/p90): flash 176/292 base vs 102/213 storm; pro 5/15 vs 1/5. Error rows: flash 41 → 133,
+pro 0 → 5. Retries from transcripts (a turn whose wall exceeds its latency by > 1 s):
+
+| corpus | track | runs | calls/run | calls with ≥1 retry /run (share) | wait per retried call med/p90 | retry-wait per run med/p90 | 900-s give-up rows |
+|---|---|---|---|---|---|---|---|
+| base | static/blender | 22 | 186/302 | 10/31 (6 %) | 10.9/63.8 s | 263/1712 s | 1 |
+| base | scene | 8 | 216/265 | 28/43 (13 %) | 19.8/116 s | 2120/4096 s | 4 |
+| base | graphics | 9 | 25/28 | 3/8 (20 %) | 32/158 s | 275/1019 s | 17 |
+| storm | static/blender | 23 | 125/257 | 33/65 (24 %) | **47/333 s** | **5030/10333 s** | 30 |
+| storm | static/cadquery | 7 | 87/148 | 18/30 (21 %) | 55/389 s | 3103/4807 s | 3 |
+| storm | static/threejs | 9 | 160/275 | 59/71 (32 %) | 40/198 s | 6579/8401 s | 8 |
+| storm | graphics | 6 | 44/49 | 10/14 (33 %) | 76/387 s | 3223/5897 s | 24 |
+| storm | scene | 3 | 36/243 | 4/49 (20 %) | 12/70 s | 1130/2823 s | 1 |
+
+Where the storm-day wait goes (246 297 s over 5 799 turns): **30 % (72 921 s) is 66 `model_error` spans** — a call that
+retried until the 900-s `RETRY_DEADLINE_S` (median span 923 s, p90 2 743 s = three consecutive give-ups on one turn
+via `api_agent.MODEL_RETRIES=3`); 45 sessions were hit, ~1 430 s per storm run on average. The other 70 % is ordinary
+storm streaks. From the 28 fancy worker.logs: 5 143 flash storm lines + 158 pro, 46 flash + 10 pro give-ups, 44 pro
+read-timeouts (300 s each). Logged **sleep is only 645 s/cell median (1 525 p90), 21 943 s in total = ~13 % of the
+wait**; `(wait − sleep) / storm lines` = **21.5 s per failed attempt** (p90 28.5), and the streak depth at give-up
+(median 18 of 60) says 900/18 = ~50 s per attempt late in a storm. The cost of a 503 is the round-trip the provider
+holds before rejecting, not the ≤ 5 s backoff. Judge: 2 of 56 storm rounds lost 1 162 s and 927 s (3 × 300-s
+timeouts before the deadline) then a second sample answered in 128 s → `judge_s` 1 818 / 1 593 s; judge wait sum
+6 847 s / 56 rounds (median 1.8 s, p90 370 s). Planner: storm plan stage 548 s median (p90 995) for 39 s of model
+time → **492 s of waiting per run that plans**; baseline 69 s (17 s model). Outside the run, the fancy cells spend a
+further 252 s median / 935 s p90 in the driver's pairwise + final pro scoring (`cell.wall_s` − run span).
+
+## 3. Per agent session (api-agent transcripts): thinking vs tools vs waiting
+
+| corpus | track | sessions | turns med/p90 | duration med/p90 | think s | tool s | wait s | think % | tool % | wait % |
+|---|---|---|---|---|---|---|---|---|---|---|
+| base | static/blender | 155 | 25/60 | 158/446 | 106 | 6 | 12 | 57 | 11 | 32 |
+| base | static/cadquery | 10 | 30/60 | 511/1659 | 277 | 23 | 129 | 56 | 9 | 34 |
+| base | static/threejs | 32 | 19/38 | 298/715 | 137 | 25 | 117 | 43 | 12 | 45 |
+| base | graphics | 24 | 8/17 | 141/754 | 87 | 5 | 31 | 42 | 2 | 56 |
+| base | scene | 82 | 20/30 | 336/933 | 128 | 32 | 90 | 35 | 14 | 52 |
+| base | articulated | 6 | 34/38 | 1259/1825 | 602 | 213 | 361 | 45 | 23 | 32 |
+| storm | static/blender | 130 | 20/45 | **1528**/2408 | 283 | 4 | **976** | 22 | 2 | **77** |
+| storm | static/cadquery | 22 | 26/49 | 1802/2403 | 417 | 28 | 1025 | 30 | 2 | 68 |
+| storm | static/threejs | 57 | 23/46 | 1288/2181 | 340 | 10 | 707 | 29 | 3 | 69 |
+| storm | graphics | 23 | 9/17 | 841/2932 | 179 | 5 | 612 | 17 | 1 | 83 |
+| storm | scene | 11 | 19/57 | 581/890 | 194 | 15 | 207 | 29 | 4 | 67 |
+
+Tool execution (pooled, median / p90 / seconds per session): blender `render_views` 0.8/61 → 9.6 s, `render_sheet`
+0.9/61 → 7.4, `isolate` 0.7/61 → 3.9, `build` 0.5/0.7 → 2.3, `check_connectivity` 0.13/0.5 → 0.6; cadquery `build`
+3.3/5.9 → 17 s; threejs `render_views` 2.5/87 → 16 s, `build` 1.0/1.7 → 5; graphics `gl_probe` 1.0/1.3 → 2.2,
+`build` 4.8/6 → 2.0, `gl_frames` 2.8/3.2 → 1.9; scene `scene_views` 6.1/16 → 25 s, `scene_probe` 2.4/16 → 17,
+`build` 3.1/8.3 → 15; articulated `joint_sweep` 50/93 → 259 s. read/write/edit/list/check_contract/measure ≈ 0.
+Renders have a 60-s-cap tail (p90 = 61 s) but total tool time is ≤ 30 s per session everywhere except `joint_sweep`.
+Sessions past their 1 800-s job timeout (exit timeout/error): 97 on storm day, overshoot 182 s med / 1 154 s p90,
+41 355 s in total — the deadline is only checked at the top of a turn, never inside the retry loop.
+
+## 4. Key rotation — evidence and mechanism
+
+**No telemetry row records which key served a call.** `keys.py` scanned 1 542 files (cost.jsonl, transcripts,
+result.json, rounds, record.json): zero occurrences of the `"key": "…xxxx"` that `gemini.py:_once` puts into
+`ChatResponse.raw`, and no key-like field anywhere (only `settings.key_pool_size`). Per-key distribution and
+"is one key hammered" are therefore unanswerable from the corpus — that is the finding; `KeyPool.stats()` exists but is
+only surfaced by `3dcv doctor --live`, never persisted. How rotation works today (`keypool.py`, `retry.py`,
+`storm.py`, `gemini.py`, `agents/api_agent.py`, `judges/vlm_judge.py`):
+1. `KeyPool.acquire` is **round-robin per call**: the cursor advances past the chosen key, so consecutive calls (and
+ consecutive retries of one call) land on different keys by construction, subject to per-key RPM (1 000/min) and TPM
+ (1 M/min, reserved from the estimated prompt tokens, reconciled on report), a ≤ 5 s 429 cooldown, health ≥ 0.3,
+ a 1-h bench for dead keys, and a process-wide `max_in_flight = 64` semaphore; it blocks ≤ 120 s if nothing is ready.
+2. Key-scoped errors (401/403, invalid key) rotate at once for free; 429 rotates for free with a 0.5 s pause while an
+ untried key remains, then counts against `max_attempts = 6` with backoff 1·2^n capped at `MAX_WAIT_S = 5`.
+3. **503/529 = "storm" branch**: sleep `min(5, 2^storm × [0.75, 1.25))` s, re-acquire (a new key), up to 60 storm
+ attempts that do NOT consume `max_attempts`, bounded only by `RETRY_DEADLINE_S = 900` — checked **after** a failed
+ attempt returns, so a 300-s read timeout (`model_timeout_s`) can push one call to ~1 200 s. Other retryables
+ (timeouts, bad JSON, empty candidates) get the 6-attempt/900-s budget. `StormGate` exists but is OFF (lost its A/B).
+4. Above that, `api_agent._generate` retries a failed *turn* 3x (sleep 2, 4 s) and `VlmJudge._sample` retries a
+ sample 3x, each retry re-entering the full 900-s machine: one turn can burn 2 700 s; a session's `timeout_s` and the
+ run's `max_minutes` are only checked between turns / stages. Each `ab_plan` cell is its own process (own pool).
+What costs wall clock under a 503 storm: (a) every attempt is **sequential and single-flight** — one request in the
+air per logical call, and a failed 503 costs 20–50 s of held round-trip before the ≤ 5 s sleep even starts; (b) no
+hedging across keys, though each attempt already lands on a fresh key; (c) the 900-s budget is per *call*, multiplied
+by the outer 3x loops, and never clipped to the remaining session/round/run budget; (d) the judge's 200 k-token call
+gets the same 300-s read timeout x 3 before its deadline; (e) a scoped baseline round waits for the slowest of 3–4
+sessions, so one session's give-up chain stalls the round and the run runs into the 60-min ceiling with nothing built.
+
+## 5. Accelerations, ranked by measured seconds per storm-day run (base-day gain in brackets)
+
+1. **Clip the per-call retry budget to the remaining session/round/run budget and cap it (~120 s inside an agent
+ turn; a turn that still fails ends the session as `error` and the round salvages what exists).** Removes the 66
+ give-up spans: 72 921 s / 51 runs = **~1 430 s/run mean** (~800 s per span at a 120-s cap), plus the 837 s median
+ overshoot on the 32 ceiling-killed runs and the 41 355 s of session overshoot. Code: `retry.py` (`max_total_s`
+ from caller), `api_agent._generate` (check `t_deadline` inside `MODEL_RETRIES`), `vlm_judge._sample`,
+ `tracks/generation.py` (`timeout_s = min(agent_timeout_s, budget remaining)`). [base: 7 137 s / 52 runs ≈ 140 s]
+2. **Hedge the retry of a 503: after the first 503, issue the next attempt on 2–3 keys concurrently, take the first
+ success (a 503 bills nothing, so the hedge is free while storming).** Storm streaks average 4.7 attempts (1 101
+ episodes / 5 143 lines) at 21–50 s each; a 3-way hedge cuts expected rounds to ~1.7, i.e. ~60 % of the retry
+ wait: 33 retried calls × 47 s × 0.6 ≈ **930 s/run blender**, 59 × 40 × 0.6 ≈ **1 400 s threejs**, ~600 s cadquery,
+ ~450 s graphics. Code: `retry.py` + `gemini.py` (a small thread fan-out), validate with `concurrency_probe.py`.
+3. **Planner and judge: hedge on a second key after ~1.5 × p90 (60 s planner, 150 s judge) and cap the sample at
+ ~200 s instead of 3 × 300 s.** Planner: 492 s median waiting per run that plans (h2h/refs/scene; **~400 s/run**);
+ judge: 3 150 s over the 2 timed-out rounds = **~56 s/round mean**, `judge_s` p90 451 → ~170 s. Code
+ (`vlm_judge._sample`, planner call site) or a role-specific `model_timeout_s` (today one global 300 s).
+4. **Pre-compute / cache the plan per prompt** (settings — `ab_plan` already does it: fancy plan stage = 0 s):
+ **548 s/run** saved on any re-run or second arm of the same prompt on storm day; 69 s baseline.
+5. **Rotate to the next key with no (or 0.5 s) sleep on 503** (settings: `GeminiModel(base_delay, storm_max_delay)`):
+ the sleeps are 645 s/cell median, 1 525 p90 — **≤ 13 % of the wait**; only worth it together with 2.
+6. **Enforce the bench ceiling inside a round** (code, `lifecycle` budget check in the session fan-out): the 14
+ zero-round killed runs would fail **837 s (p90 1 560 s) sooner**; they produce nothing either way unless salvage
+ builds the parts that did finish (2 of 4 sessions typically complete at ~1 500–1 900 s).
+7. **Cheaper loop judge** (setting `--loop-judge gemini:gemini-3.7-flash`, pro for the final): judge model time is
+ 12 % of baseline blender wall (116 s/run); flash p50 21 s vs pro 42 s → **~60 s/run base, ~20 s storm**.
+8. **Shorter render tools** (code: 30-s render cap): render/isolate p90 = 61–91 s tail, total ≤ 30 s/session →
+ **≤ ~100 s/run**; `joint_sweep` (259 s/session, articulated) is the only tool worth batching.
+9. Parallel judge samples — already parallel (`fan_out`; baseline `judge_s` 52 s vs Σ samples 100 s): **0 s**.
+Not fixable from here: successful flash calls are 2.5x slower during the storm (p50 8.3 s), which alone adds ~180 s
+of think time per session; the only lever is fewer turns.