Official Codebase of 3DCodeVerse
Python
bench: session_stats stops re-parsing the envelope, counting killed sessions and absorbing sub-runs
ziyaod8c5718aa50a
diff --git a/harness/bench/session_stats.py b/harness/bench/session_stats.pyindex 7a32e021..6caa253a 100644--- a/harness/bench/session_stats.py+++ b/harness/bench/session_stats.py@@ -23,6 +23,18 @@ from pathlib import Path#: gemini-cli names an MCP tool ``mcp_<server>_<tool>``; ours is the ``3dcv`` server.MCP_PREFIX = "mcp_3dcv_"+#: run-layout directories that hold a SUB-workspace (a scene asset candidate, a rejected+#: candidate): their sessions belong to that sub-run, not to the battery cell above them,+#: and ``flywheel.record.find_runs`` skips them for the same reason.+SUBRUN_DIRS = frozenset({"_cand", "_assets"})+++def _killed(root: Path) -> int:+ """Sessions whose ``stdout.json`` is empty: the CLI died or was killed before printing+ its stats block. They carry no numbers, so they are not sessions for the rates above —+ but they are not nothing either, and a battery with many of them was a bad window."""+ return sum(1 for p in root.rglob("stdout.json", recurse_symlinks=True)+ if "trajectories" in p.parts and not SUBRUN_DIRS & set(p.parts) and p.stat().st_size <= 2)def _sessions(root: Path) -> list[Path]:@@ -36,8 +48,10 @@ def _sessions(root: Path) -> list[Path]:"""seen: dict[Path, Path] = {}for p in sorted(root.rglob("stdout.json", recurse_symlinks=True)):- if "trajectories" not in p.parts:+ if "trajectories" not in p.parts or SUBRUN_DIRS & set(p.parts):continue+ if p.stat().st_size <= 2:+ continue # the CLI was killed before printing its stats: not a session's worthseen.setdefault(p.resolve(), p)return sorted(seen.values())@@ -60,18 +74,33 @@ def tool_rates(sessions: list[Path]) -> tuple[Counter, Counter]:def token_rates(sessions: list[Path]) -> dict[str, float]:- """Prompt/cached tokens and requests summed over every model a session used."""+ """Prompt/cached tokens and requests for each session's MAIN model.++ ``cost.reconstruct._gemini_cli_usages`` owns the envelope — including the older+ ``tokens.input`` schema this file would otherwise read as zero — so the tokens come+ from it rather than from a second parser. A session's ``models`` map also carries+ gemini-cli's own bookkeeping model (one request per session for the session title),+ and docs/COST.md §30 counts main-role requests: the model with the most requests IS+ the main role, and mixing the other in moves ``uncached per request`` by ~3 %.+ """+ from codeverse.cost.reconstruct import _gemini_cli_usages+prompt = cached = requests = 0for path in sessions:try:- models = ((json.loads(path.read_text()) or {}).get("stats") or {}).get("models") or {}+ stdout = json.loads(path.read_text()) or {}except (OSError, ValueError):continue- for m in models.values():- tok, api = m.get("tokens") or {}, m.get("api") or {}- prompt += int(tok.get("prompt") or 0)- cached += int(tok.get("cached") or 0)- requests += int(api.get("totalRequests") or 0)+ models = ((stdout.get("stats") or {}).get("models")) or {}+ if not models:+ continue+ main = max(models, key=lambda n: int(((models[n] or {}).get("api") or {}).get("totalRequests") or 0))+ usage = next((u for u in _gemini_cli_usages(stdout) if u.model == main), None)+ if usage is None:+ continue+ prompt += usage.input_tokens+ cached += usage.cached_tokens+ requests += int(((models[main] or {}).get("api") or {}).get("totalRequests") or 0)return {"requests": requests, "prompt": prompt, "cached": cached,"cache_hit": cached / prompt if prompt else float("nan"),"uncached_per_request": (prompt - cached) / requests if requests else float("nan")}@@ -82,7 +111,7 @@ def round_costs(root: Path) -> list[float]:out: list[float] = []seen: set[Path] = set()for rec in root.rglob("record.json", recurse_symlinks=True):- if rec.resolve() in seen:+ if rec.resolve() in seen or SUBRUN_DIRS & set(rec.parts):continueseen.add(rec.resolve())try:@@ -95,8 +124,8 @@ def round_costs(root: Path) -> list[float]:def report(roots: list[Path]) -> str:- lines = ["| battery | sessions | MCP calls | reported as errors | requests | cache hit | "- "uncached/req | rounds | $ per round (median) |", "|---|--:|--:|--:|--:|--:|--:|--:|--:|"]+ lines = ["| battery | sessions | killed | MCP calls | reported as errors | requests | cache hit | "+ "uncached/req | rounds | $ per round (median) |", "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|"]per_tool: dict[str, tuple[Counter, Counter]] = {}for root in roots:sessions = _sessions(root)@@ -106,11 +135,12 @@ def report(roots: list[Path]) -> str:costs = round_costs(root)n_calls, n_failed = sum(calls.values()), sum(failed.values())lines.append(- f"| {root.name} | {len(sessions)} | {n_calls} | "+ f"| {root.name} | {len(sessions)} | {_killed(root)} | {n_calls} | "f"{n_failed / n_calls:.3f} ({n_failed}) | {tok['requests']} | {tok['cache_hit']:.2f} | "f"{tok['uncached_per_request']:,.0f} | {len(costs)} | "f"{statistics.median(costs):.3f} |" if n_calls and costs else- f"| {root.name} | {len(sessions)} | {n_calls} | — | {tok['requests']} | — | — | {len(costs)} | — |")+ f"| {root.name} | {len(sessions)} | {_killed(root)} | {n_calls} | — | {tok['requests']} | — | — | "+ f"{len(costs)} | — |")for name, (calls, failed) in per_tool.items():if not calls:continuediff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 4cfee6aa..49da7772 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1491,10 +1491,11 @@ block and each round's recorded `usage`:| per generator request | before (`aa_articulated`) | after (`wave2_lean`) ||---|--:|--:|-| sessions / requests | 111 / 2 594 | 102 / 2 722 |+| sessions with stats / killed before printing | 60 / 102 | 59 / 86 |+| main-role requests | 2 537 | 2 657 || MCP tool calls reported as errors | 26.6 % (482 / 1 814) | **1.3 %** (25 / 1 863) |-| cache hit | 68.8 % | **89.2 %** |-| uncached prompt tokens | 38 390 | **13 945** (−64 %) |+| cache hit | 69 % | **90 %** |+| uncached prompt tokens per request | 38 521 | **13 480** (−65 %) || generator $ per round | mean 1.900, median 1.572 | **mean 1.242, median 0.950** (−35 % / −40 %) |**Correction (2026-09-03), and what it does NOT explain.** The after column first readdiff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.mdindex f640f58d..5b537511 100644--- a/harness/docs/DECISIONS.md+++ b/harness/docs/DECISIONS.md@@ -706,8 +706,8 @@ written) that were accepted because the code works that way and the tests pin itat n=14, found by reading the recorded tool stats. **Measured after the fix**(`aa_articulated` before, `wave2_lean` after, same battery and config): calls reportedas errors 26.6 % (482/1 814) → **1.3 %** (25/1 863; the remainder are genuine- `Observation.error` cases), cache hit 68.8 % → **89.2 %**, uncached prompt tokens per- request 38 390 → **13 945**, generator dollars per round median 1.572 → **0.950**+ `Observation.error` cases), cache hit 69 % → **90 %**, uncached prompt tokens per+ main-role request 38 521 → **13 480**, generator dollars per round median 1.572 → **0.950**(−40 %). `bench/session_stats.py` computes both columns from the recorded sessions —the first, hand-computed after column (1.4 %, 13 736, $0.967 over "108 sessions") isNOT reproduced by it and is withdrawn: no tested mechanism explains its shape (1.87x ondiff --git a/harness/docs/PAPER_WRITING.md b/harness/docs/PAPER_WRITING.mdindex be03c61e..02775cb9 100644--- a/harness/docs/PAPER_WRITING.md+++ b/harness/docs/PAPER_WRITING.md@@ -249,7 +249,7 @@ paper needs from this section:verdict; the MCP boundary bounds text characters, image count and encoded image bytes.* **The readout** (`bench/session_stats.py aa_articulated wave2_lean`, both columnsprinted by the script): calls reported as errors **26.6 % → 1.3 %**, cache- **68.8 % → 89.2 %**, uncached prompt tokens per request **38 390 → 13 945**, generator+ **69 % → 90 %**, uncached prompt tokens per request **38 521 → 13 480**, generatordollars per round median **1.572 → 0.950**. The residual 1.3 % are genuine`Observation.error` cases. An earlier hand-computed after column (1.4 %, 13 736,\$0.967) is **withdrawn**: the script does not reproduce it and no tested mechanism@@ -420,7 +420,7 @@ data the repo does not carry; they are marked in the caveat column.| planner mortality | 4.7 % → 0.7 %, Fisher p = 0.0067, n = 560 calls | `bench/data/plan_stage/restart_{on,off}.jsonl` (in the repo) | plan stage only, both arms same window || tool calls reported as errors | 26.6 % → 1.3 % | `aa_articulated` vs `wave2_lean` stats envelopes | same battery/config, different weather || cache hit | 69 % → 89 % | same | same |-| uncached tokens per request | 38 390 → 13 945 | same | same |+| uncached tokens per request | 38 521 → 13 480 (main-role) | same | same || generator $ per round | median 1.572 → 0.950 | round records of both runs | same || base64 amplification | 261 k tokens vs 516 for one 275 kB sheet | measured on a recorded blob + vendor bundle | arithmetic + matching recorded prompt jumps; **no repo script** || token growth in turns | input(t) = 12 364 + 1 822·t; total ∝ n^1.60; dollars ∝ n^1.19–1.34 | 1 063 in-process sessions, 30 401 turns | **no repo script** — the in-process agent, since gemini-cli emits no per-turn usage |diff --git a/harness/tests/compare_bench/test_session_stats.py b/harness/tests/compare_bench/test_session_stats.pyindex c5d5ed84..921120c9 100644--- a/harness/tests/compare_bench/test_session_stats.py+++ b/harness/tests/compare_bench/test_session_stats.py@@ -58,16 +58,35 @@ def test_rates_and_costs_over_two_sessions(tmp_path: Path) -> None:assert "| 3.000 |" in report([tmp_path]) # the median round, not the mean-def test_a_dead_session_is_skipped_not_fatal(tmp_path: Path) -> None:+def test_a_killed_session_is_counted_as_killed_not_as_a_session(tmp_path: Path) -> None:+ """An empty stdout.json is the CLI dying before it printed its stats. Counting it as a+ session makes every per-session number smaller for a reason that has nothing to do with+ the code under test — 102 of aa_articulated's 162 files are that."""+ from bench.session_stats import _killed+run = _run(tmp_path, "cell_a", [2.0])_session(run, "baseline_r00", tools={"build": (3, 0)}, prompt=100, cached=50, requests=1)dead = run / "trajectories" / "refine_r01"dead.mkdir(parents=True)- (dead / "stdout.json").write_text("") # the CLI was killed before it printed+ (dead / "stdout.json").write_text("")- assert len(_sessions(tmp_path)) == 2+ assert len(_sessions(tmp_path)) == 1 and _killed(tmp_path) == 1calls, _ = tool_rates(_sessions(tmp_path))assert calls == {"build": 3}+ assert "| 1 | 1 |" in report([tmp_path]) # sessions | killed+++def test_a_sub_workspace_is_not_the_battery_cell_above_it(tmp_path: Path) -> None:+ """A scene asset's own run lives under ``_cand/`` / ``_assets/`` with its own+ trajectories; ``flywheel.record.find_runs`` skips those and so must this, or one cell's+ numbers absorb every candidate it rejected."""+ run = _run(tmp_path, "cell_a", [1.0])+ _session(run, "baseline_r00", tools={"build": (2, 0)}, prompt=100, cached=50, requests=1)+ sub = run / "_cand" / "c1" / "run"+ _session(sub, "baseline_r00", tools={"build": (99, 9)}, prompt=999, cached=0, requests=9)++ calls, failed = tool_rates(_sessions(tmp_path))+ assert dict(calls) == {"build": 2} and failed["build"] == 0def test_the_coupling_survey_counts_degrees_of_freedom_not_joints(tmp_path: Path) -> None: