Skip to content

Official Codebase of 3DCodeVerse

Python

bench: session_stats stops re-parsing the envelope, counting killed sessions and absorbing sub-runs

ziyaod8c5718aa50a
diff --git a/harness/bench/session_stats.py b/harness/bench/session_stats.py
index 7a32e021..6caa253a 100644
--- a/harness/bench/session_stats.py
+++ b/harness/bench/session_stats.py
@@ -23,6 +23,18 @@ from pathlib import Path
#: gemini-cli names an MCP tool ``mcp_<server>_<tool>``; ours is the ``3dcv`` server.
MCP_PREFIX = "mcp_3dcv_"
+#: run-layout directories that hold a SUB-workspace (a scene asset candidate, a rejected
+#: candidate): their sessions belong to that sub-run, not to the battery cell above them,
+#: and ``flywheel.record.find_runs`` skips them for the same reason.
+SUBRUN_DIRS = frozenset({"_cand", "_assets"})
+
+
+def _killed(root: Path) -> int:
+ """Sessions whose ``stdout.json`` is empty: the CLI died or was killed before printing
+ its stats block. They carry no numbers, so they are not sessions for the rates above —
+ but they are not nothing either, and a battery with many of them was a bad window."""
+ return sum(1 for p in root.rglob("stdout.json", recurse_symlinks=True)
+ if "trajectories" in p.parts and not SUBRUN_DIRS & set(p.parts) and p.stat().st_size <= 2)
def _sessions(root: Path) -> list[Path]:
@@ -36,8 +48,10 @@ def _sessions(root: Path) -> list[Path]:
"""
seen: dict[Path, Path] = {}
for p in sorted(root.rglob("stdout.json", recurse_symlinks=True)):
- if "trajectories" not in p.parts:
+ if "trajectories" not in p.parts or SUBRUN_DIRS & set(p.parts):
continue
+ if p.stat().st_size <= 2:
+ continue # the CLI was killed before printing its stats: not a session's worth
seen.setdefault(p.resolve(), p)
return sorted(seen.values())
@@ -60,18 +74,33 @@ def tool_rates(sessions: list[Path]) -> tuple[Counter, Counter]:
def token_rates(sessions: list[Path]) -> dict[str, float]:
- """Prompt/cached tokens and requests summed over every model a session used."""
+ """Prompt/cached tokens and requests for each session's MAIN model.
+
+ ``cost.reconstruct._gemini_cli_usages`` owns the envelope — including the older
+ ``tokens.input`` schema this file would otherwise read as zero — so the tokens come
+ from it rather than from a second parser. A session's ``models`` map also carries
+ gemini-cli's own bookkeeping model (one request per session for the session title),
+ and docs/COST.md §30 counts main-role requests: the model with the most requests IS
+ the main role, and mixing the other in moves ``uncached per request`` by ~3 %.
+ """
+ from codeverse.cost.reconstruct import _gemini_cli_usages
+
prompt = cached = requests = 0
for path in sessions:
try:
- models = ((json.loads(path.read_text()) or {}).get("stats") or {}).get("models") or {}
+ stdout = json.loads(path.read_text()) or {}
except (OSError, ValueError):
continue
- for m in models.values():
- tok, api = m.get("tokens") or {}, m.get("api") or {}
- prompt += int(tok.get("prompt") or 0)
- cached += int(tok.get("cached") or 0)
- requests += int(api.get("totalRequests") or 0)
+ models = ((stdout.get("stats") or {}).get("models")) or {}
+ if not models:
+ continue
+ main = max(models, key=lambda n: int(((models[n] or {}).get("api") or {}).get("totalRequests") or 0))
+ usage = next((u for u in _gemini_cli_usages(stdout) if u.model == main), None)
+ if usage is None:
+ continue
+ prompt += usage.input_tokens
+ cached += usage.cached_tokens
+ requests += int(((models[main] or {}).get("api") or {}).get("totalRequests") or 0)
return {"requests": requests, "prompt": prompt, "cached": cached,
"cache_hit": cached / prompt if prompt else float("nan"),
"uncached_per_request": (prompt - cached) / requests if requests else float("nan")}
@@ -82,7 +111,7 @@ def round_costs(root: Path) -> list[float]:
out: list[float] = []
seen: set[Path] = set()
for rec in root.rglob("record.json", recurse_symlinks=True):
- if rec.resolve() in seen:
+ if rec.resolve() in seen or SUBRUN_DIRS & set(rec.parts):
continue
seen.add(rec.resolve())
try:
@@ -95,8 +124,8 @@ def round_costs(root: Path) -> list[float]:
def report(roots: list[Path]) -> str:
- lines = ["| battery | sessions | MCP calls | reported as errors | requests | cache hit | "
- "uncached/req | rounds | $ per round (median) |", "|---|--:|--:|--:|--:|--:|--:|--:|--:|"]
+ lines = ["| battery | sessions | killed | MCP calls | reported as errors | requests | cache hit | "
+ "uncached/req | rounds | $ per round (median) |", "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|"]
per_tool: dict[str, tuple[Counter, Counter]] = {}
for root in roots:
sessions = _sessions(root)
@@ -106,11 +135,12 @@ def report(roots: list[Path]) -> str:
costs = round_costs(root)
n_calls, n_failed = sum(calls.values()), sum(failed.values())
lines.append(
- f"| {root.name} | {len(sessions)} | {n_calls} | "
+ f"| {root.name} | {len(sessions)} | {_killed(root)} | {n_calls} | "
f"{n_failed / n_calls:.3f} ({n_failed}) | {tok['requests']} | {tok['cache_hit']:.2f} | "
f"{tok['uncached_per_request']:,.0f} | {len(costs)} | "
f"{statistics.median(costs):.3f} |" if n_calls and costs else
- f"| {root.name} | {len(sessions)} | {n_calls} | — | {tok['requests']} | — | — | {len(costs)} | — |")
+ f"| {root.name} | {len(sessions)} | {_killed(root)} | {n_calls} | — | {tok['requests']} | — | — | "
+ f"{len(costs)} | — |")
for name, (calls, failed) in per_tool.items():
if not calls:
continue
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 4cfee6aa..49da7772 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1491,10 +1491,11 @@ block and each round's recorded `usage`:
| per generator request | before (`aa_articulated`) | after (`wave2_lean`) |
|---|--:|--:|
-| sessions / requests | 111 / 2 594 | 102 / 2 722 |
+| sessions with stats / killed before printing | 60 / 102 | 59 / 86 |
+| main-role requests | 2 537 | 2 657 |
| MCP tool calls reported as errors | 26.6 % (482 / 1 814) | **1.3 %** (25 / 1 863) |
-| cache hit | 68.8 % | **89.2 %** |
-| uncached prompt tokens | 38 390 | **13 945** (−64 %) |
+| cache hit | 69 % | **90 %** |
+| uncached prompt tokens per request | 38 521 | **13 480** (−65 %) |
| generator $ per round | mean 1.900, median 1.572 | **mean 1.242, median 0.950** (−35 % / −40 %) |
**Correction (2026-09-03), and what it does NOT explain.** The after column first read
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index f640f58d..5b537511 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -706,8 +706,8 @@ written) that were accepted because the code works that way and the tests pin it
at n=14, found by reading the recorded tool stats. **Measured after the fix**
(`aa_articulated` before, `wave2_lean` after, same battery and config): calls reported
as errors 26.6 % (482/1 814) → **1.3 %** (25/1 863; the remainder are genuine
- `Observation.error` cases), cache hit 68.8 % → **89.2 %**, uncached prompt tokens per
- request 38 390 → **13 945**, generator dollars per round median 1.572 → **0.950**
+ `Observation.error` cases), cache hit 69 % → **90 %**, uncached prompt tokens per
+ main-role request 38 521 → **13 480**, generator dollars per round median 1.572 → **0.950**
(−40 %). `bench/session_stats.py` computes both columns from the recorded sessions —
the first, hand-computed after column (1.4 %, 13 736, $0.967 over "108 sessions") is
NOT reproduced by it and is withdrawn: no tested mechanism explains its shape (1.87x on
diff --git a/harness/docs/PAPER_WRITING.md b/harness/docs/PAPER_WRITING.md
index be03c61e..02775cb9 100644
--- a/harness/docs/PAPER_WRITING.md
+++ b/harness/docs/PAPER_WRITING.md
@@ -249,7 +249,7 @@ paper needs from this section:
verdict; the MCP boundary bounds text characters, image count and encoded image bytes.
* **The readout** (`bench/session_stats.py aa_articulated wave2_lean`, both columns
printed by the script): calls reported as errors **26.6 % → 1.3 %**, cache
- **68.8 % → 89.2 %**, uncached prompt tokens per request **38 390 → 13 945**, generator
+ **69 % → 90 %**, uncached prompt tokens per request **38 521 → 13 480**, generator
dollars per round median **1.572 → 0.950**. The residual 1.3 % are genuine
`Observation.error` cases. An earlier hand-computed after column (1.4 %, 13 736,
\$0.967) is **withdrawn**: the script does not reproduce it and no tested mechanism
@@ -420,7 +420,7 @@ data the repo does not carry; they are marked in the caveat column.
| planner mortality | 4.7 % → 0.7 %, Fisher p = 0.0067, n = 560 calls | `bench/data/plan_stage/restart_{on,off}.jsonl` (in the repo) | plan stage only, both arms same window |
| tool calls reported as errors | 26.6 % → 1.3 % | `aa_articulated` vs `wave2_lean` stats envelopes | same battery/config, different weather |
| cache hit | 69 % → 89 % | same | same |
-| uncached tokens per request | 38 390 → 13 945 | same | same |
+| uncached tokens per request | 38 521 → 13 480 (main-role) | same | same |
| generator $ per round | median 1.572 → 0.950 | round records of both runs | same |
| base64 amplification | 261 k tokens vs 516 for one 275 kB sheet | measured on a recorded blob + vendor bundle | arithmetic + matching recorded prompt jumps; **no repo script** |
| token growth in turns | input(t) = 12 364 + 1 822·t; total ∝ n^1.60; dollars ∝ n^1.19–1.34 | 1 063 in-process sessions, 30 401 turns | **no repo script** — the in-process agent, since gemini-cli emits no per-turn usage |
diff --git a/harness/tests/compare_bench/test_session_stats.py b/harness/tests/compare_bench/test_session_stats.py
index c5d5ed84..921120c9 100644
--- a/harness/tests/compare_bench/test_session_stats.py
+++ b/harness/tests/compare_bench/test_session_stats.py
@@ -58,16 +58,35 @@ def test_rates_and_costs_over_two_sessions(tmp_path: Path) -> None:
assert "| 3.000 |" in report([tmp_path]) # the median round, not the mean
-def test_a_dead_session_is_skipped_not_fatal(tmp_path: Path) -> None:
+def test_a_killed_session_is_counted_as_killed_not_as_a_session(tmp_path: Path) -> None:
+ """An empty stdout.json is the CLI dying before it printed its stats. Counting it as a
+ session makes every per-session number smaller for a reason that has nothing to do with
+ the code under test — 102 of aa_articulated's 162 files are that."""
+ from bench.session_stats import _killed
+
run = _run(tmp_path, "cell_a", [2.0])
_session(run, "baseline_r00", tools={"build": (3, 0)}, prompt=100, cached=50, requests=1)
dead = run / "trajectories" / "refine_r01"
dead.mkdir(parents=True)
- (dead / "stdout.json").write_text("") # the CLI was killed before it printed
+ (dead / "stdout.json").write_text("")
- assert len(_sessions(tmp_path)) == 2
+ assert len(_sessions(tmp_path)) == 1 and _killed(tmp_path) == 1
calls, _ = tool_rates(_sessions(tmp_path))
assert calls == {"build": 3}
+ assert "| 1 | 1 |" in report([tmp_path]) # sessions | killed
+
+
+def test_a_sub_workspace_is_not_the_battery_cell_above_it(tmp_path: Path) -> None:
+ """A scene asset's own run lives under ``_cand/`` / ``_assets/`` with its own
+ trajectories; ``flywheel.record.find_runs`` skips those and so must this, or one cell's
+ numbers absorb every candidate it rejected."""
+ run = _run(tmp_path, "cell_a", [1.0])
+ _session(run, "baseline_r00", tools={"build": (2, 0)}, prompt=100, cached=50, requests=1)
+ sub = run / "_cand" / "c1" / "run"
+ _session(sub, "baseline_r00", tools={"build": (99, 9)}, prompt=999, cached=0, requests=9)
+
+ calls, failed = tool_rates(_sessions(tmp_path))
+ assert dict(calls) == {"build": 2} and failed["build"] == 0
def test_the_coupling_survey_counts_degrees_of_freedom_not_joints(tmp_path: Path) -> None: