Official Codebase of 3DCodeVerse
Python
bench: one journal reader and one dedup rule — every append-only file reads the same way
gaoypengbf310db96d5b
diff --git a/eval/bench/_ab_report.py b/eval/bench/_ab_report.pyindex c30f6e16..ca06c355 100644--- a/eval/bench/_ab_report.py+++ b/eval/bench/_ab_report.py@@ -21,6 +21,7 @@ from pathlib import Pathfrom pydantic import BaseModel, Fieldfrom bench._compare_report import CellResult+from bench._jsonl import latestfrom bench.stats import mean_ci, n_to_resolve, sign_testCONTROL = "control"@@ -109,7 +110,7 @@ def pair_up(rows: list[CellResult], prompt_order: list[tuple[str, str]] | None =prompts absent from the order are appended first-seen. Latest row wins per(prompt, arm) — the results file is append-only and a redo re-appends."""- by: dict[tuple[str, str], CellResult] = {(r.prompt_id, r.arm): r for r in rows}+ by = latest(rows)present = {r.prompt_id for r in rows}ids: list[tuple[str, str]] = [(i, t) for i, t in (prompt_order or []) if i in present]seen = {i for i, _ in ids}diff --git a/eval/bench/_compare_report.py b/eval/bench/_compare_report.pyindex 4c610f01..4c455772 100644--- a/eval/bench/_compare_report.py+++ b/eval/bench/_compare_report.py@@ -13,16 +13,13 @@ import jsonimport shutilimport statisticsfrom pathlib import Path-from typing import Any, TypeVarfrom pydantic import BaseModel, Fieldfrom bench._fixed_eval import RUBRIC, rubric_for-from bench._jsonl import read_jsonl+from bench._jsonl import latest, read_jsonlfrom codeverse3d.contracts.common import Track-T = TypeVar("T", bound=BaseModel)-def battery_rubric(meta: dict) -> str:"""The fixed judge's rubric the report header names: ``rubric_for`` on the battery's track."""@@ -104,27 +101,6 @@ class PairRow(BaseModel):return (self.prompt_id, self.arm_a, self.arm_b)-def load_jsonl[T: BaseModel](path: Path, model: type[T]) -> list[T]:- """Rows in first-seen order, latest row winning per ``natural_key`` (models without- one keep every row). Unparseable lines are skipped, not fatal — see ``bench/_jsonl.py``.-- The dedup lives HERE, once, because the files are append-only and every reader needs- the same rows: ``ab_plan --report-only`` used to hand the raw rows to ``render_summary``- while the live driver passed deduped ones, so one summary.md printed the variant's mean- as 0.400 in the arms table, 0.800 per prompt, and based its verdict on 0.800 — and the- "lost to outages" footer counted infra_failed cells that had already been re-run and- scored, advising ``--redo-status infra_failed`` for nothing.-- Tolerance and dedup compose: a killed battery's truncated last line is dropped by- ``read_jsonl`` and the surviving rows are still collapsed to one per cell.- """- rows: dict[Any, T] = {}- for i, row in enumerate(read_jsonl(path, model)):- key = row.natural_key() if hasattr(row, "natural_key") else i- rows[key] = row # first appearance fixes the order, the last row wins the slot- return list(rows.values())--# ----------------------------------------------------------------------------- aggregatesclass ArmStats(BaseModel):arm: str@@ -233,7 +209,7 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], meopts = meta.get("options", {})arms = _arm_order(rows, meta)prompts = _prompt_order(rows, meta)- cells = {(r.prompt_id, r.arm): r for r in rows}+ cells = latest(rows)md = [f"# harness vs one-shot — {meta.get('battery', {}).get('name', out.name)}", "",f"fixed judge: **{opts.get('judge', '?')}** (rubric {battery_rubric(meta)}, n_samples={opts.get('n_samples', 2)}, "f"acceptance = must_have list) · harness loop judge: {opts.get('loop_judge') or 'settings default'} · "@@ -297,7 +273,7 @@ def compare_html(out: Path, rows: list[CellResult], pairs: list[PairRow], meta:assets.mkdir(exist_ok=True)arms = _arm_order(rows, meta)prompts = _prompt_order(rows, meta)- cells = {(r.prompt_id, r.arm): r for r in rows}+ cells = latest(rows)head = "<tr><th>arm</th><th>n</th><th>mean</th><th>pass</th><th>build ok</th><th>$gen</th><th>min</th></tr>"summ = "".join(f"<tr><td>{html.escape(s.arm)}</td><td>{s.n}</td><td>{_f(s.mean_score)}</td><td>{_f(s.pass_rate, '.0%')}</td>"f"<td>{s.build_ok_rate:.0%}</td><td>{s.mean_gen_usd:.2f}</td><td>{s.mean_minutes:.1f}</td></tr>" for s in arm_stats(rows))@@ -331,8 +307,8 @@ def compare_html(out: Path, rows: list[CellResult], pairs: list[PairRow], meta:def build_compare_report(out_dir: Path | str) -> tuple[str, str]:out = Path(out_dir)- rows = list({(r.prompt_id, r.arm): r for r in load_jsonl(out / "results.jsonl", CellResult)}.values())- pairs = list({(p.prompt_id, p.arm_a, p.arm_b): p for p in load_jsonl(out / "pairwise.jsonl", PairRow)}.values())+ rows = list(latest(read_jsonl(out / "results.jsonl", CellResult)).values())+ pairs = list(latest(read_jsonl(out / "pairwise.jsonl", PairRow)).values())meta = json.loads((out / "matrix.json").read_text()) if (out / "matrix.json").is_file() else {}md = compare_markdown(out, rows, pairs, meta)page = compare_html(out, rows, pairs, meta)@@ -341,4 +317,4 @@ def build_compare_report(out_dir: Path | str) -> tuple[str, str]:return md, page-__all__ = ["ArmStats", "CellResult", "PairRow", "PairStats", "arm_stats", "build_compare_report", "load_jsonl", "pair_stats"]+__all__ = ["ArmStats", "CellResult", "PairRow", "PairStats", "arm_stats", "build_compare_report", "pair_stats"]diff --git a/eval/bench/_jsonl.py b/eval/bench/_jsonl.pyindex f7730f83..c70f8380 100644--- a/eval/bench/_jsonl.py+++ b/eval/bench/_jsonl.py@@ -1,23 +1,25 @@-"""One tolerant reader for the append-only ``*.jsonl`` files the bench drivers-resume and report from.+"""The one reader and the one dedup rule for the append-only ``*.jsonl`` journals the+bench drivers resume and report from.Every one of those files is written a line at a time with a flush per row-(``run_bench.py``, ``compare_backends.py``, ``ab_plan.py``), which is exactly the-shape a SIGKILL truncates mid-append. Each driver used to validate every line-strictly, so a single half-written last line made the whole file unusable: a killed+(``run_bench.py``, ``compare_backends.py``, ``ab_plan.py``, ``plan_stage_bench.py``),+which is exactly the shape a SIGKILL truncates mid-append. Readers used to parse every+line strictly, so a single half-written last line made the whole file unusable: a killedbattery could not be resumed *at all* and its already-paid rows could not even bereported — the operator had to hand-edit ``results.jsonl`` or re-buy the battery.-Skip the bad line, keep the rest, say so once — the same contract+Skip the bad line, keep the rest, say so — the same contract``codeverse3d.cost.ledger.load_ledger`` has always had ("a truncated last line neverloses the rest of the file")."""from __future__ import annotations+import jsonimport logging+from collections.abc import Callable, Hashable, Iterablefrom pathlib import Path-from typing import TypeVar+from typing import Anyfrom pydantic import BaseModel@@ -25,25 +27,45 @@ from codeverse3d.proc import iter_jsonl_lineslog = logging.getLogger(__name__)-T = TypeVar("T", bound=BaseModel)--def read_jsonl[T: BaseModel](path: Path, model: type[T]) -> list[T]:- """Parse ``path`` as one ``model`` per line, skipping unparseable lines with a- warning. Missing file → ``[]``."""- rows: list[T] = []+def read_jsonl(path: Path, model: type[BaseModel] | None = None) -> list[Any]:+ """Every row of ``path`` — a ``model`` per line, or a JSON object when no model is+ given — skipping unreadable lines with a warning. Missing file → ``[]``."""+ rows: list[Any] = []bad = 0for i, line in iter_jsonl_lines(path):try:- rows.append(model.model_validate_json(line))+ row = model.model_validate_json(line) if model is not None else json.loads(line)+ if not isinstance(row, (BaseModel, dict)):+ raise ValueError(f"a JSON {type(row).__name__}, not an object")except Exception as e: # a truncated/corrupt row must not cost us the good onesbad += 1log.warning("%s:%d unreadable, skipping (%s: %s)", path, i, type(e).__name__, str(e)[:120])+ continue+ rows.append(row)if bad:log.warning("%s: skipped %d unreadable line(s), kept %d", path, bad, len(rows))return rows+def latest[R](rows: Iterable[R], key: Callable[[R], Hashable] | None = None) -> dict[Hashable, R]:+ """THE dedup rule of every journal: one row per key (``row.natural_key()`` unless+ ``key`` says otherwise), the LAST row winning and the first appearance fixing the+ order. A dict, so a cell is looked up by its key; ``.values()`` is the journal as it+ stands.++ A resume or a ``--redo-status`` re-run appends a second row for the same key, so a+ reader that skips this rule reads a superseded row: ``ab_plan --report-only`` once+ handed the raw rows to ``render_summary`` while the live driver passed deduped ones,+ and one summary.md printed the variant's mean as 0.400 in the arms table, 0.800 per+ prompt, and advised ``--redo-status infra_failed`` for cells already re-run and scored.+ """+ out: dict[Hashable, R] = {}+ for r in rows:+ out[key(r) if key is not None else r.natural_key()] = r # type: ignore[attr-defined]+ return out++def seal_for_append(path: Path) -> None:"""Terminate a partial last line before appending to a journal.diff --git a/eval/bench/ab_plan.py b/eval/bench/ab_plan.pyindex e2d4c15a..f92bd47d 100644--- a/eval/bench/ab_plan.py+++ b/eval/bench/ab_plan.py@@ -104,9 +104,9 @@ def _assert_local_codeverse() -> None:_assert_local_codeverse()from bench._ab_report import ARMS, CONTROL, VARIANT, Verdict, write_report # noqa: E402-from bench._compare_report import CellResult, load_jsonl # noqa: E402+from bench._compare_report import CellResult # noqa: E402from bench._infra import is_infra_failure # noqa: E402-from bench._jsonl import seal_for_append # noqa: E402+from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402from bench.compare_backends import ( # noqa: E402Arm,CompareDeps,@@ -475,7 +475,7 @@ def run_ab())results = out / "results.jsonl"- done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}+ done = latest(read_jsonl(results, CellResult)) if opts.resume else {}todo = _plan_todo(battery, done, opts)seal_for_append(results) # a kill left the last row unterminated; do not glue onto itwith ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:@@ -516,7 +516,7 @@ def run_ab(status=status,error=f"pinned plan: {type(e).__name__}: {e}"[:600],)- done[(r.prompt_id, r.arm)] = r+ done[r.natural_key()] = rfh.write(r.model_dump_json() + "\n")fh.flush()if on_result:@@ -525,7 +525,7 @@ def run_ab(futs = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]for fut in futs:r = fut.result()- done[(r.prompt_id, r.arm)] = r+ done[r.natural_key()] = rfh.write(r.model_dump_json() + "\n")fh.flush()if on_result:@@ -688,7 +688,7 @@ def main(argv: Sequence[str] | None = None) -> int:opts = _stored_options(out) or optsv = write_report(out,- load_jsonl(out / "results.jsonl", CellResult),+ list(latest(read_jsonl(out / "results.jsonl", CellResult)).values()),[(p.id, p.tier) for p in battery.prompts],title=f"{battery.name} / {out.name}",variant_env={} if opts.aa else opts.variant_env,@@ -755,7 +755,7 @@ def main(argv: Sequence[str] | None = None) -> int:+ (f"\nCAUTION: {v.caution}" if v.caution else "")+ f"\n{out / 'summary.md'}")- lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]+ lost = [r for r in latest(read_jsonl(out / "results.jsonl", CellResult)).values() if r.status == "infra_failed"]if lost:print(f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "diff --git a/eval/bench/ab_view.py b/eval/bench/ab_view.pyindex 6cd4b8eb..7b74e095 100644--- a/eval/bench/ab_view.py+++ b/eval/bench/ab_view.py@@ -30,6 +30,7 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._jsonl import latest # noqa: E402from bench.stats import mean_ci # noqa: E402from codeverse3d.addons import select # noqa: E402@@ -108,11 +109,7 @@ MIN_PAIRS = 3def paired_by_brief(a: list[Run], b: list[Run]) -> dict[str, tuple[Run | None, Run | None]]:"""``{brief: (A run, B run)}`` — the pairing the page shows and the verdict reads. Abrief run twice in one arm (a redo) is its LAST run, the rule of every bench journal."""- ka: dict[str, Run] = {}- kb: dict[str, Run] = {}- for rs, by in ((a, ka), (b, kb)):- for r in rs:- by[r.brief or r.slug] = r+ ka, kb = (latest(rs, key=lambda r: r.brief or r.slug) for rs in (a, b))return {k: (ka.get(k), kb.get(k)) for k in {**ka, **kb}}diff --git a/eval/bench/bestofk_report.py b/eval/bench/bestofk_report.pyindex a5f58544..626d7081 100644--- a/eval/bench/bestofk_report.py+++ b/eval/bench/bestofk_report.py@@ -38,36 +38,35 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._jsonl import latest, read_jsonl # noqa: E402from bench.stats import mean_ci # noqa: E402-from codeverse3d.proc import read_jsonl_lenient # noqa: E402#: judge repeatability on the calibration set (docs/PAPER_WRITING.md §2), used only to#: state how much of a best-of-k maximum is selection noiseJUDGE_SIGMA = 0.030+def _last_rows(path: Path, arm_prefix: str = "") -> list[dict]:+ """One row per prompt of the arms starting with ``arm_prefix`` — the LAST, as+ ``paired_compare`` pairs; a prompt whose last row is unscored has no score."""+ rows = (r for r in read_jsonl(path) if str(r.get("arm", "")).startswith(arm_prefix))+ return list(latest(rows, key=lambda r: r["prompt_id"]).values())++def one_shot_samples(root: Path) -> dict[str, list[float]]:"""``{prompt: [score per rep]}`` — one entry per rep that scored that prompt."""out: dict[str, list[float]] = defaultdict(list)for rep in sorted(root.glob("rep*/results.jsonl")):- seen: dict[str, float] = {}- for r in read_jsonl_lenient(rep, dicts_only=True):- if r.get("score") is None:- continue- seen[r["prompt_id"]] = float(r["score"]) # last row wins, as paired_compare does- for pid, sc in seen.items():- out[pid].append(sc)+ for r in _last_rows(rep):+ if r.get("score") is not None:+ out[r["prompt_id"]].append(float(r["score"]))return dict(out)def harness_scores(root: Path, *, arm_prefix: str = "harness:") -> dict[str, float]:"""``{prompt: score}`` for the recorded harness arm, last row per prompt."""- out: dict[str, float] = {}- for r in read_jsonl_lenient(root / "results.jsonl", dicts_only=True):- if not str(r.get("arm", "")).startswith(arm_prefix) or r.get("score") is None:- continue- out[r["prompt_id"]] = float(r["score"])- return out+ return {r["prompt_id"]: float(r["score"]) for r in _last_rows(root / "results.jsonl", arm_prefix)+ if r.get("score") is not None}def harness_repeat_spread(root: Path, *, arm_prefix: str = "harness:") -> tuple[int, int, float]:@@ -82,7 +81,7 @@ def harness_repeat_spread(root: Path, *, arm_prefix: str = "harness:") -> tuple["""per: dict[str, list[float]] = {}rows = 0- for r in read_jsonl_lenient(root / "results.jsonl", dicts_only=True):+ for r in read_jsonl(root / "results.jsonl"):if not str(r.get("arm", "")).startswith(arm_prefix):continuerows += 1@@ -96,11 +95,8 @@ def gen_costs(root: Path, arm_prefix: str) -> list[float]:"""Generation cost per PROMPT for an arm — last row per prompt, the basis`paired_compare` pairs on. Taken over every appended row instead, resumed and failedcells drag the median down and the equal-compute k comes out too small."""- last: dict[str, float] = {}- for r in read_jsonl_lenient(root / "results.jsonl", dicts_only=True):- if str(r.get("arm", "")).startswith(arm_prefix):- last[r["prompt_id"]] = float(r.get("gen_cost_usd") or 0.0)- return [v for v in last.values() if v > 0]+ costs = (float(r.get("gen_cost_usd") or 0.0) for r in _last_rows(root / "results.jsonl", arm_prefix))+ return [c for c in costs if c > 0]def equal_compute_k(root: Path, baseline_prefix: str) -> tuple[float, float, int] | None:diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex 9965abf6..a1a931d8 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -56,15 +56,10 @@ for _p in (REPO, Path(__file__).resolve().parents[1]): # its codeverse3d +from datetime import UTC # noqa: E402-from bench._compare_report import ( # noqa: E402- CellResult,- PairRow,- build_compare_report,- load_jsonl,-)+from bench._compare_report import CellResult, PairRow, build_compare_report # noqa: E402from bench._fixed_eval import RUBRIC, EvalOutcome, FixedEvaluator # noqa: E402from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402-from bench._jsonl import seal_for_append # noqa: E402+from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402from bench._oneshot import ( # noqa: E402OneShotBackend,OneShotResult,@@ -430,7 +425,7 @@ def _renders_of(out: Path, r: CellResult) -> RenderSet | None:def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], arms: Sequence[Arm], out: Path,opts: CompareOptions, deps: CompareDeps) -> list[PairRow]:path = out / "pairwise.jsonl"- done = {(p.prompt_id, p.arm_a, p.arm_b): p for p in load_jsonl(path, PairRow)} if opts.resume else {}+ done = latest(read_jsonl(path, PairRow)) if opts.resume else {}harness = [a for a in arms if a.kind == "harness"]oneshot = [a for a in arms if a.kind != "harness"]judge = None@@ -464,7 +459,8 @@ def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], arm# ----------------------------------------------------------------------------- matrixdef _drop_pairs(path: Path, key: tuple[str, str]) -> None:"""Forget pairwise rows touching a cell that is about to be re-run."""- rows = [p for p in load_jsonl(path, PairRow) if not (p.prompt_id == key[0] and key[1] in (p.arm_a, p.arm_b))]+ rows = [p for p in latest(read_jsonl(path, PairRow)).values()+ if not (p.prompt_id == key[0] and key[1] in (p.arm_a, p.arm_b))]if path.is_file():path.write_text("".join(p.model_dump_json() + "\n" for p in rows))@@ -478,7 +474,7 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm"options": opts.model_dump(mode="json"),"started_at": datetime.now(UTC).isoformat()}, indent=2))results = out / "results.jsonl"- done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}+ done = latest(read_jsonl(results, CellResult)) if opts.resume else {}for key in [k for k, r in done.items() if r.status in set(opts.redo_status)]:_drop_pairs(out / "pairwise.jsonl", key)del done[key]@@ -497,7 +493,7 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm_, r = _new_cell(p, a, out, opts)r.status = "error"r.note(f"run_cell raised {type(e).__name__}: {e}"[:800])- done[(r.prompt_id, r.arm)] = r+ done[r.natural_key()] = rfh.write(r.model_dump_json() + "\n")fh.flush()if on_result:@@ -607,7 +603,8 @@ def main(argv: Sequence[str] | None = None) -> int:run_matrix(ns.prompts, ns.out, arms, opts, deps, on_result=_log)print(f"report: {Path(ns.out) / 'report.md'}")- dropped = [r for r in load_jsonl(Path(ns.out) / "results.jsonl", CellResult) if r.status == "infra_failed"]+ dropped = [r for r in latest(read_jsonl(Path(ns.out) / "results.jsonl", CellResult)).values()+ if r.status == "infra_failed"]if dropped:# never let downtime pass as a result: say what was lost and how to get it backprint(f"\n{len(dropped)} cell(s) lost to provider outages and EXCLUDED from every rate: "diff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.pyindex f4ccd58d..c8c3ed0f 100644--- a/eval/bench/h2h_scene.py+++ b/eval/bench/h2h_scene.py@@ -40,13 +40,13 @@ from pydantic import BaseModelfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._jsonl import latest, read_jsonl # noqa: E402from bench.run_bench import Battery, BenchPrompt # noqa: E402from bench.stats import sign_test # noqa: E402from codeverse3d.addons import select # noqa: E402from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402from codeverse3d.contracts.common import Language, Track # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402-from codeverse3d.proc import read_jsonl_lenient # noqa: E402HERE = Path(__file__).resolve().parentBATTERY = HERE / "prompts" / "h2h_scene_v1.yaml"@@ -148,10 +148,8 @@ def our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:stills = [v for v in views if (v.time_s or 0.0) == 0.0 and v.mode == "shaded" and Path(v.path).is_file()]flagged = [v for v in stills if v.judge] or stillschosen = flagged[:MAX_VIEWS]- minutes = None- for r in read_jsonl_lenient(run_dir.parent.parent / "results.jsonl", dicts_only=True):- if r.get("id") == run_dir.name:- minutes = float(r.get("minutes") or 0.0)+ row = latest(read_jsonl(run_dir.parent.parent / "results.jsonl"), key=lambda r: r.get("id")).get(run_dir.name)+ minutes = None if row is None else float(row.get("minutes") or 0.0)if minutes is None and rec.get("started_at") and rec.get("finished_at"):t0, t1 = (datetime.fromisoformat(rec[k]) for k in ("started_at", "finished_at"))minutes = (t1 - t0).total_seconds() / 60diff --git a/eval/bench/paired_compare.py b/eval/bench/paired_compare.pyindex cf6d2b71..568ab1d6 100644--- a/eval/bench/paired_compare.py+++ b/eval/bench/paired_compare.py@@ -35,7 +35,7 @@ for _p in (REPO, Path(__file__).resolve().parents[1]): # its codeverse3d +from bench._ab_report import _fmt as _f # noqa: E402from bench._compare_report import CellResult # noqa: E402-from bench._jsonl import read_jsonl # noqa: E402+from bench._jsonl import latest, read_jsonl # noqa: E402from bench.stats import mean_ci, sign_test # noqa: E402@@ -69,14 +69,6 @@ class PairedStats(BaseModel):deltas: dict[str, float] = Field(default_factory=dict, description="prompt id → harness − one-shot")-def latest_cells(rows: list[CellResult]) -> dict[tuple[str, str], CellResult]:- """(prompt_id, arm) → last recorded row (the journal is append-only; a re-run supersedes)."""- out: dict[tuple[str, str], CellResult] = {}- for r in rows:- out[(r.prompt_id, r.arm)] = r- return out--def _usable(r: CellResult | None) -> tuple[bool, str]:if r is None:return False, "missing"@@ -89,6 +81,7 @@ def _usable(r: CellResult | None) -> tuple[bool, str]:def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_arm: str, *,tier: str = "all", exclude_degraded: bool = False) -> PairedStats:+ """One comparison over ``cells`` — ``bench/_jsonl.latest``: (prompt, arm) → last row."""prompts = sorted({p for (p, a) in cells if a in (harness_arm, oneshot_arm)})st = PairedStats(harness_arm=harness_arm, oneshot_arm=oneshot_arm, tier=tier, n=0)hs: list[float] = []@@ -148,7 +141,7 @@ def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_adef analyse(rows: list[CellResult]) -> list[PairedStats]:- cells = latest_cells(rows)+ cells = latest(rows)arms = sorted({a for (_, a) in cells})harness = [a for a in arms if a.startswith("harness:")]oneshot = [a for a in arms if a.startswith("oneshot")]@@ -241,10 +234,7 @@ def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:the same for both shapes rather than recomputed by hand."""rows: list[CellResult] = []- for line in (out_dir / "results.jsonl").read_text().splitlines():- if not line.strip():- continue- raw = json.loads(line)+ for raw in latest(read_jsonl(out_dir / "results.jsonl"), key=lambda r: r["id"]).values():# a bench-run row records no build flag: a cell with a verdict was built and judged,# one without (score None) was not — so `build_ok` here means "judged", and# `gen_cost_usd` is the run's WHOLE cost (plan + loop judge), as the field says. A@@ -282,14 +272,14 @@ def main(argv: list[str] | None = None) -> int:if ns.against is not None:name_a, name_b = arm_names(out, ns.against)rows = rows_from_bench_run(out, name_a) + rows_from_bench_run(ns.against, name_b)- stats = [paired(latest_cells(rows), name_a, name_b)]+ stats = [paired(latest(rows), name_a, name_b)]md = render_markdown(stats, f"{name_a} vs {name_b}")(out / "paired.md").write_text(md)print(md)return 0rows = read_jsonl(out / "results.jsonl", CellResult)stats = analyse(rows)- gates = gate_stats(latest_cells(rows))+ gates = gate_stats(latest(rows))md = render_markdown(stats, out.name, gates)(out / "paired.md").write_text(md)(out / "paired.json").write_text(json.dumps({"paired": [s.model_dump(mode="json") for s in stats],diff --git a/eval/bench/plan_stage_bench.py b/eval/bench/plan_stage_bench.pyindex 70766977..729c1c1e 100644--- a/eval/bench/plan_stage_bench.py+++ b/eval/bench/plan_stage_bench.py@@ -38,16 +38,9 @@ DEFAULT_JUDGE = "gemini:gemini-3.1-pro-preview"def _done(out: Path) -> set[tuple[str, int]]:"""(prompt, rep) pairs already recorded, so a killed run resumes instead of repeating."""- if not out.is_file():- return set()- seen: set[tuple[str, int]] = set()- for line in out.read_text().splitlines():- try:- row = json.loads(line)- except ValueError:- continue- seen.add((row["prompt"], row["rep"]))- return seen+ from bench._jsonl import read_jsonl # after main's path insert, like every bench import here++ return {(row["prompt"], row["rep"]) for row in read_jsonl(out)}def _plan_shape(ws_root: Path) -> dict:@@ -68,15 +61,10 @@ def _plan_shape(ws_root: Path) -> dict:def _stats(ws_root: Path) -> dict:"""What the plan stage's event log says about the call that just ran."""+ from bench._jsonl import read_jsonl # after main's path insert, like every bench import here+out: dict = {"invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0}- events = ws_root / "events.jsonl"- if not events.is_file():- return out- for line in events.read_text().splitlines():- try:- e = json.loads(line)- except ValueError:- continue+ for e in read_jsonl(ws_root / "events.jsonl"):kind = e.get("event")if kind == "plan.invalid":out["invalid_reasks"] += 1@@ -194,7 +182,7 @@ def main(argv: list[str] | None = None) -> int:sys.path.insert(0, str(ns.tree.resolve())) # the harness tree under test wins for `codeverse3d`import codeverse3d- from bench._jsonl import seal_for_append # after the path insert: `python bench/x.py`+ from bench._jsonl import read_jsonl, seal_for_append # after the path insert (script run)from bench.run_bench import Batteryfrom codeverse3d.config import get_settingsif not Path(codeverse3d.__file__).resolve().is_relative_to(ns.tree.resolve()):@@ -224,7 +212,7 @@ def main(argv: list[str] | None = None) -> int:fh.write(json.dumps(row) + "\n")fh.flush()print(f" {row['prompt']:28} r{row['rep']:<3} ok={row['ok']} restart={row['restarts']} {row['error'][:60]}")- rows = [json.loads(x) for x in ns.out.read_text().splitlines() if x.strip()]+ rows = read_jsonl(ns.out)spent = sum(float(r.get("cost_usd") or 0.0) for r in rows)print(f"{ns.label}: {sum(1 for r in rows if r['ok'])}/{len(rows)} plans valid, "f"${spent:.2f} over {len(rows)} calls (the rows are the ledger: plan_once does not "diff --git a/eval/bench/plan_stage_report.py b/eval/bench/plan_stage_report.pyindex de17d16a..cc859fb5 100644--- a/eval/bench/plan_stage_report.py+++ b/eval/bench/plan_stage_report.py@@ -10,11 +10,16 @@ are excluded from the denominator — they say nothing about the code under testfrom __future__ import annotationsimport argparse-import jsonimport math+import sysfrom collections import Counterfrom pathlib import Path+for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):+ sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)++from bench._jsonl import read_jsonl # noqa: E402+#: what a validation failure was ABOUT, read off the error text. The mechanism under test#: (``C3D_PLAN_RESTART``) only ever addresses ``dangling_link``; every other class is a#: bystander, and a total that mixes them hides both the effect and its residue.@@ -82,10 +87,6 @@ def fisher_exact(a: int, b: int, c: int, d: int) -> float:return min(1.0, sum(prob(x) for x in range(lo, hi + 1) if prob(x) <= observed * (1 + 1e-9)))-def load(path: Path) -> list[dict]:- return [json.loads(x) for x in path.read_text().splitlines() if x.strip()] if path.is_file() else []--def report(arms: dict[str, list[dict]]) -> str:lines = ["| arm | calls | valid | planning_error | provider | failure rate | 95 % CI |","|---|--:|--:|--:|--:|--:|---|"]@@ -156,7 +157,7 @@ def main(argv: list[str] | None = None) -> int:ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])ap.add_argument("rows", nargs="+", type=Path, help="one JSONL per arm (the file name is the arm name)")ns = ap.parse_args(argv)- print(report({p.stem: load(p) for p in ns.rows}))+ print(report({p.stem: read_jsonl(p) for p in ns.rows}))return 0diff --git a/eval/bench/report.py b/eval/bench/report.pyindex 35ef6373..9b8dc7ff 100644--- a/eval/bench/report.py+++ b/eval/bench/report.py@@ -19,7 +19,7 @@ from pydantic import BaseModel, Fieldfrom bench._compare_report import _f as _fmtfrom bench._compare_report import _mean-from bench._jsonl import read_jsonl+from bench._jsonl import latest, read_jsonlfrom bench.run_bench import BenchItemResultfrom codeverse3d.addons.gallery import GalleryIndex, RootSection, RunEntry, render_staticfrom codeverse3d.addons.gallery.index import entry_from_record@@ -57,8 +57,7 @@ class BenchReport(BaseModel):def load_results(out_dir: Path) -> list[BenchItemResult]:jl = out_dir / "results.jsonl"if jl.is_file():- latest: dict[str, BenchItemResult] = {r.id: r for r in read_jsonl(jl, BenchItemResult)}- return list(latest.values())+ return list(latest(read_jsonl(jl, BenchItemResult)).values())js = out_dir / "results.json"if js.is_file():return [BenchItemResult.model_validate(r) for r in json.loads(js.read_text())]diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.pyindex 18c659b8..480b875e 100644--- a/eval/bench/run_bench.py+++ b/eval/bench/run_bench.py@@ -36,7 +36,7 @@ import yamlfrom pydantic import AliasChoices, BaseModel, Fieldfrom bench._infra import is_infra_failure-from bench._jsonl import read_jsonl, seal_for_append+from bench._jsonl import latest, read_jsonl, seal_for_appendfrom codeverse3d.addons import selectfrom codeverse3d.config import get_settingsfrom codeverse3d.contracts.common import Backends, Budget, Language, Track@@ -99,6 +99,11 @@ class BenchItemResult(BaseModel):generator: str = ""judge: str = ""+ def natural_key(self) -> str:+ """Row identity: one row per prompt id — results.jsonl is append-only, so a resume+ or a ``--redo-status`` re-run appends a second row for the same id."""+ return self.id+class BenchOptions(BaseModel):generator: str | None = None@@ -198,7 +203,7 @@ def _load_done(results_jsonl: Path) -> dict[str, BenchItemResult]:"""Rows already paid for, latest wins. Tolerant by design: this file is theresume source and it is appended to a line at a time, so the run that a SIGKILLended is precisely the one whose last line is half-written."""- return {r.id: r for r in read_jsonl(results_jsonl, BenchItemResult)}+ return latest(read_jsonl(results_jsonl, BenchItemResult))def select_prompts(diff --git a/eval/bench/scene_stats.py b/eval/bench/scene_stats.pyindex 50f5bf6f..af60017c 100644--- a/eval/bench/scene_stats.py+++ b/eval/bench/scene_stats.py@@ -27,7 +27,7 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)-from codeverse3d.proc import read_jsonl_lenient # noqa: E402+from bench._jsonl import read_jsonl # noqa: E402from codeverse3d.record.record import unique_files # noqa: E402from codeverse3d.spatial.node import browser_was_lost # noqa: E402@@ -56,7 +56,7 @@ def runs(root: Path) -> list[tuple[str, dict, list[dict]]]:"""``(prompt id, record, events)`` per scene run under ``root``, once per run."""out = []for run, data in scene_records(root):- events = read_jsonl_lenient(run / "events.jsonl", dicts_only=True)+ events = read_jsonl(run / "events.jsonl")out.append((str((data.get("spec") or {}).get("id", run.name)).split("/")[-1], data, events))# by id, then by path: two runs of one prompt (a battery with reps) share the id, and a# bare sorted() would then compare their record dicts and raise. unique_files yieldsdiff --git a/eval/tests/test_ab_plan.py b/eval/tests/test_ab_plan.pyindex e8fea592..4dcdef62 100644--- a/eval/tests/test_ab_plan.py+++ b/eval/tests/test_ab_plan.py@@ -22,7 +22,8 @@ from bench._ab_report import ( # noqa: E402render_summary,verdict_of,)-from bench._compare_report import CellResult, load_jsonl # noqa: E402+from bench._compare_report import CellResult # noqa: E402+from bench._jsonl import latest, read_jsonl # noqa: E402from bench.ab_plan import ( # noqa: E402DEFAULT_MAX_IN_FLIGHT,MAX_IN_FLIGHT_ENV,@@ -51,6 +52,10 @@ def _row(pid: str, arm: str, score: float | None, status: str = "scored", **kw)gen_cost_usd=kw.pop("cost", 1.0), judge_cost_usd=0.1, wall_s=kw.pop("wall", 120.0), **kw)+def _journal(out: Path) -> list[CellResult]:+ return list(latest(read_jsonl(out / "results.jsonl", CellResult)).values())++def _pairs(*deltas: float) -> list[PairOutcome]:return [PairOutcome(prompt_id=f"p{i}", paired=True, delta=d) for i, d in enumerate(deltas)]@@ -278,7 +283,7 @@ def test_driver_runs_pairs_in_prompt_order_and_writes_the_verdict(tmp_path: Pathassert [c[0] for c in fake.calls] == [i for i in ids for _ in range(2)]assert {c[1] for c in fake.calls} == {CONTROL, VARIANT}assert v.decision == "keep" and v.n_pairs == 3 and v.mean_delta == pytest.approx(0.0333, abs=1e-3)- rows = load_jsonl(tmp_path / "results.jsonl", CellResult)+ rows = _journal(tmp_path)assert len(rows) == 6# the env reached only the variant cellsassert {r.workspace for r in rows if r.arm == VARIANT} == {"on"}@@ -306,9 +311,9 @@ def test_resume_skips_done_pairs_and_redo_reruns_both_arms(tmp_path: Path):assert sorted(redo.calls) == [(ids[1], CONTROL), (ids[1], VARIANT)]assert v.n_pairs == 2assert len((tmp_path / "results.jsonl").read_text().strip().splitlines()) == 6, "the file is append-only"- rows = load_jsonl(tmp_path / "results.jsonl", CellResult)- latest = {(r.prompt_id, r.arm): r for r in rows}- assert latest[(ids[1], VARIANT)].status == "scored"+ cells = latest(read_jsonl(tmp_path / "results.jsonl", CellResult))+ rows = list(cells.values())+ assert cells[(ids[1], VARIANT)].status == "scored"assert len(rows) == 4, "readers see one row per (prompt, arm): the redo replaces the stale attempt"@@ -356,7 +361,7 @@ def test_aa_run_records_two_empty_arms(tmp_path: Path):run_ab(BATTERY, tmp_path, AbOptions(variant_env={"C3D_PLAN_FEATURES": "all"}, aa=True, ids=ids), run_cell_fn=fake)meta = json.loads((tmp_path / "ab.json").read_text())assert meta["aa"] is True and meta["arms"] == {"control": {}, "variant": {}}- assert {r.workspace for r in load_jsonl(tmp_path / "results.jsonl", CellResult)} == {""}, "no arm saw the switch"+ assert {r.workspace for r in _journal(tmp_path)} == {""}, "no arm saw the switch"assert (tmp_path / "summary.md").read_text().startswith("# A/A: ")@@ -406,7 +411,7 @@ def test_a_pinned_plan_that_dies_in_a_storm_records_the_pair_and_continues(tmp_popts = AbOptions(variant_env={"C3D_SKILLS": "0"}, pin_plan=True, ids=ids)run_ab(BATTERY, tmp_path, opts, run_cell_fn=fake)- rows = load_jsonl(tmp_path / "results.jsonl", CellResult)+ rows = _journal(tmp_path)first = [r for r in rows if r.prompt_id == ids[0]]assert {r.arm for r in first} == {CONTROL, VARIANT} and all(r.status == "infra_failed" for r in first)assert all("pinned plan" in r.error for r in first)diff --git a/eval/tests/test_bench_shared.py b/eval/tests/test_bench_shared.pyindex 6df35ef7..9f203779 100644--- a/eval/tests/test_bench_shared.py+++ b/eval/tests/test_bench_shared.py@@ -113,7 +113,8 @@ def test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:interval crosses zero" — rather than being recomputed by hand."""import json as _json- from bench.paired_compare import latest_cells, paired, rows_from_bench_run+ from bench._jsonl import latest+ from bench.paired_compare import paired, rows_from_bench_rundef write(d: Path, scores: dict[str, float]) -> Path:d.mkdir(parents=True)@@ -126,7 +127,7 @@ def test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:b = write(tmp_path / "arm_b", {"p1": 0.5, "p2": 0.3, "p3": 0.5})rows = rows_from_bench_run(a, "arm_a") + rows_from_bench_run(b, "arm_b")- st = paired(latest_cells(rows), "arm_a", "arm_b")+ st = paired(latest(rows), "arm_a", "arm_b")assert st.n == 3 and st.mean_delta == pytest.approx(0.0667, abs=1e-3)assert st.wins == 2 and st.losses == 0 and st.ties == 1assert st.verdict == "unsupported" # CI [-0.077, +0.21] crosses zero at n = 3diff --git a/eval/tests/test_compare_matrix.py b/eval/tests/test_compare_matrix.pyindex bb535bfa..d7fd6b0b 100644--- a/eval/tests/test_compare_matrix.py+++ b/eval/tests/test_compare_matrix.py@@ -16,10 +16,10 @@ from bench._compare_report import ( # noqa: E402CellResult,PairRow,arm_stats,- load_jsonl,pair_stats,)from bench._fixed_eval import acceptance_from_spec # noqa: E402+from bench._jsonl import latest, read_jsonl # noqa: E402from bench.compare_backends import ( # noqa: E402CompareDeps,CompareOptions,@@ -120,7 +120,7 @@ def test_matrix_end_to_end_with_fakes(tmp_path: Path):# the fixed evaluator ran once per cell that had code (6 of 8)assert len(ev.evaluated) == 6# pairwise: 1 harness arm × 3 one-shot arms × 2 prompts = 6 rows; the no-code arm is auto-decided- pairs = load_jsonl(out / "pairwise.jsonl", PairRow)+ pairs = list(latest(read_jsonl(out / "pairwise.jsonl", PairRow)).values())assert len(pairs) == 6auto = [p for p in pairs if p.arm_b == "oneshot:gemini:gemini-3.7-flash"]assert all(p.winner == "a" and not p.judged for p in auto)@@ -149,7 +149,7 @@ def test_matrix_end_to_end_with_fakes(tmp_path: Path):# ---- adding a prompt (limit 3) only runs the new cellsrows3 = run_matrix(BATTERY, out, parse_arms(ARMS), opts.model_copy(update={"limit": 3}), deps2)assert len(rows3) == 12 and run_track2.calls == ["compare_v1/cmp_med_dining_chair"]- assert len(load_jsonl(out / "pairwise.jsonl", PairRow)) == 9+ assert len(latest(read_jsonl(out / "pairwise.jsonl", PairRow))) == 9def test_cell_errors_are_recorded_not_raised(tmp_path: Path):diff --git a/eval/tests/test_paired_compare.py b/eval/tests/test_paired_compare.pyindex 9da16936..05dfefa1 100644--- a/eval/tests/test_paired_compare.py+++ b/eval/tests/test_paired_compare.py@@ -11,7 +11,8 @@ if str(REPO) not in sys.path:sys.path.insert(0, str(REPO))from bench._compare_report import CellResult # noqa: E402-from bench.paired_compare import analyse, latest_cells, main, paired # noqa: E402+from bench._jsonl import latest # noqa: E402+from bench.paired_compare import analyse, main, paired, rows_from_bench_run # noqa: E402from bench.stats import t975 # noqa: E402HA, OA = "harness:gemini-cli:gemini-3.6-flash", "oneshot:gemini:x"@@ -33,7 +34,7 @@ def test_paired_stats_ci_sign_test_and_drops():rows += [_cell("p7", HA, None, status="budget_exhausted"), _cell("p7", OA, 0.4)]# the journal is append-only: a re-run of p0's harness cell supersedes the first rowrows += [_cell("p0", HA, 0.82, passed=True)]- st = paired(latest_cells(rows), HA, OA)+ st = paired(latest(rows), HA, OA)assert st.n == 6 and st.dropped_infra == 1 and st.dropped_unscored == 1assert st.deltas["p0"] == 0.22 and st.wins == 5 and st.losses == 1 and st.ties == 0assert st.sign_p == 0.2188@@ -83,3 +84,14 @@ def test_degraded_harness_cells_are_reported_and_can_be_excluded():plain = analyse([_cell("q", HA, 0.7), _cell("q", OA, 0.5)])assert "all −degraded" not in {s.tier for s in plain}+++def test_a_killed_bench_run_journal_costs_its_last_row_not_the_comparison(tmp_path):+ """`rows_from_bench_run` parsed every line strictly, so the half-written row a SIGKILL+ leaves made `--against` crash on the whole battery; and a redo re-appends a row."""+ rows = [{"id": "p1", "tier": "hard", "score_picked": 0.4}, {"id": "p2", "tier": "hard", "score_picked": 0.5},+ {"id": "p1", "tier": "hard", "score_picked": 0.9}]+ text = "".join(json.dumps(r) + "\n" for r in rows) + '{"id": "p3", "tier": "ha'+ (tmp_path / "results.jsonl").write_text(text)+ cells = rows_from_bench_run(tmp_path, "arm")+ assert [(c.prompt_id, c.score) for c in cells] == [("p1", 0.9), ("p2", 0.5)]diff --git a/eval/tests/test_plan_stage.py b/eval/tests/test_plan_stage.pyindex 9f11c66e..3a00dde2 100644--- a/eval/tests/test_plan_stage.py+++ b/eval/tests/test_plan_stage.py@@ -75,3 +75,14 @@ def test_a_harness_side_death_is_a_loss_not_provider_weather() -> None:assert outcome({"ok": False, "error": "ModelError: Gemini prompt blocked: BlockedReason.OTHER"}) == "provider"assert outcome({"ok": False, "error": "KeyError: 'parts'"}) == "planning_error"+++def test_a_killed_arm_file_still_reports(tmp_path: Path, capsys) -> None:+ """The report's loader parsed every line strictly: the half-written row a killed+ ``plan_stage_bench`` leaves made the whole arm unreadable."""+ from bench.plan_stage_report import main++ arm = tmp_path / "arm.jsonl"+ arm.write_text(json.dumps({"prompt": "p", "rep": 0, "ok": True, "error": ""}) + '\n{"prompt": "p", "re')+ assert main([str(arm)]) == 0+ assert "| arm | 1 | 1 | 0 | 0 | 0.000 |" in capsys.readouterr().out