Skip to content

Official Codebase of 3DCodeVerse

Python

bench: one journal reader and one dedup rule — every append-only file reads the same way

gaoypengbf310db96d5b
diff --git a/eval/bench/_ab_report.py b/eval/bench/_ab_report.py
index c30f6e16..ca06c355 100644
--- a/eval/bench/_ab_report.py
+++ b/eval/bench/_ab_report.py
@@ -21,6 +21,7 @@ from pathlib import Path
from pydantic import BaseModel, Field
from bench._compare_report import CellResult
+from bench._jsonl import latest
from bench.stats import mean_ci, n_to_resolve, sign_test
CONTROL = "control"
@@ -109,7 +110,7 @@ def pair_up(rows: list[CellResult], prompt_order: list[tuple[str, str]] | None =
prompts absent from the order are appended first-seen. Latest row wins per
(prompt, arm) — the results file is append-only and a redo re-appends.
"""
- by: dict[tuple[str, str], CellResult] = {(r.prompt_id, r.arm): r for r in rows}
+ by = latest(rows)
present = {r.prompt_id for r in rows}
ids: list[tuple[str, str]] = [(i, t) for i, t in (prompt_order or []) if i in present]
seen = {i for i, _ in ids}
diff --git a/eval/bench/_compare_report.py b/eval/bench/_compare_report.py
index 4c610f01..4c455772 100644
--- a/eval/bench/_compare_report.py
+++ b/eval/bench/_compare_report.py
@@ -13,16 +13,13 @@ import json
import shutil
import statistics
from pathlib import Path
-from typing import Any, TypeVar
from pydantic import BaseModel, Field
from bench._fixed_eval import RUBRIC, rubric_for
-from bench._jsonl import read_jsonl
+from bench._jsonl import latest, read_jsonl
from codeverse3d.contracts.common import Track
-T = TypeVar("T", bound=BaseModel)
-
def battery_rubric(meta: dict) -> str:
"""The fixed judge's rubric the report header names: ``rubric_for`` on the battery's track."""
@@ -104,27 +101,6 @@ class PairRow(BaseModel):
return (self.prompt_id, self.arm_a, self.arm_b)
-def load_jsonl[T: BaseModel](path: Path, model: type[T]) -> list[T]:
- """Rows in first-seen order, latest row winning per ``natural_key`` (models without
- one keep every row). Unparseable lines are skipped, not fatal — see ``bench/_jsonl.py``.
-
- The dedup lives HERE, once, because the files are append-only and every reader needs
- the same rows: ``ab_plan --report-only`` used to hand the raw rows to ``render_summary``
- while the live driver passed deduped ones, so one summary.md printed the variant's mean
- as 0.400 in the arms table, 0.800 per prompt, and based its verdict on 0.800 — and the
- "lost to outages" footer counted infra_failed cells that had already been re-run and
- scored, advising ``--redo-status infra_failed`` for nothing.
-
- Tolerance and dedup compose: a killed battery's truncated last line is dropped by
- ``read_jsonl`` and the surviving rows are still collapsed to one per cell.
- """
- rows: dict[Any, T] = {}
- for i, row in enumerate(read_jsonl(path, model)):
- key = row.natural_key() if hasattr(row, "natural_key") else i
- rows[key] = row # first appearance fixes the order, the last row wins the slot
- return list(rows.values())
-
-
# ----------------------------------------------------------------------------- aggregates
class ArmStats(BaseModel):
arm: str
@@ -233,7 +209,7 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], me
opts = meta.get("options", {})
arms = _arm_order(rows, meta)
prompts = _prompt_order(rows, meta)
- cells = {(r.prompt_id, r.arm): r for r in rows}
+ cells = latest(rows)
md = [f"# harness vs one-shot — {meta.get('battery', {}).get('name', out.name)}", "",
f"fixed judge: **{opts.get('judge', '?')}** (rubric {battery_rubric(meta)}, n_samples={opts.get('n_samples', 2)}, "
f"acceptance = must_have list) · harness loop judge: {opts.get('loop_judge') or 'settings default'} · "
@@ -297,7 +273,7 @@ def compare_html(out: Path, rows: list[CellResult], pairs: list[PairRow], meta:
assets.mkdir(exist_ok=True)
arms = _arm_order(rows, meta)
prompts = _prompt_order(rows, meta)
- cells = {(r.prompt_id, r.arm): r for r in rows}
+ cells = latest(rows)
head = "<tr><th>arm</th><th>n</th><th>mean</th><th>pass</th><th>build ok</th><th>$gen</th><th>min</th></tr>"
summ = "".join(f"<tr><td>{html.escape(s.arm)}</td><td>{s.n}</td><td>{_f(s.mean_score)}</td><td>{_f(s.pass_rate, '.0%')}</td>"
f"<td>{s.build_ok_rate:.0%}</td><td>{s.mean_gen_usd:.2f}</td><td>{s.mean_minutes:.1f}</td></tr>" for s in arm_stats(rows))
@@ -331,8 +307,8 @@ def compare_html(out: Path, rows: list[CellResult], pairs: list[PairRow], meta:
def build_compare_report(out_dir: Path | str) -> tuple[str, str]:
out = Path(out_dir)
- rows = list({(r.prompt_id, r.arm): r for r in load_jsonl(out / "results.jsonl", CellResult)}.values())
- pairs = list({(p.prompt_id, p.arm_a, p.arm_b): p for p in load_jsonl(out / "pairwise.jsonl", PairRow)}.values())
+ rows = list(latest(read_jsonl(out / "results.jsonl", CellResult)).values())
+ pairs = list(latest(read_jsonl(out / "pairwise.jsonl", PairRow)).values())
meta = json.loads((out / "matrix.json").read_text()) if (out / "matrix.json").is_file() else {}
md = compare_markdown(out, rows, pairs, meta)
page = compare_html(out, rows, pairs, meta)
@@ -341,4 +317,4 @@ def build_compare_report(out_dir: Path | str) -> tuple[str, str]:
return md, page
-__all__ = ["ArmStats", "CellResult", "PairRow", "PairStats", "arm_stats", "build_compare_report", "load_jsonl", "pair_stats"]
+__all__ = ["ArmStats", "CellResult", "PairRow", "PairStats", "arm_stats", "build_compare_report", "pair_stats"]
diff --git a/eval/bench/_jsonl.py b/eval/bench/_jsonl.py
index f7730f83..c70f8380 100644
--- a/eval/bench/_jsonl.py
+++ b/eval/bench/_jsonl.py
@@ -1,23 +1,25 @@
-"""One tolerant reader for the append-only ``*.jsonl`` files the bench drivers
-resume and report from.
+"""The one reader and the one dedup rule for the append-only ``*.jsonl`` journals the
+bench drivers resume and report from.
Every one of those files is written a line at a time with a flush per row
-(``run_bench.py``, ``compare_backends.py``, ``ab_plan.py``), which is exactly the
-shape a SIGKILL truncates mid-append. Each driver used to validate every line
-strictly, so a single half-written last line made the whole file unusable: a killed
+(``run_bench.py``, ``compare_backends.py``, ``ab_plan.py``, ``plan_stage_bench.py``),
+which is exactly the shape a SIGKILL truncates mid-append. Readers used to parse every
+line strictly, so a single half-written last line made the whole file unusable: a killed
battery could not be resumed *at all* and its already-paid rows could not even be
reported — the operator had to hand-edit ``results.jsonl`` or re-buy the battery.
-Skip the bad line, keep the rest, say so once — the same contract
+Skip the bad line, keep the rest, say so — the same contract
``codeverse3d.cost.ledger.load_ledger`` has always had ("a truncated last line never
loses the rest of the file").
"""
from __future__ import annotations
+import json
import logging
+from collections.abc import Callable, Hashable, Iterable
from pathlib import Path
-from typing import TypeVar
+from typing import Any
from pydantic import BaseModel
@@ -25,25 +27,45 @@ from codeverse3d.proc import iter_jsonl_lines
log = logging.getLogger(__name__)
-T = TypeVar("T", bound=BaseModel)
-
-def read_jsonl[T: BaseModel](path: Path, model: type[T]) -> list[T]:
- """Parse ``path`` as one ``model`` per line, skipping unparseable lines with a
- warning. Missing file → ``[]``."""
- rows: list[T] = []
+def read_jsonl(path: Path, model: type[BaseModel] | None = None) -> list[Any]:
+ """Every row of ``path`` — a ``model`` per line, or a JSON object when no model is
+ given — skipping unreadable lines with a warning. Missing file → ``[]``."""
+ rows: list[Any] = []
bad = 0
for i, line in iter_jsonl_lines(path):
try:
- rows.append(model.model_validate_json(line))
+ row = model.model_validate_json(line) if model is not None else json.loads(line)
+ if not isinstance(row, (BaseModel, dict)):
+ raise ValueError(f"a JSON {type(row).__name__}, not an object")
except Exception as e: # a truncated/corrupt row must not cost us the good ones
bad += 1
log.warning("%s:%d unreadable, skipping (%s: %s)", path, i, type(e).__name__, str(e)[:120])
+ continue
+ rows.append(row)
if bad:
log.warning("%s: skipped %d unreadable line(s), kept %d", path, bad, len(rows))
return rows
+def latest[R](rows: Iterable[R], key: Callable[[R], Hashable] | None = None) -> dict[Hashable, R]:
+ """THE dedup rule of every journal: one row per key (``row.natural_key()`` unless
+ ``key`` says otherwise), the LAST row winning and the first appearance fixing the
+ order. A dict, so a cell is looked up by its key; ``.values()`` is the journal as it
+ stands.
+
+ A resume or a ``--redo-status`` re-run appends a second row for the same key, so a
+ reader that skips this rule reads a superseded row: ``ab_plan --report-only`` once
+ handed the raw rows to ``render_summary`` while the live driver passed deduped ones,
+ and one summary.md printed the variant's mean as 0.400 in the arms table, 0.800 per
+ prompt, and advised ``--redo-status infra_failed`` for cells already re-run and scored.
+ """
+ out: dict[Hashable, R] = {}
+ for r in rows:
+ out[key(r) if key is not None else r.natural_key()] = r # type: ignore[attr-defined]
+ return out
+
+
def seal_for_append(path: Path) -> None:
"""Terminate a partial last line before appending to a journal.
diff --git a/eval/bench/ab_plan.py b/eval/bench/ab_plan.py
index e2d4c15a..f92bd47d 100644
--- a/eval/bench/ab_plan.py
+++ b/eval/bench/ab_plan.py
@@ -104,9 +104,9 @@ def _assert_local_codeverse() -> None:
_assert_local_codeverse()
from bench._ab_report import ARMS, CONTROL, VARIANT, Verdict, write_report # noqa: E402
-from bench._compare_report import CellResult, load_jsonl # noqa: E402
+from bench._compare_report import CellResult # noqa: E402
from bench._infra import is_infra_failure # noqa: E402
-from bench._jsonl import seal_for_append # noqa: E402
+from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402
from bench.compare_backends import ( # noqa: E402
Arm,
CompareDeps,
@@ -475,7 +475,7 @@ def run_ab(
)
)
results = out / "results.jsonl"
- done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}
+ done = latest(read_jsonl(results, CellResult)) if opts.resume else {}
todo = _plan_todo(battery, done, opts)
seal_for_append(results) # a kill left the last row unterminated; do not glue onto it
with ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:
@@ -516,7 +516,7 @@ def run_ab(
status=status,
error=f"pinned plan: {type(e).__name__}: {e}"[:600],
)
- done[(r.prompt_id, r.arm)] = r
+ done[r.natural_key()] = r
fh.write(r.model_dump_json() + "\n")
fh.flush()
if on_result:
@@ -525,7 +525,7 @@ def run_ab(
futs = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]
for fut in futs:
r = fut.result()
- done[(r.prompt_id, r.arm)] = r
+ done[r.natural_key()] = r
fh.write(r.model_dump_json() + "\n")
fh.flush()
if on_result:
@@ -688,7 +688,7 @@ def main(argv: Sequence[str] | None = None) -> int:
opts = _stored_options(out) or opts
v = write_report(
out,
- load_jsonl(out / "results.jsonl", CellResult),
+ list(latest(read_jsonl(out / "results.jsonl", CellResult)).values()),
[(p.id, p.tier) for p in battery.prompts],
title=f"{battery.name} / {out.name}",
variant_env={} if opts.aa else opts.variant_env,
@@ -755,7 +755,7 @@ def main(argv: Sequence[str] | None = None) -> int:
+ (f"\nCAUTION: {v.caution}" if v.caution else "")
+ f"\n{out / 'summary.md'}"
)
- lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]
+ lost = [r for r in latest(read_jsonl(out / "results.jsonl", CellResult)).values() if r.status == "infra_failed"]
if lost:
print(
f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "
diff --git a/eval/bench/ab_view.py b/eval/bench/ab_view.py
index 6cd4b8eb..7b74e095 100644
--- a/eval/bench/ab_view.py
+++ b/eval/bench/ab_view.py
@@ -30,6 +30,7 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._jsonl import latest # noqa: E402
from bench.stats import mean_ci # noqa: E402
from codeverse3d.addons import select # noqa: E402
@@ -108,11 +109,7 @@ MIN_PAIRS = 3
def paired_by_brief(a: list[Run], b: list[Run]) -> dict[str, tuple[Run | None, Run | None]]:
"""``{brief: (A run, B run)}`` — the pairing the page shows and the verdict reads. A
brief run twice in one arm (a redo) is its LAST run, the rule of every bench journal."""
- ka: dict[str, Run] = {}
- kb: dict[str, Run] = {}
- for rs, by in ((a, ka), (b, kb)):
- for r in rs:
- by[r.brief or r.slug] = r
+ ka, kb = (latest(rs, key=lambda r: r.brief or r.slug) for rs in (a, b))
return {k: (ka.get(k), kb.get(k)) for k in {**ka, **kb}}
diff --git a/eval/bench/bestofk_report.py b/eval/bench/bestofk_report.py
index a5f58544..626d7081 100644
--- a/eval/bench/bestofk_report.py
+++ b/eval/bench/bestofk_report.py
@@ -38,36 +38,35 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._jsonl import latest, read_jsonl # noqa: E402
from bench.stats import mean_ci # noqa: E402
-from codeverse3d.proc import read_jsonl_lenient # noqa: E402
#: judge repeatability on the calibration set (docs/PAPER_WRITING.md §2), used only to
#: state how much of a best-of-k maximum is selection noise
JUDGE_SIGMA = 0.030
+def _last_rows(path: Path, arm_prefix: str = "") -> list[dict]:
+ """One row per prompt of the arms starting with ``arm_prefix`` — the LAST, as
+ ``paired_compare`` pairs; a prompt whose last row is unscored has no score."""
+ rows = (r for r in read_jsonl(path) if str(r.get("arm", "")).startswith(arm_prefix))
+ return list(latest(rows, key=lambda r: r["prompt_id"]).values())
+
+
def one_shot_samples(root: Path) -> dict[str, list[float]]:
"""``{prompt: [score per rep]}`` — one entry per rep that scored that prompt."""
out: dict[str, list[float]] = defaultdict(list)
for rep in sorted(root.glob("rep*/results.jsonl")):
- seen: dict[str, float] = {}
- for r in read_jsonl_lenient(rep, dicts_only=True):
- if r.get("score") is None:
- continue
- seen[r["prompt_id"]] = float(r["score"]) # last row wins, as paired_compare does
- for pid, sc in seen.items():
- out[pid].append(sc)
+ for r in _last_rows(rep):
+ if r.get("score") is not None:
+ out[r["prompt_id"]].append(float(r["score"]))
return dict(out)
def harness_scores(root: Path, *, arm_prefix: str = "harness:") -> dict[str, float]:
"""``{prompt: score}`` for the recorded harness arm, last row per prompt."""
- out: dict[str, float] = {}
- for r in read_jsonl_lenient(root / "results.jsonl", dicts_only=True):
- if not str(r.get("arm", "")).startswith(arm_prefix) or r.get("score") is None:
- continue
- out[r["prompt_id"]] = float(r["score"])
- return out
+ return {r["prompt_id"]: float(r["score"]) for r in _last_rows(root / "results.jsonl", arm_prefix)
+ if r.get("score") is not None}
def harness_repeat_spread(root: Path, *, arm_prefix: str = "harness:") -> tuple[int, int, float]:
@@ -82,7 +81,7 @@ def harness_repeat_spread(root: Path, *, arm_prefix: str = "harness:") -> tuple[
"""
per: dict[str, list[float]] = {}
rows = 0
- for r in read_jsonl_lenient(root / "results.jsonl", dicts_only=True):
+ for r in read_jsonl(root / "results.jsonl"):
if not str(r.get("arm", "")).startswith(arm_prefix):
continue
rows += 1
@@ -96,11 +95,8 @@ def gen_costs(root: Path, arm_prefix: str) -> list[float]:
"""Generation cost per PROMPT for an arm — last row per prompt, the basis
`paired_compare` pairs on. Taken over every appended row instead, resumed and failed
cells drag the median down and the equal-compute k comes out too small."""
- last: dict[str, float] = {}
- for r in read_jsonl_lenient(root / "results.jsonl", dicts_only=True):
- if str(r.get("arm", "")).startswith(arm_prefix):
- last[r["prompt_id"]] = float(r.get("gen_cost_usd") or 0.0)
- return [v for v in last.values() if v > 0]
+ costs = (float(r.get("gen_cost_usd") or 0.0) for r in _last_rows(root / "results.jsonl", arm_prefix))
+ return [c for c in costs if c > 0]
def equal_compute_k(root: Path, baseline_prefix: str) -> tuple[float, float, int] | None:
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.py
index 9965abf6..a1a931d8 100644
--- a/eval/bench/compare_backends.py
+++ b/eval/bench/compare_backends.py
@@ -56,15 +56,10 @@ for _p in (REPO, Path(__file__).resolve().parents[1]): # its codeverse3d +
from datetime import UTC # noqa: E402
-from bench._compare_report import ( # noqa: E402
- CellResult,
- PairRow,
- build_compare_report,
- load_jsonl,
-)
+from bench._compare_report import CellResult, PairRow, build_compare_report # noqa: E402
from bench._fixed_eval import RUBRIC, EvalOutcome, FixedEvaluator # noqa: E402
from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402
-from bench._jsonl import seal_for_append # noqa: E402
+from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402
from bench._oneshot import ( # noqa: E402
OneShotBackend,
OneShotResult,
@@ -430,7 +425,7 @@ def _renders_of(out: Path, r: CellResult) -> RenderSet | None:
def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], arms: Sequence[Arm], out: Path,
opts: CompareOptions, deps: CompareDeps) -> list[PairRow]:
path = out / "pairwise.jsonl"
- done = {(p.prompt_id, p.arm_a, p.arm_b): p for p in load_jsonl(path, PairRow)} if opts.resume else {}
+ done = latest(read_jsonl(path, PairRow)) if opts.resume else {}
harness = [a for a in arms if a.kind == "harness"]
oneshot = [a for a in arms if a.kind != "harness"]
judge = None
@@ -464,7 +459,8 @@ def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], arm
# ----------------------------------------------------------------------------- matrix
def _drop_pairs(path: Path, key: tuple[str, str]) -> None:
"""Forget pairwise rows touching a cell that is about to be re-run."""
- rows = [p for p in load_jsonl(path, PairRow) if not (p.prompt_id == key[0] and key[1] in (p.arm_a, p.arm_b))]
+ rows = [p for p in latest(read_jsonl(path, PairRow)).values()
+ if not (p.prompt_id == key[0] and key[1] in (p.arm_a, p.arm_b))]
if path.is_file():
path.write_text("".join(p.model_dump_json() + "\n" for p in rows))
@@ -478,7 +474,7 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm
"options": opts.model_dump(mode="json"),
"started_at": datetime.now(UTC).isoformat()}, indent=2))
results = out / "results.jsonl"
- done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}
+ done = latest(read_jsonl(results, CellResult)) if opts.resume else {}
for key in [k for k, r in done.items() if r.status in set(opts.redo_status)]:
_drop_pairs(out / "pairwise.jsonl", key)
del done[key]
@@ -497,7 +493,7 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm
_, r = _new_cell(p, a, out, opts)
r.status = "error"
r.note(f"run_cell raised {type(e).__name__}: {e}"[:800])
- done[(r.prompt_id, r.arm)] = r
+ done[r.natural_key()] = r
fh.write(r.model_dump_json() + "\n")
fh.flush()
if on_result:
@@ -607,7 +603,8 @@ def main(argv: Sequence[str] | None = None) -> int:
run_matrix(ns.prompts, ns.out, arms, opts, deps, on_result=_log)
print(f"report: {Path(ns.out) / 'report.md'}")
- dropped = [r for r in load_jsonl(Path(ns.out) / "results.jsonl", CellResult) if r.status == "infra_failed"]
+ dropped = [r for r in latest(read_jsonl(Path(ns.out) / "results.jsonl", CellResult)).values()
+ if r.status == "infra_failed"]
if dropped:
# never let downtime pass as a result: say what was lost and how to get it back
print(f"\n{len(dropped)} cell(s) lost to provider outages and EXCLUDED from every rate: "
diff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.py
index f4ccd58d..c8c3ed0f 100644
--- a/eval/bench/h2h_scene.py
+++ b/eval/bench/h2h_scene.py
@@ -40,13 +40,13 @@ from pydantic import BaseModel
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._jsonl import latest, read_jsonl # noqa: E402
from bench.run_bench import Battery, BenchPrompt # noqa: E402
from bench.stats import sign_test # noqa: E402
from codeverse3d.addons import select # noqa: E402
from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402
from codeverse3d.contracts.common import Language, Track # noqa: E402
from codeverse3d.contracts.spec import Spec # noqa: E402
-from codeverse3d.proc import read_jsonl_lenient # noqa: E402
HERE = Path(__file__).resolve().parent
BATTERY = HERE / "prompts" / "h2h_scene_v1.yaml"
@@ -148,10 +148,8 @@ def our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:
stills = [v for v in views if (v.time_s or 0.0) == 0.0 and v.mode == "shaded" and Path(v.path).is_file()]
flagged = [v for v in stills if v.judge] or stills
chosen = flagged[:MAX_VIEWS]
- minutes = None
- for r in read_jsonl_lenient(run_dir.parent.parent / "results.jsonl", dicts_only=True):
- if r.get("id") == run_dir.name:
- minutes = float(r.get("minutes") or 0.0)
+ row = latest(read_jsonl(run_dir.parent.parent / "results.jsonl"), key=lambda r: r.get("id")).get(run_dir.name)
+ minutes = None if row is None else float(row.get("minutes") or 0.0)
if minutes is None and rec.get("started_at") and rec.get("finished_at"):
t0, t1 = (datetime.fromisoformat(rec[k]) for k in ("started_at", "finished_at"))
minutes = (t1 - t0).total_seconds() / 60
diff --git a/eval/bench/paired_compare.py b/eval/bench/paired_compare.py
index cf6d2b71..568ab1d6 100644
--- a/eval/bench/paired_compare.py
+++ b/eval/bench/paired_compare.py
@@ -35,7 +35,7 @@ for _p in (REPO, Path(__file__).resolve().parents[1]): # its codeverse3d +
from bench._ab_report import _fmt as _f # noqa: E402
from bench._compare_report import CellResult # noqa: E402
-from bench._jsonl import read_jsonl # noqa: E402
+from bench._jsonl import latest, read_jsonl # noqa: E402
from bench.stats import mean_ci, sign_test # noqa: E402
@@ -69,14 +69,6 @@ class PairedStats(BaseModel):
deltas: dict[str, float] = Field(default_factory=dict, description="prompt id → harness − one-shot")
-def latest_cells(rows: list[CellResult]) -> dict[tuple[str, str], CellResult]:
- """(prompt_id, arm) → last recorded row (the journal is append-only; a re-run supersedes)."""
- out: dict[tuple[str, str], CellResult] = {}
- for r in rows:
- out[(r.prompt_id, r.arm)] = r
- return out
-
-
def _usable(r: CellResult | None) -> tuple[bool, str]:
if r is None:
return False, "missing"
@@ -89,6 +81,7 @@ def _usable(r: CellResult | None) -> tuple[bool, str]:
def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_arm: str, *,
tier: str = "all", exclude_degraded: bool = False) -> PairedStats:
+ """One comparison over ``cells`` — ``bench/_jsonl.latest``: (prompt, arm) → last row."""
prompts = sorted({p for (p, a) in cells if a in (harness_arm, oneshot_arm)})
st = PairedStats(harness_arm=harness_arm, oneshot_arm=oneshot_arm, tier=tier, n=0)
hs: list[float] = []
@@ -148,7 +141,7 @@ def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_a
def analyse(rows: list[CellResult]) -> list[PairedStats]:
- cells = latest_cells(rows)
+ cells = latest(rows)
arms = sorted({a for (_, a) in cells})
harness = [a for a in arms if a.startswith("harness:")]
oneshot = [a for a in arms if a.startswith("oneshot")]
@@ -241,10 +234,7 @@ def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:
the same for both shapes rather than recomputed by hand.
"""
rows: list[CellResult] = []
- for line in (out_dir / "results.jsonl").read_text().splitlines():
- if not line.strip():
- continue
- raw = json.loads(line)
+ for raw in latest(read_jsonl(out_dir / "results.jsonl"), key=lambda r: r["id"]).values():
# a bench-run row records no build flag: a cell with a verdict was built and judged,
# one without (score None) was not — so `build_ok` here means "judged", and
# `gen_cost_usd` is the run's WHOLE cost (plan + loop judge), as the field says. A
@@ -282,14 +272,14 @@ def main(argv: list[str] | None = None) -> int:
if ns.against is not None:
name_a, name_b = arm_names(out, ns.against)
rows = rows_from_bench_run(out, name_a) + rows_from_bench_run(ns.against, name_b)
- stats = [paired(latest_cells(rows), name_a, name_b)]
+ stats = [paired(latest(rows), name_a, name_b)]
md = render_markdown(stats, f"{name_a} vs {name_b}")
(out / "paired.md").write_text(md)
print(md)
return 0
rows = read_jsonl(out / "results.jsonl", CellResult)
stats = analyse(rows)
- gates = gate_stats(latest_cells(rows))
+ gates = gate_stats(latest(rows))
md = render_markdown(stats, out.name, gates)
(out / "paired.md").write_text(md)
(out / "paired.json").write_text(json.dumps({"paired": [s.model_dump(mode="json") for s in stats],
diff --git a/eval/bench/plan_stage_bench.py b/eval/bench/plan_stage_bench.py
index 70766977..729c1c1e 100644
--- a/eval/bench/plan_stage_bench.py
+++ b/eval/bench/plan_stage_bench.py
@@ -38,16 +38,9 @@ DEFAULT_JUDGE = "gemini:gemini-3.1-pro-preview"
def _done(out: Path) -> set[tuple[str, int]]:
"""(prompt, rep) pairs already recorded, so a killed run resumes instead of repeating."""
- if not out.is_file():
- return set()
- seen: set[tuple[str, int]] = set()
- for line in out.read_text().splitlines():
- try:
- row = json.loads(line)
- except ValueError:
- continue
- seen.add((row["prompt"], row["rep"]))
- return seen
+ from bench._jsonl import read_jsonl # after main's path insert, like every bench import here
+
+ return {(row["prompt"], row["rep"]) for row in read_jsonl(out)}
def _plan_shape(ws_root: Path) -> dict:
@@ -68,15 +61,10 @@ def _plan_shape(ws_root: Path) -> dict:
def _stats(ws_root: Path) -> dict:
"""What the plan stage's event log says about the call that just ran."""
+ from bench._jsonl import read_jsonl # after main's path insert, like every bench import here
+
out: dict = {"invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0}
- events = ws_root / "events.jsonl"
- if not events.is_file():
- return out
- for line in events.read_text().splitlines():
- try:
- e = json.loads(line)
- except ValueError:
- continue
+ for e in read_jsonl(ws_root / "events.jsonl"):
kind = e.get("event")
if kind == "plan.invalid":
out["invalid_reasks"] += 1
@@ -194,7 +182,7 @@ def main(argv: list[str] | None = None) -> int:
sys.path.insert(0, str(ns.tree.resolve())) # the harness tree under test wins for `codeverse3d`
import codeverse3d
- from bench._jsonl import seal_for_append # after the path insert: `python bench/x.py`
+ from bench._jsonl import read_jsonl, seal_for_append # after the path insert (script run)
from bench.run_bench import Battery
from codeverse3d.config import get_settings
if not Path(codeverse3d.__file__).resolve().is_relative_to(ns.tree.resolve()):
@@ -224,7 +212,7 @@ def main(argv: list[str] | None = None) -> int:
fh.write(json.dumps(row) + "\n")
fh.flush()
print(f" {row['prompt']:28} r{row['rep']:<3} ok={row['ok']} restart={row['restarts']} {row['error'][:60]}")
- rows = [json.loads(x) for x in ns.out.read_text().splitlines() if x.strip()]
+ rows = read_jsonl(ns.out)
spent = sum(float(r.get("cost_usd") or 0.0) for r in rows)
print(f"{ns.label}: {sum(1 for r in rows if r['ok'])}/{len(rows)} plans valid, "
f"${spent:.2f} over {len(rows)} calls (the rows are the ledger: plan_once does not "
diff --git a/eval/bench/plan_stage_report.py b/eval/bench/plan_stage_report.py
index de17d16a..cc859fb5 100644
--- a/eval/bench/plan_stage_report.py
+++ b/eval/bench/plan_stage_report.py
@@ -10,11 +10,16 @@ are excluded from the denominator — they say nothing about the code under test
from __future__ import annotations
import argparse
-import json
import math
+import sys
from collections import Counter
from pathlib import Path
+for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
+ sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+
+from bench._jsonl import read_jsonl # noqa: E402
+
#: what a validation failure was ABOUT, read off the error text. The mechanism under test
#: (``C3D_PLAN_RESTART``) only ever addresses ``dangling_link``; every other class is a
#: bystander, and a total that mixes them hides both the effect and its residue.
@@ -82,10 +87,6 @@ def fisher_exact(a: int, b: int, c: int, d: int) -> float:
return min(1.0, sum(prob(x) for x in range(lo, hi + 1) if prob(x) <= observed * (1 + 1e-9)))
-def load(path: Path) -> list[dict]:
- return [json.loads(x) for x in path.read_text().splitlines() if x.strip()] if path.is_file() else []
-
-
def report(arms: dict[str, list[dict]]) -> str:
lines = ["| arm | calls | valid | planning_error | provider | failure rate | 95 % CI |",
"|---|--:|--:|--:|--:|--:|---|"]
@@ -156,7 +157,7 @@ def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("rows", nargs="+", type=Path, help="one JSONL per arm (the file name is the arm name)")
ns = ap.parse_args(argv)
- print(report({p.stem: load(p) for p in ns.rows}))
+ print(report({p.stem: read_jsonl(p) for p in ns.rows}))
return 0
diff --git a/eval/bench/report.py b/eval/bench/report.py
index 35ef6373..9b8dc7ff 100644
--- a/eval/bench/report.py
+++ b/eval/bench/report.py
@@ -19,7 +19,7 @@ from pydantic import BaseModel, Field
from bench._compare_report import _f as _fmt
from bench._compare_report import _mean
-from bench._jsonl import read_jsonl
+from bench._jsonl import latest, read_jsonl
from bench.run_bench import BenchItemResult
from codeverse3d.addons.gallery import GalleryIndex, RootSection, RunEntry, render_static
from codeverse3d.addons.gallery.index import entry_from_record
@@ -57,8 +57,7 @@ class BenchReport(BaseModel):
def load_results(out_dir: Path) -> list[BenchItemResult]:
jl = out_dir / "results.jsonl"
if jl.is_file():
- latest: dict[str, BenchItemResult] = {r.id: r for r in read_jsonl(jl, BenchItemResult)}
- return list(latest.values())
+ return list(latest(read_jsonl(jl, BenchItemResult)).values())
js = out_dir / "results.json"
if js.is_file():
return [BenchItemResult.model_validate(r) for r in json.loads(js.read_text())]
diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.py
index 18c659b8..480b875e 100644
--- a/eval/bench/run_bench.py
+++ b/eval/bench/run_bench.py
@@ -36,7 +36,7 @@ import yaml
from pydantic import AliasChoices, BaseModel, Field
from bench._infra import is_infra_failure
-from bench._jsonl import read_jsonl, seal_for_append
+from bench._jsonl import latest, read_jsonl, seal_for_append
from codeverse3d.addons import select
from codeverse3d.config import get_settings
from codeverse3d.contracts.common import Backends, Budget, Language, Track
@@ -99,6 +99,11 @@ class BenchItemResult(BaseModel):
generator: str = ""
judge: str = ""
+ def natural_key(self) -> str:
+ """Row identity: one row per prompt id — results.jsonl is append-only, so a resume
+ or a ``--redo-status`` re-run appends a second row for the same id."""
+ return self.id
+
class BenchOptions(BaseModel):
generator: str | None = None
@@ -198,7 +203,7 @@ def _load_done(results_jsonl: Path) -> dict[str, BenchItemResult]:
"""Rows already paid for, latest wins. Tolerant by design: this file is the
resume source and it is appended to a line at a time, so the run that a SIGKILL
ended is precisely the one whose last line is half-written."""
- return {r.id: r for r in read_jsonl(results_jsonl, BenchItemResult)}
+ return latest(read_jsonl(results_jsonl, BenchItemResult))
def select_prompts(
diff --git a/eval/bench/scene_stats.py b/eval/bench/scene_stats.py
index 50f5bf6f..af60017c 100644
--- a/eval/bench/scene_stats.py
+++ b/eval/bench/scene_stats.py
@@ -27,7 +27,7 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
-from codeverse3d.proc import read_jsonl_lenient # noqa: E402
+from bench._jsonl import read_jsonl # noqa: E402
from codeverse3d.record.record import unique_files # noqa: E402
from codeverse3d.spatial.node import browser_was_lost # noqa: E402
@@ -56,7 +56,7 @@ def runs(root: Path) -> list[tuple[str, dict, list[dict]]]:
"""``(prompt id, record, events)`` per scene run under ``root``, once per run."""
out = []
for run, data in scene_records(root):
- events = read_jsonl_lenient(run / "events.jsonl", dicts_only=True)
+ events = read_jsonl(run / "events.jsonl")
out.append((str((data.get("spec") or {}).get("id", run.name)).split("/")[-1], data, events))
# by id, then by path: two runs of one prompt (a battery with reps) share the id, and a
# bare sorted() would then compare their record dicts and raise. unique_files yields
diff --git a/eval/tests/test_ab_plan.py b/eval/tests/test_ab_plan.py
index e8fea592..4dcdef62 100644
--- a/eval/tests/test_ab_plan.py
+++ b/eval/tests/test_ab_plan.py
@@ -22,7 +22,8 @@ from bench._ab_report import ( # noqa: E402
render_summary,
verdict_of,
)
-from bench._compare_report import CellResult, load_jsonl # noqa: E402
+from bench._compare_report import CellResult # noqa: E402
+from bench._jsonl import latest, read_jsonl # noqa: E402
from bench.ab_plan import ( # noqa: E402
DEFAULT_MAX_IN_FLIGHT,
MAX_IN_FLIGHT_ENV,
@@ -51,6 +52,10 @@ def _row(pid: str, arm: str, score: float | None, status: str = "scored", **kw)
gen_cost_usd=kw.pop("cost", 1.0), judge_cost_usd=0.1, wall_s=kw.pop("wall", 120.0), **kw)
+def _journal(out: Path) -> list[CellResult]:
+ return list(latest(read_jsonl(out / "results.jsonl", CellResult)).values())
+
+
def _pairs(*deltas: float) -> list[PairOutcome]:
return [PairOutcome(prompt_id=f"p{i}", paired=True, delta=d) for i, d in enumerate(deltas)]
@@ -278,7 +283,7 @@ def test_driver_runs_pairs_in_prompt_order_and_writes_the_verdict(tmp_path: Path
assert [c[0] for c in fake.calls] == [i for i in ids for _ in range(2)]
assert {c[1] for c in fake.calls} == {CONTROL, VARIANT}
assert v.decision == "keep" and v.n_pairs == 3 and v.mean_delta == pytest.approx(0.0333, abs=1e-3)
- rows = load_jsonl(tmp_path / "results.jsonl", CellResult)
+ rows = _journal(tmp_path)
assert len(rows) == 6
# the env reached only the variant cells
assert {r.workspace for r in rows if r.arm == VARIANT} == {"on"}
@@ -306,9 +311,9 @@ def test_resume_skips_done_pairs_and_redo_reruns_both_arms(tmp_path: Path):
assert sorted(redo.calls) == [(ids[1], CONTROL), (ids[1], VARIANT)]
assert v.n_pairs == 2
assert len((tmp_path / "results.jsonl").read_text().strip().splitlines()) == 6, "the file is append-only"
- rows = load_jsonl(tmp_path / "results.jsonl", CellResult)
- latest = {(r.prompt_id, r.arm): r for r in rows}
- assert latest[(ids[1], VARIANT)].status == "scored"
+ cells = latest(read_jsonl(tmp_path / "results.jsonl", CellResult))
+ rows = list(cells.values())
+ assert cells[(ids[1], VARIANT)].status == "scored"
assert len(rows) == 4, "readers see one row per (prompt, arm): the redo replaces the stale attempt"
@@ -356,7 +361,7 @@ def test_aa_run_records_two_empty_arms(tmp_path: Path):
run_ab(BATTERY, tmp_path, AbOptions(variant_env={"C3D_PLAN_FEATURES": "all"}, aa=True, ids=ids), run_cell_fn=fake)
meta = json.loads((tmp_path / "ab.json").read_text())
assert meta["aa"] is True and meta["arms"] == {"control": {}, "variant": {}}
- assert {r.workspace for r in load_jsonl(tmp_path / "results.jsonl", CellResult)} == {""}, "no arm saw the switch"
+ assert {r.workspace for r in _journal(tmp_path)} == {""}, "no arm saw the switch"
assert (tmp_path / "summary.md").read_text().startswith("# A/A: ")
@@ -406,7 +411,7 @@ def test_a_pinned_plan_that_dies_in_a_storm_records_the_pair_and_continues(tmp_p
opts = AbOptions(variant_env={"C3D_SKILLS": "0"}, pin_plan=True, ids=ids)
run_ab(BATTERY, tmp_path, opts, run_cell_fn=fake)
- rows = load_jsonl(tmp_path / "results.jsonl", CellResult)
+ rows = _journal(tmp_path)
first = [r for r in rows if r.prompt_id == ids[0]]
assert {r.arm for r in first} == {CONTROL, VARIANT} and all(r.status == "infra_failed" for r in first)
assert all("pinned plan" in r.error for r in first)
diff --git a/eval/tests/test_bench_shared.py b/eval/tests/test_bench_shared.py
index 6df35ef7..9f203779 100644
--- a/eval/tests/test_bench_shared.py
+++ b/eval/tests/test_bench_shared.py
@@ -113,7 +113,8 @@ def test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:
interval crosses zero" — rather than being recomputed by hand."""
import json as _json
- from bench.paired_compare import latest_cells, paired, rows_from_bench_run
+ from bench._jsonl import latest
+ from bench.paired_compare import paired, rows_from_bench_run
def write(d: Path, scores: dict[str, float]) -> Path:
d.mkdir(parents=True)
@@ -126,7 +127,7 @@ def test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:
b = write(tmp_path / "arm_b", {"p1": 0.5, "p2": 0.3, "p3": 0.5})
rows = rows_from_bench_run(a, "arm_a") + rows_from_bench_run(b, "arm_b")
- st = paired(latest_cells(rows), "arm_a", "arm_b")
+ st = paired(latest(rows), "arm_a", "arm_b")
assert st.n == 3 and st.mean_delta == pytest.approx(0.0667, abs=1e-3)
assert st.wins == 2 and st.losses == 0 and st.ties == 1
assert st.verdict == "unsupported" # CI [-0.077, +0.21] crosses zero at n = 3
diff --git a/eval/tests/test_compare_matrix.py b/eval/tests/test_compare_matrix.py
index bb535bfa..d7fd6b0b 100644
--- a/eval/tests/test_compare_matrix.py
+++ b/eval/tests/test_compare_matrix.py
@@ -16,10 +16,10 @@ from bench._compare_report import ( # noqa: E402
CellResult,
PairRow,
arm_stats,
- load_jsonl,
pair_stats,
)
from bench._fixed_eval import acceptance_from_spec # noqa: E402
+from bench._jsonl import latest, read_jsonl # noqa: E402
from bench.compare_backends import ( # noqa: E402
CompareDeps,
CompareOptions,
@@ -120,7 +120,7 @@ def test_matrix_end_to_end_with_fakes(tmp_path: Path):
# the fixed evaluator ran once per cell that had code (6 of 8)
assert len(ev.evaluated) == 6
# pairwise: 1 harness arm × 3 one-shot arms × 2 prompts = 6 rows; the no-code arm is auto-decided
- pairs = load_jsonl(out / "pairwise.jsonl", PairRow)
+ pairs = list(latest(read_jsonl(out / "pairwise.jsonl", PairRow)).values())
assert len(pairs) == 6
auto = [p for p in pairs if p.arm_b == "oneshot:gemini:gemini-3.7-flash"]
assert all(p.winner == "a" and not p.judged for p in auto)
@@ -149,7 +149,7 @@ def test_matrix_end_to_end_with_fakes(tmp_path: Path):
# ---- adding a prompt (limit 3) only runs the new cells
rows3 = run_matrix(BATTERY, out, parse_arms(ARMS), opts.model_copy(update={"limit": 3}), deps2)
assert len(rows3) == 12 and run_track2.calls == ["compare_v1/cmp_med_dining_chair"]
- assert len(load_jsonl(out / "pairwise.jsonl", PairRow)) == 9
+ assert len(latest(read_jsonl(out / "pairwise.jsonl", PairRow))) == 9
def test_cell_errors_are_recorded_not_raised(tmp_path: Path):
diff --git a/eval/tests/test_paired_compare.py b/eval/tests/test_paired_compare.py
index 9da16936..05dfefa1 100644
--- a/eval/tests/test_paired_compare.py
+++ b/eval/tests/test_paired_compare.py
@@ -11,7 +11,8 @@ if str(REPO) not in sys.path:
sys.path.insert(0, str(REPO))
from bench._compare_report import CellResult # noqa: E402
-from bench.paired_compare import analyse, latest_cells, main, paired # noqa: E402
+from bench._jsonl import latest # noqa: E402
+from bench.paired_compare import analyse, main, paired, rows_from_bench_run # noqa: E402
from bench.stats import t975 # noqa: E402
HA, OA = "harness:gemini-cli:gemini-3.6-flash", "oneshot:gemini:x"
@@ -33,7 +34,7 @@ def test_paired_stats_ci_sign_test_and_drops():
rows += [_cell("p7", HA, None, status="budget_exhausted"), _cell("p7", OA, 0.4)]
# the journal is append-only: a re-run of p0's harness cell supersedes the first row
rows += [_cell("p0", HA, 0.82, passed=True)]
- st = paired(latest_cells(rows), HA, OA)
+ st = paired(latest(rows), HA, OA)
assert st.n == 6 and st.dropped_infra == 1 and st.dropped_unscored == 1
assert st.deltas["p0"] == 0.22 and st.wins == 5 and st.losses == 1 and st.ties == 0
assert st.sign_p == 0.2188
@@ -83,3 +84,14 @@ def test_degraded_harness_cells_are_reported_and_can_be_excluded():
plain = analyse([_cell("q", HA, 0.7), _cell("q", OA, 0.5)])
assert "all −degraded" not in {s.tier for s in plain}
+
+
+def test_a_killed_bench_run_journal_costs_its_last_row_not_the_comparison(tmp_path):
+ """`rows_from_bench_run` parsed every line strictly, so the half-written row a SIGKILL
+ leaves made `--against` crash on the whole battery; and a redo re-appends a row."""
+ rows = [{"id": "p1", "tier": "hard", "score_picked": 0.4}, {"id": "p2", "tier": "hard", "score_picked": 0.5},
+ {"id": "p1", "tier": "hard", "score_picked": 0.9}]
+ text = "".join(json.dumps(r) + "\n" for r in rows) + '{"id": "p3", "tier": "ha'
+ (tmp_path / "results.jsonl").write_text(text)
+ cells = rows_from_bench_run(tmp_path, "arm")
+ assert [(c.prompt_id, c.score) for c in cells] == [("p1", 0.9), ("p2", 0.5)]
diff --git a/eval/tests/test_plan_stage.py b/eval/tests/test_plan_stage.py
index 9f11c66e..3a00dde2 100644
--- a/eval/tests/test_plan_stage.py
+++ b/eval/tests/test_plan_stage.py
@@ -75,3 +75,14 @@ def test_a_harness_side_death_is_a_loss_not_provider_weather() -> None:
assert outcome({"ok": False, "error": "ModelError: Gemini prompt blocked: BlockedReason.OTHER"}) == "provider"
assert outcome({"ok": False, "error": "KeyError: 'parts'"}) == "planning_error"
+
+
+def test_a_killed_arm_file_still_reports(tmp_path: Path, capsys) -> None:
+ """The report's loader parsed every line strictly: the half-written row a killed
+ ``plan_stage_bench`` leaves made the whole arm unreadable."""
+ from bench.plan_stage_report import main
+
+ arm = tmp_path / "arm.jsonl"
+ arm.write_text(json.dumps({"prompt": "p", "rep": 0, "ok": True, "error": ""}) + '\n{"prompt": "p", "re')
+ assert main([str(arm)]) == 0
+ assert "| arm | 1 | 1 | 0 | 0 | 0.000 |" in capsys.readouterr().out