Official Codebase of 3DCodeVerse
Python
bench: one truncated line in results.jsonl made a killed battery unresumable
gaoypengf84ecb0ed030
diff --git a/harness/bench/_compare_report.py b/harness/bench/_compare_report.pyindex 6962e5ab..fe05a618 100644--- a/harness/bench/_compare_report.py+++ b/harness/bench/_compare_report.py@@ -17,6 +17,8 @@ from typing import Any, TypeVarfrom pydantic import BaseModel, Field+from bench._jsonl import read_jsonl+T = TypeVar("T", bound=BaseModel)@@ -76,7 +78,7 @@ class PairRow(BaseModel):def load_jsonl(path: Path, model: type[T]) -> list[T]:"""Rows in first-seen order, latest row winning per ``natural_key`` (models without- one keep every row).+ one keep every row). Unparseable lines are skipped, not fatal — see ``bench/_jsonl.py``.The dedup lives HERE, once, because the files are append-only and every reader needsthe same rows: ``ab_plan --report-only`` used to hand the raw rows to ``render_summary``@@ -84,14 +86,12 @@ def load_jsonl(path: Path, model: type[T]) -> list[T]:as 0.400 in the arms table, 0.800 per prompt, and based its verdict on 0.800 — and the"lost to outages" footer counted infra_failed cells that had already been re-run andscored, advising ``--redo-status infra_failed`` for nothing.++ Tolerance and dedup compose: a killed battery's truncated last line is dropped by+ ``read_jsonl`` and the surviving rows are still collapsed to one per cell."""- if not path.is_file():- return []rows: dict[Any, T] = {}- for i, line in enumerate(path.read_text().splitlines()):- if not line.strip():- continue- row = model.model_validate_json(line)+ for i, row in enumerate(read_jsonl(path, model)):key = row.natural_key() if hasattr(row, "natural_key") else irows[key] = row # first appearance fixes the order, the last row wins the slotreturn list(rows.values())diff --git a/harness/bench/_jsonl.py b/harness/bench/_jsonl.pynew file mode 100644index 00000000..67784638--- /dev/null+++ b/harness/bench/_jsonl.py@@ -0,0 +1,64 @@+"""One tolerant reader for the append-only ``*.jsonl`` files the bench drivers+resume and report from.++Every one of those files is written a line at a time with a flush per row+(``run_bench.py``, ``compare_backends.py``, ``ab_plan.py``), which is exactly the+shape a SIGKILL truncates mid-append. Each driver used to validate every line+strictly, so a single half-written last line made the whole file unusable: a killed+battery could not be resumed *at all* and its already-paid rows could not even be+reported — the operator had to hand-edit ``results.jsonl`` or re-buy the battery.++Skip the bad line, keep the rest, say so once — the same contract+``codeverse.cost.ledger.load_ledger`` has always had ("a truncated last line never+loses the rest of the file").+"""++from __future__ import annotations++import logging+from pathlib import Path+from typing import TypeVar++from pydantic import BaseModel++log = logging.getLogger(__name__)++T = TypeVar("T", bound=BaseModel)+++def read_jsonl(path: Path, model: type[T]) -> list[T]:+ """Parse ``path`` as one ``model`` per line, skipping unparseable lines with a+ warning. Missing file → ``[]``."""+ if not Path(path).is_file():+ return []+ rows: list[T] = []+ bad = 0+ for i, line in enumerate(Path(path).read_text(encoding="utf-8", errors="replace").splitlines()):+ if not line.strip():+ continue+ try:+ rows.append(model.model_validate_json(line))+ except Exception as e: # a truncated/corrupt row must not cost us the good ones+ bad += 1+ log.warning("%s:%d unreadable, skipping (%s: %s)", path, i + 1, type(e).__name__, str(e)[:120])+ if bad:+ log.warning("%s: skipped %d unreadable line(s), kept %d", path, bad, len(rows))+ return rows+++def seal_for_append(path: Path) -> None:+ """Terminate a partial last line before appending to a journal.++ A SIGKILL leaves ``results.jsonl`` with a last line that has no ``\n``. Opening+ it ``"a"`` and writing the next row would then glue the new row onto the broken+ one and lose BOTH — the resumed cell would be paid for and then dropped on every+ subsequent read. One newline keeps the damage to the single row that was+ actually interrupted.+ """+ p = Path(path)+ if not p.is_file() or p.stat().st_size == 0:+ return+ with p.open("rb+") as fh:+ fh.seek(-1, 2)+ if fh.read(1) != b"\n":+ fh.write(b"\n")diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.pyindex c84e9955..47ef27d0 100644--- a/harness/bench/ab_plan.py+++ b/harness/bench/ab_plan.py@@ -66,6 +66,7 @@ if str(REPO) not in sys.path: # `python bench/ab_plan.py` from anywherefrom bench._ab_report import ARMS, CONTROL, VARIANT, Verdict, write_report # noqa: E402from bench._compare_report import CellResult, load_jsonl # noqa: E402from bench._infra import is_infra_failure # noqa: E402+from bench._jsonl import seal_for_append # noqa: E402from bench.compare_backends import ( # noqa: E402Arm,CompareDeps,@@ -307,6 +308,7 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, ruresults = out / "results.jsonl"done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}todo = _plan_todo(battery, done, opts)+ seal_for_append(results) # a kill left the last row unterminated; do not glue onto itwith ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:for item, arms, fresh in todo: # one pair at a time: both arms of a prompt see the same weatherif fresh:diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.pyindex 3934ba91..18129b24 100644--- a/harness/bench/compare_backends.py+++ b/harness/bench/compare_backends.py@@ -59,6 +59,7 @@ from bench._compare_report import ( # noqa: E402)from bench._fixed_eval import RUBRIC, EvalOutcome, FixedEvaluator # noqa: E402from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402+from bench._jsonl import seal_for_append # noqa: E402from bench._oneshot import ( # noqa: E402MODEL_FILE,OneShotBackend,@@ -303,6 +304,7 @@ def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], armharness = [a for a in arms if a.kind == "harness"]oneshot = [a for a in arms if a.kind != "harness"]judge = None+ seal_for_append(path) # a kill left the last row unterminated; do not glue onto itwith path.open("a") as fh:for item in battery.prompts:for ha in harness:@@ -353,6 +355,7 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Armselected = select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit)todo = [(p, a) for a in sorted(arms, key=lambda a: a.kind == "harness") # cheap one-shots firstfor p in selected if (p.id, a.raw) not in done]+ seal_for_append(results) # a kill left the last row unterminated; do not glue onto itwith ThreadPoolExecutor(max_workers=max(1, opts.parallel)) as pool, results.open("a") as fh:futs = {pool.submit(run_cell, battery, p, a, out, opts, deps): (p, a) for p, a in todo}for fut in as_completed(futs):diff --git a/harness/bench/report.py b/harness/bench/report.pyindex bd0fb3f7..dccea21e 100644--- a/harness/bench/report.py+++ b/harness/bench/report.py@@ -16,6 +16,7 @@ from typing import Anyfrom pydantic import BaseModel, Field+from bench._jsonl import read_jsonlfrom bench.run_bench import BenchItemResultfrom codeverse.flywheel.gallery import GalleryItem, item_from_run, render_galleryfrom codeverse.flywheel.record import RecordError, load_record@@ -51,11 +52,7 @@ class BenchReport(BaseModel):def load_results(out_dir: Path) -> list[BenchItemResult]:jl = out_dir / "results.jsonl"if jl.is_file():- latest: dict[str, BenchItemResult] = {}- for line in jl.read_text().splitlines():- if line.strip():- r = BenchItemResult.model_validate_json(line)- latest[r.id] = r+ latest: dict[str, BenchItemResult] = {r.id: r for r in read_jsonl(jl, BenchItemResult)}return list(latest.values())js = out_dir / "results.json"if js.is_file():diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.pyindex 2734e018..8bbe763d 100644--- a/harness/bench/run_bench.py+++ b/harness/bench/run_bench.py@@ -32,6 +32,7 @@ from typing import Anyimport yamlfrom pydantic import BaseModel, Field+from bench._jsonl import read_jsonl, seal_for_appendfrom codeverse._compat import UTCfrom codeverse.config import get_settingsfrom codeverse.contracts.common import Backends, Budget, Language, Track@@ -136,13 +137,10 @@ def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Wodef _load_done(results_jsonl: Path) -> dict[str, BenchItemResult]:- done: dict[str, BenchItemResult] = {}- if results_jsonl.is_file():- for line in results_jsonl.read_text().splitlines():- if line.strip():- r = BenchItemResult.model_validate_json(line)- done[r.id] = r- return done+ """Rows already paid for, latest wins. Tolerant by design: this file is the+ resume source and it is appended to a line at a time, so the run that a SIGKILL+ ended is precisely the one whose last line is half-written."""+ return {r.id: r for r in read_jsonl(results_jsonl, BenchItemResult)}def select_prompts(@@ -213,6 +211,7 @@ def run_battery(generator=spec.backends.generator, judge=spec.backends.judge)return result_from_record(item, rec, (time.time() - t0) / 60, ws)+ seal_for_append(results_jsonl) # a kill left the last row unterminated; do not glue onto itwith ThreadPoolExecutor(max_workers=max(1, opts.parallel)) as pool, results_jsonl.open("a") as fh:futs = {pool.submit(_one, item): item for item in todo}for fut in as_completed(futs):diff --git a/harness/tests/flywheel_cli/test_bench.py b/harness/tests/flywheel_cli/test_bench.pyindex 405a4946..d9080014 100644--- a/harness/tests/flywheel_cli/test_bench.py+++ b/harness/tests/flywheel_cli/test_bench.py@@ -118,3 +118,32 @@ def test_every_bench_prompt_opens_its_own_run_ledger(tmp_path: Path):rows = load_ledger(Path(r.workspace))assert len(rows) == 1, f"{r.id}: {rows}"assert rows[0].run == r.id and rows[0].cost_usd > 0 # and not a sibling's row+++def test_a_truncated_last_line_does_not_cost_the_whole_resume(tmp_path: Path):+ """RS-3: results.jsonl is the resume source AND it is appended a line at a time,+ so the run a SIGKILL ended is exactly the one whose last line is half-written.+ Strict per-line validation made that file unusable: the battery could not be+ resumed at all and its already-paid rows could not even be reported, so the+ operator had to hand-edit the file or re-buy the battery."""+ battery = REPO / "bench" / "prompts" / "static_objects_v1.yaml"+ out = tmp_path / "bench_out"+ run_battery(battery, out, BenchOptions(parallel=1, limit=2), run_fn=_fake_run_fn({}))+ jl = out / "results.jsonl"+ good = [ln for ln in jl.read_text().splitlines() if ln.strip()]+ assert len(good) == 2+ # SIGKILL mid-append: the last row is truncated inside a JSON string+ jl.write_text("\n".join(good) [: -30])++ calls: list[str] = []++ def counting(spec, ws, resume):+ calls.append(spec.id.split("/")[-1])+ return _fake_run_fn({})(spec, ws, resume)++ res = run_battery(battery, out, BenchOptions(parallel=1, limit=2), run_fn=counting)+ # the intact first row is still credited: only the truncated prompt is re-run+ assert calls == [good and json.loads(good[1])["id"]]+ assert len(res) == 2+ # and the already-paid rows remain reportable+ assert {r.id for r in load_results(out)} == {json.loads(ln)["id"] for ln in good}