Skip to content

Official Codebase of 3DCodeVerse

Python

bench: one truncated line in results.jsonl made a killed battery unresumable

gaoypengf84ecb0ed030
diff --git a/harness/bench/_compare_report.py b/harness/bench/_compare_report.py
index 6962e5ab..fe05a618 100644
--- a/harness/bench/_compare_report.py
+++ b/harness/bench/_compare_report.py
@@ -17,6 +17,8 @@ from typing import Any, TypeVar
from pydantic import BaseModel, Field
+from bench._jsonl import read_jsonl
+
T = TypeVar("T", bound=BaseModel)
@@ -76,7 +78,7 @@ class PairRow(BaseModel):
def load_jsonl(path: Path, model: type[T]) -> list[T]:
"""Rows in first-seen order, latest row winning per ``natural_key`` (models without
- one keep every row).
+ one keep every row). Unparseable lines are skipped, not fatal — see ``bench/_jsonl.py``.
The dedup lives HERE, once, because the files are append-only and every reader needs
the same rows: ``ab_plan --report-only`` used to hand the raw rows to ``render_summary``
@@ -84,14 +86,12 @@ def load_jsonl(path: Path, model: type[T]) -> list[T]:
as 0.400 in the arms table, 0.800 per prompt, and based its verdict on 0.800 — and the
"lost to outages" footer counted infra_failed cells that had already been re-run and
scored, advising ``--redo-status infra_failed`` for nothing.
+
+ Tolerance and dedup compose: a killed battery's truncated last line is dropped by
+ ``read_jsonl`` and the surviving rows are still collapsed to one per cell.
"""
- if not path.is_file():
- return []
rows: dict[Any, T] = {}
- for i, line in enumerate(path.read_text().splitlines()):
- if not line.strip():
- continue
- row = model.model_validate_json(line)
+ for i, row in enumerate(read_jsonl(path, model)):
key = row.natural_key() if hasattr(row, "natural_key") else i
rows[key] = row # first appearance fixes the order, the last row wins the slot
return list(rows.values())
diff --git a/harness/bench/_jsonl.py b/harness/bench/_jsonl.py
new file mode 100644
index 00000000..67784638
--- /dev/null
+++ b/harness/bench/_jsonl.py
@@ -0,0 +1,64 @@
+"""One tolerant reader for the append-only ``*.jsonl`` files the bench drivers
+resume and report from.
+
+Every one of those files is written a line at a time with a flush per row
+(``run_bench.py``, ``compare_backends.py``, ``ab_plan.py``), which is exactly the
+shape a SIGKILL truncates mid-append. Each driver used to validate every line
+strictly, so a single half-written last line made the whole file unusable: a killed
+battery could not be resumed *at all* and its already-paid rows could not even be
+reported — the operator had to hand-edit ``results.jsonl`` or re-buy the battery.
+
+Skip the bad line, keep the rest, say so once — the same contract
+``codeverse.cost.ledger.load_ledger`` has always had ("a truncated last line never
+loses the rest of the file").
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TypeVar
+
+from pydantic import BaseModel
+
+log = logging.getLogger(__name__)
+
+T = TypeVar("T", bound=BaseModel)
+
+
+def read_jsonl(path: Path, model: type[T]) -> list[T]:
+ """Parse ``path`` as one ``model`` per line, skipping unparseable lines with a
+ warning. Missing file → ``[]``."""
+ if not Path(path).is_file():
+ return []
+ rows: list[T] = []
+ bad = 0
+ for i, line in enumerate(Path(path).read_text(encoding="utf-8", errors="replace").splitlines()):
+ if not line.strip():
+ continue
+ try:
+ rows.append(model.model_validate_json(line))
+ except Exception as e: # a truncated/corrupt row must not cost us the good ones
+ bad += 1
+ log.warning("%s:%d unreadable, skipping (%s: %s)", path, i + 1, type(e).__name__, str(e)[:120])
+ if bad:
+ log.warning("%s: skipped %d unreadable line(s), kept %d", path, bad, len(rows))
+ return rows
+
+
+def seal_for_append(path: Path) -> None:
+ """Terminate a partial last line before appending to a journal.
+
+ A SIGKILL leaves ``results.jsonl`` with a last line that has no ``\n``. Opening
+ it ``"a"`` and writing the next row would then glue the new row onto the broken
+ one and lose BOTH — the resumed cell would be paid for and then dropped on every
+ subsequent read. One newline keeps the damage to the single row that was
+ actually interrupted.
+ """
+ p = Path(path)
+ if not p.is_file() or p.stat().st_size == 0:
+ return
+ with p.open("rb+") as fh:
+ fh.seek(-1, 2)
+ if fh.read(1) != b"\n":
+ fh.write(b"\n")
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.py
index c84e9955..47ef27d0 100644
--- a/harness/bench/ab_plan.py
+++ b/harness/bench/ab_plan.py
@@ -66,6 +66,7 @@ if str(REPO) not in sys.path: # `python bench/ab_plan.py` from anywhere
from bench._ab_report import ARMS, CONTROL, VARIANT, Verdict, write_report # noqa: E402
from bench._compare_report import CellResult, load_jsonl # noqa: E402
from bench._infra import is_infra_failure # noqa: E402
+from bench._jsonl import seal_for_append # noqa: E402
from bench.compare_backends import ( # noqa: E402
Arm,
CompareDeps,
@@ -307,6 +308,7 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, ru
results = out / "results.jsonl"
done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}
todo = _plan_todo(battery, done, opts)
+ seal_for_append(results) # a kill left the last row unterminated; do not glue onto it
with ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:
for item, arms, fresh in todo: # one pair at a time: both arms of a prompt see the same weather
if fresh:
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index 3934ba91..18129b24 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -59,6 +59,7 @@ from bench._compare_report import ( # noqa: E402
)
from bench._fixed_eval import RUBRIC, EvalOutcome, FixedEvaluator # noqa: E402
from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402
+from bench._jsonl import seal_for_append # noqa: E402
from bench._oneshot import ( # noqa: E402
MODEL_FILE,
OneShotBackend,
@@ -303,6 +304,7 @@ def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], arm
harness = [a for a in arms if a.kind == "harness"]
oneshot = [a for a in arms if a.kind != "harness"]
judge = None
+ seal_for_append(path) # a kill left the last row unterminated; do not glue onto it
with path.open("a") as fh:
for item in battery.prompts:
for ha in harness:
@@ -353,6 +355,7 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm
selected = select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit)
todo = [(p, a) for a in sorted(arms, key=lambda a: a.kind == "harness") # cheap one-shots first
for p in selected if (p.id, a.raw) not in done]
+ seal_for_append(results) # a kill left the last row unterminated; do not glue onto it
with ThreadPoolExecutor(max_workers=max(1, opts.parallel)) as pool, results.open("a") as fh:
futs = {pool.submit(run_cell, battery, p, a, out, opts, deps): (p, a) for p, a in todo}
for fut in as_completed(futs):
diff --git a/harness/bench/report.py b/harness/bench/report.py
index bd0fb3f7..dccea21e 100644
--- a/harness/bench/report.py
+++ b/harness/bench/report.py
@@ -16,6 +16,7 @@ from typing import Any
from pydantic import BaseModel, Field
+from bench._jsonl import read_jsonl
from bench.run_bench import BenchItemResult
from codeverse.flywheel.gallery import GalleryItem, item_from_run, render_gallery
from codeverse.flywheel.record import RecordError, load_record
@@ -51,11 +52,7 @@ class BenchReport(BaseModel):
def load_results(out_dir: Path) -> list[BenchItemResult]:
jl = out_dir / "results.jsonl"
if jl.is_file():
- latest: dict[str, BenchItemResult] = {}
- for line in jl.read_text().splitlines():
- if line.strip():
- r = BenchItemResult.model_validate_json(line)
- latest[r.id] = r
+ latest: dict[str, BenchItemResult] = {r.id: r for r in read_jsonl(jl, BenchItemResult)}
return list(latest.values())
js = out_dir / "results.json"
if js.is_file():
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.py
index 2734e018..8bbe763d 100644
--- a/harness/bench/run_bench.py
+++ b/harness/bench/run_bench.py
@@ -32,6 +32,7 @@ from typing import Any
import yaml
from pydantic import BaseModel, Field
+from bench._jsonl import read_jsonl, seal_for_append
from codeverse._compat import UTC
from codeverse.config import get_settings
from codeverse.contracts.common import Backends, Budget, Language, Track
@@ -136,13 +137,10 @@ def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Wo
def _load_done(results_jsonl: Path) -> dict[str, BenchItemResult]:
- done: dict[str, BenchItemResult] = {}
- if results_jsonl.is_file():
- for line in results_jsonl.read_text().splitlines():
- if line.strip():
- r = BenchItemResult.model_validate_json(line)
- done[r.id] = r
- return done
+ """Rows already paid for, latest wins. Tolerant by design: this file is the
+ resume source and it is appended to a line at a time, so the run that a SIGKILL
+ ended is precisely the one whose last line is half-written."""
+ return {r.id: r for r in read_jsonl(results_jsonl, BenchItemResult)}
def select_prompts(
@@ -213,6 +211,7 @@ def run_battery(
generator=spec.backends.generator, judge=spec.backends.judge)
return result_from_record(item, rec, (time.time() - t0) / 60, ws)
+ seal_for_append(results_jsonl) # a kill left the last row unterminated; do not glue onto it
with ThreadPoolExecutor(max_workers=max(1, opts.parallel)) as pool, results_jsonl.open("a") as fh:
futs = {pool.submit(_one, item): item for item in todo}
for fut in as_completed(futs):
diff --git a/harness/tests/flywheel_cli/test_bench.py b/harness/tests/flywheel_cli/test_bench.py
index 405a4946..d9080014 100644
--- a/harness/tests/flywheel_cli/test_bench.py
+++ b/harness/tests/flywheel_cli/test_bench.py
@@ -118,3 +118,32 @@ def test_every_bench_prompt_opens_its_own_run_ledger(tmp_path: Path):
rows = load_ledger(Path(r.workspace))
assert len(rows) == 1, f"{r.id}: {rows}"
assert rows[0].run == r.id and rows[0].cost_usd > 0 # and not a sibling's row
+
+
+def test_a_truncated_last_line_does_not_cost_the_whole_resume(tmp_path: Path):
+ """RS-3: results.jsonl is the resume source AND it is appended a line at a time,
+ so the run a SIGKILL ended is exactly the one whose last line is half-written.
+ Strict per-line validation made that file unusable: the battery could not be
+ resumed at all and its already-paid rows could not even be reported, so the
+ operator had to hand-edit the file or re-buy the battery."""
+ battery = REPO / "bench" / "prompts" / "static_objects_v1.yaml"
+ out = tmp_path / "bench_out"
+ run_battery(battery, out, BenchOptions(parallel=1, limit=2), run_fn=_fake_run_fn({}))
+ jl = out / "results.jsonl"
+ good = [ln for ln in jl.read_text().splitlines() if ln.strip()]
+ assert len(good) == 2
+ # SIGKILL mid-append: the last row is truncated inside a JSON string
+ jl.write_text("\n".join(good) [: -30])
+
+ calls: list[str] = []
+
+ def counting(spec, ws, resume):
+ calls.append(spec.id.split("/")[-1])
+ return _fake_run_fn({})(spec, ws, resume)
+
+ res = run_battery(battery, out, BenchOptions(parallel=1, limit=2), run_fn=counting)
+ # the intact first row is still credited: only the truncated prompt is re-run
+ assert calls == [good and json.loads(good[1])["id"]]
+ assert len(res) == 2
+ # and the already-paid rows remain reportable
+ assert {r.id for r in load_results(out)} == {json.loads(ln)["id"] for ln in good}