Skip to content

Official Codebase of 3DCodeVerse

Python

eval: every resume cache is published in one rename (codeverse3d.proc.write_text_atomic)

gaoypeng386dfb6f8883
diff --git a/eval/bench/_bare_agent.py b/eval/bench/_bare_agent.py
index fcc61d0a..4c989f77 100644
--- a/eval/bench/_bare_agent.py
+++ b/eval/bench/_bare_agent.py
@@ -23,6 +23,7 @@ from codeverse3d.config import get_settings
from codeverse3d.contracts.agent import AgentJob, AgentResult
from codeverse3d.contracts.common import Language
from codeverse3d.contracts.spec import Spec
+from codeverse3d.proc import write_text_atomic
from codeverse3d.spatial.node import node_modules_dir
from codeverse3d.workspace import Workspace
@@ -85,7 +86,7 @@ def run_bare_agent(spec: Spec, target: str, cell: Path, eval_ws: Workspace, *, m
env={"NODE_PATH": str(node_modules_dir())})
result = get_coding_agent(target).run(job)
if not result.transient: # a session a 503 storm killed is not finished: a --redo must run it again
- done.write_text(result.model_dump_json(indent=1))
+ write_text_atomic(done, result.model_dump_json(indent=1))
for rel in files_for(spec.language):
src = ws.root / rel
if src.is_file():
diff --git a/eval/bench/ab_plan.py b/eval/bench/ab_plan.py
index 654219ed..a895bf83 100644
--- a/eval/bench/ab_plan.py
+++ b/eval/bench/ab_plan.py
@@ -123,7 +123,7 @@ from bench.run_bench import ( # noqa: E402
select_prompts,
)
from codeverse3d.contracts.common import Backends # noqa: E402
-from codeverse3d.proc import exclusive # noqa: E402
+from codeverse3d.proc import exclusive, write_text_atomic # noqa: E402
from codeverse3d.tracks.plan_features import pin_plan_blockers # noqa: E402
from codeverse3d.workspace import Workspace # noqa: E402
@@ -352,7 +352,7 @@ def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOpt
battery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps
)
res.arm, res.target = arm, opts.generator # the row is keyed by A/B arm, not by generator
- (Path(res.workspace) / "cell.json").write_text(res.model_dump_json(indent=1))
+ write_text_atomic(Path(res.workspace) / "cell.json", res.model_dump_json(indent=1)) # the parent's marker
print(
f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}"
)
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.py
index 22111245..0ebf58c2 100644
--- a/eval/bench/compare_backends.py
+++ b/eval/bench/compare_backends.py
@@ -86,7 +86,7 @@ from codeverse3d.contracts.run import RunRecord, RunStatus # noqa: E402
from codeverse3d.contracts.spec import Spec # noqa: E402
from codeverse3d.cost import run_ledger # noqa: E402
from codeverse3d.cost.tally import tally # noqa: E402
-from codeverse3d.proc import exclusive # noqa: E402
+from codeverse3d.proc import exclusive, write_text_atomic # noqa: E402
from codeverse3d.tracks.generation import MultiFileParseError # noqa: E402
from codeverse3d.tracks.planner import PlanningError # noqa: E402
from codeverse3d.workspace import Workspace # noqa: E402
@@ -216,7 +216,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts
label=f"oneshot_{spec.id.split('/')[-1]}")
if gen.ok: # failures (timeouts, 5xx) are not cached so a redo regenerates
gen_dir.mkdir(parents=True, exist_ok=True)
- cached.write_text(gen.model_dump_json(indent=1))
+ write_text_atomic(cached, gen.model_dump_json(indent=1))
res.attempts = attempt + 1
res.gen_cost_usd += gen.usage.cost_usd
res.tool_calls += gen.tool_calls
@@ -520,7 +520,7 @@ def _drop_pairs(path: Path, key: tuple[str, str]) -> None:
rows = [p for p in latest(read_jsonl(path, PairRow)).values()
if not (p.prompt_id == key[0] and key[1] in (p.arm_a, p.arm_b))]
if path.is_file():
- path.write_text("".join(p.model_dump_json() + "\n" for p in rows))
+ write_text_atomic(path, "".join(p.model_dump_json() + "\n" for p in rows))
def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm], opts: CompareOptions,
diff --git a/eval/bench/h2h_glb.py b/eval/bench/h2h_glb.py
index 4e9e5451..c368de8e 100644
--- a/eval/bench/h2h_glb.py
+++ b/eval/bench/h2h_glb.py
@@ -49,6 +49,7 @@ from bench._fixed_eval import RUBRIC # noqa: E402
from bench.run_bench import Battery, BenchPrompt # noqa: E402
from bench.stats import sign_test # noqa: E402
from codeverse3d.addons import select # noqa: E402
+from codeverse3d.proc import write_text_atomic # noqa: E402
from codeverse3d.record.record import load_record # noqa: E402
from codeverse3d.workspace import Workspace # noqa: E402
@@ -185,7 +186,7 @@ def cached_eval(side: Side, spec: Any, out_dir: Path, judge: Any, *, force: bool
return prev.model_copy(update={k: getattr(side, k) for k in ("rounds", "cost_usd", "minutes", "status")})
out_dir.mkdir(parents=True, exist_ok=True)
side = evaluate(side, spec, out_dir, judge)
- cache.write_text(side.model_dump_json(indent=2))
+ write_text_atomic(cache, side.model_dump_json(indent=2))
return side
diff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.py
index 4f9b12f7..45b9f763 100644
--- a/eval/bench/h2h_scene.py
+++ b/eval/bench/h2h_scene.py
@@ -45,6 +45,7 @@ from codeverse3d.addons import select # noqa: E402
from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402
from codeverse3d.contracts.common import Language, Track # noqa: E402
from codeverse3d.contracts.spec import Spec # noqa: E402
+from codeverse3d.proc import write_text_atomic # noqa: E402
from codeverse3d.record.record import load_record # noqa: E402
HERE = Path(__file__).resolve().parent
@@ -169,7 +170,7 @@ def judge_side(judge: Any, spec: Spec, views: list[RenderView], cache: Path) ->
inp = JudgeInput(spec=spec, renders=render_set(views), gates=[], acceptance=[], round_index=0, extra_context=EXTRA_CONTEXT)
j = judge.judge(inp)
cache.parent.mkdir(parents=True, exist_ok=True)
- cache.write_text(j.model_dump_json(indent=1))
+ write_text_atomic(cache, j.model_dump_json(indent=1))
return json.loads(cache.read_text())
diff --git a/eval/bench/pin_plan.py b/eval/bench/pin_plan.py
index 5d528b10..7ab31916 100644
--- a/eval/bench/pin_plan.py
+++ b/eval/bench/pin_plan.py
@@ -23,6 +23,8 @@ import shutil
from pathlib import Path
from typing import Any
+from codeverse3d.proc import write_text_atomic
+
__all__ = ["PinError", "plan_artifacts", "plan_once", "seed_plan"]
#: what a planned run leaves behind that a later run resumes from
@@ -73,7 +75,7 @@ def seed_plan(src_run: Path, dst_run: Path) -> str:
stages = dict(target.get("stages") or {})
stages[STAGE_NAME] = {**entry, "result_path": str(dst_stage.resolve())}
target["stages"] = stages
- dst_state.write_text(json.dumps(target, indent=1))
+ write_text_atomic(dst_state, json.dumps(target, indent=1)) # a torn run_state.json is an unresumable run
return str(entry["inputs_hash"])
diff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.py
index fb02ee4e..90b8ca0a 100644
--- a/eval/bench/skill_targets.py
+++ b/eval/bench/skill_targets.py
@@ -44,6 +44,7 @@ from codeverse3d.addons.skill_targets import ( # noqa: E402
TARGETS,
Target,
)
+from codeverse3d.proc import write_text_atomic # noqa: E402
from codeverse3d.skills.registry import finding_kind # noqa: E402
CACHE_NAME = ".skill_targets_cache.json"
@@ -405,7 +406,7 @@ def main(argv: list[str] | None = None) -> int:
if cache is not None:
with contextlib.suppress(OSError):
- cache_path.write_text(json.dumps(cache))
+ write_text_atomic(cache_path, json.dumps(cache))
if a.json:
print(json.dumps({"out": str(a.out), "mode": "ab" if arms else "battery",
diff --git a/eval/llm/_jsonl.py b/eval/llm/_jsonl.py
index aef106c6..afcba63b 100644
--- a/eval/llm/_jsonl.py
+++ b/eval/llm/_jsonl.py
@@ -1,15 +1,18 @@
-"""The tolerant JSONL reader every resumable stage starts from.
+"""The tolerant JSONL reader every resumable stage starts from, and the one writer.
-The stages rewrite their ``*.jsonl`` whole at the end; a process killed during that write
-leaves a torn last line, and a strict ``json.loads`` per line then crashed every later
-``--resume`` of the directory. A row that does not parse was not finished: the stage
-simply runs it again."""
+The stages rewrite their ``*.jsonl`` whole at the end. That rewrite truncated the file
+first, so a process killed during it lost every row after the tear (re-paid on the next
+``--resume``) and left a torn line that a strict ``json.loads`` per line then crashed on.
+``write_rows`` publishes the new file in one rename (``codeverse3d.proc.write_text_atomic``);
+``read_rows`` still skips a line that does not parse — that row simply runs again."""
from __future__ import annotations
import json
from pathlib import Path
+from codeverse3d.proc import write_text_atomic
+
def read_rows(path: Path) -> list[dict]:
"""Every JSON object line of ``path``, in order; blank and torn lines are skipped."""
@@ -23,3 +26,9 @@ def read_rows(path: Path) -> list[dict]:
if isinstance(row, dict):
rows.append(row)
return rows
+
+
+def write_rows(path: Path, rows: list[dict], *, ensure_ascii: bool = True) -> None:
+ """Rewrite ``path`` as one JSON line per row, atomically (the old file stays whole until
+ the new one is complete)."""
+ write_text_atomic(Path(path), "".join(json.dumps(r, ensure_ascii=ensure_ascii) + "\n" for r in rows))
diff --git a/eval/llm/execute.py b/eval/llm/execute.py
index 21ac5049..6888acca 100644
--- a/eval/llm/execute.py
+++ b/eval/llm/execute.py
@@ -1,13 +1,12 @@
"""Execute every generated program in a gen dir with the suite's executor → <gen_dir>/exec_results.jsonl."""
from __future__ import annotations
-import json
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from . import executors
-from ._jsonl import read_rows
+from ._jsonl import read_rows, write_rows
from .executors import CODE_EXT
# Blender and Chromium are heavy; CadQuery/OpenSCAD/glslang are light.
@@ -44,9 +43,7 @@ def execute_dir(gen_dir: Path, dialect: str, workers: int | None = None, timeout
if i % 25 == 0:
print(f"[exec] {gen_dir.name} {i}/{len(todo)}", flush=True)
res.sort(key=lambda r: r["id"])
- with out_path.open("w") as f:
- for r in res:
- f.write(json.dumps(r) + "\n")
+ write_rows(out_path, res)
c = Counter(r["status"] for r in res)
print(f"[exec] {gen_dir.name}: {dict(c)} | OK {c.get('OK', 0)}/{len(res)} = {100 * c.get('OK', 0) / max(1, len(res)):.1f}%", flush=True)
return res
diff --git a/eval/llm/generate.py b/eval/llm/generate.py
index df6e9409..a2d82beb 100644
--- a/eval/llm/generate.py
+++ b/eval/llm/generate.py
@@ -24,7 +24,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from . import config
-from ._jsonl import read_rows
+from ._jsonl import read_rows, write_rows
from .executors import CODE_EXT
from .extract import extract, summarize
from .suites import resolve
@@ -69,9 +69,7 @@ def write_answer(gen_dir: Path, row: dict, text: str, meta: dict) -> dict:
def finish(gen_dir: Path, recs: list[dict], info: dict) -> None:
recs.sort(key=lambda r: r["id"])
- with (gen_dir / "gens.jsonl").open("w") as f:
- for r in recs:
- f.write(json.dumps(r) + "\n")
+ write_rows(gen_dir / "gens.jsonl", recs) # run_eval skips a stage whose gens.jsonl exists
stats = summarize([r["extract"] for r in recs])
n = max(1, len(recs))
stats.update(info, n_rows=len(recs), truncated=sum(bool(r.get("truncated")) for r in recs),
diff --git a/eval/llm/judge.py b/eval/llm/judge.py
index 51069b6b..641ccae8 100644
--- a/eval/llm/judge.py
+++ b/eval/llm/judge.py
@@ -31,7 +31,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from . import config
-from ._jsonl import read_rows
+from ._jsonl import read_rows, write_rows
from .render_views import VIEWS
from .suites import load_prompts, resolve
@@ -159,9 +159,7 @@ def judge_absolute(run: str, suite: str, jc: JudgeClient, tag: str, workers: int
if i % 20 == 0:
print(f"[judge] {run}/{suite} {i}/{len(todo)}", flush=True)
recs.sort(key=lambda r: r["id"])
- with out_path.open("w") as f:
- for r in recs:
- f.write(json.dumps(r, ensure_ascii=False) + "\n")
+ write_rows(out_path, recs, ensure_ascii=False)
ok = [r for r in recs if r["status"] == "OK"]
n = len(rows)
diff --git a/eval/llm/render_views.py b/eval/llm/render_views.py
index 5545e3bb..cd05cfa9 100644
--- a/eval/llm/render_views.py
+++ b/eval/llm/render_views.py
@@ -21,6 +21,8 @@ from pathlib import Path
from . import config
from ._jsonl import read_rows
+from codeverse3d.proc import write_text_atomic
+
VIEWS = ["Image_005.png", "Image_015.png", "Image_025.png", "Image_035.png"]
BLENDER_SCRIPT = r'''
@@ -104,7 +106,9 @@ try:
except Exception as e:
rec["status"] = "ERR_RENDER"; rec["error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()[-800:]}"
rec["latency_s"] = round(time.time() - t0, 2)
-open(os.path.join(out_dir, "render_log.json"), "w").write(json.dumps(rec, indent=1))
+# tmp + rename (Blender's python cannot import codeverse3d.proc): the log is the cache marker
+open(os.path.join(out_dir, "render_log.json.tmp"), "w").write(json.dumps(rec, indent=1))
+os.replace(os.path.join(out_dir, "render_log.json.tmp"), os.path.join(out_dir, "render_log.json"))
'''
@@ -127,12 +131,12 @@ def render_glb_views(glb: str | Path, out_dir: str | Path, samples: int = 64, re
subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=config.tool_env(HOME="/tmp", XDG_CONFIG_HOME="/tmp/.bcfg"))
except subprocess.TimeoutExpired:
rec = {"status": "ERR_TIMEOUT", "error": f"render > {timeout}s", "n_views": 0, "latency_s": round(time.time() - t0, 1)}
- log.write_text(json.dumps(rec))
+ write_text_atomic(log, json.dumps(rec))
return rec
if log.exists():
return json.loads(log.read_text())
rec = {"status": "ERR_NOLOG", "error": "blender exited without a log", "n_views": 0, "latency_s": round(time.time() - t0, 1)}
- log.write_text(json.dumps(rec))
+ write_text_atomic(log, json.dumps(rec))
return rec
diff --git a/eval/tests/test_cache_writes.py b/eval/tests/test_cache_writes.py
new file mode 100644
index 00000000..2dca1d08
--- /dev/null
+++ b/eval/tests/test_cache_writes.py
@@ -0,0 +1,27 @@
+"""A resume cache survives a crash in its own rewrite (eval/llm stages, eval/bench caches)."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+
+def test_a_failed_rewrite_keeps_the_resume_cache_whole(tmp_path, monkeypatch):
+ """``execute_dir`` rewrites exec_results.jsonl whole. The rewrite used to truncate the
+ file first, so a crash part-way lost every cached row (re-paid on the next --resume) —
+ the torn-cache bug; now the old file stays until the new one is complete."""
+ from llm import execute
+
+ gen = tmp_path / "gen"
+ for task in ("a_new", "b_cached", "c_cached"):
+ (gen / task).mkdir(parents=True)
+ cached = [{"id": "b_cached", "status": "OK"}, {"id": "c_cached", "status": "OK"}]
+ out = gen / "exec_results.jsonl"
+ out.write_text("".join(json.dumps(r) + "\n" for r in cached))
+ # the new row sorts first and cannot be serialised: the rewrite dies on its first line
+ monkeypatch.setattr(execute, "_one", lambda dialect, p, timeout: {"id": p.name, "status": "OK", "x": object()})
+ with pytest.raises(TypeError):
+ execute.execute_dir(gen, "cadquery", workers=1)
+ assert [json.loads(line) for line in out.read_text().splitlines()] == cached
+ assert not list(gen.glob("*.tmp"))