Official Codebase of 3DCodeVerse
Python
eval: every resume cache is published in one rename (codeverse3d.proc.write_text_atomic)
gaoypeng386dfb6f8883
diff --git a/eval/bench/_bare_agent.py b/eval/bench/_bare_agent.pyindex fcc61d0a..4c989f77 100644--- a/eval/bench/_bare_agent.py+++ b/eval/bench/_bare_agent.py@@ -23,6 +23,7 @@ from codeverse3d.config import get_settingsfrom codeverse3d.contracts.agent import AgentJob, AgentResultfrom codeverse3d.contracts.common import Languagefrom codeverse3d.contracts.spec import Spec+from codeverse3d.proc import write_text_atomicfrom codeverse3d.spatial.node import node_modules_dirfrom codeverse3d.workspace import Workspace@@ -85,7 +86,7 @@ def run_bare_agent(spec: Spec, target: str, cell: Path, eval_ws: Workspace, *, menv={"NODE_PATH": str(node_modules_dir())})result = get_coding_agent(target).run(job)if not result.transient: # a session a 503 storm killed is not finished: a --redo must run it again- done.write_text(result.model_dump_json(indent=1))+ write_text_atomic(done, result.model_dump_json(indent=1))for rel in files_for(spec.language):src = ws.root / relif src.is_file():diff --git a/eval/bench/ab_plan.py b/eval/bench/ab_plan.pyindex 654219ed..a895bf83 100644--- a/eval/bench/ab_plan.py+++ b/eval/bench/ab_plan.py@@ -123,7 +123,7 @@ from bench.run_bench import ( # noqa: E402select_prompts,)from codeverse3d.contracts.common import Backends # noqa: E402-from codeverse3d.proc import exclusive # noqa: E402+from codeverse3d.proc import exclusive, write_text_atomic # noqa: E402from codeverse3d.tracks.plan_features import pin_plan_blockers # noqa: E402from codeverse3d.workspace import Workspace # noqa: E402@@ -352,7 +352,7 @@ def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptbattery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps)res.arm, res.target = arm, opts.generator # the row is keyed by A/B arm, not by generator- (Path(res.workspace) / "cell.json").write_text(res.model_dump_json(indent=1))+ write_text_atomic(Path(res.workspace) / "cell.json", res.model_dump_json(indent=1)) # the parent's markerprint(f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}")diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex 22111245..0ebf58c2 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -86,7 +86,7 @@ from codeverse3d.contracts.run import RunRecord, RunStatus # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402from codeverse3d.cost import run_ledger # noqa: E402from codeverse3d.cost.tally import tally # noqa: E402-from codeverse3d.proc import exclusive # noqa: E402+from codeverse3d.proc import exclusive, write_text_atomic # noqa: E402from codeverse3d.tracks.generation import MultiFileParseError # noqa: E402from codeverse3d.tracks.planner import PlanningError # noqa: E402from codeverse3d.workspace import Workspace # noqa: E402@@ -216,7 +216,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, optslabel=f"oneshot_{spec.id.split('/')[-1]}")if gen.ok: # failures (timeouts, 5xx) are not cached so a redo regeneratesgen_dir.mkdir(parents=True, exist_ok=True)- cached.write_text(gen.model_dump_json(indent=1))+ write_text_atomic(cached, gen.model_dump_json(indent=1))res.attempts = attempt + 1res.gen_cost_usd += gen.usage.cost_usdres.tool_calls += gen.tool_calls@@ -520,7 +520,7 @@ def _drop_pairs(path: Path, key: tuple[str, str]) -> None:rows = [p for p in latest(read_jsonl(path, PairRow)).values()if not (p.prompt_id == key[0] and key[1] in (p.arm_a, p.arm_b))]if path.is_file():- path.write_text("".join(p.model_dump_json() + "\n" for p in rows))+ write_text_atomic(path, "".join(p.model_dump_json() + "\n" for p in rows))def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm], opts: CompareOptions,diff --git a/eval/bench/h2h_glb.py b/eval/bench/h2h_glb.pyindex 4e9e5451..c368de8e 100644--- a/eval/bench/h2h_glb.py+++ b/eval/bench/h2h_glb.py@@ -49,6 +49,7 @@ from bench._fixed_eval import RUBRIC # noqa: E402from bench.run_bench import Battery, BenchPrompt # noqa: E402from bench.stats import sign_test # noqa: E402from codeverse3d.addons import select # noqa: E402+from codeverse3d.proc import write_text_atomic # noqa: E402from codeverse3d.record.record import load_record # noqa: E402from codeverse3d.workspace import Workspace # noqa: E402@@ -185,7 +186,7 @@ def cached_eval(side: Side, spec: Any, out_dir: Path, judge: Any, *, force: boolreturn prev.model_copy(update={k: getattr(side, k) for k in ("rounds", "cost_usd", "minutes", "status")})out_dir.mkdir(parents=True, exist_ok=True)side = evaluate(side, spec, out_dir, judge)- cache.write_text(side.model_dump_json(indent=2))+ write_text_atomic(cache, side.model_dump_json(indent=2))return sidediff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.pyindex 4f9b12f7..45b9f763 100644--- a/eval/bench/h2h_scene.py+++ b/eval/bench/h2h_scene.py@@ -45,6 +45,7 @@ from codeverse3d.addons import select # noqa: E402from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402from codeverse3d.contracts.common import Language, Track # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402+from codeverse3d.proc import write_text_atomic # noqa: E402from codeverse3d.record.record import load_record # noqa: E402HERE = Path(__file__).resolve().parent@@ -169,7 +170,7 @@ def judge_side(judge: Any, spec: Spec, views: list[RenderView], cache: Path) ->inp = JudgeInput(spec=spec, renders=render_set(views), gates=[], acceptance=[], round_index=0, extra_context=EXTRA_CONTEXT)j = judge.judge(inp)cache.parent.mkdir(parents=True, exist_ok=True)- cache.write_text(j.model_dump_json(indent=1))+ write_text_atomic(cache, j.model_dump_json(indent=1))return json.loads(cache.read_text())diff --git a/eval/bench/pin_plan.py b/eval/bench/pin_plan.pyindex 5d528b10..7ab31916 100644--- a/eval/bench/pin_plan.py+++ b/eval/bench/pin_plan.py@@ -23,6 +23,8 @@ import shutilfrom pathlib import Pathfrom typing import Any+from codeverse3d.proc import write_text_atomic+__all__ = ["PinError", "plan_artifacts", "plan_once", "seed_plan"]#: what a planned run leaves behind that a later run resumes from@@ -73,7 +75,7 @@ def seed_plan(src_run: Path, dst_run: Path) -> str:stages = dict(target.get("stages") or {})stages[STAGE_NAME] = {**entry, "result_path": str(dst_stage.resolve())}target["stages"] = stages- dst_state.write_text(json.dumps(target, indent=1))+ write_text_atomic(dst_state, json.dumps(target, indent=1)) # a torn run_state.json is an unresumable runreturn str(entry["inputs_hash"])diff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.pyindex fb02ee4e..90b8ca0a 100644--- a/eval/bench/skill_targets.py+++ b/eval/bench/skill_targets.py@@ -44,6 +44,7 @@ from codeverse3d.addons.skill_targets import ( # noqa: E402TARGETS,Target,)+from codeverse3d.proc import write_text_atomic # noqa: E402from codeverse3d.skills.registry import finding_kind # noqa: E402CACHE_NAME = ".skill_targets_cache.json"@@ -405,7 +406,7 @@ def main(argv: list[str] | None = None) -> int:if cache is not None:with contextlib.suppress(OSError):- cache_path.write_text(json.dumps(cache))+ write_text_atomic(cache_path, json.dumps(cache))if a.json:print(json.dumps({"out": str(a.out), "mode": "ab" if arms else "battery",diff --git a/eval/llm/_jsonl.py b/eval/llm/_jsonl.pyindex aef106c6..afcba63b 100644--- a/eval/llm/_jsonl.py+++ b/eval/llm/_jsonl.py@@ -1,15 +1,18 @@-"""The tolerant JSONL reader every resumable stage starts from.+"""The tolerant JSONL reader every resumable stage starts from, and the one writer.-The stages rewrite their ``*.jsonl`` whole at the end; a process killed during that write-leaves a torn last line, and a strict ``json.loads`` per line then crashed every later-``--resume`` of the directory. A row that does not parse was not finished: the stage-simply runs it again."""+The stages rewrite their ``*.jsonl`` whole at the end. That rewrite truncated the file+first, so a process killed during it lost every row after the tear (re-paid on the next+``--resume``) and left a torn line that a strict ``json.loads`` per line then crashed on.+``write_rows`` publishes the new file in one rename (``codeverse3d.proc.write_text_atomic``);+``read_rows`` still skips a line that does not parse — that row simply runs again."""from __future__ import annotationsimport jsonfrom pathlib import Path+from codeverse3d.proc import write_text_atomic+def read_rows(path: Path) -> list[dict]:"""Every JSON object line of ``path``, in order; blank and torn lines are skipped."""@@ -23,3 +26,9 @@ def read_rows(path: Path) -> list[dict]:if isinstance(row, dict):rows.append(row)return rows+++def write_rows(path: Path, rows: list[dict], *, ensure_ascii: bool = True) -> None:+ """Rewrite ``path`` as one JSON line per row, atomically (the old file stays whole until+ the new one is complete)."""+ write_text_atomic(Path(path), "".join(json.dumps(r, ensure_ascii=ensure_ascii) + "\n" for r in rows))diff --git a/eval/llm/execute.py b/eval/llm/execute.pyindex 21ac5049..6888acca 100644--- a/eval/llm/execute.py+++ b/eval/llm/execute.py@@ -1,13 +1,12 @@"""Execute every generated program in a gen dir with the suite's executor → <gen_dir>/exec_results.jsonl."""from __future__ import annotations-import jsonfrom collections import Counterfrom concurrent.futures import ThreadPoolExecutor, as_completedfrom pathlib import Pathfrom . import executors-from ._jsonl import read_rows+from ._jsonl import read_rows, write_rowsfrom .executors import CODE_EXT# Blender and Chromium are heavy; CadQuery/OpenSCAD/glslang are light.@@ -44,9 +43,7 @@ def execute_dir(gen_dir: Path, dialect: str, workers: int | None = None, timeoutif i % 25 == 0:print(f"[exec] {gen_dir.name} {i}/{len(todo)}", flush=True)res.sort(key=lambda r: r["id"])- with out_path.open("w") as f:- for r in res:- f.write(json.dumps(r) + "\n")+ write_rows(out_path, res)c = Counter(r["status"] for r in res)print(f"[exec] {gen_dir.name}: {dict(c)} | OK {c.get('OK', 0)}/{len(res)} = {100 * c.get('OK', 0) / max(1, len(res)):.1f}%", flush=True)return resdiff --git a/eval/llm/generate.py b/eval/llm/generate.pyindex df6e9409..a2d82beb 100644--- a/eval/llm/generate.py+++ b/eval/llm/generate.py@@ -24,7 +24,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completedfrom pathlib import Pathfrom . import config-from ._jsonl import read_rows+from ._jsonl import read_rows, write_rowsfrom .executors import CODE_EXTfrom .extract import extract, summarizefrom .suites import resolve@@ -69,9 +69,7 @@ def write_answer(gen_dir: Path, row: dict, text: str, meta: dict) -> dict:def finish(gen_dir: Path, recs: list[dict], info: dict) -> None:recs.sort(key=lambda r: r["id"])- with (gen_dir / "gens.jsonl").open("w") as f:- for r in recs:- f.write(json.dumps(r) + "\n")+ write_rows(gen_dir / "gens.jsonl", recs) # run_eval skips a stage whose gens.jsonl existsstats = summarize([r["extract"] for r in recs])n = max(1, len(recs))stats.update(info, n_rows=len(recs), truncated=sum(bool(r.get("truncated")) for r in recs),diff --git a/eval/llm/judge.py b/eval/llm/judge.pyindex 51069b6b..641ccae8 100644--- a/eval/llm/judge.py+++ b/eval/llm/judge.py@@ -31,7 +31,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completedfrom pathlib import Pathfrom . import config-from ._jsonl import read_rows+from ._jsonl import read_rows, write_rowsfrom .render_views import VIEWSfrom .suites import load_prompts, resolve@@ -159,9 +159,7 @@ def judge_absolute(run: str, suite: str, jc: JudgeClient, tag: str, workers: intif i % 20 == 0:print(f"[judge] {run}/{suite} {i}/{len(todo)}", flush=True)recs.sort(key=lambda r: r["id"])- with out_path.open("w") as f:- for r in recs:- f.write(json.dumps(r, ensure_ascii=False) + "\n")+ write_rows(out_path, recs, ensure_ascii=False)ok = [r for r in recs if r["status"] == "OK"]n = len(rows)diff --git a/eval/llm/render_views.py b/eval/llm/render_views.pyindex 5545e3bb..cd05cfa9 100644--- a/eval/llm/render_views.py+++ b/eval/llm/render_views.py@@ -21,6 +21,8 @@ from pathlib import Pathfrom . import configfrom ._jsonl import read_rows+from codeverse3d.proc import write_text_atomic+VIEWS = ["Image_005.png", "Image_015.png", "Image_025.png", "Image_035.png"]BLENDER_SCRIPT = r'''@@ -104,7 +106,9 @@ try:except Exception as e:rec["status"] = "ERR_RENDER"; rec["error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()[-800:]}"rec["latency_s"] = round(time.time() - t0, 2)-open(os.path.join(out_dir, "render_log.json"), "w").write(json.dumps(rec, indent=1))+# tmp + rename (Blender's python cannot import codeverse3d.proc): the log is the cache marker+open(os.path.join(out_dir, "render_log.json.tmp"), "w").write(json.dumps(rec, indent=1))+os.replace(os.path.join(out_dir, "render_log.json.tmp"), os.path.join(out_dir, "render_log.json"))'''@@ -127,12 +131,12 @@ def render_glb_views(glb: str | Path, out_dir: str | Path, samples: int = 64, resubprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=config.tool_env(HOME="/tmp", XDG_CONFIG_HOME="/tmp/.bcfg"))except subprocess.TimeoutExpired:rec = {"status": "ERR_TIMEOUT", "error": f"render > {timeout}s", "n_views": 0, "latency_s": round(time.time() - t0, 1)}- log.write_text(json.dumps(rec))+ write_text_atomic(log, json.dumps(rec))return recif log.exists():return json.loads(log.read_text())rec = {"status": "ERR_NOLOG", "error": "blender exited without a log", "n_views": 0, "latency_s": round(time.time() - t0, 1)}- log.write_text(json.dumps(rec))+ write_text_atomic(log, json.dumps(rec))return recdiff --git a/eval/tests/test_cache_writes.py b/eval/tests/test_cache_writes.pynew file mode 100644index 00000000..2dca1d08--- /dev/null+++ b/eval/tests/test_cache_writes.py@@ -0,0 +1,27 @@+"""A resume cache survives a crash in its own rewrite (eval/llm stages, eval/bench caches)."""++from __future__ import annotations++import json++import pytest+++def test_a_failed_rewrite_keeps_the_resume_cache_whole(tmp_path, monkeypatch):+ """``execute_dir`` rewrites exec_results.jsonl whole. The rewrite used to truncate the+ file first, so a crash part-way lost every cached row (re-paid on the next --resume) —+ the torn-cache bug; now the old file stays until the new one is complete."""+ from llm import execute++ gen = tmp_path / "gen"+ for task in ("a_new", "b_cached", "c_cached"):+ (gen / task).mkdir(parents=True)+ cached = [{"id": "b_cached", "status": "OK"}, {"id": "c_cached", "status": "OK"}]+ out = gen / "exec_results.jsonl"+ out.write_text("".join(json.dumps(r) + "\n" for r in cached))+ # the new row sorts first and cannot be serialised: the rewrite dies on its first line+ monkeypatch.setattr(execute, "_one", lambda dialect, p, timeout: {"id": p.name, "status": "OK", "x": object()})+ with pytest.raises(TypeError):+ execute.execute_dir(gen, "cadquery", workers=1)+ assert [json.loads(line) for line in out.read_text().splitlines()] == cached+ assert not list(gen.glob("*.tmp"))