Official Codebase of 3DCodeVerse
Python
atomic writes: one fixed temp filename raced, silently disabling the three.js asset gate
gaoypeng2111e096214f
diff --git a/harness/codeverse/flywheel/telemetry.py b/harness/codeverse/flywheel/telemetry.pyindex f9ddda49..69ec60f9 100644--- a/harness/codeverse/flywheel/telemetry.py+++ b/harness/codeverse/flywheel/telemetry.py@@ -38,7 +38,7 @@ from codeverse.contracts.run import (SettingsSnapshot,StageCost,)-from codeverse.proc import write_json_atomic+from codeverse.proc import write_json_atomic, write_text_atomicfrom codeverse.workspace import Workspacelog = logging.getLogger(__name__)@@ -310,11 +310,7 @@ def read_json(path: Path) -> dict[str, Any] | None:# --------------------------------------------------------------------------- iodef write_usage_jsonl(path: Path, rows: list[dict[str, Any]]) -> Path:- path.parent.mkdir(parents=True, exist_ok=True)- tmp = path.with_suffix(".jsonl.tmp")- tmp.write_text("".join(json.dumps(r, ensure_ascii=False, default=str) + "\n" for r in rows))- tmp.replace(path)- return path+ return write_text_atomic(path, "".join(json.dumps(r, ensure_ascii=False, default=str) + "\n" for r in rows))def _place_usage_rows(ws: Workspace, rows: list[dict[str, Any]], source: str) -> str:diff --git a/harness/codeverse/gallery/static_site.py b/harness/codeverse/gallery/static_site.pyindex 554ad3ce..4fe75691 100644--- a/harness/codeverse/gallery/static_site.py+++ b/harness/codeverse/gallery/static_site.py@@ -16,6 +16,7 @@ from codeverse.gallery.index import build_indexfrom codeverse.gallery.model import GalleryIndexfrom codeverse.gallery.page import render_indexfrom codeverse.gallery.urls import THUMB_PX, StaticUrls+from codeverse.proc import write_text_atomicdef render_static(index: GalleryIndex, *, title: str = "3dcv gallery", embed: bool = False,@@ -30,11 +31,7 @@ def render_static(index: GalleryIndex, *, title: str = "3dcv gallery", embed: bodef write_atomic(path: Path, text: str) -> Path:"""tmp + rename, so a reader never sees a partial gallery."""- path.parent.mkdir(parents=True, exist_ok=True)- tmp = path.with_name(path.name + ".tmp")- tmp.write_text(text, encoding="utf-8")- tmp.replace(path)- return path+ return write_text_atomic(path, text)def build_static(roots: list[Path] | list[str], out_html: Path | str, *, title: str | None = None,diff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.pyindex aa78fff1..6619237d 100644--- a/harness/codeverse/proc.py+++ b/harness/codeverse/proc.py@@ -92,26 +92,43 @@ def tail(text: str, *, max_lines: int = 40, max_chars: int = 4000) -> str:return s[-max_chars:] if len(s) > max_chars else s+def unique_tmp(path: Path) -> Path:+ """A temp sibling of ``path`` unique to this process AND thread.++ A FIXED ``<name>.tmp`` is a race: two writers of the same destination both+ create it and the second ``replace()`` finds its source already renamed away,+ raising FileNotFoundError. The harness fans work out over thread pools, so+ "one writer per destination" does not hold — see ``tracks/scene_asset_gen``,+ where the losing thread's asset gate was silently skipped.+ """+ return path.with_name(f"{path.name}.{os.getpid()}-{threading.get_ident()}.tmp")+++def write_text_atomic(path: Path, text: str, *, encoding: str = "utf-8") -> Path:+ """tmp + rename so readers never see a partial file, safe under concurrency."""+ path.parent.mkdir(parents=True, exist_ok=True)+ tmp = unique_tmp(path)+ try:+ tmp.write_text(text, encoding=encoding)+ tmp.replace(path)+ except BaseException:+ tmp.unlink(missing_ok=True) # a failed write must not leave litter behind+ raise+ return path++def write_json_atomic(path: Path, data: Any) -> None:"""tmp + rename so readers never see a partial file — including when severalwriters race for the same path.- The tmp name is private to this writer (pid + thread id, the convention- ``judges/images.py`` already uses for its shared cache). One shared- ``<path>.tmp`` was NOT atomic across writers: A could rename B's half-written- tmp into place — readers then observed truncated or zero-byte JSON at the- published path — and B's own ``replace()`` died with FileNotFoundError. That- is not hypothetical here: the scene track fans zone agents out over ONE- workspace root and every agent's MCP ``build`` tool writes+ Delegates to :func:`write_text_atomic`, whose tmp name is private to this writer+ (pid + thread id). One shared ``<path>.tmp`` was NOT atomic across writers: A could+ rename B's half-written tmp into place — readers then observed truncated or+ zero-byte JSON at the published path — and B's own ``replace()`` died with+ FileNotFoundError. That is not hypothetical here: the scene track fans zone agents+ out over ONE workspace root and every agent's MCP ``build`` tool writes``artifacts/build_last.json`` and ``artifacts/measurement.json`` through thisfunction, and a clobbered ``run_state.json`` is an unresumable run(``RunState.load`` refuses to guess). Last writer to rename still wins."""- path.parent.mkdir(parents=True, exist_ok=True)- tmp = path.with_name(f"{path.name}.{os.getpid()}-{threading.get_ident()}.tmp")- try:- tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False, default=str))- tmp.replace(path)- except BaseException:- tmp.unlink(missing_ok=True) # a failed write must not leave litter behind- raise+ write_text_atomic(path, json.dumps(data, indent=2, ensure_ascii=False, default=str))diff --git a/harness/codeverse/tracks/scene_asset_gen.py b/harness/codeverse/tracks/scene_asset_gen.pyindex a99c396f..4c111bbd 100644--- a/harness/codeverse/tracks/scene_asset_gen.py+++ b/harness/codeverse/tracks/scene_asset_gen.py@@ -30,7 +30,7 @@ from pydantic import BaseModel, Fieldfrom codeverse.contracts.plan import AssetPlanfrom codeverse.conventions import to_pascal, to_snake-from codeverse.proc import write_json_atomic+from codeverse.proc import write_json_atomic, write_text_atomicfrom codeverse.tracks.common import RunContextfrom codeverse.tracks.generation import SINGLE_SHOT_PREFIX, is_single_shotfrom codeverse.workspace import Workspace@@ -148,10 +148,10 @@ def _checker_path(ctx: RunContext) -> Path:"""The checker script lives in the harness cache, never in the workspace."""p = Path(ctx.settings.cache_dir) / "scene_asset_check.mjs"if not p.is_file() or p.read_text() != _CHECK_JS:- p.parent.mkdir(parents=True, exist_ok=True)- tmp = p.with_suffix(".mjs.tmp")- tmp.write_text(_CHECK_JS)- tmp.replace(p)+ # run_asset_stage fans the assets out over a thread pool, so every thread+ # used to write the SAME '<cache>/scene_asset_check.mjs.tmp' and the loser's+ # replace() raised FileNotFoundError -- swallowed below into ok=True, ran=False.+ write_text_atomic(p, _CHECK_JS)return pdiff --git a/harness/tests/core/test_proc.py b/harness/tests/core/test_proc.pyindex 44e0835b..52a38c9c 100644--- a/harness/tests/core/test_proc.py+++ b/harness/tests/core/test_proc.py@@ -6,6 +6,7 @@ import jsonimport osimport resourceimport sys+import threadingimport timefrom pathlib import Path@@ -130,3 +131,54 @@ def test_write_json_atomic_survives_concurrent_writers(tmp_path: Path):assert out.strip() == "0", f"a reader saw a partial file {out.strip()} time(s)"assert json.loads(target.read_text())["writer"] in ("A", "B", "C") # last writer wins, wholeassert not list(tmp_path.glob("*.tmp")), "no temp file left behind"+# --------------------------------------------------------------------------- atomic writes+def test_unique_tmp_is_per_process_and_per_thread(tmp_path: Path):+ """A FIXED '<name>.tmp' is what made concurrent writers of one destination race."""+ from codeverse.proc import unique_tmp++ out = tmp_path / "cache" / "checker.mjs"+ seen: list[Path] = []+ threads = [threading.Thread(target=lambda: seen.append(unique_tmp(out))) for _ in range(8)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()++ assert len({str(p) for p in seen}) == 8+ assert all(p.parent == out.parent and p.name.startswith("checker.mjs.") for p in seen)+ assert str(os.getpid()) in seen[0].name+++def test_concurrent_writers_of_one_destination_all_succeed(tmp_path: Path):+ """CQ-2: the second writer's replace() used to find its source already renamed+ away -> FileNotFoundError. Barrier-synchronised, this failed ~half the time."""+ from codeverse.proc import write_text_atomic++ out = tmp_path / "shared.txt"+ n = 8+ bar = threading.Barrier(n)+ errors: list[BaseException] = []++ def writer() -> None:+ try:+ bar.wait()+ write_text_atomic(out, "payload\n")+ except BaseException as e: # noqa: BLE001 — the point of the test+ errors.append(e)++ threads = [threading.Thread(target=writer) for _ in range(n)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()++ assert not errors, errors+ assert out.read_text() == "payload\n"+ assert list(tmp_path.iterdir()) == [out] # no temp litter left behind+++def test_write_json_atomic_round_trips(tmp_path: Path):+ out = tmp_path / "d" / "x.json"+ write_json_atomic(out, {"a": 1})+ assert json.loads(out.read_text()) == {"a": 1}+ assert [p.name for p in out.parent.iterdir()] == ["x.json"]diff --git a/harness/tests/orchestrator_tracks/test_scene_asset_checker_race.py b/harness/tests/orchestrator_tracks/test_scene_asset_checker_race.pynew file mode 100644index 00000000..6e4196ee--- /dev/null+++ b/harness/tests/orchestrator_tracks/test_scene_asset_checker_race.py@@ -0,0 +1,49 @@+"""CQ-2: run_asset_stage fans assets out over a thread pool, so _checker_path runs+concurrently. Every thread wrote the same '<cache>/scene_asset_check.mjs.tmp' and the+loser's replace() raised FileNotFoundError — swallowed by the broad `except Exception`+in check_threejs_asset into AssetCheck(ok=True, ran=False). The losing asset was then+reported as PASSING while the min_y sink, the ASSET_MAX_TRIS budget and the+single-low-poly-box detector were never computed, and size_m stayed None.++Offline: _checker_path only writes the script, it never launches node.+"""++from __future__ import annotations++import threading+from concurrent.futures import ThreadPoolExecutor+from pathlib import Path+from types import SimpleNamespace++from codeverse.tracks.scene_asset_gen import _CHECK_JS, _checker_path+++def _ctx(cache: Path) -> SimpleNamespace:+ """_checker_path reads exactly one thing off the context."""+ return SimpleNamespace(settings=SimpleNamespace(cache_dir=str(cache)))+++def test_concurrent_asset_checks_all_get_the_script(tmp_path: Path) -> None:+ ctx = _ctx(tmp_path / "cache")+ n = 8+ bar = threading.Barrier(n)++ def one() -> Path:+ bar.wait() # every thread hits the write at once, as the asset fan-out does+ return _checker_path(ctx)++ with ThreadPoolExecutor(n) as ex:+ paths = [f.result() for f in [ex.submit(one) for _ in range(n)]]++ assert len({str(p) for p in paths}) == 1+ assert paths[0].read_text() == _CHECK_JS+ assert [p.name for p in paths[0].parent.iterdir()] == ["scene_asset_check.mjs"]+++def test_the_checker_is_rewritten_when_it_is_stale_and_reused_when_it_is_not(tmp_path: Path) -> None:+ ctx = _ctx(tmp_path / "cache")+ p = _checker_path(ctx)+ p.write_text("// stale\n")+ assert _checker_path(ctx).read_text() == _CHECK_JS+ mtime = p.stat().st_mtime_ns+ assert _checker_path(ctx).stat().st_mtime_ns == mtime # unchanged → not rewritten