Official Codebase of 3DCodeVerse
Python
runlock: one authority, and it covers every mutating command (net -82)
gaoypeng825c478931cd
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.pyindex 5a342364..b3cf1b96 100644--- a/harness/bench/ab_plan.py+++ b/harness/bench/ab_plan.py@@ -108,6 +108,7 @@ from bench.compare_backends import ( # noqa: E402)from bench.pin_plan import PLAN_JSON, PinError, plan_once, seed_plan # noqa: E402from bench.run_bench import Battery, BenchPrompt, select_prompts # noqa: E402+from codeverse.runlock import exclusive # noqa: E402from codeverse.tracks.plan_features import pin_plan_blockers # noqa: E402from codeverse.workspace import Workspace # noqa: E402@@ -423,7 +424,8 @@ def pin_pair(spec = spec_for(battery, item, _harness_arm(opts.generator), opts.compare_options())src = Workspace(out / "plans" / item.id / "run")if not (src.root / PLAN_JSON).is_file():- plan_once(spec, src.root)+ with exclusive(src.root, what=f"ab pin {item.id}"): # one writer per run dir+ plan_once(spec, src.root)want = json.loads((src.root / "run_state.json").read_text())["stages"]["plan"]["inputs_hash"]for arm in arms:dst = Workspace(cell_dir(out, arm, item.id, opts.generator) / "run")diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.pyindex 2bc9c96c..87f06e0e 100644--- a/harness/bench/compare_backends.py+++ b/harness/bench/compare_backends.py@@ -85,6 +85,7 @@ from codeverse.contracts.common import ENTRY_FILE # noqa: E402from codeverse.contracts.run import RunRecord # noqa: E402from codeverse.contracts.spec import Spec # noqa: E402from codeverse.cost import run_ledger # noqa: E402+from codeverse.runlock import exclusive # noqa: E402from codeverse.tracks.generation import MultiFileParseError # noqa: E402from codeverse.tracks.planner import PlanningError # noqa: E402from codeverse.workspace import Workspace # noqa: E402@@ -236,10 +237,11 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, deps: CompareDeps,if not resume:run_ws.create()run_ws.write_json(run_ws.spec_path, spec)- # the harness arm is a real run: give it its own ledger, nested inside the cell's- # (run_ledger restores the outer one on the way out, so the fixed evaluation that- # follows keeps landing in the cell ledger)- with run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}"):+ # the harness arm is a real run: one writer per run dir (--parallel is threads of ONE+ # process), and its own ledger nested inside the cell's (run_ledger restores the outer+ # one on the way out, so the fixed evaluation that follows lands in the cell ledger)+ with (exclusive(run_ws.root, what=f"compare {spec.id}:{run_ws.root.parent.name}"),+ run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}")):rec = deps.run_track(spec, run_ws, resume)res.gen_cost_usd = rec.total_usage.cost_usdres.tool_calls = rec.total_usage.tool_callsdiff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.pyindex 451a0530..a9219363 100644--- a/harness/bench/run_bench.py+++ b/harness/bench/run_bench.py@@ -39,6 +39,7 @@ from codeverse.contracts.common import Backends, Budget, Language, Trackfrom codeverse.contracts.run import RunRecordfrom codeverse.contracts.spec import Constraints, ReferenceImage, Specfrom codeverse.cost import run_ledger+from codeverse.runlock import exclusivefrom codeverse.workspace import WorkspaceRESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_final", "passed", "rounds", "cost_usd",@@ -255,7 +256,9 @@ def run_battery(ws.write_json(ws.spec_path, spec)t0 = time.time()try:- with run_ledger(ws.root, run=item.id):+ # one writer per run dir: --parallel runs these in threads of ONE process, so+ # the run mutex is what keeps two cells off the same workspace+ with exclusive(ws.root, what=f"bench {item.id}"), run_ledger(ws.root, run=item.id):rec = run_fn(spec, ws, resume)except Exception as e: # one failing prompt must not kill the battery# A provider outage is not a result: an hour spent retrying a 503 is not modeldiff --git a/harness/codeverse/cli/_common.py b/harness/codeverse/cli/_common.pyindex ee8dce00..905a7cab 100644--- a/harness/codeverse/cli/_common.py+++ b/harness/codeverse/cli/_common.py@@ -7,6 +7,8 @@ import hashlibimport importlibimport shutilimport sys+from collections.abc import Iterator+from contextlib import ExitStack, contextmanagerfrom dataclasses import dataclassfrom pathlib import Pathfrom typing import Any@@ -87,19 +89,29 @@ def make_slug(prompt: str, track: str, language: str, explicit: str | None = Nonreturn f"{slugify(prompt, 40)}_{h}"-def create_workspace(root: Path, *, force: bool) -> Workspace:- from codeverse.runlock import RunLocked, assert_free+@contextmanager+def mutating(target: Any, *, what: str, action: str = "enter") -> Iterator[None]:+ """The run mutex (``runlock.exclusive``) around anything that writes into a run+ directory, reported as a typed exit. ``target`` is a Workspace or a run root."""+ from codeverse.runlock import RunLocked, exclusive++ root = target if isinstance(target, (str, Path)) else target.root # NB: Path.root is "/"+ stack = ExitStack()+ try:+ stack.enter_context(exclusive(root, what=what, action=action))+ except RunLocked as e:+ raise CliError(str(e), code=2) from None+ with stack:+ yield++def create_workspace(root: Path, *, force: bool) -> Workspace:+ """Create (or ``--force``-replace) a run directory. The caller holds the run mutex —+ the rmtree below would otherwise wipe a live holder's workspace."""ws = Workspace(root)if ws.root.exists() and any(ws.root.iterdir()):if not force:raise CliError(f"{ws.root} already exists; use --force to overwrite or --slug for a new name")- # --force overwrites a DEAD run, never a live one: the rmtree below would delete- # that process's lock file along with its workspace (codeverse/runlock.py)- try:- assert_free(ws.root)- except RunLocked as e:- raise CliError(str(e), code=2) from Noneshutil.rmtree(ws.root)ws.create()return wsdiff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.pyindex 12b4789e..efb5cef9 100644--- a/harness/codeverse/cli/flywheel_cmd.py+++ b/harness/codeverse/cli/flywheel_cmd.py@@ -2,6 +2,7 @@from __future__ import annotations+from contextlib import nullcontextfrom pathlib import Pathfrom typing import Annotated@@ -24,7 +25,7 @@ def export_cmd(tar_prefix: Annotated[str, typer.Option("--tar-prefix", help="repo-root-relative prefix for the tar column")] = "",include_unbuilt: Annotated[bool, typer.Option("--include-unbuilt", help="also export runs whose best round never built")] = False,captions_dir: Annotated[Path | None, typer.Option("--captions-dir", help="side-car captions written by `caption --out`")] = None,- drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave normalised duplicates (normalised-code fingerprint + prompt) out of the index, manifest and tars (recorded under the manifest's dropped)")] = False,+ drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave byte-identical duplicates (raw code sha256 + prompt) out of the index, manifest and tars (recorded under the manifest's dropped); normalised ones are only marked near_duplicate_of")] = False,) -> None:"""Export runs → sample folders + dataset_manifest.json + metadata.jsonl/.parquet(+ optional plain tars, packed FROM the manifest)."""@@ -87,9 +88,11 @@ def caption_cmd(out: Annotated[Path | None, typer.Option("--out", help="write side-car <out>/<slug>.json instead of touching the run (read-only runs)")] = None,) -> None:"""Caption run(s): {detailed, instruction, factory} via a chat model."""+ from codeverse.cost.instrument import run_ledgerfrom codeverse.flywheel.captions import CaptionError, caption_samplefrom codeverse.flywheel.export import load_captionsfrom codeverse.flywheel.record import iter_runs, load_record+ from codeverse.runlock import RunLocked, exclusiveif all_runs:targets = [(fr.ws, fr.record, fr.run_id.slug)@@ -102,9 +105,15 @@ def caption_cmd(if not force and load_captions(ws, rec, out, slug=slug).get("detailed"):skipped += 1continue+ # PER RUN: captioning rewrites record.json (--out writes a side-car instead and+ # touches nothing), and a locked run is skipped with a warning — one live run must+ # not abort a --all batch of hundreds+ lock = nullcontext() if out else exclusive(ws.root, what=f"3dcv flywheel caption {ws.root.name}")try:- caps = caption_sample(ws, rec, model, out_dir=out, slug=slug)- except CaptionError as e:+ # the priced captioner call joins the run's ledger (as `3dcv judge` does)+ with lock, run_ledger(ws.root, run=ws.root.name, create=False):+ caps = caption_sample(ws, rec, model, out_dir=out, slug=slug)+ except (CaptionError, RunLocked) as e:failed += 1warn(str(e))continuediff --git a/harness/codeverse/cli/inspect_cmd.py b/harness/codeverse/cli/inspect_cmd.pyindex 86a76406..89261ab5 100644--- a/harness/codeverse/cli/inspect_cmd.py+++ b/harness/codeverse/cli/inspect_cmd.py@@ -39,6 +39,7 @@ def status("""Show spec / run_state / record / recent events of a run."""from codeverse.events import EventLogfrom codeverse.flywheel.record import RecordError, load_record+ from codeverse.runlock import holder_ofws = C.open_workspace(slug, runs_dir)spec = C.load_spec(ws)@@ -50,6 +51,8 @@ def status("judge": spec.backends.judge,"prompt": spec.prompt,}+ if (held := holder_of(ws.root)) is not None: # kill THAT pid, never `pkill -f 3dcv`+ rows["RUNNING NOW"] = f"pid {held.get('pid', '?')} ({held.get('what') or '3dcv'})"if ws.state_path.is_file():try:state = json.loads(ws.state_path.read_text())@@ -150,24 +153,25 @@ def render(spec = C.load_spec(ws)idx = _render_round_or_refuse(ws, round_index)out_dir = out or ws.renders_dir(idx) / ("cli" if mode == "shaded" else f"cli_{mode}")- if spec.track is Track.GRAPHICS:- _render_graphics(ws, spec, out_dir)- return- # the size settings are honoured by the in-run renders (tracks/static_object.py,- # tracks/scene.py) and were silently dropped by the one command whose whole job is- # rendering, so a CLI render did not match the one the judge saw- r = get_settings().render- if spec.track is Track.SCENE:- render_scene = C.lazy("codeverse.spatial.render_scene", "render_scene")- rs = render_scene(- ws, out_dir, cameras=None, width=width or r.scene_width, height=height or r.scene_height- )- else:- glb = ws.artifacts / "object.glb"- if not glb.is_file():- raise C.CliError(f"no artifact to render: {glb} (run a build first)")- render_glb = C.lazy("codeverse.spatial.render", "render_glb")- rs = render_glb(glb, out_dir, mode=mode, width=width or r.width, height=height or r.height)+ # writes into the run (a graphics render even rebuilds it): one writer per run dir+ with C.mutating(ws, what=f"3dcv render {ws.root.name}", action="render"):+ if spec.track is Track.GRAPHICS:+ _render_graphics(ws, spec, out_dir)+ return+ # the size settings are honoured by the in-run renders (tracks/static_object.py,+ # tracks/scene.py) and were silently dropped by the one command whose whole job is+ # rendering, so a CLI render did not match the one the judge saw+ r = get_settings().render+ if spec.track is Track.SCENE:+ render_scene = C.lazy("codeverse.spatial.render_scene", "render_scene")+ rs = render_scene(ws, out_dir, cameras=None, width=width or r.scene_width,+ height=height or r.scene_height)+ else:+ glb = ws.artifacts / "object.glb"+ if not glb.is_file():+ raise C.CliError(f"no artifact to render: {glb} (run a build first)")+ render_glb = C.lazy("codeverse.spatial.render", "render_glb")+ rs = render_glb(glb, out_dir, mode=mode, width=width or r.width, height=height or r.height)console.print(kv_table("renders",@@ -214,19 +218,21 @@ def judge(n_images = J.count_prompt_images(inp, rubric_name)from codeverse.cost.instrument import run_ledger- try:- # a re-judge joins the run's ledger when it has one; otherwise the per-process log- # (a ledger holding only this verdict would be read as the whole run's cost)- with run_ledger(ws.root, run=ws.root.name, create=False):- verdict = judge_obj.judge(inp)- except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothing- raise C.CliError(f"judge failed: {e}") from e- except Exception as e:- ReferenceJudgeError = C.lazy("codeverse.judges.reference", "ReferenceJudgeError")- if isinstance(e, ReferenceJudgeError):+ # writes artifacts/judge/rNN_cli.json into the run: one writer per run dir+ with C.mutating(ws, what=f"3dcv judge {ws.root.name}", action="judge"):+ try:+ # a re-judge joins the run's ledger when it has one; otherwise the per-process+ # log (a ledger holding only this verdict would be read as the whole run's cost)+ with run_ledger(ws.root, run=ws.root.name, create=False):+ verdict = judge_obj.judge(inp)+ except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothingraise C.CliError(f"judge failed: {e}") from e- raise- ws.write_json(ws.judge_path(idx, "_cli"), verdict)+ except Exception as e:+ ReferenceJudgeError = C.lazy("codeverse.judges.reference", "ReferenceJudgeError")+ if isinstance(e, ReferenceJudgeError):+ raise C.CliError(f"judge failed: {e}") from e+ raise+ ws.write_json(ws.judge_path(idx, "_cli"), verdict)console.print(kv_table("judgment",diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.pyindex f494b80f..e1e1f868 100644--- a/harness/codeverse/cli/main.py+++ b/harness/codeverse/cli/main.py@@ -15,7 +15,6 @@ from __future__ import annotationsimport osimport sys-from contextlib import ExitStackfrom pathlib import Pathfrom typing import Annotated@@ -35,7 +34,6 @@ from codeverse.cli.tools_cmd import toolsfrom codeverse.config import get_settingsfrom codeverse.contracts.common import TRACK_LANGUAGES, Budget, Language, Trackfrom codeverse.contracts.spec import Constraints, ReferenceImage, RunOptions, Spec-from codeverse.runlock import RunLocked, exclusiveapp = typer.Typer(name="3dcv",@@ -255,62 +253,44 @@ def make(# THE run mutex, taken BEFORE the workspace is created (or --force wipes one) and# held through the paid reference call and the run itself: everything below mutates# this run directory, and two 3dcv on one slug corrupt each other (runlock.py).- stack = ExitStack()- try:- stack.enter_context(exclusive(C.runs_root(runs_dir) / run_slug,- what=f"3dcv make {run_slug}", action="create"))- except RunLocked as e:- raise C.CliError(str(e), code=2) from None- with stack:- _make_in_lock(spec, run_slug, runs_dir, force=force, no_run=no_run, reference=reference,- reference_views=reference_views, candidates=candidates, rounds=rounds,- max_usd=max_usd, texture=texture, dial=dial, settings=settings,- backends=backends, track=track, language=language)---def _make_in_lock(spec, run_slug: str, runs_dir, *, force: bool, no_run: bool, reference: bool,- reference_views: int, candidates, rounds, max_usd, texture, dial, settings,- backends, track, language) -> None:- """The body of ``make`` that mutates the run directory — always under the run mutex- (``runlock.exclusive``), which is why creating, wiping, writing the spec and the paid- reference call can no longer race a second ``3dcv make`` on the same slug."""- ws = C.create_workspace(C.runs_root(runs_dir) / run_slug, force=force)- ws.write_json(ws.spec_path, spec)- ws.commit("spec")- console.print(- kv_table(- "run",- {- "slug": run_slug,- "workspace": ws.root,- "track": track.value,- "language": language.value,- "generator": backends.generator,- "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",- "judge": f"{backends.judge} n={dial.judge_samples} "- f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",- "rounds": rounds,- "max_usd": max_usd,- "candidates": candidates,- "texture": texture,- },+ with C.mutating(C.runs_root(runs_dir) / run_slug, what=f"3dcv make {run_slug}", action="create"):+ ws = C.create_workspace(C.runs_root(runs_dir) / run_slug, force=force)+ ws.write_json(ws.spec_path, spec)+ ws.commit("spec")+ console.print(+ kv_table(+ "run",+ {+ "slug": run_slug,+ "workspace": ws.root,+ "track": track.value,+ "language": language.value,+ "generator": backends.generator,+ "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",+ "judge": f"{backends.judge} n={dial.judge_samples} "+ f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",+ "rounds": rounds,+ "max_usd": max_usd,+ "candidates": candidates,+ "texture": texture,+ },+ ))- )- if reference:- # --reference runs even under --no-run, because the grounded spec IS the artifact- # it produces (reference/run.py writes it back to spec.json) — but the combination- # is otherwise read as "filesystem only", so say out loud that this part spends- # money and can block on a degraded provider before anything is written.+ if reference:+ # --reference runs even under --no-run, because the grounded spec IS the artifact+ # it produces (reference/run.py writes it back to spec.json) — but the combination+ # is otherwise read as "filesystem only", so say out loud that this part spends+ # money and can block on a degraded provider before anything is written.+ if no_run:+ warn(+ f"--reference makes model calls now (1 planner call + {reference_views} image generation(s) "+ f"+ {reference_views} vision check(s), ~$0.15 for 2 views) — --no-run stops after that"+ )+ spec = _ground_in_reference(spec, ws, n_views=reference_views)if no_run:- warn(- f"--reference makes model calls now (1 planner call + {reference_views} image generation(s) "- f"+ {reference_views} vision check(s), ~$0.15 for 2 views) — --no-run stops after that"- )- spec = _ground_in_reference(spec, ws, n_views=reference_views)- if no_run:- ok(f"spec written: {ws.spec_path} (not run; `3dcv resume {run_slug}` to start)")- return- _run_track(spec, ws, resume=False, candidates=candidates)+ ok(f"spec written: {ws.spec_path} (not run; `3dcv resume {run_slug}` to start)")+ return+ _run_track(spec, ws, resume=False, candidates=candidates)def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:@@ -347,21 +327,15 @@ def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None, force: bool = False) -> None:from codeverse.cost.instrument import run_ledger- from codeverse.runlock import RunLocked, run_lockget_track = C.lazy("codeverse.tracks", "get_track")options: dict = {"n_candidates": candidates} if candidates else {}options.update(C.round_policy_options(spec))try:- # ONE writer per run directory (codeverse/runlock.py), and every model call and+ # already inside the run mutex (make / resume take it); every model call and# agent session of this run lands in telemetry/cost.jsonl- with (- run_lock(ws.root, what=f"3dcv {'resume' if resume else 'make'} {ws.root.name}"),- run_ledger(ws.root, run=ws.root.name),- ):+ with run_ledger(ws.root, run=ws.root.name):record = get_track(spec.track, **options).run(spec, ws, resume=resume, force=force)- except RunLocked as e:- raise C.CliError(str(e), code=2) from Noneexcept KeyboardInterrupt:raise C.CliError(f"interrupted; resume with `3dcv resume {ws.root.name}`", code=130@@ -456,44 +430,34 @@ def resume(reached a terminal state is refused unless ``--force``: re-entering it spendsmoney and overwrites its final state."""ws = C.open_workspace(slug, runs_dir)- try: # the run mutex, before the spec is rewritten (a budget raise is a mutation)- stack = ExitStack()- stack.enter_context(exclusive(ws.root, what=f"3dcv resume {ws.root.name}"))- except RunLocked as e:- raise C.CliError(str(e), code=2) from None- with stack:- _resume_in_lock(ws, runs_dir, force=force, max_usd=max_usd, max_minutes=max_minutes,- rounds=rounds, candidates=candidates)---def _resume_in_lock(ws, runs_dir, *, force: bool, max_usd, max_minutes, rounds, candidates) -> None:- """The body of ``resume`` that mutates the run — always under the run mutex."""- spec = C.load_spec(ws)- if (- spec.options.profile- ): # the dial the run was created with (judge samples, montage px, turn cap)- try:- get_settings().apply_profile(spec.options.profile, force=True)- except ValueError as e: # a spec.json naming a profile this build no longer has- raise C.CliError(f"{ws.spec_path}: {e}", code=2) from e- raised = {- k: v- for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items()- if v is not None- }- if not force and (why := _finished_reason(ws, raised)):- raise C.CliError(- f"run {ws.root.name} already finished ({why}); nothing to resume. "- f"`3dcv status {ws.root.name}` to look at it, or --force to re-enter it "- f"(that re-plans, re-scores and overwrites the final state)."- )- if raised:- from codeverse.events import EventLog+ # the run mutex, before the spec is rewritten (a budget raise is a mutation)+ with C.mutating(ws, what=f"3dcv resume {ws.root.name}"):+ spec = C.load_spec(ws)+ if (+ spec.options.profile+ ): # the dial the run was created with (judge samples, montage px, turn cap)+ try:+ get_settings().apply_profile(spec.options.profile, force=True)+ except ValueError as e: # a spec.json naming a profile this build no longer has+ raise C.CliError(f"{ws.spec_path}: {e}", code=2) from e+ raised = {+ k: v+ for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items()+ if v is not None+ }+ if not force and (why := _finished_reason(ws, raised)):+ raise C.CliError(+ f"run {ws.root.name} already finished ({why}); nothing to resume. "+ f"`3dcv status {ws.root.name}` to look at it, or --force to re-enter it "+ f"(that re-plans, re-scores and overwrites the final state)."+ )+ if raised:+ from codeverse.events import EventLog- spec = spec.model_copy(update={"budget": spec.budget.model_copy(update=raised)})- ws.write_json(ws.spec_path, spec)- EventLog(ws.events_path).emit("budget.raised", **raised)- _run_track(spec, ws, resume=True, candidates=candidates, force=force)+ spec = spec.model_copy(update={"budget": spec.budget.model_copy(update=raised)})+ ws.write_json(ws.spec_path, spec)+ EventLog(ws.events_path).emit("budget.raised", **raised)+ _run_track(spec, ws, resume=True, candidates=candidates, force=force)# --------------------------------------------------------------------------- status / render / judgediff --git a/harness/codeverse/cli/texture_cmd.py b/harness/codeverse/cli/texture_cmd.pyindex f3a85b87..a0f26276 100644--- a/harness/codeverse/cli/texture_cmd.py+++ b/harness/codeverse/cli/texture_cmd.py@@ -47,8 +47,10 @@ def pass_(texture_pass = C.lazy("codeverse.texturing.run", "texture_pass")from codeverse.cost.instrument import run_ledger- # a post-hoc pass joins the run's ledger when it has one, else the per-process log- with run_ledger(ws.root, run=ws.root.name, create=False):+ # a post-hoc pass joins the run's ledger when it has one, else the per-process log;+ # it rewrites the run's artifacts, so it holds the run mutex (one writer per run dir)+ with (C.mutating(ws, what=f"3dcv texture pass {ws.root.name}", action="texture"),+ run_ledger(ws.root, run=ws.root.name, create=False)):rep = texture_pass(ws, spec, plan, model_id=model or spec.backends.planner,image_model=_image_model(image_model), judge=judge,judge_model_id=judge_model, size=size)@@ -105,7 +107,8 @@ def scene_pack(scene_texture_pack = C.lazy("codeverse.texturing.scene_pack", "scene_texture_pack")texture_pack_prompt = C.lazy("codeverse.texturing.scene_pack", "texture_pack_prompt")model_id = spec.backends.planner if model is None else model- pack = scene_texture_pack(plan, out or ws.public / "textures", _image_model(image_model), model_id, size=size, n_max=n)+ with C.mutating(ws, what=f"3dcv texture scene-pack {ws.root.name}", action="texture"):+ pack = scene_texture_pack(plan, out or ws.public / "textures", _image_model(image_model), model_id, size=size, n_max=n)rows = {name: f"{e.file or 'FAILED'} tile={e.tile_size_m:.2f}m {e.material_family}/{e.role} seam={e.seam_score:.3f}"+ (f" {e.error}" if e.error else "") for name, e in pack.entries.items()}console.print(kv_table("scene texture pack", rows))diff --git a/harness/codeverse/flywheel/migrate.py b/harness/codeverse/flywheel/migrate.pyindex c072b8fa..c0aa5fbe 100644--- a/harness/codeverse/flywheel/migrate.py+++ b/harness/codeverse/flywheel/migrate.py@@ -17,11 +17,13 @@ history. Running it twice is a no-op (second pass reports ``up_to_date``).from __future__ import annotationsimport logging+from contextlib import ExitStackfrom pathlib import Pathfrom pydantic import BaseModel, Fieldfrom codeverse.flywheel.record import RecordError, load_record, package_run+from codeverse.runlock import RunLocked, exclusivefrom codeverse.workspace import Workspacelog = logging.getLogger(__name__)@@ -64,43 +66,53 @@ class MigrationReport(BaseModel):def migrate_run(ws: Workspace, *, dry_run: bool = False) -> RunMigration:- """Reorganise one run in place; idempotent, additive, never destructive."""+ """Reorganise one run in place; idempotent, additive, never destructive.++ Under the run mutex unless ``dry_run`` (it relinks the layout and rewrites+ ``record.json``); a run someone is using is reported SKIPPED, not migrated."""m = RunMigration(run=str(ws.root))if not ws.exists():m.status = SKIPPEDm.reason = "no spec.json"return m- changed = False- try:- m.layout = ws.ensure_layout(dry_run=dry_run)- m.gitignore = ws.ensure_gitignore(dry_run=dry_run)- except OSError as e:- m.status = FAILED- m.reason = f"{type(e).__name__}: {e}"- return m- changed = bool(m.layout) or m.gitignore+ stack = ExitStack()try:- record = load_record(ws)- except RecordError as e:- m.status = MIGRATED if changed else UP_TO_DATE- m.reason = str(e) # layout only: an unfinished run has no record yet+ if not dry_run:+ stack.enter_context(exclusive(ws.root, what=f"3dcv migrate-runs {ws.root.name}"))+ except RunLocked as e:+ m.status, m.reason = SKIPPED, str(e)return m- if dry_run:- m.telemetry_rows = _row_count(ws, record)- m.deliverable_files = None- m.status = MIGRATED if (changed or record.telemetry is None or record.deliverable is None) else UP_TO_DATE+ with stack:+ try:+ m.layout = ws.ensure_layout(dry_run=dry_run)+ m.gitignore = ws.ensure_gitignore(dry_run=dry_run)+ except OSError as e:+ m.status = FAILED+ m.reason = f"{type(e).__name__}: {e}"+ return m+ changed = bool(m.layout) or m.gitignore+ try:+ record = load_record(ws)+ except RecordError as e:+ m.status = MIGRATED if changed else UP_TO_DATE+ m.reason = str(e) # layout only: an unfinished run has no record yet+ return m+ if dry_run:+ m.telemetry_rows = _row_count(ws, record)+ m.deliverable_files = None+ m.status = MIGRATED if (changed or record.telemetry is None or record.deliverable is None) else UP_TO_DATE+ return m+ before = ws.record_path.read_bytes()+ package_run(ws, record)+ if record.deliverable is not None:+ m.deliverable_files = len(record.deliverable.files)+ m.deliverable_bytes = record.deliverable.total_bytes+ if record.telemetry is not None and record.telemetry.cost is not None:+ m.telemetry_rows = record.telemetry.cost.n_calls+ ws.write_json(ws.record_path, record)+ m.record_updated = ws.record_path.read_bytes() != before+ m.status = MIGRATED if (changed or m.record_updated) else UP_TO_DATEreturn m- before = ws.record_path.read_bytes()- package_run(ws, record)- if record.deliverable is not None:- m.deliverable_files = len(record.deliverable.files)- m.deliverable_bytes = record.deliverable.total_bytes- if record.telemetry is not None and record.telemetry.cost is not None:- m.telemetry_rows = record.telemetry.cost.n_calls- ws.write_json(ws.record_path, record)- m.record_updated = ws.record_path.read_bytes() != before- m.status = MIGRATED if (changed or m.record_updated) else UP_TO_DATE- return mdef _row_count(ws: Workspace, record: object) -> int | None:diff --git a/harness/codeverse/runlock.py b/harness/codeverse/runlock.pyindex daa69e0b..b7405132 100644--- a/harness/codeverse/runlock.py+++ b/harness/codeverse/runlock.py@@ -1,33 +1,15 @@"""One writer per run directory.-Nothing stopped two ``3dcv`` processes from entering the same run. Both write-``run_state.json``, both snapshot ``src/`` into the same git repo, both append to the-same event log and both spend the run's budget. ``workspace.py`` already saw the-symptom and retried around a contended ``.git/index.lock``; this is the cause.--Measured 2026-08-25: a lane launched ``3dcv resume tsr_scn_temple_night`` three times-after mistaking a surviving process for someone else's, and two of them ran concurrently-against the same workspace for four minutes.--TWO layers, because each does what the other cannot (2026-08-27):--* :func:`exclusive` — an ``fcntl.flock`` on ``<runs>/.locks/<slug>.lock``, i.e. OUTSIDE- the directory a ``--force`` run deletes. This is the actual mutex: the PID file below- is written after a check, so two processes could both pass it (and a forced wipe took- the lock file with it). The kernel releases a flock when the holder dies, so a crash- never leaves a run unenterable.-* the PID file (:func:`run_lock`) — the human-readable holder record, kept because the- useful message here is *which* process holds the run, so a human can look at it or kill- that one PID: the lesson from the same day's ``pkill -f "3dcv make"``, which killed- thirteen unrelated runs because the operator had no way to name just theirs. A lock- that says "held by pid 2992370, started 15:56:25" is worth more than one that only says- "busy". The same record is written into the flock file, so a refusal can name the- holder either way.--``exclusive`` is re-entrant WITHIN one process (flock is per open file description, so a-second ``open``+``flock`` from the same process would refuse itself): the CLI takes it at-the mutation boundary — before a workspace is created, wiped or its spec rewritten — and-the track's own :func:`run_lock` nests inside it.+Two ``3dcv`` on one run both write ``run_state.json``, both snapshot ``src/`` into the+same git repo and both spend the budget (measured 2026-08-25: two ``3dcv resume`` ran+four minutes side by side on tsr_scn_temple_night).++:func:`exclusive` is the ONE authority: an ``fcntl.flock`` on+``<runs>/.locks/<slug>.lock``, beside the run dirs because ``--force`` ``rmtree``s the+run root and a lock deleted mid-hold is no lock. The kernel drops it when the holder+dies, so a crash never wedges a run. Its ``{pid, started, what}`` record (also read by+:func:`holder_of`, without locking) is what a refusal prints, so a human can ``kill``+THAT pid instead of ``pkill -f 3dcv`` — which took out thirteen unrelated runs."""from __future__ import annotations@@ -41,71 +23,12 @@ from collections.abc import Iteratorfrom contextlib import contextmanagerfrom pathlib import Path-from codeverse.proc import read_json_or_none--LOCK_NAME = "run.lock"#: directory (beside the run dirs, never inside one) holding the flock filesLOCKS_DIR = ".locks"class RunLocked(RuntimeError):- """Another live process is already running this run."""---def _lock_path(run_root: Path) -> Path:- return Path(run_root) / ".3dcv" / LOCK_NAME---def _alive(pid: int) -> bool:- """Is this PID a live process we could signal?-- ``os.kill(pid, 0)`` raising ``ProcessLookupError`` means gone; ``PermissionError``- means alive but owned by someone else, which still counts as held.- """- try:- os.kill(pid, 0)- except ProcessLookupError:- return False- except PermissionError:- return True- except OSError:- return False- return True---def _holder(path: Path) -> dict | None:- """The live holder recorded in ``path``, or None when absent/stale/unreadable.-- An unreadable or truncated lock is treated as stale rather than fatal: a crashed- process must never leave a run permanently unenterable.- """- rec = read_json_or_none(path)- if rec is None:- return None- pid = rec.get("pid")- if not isinstance(pid, int) or not _alive(pid):- return None- return rec---def assert_free(run_root: Path | str, *, action: str = "overwrite") -> None:- """Raise :class:`RunLocked` if a live process holds this run.-- Call this BEFORE destroying or recreating a run directory. ``--force`` must mean- "overwrite a DEAD run", never "evict a running one": ``create_workspace(force=True)``- does ``shutil.rmtree`` on the run root, which deletes the lock file itself, so without- this check a forced run silently wipes a live holder's workspace out from under it and- then takes a fresh lock with no error. Measured 2026-08-25 on- tsr_scn_boat_workshop_v2: two independent ``3dcv make`` on one slug, 33 seconds apart.-- This narrows the window rather than closing it completely — two ``--force`` calls that- interleave between this check and the wipe can still both proceed. Closing that- properly needs a lock held outside the directory being deleted; the check here removes- the case that actually happens, which is a force against a run someone is already- using.- """- if (held := _holder(_lock_path(run_root))) is not None:- raise RunLocked(_held_message(run_root, held, action=action))+ """Another live holder — process or thread — is already running this run."""def _held_message(run_root: Path | str, held: dict, *, action: str) -> str:@@ -121,15 +44,12 @@ def _held_message(run_root: Path | str, held: dict, *, action: str) -> str:def flock_path(run_root: Path | str) -> Path:- """The flock file for this run: ``<runs>/.locks/<slug>.lock``.-- Beside the run directory, never inside it — ``create_workspace(force=True)``- ``rmtree``s the run root, and a lock that gets deleted mid-hold is no lock."""+ """The flock file for this run: ``<runs>/.locks/<slug>.lock``."""root = Path(run_root).resolve()return root.parent / LOCKS_DIR / f"{root.name}.lock"-_HELD: dict[str, list[int]] = {} # flock path -> [fd, recursion depth] held by THIS process+_HELD: dict[str, list] = {} # flock path -> [fd, owning thread ident] held by THIS process_HELD_GUARD = threading.Lock()@@ -150,29 +70,44 @@ def _write_record(fd: int, rec: dict) -> None:os.fsync(fd)+def holder_of(run_root: Path | str) -> dict | None:+ """The ``{pid, started, what}`` of whoever holds this run — without taking it.++ ``None`` when nobody does, including after a SIGKILL: the kernel dropped the flock but+ the record is still in the file, so a probe (shared, non-blocking, released at once)+ tells the difference and ``3dcv status`` never names a pid that is already gone."""+ try:+ fd = os.open(flock_path(run_root), os.O_RDONLY)+ except OSError:+ return None+ try:+ try:+ fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)+ except OSError:+ return _record_of(fd) # refused: somebody is holding it right now+ fcntl.flock(fd, fcntl.LOCK_UN)+ return None+ finally:+ os.close(fd)++@contextmanagerdef exclusive(run_root: Path | str, *, what: str = "", action: str = "enter") -> Iterator[None]:"""THE mutex for one run: hold it around every mutation of the run directory.- Raises :class:`RunLocked` naming the holder when another live process has it.- Re-entrant within this process (nested :func:`run_lock` is a no-op); released by- the kernel if this process dies, so a crash never wedges the run."""- path = flock_path(run_root)- key = str(path)+ Raises :class:`RunLocked`, naming the holder, when anyone else has it — another+ process, or another THREAD of this one (the bench drivers run runs in a pool). Only+ the SAME thread re-enters, so the CLI can take it at the mutation boundary and the+ code under it can take it again."""+ key, me = str(flock_path(run_root)), threading.get_ident()with _HELD_GUARD:- if (entry := _HELD.get(key)) is not None:- entry[1] += 1- nested = True- else:- nested = False- if nested:- try:- yield- finally:- with _HELD_GUARD:- if (entry := _HELD.get(key)) is not None:- entry[1] -= 1+ entry = _HELD.get(key)+ if entry is not None:+ if entry[1] != me:+ raise RunLocked(_held_message(run_root, holder_of(run_root) or {}, action=action))+ yield # the SAME thread re-entering: the outermost `with` owns the releasereturn+ path = Path(key)path.parent.mkdir(parents=True, exist_ok=True)fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o644)try:@@ -184,10 +119,10 @@ def exclusive(run_root: Path | str, *, what: str = "", action: str = "enter") ->except BaseException:os.close(fd)raise- _write_record(fd, {"pid": os.getpid(), "started": time.time(), "what": what})- with _HELD_GUARD:- _HELD[key] = [fd, 1]- try:+ try: # inside the try that owns the fd: a failed write must not leak an unreleasable lock+ _write_record(fd, {"pid": os.getpid(), "started": time.time(), "what": what})+ with _HELD_GUARD:+ _HELD[key] = [fd, me]yieldfinally:with _HELD_GUARD:@@ -199,29 +134,3 @@ def exclusive(run_root: Path | str, *, what: str = "", action: str = "enter") ->passfinally:os.close(fd)---@contextmanager-def run_lock(run_root: Path | str, *, what: str = "") -> Iterator[None]:- """Hold the run directory for this process, or raise :class:`RunLocked`.-- Takes over a lock whose PID is gone (a killed or crashed run must stay resumable).- Releasing is best-effort and never masks the body's own exception.- """- path = _lock_path(run_root)- path.parent.mkdir(parents=True, exist_ok=True)- if (held := _holder(path)) is not None:- raise RunLocked(_held_message(run_root, held, action="enter"))- with exclusive(run_root, what=what): # the real mutex; a no-op when the CLI holds it- rec = {"pid": os.getpid(), "started": time.time(), "what": what}- tmp = path.with_suffix(".lock.tmp")- tmp.write_text(json.dumps(rec))- os.replace(tmp, path) # the human-readable holder record `3dcv status` reads- try:- yield- finally:- try:- if (cur := _holder(path)) is not None and cur.get("pid") == os.getpid():- path.unlink(missing_ok=True)- except OSError:- passdiff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex 1b264e5e..c7a97354 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -80,8 +80,8 @@ codeverse/workspace.py run-dir layout + git snapshots ; events.py JSONL event logrunlock.py ONE writer per run dir: an fcntl.flock on <runs>/.locks/<slug>.lock(outside the dir a --force wipe deletes; released by the kernel if- the holder dies) taken at every CLI mutation boundary, plus the PID- file that NAMES the holder so a human can kill that one pid+ the holder dies) taken at every mutation boundary; its record NAMES+ the holder (`holder_of`, printed by `3dcv status`) so a human can kill that pidproc.py stdlib-only subprocess + atomic-JSON primitives (ManagedProcess owns everychild's lifecycle: group kill on ANY exception, bounded pumps, stdin writer;run_subprocess, kill_group, tail, write_json_atomic, scrub_secrets) and the tolerant readers/writer@@ -243,7 +243,7 @@ runs/<slug>/tool_renders/rNN_<hash>/trajectories/<label>_rNN/ prompt.md transcript.jsonl stdout.json stderr.log result.json(a retried label lands in <label>.a2_rNN — first attempt preserved)- AGENTS.md GEMINI.md CLAUDE.md .mcp.json .gemini/settings.json .3dcv/cookbook.md .geminiignore .aiexclude+ AGENTS.md GEMINI.md CLAUDE.md .gemini/settings.json .3dcv/cookbook.md .geminiignore .aiexclude```## 4. Per-language authoring contracts (raw code; the harness owns export)diff --git a/harness/tests/core/test_runlock.py b/harness/tests/core/test_runlock.pyindex 09c00863..bea59e24 100644--- a/harness/tests/core/test_runlock.py+++ b/harness/tests/core/test_runlock.py@@ -2,116 +2,24 @@from __future__ import annotations-import jsonimport os+import threadingimport timefrom pathlib import Pathimport pytestfrom codeverse.runlock import (- LOCK_NAME,LOCKS_DIR,RunLocked,- _lock_path,exclusive,flock_path,- run_lock,+ holder_of,)HARNESS_ROOT = Path(__file__).resolve().parents[2]-def test_a_second_process_cannot_enter_a_locked_run(tmp_path):- """Measured 2026-08-25: a lane launched `3dcv resume tsr_scn_temple_night` three- times and two ran concurrently on the same workspace for four minutes, both writing- run_state.json, both snapshotting src/ into the same git repo, both spending the- run's budget."""- with (- run_lock(tmp_path, what="3dcv make demo"),- pytest.raises(RunLocked) as ei,- run_lock(tmp_path), # raises on __enter__, so the body below never runs- ):- pytest.fail("the second entry must not be granted")- msg = str(ei.value)- assert str(os.getpid()) in msg, "the message must name the PID so ONE process can be killed"- assert "pkill" in msg, "and must warn against the pattern kill that took out 13 runs"---def test_the_lock_is_released_on_the_way_out(tmp_path):- with run_lock(tmp_path):- pass- with run_lock(tmp_path): # must not raise- pass- assert not _lock_path(tmp_path).exists()---def test_an_exception_still_releases_the_lock(tmp_path):- with pytest.raises(ZeroDivisionError), run_lock(tmp_path):- raise ZeroDivisionError- with run_lock(tmp_path):- pass---def test_a_dead_holder_is_taken_over_not_honoured(tmp_path):- """A killed or crashed run must stay resumable — the pkill incident left 13 runs- that all needed to be re-entered."""- p = _lock_path(tmp_path)- p.parent.mkdir(parents=True, exist_ok=True)- # PID 2^22 is above /proc/sys/kernel/pid_max on any normal box, so it cannot exist- p.write_text(json.dumps({"pid": 4194303, "started": 0, "what": "3dcv make ghost"}))- with run_lock(tmp_path):- assert json.loads(p.read_text())["pid"] == os.getpid()--[email protected]("junk", ["", "not json", '{"no pid": 1}', '{"pid": "abc"}'])-def test_an_unreadable_lock_never_bricks_a_run(tmp_path, junk):- """A truncated write from a crash must not make the run permanently unenterable."""- p = _lock_path(tmp_path)- p.parent.mkdir(parents=True, exist_ok=True)- p.write_text(junk)- with run_lock(tmp_path):- pass---# --------------------------------------------------------- --force must not evict a live run-def test_force_refuses_to_wipe_a_run_a_live_process_is_holding(tmp_path):- """`--force` means "overwrite a DEAD run", never "evict a running one".-- create_workspace(force=True) does shutil.rmtree on the run root, which deletes the- lock file itself — so before this check a forced run wiped a live holder's workspace- out from under it and then took a fresh lock, with no error and no message.- Reproduced 2026-08-25 on tsr_scn_boat_workshop_v2: two independent `3dcv make` on one- slug, 33 seconds apart, both writing.- """- from codeverse.cli._common import CliError, create_workspace-- root = tmp_path / "held_run"- root.mkdir()- (root / "spec.json").write_text("{}") # non-empty, so --force would rmtree it- with run_lock(root, what="3dcv make held_run"):- with pytest.raises(CliError) as ei:- create_workspace(root, force=True)- assert str(os.getpid()) in str(ei.value)- assert (root / "spec.json").exists(), "the live holder's workspace must survive"---def test_force_still_overwrites_a_dead_run(tmp_path):- """The check must not make a crashed run un-forceable — that would be worse than the- bug it fixes, because 13 runs needed exactly this after the pkill incident."""- import json-- from codeverse.cli._common import create_workspace-- root = tmp_path / "dead_run"- (root / ".3dcv").mkdir(parents=True)- (root / "spec.json").write_text("{}")- _lock_path(root).write_text(json.dumps({"pid": 4194303, "started": 0, "what": "3dcv make ghost"}))- create_workspace(root, force=True) # must not raise- assert not (root / "spec.json").exists(), "a dead run's directory is replaced as before"---# ------------------------------------------------------- the flock mutex (2026-08-27)def _child_script(run_root: Path, ready: Path, go: Path) -> str:"""A second PROCESS that takes the run mutex, announces it, and waits to be told to let go."""return (@@ -130,7 +38,10 @@ def _child_script(run_root: Path, ready: Path, go: Path) -> str:def test_two_processes_cannot_hold_one_run(tmp_path: Path):- """The PID file could be passed by BOTH racers (check, then write); the flock cannot."""+ """Measured 2026-08-25: a lane launched `3dcv resume tsr_scn_temple_night` three+ times and two ran concurrently on the same workspace for four minutes, both writing+ run_state.json, both snapshotting src/ into the same git repo, both spending the+ run's budget."""import subprocessimport sys@@ -146,7 +57,9 @@ def test_two_processes_cannot_hold_one_run(tmp_path: Path):assert ready.read_text() == "held", "child never took the lock"with pytest.raises(RunLocked) as got, exclusive(run_root):pass- assert str(child.pid) in str(got.value) # names the holder, so a human can kill THAT pid+ msg = str(got.value)+ assert str(child.pid) in msg # names the holder, so a human can kill THAT pid+ assert "pkill" in msg, "and must warn against the pattern kill that took out 13 runs"finally:go.write_text("release")child.wait(timeout=10)@@ -171,23 +84,78 @@ def test_a_dead_holder_never_wedges_a_run(tmp_path: Path):assert ready.read_text() == "held"child.send_signal(signal.SIGKILL)child.wait(timeout=10)+ assert holder_of(run_root) is None, "a SIGKILLed holder cannot leave a record behind"with exclusive(run_root): # no stale-PID reasoning needed: the kernel already released itpass-def test_the_mutex_is_reentrant_in_one_process(tmp_path: Path):- """The CLI holds it at the mutation boundary and the track's run_lock nests inside."""+def test_a_second_thread_is_refused_and_the_first_keeps_the_lock(tmp_path: Path):+ """The bench drivers run runs in a THREAD pool of one process.++ Keying the held set on the path alone made a second thread walk straight into a run+ the first was inside — and worse, whichever left first unlocked and closed the fd+ while the other was still working, handing the run to any outside process.+ """run_root = tmp_path / "runs" / "slug"run_root.mkdir(parents=True)- with exclusive(run_root, what="outer"):- with run_lock(run_root, what="inner"): # would self-refuse without re-entrancy- assert (run_root / ".3dcv" / LOCK_NAME).is_file()- with exclusive(run_root, what="inner2"):+ entered, refused = threading.Event(), []+ release = threading.Event()++ def t2() -> None:+ try:+ with exclusive(run_root, what="T2"):+ refused.append("ENTERED WHILE T1 HELD IT")+ except RunLocked as e:+ refused.append(str(e))++ with exclusive(run_root, what="T1"):+ with exclusive(run_root, what="T1 again"): # the SAME thread re-enterspass- with exclusive(run_root): # fully released after the outermost exit+ th = threading.Thread(target=t2)+ th.start()+ th.join(timeout=10)+ entered.set()+ assert refused and "refusing to enter" in refused[0], refused+ # T2's refusal must not have released T1's lock: the run is still held+ assert (holder_of(run_root) or {}).get("what") == "T1"+ release.set()+ assert entered.is_set()+ with exclusive(run_root): # fully released after the owning thread exitspass+def test_a_failed_record_write_leaks_nothing(tmp_path: Path, monkeypatch):+ """The record write lives INSIDE the try that owns the fd.++ Outside it, a failure left an fd holding the flock with nothing registered to release+ it — every process was refused for the lifetime of this one, by a message naming an+ empty holder.+ """+ import codeverse.runlock as R++ run_root = tmp_path / "runs" / "slug"+ run_root.mkdir(parents=True)+ monkeypatch.setattr(R, "_write_record", lambda fd, rec: (_ for _ in ()).throw(OSError("ENOSPC")))+ with pytest.raises(OSError, match="ENOSPC"), R.exclusive(run_root):+ pass+ monkeypatch.undo()+ assert not R._HELD, "nothing may stay registered after a failed acquisition"+ with R.exclusive(run_root): # still enterable, in this process and any other+ pass+++def test_holder_of_names_the_holder_and_is_empty_otherwise(tmp_path: Path):+ """`3dcv status` prints this so a human can kill THAT pid, never `pkill -f 3dcv`."""+ run_root = tmp_path / "runs" / "slug"+ run_root.mkdir(parents=True)+ assert holder_of(run_root) is None+ with exclusive(run_root, what="3dcv make slug"):+ held = holder_of(run_root)+ assert held is not None and held["pid"] == os.getpid()+ assert held["what"] == "3dcv make slug"+ assert holder_of(run_root) is None, "the record is truncated on release, never left to lie"++def test_the_lock_file_lives_outside_the_run_directory(tmp_path: Path):"""A --force wipe rmtree's the run root; a lock inside it would be deleted mid-hold."""run_root = tmp_path / "runs" / "slug"@@ -197,3 +165,31 @@ def test_the_lock_file_lives_outside_the_run_directory(tmp_path: Path):assert lock.is_file()assert run_root not in lock.parentsassert lock.parent == run_root.parent / LOCKS_DIR+++def test_force_refuses_to_wipe_a_run_another_holder_is_using(tmp_path: Path):+ """`--force` means "overwrite a DEAD run", never "evict a running one": create_workspace+ rmtree's the run root, and `make` holds the mutex around it (cli/main.py)."""+ import subprocess+ import sys++ from codeverse.cli._common import CliError, mutating++ run_root = tmp_path / "runs" / "held_run"+ run_root.mkdir(parents=True)+ (run_root / "spec.json").write_text("{}") # non-empty, so --force would rmtree it+ ready, go = tmp_path / "ready", tmp_path / "go"+ child = subprocess.Popen([sys.executable, "-c", _child_script(run_root, ready, go)])+ try:+ for _ in range(500):+ if ready.exists():+ break+ time.sleep(0.02)+ assert ready.read_text() == "held"+ with pytest.raises(CliError) as ei, mutating(run_root, what="3dcv make held_run"):+ pytest.fail("the mutation boundary must not be entered")+ assert ei.value.exit_code == 2+ assert (run_root / "spec.json").exists(), "the live holder's workspace must survive"+ finally:+ go.write_text("release")+ child.wait(timeout=10)