Skip to content

Official Codebase of 3DCodeVerse

Python

runlock: one authority, and it covers every mutating command (net -82)

gaoypeng825c478931cd
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.py
index 5a342364..b3cf1b96 100644
--- a/harness/bench/ab_plan.py
+++ b/harness/bench/ab_plan.py
@@ -108,6 +108,7 @@ from bench.compare_backends import ( # noqa: E402
)
from bench.pin_plan import PLAN_JSON, PinError, plan_once, seed_plan # noqa: E402
from bench.run_bench import Battery, BenchPrompt, select_prompts # noqa: E402
+from codeverse.runlock import exclusive # noqa: E402
from codeverse.tracks.plan_features import pin_plan_blockers # noqa: E402
from codeverse.workspace import Workspace # noqa: E402
@@ -423,7 +424,8 @@ def pin_pair(
spec = spec_for(battery, item, _harness_arm(opts.generator), opts.compare_options())
src = Workspace(out / "plans" / item.id / "run")
if not (src.root / PLAN_JSON).is_file():
- plan_once(spec, src.root)
+ with exclusive(src.root, what=f"ab pin {item.id}"): # one writer per run dir
+ plan_once(spec, src.root)
want = json.loads((src.root / "run_state.json").read_text())["stages"]["plan"]["inputs_hash"]
for arm in arms:
dst = Workspace(cell_dir(out, arm, item.id, opts.generator) / "run")
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index 2bc9c96c..87f06e0e 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -85,6 +85,7 @@ from codeverse.contracts.common import ENTRY_FILE # noqa: E402
from codeverse.contracts.run import RunRecord # noqa: E402
from codeverse.contracts.spec import Spec # noqa: E402
from codeverse.cost import run_ledger # noqa: E402
+from codeverse.runlock import exclusive # noqa: E402
from codeverse.tracks.generation import MultiFileParseError # noqa: E402
from codeverse.tracks.planner import PlanningError # noqa: E402
from codeverse.workspace import Workspace # noqa: E402
@@ -236,10 +237,11 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, deps: CompareDeps,
if not resume:
run_ws.create()
run_ws.write_json(run_ws.spec_path, spec)
- # the harness arm is a real run: give it its own ledger, nested inside the cell's
- # (run_ledger restores the outer one on the way out, so the fixed evaluation that
- # follows keeps landing in the cell ledger)
- with run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}"):
+ # the harness arm is a real run: one writer per run dir (--parallel is threads of ONE
+ # process), and its own ledger nested inside the cell's (run_ledger restores the outer
+ # one on the way out, so the fixed evaluation that follows lands in the cell ledger)
+ with (exclusive(run_ws.root, what=f"compare {spec.id}:{run_ws.root.parent.name}"),
+ run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}")):
rec = deps.run_track(spec, run_ws, resume)
res.gen_cost_usd = rec.total_usage.cost_usd
res.tool_calls = rec.total_usage.tool_calls
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.py
index 451a0530..a9219363 100644
--- a/harness/bench/run_bench.py
+++ b/harness/bench/run_bench.py
@@ -39,6 +39,7 @@ from codeverse.contracts.common import Backends, Budget, Language, Track
from codeverse.contracts.run import RunRecord
from codeverse.contracts.spec import Constraints, ReferenceImage, Spec
from codeverse.cost import run_ledger
+from codeverse.runlock import exclusive
from codeverse.workspace import Workspace
RESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_final", "passed", "rounds", "cost_usd",
@@ -255,7 +256,9 @@ def run_battery(
ws.write_json(ws.spec_path, spec)
t0 = time.time()
try:
- with run_ledger(ws.root, run=item.id):
+ # one writer per run dir: --parallel runs these in threads of ONE process, so
+ # the run mutex is what keeps two cells off the same workspace
+ with exclusive(ws.root, what=f"bench {item.id}"), run_ledger(ws.root, run=item.id):
rec = run_fn(spec, ws, resume)
except Exception as e: # one failing prompt must not kill the battery
# A provider outage is not a result: an hour spent retrying a 503 is not model
diff --git a/harness/codeverse/cli/_common.py b/harness/codeverse/cli/_common.py
index ee8dce00..905a7cab 100644
--- a/harness/codeverse/cli/_common.py
+++ b/harness/codeverse/cli/_common.py
@@ -7,6 +7,8 @@ import hashlib
import importlib
import shutil
import sys
+from collections.abc import Iterator
+from contextlib import ExitStack, contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Any
@@ -87,19 +89,29 @@ def make_slug(prompt: str, track: str, language: str, explicit: str | None = Non
return f"{slugify(prompt, 40)}_{h}"
-def create_workspace(root: Path, *, force: bool) -> Workspace:
- from codeverse.runlock import RunLocked, assert_free
+@contextmanager
+def mutating(target: Any, *, what: str, action: str = "enter") -> Iterator[None]:
+ """The run mutex (``runlock.exclusive``) around anything that writes into a run
+ directory, reported as a typed exit. ``target`` is a Workspace or a run root."""
+ from codeverse.runlock import RunLocked, exclusive
+
+ root = target if isinstance(target, (str, Path)) else target.root # NB: Path.root is "/"
+ stack = ExitStack()
+ try:
+ stack.enter_context(exclusive(root, what=what, action=action))
+ except RunLocked as e:
+ raise CliError(str(e), code=2) from None
+ with stack:
+ yield
+
+def create_workspace(root: Path, *, force: bool) -> Workspace:
+ """Create (or ``--force``-replace) a run directory. The caller holds the run mutex —
+ the rmtree below would otherwise wipe a live holder's workspace."""
ws = Workspace(root)
if ws.root.exists() and any(ws.root.iterdir()):
if not force:
raise CliError(f"{ws.root} already exists; use --force to overwrite or --slug for a new name")
- # --force overwrites a DEAD run, never a live one: the rmtree below would delete
- # that process's lock file along with its workspace (codeverse/runlock.py)
- try:
- assert_free(ws.root)
- except RunLocked as e:
- raise CliError(str(e), code=2) from None
shutil.rmtree(ws.root)
ws.create()
return ws
diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.py
index 12b4789e..efb5cef9 100644
--- a/harness/codeverse/cli/flywheel_cmd.py
+++ b/harness/codeverse/cli/flywheel_cmd.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from contextlib import nullcontext
from pathlib import Path
from typing import Annotated
@@ -24,7 +25,7 @@ def export_cmd(
tar_prefix: Annotated[str, typer.Option("--tar-prefix", help="repo-root-relative prefix for the tar column")] = "",
include_unbuilt: Annotated[bool, typer.Option("--include-unbuilt", help="also export runs whose best round never built")] = False,
captions_dir: Annotated[Path | None, typer.Option("--captions-dir", help="side-car captions written by `caption --out`")] = None,
- drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave normalised duplicates (normalised-code fingerprint + prompt) out of the index, manifest and tars (recorded under the manifest's dropped)")] = False,
+ drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave byte-identical duplicates (raw code sha256 + prompt) out of the index, manifest and tars (recorded under the manifest's dropped); normalised ones are only marked near_duplicate_of")] = False,
) -> None:
"""Export runs → sample folders + dataset_manifest.json + metadata.jsonl/.parquet
(+ optional plain tars, packed FROM the manifest)."""
@@ -87,9 +88,11 @@ def caption_cmd(
out: Annotated[Path | None, typer.Option("--out", help="write side-car <out>/<slug>.json instead of touching the run (read-only runs)")] = None,
) -> None:
"""Caption run(s): {detailed, instruction, factory} via a chat model."""
+ from codeverse.cost.instrument import run_ledger
from codeverse.flywheel.captions import CaptionError, caption_sample
from codeverse.flywheel.export import load_captions
from codeverse.flywheel.record import iter_runs, load_record
+ from codeverse.runlock import RunLocked, exclusive
if all_runs:
targets = [(fr.ws, fr.record, fr.run_id.slug)
@@ -102,9 +105,15 @@ def caption_cmd(
if not force and load_captions(ws, rec, out, slug=slug).get("detailed"):
skipped += 1
continue
+ # PER RUN: captioning rewrites record.json (--out writes a side-car instead and
+ # touches nothing), and a locked run is skipped with a warning — one live run must
+ # not abort a --all batch of hundreds
+ lock = nullcontext() if out else exclusive(ws.root, what=f"3dcv flywheel caption {ws.root.name}")
try:
- caps = caption_sample(ws, rec, model, out_dir=out, slug=slug)
- except CaptionError as e:
+ # the priced captioner call joins the run's ledger (as `3dcv judge` does)
+ with lock, run_ledger(ws.root, run=ws.root.name, create=False):
+ caps = caption_sample(ws, rec, model, out_dir=out, slug=slug)
+ except (CaptionError, RunLocked) as e:
failed += 1
warn(str(e))
continue
diff --git a/harness/codeverse/cli/inspect_cmd.py b/harness/codeverse/cli/inspect_cmd.py
index 86a76406..89261ab5 100644
--- a/harness/codeverse/cli/inspect_cmd.py
+++ b/harness/codeverse/cli/inspect_cmd.py
@@ -39,6 +39,7 @@ def status(
"""Show spec / run_state / record / recent events of a run."""
from codeverse.events import EventLog
from codeverse.flywheel.record import RecordError, load_record
+ from codeverse.runlock import holder_of
ws = C.open_workspace(slug, runs_dir)
spec = C.load_spec(ws)
@@ -50,6 +51,8 @@ def status(
"judge": spec.backends.judge,
"prompt": spec.prompt,
}
+ if (held := holder_of(ws.root)) is not None: # kill THAT pid, never `pkill -f 3dcv`
+ rows["RUNNING NOW"] = f"pid {held.get('pid', '?')} ({held.get('what') or '3dcv'})"
if ws.state_path.is_file():
try:
state = json.loads(ws.state_path.read_text())
@@ -150,24 +153,25 @@ def render(
spec = C.load_spec(ws)
idx = _render_round_or_refuse(ws, round_index)
out_dir = out or ws.renders_dir(idx) / ("cli" if mode == "shaded" else f"cli_{mode}")
- if spec.track is Track.GRAPHICS:
- _render_graphics(ws, spec, out_dir)
- return
- # the size settings are honoured by the in-run renders (tracks/static_object.py,
- # tracks/scene.py) and were silently dropped by the one command whose whole job is
- # rendering, so a CLI render did not match the one the judge saw
- r = get_settings().render
- if spec.track is Track.SCENE:
- render_scene = C.lazy("codeverse.spatial.render_scene", "render_scene")
- rs = render_scene(
- ws, out_dir, cameras=None, width=width or r.scene_width, height=height or r.scene_height
- )
- else:
- glb = ws.artifacts / "object.glb"
- if not glb.is_file():
- raise C.CliError(f"no artifact to render: {glb} (run a build first)")
- render_glb = C.lazy("codeverse.spatial.render", "render_glb")
- rs = render_glb(glb, out_dir, mode=mode, width=width or r.width, height=height or r.height)
+ # writes into the run (a graphics render even rebuilds it): one writer per run dir
+ with C.mutating(ws, what=f"3dcv render {ws.root.name}", action="render"):
+ if spec.track is Track.GRAPHICS:
+ _render_graphics(ws, spec, out_dir)
+ return
+ # the size settings are honoured by the in-run renders (tracks/static_object.py,
+ # tracks/scene.py) and were silently dropped by the one command whose whole job is
+ # rendering, so a CLI render did not match the one the judge saw
+ r = get_settings().render
+ if spec.track is Track.SCENE:
+ render_scene = C.lazy("codeverse.spatial.render_scene", "render_scene")
+ rs = render_scene(ws, out_dir, cameras=None, width=width or r.scene_width,
+ height=height or r.scene_height)
+ else:
+ glb = ws.artifacts / "object.glb"
+ if not glb.is_file():
+ raise C.CliError(f"no artifact to render: {glb} (run a build first)")
+ render_glb = C.lazy("codeverse.spatial.render", "render_glb")
+ rs = render_glb(glb, out_dir, mode=mode, width=width or r.width, height=height or r.height)
console.print(
kv_table(
"renders",
@@ -214,19 +218,21 @@ def judge(
n_images = J.count_prompt_images(inp, rubric_name)
from codeverse.cost.instrument import run_ledger
- try:
- # a re-judge joins the run's ledger when it has one; otherwise the per-process log
- # (a ledger holding only this verdict would be read as the whole run's cost)
- with run_ledger(ws.root, run=ws.root.name, create=False):
- verdict = judge_obj.judge(inp)
- except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothing
- raise C.CliError(f"judge failed: {e}") from e
- except Exception as e:
- ReferenceJudgeError = C.lazy("codeverse.judges.reference", "ReferenceJudgeError")
- if isinstance(e, ReferenceJudgeError):
+ # writes artifacts/judge/rNN_cli.json into the run: one writer per run dir
+ with C.mutating(ws, what=f"3dcv judge {ws.root.name}", action="judge"):
+ try:
+ # a re-judge joins the run's ledger when it has one; otherwise the per-process
+ # log (a ledger holding only this verdict would be read as the whole run's cost)
+ with run_ledger(ws.root, run=ws.root.name, create=False):
+ verdict = judge_obj.judge(inp)
+ except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothing
raise C.CliError(f"judge failed: {e}") from e
- raise
- ws.write_json(ws.judge_path(idx, "_cli"), verdict)
+ except Exception as e:
+ ReferenceJudgeError = C.lazy("codeverse.judges.reference", "ReferenceJudgeError")
+ if isinstance(e, ReferenceJudgeError):
+ raise C.CliError(f"judge failed: {e}") from e
+ raise
+ ws.write_json(ws.judge_path(idx, "_cli"), verdict)
console.print(
kv_table(
"judgment",
diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.py
index f494b80f..e1e1f868 100644
--- a/harness/codeverse/cli/main.py
+++ b/harness/codeverse/cli/main.py
@@ -15,7 +15,6 @@ from __future__ import annotations
import os
import sys
-from contextlib import ExitStack
from pathlib import Path
from typing import Annotated
@@ -35,7 +34,6 @@ from codeverse.cli.tools_cmd import tools
from codeverse.config import get_settings
from codeverse.contracts.common import TRACK_LANGUAGES, Budget, Language, Track
from codeverse.contracts.spec import Constraints, ReferenceImage, RunOptions, Spec
-from codeverse.runlock import RunLocked, exclusive
app = typer.Typer(
name="3dcv",
@@ -255,62 +253,44 @@ def make(
# THE run mutex, taken BEFORE the workspace is created (or --force wipes one) and
# held through the paid reference call and the run itself: everything below mutates
# this run directory, and two 3dcv on one slug corrupt each other (runlock.py).
- stack = ExitStack()
- try:
- stack.enter_context(exclusive(C.runs_root(runs_dir) / run_slug,
- what=f"3dcv make {run_slug}", action="create"))
- except RunLocked as e:
- raise C.CliError(str(e), code=2) from None
- with stack:
- _make_in_lock(spec, run_slug, runs_dir, force=force, no_run=no_run, reference=reference,
- reference_views=reference_views, candidates=candidates, rounds=rounds,
- max_usd=max_usd, texture=texture, dial=dial, settings=settings,
- backends=backends, track=track, language=language)
-
-
-def _make_in_lock(spec, run_slug: str, runs_dir, *, force: bool, no_run: bool, reference: bool,
- reference_views: int, candidates, rounds, max_usd, texture, dial, settings,
- backends, track, language) -> None:
- """The body of ``make`` that mutates the run directory — always under the run mutex
- (``runlock.exclusive``), which is why creating, wiping, writing the spec and the paid
- reference call can no longer race a second ``3dcv make`` on the same slug."""
- ws = C.create_workspace(C.runs_root(runs_dir) / run_slug, force=force)
- ws.write_json(ws.spec_path, spec)
- ws.commit("spec")
- console.print(
- kv_table(
- "run",
- {
- "slug": run_slug,
- "workspace": ws.root,
- "track": track.value,
- "language": language.value,
- "generator": backends.generator,
- "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",
- "judge": f"{backends.judge} n={dial.judge_samples} "
- f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",
- "rounds": rounds,
- "max_usd": max_usd,
- "candidates": candidates,
- "texture": texture,
- },
+ with C.mutating(C.runs_root(runs_dir) / run_slug, what=f"3dcv make {run_slug}", action="create"):
+ ws = C.create_workspace(C.runs_root(runs_dir) / run_slug, force=force)
+ ws.write_json(ws.spec_path, spec)
+ ws.commit("spec")
+ console.print(
+ kv_table(
+ "run",
+ {
+ "slug": run_slug,
+ "workspace": ws.root,
+ "track": track.value,
+ "language": language.value,
+ "generator": backends.generator,
+ "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",
+ "judge": f"{backends.judge} n={dial.judge_samples} "
+ f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",
+ "rounds": rounds,
+ "max_usd": max_usd,
+ "candidates": candidates,
+ "texture": texture,
+ },
+ )
)
- )
- if reference:
- # --reference runs even under --no-run, because the grounded spec IS the artifact
- # it produces (reference/run.py writes it back to spec.json) — but the combination
- # is otherwise read as "filesystem only", so say out loud that this part spends
- # money and can block on a degraded provider before anything is written.
+ if reference:
+ # --reference runs even under --no-run, because the grounded spec IS the artifact
+ # it produces (reference/run.py writes it back to spec.json) — but the combination
+ # is otherwise read as "filesystem only", so say out loud that this part spends
+ # money and can block on a degraded provider before anything is written.
+ if no_run:
+ warn(
+ f"--reference makes model calls now (1 planner call + {reference_views} image generation(s) "
+ f"+ {reference_views} vision check(s), ~$0.15 for 2 views) — --no-run stops after that"
+ )
+ spec = _ground_in_reference(spec, ws, n_views=reference_views)
if no_run:
- warn(
- f"--reference makes model calls now (1 planner call + {reference_views} image generation(s) "
- f"+ {reference_views} vision check(s), ~$0.15 for 2 views) — --no-run stops after that"
- )
- spec = _ground_in_reference(spec, ws, n_views=reference_views)
- if no_run:
- ok(f"spec written: {ws.spec_path} (not run; `3dcv resume {run_slug}` to start)")
- return
- _run_track(spec, ws, resume=False, candidates=candidates)
+ ok(f"spec written: {ws.spec_path} (not run; `3dcv resume {run_slug}` to start)")
+ return
+ _run_track(spec, ws, resume=False, candidates=candidates)
def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:
@@ -347,21 +327,15 @@ def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:
def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None, force: bool = False) -> None:
from codeverse.cost.instrument import run_ledger
- from codeverse.runlock import RunLocked, run_lock
get_track = C.lazy("codeverse.tracks", "get_track")
options: dict = {"n_candidates": candidates} if candidates else {}
options.update(C.round_policy_options(spec))
try:
- # ONE writer per run directory (codeverse/runlock.py), and every model call and
+ # already inside the run mutex (make / resume take it); every model call and
# agent session of this run lands in telemetry/cost.jsonl
- with (
- run_lock(ws.root, what=f"3dcv {'resume' if resume else 'make'} {ws.root.name}"),
- run_ledger(ws.root, run=ws.root.name),
- ):
+ with run_ledger(ws.root, run=ws.root.name):
record = get_track(spec.track, **options).run(spec, ws, resume=resume, force=force)
- except RunLocked as e:
- raise C.CliError(str(e), code=2) from None
except KeyboardInterrupt:
raise C.CliError(
f"interrupted; resume with `3dcv resume {ws.root.name}`", code=130
@@ -456,44 +430,34 @@ def resume(
reached a terminal state is refused unless ``--force``: re-entering it spends
money and overwrites its final state."""
ws = C.open_workspace(slug, runs_dir)
- try: # the run mutex, before the spec is rewritten (a budget raise is a mutation)
- stack = ExitStack()
- stack.enter_context(exclusive(ws.root, what=f"3dcv resume {ws.root.name}"))
- except RunLocked as e:
- raise C.CliError(str(e), code=2) from None
- with stack:
- _resume_in_lock(ws, runs_dir, force=force, max_usd=max_usd, max_minutes=max_minutes,
- rounds=rounds, candidates=candidates)
-
-
-def _resume_in_lock(ws, runs_dir, *, force: bool, max_usd, max_minutes, rounds, candidates) -> None:
- """The body of ``resume`` that mutates the run — always under the run mutex."""
- spec = C.load_spec(ws)
- if (
- spec.options.profile
- ): # the dial the run was created with (judge samples, montage px, turn cap)
- try:
- get_settings().apply_profile(spec.options.profile, force=True)
- except ValueError as e: # a spec.json naming a profile this build no longer has
- raise C.CliError(f"{ws.spec_path}: {e}", code=2) from e
- raised = {
- k: v
- for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items()
- if v is not None
- }
- if not force and (why := _finished_reason(ws, raised)):
- raise C.CliError(
- f"run {ws.root.name} already finished ({why}); nothing to resume. "
- f"`3dcv status {ws.root.name}` to look at it, or --force to re-enter it "
- f"(that re-plans, re-scores and overwrites the final state)."
- )
- if raised:
- from codeverse.events import EventLog
+ # the run mutex, before the spec is rewritten (a budget raise is a mutation)
+ with C.mutating(ws, what=f"3dcv resume {ws.root.name}"):
+ spec = C.load_spec(ws)
+ if (
+ spec.options.profile
+ ): # the dial the run was created with (judge samples, montage px, turn cap)
+ try:
+ get_settings().apply_profile(spec.options.profile, force=True)
+ except ValueError as e: # a spec.json naming a profile this build no longer has
+ raise C.CliError(f"{ws.spec_path}: {e}", code=2) from e
+ raised = {
+ k: v
+ for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items()
+ if v is not None
+ }
+ if not force and (why := _finished_reason(ws, raised)):
+ raise C.CliError(
+ f"run {ws.root.name} already finished ({why}); nothing to resume. "
+ f"`3dcv status {ws.root.name}` to look at it, or --force to re-enter it "
+ f"(that re-plans, re-scores and overwrites the final state)."
+ )
+ if raised:
+ from codeverse.events import EventLog
- spec = spec.model_copy(update={"budget": spec.budget.model_copy(update=raised)})
- ws.write_json(ws.spec_path, spec)
- EventLog(ws.events_path).emit("budget.raised", **raised)
- _run_track(spec, ws, resume=True, candidates=candidates, force=force)
+ spec = spec.model_copy(update={"budget": spec.budget.model_copy(update=raised)})
+ ws.write_json(ws.spec_path, spec)
+ EventLog(ws.events_path).emit("budget.raised", **raised)
+ _run_track(spec, ws, resume=True, candidates=candidates, force=force)
# --------------------------------------------------------------------------- status / render / judge
diff --git a/harness/codeverse/cli/texture_cmd.py b/harness/codeverse/cli/texture_cmd.py
index f3a85b87..a0f26276 100644
--- a/harness/codeverse/cli/texture_cmd.py
+++ b/harness/codeverse/cli/texture_cmd.py
@@ -47,8 +47,10 @@ def pass_(
texture_pass = C.lazy("codeverse.texturing.run", "texture_pass")
from codeverse.cost.instrument import run_ledger
- # a post-hoc pass joins the run's ledger when it has one, else the per-process log
- with run_ledger(ws.root, run=ws.root.name, create=False):
+ # a post-hoc pass joins the run's ledger when it has one, else the per-process log;
+ # it rewrites the run's artifacts, so it holds the run mutex (one writer per run dir)
+ with (C.mutating(ws, what=f"3dcv texture pass {ws.root.name}", action="texture"),
+ run_ledger(ws.root, run=ws.root.name, create=False)):
rep = texture_pass(ws, spec, plan, model_id=model or spec.backends.planner,
image_model=_image_model(image_model), judge=judge,
judge_model_id=judge_model, size=size)
@@ -105,7 +107,8 @@ def scene_pack(
scene_texture_pack = C.lazy("codeverse.texturing.scene_pack", "scene_texture_pack")
texture_pack_prompt = C.lazy("codeverse.texturing.scene_pack", "texture_pack_prompt")
model_id = spec.backends.planner if model is None else model
- pack = scene_texture_pack(plan, out or ws.public / "textures", _image_model(image_model), model_id, size=size, n_max=n)
+ with C.mutating(ws, what=f"3dcv texture scene-pack {ws.root.name}", action="texture"):
+ pack = scene_texture_pack(plan, out or ws.public / "textures", _image_model(image_model), model_id, size=size, n_max=n)
rows = {name: f"{e.file or 'FAILED'} tile={e.tile_size_m:.2f}m {e.material_family}/{e.role} seam={e.seam_score:.3f}"
+ (f" {e.error}" if e.error else "") for name, e in pack.entries.items()}
console.print(kv_table("scene texture pack", rows))
diff --git a/harness/codeverse/flywheel/migrate.py b/harness/codeverse/flywheel/migrate.py
index c072b8fa..c0aa5fbe 100644
--- a/harness/codeverse/flywheel/migrate.py
+++ b/harness/codeverse/flywheel/migrate.py
@@ -17,11 +17,13 @@ history. Running it twice is a no-op (second pass reports ``up_to_date``).
from __future__ import annotations
import logging
+from contextlib import ExitStack
from pathlib import Path
from pydantic import BaseModel, Field
from codeverse.flywheel.record import RecordError, load_record, package_run
+from codeverse.runlock import RunLocked, exclusive
from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
@@ -64,43 +66,53 @@ class MigrationReport(BaseModel):
def migrate_run(ws: Workspace, *, dry_run: bool = False) -> RunMigration:
- """Reorganise one run in place; idempotent, additive, never destructive."""
+ """Reorganise one run in place; idempotent, additive, never destructive.
+
+ Under the run mutex unless ``dry_run`` (it relinks the layout and rewrites
+ ``record.json``); a run someone is using is reported SKIPPED, not migrated."""
m = RunMigration(run=str(ws.root))
if not ws.exists():
m.status = SKIPPED
m.reason = "no spec.json"
return m
- changed = False
- try:
- m.layout = ws.ensure_layout(dry_run=dry_run)
- m.gitignore = ws.ensure_gitignore(dry_run=dry_run)
- except OSError as e:
- m.status = FAILED
- m.reason = f"{type(e).__name__}: {e}"
- return m
- changed = bool(m.layout) or m.gitignore
+ stack = ExitStack()
try:
- record = load_record(ws)
- except RecordError as e:
- m.status = MIGRATED if changed else UP_TO_DATE
- m.reason = str(e) # layout only: an unfinished run has no record yet
+ if not dry_run:
+ stack.enter_context(exclusive(ws.root, what=f"3dcv migrate-runs {ws.root.name}"))
+ except RunLocked as e:
+ m.status, m.reason = SKIPPED, str(e)
return m
- if dry_run:
- m.telemetry_rows = _row_count(ws, record)
- m.deliverable_files = None
- m.status = MIGRATED if (changed or record.telemetry is None or record.deliverable is None) else UP_TO_DATE
+ with stack:
+ try:
+ m.layout = ws.ensure_layout(dry_run=dry_run)
+ m.gitignore = ws.ensure_gitignore(dry_run=dry_run)
+ except OSError as e:
+ m.status = FAILED
+ m.reason = f"{type(e).__name__}: {e}"
+ return m
+ changed = bool(m.layout) or m.gitignore
+ try:
+ record = load_record(ws)
+ except RecordError as e:
+ m.status = MIGRATED if changed else UP_TO_DATE
+ m.reason = str(e) # layout only: an unfinished run has no record yet
+ return m
+ if dry_run:
+ m.telemetry_rows = _row_count(ws, record)
+ m.deliverable_files = None
+ m.status = MIGRATED if (changed or record.telemetry is None or record.deliverable is None) else UP_TO_DATE
+ return m
+ before = ws.record_path.read_bytes()
+ package_run(ws, record)
+ if record.deliverable is not None:
+ m.deliverable_files = len(record.deliverable.files)
+ m.deliverable_bytes = record.deliverable.total_bytes
+ if record.telemetry is not None and record.telemetry.cost is not None:
+ m.telemetry_rows = record.telemetry.cost.n_calls
+ ws.write_json(ws.record_path, record)
+ m.record_updated = ws.record_path.read_bytes() != before
+ m.status = MIGRATED if (changed or m.record_updated) else UP_TO_DATE
return m
- before = ws.record_path.read_bytes()
- package_run(ws, record)
- if record.deliverable is not None:
- m.deliverable_files = len(record.deliverable.files)
- m.deliverable_bytes = record.deliverable.total_bytes
- if record.telemetry is not None and record.telemetry.cost is not None:
- m.telemetry_rows = record.telemetry.cost.n_calls
- ws.write_json(ws.record_path, record)
- m.record_updated = ws.record_path.read_bytes() != before
- m.status = MIGRATED if (changed or m.record_updated) else UP_TO_DATE
- return m
def _row_count(ws: Workspace, record: object) -> int | None:
diff --git a/harness/codeverse/runlock.py b/harness/codeverse/runlock.py
index daa69e0b..b7405132 100644
--- a/harness/codeverse/runlock.py
+++ b/harness/codeverse/runlock.py
@@ -1,33 +1,15 @@
"""One writer per run directory.
-Nothing stopped two ``3dcv`` processes from entering the same run. Both write
-``run_state.json``, both snapshot ``src/`` into the same git repo, both append to the
-same event log and both spend the run's budget. ``workspace.py`` already saw the
-symptom and retried around a contended ``.git/index.lock``; this is the cause.
-
-Measured 2026-08-25: a lane launched ``3dcv resume tsr_scn_temple_night`` three times
-after mistaking a surviving process for someone else's, and two of them ran concurrently
-against the same workspace for four minutes.
-
-TWO layers, because each does what the other cannot (2026-08-27):
-
-* :func:`exclusive` — an ``fcntl.flock`` on ``<runs>/.locks/<slug>.lock``, i.e. OUTSIDE
- the directory a ``--force`` run deletes. This is the actual mutex: the PID file below
- is written after a check, so two processes could both pass it (and a forced wipe took
- the lock file with it). The kernel releases a flock when the holder dies, so a crash
- never leaves a run unenterable.
-* the PID file (:func:`run_lock`) — the human-readable holder record, kept because the
- useful message here is *which* process holds the run, so a human can look at it or kill
- that one PID: the lesson from the same day's ``pkill -f "3dcv make"``, which killed
- thirteen unrelated runs because the operator had no way to name just theirs. A lock
- that says "held by pid 2992370, started 15:56:25" is worth more than one that only says
- "busy". The same record is written into the flock file, so a refusal can name the
- holder either way.
-
-``exclusive`` is re-entrant WITHIN one process (flock is per open file description, so a
-second ``open``+``flock`` from the same process would refuse itself): the CLI takes it at
-the mutation boundary — before a workspace is created, wiped or its spec rewritten — and
-the track's own :func:`run_lock` nests inside it.
+Two ``3dcv`` on one run both write ``run_state.json``, both snapshot ``src/`` into the
+same git repo and both spend the budget (measured 2026-08-25: two ``3dcv resume`` ran
+four minutes side by side on tsr_scn_temple_night).
+
+:func:`exclusive` is the ONE authority: an ``fcntl.flock`` on
+``<runs>/.locks/<slug>.lock``, beside the run dirs because ``--force`` ``rmtree``s the
+run root and a lock deleted mid-hold is no lock. The kernel drops it when the holder
+dies, so a crash never wedges a run. Its ``{pid, started, what}`` record (also read by
+:func:`holder_of`, without locking) is what a refusal prints, so a human can ``kill``
+THAT pid instead of ``pkill -f 3dcv`` — which took out thirteen unrelated runs.
"""
from __future__ import annotations
@@ -41,71 +23,12 @@ from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
-from codeverse.proc import read_json_or_none
-
-LOCK_NAME = "run.lock"
#: directory (beside the run dirs, never inside one) holding the flock files
LOCKS_DIR = ".locks"
class RunLocked(RuntimeError):
- """Another live process is already running this run."""
-
-
-def _lock_path(run_root: Path) -> Path:
- return Path(run_root) / ".3dcv" / LOCK_NAME
-
-
-def _alive(pid: int) -> bool:
- """Is this PID a live process we could signal?
-
- ``os.kill(pid, 0)`` raising ``ProcessLookupError`` means gone; ``PermissionError``
- means alive but owned by someone else, which still counts as held.
- """
- try:
- os.kill(pid, 0)
- except ProcessLookupError:
- return False
- except PermissionError:
- return True
- except OSError:
- return False
- return True
-
-
-def _holder(path: Path) -> dict | None:
- """The live holder recorded in ``path``, or None when absent/stale/unreadable.
-
- An unreadable or truncated lock is treated as stale rather than fatal: a crashed
- process must never leave a run permanently unenterable.
- """
- rec = read_json_or_none(path)
- if rec is None:
- return None
- pid = rec.get("pid")
- if not isinstance(pid, int) or not _alive(pid):
- return None
- return rec
-
-
-def assert_free(run_root: Path | str, *, action: str = "overwrite") -> None:
- """Raise :class:`RunLocked` if a live process holds this run.
-
- Call this BEFORE destroying or recreating a run directory. ``--force`` must mean
- "overwrite a DEAD run", never "evict a running one": ``create_workspace(force=True)``
- does ``shutil.rmtree`` on the run root, which deletes the lock file itself, so without
- this check a forced run silently wipes a live holder's workspace out from under it and
- then takes a fresh lock with no error. Measured 2026-08-25 on
- tsr_scn_boat_workshop_v2: two independent ``3dcv make`` on one slug, 33 seconds apart.
-
- This narrows the window rather than closing it completely — two ``--force`` calls that
- interleave between this check and the wipe can still both proceed. Closing that
- properly needs a lock held outside the directory being deleted; the check here removes
- the case that actually happens, which is a force against a run someone is already
- using.
- """
- if (held := _holder(_lock_path(run_root))) is not None:
- raise RunLocked(_held_message(run_root, held, action=action))
+ """Another live holder — process or thread — is already running this run."""
def _held_message(run_root: Path | str, held: dict, *, action: str) -> str:
@@ -121,15 +44,12 @@ def _held_message(run_root: Path | str, held: dict, *, action: str) -> str:
def flock_path(run_root: Path | str) -> Path:
- """The flock file for this run: ``<runs>/.locks/<slug>.lock``.
-
- Beside the run directory, never inside it — ``create_workspace(force=True)``
- ``rmtree``s the run root, and a lock that gets deleted mid-hold is no lock."""
+ """The flock file for this run: ``<runs>/.locks/<slug>.lock``."""
root = Path(run_root).resolve()
return root.parent / LOCKS_DIR / f"{root.name}.lock"
-_HELD: dict[str, list[int]] = {} # flock path -> [fd, recursion depth] held by THIS process
+_HELD: dict[str, list] = {} # flock path -> [fd, owning thread ident] held by THIS process
_HELD_GUARD = threading.Lock()
@@ -150,29 +70,44 @@ def _write_record(fd: int, rec: dict) -> None:
os.fsync(fd)
+def holder_of(run_root: Path | str) -> dict | None:
+ """The ``{pid, started, what}`` of whoever holds this run — without taking it.
+
+ ``None`` when nobody does, including after a SIGKILL: the kernel dropped the flock but
+ the record is still in the file, so a probe (shared, non-blocking, released at once)
+ tells the difference and ``3dcv status`` never names a pid that is already gone."""
+ try:
+ fd = os.open(flock_path(run_root), os.O_RDONLY)
+ except OSError:
+ return None
+ try:
+ try:
+ fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
+ except OSError:
+ return _record_of(fd) # refused: somebody is holding it right now
+ fcntl.flock(fd, fcntl.LOCK_UN)
+ return None
+ finally:
+ os.close(fd)
+
+
@contextmanager
def exclusive(run_root: Path | str, *, what: str = "", action: str = "enter") -> Iterator[None]:
"""THE mutex for one run: hold it around every mutation of the run directory.
- Raises :class:`RunLocked` naming the holder when another live process has it.
- Re-entrant within this process (nested :func:`run_lock` is a no-op); released by
- the kernel if this process dies, so a crash never wedges the run."""
- path = flock_path(run_root)
- key = str(path)
+ Raises :class:`RunLocked`, naming the holder, when anyone else has it — another
+ process, or another THREAD of this one (the bench drivers run runs in a pool). Only
+ the SAME thread re-enters, so the CLI can take it at the mutation boundary and the
+ code under it can take it again."""
+ key, me = str(flock_path(run_root)), threading.get_ident()
with _HELD_GUARD:
- if (entry := _HELD.get(key)) is not None:
- entry[1] += 1
- nested = True
- else:
- nested = False
- if nested:
- try:
- yield
- finally:
- with _HELD_GUARD:
- if (entry := _HELD.get(key)) is not None:
- entry[1] -= 1
+ entry = _HELD.get(key)
+ if entry is not None:
+ if entry[1] != me:
+ raise RunLocked(_held_message(run_root, holder_of(run_root) or {}, action=action))
+ yield # the SAME thread re-entering: the outermost `with` owns the release
return
+ path = Path(key)
path.parent.mkdir(parents=True, exist_ok=True)
fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o644)
try:
@@ -184,10 +119,10 @@ def exclusive(run_root: Path | str, *, what: str = "", action: str = "enter") ->
except BaseException:
os.close(fd)
raise
- _write_record(fd, {"pid": os.getpid(), "started": time.time(), "what": what})
- with _HELD_GUARD:
- _HELD[key] = [fd, 1]
- try:
+ try: # inside the try that owns the fd: a failed write must not leak an unreleasable lock
+ _write_record(fd, {"pid": os.getpid(), "started": time.time(), "what": what})
+ with _HELD_GUARD:
+ _HELD[key] = [fd, me]
yield
finally:
with _HELD_GUARD:
@@ -199,29 +134,3 @@ def exclusive(run_root: Path | str, *, what: str = "", action: str = "enter") ->
pass
finally:
os.close(fd)
-
-
-@contextmanager
-def run_lock(run_root: Path | str, *, what: str = "") -> Iterator[None]:
- """Hold the run directory for this process, or raise :class:`RunLocked`.
-
- Takes over a lock whose PID is gone (a killed or crashed run must stay resumable).
- Releasing is best-effort and never masks the body's own exception.
- """
- path = _lock_path(run_root)
- path.parent.mkdir(parents=True, exist_ok=True)
- if (held := _holder(path)) is not None:
- raise RunLocked(_held_message(run_root, held, action="enter"))
- with exclusive(run_root, what=what): # the real mutex; a no-op when the CLI holds it
- rec = {"pid": os.getpid(), "started": time.time(), "what": what}
- tmp = path.with_suffix(".lock.tmp")
- tmp.write_text(json.dumps(rec))
- os.replace(tmp, path) # the human-readable holder record `3dcv status` reads
- try:
- yield
- finally:
- try:
- if (cur := _holder(path)) is not None and cur.get("pid") == os.getpid():
- path.unlink(missing_ok=True)
- except OSError:
- pass
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 1b264e5e..c7a97354 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -80,8 +80,8 @@ codeverse/
workspace.py run-dir layout + git snapshots ; events.py JSONL event log
runlock.py ONE writer per run dir: an fcntl.flock on <runs>/.locks/<slug>.lock
(outside the dir a --force wipe deletes; released by the kernel if
- the holder dies) taken at every CLI mutation boundary, plus the PID
- file that NAMES the holder so a human can kill that one pid
+ the holder dies) taken at every mutation boundary; its record NAMES
+ the holder (`holder_of`, printed by `3dcv status`) so a human can kill that pid
proc.py stdlib-only subprocess + atomic-JSON primitives (ManagedProcess owns every
child's lifecycle: group kill on ANY exception, bounded pumps, stdin writer;
run_subprocess, kill_group, tail, write_json_atomic, scrub_secrets) and the tolerant readers/writer
@@ -243,7 +243,7 @@ runs/<slug>/
tool_renders/rNN_<hash>/
trajectories/<label>_rNN/ prompt.md transcript.jsonl stdout.json stderr.log result.json
(a retried label lands in <label>.a2_rNN — first attempt preserved)
- AGENTS.md GEMINI.md CLAUDE.md .mcp.json .gemini/settings.json .3dcv/cookbook.md .geminiignore .aiexclude
+ AGENTS.md GEMINI.md CLAUDE.md .gemini/settings.json .3dcv/cookbook.md .geminiignore .aiexclude
```
## 4. Per-language authoring contracts (raw code; the harness owns export)
diff --git a/harness/tests/core/test_runlock.py b/harness/tests/core/test_runlock.py
index 09c00863..bea59e24 100644
--- a/harness/tests/core/test_runlock.py
+++ b/harness/tests/core/test_runlock.py
@@ -2,116 +2,24 @@
from __future__ import annotations
-import json
import os
+import threading
import time
from pathlib import Path
import pytest
from codeverse.runlock import (
- LOCK_NAME,
LOCKS_DIR,
RunLocked,
- _lock_path,
exclusive,
flock_path,
- run_lock,
+ holder_of,
)
HARNESS_ROOT = Path(__file__).resolve().parents[2]
-def test_a_second_process_cannot_enter_a_locked_run(tmp_path):
- """Measured 2026-08-25: a lane launched `3dcv resume tsr_scn_temple_night` three
- times and two ran concurrently on the same workspace for four minutes, both writing
- run_state.json, both snapshotting src/ into the same git repo, both spending the
- run's budget."""
- with (
- run_lock(tmp_path, what="3dcv make demo"),
- pytest.raises(RunLocked) as ei,
- run_lock(tmp_path), # raises on __enter__, so the body below never runs
- ):
- pytest.fail("the second entry must not be granted")
- msg = str(ei.value)
- assert str(os.getpid()) in msg, "the message must name the PID so ONE process can be killed"
- assert "pkill" in msg, "and must warn against the pattern kill that took out 13 runs"
-
-
-def test_the_lock_is_released_on_the_way_out(tmp_path):
- with run_lock(tmp_path):
- pass
- with run_lock(tmp_path): # must not raise
- pass
- assert not _lock_path(tmp_path).exists()
-
-
-def test_an_exception_still_releases_the_lock(tmp_path):
- with pytest.raises(ZeroDivisionError), run_lock(tmp_path):
- raise ZeroDivisionError
- with run_lock(tmp_path):
- pass
-
-
-def test_a_dead_holder_is_taken_over_not_honoured(tmp_path):
- """A killed or crashed run must stay resumable — the pkill incident left 13 runs
- that all needed to be re-entered."""
- p = _lock_path(tmp_path)
- p.parent.mkdir(parents=True, exist_ok=True)
- # PID 2^22 is above /proc/sys/kernel/pid_max on any normal box, so it cannot exist
- p.write_text(json.dumps({"pid": 4194303, "started": 0, "what": "3dcv make ghost"}))
- with run_lock(tmp_path):
- assert json.loads(p.read_text())["pid"] == os.getpid()
-
-
[email protected]("junk", ["", "not json", '{"no pid": 1}', '{"pid": "abc"}'])
-def test_an_unreadable_lock_never_bricks_a_run(tmp_path, junk):
- """A truncated write from a crash must not make the run permanently unenterable."""
- p = _lock_path(tmp_path)
- p.parent.mkdir(parents=True, exist_ok=True)
- p.write_text(junk)
- with run_lock(tmp_path):
- pass
-
-
-# --------------------------------------------------------- --force must not evict a live run
-def test_force_refuses_to_wipe_a_run_a_live_process_is_holding(tmp_path):
- """`--force` means "overwrite a DEAD run", never "evict a running one".
-
- create_workspace(force=True) does shutil.rmtree on the run root, which deletes the
- lock file itself — so before this check a forced run wiped a live holder's workspace
- out from under it and then took a fresh lock, with no error and no message.
- Reproduced 2026-08-25 on tsr_scn_boat_workshop_v2: two independent `3dcv make` on one
- slug, 33 seconds apart, both writing.
- """
- from codeverse.cli._common import CliError, create_workspace
-
- root = tmp_path / "held_run"
- root.mkdir()
- (root / "spec.json").write_text("{}") # non-empty, so --force would rmtree it
- with run_lock(root, what="3dcv make held_run"):
- with pytest.raises(CliError) as ei:
- create_workspace(root, force=True)
- assert str(os.getpid()) in str(ei.value)
- assert (root / "spec.json").exists(), "the live holder's workspace must survive"
-
-
-def test_force_still_overwrites_a_dead_run(tmp_path):
- """The check must not make a crashed run un-forceable — that would be worse than the
- bug it fixes, because 13 runs needed exactly this after the pkill incident."""
- import json
-
- from codeverse.cli._common import create_workspace
-
- root = tmp_path / "dead_run"
- (root / ".3dcv").mkdir(parents=True)
- (root / "spec.json").write_text("{}")
- _lock_path(root).write_text(json.dumps({"pid": 4194303, "started": 0, "what": "3dcv make ghost"}))
- create_workspace(root, force=True) # must not raise
- assert not (root / "spec.json").exists(), "a dead run's directory is replaced as before"
-
-
-# ------------------------------------------------------- the flock mutex (2026-08-27)
def _child_script(run_root: Path, ready: Path, go: Path) -> str:
"""A second PROCESS that takes the run mutex, announces it, and waits to be told to let go."""
return (
@@ -130,7 +38,10 @@ def _child_script(run_root: Path, ready: Path, go: Path) -> str:
def test_two_processes_cannot_hold_one_run(tmp_path: Path):
- """The PID file could be passed by BOTH racers (check, then write); the flock cannot."""
+ """Measured 2026-08-25: a lane launched `3dcv resume tsr_scn_temple_night` three
+ times and two ran concurrently on the same workspace for four minutes, both writing
+ run_state.json, both snapshotting src/ into the same git repo, both spending the
+ run's budget."""
import subprocess
import sys
@@ -146,7 +57,9 @@ def test_two_processes_cannot_hold_one_run(tmp_path: Path):
assert ready.read_text() == "held", "child never took the lock"
with pytest.raises(RunLocked) as got, exclusive(run_root):
pass
- assert str(child.pid) in str(got.value) # names the holder, so a human can kill THAT pid
+ msg = str(got.value)
+ assert str(child.pid) in msg # names the holder, so a human can kill THAT pid
+ assert "pkill" in msg, "and must warn against the pattern kill that took out 13 runs"
finally:
go.write_text("release")
child.wait(timeout=10)
@@ -171,23 +84,78 @@ def test_a_dead_holder_never_wedges_a_run(tmp_path: Path):
assert ready.read_text() == "held"
child.send_signal(signal.SIGKILL)
child.wait(timeout=10)
+ assert holder_of(run_root) is None, "a SIGKILLed holder cannot leave a record behind"
with exclusive(run_root): # no stale-PID reasoning needed: the kernel already released it
pass
-def test_the_mutex_is_reentrant_in_one_process(tmp_path: Path):
- """The CLI holds it at the mutation boundary and the track's run_lock nests inside."""
+def test_a_second_thread_is_refused_and_the_first_keeps_the_lock(tmp_path: Path):
+ """The bench drivers run runs in a THREAD pool of one process.
+
+ Keying the held set on the path alone made a second thread walk straight into a run
+ the first was inside — and worse, whichever left first unlocked and closed the fd
+ while the other was still working, handing the run to any outside process.
+ """
run_root = tmp_path / "runs" / "slug"
run_root.mkdir(parents=True)
- with exclusive(run_root, what="outer"):
- with run_lock(run_root, what="inner"): # would self-refuse without re-entrancy
- assert (run_root / ".3dcv" / LOCK_NAME).is_file()
- with exclusive(run_root, what="inner2"):
+ entered, refused = threading.Event(), []
+ release = threading.Event()
+
+ def t2() -> None:
+ try:
+ with exclusive(run_root, what="T2"):
+ refused.append("ENTERED WHILE T1 HELD IT")
+ except RunLocked as e:
+ refused.append(str(e))
+
+ with exclusive(run_root, what="T1"):
+ with exclusive(run_root, what="T1 again"): # the SAME thread re-enters
pass
- with exclusive(run_root): # fully released after the outermost exit
+ th = threading.Thread(target=t2)
+ th.start()
+ th.join(timeout=10)
+ entered.set()
+ assert refused and "refusing to enter" in refused[0], refused
+ # T2's refusal must not have released T1's lock: the run is still held
+ assert (holder_of(run_root) or {}).get("what") == "T1"
+ release.set()
+ assert entered.is_set()
+ with exclusive(run_root): # fully released after the owning thread exits
pass
+def test_a_failed_record_write_leaks_nothing(tmp_path: Path, monkeypatch):
+ """The record write lives INSIDE the try that owns the fd.
+
+ Outside it, a failure left an fd holding the flock with nothing registered to release
+ it — every process was refused for the lifetime of this one, by a message naming an
+ empty holder.
+ """
+ import codeverse.runlock as R
+
+ run_root = tmp_path / "runs" / "slug"
+ run_root.mkdir(parents=True)
+ monkeypatch.setattr(R, "_write_record", lambda fd, rec: (_ for _ in ()).throw(OSError("ENOSPC")))
+ with pytest.raises(OSError, match="ENOSPC"), R.exclusive(run_root):
+ pass
+ monkeypatch.undo()
+ assert not R._HELD, "nothing may stay registered after a failed acquisition"
+ with R.exclusive(run_root): # still enterable, in this process and any other
+ pass
+
+
+def test_holder_of_names_the_holder_and_is_empty_otherwise(tmp_path: Path):
+ """`3dcv status` prints this so a human can kill THAT pid, never `pkill -f 3dcv`."""
+ run_root = tmp_path / "runs" / "slug"
+ run_root.mkdir(parents=True)
+ assert holder_of(run_root) is None
+ with exclusive(run_root, what="3dcv make slug"):
+ held = holder_of(run_root)
+ assert held is not None and held["pid"] == os.getpid()
+ assert held["what"] == "3dcv make slug"
+ assert holder_of(run_root) is None, "the record is truncated on release, never left to lie"
+
+
def test_the_lock_file_lives_outside_the_run_directory(tmp_path: Path):
"""A --force wipe rmtree's the run root; a lock inside it would be deleted mid-hold."""
run_root = tmp_path / "runs" / "slug"
@@ -197,3 +165,31 @@ def test_the_lock_file_lives_outside_the_run_directory(tmp_path: Path):
assert lock.is_file()
assert run_root not in lock.parents
assert lock.parent == run_root.parent / LOCKS_DIR
+
+
+def test_force_refuses_to_wipe_a_run_another_holder_is_using(tmp_path: Path):
+ """`--force` means "overwrite a DEAD run", never "evict a running one": create_workspace
+ rmtree's the run root, and `make` holds the mutex around it (cli/main.py)."""
+ import subprocess
+ import sys
+
+ from codeverse.cli._common import CliError, mutating
+
+ run_root = tmp_path / "runs" / "held_run"
+ run_root.mkdir(parents=True)
+ (run_root / "spec.json").write_text("{}") # non-empty, so --force would rmtree it
+ ready, go = tmp_path / "ready", tmp_path / "go"
+ child = subprocess.Popen([sys.executable, "-c", _child_script(run_root, ready, go)])
+ try:
+ for _ in range(500):
+ if ready.exists():
+ break
+ time.sleep(0.02)
+ assert ready.read_text() == "held"
+ with pytest.raises(CliError) as ei, mutating(run_root, what="3dcv make held_run"):
+ pytest.fail("the mutation boundary must not be entered")
+ assert ei.value.exit_code == 2
+ assert (run_root / "spec.json").exists(), "the live holder's workspace must survive"
+ finally:
+ go.write_text("release")
+ child.wait(timeout=10)