Skip to content

Official Codebase of 3DCodeVerse

Python

runlock: one writer per run directory

gaoypengc7c91ca65ecf
diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.py
index 989d3912..6d259dc6 100644
--- a/harness/codeverse/cli/main.py
+++ b/harness/codeverse/cli/main.py
@@ -30,22 +30,42 @@ from codeverse.config import get_settings
from codeverse.contracts.common import TRACK_LANGUAGES, Budget, Language, Track
from codeverse.contracts.spec import Constraints, ReferenceImage, RunOptions, Spec
-app = typer.Typer(name="3dcv", help="3dcodeverse: LLMs write raw 3D code; the harness builds, judges, refines, records.",
- pretty_exceptions_enable=False)
+app = typer.Typer(
+ name="3dcv",
+ help="3dcodeverse: LLMs write raw 3D code; the harness builds, judges, refines, records.",
+ pretty_exceptions_enable=False,
+)
app.add_typer(flywheel_app, name="flywheel", help="Dataset export / pairs / captions / index.")
-app.add_typer(gallery_app, name="gallery", help="Look at runs locally: `serve` on localhost, `build` one shareable HTML file.")
+app.add_typer(
+ gallery_app,
+ name="gallery",
+ help="Look at runs locally: `serve` on localhost, `build` one shareable HTML file.",
+)
app.add_typer(bench_app, name="bench", help="Prompt batteries: run + report.")
-app.add_typer(cost_app, name="cost", help="Cost audit: per stage/role/model, waste, $ per passing artifact.")
+app.add_typer(
+ cost_app, name="cost", help="Cost audit: per stage/role/model, waste, $ per passing artifact."
+)
app.add_typer(doctor_app, name="doctor", help="Environment checks.")
-app.add_typer(skills_app, name="skills", help="The skill library: list / show / validate / read-rate report.")
-app.add_typer(texture_app, name="texture", help="Text-to-image texturing: object pass / scene pack.")
-app.command("tools", help="List spatial tools or run one: `3dcv tools list` | `3dcv tools <name> --json '{...}' --workspace ws`.")(tools)
-
-RunsDirOpt = Annotated[Path | None, typer.Option("--runs-dir", help="runs root (default: settings.runs_dir)")]
+app.add_typer(
+ skills_app, name="skills", help="The skill library: list / show / validate / read-rate report."
+)
+app.add_typer(
+ texture_app, name="texture", help="Text-to-image texturing: object pass / scene pack."
+)
+app.command(
+ "tools",
+ help="List spatial tools or run one: `3dcv tools list` | `3dcv tools <name> --json '{...}' --workspace ws`.",
+)(tools)
+
+RunsDirOpt = Annotated[
+ Path | None, typer.Option("--runs-dir", help="runs root (default: settings.runs_dir)")
+]
@app.callback(invoke_without_command=True)
-def _root(ctx: typer.Context, version: Annotated[bool, typer.Option("--version", is_eager=True)] = False) -> None:
+def _root(
+ ctx: typer.Context, version: Annotated[bool, typer.Option("--version", is_eager=True)] = False
+) -> None:
if version:
console.print(f"3dcv {__version__}")
raise typer.Exit()
@@ -68,39 +88,98 @@ def _root(ctx: typer.Context, version: Annotated[bool, typer.Option("--version",
def make(
prompt: Annotated[str, typer.Argument(help="what to build")],
track: Annotated[Track, typer.Option("--track")] = Track.STATIC_OBJECT,
- language: Annotated[Language | None, typer.Option("--language", help="default: the track's first language "
- "(blender / urdf_blender / scene_threejs / glsl_shader)")] = None,
- generator: Annotated[str | None, typer.Option(help="api-agent:gemini:gemini-3.7-flash | gemini-cli:... | claude-code:... | codex:... | agy:...")] = None,
+ language: Annotated[
+ Language | None,
+ typer.Option(
+ "--language",
+ help="default: the track's first language "
+ "(blender / urdf_blender / scene_threejs / glsl_shader)",
+ ),
+ ] = None,
+ generator: Annotated[
+ str | None,
+ typer.Option(
+ help="api-agent:gemini:gemini-3.7-flash | gemini-cli:... | claude-code:... | codex:... | agy:..."
+ ),
+ ] = None,
planner: Annotated[str | None, typer.Option()] = None,
judge: Annotated[str | None, typer.Option()] = None,
captioner: Annotated[str | None, typer.Option()] = None,
image: Annotated[list[Path] | None, typer.Option("--image", help="reference image(s)")] = None,
- reference: Annotated[bool, typer.Option("--reference/--no-reference", help="reference grounding: synthesize a "
- "neutral studio product shot of the prompt with the image model, validate it, and use it as "
- "the fidelity anchor for planning, the compare_reference tool, the silhouette gate and the "
- "judge. Ignored when --image is given (your own references always win); ~$0.15 for 2 views")] = False,
- reference_views: Annotated[int, typer.Option("--reference-views", min=1, max=4, help="how many reference views to "
- "synthesize (1 = 3/4 only, 2 = + straight front elevation)")] = 2,
- profile: Annotated[str | None, typer.Option("--profile", help="cost/quality dial: economy | balanced | quality "
- "(sets models, judge samples, rounds, candidates, turn cap, "
- "montage size and the texture pass together)")] = None,
- rounds: Annotated[int | None, typer.Option("--rounds", min=0, help="refine rounds after the baseline (default: the profile's)")] = None,
+ reference: Annotated[
+ bool,
+ typer.Option(
+ "--reference/--no-reference",
+ help="reference grounding: synthesize a "
+ "neutral studio product shot of the prompt with the image model, validate it, and use it as "
+ "the fidelity anchor for planning, the compare_reference tool, the silhouette gate and the "
+ "judge. Ignored when --image is given (your own references always win); ~$0.15 for 2 views",
+ ),
+ ] = False,
+ reference_views: Annotated[
+ int,
+ typer.Option(
+ "--reference-views",
+ min=1,
+ max=4,
+ help="how many reference views to "
+ "synthesize (1 = 3/4 only, 2 = + straight front elevation)",
+ ),
+ ] = 2,
+ profile: Annotated[
+ str | None,
+ typer.Option(
+ "--profile",
+ help="cost/quality dial: economy | balanced | quality "
+ "(sets models, judge samples, rounds, candidates, turn cap, "
+ "montage size and the texture pass together)",
+ ),
+ ] = None,
+ rounds: Annotated[
+ int | None,
+ typer.Option(
+ "--rounds", min=0, help="refine rounds after the baseline (default: the profile's)"
+ ),
+ ] = None,
max_usd: Annotated[float | None, typer.Option("--max-usd", min=0)] = None,
max_minutes: Annotated[float | None, typer.Option("--max-minutes", min=0)] = None,
- candidates: Annotated[int | None, typer.Option("--candidates", min=1, help="best-of-N baseline: N parallel candidates, keep the best (default: settings.default_candidates or 1)")] = None,
+ candidates: Annotated[
+ int | None,
+ typer.Option(
+ "--candidates",
+ min=1,
+ help="best-of-N baseline: N parallel candidates, keep the best (default: settings.default_candidates or 1)",
+ ),
+ ] = None,
slug: Annotated[str | None, typer.Option("--slug")] = None,
runs_dir: RunsDirOpt = None,
- dim: Annotated[list[str] | None, typer.Option("--dim", help="width=1.2 (meters); repeatable")] = None,
- must: Annotated[list[str] | None, typer.Option("--must", help="hard requirement; repeatable")] = None,
+ dim: Annotated[
+ list[str] | None, typer.Option("--dim", help="width=1.2 (meters); repeatable")
+ ] = None,
+ must: Annotated[
+ list[str] | None, typer.Option("--must", help="hard requirement; repeatable")
+ ] = None,
must_not: Annotated[list[str] | None, typer.Option("--must-not")] = None,
style: Annotated[str, typer.Option("--style")] = "",
tag: Annotated[list[str] | None, typer.Option("--tag")] = None,
- texture: Annotated[bool, typer.Option("--texture", help="run the text-to-image texture pass after the rounds (frozen on spec.options)")] = False,
+ texture: Annotated[
+ bool,
+ typer.Option(
+ "--texture",
+ help="run the text-to-image texture pass after the rounds (frozen on spec.options)",
+ ),
+ ] = False,
seed: Annotated[int, typer.Option("--seed")] = 0,
force: Annotated[bool, typer.Option("--force", help="overwrite an existing run dir")] = False,
- no_run: Annotated[bool, typer.Option("--no-run", help="only create the workspace + spec.json and stop "
- "(with --reference the reference pass still runs first: it makes model calls and writes "
- "the grounded spec)")] = False,
+ no_run: Annotated[
+ bool,
+ typer.Option(
+ "--no-run",
+ help="only create the workspace + spec.json and stop "
+ "(with --reference the reference pass still runs first: it makes model calls and writes "
+ "the grounded spec)",
+ ),
+ ] = False,
) -> None:
"""Create a run (workspace + spec.json) and execute the track pipeline."""
image, dim, must, must_not, tag = image or [], dim or [], must or [], must_not or [], tag or []
@@ -114,8 +193,15 @@ def make(
texture_arg = texture # the dial folds the profile in below; remember what the USER asked
# ONE resolver for the whole dial, so `--profile X` and `CV3D_PROFILE=X` land the same
# values (they used to disagree on candidates + texture); an explicit flag beats both.
- dial = C.resolve_dial(settings, profile, rounds=rounds, candidates=candidates,
- max_usd=max_usd, max_minutes=max_minutes, texture=texture)
+ dial = C.resolve_dial(
+ settings,
+ profile,
+ rounds=rounds,
+ candidates=candidates,
+ max_usd=max_usd,
+ max_minutes=max_minutes,
+ texture=texture,
+ )
rounds, max_usd, max_minutes = dial.rounds, dial.max_usd, dial.max_minutes
candidates, texture = dial.candidates, dial.texture
# A run must not record and display a pass it cannot run. `3dcv texture pass` already
@@ -127,44 +213,70 @@ def make(
if texture and not texture_supported(track):
if texture_arg: # the user asked for it explicitly: say no, and say where to go
hint = " — use `3dcv texture scene-pack`" if track is Track.SCENE else ""
- raise C.CliError(f"--texture is for object tracks; {track.value} runs have no GLB to texture{hint}")
+ raise C.CliError(
+ f"--texture is for object tracks; {track.value} runs have no GLB to texture{hint}"
+ )
texture = False # profile-implied: quality simply has no texture pass on this track
- backends = settings.backends(generator=generator, planner=planner, judge=judge, captioner=captioner)
+ backends = settings.backends(
+ generator=generator, planner=planner, judge=judge, captioner=captioner
+ )
# validate the whole Spec BEFORE touching the filesystem: an invalid
# track/language combination must not leave an orphan run directory behind.
try:
spec = Spec(
- id=run_slug, track=track, language=language, prompt=prompt,
+ id=run_slug,
+ track=track,
+ language=language,
+ prompt=prompt,
references=[ReferenceImage(path=str(p.resolve())) for p in image],
- constraints=Constraints(dimensions_m=C.parse_kv_floats(dim, "--dim") or None, must_have=must,
- must_not=must_not, style=style),
+ constraints=Constraints(
+ dimensions_m=C.parse_kv_floats(dim, "--dim") or None,
+ must_have=must,
+ must_not=must_not,
+ style=style,
+ ),
budget=Budget(max_rounds=rounds, max_usd=max_usd, max_minutes=max_minutes),
# options.profile records the dial this run resolved to, whichever way it was
# named (flag, CV3D_PROFILE, config.yaml), so `3dcv resume` re-applies it
- backends=backends, options=RunOptions(candidates=candidates, texture=texture,
- profile=dial.profile),
- seed=seed, tags=tag,
+ backends=backends,
+ options=RunOptions(candidates=candidates, texture=texture, profile=dial.profile),
+ seed=seed,
+ tags=tag,
)
except ValueError as e:
raise C.CliError(f"invalid run spec: {e}") from e
ws = C.create_workspace(C.runs_root(runs_dir) / run_slug, force=force)
ws.write_json(ws.spec_path, spec)
ws.commit("spec")
- console.print(kv_table("run", {"slug": run_slug, "workspace": ws.root, "track": track.value,
- "language": language.value, "generator": backends.generator,
- "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",
- "judge": f"{backends.judge} n={dial.judge_samples} "
- f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",
- "rounds": rounds, "max_usd": max_usd,
- "candidates": candidates, "texture": texture}))
+ console.print(
+ kv_table(
+ "run",
+ {
+ "slug": run_slug,
+ "workspace": ws.root,
+ "track": track.value,
+ "language": language.value,
+ "generator": backends.generator,
+ "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",
+ "judge": f"{backends.judge} n={dial.judge_samples} "
+ f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",
+ "rounds": rounds,
+ "max_usd": max_usd,
+ "candidates": candidates,
+ "texture": texture,
+ },
+ )
+ )
if reference:
# --reference runs even under --no-run, because the grounded spec IS the artifact
# it produces (reference/run.py writes it back to spec.json) — but the combination
# is otherwise read as "filesystem only", so say out loud that this part spends
# money and can block on a degraded provider before anything is written.
if no_run:
- warn(f"--reference makes model calls now (1 planner call + {reference_views} image generation(s) "
- f"+ {reference_views} vision check(s), ~$0.15 for 2 views) — --no-run stops after that")
+ warn(
+ f"--reference makes model calls now (1 planner call + {reference_views} image generation(s) "
+ f"+ {reference_views} vision check(s), ~$0.15 for 2 views) — --no-run stops after that"
+ )
spec = _ground_in_reference(spec, ws, n_views=reference_views)
if no_run:
ok(f"spec written: {ws.spec_path} (not run; `3dcv resume {run_slug}` to start)")
@@ -183,11 +295,18 @@ def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:
from codeverse.reference import ground_spec
events = EventLog(ws.events_path)
- with run_ledger(ws.root, run=ws.root.name), call_context(stage=Stage.PLAN, role=Role.PLANNER, label="reference"):
+ with (
+ run_ledger(ws.root, run=ws.root.name),
+ call_context(stage=Stage.PLAN, role=Role.PLANNER, label="reference"),
+ ):
grounded, refset, why = ground_spec(spec, ws, n_views=n_views, events=events)
(ok if grounded is not spec else warn)(f"reference grounding: {why}")
for v in refset.views:
- detail = v.verdict.failure() if v.verdict and not v.accepted else (Path(v.path).name if v.path else "-")
+ detail = (
+ v.verdict.failure()
+ if v.verdict and not v.accepted
+ else (Path(v.path).name if v.path else "-")
+ )
if v.accepted and v.dimension_conflict:
got = f"{v.aspect:.2f}" if v.aspect else "?"
detail += f" (aspect {got} disagrees with the stated dimensions — shape target only)"
@@ -199,19 +318,30 @@ def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:
def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None) -> None:
from codeverse.cost.instrument import run_ledger
+ from codeverse.runlock import RunLocked, run_lock
get_track = C.lazy("codeverse.tracks", "get_track")
options: dict = {"n_candidates": candidates} if candidates else {}
options.update(C.round_policy_options(spec))
try:
- # every model call and agent session of this run lands in telemetry/cost.jsonl
- with run_ledger(ws.root, run=ws.root.name):
+ # ONE writer per run directory (codeverse/runlock.py), and every model call and
+ # agent session of this run lands in telemetry/cost.jsonl
+ with (
+ run_lock(ws.root, what=f"3dcv {'resume' if resume else 'make'} {ws.root.name}"),
+ run_ledger(ws.root, run=ws.root.name),
+ ):
record = get_track(spec.track, **options).run(spec, ws, resume=resume)
+ except RunLocked as e:
+ raise C.CliError(str(e), code=2) from None
except KeyboardInterrupt:
- raise C.CliError(f"interrupted; resume with `3dcv resume {ws.root.name}`", code=130) from None
+ raise C.CliError(
+ f"interrupted; resume with `3dcv resume {ws.root.name}`", code=130
+ ) from None
except Exception as e: # the track failed outside its own error handling
err_console.print_exception(max_frames=8)
- raise C.CliError(f"run failed: {type(e).__name__}: {e} (workspace {ws.root}; see events.jsonl)") from e
+ raise C.CliError(
+ f"run failed: {type(e).__name__}: {e} (workspace {ws.root}; see events.jsonl)"
+ ) from e
print_record_summary(record, ws.root)
if record.status.value == "failed":
raise typer.Exit(code=1)
@@ -236,8 +366,12 @@ def _finished_reason(ws, raised: dict) -> str:
return "" # let the track report it the way it always has
if state is None:
return ""
- if state.status in (RunStatus.PASSED, RunStatus.PLATEAU) or (state.status is RunStatus.BUDGET and not raised):
- detail = f"status={state.status.value}" + (f" stop_reason={state.stop_reason!r}" if state.stop_reason else "")
+ if state.status in (RunStatus.PASSED, RunStatus.PLATEAU) or (
+ state.status is RunStatus.BUDGET and not raised
+ ):
+ detail = f"status={state.status.value}" + (
+ f" stop_reason={state.stop_reason!r}" if state.stop_reason else ""
+ )
if state.status is RunStatus.BUDGET:
return f"{detail}: raise a cap to continue it (--max-usd / --max-minutes / --rounds)"
return f"{detail} best_score={state.best_score}"
@@ -248,11 +382,33 @@ def _finished_reason(ws, raised: dict) -> str:
def resume(
slug: str,
runs_dir: RunsDirOpt = None,
- candidates: Annotated[int | None, typer.Option("--candidates", min=1, help="best-of-N baseline width (only matters before round 0 ran)")] = None,
- max_usd: Annotated[float | None, typer.Option("--max-usd", min=0, help="raise the budget cap before resuming (rewrites spec.json)")] = None,
- max_minutes: Annotated[float | None, typer.Option("--max-minutes", min=0, help="raise the time cap before resuming")] = None,
- rounds: Annotated[int | None, typer.Option("--rounds", min=0, help="new max refine rounds (rewrites spec.json)")] = None,
- force: Annotated[bool, typer.Option("--force", help="re-enter a run that already finished (it will be re-planned and re-scored)")] = False,
+ candidates: Annotated[
+ int | None,
+ typer.Option(
+ "--candidates", min=1, help="best-of-N baseline width (only matters before round 0 ran)"
+ ),
+ ] = None,
+ max_usd: Annotated[
+ float | None,
+ typer.Option(
+ "--max-usd", min=0, help="raise the budget cap before resuming (rewrites spec.json)"
+ ),
+ ] = None,
+ max_minutes: Annotated[
+ float | None,
+ typer.Option("--max-minutes", min=0, help="raise the time cap before resuming"),
+ ] = None,
+ rounds: Annotated[
+ int | None,
+ typer.Option("--rounds", min=0, help="new max refine rounds (rewrites spec.json)"),
+ ] = None,
+ force: Annotated[
+ bool,
+ typer.Option(
+ "--force",
+ help="re-enter a run that already finished (it will be re-planned and re-scored)",
+ ),
+ ] = False,
) -> None:
"""Resume an interrupted / partial run (or start a `--no-run` one).
@@ -262,16 +418,24 @@ def resume(
money and overwrites its final state."""
ws = C.open_workspace(slug, runs_dir)
spec = C.load_spec(ws)
- if spec.options.profile: # the dial the run was created with (judge samples, montage px, turn cap)
+ if (
+ spec.options.profile
+ ): # the dial the run was created with (judge samples, montage px, turn cap)
try:
get_settings().apply_profile(spec.options.profile, force=True)
except ValueError as e: # a spec.json naming a profile this build no longer has
raise C.CliError(f"{ws.spec_path}: {e}", code=2) from e
- raised = {k: v for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items() if v is not None}
+ raised = {
+ k: v
+ for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items()
+ if v is not None
+ }
if not force and (why := _finished_reason(ws, raised)):
- raise C.CliError(f"run {ws.root.name} already finished ({why}); nothing to resume. "
- f"`3dcv status {ws.root.name}` to look at it, or --force to re-enter it "
- f"(that re-plans, re-scores and overwrites the final state).")
+ raise C.CliError(
+ f"run {ws.root.name} already finished ({why}); nothing to resume. "
+ f"`3dcv status {ws.root.name}` to look at it, or --force to re-enter it "
+ f"(that re-plans, re-scores and overwrites the final state)."
+ )
if raised:
from codeverse.events import EventLog
@@ -283,19 +447,31 @@ def resume(
# --------------------------------------------------------------------------- status
@app.command()
-def status(slug: str, runs_dir: RunsDirOpt = None, events: Annotated[int, typer.Option("--events", help="tail N events")] = 8) -> None:
+def status(
+ slug: str,
+ runs_dir: RunsDirOpt = None,
+ events: Annotated[int, typer.Option("--events", help="tail N events")] = 8,
+) -> None:
"""Show spec / run_state / record / recent events of a run."""
from codeverse.events import EventLog
from codeverse.flywheel.record import RecordError, load_record
ws = C.open_workspace(slug, runs_dir)
spec = C.load_spec(ws)
- rows = {"workspace": ws.root, "track": spec.track.value, "language": spec.language.value,
- "generator": spec.backends.generator, "judge": spec.backends.judge, "prompt": spec.prompt}
+ rows = {
+ "workspace": ws.root,
+ "track": spec.track.value,
+ "language": spec.language.value,
+ "generator": spec.backends.generator,
+ "judge": spec.backends.judge,
+ "prompt": spec.prompt,
+ }
if ws.state_path.is_file():
try:
state = json.loads(ws.state_path.read_text())
- rows["run_state"] = ", ".join(f"{k}={v}" for k, v in state.items() if not isinstance(v, (dict, list)))[:300]
+ rows["run_state"] = ", ".join(
+ f"{k}={v}" for k, v in state.items() if not isinstance(v, (dict, list))
+ )[:300]
except ValueError:
rows["run_state"] = "(unreadable)"
console.print(kv_table("status", rows))
@@ -308,17 +484,29 @@ def status(slug: str, runs_dir: RunsDirOpt = None, events: Annotated[int, typer.
_print_candidates(ws)
if record is not None and record.extra.get("texturing"):
t = record.extra["texturing"]
- console.print(kv_table("texturing", {
- "shipped": t.get("shipped"), "delta": t.get("delta"), "reason": t.get("reason", ""),
- "textures": t.get("n_textures", len(t.get("textures", {}) or {})),
- "glb": t.get("glb_textured", "") or "-"}))
- console.print(f"[dim]`3dcv show {ws.root.name}` for the DELIVERABLE / QUALITY EVIDENCE / COST & SETTINGS view[/dim]")
+ console.print(
+ kv_table(
+ "texturing",
+ {
+ "shipped": t.get("shipped"),
+ "delta": t.get("delta"),
+ "reason": t.get("reason", ""),
+ "textures": t.get("n_textures", len(t.get("textures", {}) or {})),
+ "glb": t.get("glb_textured", "") or "-",
+ },
+ )
+ )
+ console.print(
+ f"[dim]`3dcv show {ws.root.name}` for the DELIVERABLE / QUALITY EVIDENCE / COST & SETTINGS view[/dim]"
+ )
evs = EventLog(ws.events_path).read()
if evs:
console.print(f"[dim]last {min(events, len(evs))} of {len(evs)} events:[/dim]")
for ev in evs[-events:]:
extra = {k: v for k, v in ev.items() if k not in ("t", "event")}
- console.print(f" {_event_time(ev)} {ev.get('event', '?'):<14} {json.dumps(extra, default=str)[:160]}")
+ console.print(
+ f" {_event_time(ev)} {ev.get('event', '?'):<14} {json.dumps(extra, default=str)[:160]}"
+ )
def _print_candidates(ws) -> None:
@@ -336,10 +524,13 @@ def _print_candidates(ws) -> None:
mark = " *" if c.get("index") == data.get("selected") else ""
score = c.get("score")
rows[f"{c.get('label', c.get('index'))}{mark}"] = (
- f"score {score if score is None else round(score, 3)} build_ok={c.get('build_ok')}")
+ f"score {score if score is None else round(score, 3)} build_ok={c.get('build_ok')}"
+ )
pw = data.get("pairwise")
if pw:
- rows["pairwise"] = f"{pw.get('a')} vs {pw.get('b')} → {pw.get('winner')} (confidence {pw.get('confidence')})"
+ rows["pairwise"] = (
+ f"{pw.get('a')} vs {pw.get('b')} → {pw.get('winner')} (confidence {pw.get('confidence')})"
+ )
console.print(kv_table("candidates (best-of-N, * = selected)", rows))
@@ -357,10 +548,18 @@ def _event_time(ev: dict) -> str:
def render(
slug: str,
round_index: Annotated[int | None, typer.Option("--round")] = None,
- mode: Annotated[str, typer.Option("--mode", help="shaded | wire | normals | clay | silhouette")] = "shaded",
+ mode: Annotated[
+ str, typer.Option("--mode", help="shaded | wire | normals | clay | silhouette")
+ ] = "shaded",
out: Annotated[Path | None, typer.Option("--out")] = None,
- width: Annotated[int | None, typer.Option("--width", min=1, help="default: render.width (scene: render.scene_width)")] = None,
- height: Annotated[int | None, typer.Option("--height", min=1, help="default: render.height (scene: render.scene_height)")] = None,
+ width: Annotated[
+ int | None,
+ typer.Option("--width", min=1, help="default: render.width (scene: render.scene_width)"),
+ ] = None,
+ height: Annotated[
+ int | None,
+ typer.Option("--height", min=1, help="default: render.height (scene: render.scene_height)"),
+ ] = None,
runs_dir: RunsDirOpt = None,
) -> None:
"""Render the current artifact (object.glb or the scene) with the canonical rig."""
@@ -377,23 +576,36 @@ def render(
r = get_settings().render
if spec.track is Track.SCENE:
render_scene = C.lazy("codeverse.spatial.render", "render_scene")
- rs = render_scene(ws, out_dir, cameras=None,
- width=width or r.scene_width, height=height or r.scene_height)
+ rs = render_scene(
+ ws, out_dir, cameras=None, width=width or r.scene_width, height=height or r.scene_height
+ )
else:
glb = ws.artifacts / "object.glb"
if not glb.is_file():
raise C.CliError(f"no artifact to render: {glb} (run a build first)")
render_glb = C.lazy("codeverse.spatial.render", "render_glb")
rs = render_glb(glb, out_dir, mode=mode, width=width or r.width, height=height or r.height)
- console.print(kv_table("renders", {"views": len(rs.views), "sheet": rs.contact_sheet, "dir": out_dir,
- "renderer": rs.renderer, "ms": rs.duration_ms}))
+ console.print(
+ kv_table(
+ "renders",
+ {
+ "views": len(rs.views),
+ "sheet": rs.contact_sheet,
+ "dir": out_dir,
+ "renderer": rs.renderer,
+ "ms": rs.duration_ms,
+ },
+ )
+ )
@app.command()
def judge(
slug: str,
round_index: Annotated[int | None, typer.Option("--round")] = None,
- rubric: Annotated[str | None, typer.Option("--rubric", help="default: the rubric the round was judged with")] = None,
+ rubric: Annotated[
+ str | None, typer.Option("--rubric", help="default: the rubric the round was judged with")
+ ] = None,
model: Annotated[str | None, typer.Option("--model", help="judge chat model id")] = None,
n: Annotated[int, typer.Option("--n", min=1)] = 1,
runs_dir: RunsDirOpt = None,
@@ -406,7 +618,11 @@ def judge(
ws = C.open_workspace(slug, runs_dir)
rec = load_record(ws)
- idx = round_index if round_index is not None else (rec.best_round if rec.best_round is not None else _latest_round(ws))
+ idx = (
+ round_index
+ if round_index is not None
+ else (rec.best_round if rec.best_round is not None else _latest_round(ws))
+ )
rnd = J.load_round(ws, rec, idx)
if rnd is None or rnd.renders is None or not rnd.renders.views:
raise C.CliError(f"round {idx} has no renders (rounds/r{idx:02d}.json / record.json)")
@@ -429,12 +645,22 @@ def judge(
raise C.CliError(f"judge failed: {e}") from e
raise
ws.write_json(ws.judge_path(idx, "_cli"), verdict)
- console.print(kv_table("judgment", {"rubric": verdict.rubric, "round": idx, "overall": f"{verdict.overall:.3f}",
- "passed": verdict.passed, "std": verdict.score_std,
- "prompt images": n_images if n_images is not None else "?",
- "stored score": f"{rnd.judgment.overall:.3f}" if rnd.judgment else "-",
- "summary": verdict.summary,
- "cost": f"${verdict.usage.cost_usd:.4f}"}))
+ console.print(
+ kv_table(
+ "judgment",
+ {
+ "rubric": verdict.rubric,
+ "round": idx,
+ "overall": f"{verdict.overall:.3f}",
+ "passed": verdict.passed,
+ "std": verdict.score_std,
+ "prompt images": n_images if n_images is not None else "?",
+ "stored score": f"{rnd.judgment.overall:.3f}" if rnd.judgment else "-",
+ "summary": verdict.summary,
+ "cost": f"${verdict.usage.cost_usd:.4f}",
+ },
+ )
+ )
for it in verdict.improvement_plan:
console.print(f" [{it.priority}] {it.target}: {it.instruction}")
@@ -458,13 +684,27 @@ def _render_graphics(ws, spec: Spec, out_dir: Path) -> None:
shutil.copy2(sheet, out_dir / "sheet.png")
sheet = str(out_dir / "sheet.png")
renderer = br.census.get("renderer", "moderngl") if isinstance(br.census, dict) else "moderngl"
- console.print(kv_table("renders", {"views": n, "sheet": sheet or "-", "dir": out_dir,
- "renderer": renderer, "ms": br.duration_ms}))
+ console.print(
+ kv_table(
+ "renders",
+ {
+ "views": n,
+ "sheet": sheet or "-",
+ "dir": out_dir,
+ "renderer": renderer,
+ "ms": br.duration_ms,
+ },
+ )
+ )
def _latest_round(ws) -> int:
rdir = ws.artifacts / "renders"
- idxs = sorted(int(p.name[1:]) for p in rdir.glob("r[0-9][0-9]") if p.is_dir()) if rdir.is_dir() else []
+ idxs = (
+ sorted(int(p.name[1:]) for p in rdir.glob("r[0-9][0-9]") if p.is_dir())
+ if rdir.is_dir()
+ else []
+ )
return idxs[-1] if idxs else 0
@@ -474,7 +714,10 @@ def mcp(workspace: Annotated[Path, typer.Option("--workspace")]) -> None:
"""Serve the spatial tools over stdio MCP (exec `python -m codeverse.spatial.mcp_server`)."""
if not workspace.is_dir():
raise C.CliError(f"workspace not found: {workspace}")
- os.execvp(sys.executable, [sys.executable, "-m", "codeverse.spatial.mcp_server", "--workspace", str(workspace)])
+ os.execvp(
+ sys.executable,
+ [sys.executable, "-m", "codeverse.spatial.mcp_server", "--workspace", str(workspace)],
+ )
# --------------------------------------------------------------------------- run layout (show / migrate)
@@ -482,8 +725,13 @@ def mcp(workspace: Annotated[Path, typer.Option("--workspace")]) -> None:
from codeverse.cli.layout_cmd import migrate_runs_cmd as _migrate_runs_cmd # noqa: E402
from codeverse.cli.layout_cmd import show as _show # noqa: E402
-app.command("show", help="One run in three sections: DELIVERABLE / QUALITY EVIDENCE / COST & SETTINGS.")(_show)
-app.command("migrate-runs", help="Reorganise existing runs onto deliverable/ + evidence/ + telemetry/ (idempotent).")(_migrate_runs_cmd)
+app.command(
+ "show", help="One run in three sections: DELIVERABLE / QUALITY EVIDENCE / COST & SETTINGS."
+)(_show)
+app.command(
+ "migrate-runs",
+ help="Reorganise existing runs onto deliverable/ + evidence/ + telemetry/ (idempotent).",
+)(_migrate_runs_cmd)
if __name__ == "__main__": # pragma: no cover
diff --git a/harness/codeverse/runlock.py b/harness/codeverse/runlock.py
new file mode 100644
index 00000000..bc25b56a
--- /dev/null
+++ b/harness/codeverse/runlock.py
@@ -0,0 +1,101 @@
+"""One writer per run directory.
+
+Nothing stopped two ``3dcv`` processes from entering the same run. Both write
+``run_state.json``, both snapshot ``src/`` into the same git repo, both append to the
+same event log and both spend the run's budget. ``workspace.py`` already saw the
+symptom and retried around a contended ``.git/index.lock``; this is the cause.
+
+Measured 2026-08-25: a lane launched ``3dcv resume tsr_scn_temple_night`` three times
+after mistaking a surviving process for someone else's, and two of them ran concurrently
+against the same workspace for four minutes.
+
+Deliberately a PID file and not ``fcntl.flock``: the useful message here is *which*
+process holds the run, so a human can look at it or kill that one PID — the lesson from
+the same day's ``pkill -f "3dcv make"``, which killed thirteen unrelated runs because the
+operator had no way to name just theirs. A lock that says "held by pid 2992370, started
+15:56:25" is worth more than one that only says "busy".
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+LOCK_NAME = "run.lock"
+
+
+class RunLocked(RuntimeError):
+ """Another live process is already running this run."""
+
+
+def _lock_path(run_root: Path) -> Path:
+ return Path(run_root) / ".3dcv" / LOCK_NAME
+
+
+def _alive(pid: int) -> bool:
+ """Is this PID a live process we could signal?
+
+ ``os.kill(pid, 0)`` raising ``ProcessLookupError`` means gone; ``PermissionError``
+ means alive but owned by someone else, which still counts as held.
+ """
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ return True
+ except OSError:
+ return False
+ return True
+
+
+def _holder(path: Path) -> dict | None:
+ """The live holder recorded in ``path``, or None when absent/stale/unreadable.
+
+ An unreadable or truncated lock is treated as stale rather than fatal: a crashed
+ process must never leave a run permanently unenterable.
+ """
+ try:
+ rec = json.loads(path.read_text())
+ except (OSError, ValueError):
+ return None
+ pid = rec.get("pid")
+ if not isinstance(pid, int) or not _alive(pid):
+ return None
+ return rec
+
+
+@contextmanager
+def run_lock(run_root: Path | str, *, what: str = "") -> Iterator[None]:
+ """Hold the run directory for this process, or raise :class:`RunLocked`.
+
+ Takes over a lock whose PID is gone (a killed or crashed run must stay resumable).
+ Releasing is best-effort and never masks the body's own exception.
+ """
+ path = _lock_path(run_root)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ if (held := _holder(path)) is not None:
+ started = time.strftime("%H:%M:%S", time.localtime(held.get("started", 0)))
+ raise RunLocked(
+ f"run {Path(run_root).name} is already being run by pid {held['pid']} "
+ f"(started {started}: {held.get('what') or '3dcv'}). Two processes on one run "
+ f"corrupt each other's state. Look at it with `3dcv status {Path(run_root).name}`, "
+ f"or stop that ONE process with `kill {held['pid']}` — never `pkill -f 3dcv`, "
+ f"which kills every other run on this machine too."
+ )
+ rec = {"pid": os.getpid(), "started": time.time(), "what": what}
+ tmp = path.with_suffix(".lock.tmp")
+ tmp.write_text(json.dumps(rec))
+ os.replace(tmp, path) # atomic publish; last writer in a race still leaves ONE holder
+ try:
+ yield
+ finally:
+ try:
+ if (cur := _holder(path)) is not None and cur.get("pid") == os.getpid():
+ path.unlink(missing_ok=True)
+ except OSError:
+ pass
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 7f63725b..51e2c04c 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -80,6 +80,8 @@ codeverse/
agent (typed AgentJob + ApiAgentOptions), skills.py (SkillsUsage/SkillRead on
RoundRecord: what was attached, what was read)
workspace.py run-dir layout + git snapshots ; events.py JSONL event log
+ runlock.py ONE writer per run dir — a PID lock that names its holder so a
+ human can kill that one run and not every run on the box
proc.py stdlib-only subprocess + atomic-JSON primitives (run_subprocess w/ group kill +
preexec_fn, kill_group, tail, write_json_atomic) — shared by languages/spatial/cli
models/ ChatModel; gemini_convert.py anthropic_convert.py openai_convert.py
diff --git a/harness/tests/core/test_runlock.py b/harness/tests/core/test_runlock.py
new file mode 100644
index 00000000..a3c44e97
--- /dev/null
+++ b/harness/tests/core/test_runlock.py
@@ -0,0 +1,61 @@
+"""One writer per run directory — see codeverse/runlock.py."""
+
+from __future__ import annotations
+
+import json
+import os
+
+import pytest
+
+from codeverse.runlock import RunLocked, _lock_path, run_lock
+
+
+def test_a_second_process_cannot_enter_a_locked_run(tmp_path):
+ """Measured 2026-08-25: a lane launched `3dcv resume tsr_scn_temple_night` three
+ times and two ran concurrently on the same workspace for four minutes, both writing
+ run_state.json, both snapshotting src/ into the same git repo, both spending the
+ run's budget."""
+ with run_lock(tmp_path, what="3dcv make demo"):
+ with pytest.raises(RunLocked) as ei:
+ with run_lock(tmp_path):
+ pytest.fail("the second entry must not be granted")
+ msg = str(ei.value)
+ assert str(os.getpid()) in msg, "the message must name the PID so ONE process can be killed"
+ assert "pkill" in msg, "and must warn against the pattern kill that took out 13 runs"
+
+
+def test_the_lock_is_released_on_the_way_out(tmp_path):
+ with run_lock(tmp_path):
+ pass
+ with run_lock(tmp_path): # must not raise
+ pass
+ assert not _lock_path(tmp_path).exists()
+
+
+def test_an_exception_still_releases_the_lock(tmp_path):
+ with pytest.raises(ZeroDivisionError):
+ with run_lock(tmp_path):
+ raise ZeroDivisionError
+ with run_lock(tmp_path):
+ pass
+
+
+def test_a_dead_holder_is_taken_over_not_honoured(tmp_path):
+ """A killed or crashed run must stay resumable — the pkill incident left 13 runs
+ that all needed to be re-entered."""
+ p = _lock_path(tmp_path)
+ p.parent.mkdir(parents=True, exist_ok=True)
+ # PID 2^22 is above /proc/sys/kernel/pid_max on any normal box, so it cannot exist
+ p.write_text(json.dumps({"pid": 4194303, "started": 0, "what": "3dcv make ghost"}))
+ with run_lock(tmp_path):
+ assert json.loads(p.read_text())["pid"] == os.getpid()
+
+
[email protected]("junk", ["", "not json", '{"no pid": 1}', '{"pid": "abc"}'])
+def test_an_unreadable_lock_never_bricks_a_run(tmp_path, junk):
+ """A truncated write from a crash must not make the run permanently unenterable."""
+ p = _lock_path(tmp_path)
+ p.parent.mkdir(parents=True, exist_ok=True)
+ p.write_text(junk)
+ with run_lock(tmp_path):
+ pass