Skip to content

Official Codebase of 3DCodeVerse

Python

harness v0.3: quality refactor batches 1-6 (61 verified improvements: shared registries/proc, planner+refine template methods, judge_context unification, spatial perf (fcl memoization, parts LRU), languages _gl_common/_ast_lint dedupe, scene runtime perf) + 503 capacity-storm backoff; 966 tests

gaoypengd4aca786e245
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index 87c7c12a..c481a6d8 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -65,12 +65,17 @@ from bench._oneshot import ( # noqa: E402
repair_prompt,
write_model_file,
)
-from bench.run_bench import Battery, BenchPrompt # noqa: E402
+from bench.run_bench import ( # noqa: E402
+ Battery,
+ BenchPrompt,
+ build_spec,
+ default_run_track,
+ select_prompts,
+)
from codeverse.config import get_settings # noqa: E402
from codeverse.contracts.artifacts import RenderSet # noqa: E402
-from codeverse.contracts.common import Backends, Budget # noqa: E402
from codeverse.contracts.run import RunRecord # noqa: E402
-from codeverse.contracts.spec import Constraints, Spec # noqa: E402
+from codeverse.contracts.spec import Spec # noqa: E402
from codeverse.tracks.generation import MultiFileParseError # noqa: E402
from codeverse.workspace import Workspace # noqa: E402
@@ -116,6 +121,7 @@ class CompareOptions(BaseModel):
parallel: int = 3
limit: int | None = None
ids: list[str] = Field(default_factory=list)
+ tiers: list[str] = Field(default_factory=list)
resume: bool = True
gen_timeout_s: float = 900.0
repair_attempts: int = 2
@@ -126,12 +132,9 @@ class CompareOptions(BaseModel):
def spec_for(battery: Battery, item: BenchPrompt, arm: Arm, opts: CompareOptions) -> Spec:
generator = arm.target if arm.kind == "harness" else f"single-shot:{arm.target}"
- backends = Backends(generator=generator, judge=opts.loop_judge or get_settings().default_judge,
- planner=opts.planner or get_settings().default_planner)
- return Spec(id=f"{battery.name}/{item.id}", track=battery.track, language=battery.language, prompt=item.prompt,
- constraints=Constraints(must_have=list(item.must_have), dimensions_m=item.dimensions_m),
- budget=Budget(max_rounds=opts.rounds, max_usd=opts.max_usd, max_minutes=opts.max_minutes),
- backends=backends, tags=["compare", battery.name, item.tier, item.category, arm.kind, *item.tags])
+ backends = get_settings().backends(generator=generator, judge=opts.loop_judge, planner=opts.planner)
+ return build_spec(battery, item, backends=backends, rounds=opts.rounds, max_usd=opts.max_usd,
+ max_minutes=opts.max_minutes, tag0="compare", extra_tags=(arm.kind,))
# ----------------------------------------------------------------------------- deps (fakeable seams)
@@ -145,17 +148,11 @@ class CompareDeps:
oneshot_backend: Callable[[str], OneShotBackend] | None = None,
pairwise_judge: Callable[[str], Any] | None = None):
self.evaluator = evaluator
- self.run_track = run_track or _default_run_track
+ self.run_track = run_track or default_run_track
self.oneshot_backend = oneshot_backend or get_oneshot_backend
self.pairwise_judge = pairwise_judge or _default_pairwise
-def _default_run_track(spec: Spec, ws: Workspace, resume: bool) -> RunRecord:
- from codeverse.tracks import get_track
-
- return get_track(spec.track).run(spec, ws, resume=resume)
-
-
def _default_pairwise(model_id: str) -> Any:
from codeverse.judges.pairwise import PairwiseJudge
@@ -320,11 +317,6 @@ def _drop_pairs(path: Path, key: tuple[str, str]) -> None:
path.write_text("".join(p.model_dump_json() + "\n" for p in rows))
-def _select(battery: Battery, opts: CompareOptions) -> list[BenchPrompt]:
- items = [p for p in battery.prompts if not opts.ids or p.id in set(opts.ids)]
- return items[: opts.limit] if opts.limit is not None else items
-
-
def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm], opts: CompareOptions,
deps: CompareDeps, *, on_result: Callable[[CellResult], None] | None = None) -> list[CellResult]:
battery = Battery.load(battery_path)
@@ -338,8 +330,9 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm
for key in [k for k, r in done.items() if r.status in set(opts.redo_status)]:
_drop_pairs(out / "pairwise.jsonl", key)
del done[key]
+ selected = select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit)
todo = [(p, a) for a in sorted(arms, key=lambda a: a.kind == "harness") # cheap one-shots first
- for p in _select(battery, opts) if (p.id, a.raw) not in done]
+ for p in selected if (p.id, a.raw) not in done]
with ThreadPoolExecutor(max_workers=max(1, opts.parallel)) as pool, results.open("a") as fh:
futs = {pool.submit(run_cell, battery, p, a, out, opts, deps): (p, a) for p, a in todo}
for fut in as_completed(futs):
@@ -365,6 +358,7 @@ def main(argv: Sequence[str] | None = None) -> int:
ap.add_argument("--parallel", type=int, default=3)
ap.add_argument("--limit", type=int, default=None)
ap.add_argument("--ids", default="", help="comma-separated prompt ids")
+ ap.add_argument("--tiers", default="", help="comma-separated tiers (easy,medium,hard)")
ap.add_argument("--rounds", type=int, default=3)
ap.add_argument("--max-usd", type=float, default=2.5)
ap.add_argument("--loop-judge", default=None, help="harness in-loop judge (default: settings default)")
@@ -380,7 +374,8 @@ def main(argv: Sequence[str] | None = None) -> int:
build_compare_report(Path(ns.out))
return 0
opts = CompareOptions(judge=ns.judge, loop_judge=ns.loop_judge, rounds=ns.rounds, max_usd=ns.max_usd, parallel=ns.parallel,
- limit=ns.limit, ids=[i for i in ns.ids.split(",") if i], resume=not ns.no_resume,
+ limit=ns.limit, ids=[i for i in ns.ids.split(",") if i],
+ tiers=[t for t in ns.tiers.split(",") if t], resume=not ns.no_resume,
gen_timeout_s=ns.gen_timeout, repair_attempts=ns.repair_attempts, pairwise=not ns.no_pairwise,
redo_status=[x for x in ns.redo_status.split(",") if x])
arms = parse_arms(ns.arms)
diff --git a/harness/bench/out/articulated_v1_flash/report.md b/harness/bench/out/articulated_v1_flash/report.md
new file mode 100644
index 00000000..ae488603
--- /dev/null
+++ b/harness/bench/out/articulated_v1_flash/report.md
@@ -0,0 +1,47 @@
+# bench report — articulated_v1
+
+runs: 12 · generator: api-agent:gemini:gemini-3.7-flash · judge: gemini:gemini-3.1-pro-preview · rounds ≤ 3
+
+### overall
+
+| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors |
+|---|---|---|---|---|---|---|---|---|---|---|
+| all | 12 | 9 | 0.618 | 0.800 | 0.942 | +0.182 | 67% | 0.94 | 19.1 | 6 |
+
+### by tier
+
+| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors |
+|---|---|---|---|---|---|---|---|---|---|---|
+| easy | 4 | 4 | 0.788 | 0.875 | 0.955 | +0.086 | 75% | 0.97 | 19.9 | 1 |
+| medium | 4 | 3 | 0.635 | 0.719 | 0.750 | +0.083 | 67% | 1.03 | 16.6 | 2 |
+| hard | 4 | 2 | 0.252 | 0.771 | 0.771 | +0.519 | 50% | 0.83 | 20.7 | 3 |
+
+### by category
+
+| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors |
+|---|---|---|---|---|---|---|---|---|---|---|
+| architecture | 1 | 1 | 0.066 | 0.600 | 0.600 | +0.534 | 0% | 2.32 | 54.9 | 1 |
+| electronics | 1 | 1 | 0.600 | 0.946 | 0.946 | +0.346 | 100% | 1.12 | 35.1 | 0 |
+| furniture | 4 | 3 | 0.790 | 0.790 | 0.964 | +0.000 | 67% | 0.67 | 11.2 | 2 |
+| garden | 1 | 1 | 0.988 | 0.988 | 0.988 | +0.000 | 100% | 0.72 | 7.8 | 0 |
+| lighting | 1 | 1 | 0.500 | 0.750 | 0.750 | +0.250 | 100% | 1.39 | 23.1 | 0 |
+| props | 1 | 1 | 0.438 | 0.942 | 0.942 | +0.504 | 100% | 0.98 | 25.6 | 0 |
+| tools | 2 | 1 | 0.600 | 0.600 | 0.600 | +0.000 | 0% | 1.04 | 18.1 | 2 |
+| vehicles | 1 | 0 | - | - | - | - | - | 0.00 | 1.1 | 1 |
+
+### per prompt
+
+| id | tier | category | baseline | final | passed | rounds | $ | min | status |
+|---|---|---|---|---|---|---|---|---|---|
+| art_easy_cabinet_door | easy | furniture | 0.964 | 0.964 | yes | 1 | 0.30 | 4.1 | passed |
+| art_easy_drawer_unit | easy | furniture | 0.988 | 0.988 | yes | 1 | 0.37 | 5.3 | passed |
+| art_easy_laptop | easy | electronics | 0.600 | 0.946 | yes | 3 | 1.12 | 35.1 | passed |
+| art_easy_scissors | easy | tools | 0.600 | 0.600 | no | 2 | 2.07 | 35.0 | budget ⚠ |
+| art_med_desk_lamp | medium | lighting | 0.500 | 0.750 | yes | 2 | 1.39 | 23.1 | passed |
+| art_med_folding_chair | medium | furniture | 0.418 | 0.418 | no | 1 | 2.00 | 34.5 | budget ⚠ |
+| art_med_toolbox | medium | tools | - | - | - | 0 | 0.00 | 1.1 | error ⚠ |
+| art_med_wheelbarrow | medium | garden | 0.988 | 0.988 | yes | 1 | 0.72 | 7.8 | passed |
+| art_hard_bicycle_wheel_stand | hard | vehicles | - | - | - | 0 | 0.00 | 1.1 | error ⚠ |
+| art_hard_door_handle | hard | architecture | 0.066 | 0.600 | no | 3 | 2.32 | 54.9 | budget ⚠ |
+| art_hard_swivel_chair | hard | furniture | - | - | - | 0 | 0.00 | 1.0 | error ⚠ |
+| art_hard_treasure_chest | hard | props | 0.438 | 0.942 | yes | 2 | 0.98 | 25.6 | passed |
diff --git a/harness/bench/out/articulated_v1_flash/results.csv b/harness/bench/out/articulated_v1_flash/results.csv
new file mode 100644
index 00000000..810a7be9
--- /dev/null
+++ b/harness/bench/out/articulated_v1_flash/results.csv
@@ -0,0 +1,82 @@
+id,tier,category,score_baseline,score_final,passed,rounds,cost_usd,minutes,status,errors,workspace,generator,judge
+art_easy_cabinet_door,easy,furniture,0.964,0.964,True,1,0.3002882,4.14,passed,,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_cabinet_door,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_easy_drawer_unit,easy,furniture,0.988,0.988,True,1,0.373319525,5.29,passed,,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_drawer_unit,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_easy_laptop,easy,electronics,0.6,0.946,True,3,1.1222815,35.09,passed,,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_laptop,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_easy_scissors,easy,tools,0.6,0.6,False,2,2.0728830750000005,35.02,budget,cost $2.073 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_scissors,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_med_folding_chair,medium,furniture,0.418,0.418,False,1,2.004824325,34.5,budget,cost $2.005 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_folding_chair,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_med_desk_lamp,medium,lighting,0.5,0.75,True,2,1.387640925,23.15,passed,,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_desk_lamp,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_med_toolbox,medium,tools,,,,0,0.0,1.1,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+g/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in run
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ return []
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ try:
+
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ def __init__(self, message: str, usage: Usage | None = None):
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_toolbox,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_med_wheelbarrow,medium,garden,0.988,0.988,True,1,0.7191538000000001,7.81,passed,,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_wheelbarrow,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_hard_door_handle,hard,architecture,0.066,0.6,False,3,2.31801805,54.9,budget,cost $2.318 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_door_handle,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_hard_treasure_chest,hard,props,0.438,0.942,True,2,0.983918925,25.64,passed,,/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_treasure_chest,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_hard_swivel_chair,hard,furniture,,,,0,0.0,1.0,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+g/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in run
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ return []
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ try:
+
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ def __init__(self, message: str, usage: Usage | None = None):
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_swivel_chair,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+art_hard_bicycle_wheel_stand,hard,vehicles,,,,0,0.0,1.09,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+g/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in run
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ return []
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ try:
+
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ def __init__(self, message: str, usage: Usage | None = None):
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_bicycle_wheel_stand,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
diff --git a/harness/bench/out/articulated_v1_flash/results.json b/harness/bench/out/articulated_v1_flash/results.json
new file mode 100644
index 00000000..7760efb6
--- /dev/null
+++ b/harness/bench/out/articulated_v1_flash/results.json
@@ -0,0 +1,194 @@
+[
+ {
+ "id": "art_easy_cabinet_door",
+ "tier": "easy",
+ "category": "furniture",
+ "score_baseline": 0.964,
+ "score_final": 0.964,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.3002882,
+ "minutes": 4.14,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_cabinet_door",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_easy_drawer_unit",
+ "tier": "easy",
+ "category": "furniture",
+ "score_baseline": 0.988,
+ "score_final": 0.988,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.373319525,
+ "minutes": 5.29,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_drawer_unit",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_easy_laptop",
+ "tier": "easy",
+ "category": "electronics",
+ "score_baseline": 0.6,
+ "score_final": 0.946,
+ "passed": true,
+ "rounds": 3,
+ "cost_usd": 1.1222815,
+ "minutes": 35.09,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_laptop",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_easy_scissors",
+ "tier": "easy",
+ "category": "tools",
+ "score_baseline": 0.6,
+ "score_final": 0.6,
+ "passed": false,
+ "rounds": 2,
+ "cost_usd": 2.0728830750000005,
+ "minutes": 35.02,
+ "status": "budget",
+ "errors": "cost $2.073 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_scissors",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_med_folding_chair",
+ "tier": "medium",
+ "category": "furniture",
+ "score_baseline": 0.418,
+ "score_final": 0.418,
+ "passed": false,
+ "rounds": 1,
+ "cost_usd": 2.004824325,
+ "minutes": 34.5,
+ "status": "budget",
+ "errors": "cost $2.005 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_folding_chair",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_med_desk_lamp",
+ "tier": "medium",
+ "category": "lighting",
+ "score_baseline": 0.5,
+ "score_final": 0.75,
+ "passed": true,
+ "rounds": 2,
+ "cost_usd": 1.387640925,
+ "minutes": 23.15,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_desk_lamp",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_med_toolbox",
+ "tier": "medium",
+ "category": "tools",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 1.1,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_toolbox",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_med_wheelbarrow",
+ "tier": "medium",
+ "category": "garden",
+ "score_baseline": 0.988,
+ "score_final": 0.988,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.7191538000000001,
+ "minutes": 7.81,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_wheelbarrow",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_hard_door_handle",
+ "tier": "hard",
+ "category": "architecture",
+ "score_baseline": 0.066,
+ "score_final": 0.6,
+ "passed": false,
+ "rounds": 3,
+ "cost_usd": 2.31801805,
+ "minutes": 54.9,
+ "status": "budget",
+ "errors": "cost $2.318 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_door_handle",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_hard_treasure_chest",
+ "tier": "hard",
+ "category": "props",
+ "score_baseline": 0.438,
+ "score_final": 0.942,
+ "passed": true,
+ "rounds": 2,
+ "cost_usd": 0.983918925,
+ "minutes": 25.64,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_treasure_chest",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_hard_swivel_chair",
+ "tier": "hard",
+ "category": "furniture",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 1.0,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_swivel_chair",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "art_hard_bicycle_wheel_stand",
+ "tier": "hard",
+ "category": "vehicles",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 1.09,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_bicycle_wheel_stand",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ }
+]
\ No newline at end of file
diff --git a/harness/bench/out/articulated_v1_flash/results.jsonl b/harness/bench/out/articulated_v1_flash/results.jsonl
index 973360f9..5d3d5a58 100644
--- a/harness/bench/out/articulated_v1_flash/results.jsonl
+++ b/harness/bench/out/articulated_v1_flash/results.jsonl
@@ -1,2 +1,12 @@
{"id":"art_easy_cabinet_door","tier":"easy","category":"furniture","score_baseline":0.964,"score_final":0.964,"passed":true,"rounds":1,"cost_usd":0.3002882,"minutes":4.14,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_cabinet_door","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
{"id":"art_easy_drawer_unit","tier":"easy","category":"furniture","score_baseline":0.988,"score_final":0.988,"passed":true,"rounds":1,"cost_usd":0.373319525,"minutes":5.29,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_drawer_unit","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_easy_laptop","tier":"easy","category":"electronics","score_baseline":0.6,"score_final":0.946,"passed":true,"rounds":3,"cost_usd":1.1222815,"minutes":35.09,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_laptop","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_easy_scissors","tier":"easy","category":"tools","score_baseline":0.6,"score_final":0.6,"passed":false,"rounds":2,"cost_usd":2.0728830750000005,"minutes":35.02,"status":"budget","errors":"cost $2.073 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_easy_scissors","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_med_folding_chair","tier":"medium","category":"furniture","score_baseline":0.418,"score_final":0.418,"passed":false,"rounds":1,"cost_usd":2.004824325,"minutes":34.5,"status":"budget","errors":"cost $2.005 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_folding_chair","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_med_toolbox","tier":"medium","category":"tools","score_baseline":null,"score_final":null,"passed":null,"rounds":0,"cost_usd":0.0,"minutes":1.1,"status":"error","errors":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_toolbox","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_med_wheelbarrow","tier":"medium","category":"garden","score_baseline":0.988,"score_final":0.988,"passed":true,"rounds":1,"cost_usd":0.7191538000000001,"minutes":7.81,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_wheelbarrow","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_med_desk_lamp","tier":"medium","category":"lighting","score_baseline":0.5,"score_final":0.75,"passed":true,"rounds":2,"cost_usd":1.387640925,"minutes":23.15,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_med_desk_lamp","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_hard_swivel_chair","tier":"hard","category":"furniture","score_baseline":null,"score_final":null,"passed":null,"rounds":0,"cost_usd":0.0,"minutes":1.0,"status":"error","errors":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_swivel_chair","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_hard_bicycle_wheel_stand","tier":"hard","category":"vehicles","score_baseline":null,"score_final":null,"passed":null,"rounds":0,"cost_usd":0.0,"minutes":1.09,"status":"error","errors":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_bicycle_wheel_stand","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_hard_treasure_chest","tier":"hard","category":"props","score_baseline":0.438,"score_final":0.942,"passed":true,"rounds":2,"cost_usd":0.983918925,"minutes":25.64,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_treasure_chest","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"art_hard_door_handle","tier":"hard","category":"architecture","score_baseline":0.066,"score_final":0.6,"passed":false,"rounds":3,"cost_usd":2.31801805,"minutes":54.9,"status":"budget","errors":"cost $2.318 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/articulated_v1_flash/runs/art_hard_door_handle","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/cell.json
new file mode 100644
index 00000000..2740e107
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/cell.json
@@ -0,0 +1,29 @@
+{
+ "prompt_id": "cmp_easy_desk_lamp",
+ "tier": "easy",
+ "arm": "harness:api-agent:gemini:gemini-3.7-flash",
+ "kind": "harness",
+ "target": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "error",
+ "score": null,
+ "score_std": 0.0,
+ "passed": null,
+ "build_ok": false,
+ "gate_errors": [],
+ "tris": null,
+ "gen_cost_usd": 0.0,
+ "judge_cost_usd": 0.0,
+ "gen_seconds": 0.0,
+ "wall_s": 65.0,
+ "attempts": 0,
+ "tool_calls": 0,
+ "criteria": {},
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "",
+ "glb": "",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash",
+ "error": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\nevents, runtime=runtime,\n ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n template=template, example=example, temperature=temperature,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n max_output_tokens=max_output_tokens, finalise=finalise, event_stats=event_stats)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 108, in plan_with_usage\n resp = model.generate(req)\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 203, in generate\n return rotate_with_retries(\n self.pool,\n ...<10 lines>...\n label=f\"gemini {self.model}\",\n )\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/retry.py\", line 177, in rotate_with_retries\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/eval
new file mode 160000
index 00000000..c5372685
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/eval
@@ -0,0 +1 @@
+Subproject commit c537268577793d592345e84d026a60f420f210a0
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/run b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/run
new file mode 160000
index 00000000..c5372685
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash/run
@@ -0,0 +1 @@
+Subproject commit c537268577793d592345e84d026a60f420f210a0
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_gemini-cli_gemini-3.7-flash/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_gemini-cli_gemini-3.7-flash/eval
new file mode 160000
index 00000000..0299b00a
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_gemini-cli_gemini-3.7-flash/eval
@@ -0,0 +1 @@
+Subproject commit 0299b00a3279cc745805883674f40750ad55eb8f
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_gemini-cli_gemini-3.7-flash/run b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_gemini-cli_gemini-3.7-flash/run
new file mode 160000
index 00000000..30ba0281
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_gemini-cli_gemini-3.7-flash/run
@@ -0,0 +1 @@
+Subproject commit 30ba0281452ba21d9691c9fe4f68eee89c924e4a
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/cell.json
new file mode 100644
index 00000000..a85ce036
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/cell.json
@@ -0,0 +1,42 @@
+{
+ "prompt_id": "cmp_easy_desk_lamp",
+ "tier": "easy",
+ "arm": "oneshot:claude-code",
+ "kind": "oneshot",
+ "target": "claude-code",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "scored",
+ "score": 0.7,
+ "score_std": 0.012,
+ "passed": false,
+ "build_ok": true,
+ "gate_errors": [
+ "connectivity: 'ShadeNeck' and 'ShadeJoint' interpenetrate by ≈15.5 mm (45% of surface samples inside)",
+ "connectivity: 'ShadeJoint' and 'Arm' interpenetrate by ≈14.4 mm (9% of surface samples inside)",
+ "connectivity: 'Arm' and 'ArmJoint' interpenetrate by ≈13.0 mm (8% of surface samples inside)",
+ "connectivity: 'ArmJoint' and 'BaseCollar' interpenetrate by ≈14.3 mm (41% of surface samples inside)"
+ ],
+ "tris": 5824,
+ "gen_cost_usd": 1.16671,
+ "judge_cost_usd": 0.0774098,
+ "gen_seconds": 241.05,
+ "wall_s": 290.0,
+ "attempts": 1,
+ "tool_calls": 0,
+ "criteria": {
+ "intent_fidelity": 1.0,
+ "structure_plausibility": 0.9,
+ "geometry_detail": 0.8,
+ "proportions_scale": 1.0,
+ "assembly_fit": 0.5,
+ "materials": 0.9,
+ "craftsmanship_no_artifacts": 1.0
+ },
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/eval/artifacts/renders/r00/sheet.png",
+ "glb": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/eval/artifacts/object.glb",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code",
+ "error": ""
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/eval
new file mode 160000
index 00000000..5c65600d
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/eval
@@ -0,0 +1 @@
+Subproject commit 5c65600d3e1056316565478bdee6c5acb84f780d
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/argv.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/argv.json
new file mode 100644
index 00000000..bc1109b5
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/argv.json
@@ -0,0 +1 @@
+["claude", "-p", "<prompt 1956 chars>", "--output-format", "json", "--tools", "", "--max-turns", "1", "--no-session-persistence", "--strict-mcp-config"]
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/prompt.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/prompt.md
new file mode 100644
index 00000000..ce4836b5
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/prompt.md
@@ -0,0 +1,26 @@
+Model this object in raw bpy: a desk lamp with a heavy round base, a single angled arm and a conical metal shade
+MUST HAVE: round base on the ground
+MUST HAVE: one straight arm leaning forward
+MUST HAVE: conical shade at the top of the arm
+MUST HAVE: shade opens downward
+
+You write ONE file, `src/model.py`, in raw Blender Python (bpy, Blender 5.x API).
+The harness runs it headless (`blender -b --factory-startup --python <wrapper> -- --script src/model.py`)
+inside an EMPTIED scene (no default cube / camera / light) and then exports the result itself.
+
+Frame, units and placement: Coordinate frame: Z is UP, -Y is the FRONT of the object (the face a viewer sees from the default front view), +X is the object's right. Units are meters. The object stands on the ground plane z=0 and its footprint is centred on the Z axis. Build at real-world size.
+
+Naming: one mesh object per part, `obj.name` set to a unique PascalCase part name (e.g. `Seat`,
+`LegFrontLeft`); no auto-suffixed duplicates like `Leg.001`. Link every object to
+`bpy.context.scene.collection`. Every visible mesh gets a material (Principled BSDF base colour /
+roughness / metallic).
+
+Do NOT: create cameras or lights, touch render/world settings, call `bpy.ops.render.*`,
+`bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the network, or
+import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Keep the total under
+500k triangles and finish in under 120 s.
+
+The script must build the whole object when executed top-to-bottom (call your `main()` at module
+level). Make sure the code runs without errors in headless Blender — there is no second chance.
+
+You have NO tools in this session: you cannot write files, run Blender or inspect anything — the code must appear in your reply. Reply with the COMPLETE contents of `src/model.py` as ONE ```python fenced code block and nothing else (no prose before or after, no partial snippets).
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/response.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/response.md
new file mode 100644
index 00000000..c5047101
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/response.md
@@ -0,0 +1,190 @@
+```python
+import bpy
+import bmesh
+import math
+from mathutils import Vector, Matrix
+
+SEG = 64
+
+
+# ---------------------------------------------------------------- materials
+def make_material(name, color, roughness=0.5, metallic=0.0, emission=None, emission_strength=0.0):
+ mat = bpy.data.materials.new(name)
+ mat.diffuse_color = (color[0], color[1], color[2], 1.0)
+ mat.roughness = roughness
+ mat.metallic = metallic
+ try:
+ mat.use_nodes = True
+ nt = mat.node_tree
+ bsdf = None
+ for n in nt.nodes:
+ if n.type == 'BSDF_PRINCIPLED':
+ bsdf = n
+ break
+ if bsdf is None:
+ bsdf = nt.nodes.new('ShaderNodeBsdfPrincipled')
+ out = None
+ for n in nt.nodes:
+ if n.type == 'OUTPUT_MATERIAL':
+ out = n
+ break
+ if out is None:
+ out = nt.nodes.new('ShaderNodeOutputMaterial')
+ nt.links.new(bsdf.outputs['BSDF'], out.inputs['Surface'])
+ bsdf.inputs['Base Color'].default_value = (color[0], color[1], color[2], 1.0)
+ bsdf.inputs['Roughness'].default_value = roughness
+ bsdf.inputs['Metallic'].default_value = metallic
+ if emission is not None:
+ try:
+ bsdf.inputs['Emission Color'].default_value = (emission[0], emission[1], emission[2], 1.0)
+ bsdf.inputs['Emission Strength'].default_value = emission_strength
+ except Exception:
+ pass
+ except Exception:
+ pass
+ return mat
+
+
+# ---------------------------------------------------------------- geometry helpers
+def lathe(profile, segments=SEG, closed=False, sharp_angle=35.0):
+ """Revolve a list of (radius, z) points around the local Z axis.
+ radius <= 0 gives a pole vertex. closed=True links last ring back to first."""
+ bm = bmesh.new()
+ rings = []
+ for (r, z) in profile:
+ if r <= 1e-6:
+ rings.append([bm.verts.new((0.0, 0.0, z))])
+ else:
+ ring = []
+ for i in range(segments):
+ a = 2.0 * math.pi * i / segments
+ ring.append(bm.verts.new((r * math.cos(a), r * math.sin(a), z)))
+ rings.append(ring)
+ n = len(rings)
+ pairs = [(k, k + 1) for k in range(n - 1)]
+ if closed:
+ pairs.append((n - 1, 0))
+ for (ka, kb) in pairs:
+ A, B = rings[ka], rings[kb]
+ if len(A) == 1 and len(B) == 1:
+ continue
+ for i in range(segments):
+ i2 = (i + 1) % segments
+ if len(A) == 1:
+ bm.faces.new((A[0], B[i], B[i2]))
+ elif len(B) == 1:
+ bm.faces.new((A[i], A[i2], B[0]))
+ else:
+ bm.faces.new((A[i], A[i2], B[i2], B[i]))
+ bm.normal_update()
+ bmesh.ops.recalc_face_normals(bm, faces=list(bm.faces))
+ bm.normal_update()
+ thr = math.radians(sharp_angle)
+ for f in bm.faces:
+ f.smooth = True
+ for e in bm.edges:
+ ang = e.calc_face_angle(0.0)
+ e.smooth = ang < thr
+ return bm
+
+
+def sphere_profile(r, cz, n=18):
+ pts = []
+ for j in range(n + 1):
+ phi = math.pi * j / n
+ pts.append((r * math.sin(phi), cz - r * math.cos(phi)))
+ return pts
+
+
+def cylinder_profile(r, z0, z1):
+ return [(0.0, z0), (r, z0), (r, z1), (0.0, z1)]
+
+
+def add_object(name, bm, mat, matrix=None):
+ mesh = bpy.data.meshes.new(name)
+ bm.to_mesh(mesh)
+ bm.free()
+ mesh.update()
+ obj = bpy.data.objects.new(name, mesh)
+ bpy.context.scene.collection.objects.link(obj)
+ if matrix is not None:
+ obj.matrix_world = matrix
+ mesh.materials.append(mat)
+ return obj
+
+
+# ---------------------------------------------------------------- build
+def main():
+ mat_dark = make_material("LampDarkMetal", (0.07, 0.07, 0.08), 0.35, 1.0)
+ mat_steel = make_material("LampBrushedSteel", (0.72, 0.72, 0.74), 0.45, 1.0)
+ mat_plastic = make_material("LampBlackPlastic", (0.02, 0.02, 0.02), 0.6, 0.0)
+ mat_bulb = make_material("LampBulb", (1.0, 0.93, 0.8), 0.3, 0.0,
+ emission=(1.0, 0.9, 0.7), emission_strength=3.0)
+
+ # ---- heavy round base on the ground (z = 0), centred on Z axis
+ R, H, b = 0.095, 0.032, 0.009
+ prof = [(0.0, 0.0), (R, 0.0), (R, H - b)]
+ for k in range(1, 6):
+ ang = (math.pi / 2.0) * k / 5.0
+ prof.append((R - b + b * math.cos(ang), H - b + b * math.sin(ang)))
+ prof.append((0.0, H))
+ add_object("Base", lathe(prof, SEG), mat_dark)
+
+ # ---- collar on top of the base where the arm is mounted
+ prof = [(0.0, H - 0.002), (0.021, H - 0.002), (0.021, H + 0.020),
+ (0.017, H + 0.024), (0.0, H + 0.024)]
+ add_object("BaseCollar", lathe(prof, 48), mat_dark)
+
+ # ---- arm joint (ball) on the collar
+ P0 = Vector((0.0, 0.0, 0.060))
+ add_object("ArmJoint", lathe(sphere_profile(0.019, 0.0), 32), mat_dark,
+ Matrix.Translation(P0))
+
+ # ---- single straight arm leaning forward (toward -Y)
+ tilt = math.radians(30.0)
+ L = 0.42
+ d = Vector((0.0, -math.sin(tilt), math.cos(tilt)))
+ arm_mat = Matrix.Translation(P0) @ Matrix.Rotation(tilt, 4, 'X')
+ add_object("Arm", lathe(cylinder_profile(0.009, -0.005, L), 32), mat_dark, arm_mat)
+ P1 = P0 + d * L
+
+ # ---- joint ball at the top of the arm
+ add_object("ShadeJoint", lathe(sphere_profile(0.017, 0.0), 32), mat_dark,
+ Matrix.Translation(P1))
+
+ # ---- shade frame: axis points up & slightly back, so the opening faces down/forward
+ a = math.radians(12.0)
+ axis = Vector((0.0, math.sin(a), math.cos(a)))
+ Rs, rt, h, t = 0.10, 0.03, 0.14, 0.003
+ neck_top = h + 0.032
+ B = P1 - axis * neck_top
+ shade_mat = Matrix.Translation(B) @ Matrix.Rotation(-a, 4, 'X')
+
+ # conical metal shade, hollow shell with thickness, opening at local z=0 (downward)
+ prof = [(Rs, 0.0), (rt, h), (rt - t, h), (Rs - t, 0.0)]
+ add_object("Shade", lathe(prof, SEG, closed=True), mat_steel, shade_mat)
+
+ # cap closing the top of the cone
+ prof = [(0.0, h - 0.004), (0.034, h - 0.004), (0.033, h + 0.010),
+ (0.028, h + 0.014), (0.0, h + 0.014)]
+ add_object("ShadeCap", lathe(prof, 48), mat_steel, shade_mat)
+
+ # neck between cap and the arm joint
+ add_object("ShadeNeck", lathe(cylinder_profile(0.011, h + 0.005, neck_top), 32),
+ mat_steel, shade_mat)
+
+ # bulb socket inside the shade
+ add_object("Socket", lathe(cylinder_profile(0.013, h - 0.045, h - 0.002), 32),
+ mat_plastic, shade_mat)
+
+ # bulb
+ add_object("Bulb", lathe(sphere_profile(0.026, h - 0.067), 32), mat_bulb, shade_mat)
+
+ # ---- small switch on the base
+ prof = [(0.0, 0.0), (0.007, 0.0), (0.007, 0.006), (0.005, 0.008), (0.0, 0.008)]
+ add_object("Switch", lathe(prof, 32), mat_plastic,
+ Matrix.Translation(Vector((0.05, -0.045, H - 0.001))))
+
+
+main()
+```
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/result.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/result.json
new file mode 100644
index 00000000..ad294b19
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/result.json
@@ -0,0 +1,19 @@
+{
+ "ok": true,
+ "text": "```python\nimport bpy\nimport bmesh\nimport math\nfrom mathutils import Vector, Matrix\n\nSEG = 64\n\n\n# ---------------------------------------------------------------- materials\ndef make_material(name, color, roughness=0.5, metallic=0.0, emission=None, emission_strength=0.0):\n mat = bpy.data.materials.new(name)\n mat.diffuse_color = (color[0], color[1], color[2], 1.0)\n mat.roughness = roughness\n mat.metallic = metallic\n try:\n mat.use_nodes = True\n nt = mat.node_tree\n bsdf = None\n for n in nt.nodes:\n if n.type == 'BSDF_PRINCIPLED':\n bsdf = n\n break\n if bsdf is None:\n bsdf = nt.nodes.new('ShaderNodeBsdfPrincipled')\n out = None\n for n in nt.nodes:\n if n.type == 'OUTPUT_MATERIAL':\n out = n\n break\n if out is None:\n out = nt.nodes.new('ShaderNodeOutputMaterial')\n nt.links.new(bsdf.outputs['BSDF'], out.inputs['Surface'])\n bsdf.inputs['Base Color'].default_value = (color[0], color[1], color[2], 1.0)\n bsdf.inputs['Roughness'].default_value = roughness\n bsdf.inputs['Metallic'].default_value = metallic\n if emission is not None:\n try:\n bsdf.inputs['Emission Color'].default_value = (emission[0], emission[1], emission[2], 1.0)\n bsdf.inputs['Emission Strength'].default_value = emission_strength\n except Exception:\n pass\n except Exception:\n pass\n return mat\n\n\n# ---------------------------------------------------------------- geometry helpers\ndef lathe(profile, segments=SEG, closed=False, sharp_angle=35.0):\n \"\"\"Revolve a list of (radius, z) points around the local Z axis.\n radius <= 0 gives a pole vertex. closed=True links last ring back to first.\"\"\"\n bm = bmesh.new()\n rings = []\n for (r, z) in profile:\n if r <= 1e-6:\n rings.append([bm.verts.new((0.0, 0.0, z))])\n else:\n ring = []\n for i in range(segments):\n a = 2.0 * math.pi * i / segments\n ring.append(bm.verts.new((r * math.cos(a), r * math.sin(a), z)))\n rings.append(ring)\n n = len(rings)\n pairs = [(k, k + 1) for k in range(n - 1)]\n if closed:\n pairs.append((n - 1, 0))\n for (ka, kb) in pairs:\n A, B = rings[ka], rings[kb]\n if len(A) == 1 and len(B) == 1:\n continue\n for i in range(segments):\n i2 = (i + 1) % segments\n if len(A) == 1:\n bm.faces.new((A[0], B[i], B[i2]))\n elif len(B) == 1:\n bm.faces.new((A[i], A[i2], B[0]))\n else:\n bm.faces.new((A[i], A[i2], B[i2], B[i]))\n bm.normal_update()\n bmesh.ops.recalc_face_normals(bm, faces=list(bm.faces))\n bm.normal_update()\n thr = math.radians(sharp_angle)\n for f in bm.faces:\n f.smooth = True\n for e in bm.edges:\n ang = e.calc_face_angle(0.0)\n e.smooth = ang < thr\n return bm\n\n\ndef sphere_profile(r, cz, n=18):\n pts = []\n for j in range(n + 1):\n phi = math.pi * j / n\n pts.append((r * math.sin(phi), cz - r * math.cos(phi)))\n return pts\n\n\ndef cylinder_profile(r, z0, z1):\n return [(0.0, z0), (r, z0), (r, z1), (0.0, z1)]\n\n\ndef add_object(name, bm, mat, matrix=None):\n mesh = bpy.data.meshes.new(name)\n bm.to_mesh(mesh)\n bm.free()\n mesh.update()\n obj = bpy.data.objects.new(name, mesh)\n bpy.context.scene.collection.objects.link(obj)\n if matrix is not None:\n obj.matrix_world = matrix\n mesh.materials.append(mat)\n return obj\n\n\n# ---------------------------------------------------------------- build\ndef main():\n mat_dark = make_material(\"LampDarkMetal\", (0.07, 0.07, 0.08), 0.35, 1.0)\n mat_steel = make_material(\"LampBrushedSteel\", (0.72, 0.72, 0.74), 0.45, 1.0)\n mat_plastic = make_material(\"LampBlackPlastic\", (0.02, 0.02, 0.02), 0.6, 0.0)\n mat_bulb = make_material(\"LampBulb\", (1.0, 0.93, 0.8), 0.3, 0.0,\n emission=(1.0, 0.9, 0.7), emission_strength=3.0)\n\n # ---- heavy round base on the ground (z = 0), centred on Z axis\n R, H, b = 0.095, 0.032, 0.009\n prof = [(0.0, 0.0), (R, 0.0), (R, H - b)]\n for k in range(1, 6):\n ang = (math.pi / 2.0) * k / 5.0\n prof.append((R - b + b * math.cos(ang), H - b + b * math.sin(ang)))\n prof.append((0.0, H))\n add_object(\"Base\", lathe(prof, SEG), mat_dark)\n\n # ---- collar on top of the base where the arm is mounted\n prof = [(0.0, H - 0.002), (0.021, H - 0.002), (0.021, H + 0.020),\n (0.017, H + 0.024), (0.0, H + 0.024)]\n add_object(\"BaseCollar\", lathe(prof, 48), mat_dark)\n\n # ---- arm joint (ball) on the collar\n P0 = Vector((0.0, 0.0, 0.060))\n add_object(\"ArmJoint\", lathe(sphere_profile(0.019, 0.0), 32), mat_dark,\n Matrix.Translation(P0))\n\n # ---- single straight arm leaning forward (toward -Y)\n tilt = math.radians(30.0)\n L = 0.42\n d = Vector((0.0, -math.sin(tilt), math.cos(tilt)))\n arm_mat = Matrix.Translation(P0) @ Matrix.Rotation(tilt, 4, 'X')\n add_object(\"Arm\", lathe(cylinder_profile(0.009, -0.005, L), 32), mat_dark, arm_mat)\n P1 = P0 + d * L\n\n # ---- joint ball at the top of the arm\n add_object(\"ShadeJoint\", lathe(sphere_profile(0.017, 0.0), 32), mat_dark,\n Matrix.Translation(P1))\n\n # ---- shade frame: axis points up & slightly back, so the opening faces down/forward\n a = math.radians(12.0)\n axis = Vector((0.0, math.sin(a), math.cos(a)))\n Rs, rt, h, t = 0.10, 0.03, 0.14, 0.003\n neck_top = h + 0.032\n B = P1 - axis * neck_top\n shade_mat = Matrix.Translation(B) @ Matrix.Rotation(-a, 4, 'X')\n\n # conical metal shade, hollow shell with thickness, opening at local z=0 (downward)\n prof = [(Rs, 0.0), (rt, h), (rt - t, h), (Rs - t, 0.0)]\n add_object(\"Shade\", lathe(prof, SEG, closed=True), mat_steel, shade_mat)\n\n # cap closing the top of the cone\n prof = [(0.0, h - 0.004), (0.034, h - 0.004), (0.033, h + 0.010),\n (0.028, h + 0.014), (0.0, h + 0.014)]\n add_object(\"ShadeCap\", lathe(prof, 48), mat_steel, shade_mat)\n\n # neck between cap and the arm joint\n add_object(\"ShadeNeck\", lathe(cylinder_profile(0.011, h + 0.005, neck_top), 32),\n mat_steel, shade_mat)\n\n # bulb socket inside the shade\n add_object(\"Socket\", lathe(cylinder_profile(0.013, h - 0.045, h - 0.002), 32),\n mat_plastic, shade_mat)\n\n # bulb\n add_object(\"Bulb\", lathe(sphere_profile(0.026, h - 0.067), 32), mat_bulb, shade_mat)\n\n # ---- small switch on the base\n prof = [(0.0, 0.0), (0.007, 0.0), (0.007, 0.006), (0.005, 0.008), (0.0, 0.008)]\n add_object(\"Switch\", lathe(prof, 32), mat_plastic,\n Matrix.Translation(Vector((0.05, -0.045, H - 0.001))))\n\n\nmain()\n```",
+ "usage": {
+ "backend": "claude-code",
+ "model": "claude-haiku-4-5-20251001",
+ "input_tokens": 2,
+ "output_tokens": 21522,
+ "cached_tokens": 0,
+ "thoughts_tokens": 18336,
+ "tool_calls": 0,
+ "cost_usd": 1.16671,
+ "latency_ms": 239666
+ },
+ "tool_calls": 0,
+ "notes": "",
+ "duration_s": 241.05,
+ "transcript_dir": "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/stderr.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/stderr.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/stdout.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/stdout.txt
new file mode 100644
index 00000000..b60d09d6
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/gen/attempt0/stdout.txt
@@ -0,0 +1 @@
+{"is_error":false,"duration_api_ms":240626,"num_turns":1,"stop_reason":"end_turn","session_id":"c360a9ba-7393-489f-afaa-95862bf857cc","total_cost_usd":1.16671,"usage":{"input_tokens":2,"cache_creation_input_tokens":4453,"cache_read_input_tokens":0,"output_tokens":21522,"output_tokens_details":{"thinking_tokens":18336},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":4453,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":21522,"cache_read_input_tokens":0,"cache_creation_input_tokens":4453,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":4453},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-haiku-4-5-20251001":{"inputTokens":1455,"outputTokens":15,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.00153,"contextWindow":200000,"maxOutputTokens":32000,"canonicalModel":"claude-haiku-4-5","provider":"firstParty"},"claude-fable-5":{"inputTokens":2,"outputTokens":21522,"cacheReadInputTokens":0,"cacheCreationInputTokens":4453,"webSearchRequests":0,"costUSD":1.1651799999999999,"contextWindow":1000000,"maxOutputTokens":64000,"canonicalModel":"claude-fable-5","provider":"firstParty"}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","fast_mode_disabled_reason":"sdk_opt_in_required","subagent_stats":{"spawned":0,"requested":{"background":0,"foreground":0,"unset":0},"started_in_background":0,"max_depth":0,"spawned_by_subagents":0,"completed":0,"failed":0,"killed":{"parent":0,"user":0,"system":0},"refused":{"depth_limit":0,"concurrency_limit":0,"budget":0},"by_type":{}},"subtype":"success","api_error_status":null,"result":"```python\nimport bpy\nimport bmesh\nimport math\nfrom mathutils import Vector, Matrix\n\nSEG = 64\n\n\n# ---------------------------------------------------------------- materials\ndef make_material(name, color, roughness=0.5, metallic=0.0, emission=None, emission_strength=0.0):\n mat = bpy.data.materials.new(name)\n mat.diffuse_color = (color[0], color[1], color[2], 1.0)\n mat.roughness = roughness\n mat.metallic = metallic\n try:\n mat.use_nodes = True\n nt = mat.node_tree\n bsdf = None\n for n in nt.nodes:\n if n.type == 'BSDF_PRINCIPLED':\n bsdf = n\n break\n if bsdf is None:\n bsdf = nt.nodes.new('ShaderNodeBsdfPrincipled')\n out = None\n for n in nt.nodes:\n if n.type == 'OUTPUT_MATERIAL':\n out = n\n break\n if out is None:\n out = nt.nodes.new('ShaderNodeOutputMaterial')\n nt.links.new(bsdf.outputs['BSDF'], out.inputs['Surface'])\n bsdf.inputs['Base Color'].default_value = (color[0], color[1], color[2], 1.0)\n bsdf.inputs['Roughness'].default_value = roughness\n bsdf.inputs['Metallic'].default_value = metallic\n if emission is not None:\n try:\n bsdf.inputs['Emission Color'].default_value = (emission[0], emission[1], emission[2], 1.0)\n bsdf.inputs['Emission Strength'].default_value = emission_strength\n except Exception:\n pass\n except Exception:\n pass\n return mat\n\n\n# ---------------------------------------------------------------- geometry helpers\ndef lathe(profile, segments=SEG, closed=False, sharp_angle=35.0):\n \"\"\"Revolve a list of (radius, z) points around the local Z axis.\n radius <= 0 gives a pole vertex. closed=True links last ring back to first.\"\"\"\n bm = bmesh.new()\n rings = []\n for (r, z) in profile:\n if r <= 1e-6:\n rings.append([bm.verts.new((0.0, 0.0, z))])\n else:\n ring = []\n for i in range(segments):\n a = 2.0 * math.pi * i / segments\n ring.append(bm.verts.new((r * math.cos(a), r * math.sin(a), z)))\n rings.append(ring)\n n = len(rings)\n pairs = [(k, k + 1) for k in range(n - 1)]\n if closed:\n pairs.append((n - 1, 0))\n for (ka, kb) in pairs:\n A, B = rings[ka], rings[kb]\n if len(A) == 1 and len(B) == 1:\n continue\n for i in range(segments):\n i2 = (i + 1) % segments\n if len(A) == 1:\n bm.faces.new((A[0], B[i], B[i2]))\n elif len(B) == 1:\n bm.faces.new((A[i], A[i2], B[0]))\n else:\n bm.faces.new((A[i], A[i2], B[i2], B[i]))\n bm.normal_update()\n bmesh.ops.recalc_face_normals(bm, faces=list(bm.faces))\n bm.normal_update()\n thr = math.radians(sharp_angle)\n for f in bm.faces:\n f.smooth = True\n for e in bm.edges:\n ang = e.calc_face_angle(0.0)\n e.smooth = ang < thr\n return bm\n\n\ndef sphere_profile(r, cz, n=18):\n pts = []\n for j in range(n + 1):\n phi = math.pi * j / n\n pts.append((r * math.sin(phi), cz - r * math.cos(phi)))\n return pts\n\n\ndef cylinder_profile(r, z0, z1):\n return [(0.0, z0), (r, z0), (r, z1), (0.0, z1)]\n\n\ndef add_object(name, bm, mat, matrix=None):\n mesh = bpy.data.meshes.new(name)\n bm.to_mesh(mesh)\n bm.free()\n mesh.update()\n obj = bpy.data.objects.new(name, mesh)\n bpy.context.scene.collection.objects.link(obj)\n if matrix is not None:\n obj.matrix_world = matrix\n mesh.materials.append(mat)\n return obj\n\n\n# ---------------------------------------------------------------- build\ndef main():\n mat_dark = make_material(\"LampDarkMetal\", (0.07, 0.07, 0.08), 0.35, 1.0)\n mat_steel = make_material(\"LampBrushedSteel\", (0.72, 0.72, 0.74), 0.45, 1.0)\n mat_plastic = make_material(\"LampBlackPlastic\", (0.02, 0.02, 0.02), 0.6, 0.0)\n mat_bulb = make_material(\"LampBulb\", (1.0, 0.93, 0.8), 0.3, 0.0,\n emission=(1.0, 0.9, 0.7), emission_strength=3.0)\n\n # ---- heavy round base on the ground (z = 0), centred on Z axis\n R, H, b = 0.095, 0.032, 0.009\n prof = [(0.0, 0.0), (R, 0.0), (R, H - b)]\n for k in range(1, 6):\n ang = (math.pi / 2.0) * k / 5.0\n prof.append((R - b + b * math.cos(ang), H - b + b * math.sin(ang)))\n prof.append((0.0, H))\n add_object(\"Base\", lathe(prof, SEG), mat_dark)\n\n # ---- collar on top of the base where the arm is mounted\n prof = [(0.0, H - 0.002), (0.021, H - 0.002), (0.021, H + 0.020),\n (0.017, H + 0.024), (0.0, H + 0.024)]\n add_object(\"BaseCollar\", lathe(prof, 48), mat_dark)\n\n # ---- arm joint (ball) on the collar\n P0 = Vector((0.0, 0.0, 0.060))\n add_object(\"ArmJoint\", lathe(sphere_profile(0.019, 0.0), 32), mat_dark,\n Matrix.Translation(P0))\n\n # ---- single straight arm leaning forward (toward -Y)\n tilt = math.radians(30.0)\n L = 0.42\n d = Vector((0.0, -math.sin(tilt), math.cos(tilt)))\n arm_mat = Matrix.Translation(P0) @ Matrix.Rotation(tilt, 4, 'X')\n add_object(\"Arm\", lathe(cylinder_profile(0.009, -0.005, L), 32), mat_dark, arm_mat)\n P1 = P0 + d * L\n\n # ---- joint ball at the top of the arm\n add_object(\"ShadeJoint\", lathe(sphere_profile(0.017, 0.0), 32), mat_dark,\n Matrix.Translation(P1))\n\n # ---- shade frame: axis points up & slightly back, so the opening faces down/forward\n a = math.radians(12.0)\n axis = Vector((0.0, math.sin(a), math.cos(a)))\n Rs, rt, h, t = 0.10, 0.03, 0.14, 0.003\n neck_top = h + 0.032\n B = P1 - axis * neck_top\n shade_mat = Matrix.Translation(B) @ Matrix.Rotation(-a, 4, 'X')\n\n # conical metal shade, hollow shell with thickness, opening at local z=0 (downward)\n prof = [(Rs, 0.0), (rt, h), (rt - t, h), (Rs - t, 0.0)]\n add_object(\"Shade\", lathe(prof, SEG, closed=True), mat_steel, shade_mat)\n\n # cap closing the top of the cone\n prof = [(0.0, h - 0.004), (0.034, h - 0.004), (0.033, h + 0.010),\n (0.028, h + 0.014), (0.0, h + 0.014)]\n add_object(\"ShadeCap\", lathe(prof, 48), mat_steel, shade_mat)\n\n # neck between cap and the arm joint\n add_object(\"ShadeNeck\", lathe(cylinder_profile(0.011, h + 0.005, neck_top), 32),\n mat_steel, shade_mat)\n\n # bulb socket inside the shade\n add_object(\"Socket\", lathe(cylinder_profile(0.013, h - 0.045, h - 0.002), 32),\n mat_plastic, shade_mat)\n\n # bulb\n add_object(\"Bulb\", lathe(sphere_profile(0.026, h - 0.067), 32), mat_bulb, shade_mat)\n\n # ---- small switch on the base\n prof = [(0.0, 0.0), (0.007, 0.0), (0.007, 0.006), (0.005, 0.008), (0.0, 0.008)]\n add_object(\"Switch\", lathe(prof, 32), mat_plastic,\n Matrix.Translation(Vector((0.05, -0.045, H - 0.001))))\n\n\nmain()\n```","ttft_ms":214999,"ttft_stream_ms":1138,"time_to_request_ms":32,"type":"result","duration_ms":239666,"uuid":"9c0d5db8-0f5b-427a-afa6-386cce3014f1"}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/cell.json
new file mode 100644
index 00000000..fc07b7e5
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/cell.json
@@ -0,0 +1,39 @@
+{
+ "prompt_id": "cmp_easy_desk_lamp",
+ "tier": "easy",
+ "arm": "oneshot:codex",
+ "kind": "oneshot",
+ "target": "codex",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "scored",
+ "score": 0.6675,
+ "score_std": 0.0105,
+ "passed": false,
+ "build_ok": true,
+ "gate_errors": [
+ "connectivity: 'AngledArm' and 'RoundBase' interpenetrate by ≈11.7 mm (3% of surface samples inside)"
+ ],
+ "tris": 1664,
+ "gen_cost_usd": 0.20707199999999998,
+ "judge_cost_usd": 0.093532,
+ "gen_seconds": 68.02,
+ "wall_s": 121.6,
+ "attempts": 1,
+ "tool_calls": 0,
+ "criteria": {
+ "intent_fidelity": 0.85,
+ "structure_plausibility": 0.75,
+ "geometry_detail": 0.675,
+ "proportions_scale": 0.75,
+ "assembly_fit": 0.5,
+ "materials": 0.8,
+ "craftsmanship_no_artifacts": 0.875
+ },
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/eval/artifacts/renders/r00/sheet.png",
+ "glb": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/eval/artifacts/object.glb",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex",
+ "error": ""
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/eval
new file mode 160000
index 00000000..b3621b77
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/eval
@@ -0,0 +1 @@
+Subproject commit b3621b77b7d951d629b03fd78206415dcc6cc91a
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/argv.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/argv.json
new file mode 100644
index 00000000..dc4eff74
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/argv.json
@@ -0,0 +1 @@
+["codex", "exec", "--json", "-C", "/tmp/cv3d_oneshot_codex__f6h054j", "--sandbox", "read-only", "--skip-git-repo-check", "--ephemeral", "--color", "never", "-o", "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/last_message.md", "-"]
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/prompt.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/prompt.md
new file mode 100644
index 00000000..ce4836b5
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/prompt.md
@@ -0,0 +1,26 @@
+Model this object in raw bpy: a desk lamp with a heavy round base, a single angled arm and a conical metal shade
+MUST HAVE: round base on the ground
+MUST HAVE: one straight arm leaning forward
+MUST HAVE: conical shade at the top of the arm
+MUST HAVE: shade opens downward
+
+You write ONE file, `src/model.py`, in raw Blender Python (bpy, Blender 5.x API).
+The harness runs it headless (`blender -b --factory-startup --python <wrapper> -- --script src/model.py`)
+inside an EMPTIED scene (no default cube / camera / light) and then exports the result itself.
+
+Frame, units and placement: Coordinate frame: Z is UP, -Y is the FRONT of the object (the face a viewer sees from the default front view), +X is the object's right. Units are meters. The object stands on the ground plane z=0 and its footprint is centred on the Z axis. Build at real-world size.
+
+Naming: one mesh object per part, `obj.name` set to a unique PascalCase part name (e.g. `Seat`,
+`LegFrontLeft`); no auto-suffixed duplicates like `Leg.001`. Link every object to
+`bpy.context.scene.collection`. Every visible mesh gets a material (Principled BSDF base colour /
+roughness / metallic).
+
+Do NOT: create cameras or lights, touch render/world settings, call `bpy.ops.render.*`,
+`bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the network, or
+import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Keep the total under
+500k triangles and finish in under 120 s.
+
+The script must build the whole object when executed top-to-bottom (call your `main()` at module
+level). Make sure the code runs without errors in headless Blender — there is no second chance.
+
+You have NO tools in this session: you cannot write files, run Blender or inspect anything — the code must appear in your reply. Reply with the COMPLETE contents of `src/model.py` as ONE ```python fenced code block and nothing else (no prose before or after, no partial snippets).
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/response.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/response.md
new file mode 100644
index 00000000..d9af06d2
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/response.md
@@ -0,0 +1,281 @@
+```python
+import bpy
+import math
+from mathutils import Vector
+
+
+def make_material(name, color, metallic, roughness):
+ material = bpy.data.materials.new(name=name)
+ material.diffuse_color = color
+ material.use_nodes = True
+
+ shader = material.node_tree.nodes.get("Principled BSDF")
+ shader.inputs["Base Color"].default_value = color
+ shader.inputs["Metallic"].default_value = metallic
+ shader.inputs["Roughness"].default_value = roughness
+ return material
+
+
+def create_mesh_object(name, vertices, faces, materials, material_indices=None,
+ smooth_faces=None):
+ mesh = bpy.data.meshes.new(name + "Mesh")
+ mesh.from_pydata(vertices, [], faces)
+ mesh.materials.clear()
+
+ for material in materials:
+ mesh.materials.append(material)
+
+ mesh.update()
+
+ obj = bpy.data.objects.new(name, mesh)
+ bpy.context.scene.collection.objects.link(obj)
+
+ if material_indices is not None:
+ for polygon, index in zip(mesh.polygons, material_indices):
+ polygon.material_index = index
+
+ if smooth_faces is not None:
+ for polygon, smooth in zip(mesh.polygons, smooth_faces):
+ polygon.use_smooth = smooth
+
+ return obj
+
+
+def create_rounded_base(material, segments=64):
+ profile = [
+ (0.140, 0.000),
+ (0.169, 0.005),
+ (0.183, 0.016),
+ (0.187, 0.031),
+ (0.181, 0.045),
+ (0.163, 0.057),
+ (0.128, 0.065),
+ (0.070, 0.069),
+ ]
+
+ vertices = []
+ for radius, z in profile:
+ for i in range(segments):
+ angle = 2.0 * math.pi * i / segments
+ vertices.append((
+ radius * math.cos(angle),
+ radius * math.sin(angle),
+ z,
+ ))
+
+ bottom_center = len(vertices)
+ vertices.append((0.0, 0.0, 0.0))
+ top_center = len(vertices)
+ vertices.append((0.0, 0.0, 0.070))
+
+ faces = []
+ smooth = []
+
+ for ring in range(len(profile) - 1):
+ lower = ring * segments
+ upper = (ring + 1) * segments
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((lower + i, lower + j, upper + j, upper + i))
+ smooth.append(True)
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((bottom_center, j, i))
+ smooth.append(False)
+
+ last_ring = (len(profile) - 1) * segments
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((top_center, last_ring + i, last_ring + j))
+ smooth.append(False)
+
+ return create_mesh_object(
+ "RoundBase",
+ vertices,
+ faces,
+ [material],
+ smooth_faces=smooth,
+ )
+
+
+def create_cylinder_between(name, start, end, radius, material, segments=32):
+ start = Vector(start)
+ end = Vector(end)
+ axis = end - start
+ direction = axis.normalized()
+
+ reference = Vector((0.0, 0.0, 1.0))
+ if abs(direction.dot(reference)) > 0.95:
+ reference = Vector((0.0, 1.0, 0.0))
+
+ basis_u = direction.cross(reference).normalized()
+ basis_v = direction.cross(basis_u).normalized()
+
+ vertices = []
+ for point in (start, end):
+ for i in range(segments):
+ angle = 2.0 * math.pi * i / segments
+ offset = radius * (
+ basis_u * math.cos(angle) + basis_v * math.sin(angle)
+ )
+ vertex = point + offset
+ vertices.append(tuple(vertex))
+
+ start_center = len(vertices)
+ vertices.append(tuple(start))
+ end_center = len(vertices)
+ vertices.append(tuple(end))
+
+ faces = []
+ smooth = []
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((i, j, segments + j, segments + i))
+ smooth.append(True)
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((start_center, j, i))
+ smooth.append(False)
+ faces.append((end_center, segments + i, segments + j))
+ smooth.append(False)
+
+ return create_mesh_object(
+ name,
+ vertices,
+ faces,
+ [material],
+ smooth_faces=smooth,
+ )
+
+
+def create_shade(outer_material, inner_material, segments=64):
+ center_x = 0.0
+ center_y = -0.105
+ top_z = 0.530
+ bottom_z = 0.335
+
+ outer_top_radius = 0.044
+ outer_bottom_radius = 0.137
+ inner_top_radius = 0.016
+ inner_bottom_radius = 0.130
+
+ vertices = []
+
+ rings = [
+ (outer_bottom_radius, bottom_z),
+ (outer_top_radius, top_z),
+ (inner_bottom_radius, bottom_z + 0.004),
+ (inner_top_radius, top_z - 0.004),
+ ]
+
+ for radius, z in rings:
+ for i in range(segments):
+ angle = 2.0 * math.pi * i / segments
+ vertices.append((
+ center_x + radius * math.cos(angle),
+ center_y + radius * math.sin(angle),
+ z,
+ ))
+
+ outer_bottom = 0
+ outer_top = segments
+ inner_bottom = segments * 2
+ inner_top = segments * 3
+
+ faces = []
+ material_indices = []
+ smooth = []
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((
+ outer_bottom + i,
+ outer_bottom + j,
+ outer_top + j,
+ outer_top + i,
+ ))
+ material_indices.append(0)
+ smooth.append(True)
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((
+ inner_bottom + i,
+ inner_top + i,
+ inner_top + j,
+ inner_bottom + j,
+ ))
+ material_indices.append(1)
+ smooth.append(True)
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((
+ outer_bottom + i,
+ inner_bottom + i,
+ inner_bottom + j,
+ outer_bottom + j,
+ ))
+ material_indices.append(0)
+ smooth.append(False)
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((
+ outer_top + i,
+ outer_top + j,
+ inner_top + j,
+ inner_top + i,
+ ))
+ material_indices.append(0)
+ smooth.append(False)
+
+ return create_mesh_object(
+ "ConicalShade",
+ vertices,
+ faces,
+ [outer_material, inner_material],
+ material_indices=material_indices,
+ smooth_faces=smooth,
+ )
+
+
+def main():
+ dark_green = make_material(
+ "DarkGreenEnamel",
+ (0.055, 0.145, 0.105, 1.0),
+ metallic=0.72,
+ roughness=0.24,
+ )
+ arm_metal = make_material(
+ "ArmMetal",
+ (0.105, 0.120, 0.105, 1.0),
+ metallic=0.88,
+ roughness=0.20,
+ )
+ shade_inner = make_material(
+ "ShadeReflectiveInterior",
+ (0.72, 0.68, 0.52, 1.0),
+ metallic=0.78,
+ roughness=0.25,
+ )
+
+ create_rounded_base(dark_green)
+
+ create_cylinder_between(
+ "AngledArm",
+ (0.0, 0.038, 0.061),
+ (0.0, -0.105, 0.540),
+ 0.0125,
+ arm_metal,
+ segments=32,
+ )
+
+ create_shade(dark_green, shade_inner)
+
+
+main()
+```
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/result.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/result.json
new file mode 100644
index 00000000..3ba0cec7
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/result.json
@@ -0,0 +1,19 @@
+{
+ "ok": true,
+ "text": "```python\nimport bpy\nimport math\nfrom mathutils import Vector\n\n\ndef make_material(name, color, metallic, roughness):\n material = bpy.data.materials.new(name=name)\n material.diffuse_color = color\n material.use_nodes = True\n\n shader = material.node_tree.nodes.get(\"Principled BSDF\")\n shader.inputs[\"Base Color\"].default_value = color\n shader.inputs[\"Metallic\"].default_value = metallic\n shader.inputs[\"Roughness\"].default_value = roughness\n return material\n\n\ndef create_mesh_object(name, vertices, faces, materials, material_indices=None,\n smooth_faces=None):\n mesh = bpy.data.meshes.new(name + \"Mesh\")\n mesh.from_pydata(vertices, [], faces)\n mesh.materials.clear()\n\n for material in materials:\n mesh.materials.append(material)\n\n mesh.update()\n\n obj = bpy.data.objects.new(name, mesh)\n bpy.context.scene.collection.objects.link(obj)\n\n if material_indices is not None:\n for polygon, index in zip(mesh.polygons, material_indices):\n polygon.material_index = index\n\n if smooth_faces is not None:\n for polygon, smooth in zip(mesh.polygons, smooth_faces):\n polygon.use_smooth = smooth\n\n return obj\n\n\ndef create_rounded_base(material, segments=64):\n profile = [\n (0.140, 0.000),\n (0.169, 0.005),\n (0.183, 0.016),\n (0.187, 0.031),\n (0.181, 0.045),\n (0.163, 0.057),\n (0.128, 0.065),\n (0.070, 0.069),\n ]\n\n vertices = []\n for radius, z in profile:\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n vertices.append((\n radius * math.cos(angle),\n radius * math.sin(angle),\n z,\n ))\n\n bottom_center = len(vertices)\n vertices.append((0.0, 0.0, 0.0))\n top_center = len(vertices)\n vertices.append((0.0, 0.0, 0.070))\n\n faces = []\n smooth = []\n\n for ring in range(len(profile) - 1):\n lower = ring * segments\n upper = (ring + 1) * segments\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((lower + i, lower + j, upper + j, upper + i))\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((bottom_center, j, i))\n smooth.append(False)\n\n last_ring = (len(profile) - 1) * segments\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((top_center, last_ring + i, last_ring + j))\n smooth.append(False)\n\n return create_mesh_object(\n \"RoundBase\",\n vertices,\n faces,\n [material],\n smooth_faces=smooth,\n )\n\n\ndef create_cylinder_between(name, start, end, radius, material, segments=32):\n start = Vector(start)\n end = Vector(end)\n axis = end - start\n direction = axis.normalized()\n\n reference = Vector((0.0, 0.0, 1.0))\n if abs(direction.dot(reference)) > 0.95:\n reference = Vector((0.0, 1.0, 0.0))\n\n basis_u = direction.cross(reference).normalized()\n basis_v = direction.cross(basis_u).normalized()\n\n vertices = []\n for point in (start, end):\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n offset = radius * (\n basis_u * math.cos(angle) + basis_v * math.sin(angle)\n )\n vertex = point + offset\n vertices.append(tuple(vertex))\n\n start_center = len(vertices)\n vertices.append(tuple(start))\n end_center = len(vertices)\n vertices.append(tuple(end))\n\n faces = []\n smooth = []\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((i, j, segments + j, segments + i))\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((start_center, j, i))\n smooth.append(False)\n faces.append((end_center, segments + i, segments + j))\n smooth.append(False)\n\n return create_mesh_object(\n name,\n vertices,\n faces,\n [material],\n smooth_faces=smooth,\n )\n\n\ndef create_shade(outer_material, inner_material, segments=64):\n center_x = 0.0\n center_y = -0.105\n top_z = 0.530\n bottom_z = 0.335\n\n outer_top_radius = 0.044\n outer_bottom_radius = 0.137\n inner_top_radius = 0.016\n inner_bottom_radius = 0.130\n\n vertices = []\n\n rings = [\n (outer_bottom_radius, bottom_z),\n (outer_top_radius, top_z),\n (inner_bottom_radius, bottom_z + 0.004),\n (inner_top_radius, top_z - 0.004),\n ]\n\n for radius, z in rings:\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n vertices.append((\n center_x + radius * math.cos(angle),\n center_y + radius * math.sin(angle),\n z,\n ))\n\n outer_bottom = 0\n outer_top = segments\n inner_bottom = segments * 2\n inner_top = segments * 3\n\n faces = []\n material_indices = []\n smooth = []\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n outer_bottom + i,\n outer_bottom + j,\n outer_top + j,\n outer_top + i,\n ))\n material_indices.append(0)\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n inner_bottom + i,\n inner_top + i,\n inner_top + j,\n inner_bottom + j,\n ))\n material_indices.append(1)\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n outer_bottom + i,\n inner_bottom + i,\n inner_bottom + j,\n outer_bottom + j,\n ))\n material_indices.append(0)\n smooth.append(False)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n outer_top + i,\n outer_top + j,\n inner_top + j,\n inner_top + i,\n ))\n material_indices.append(0)\n smooth.append(False)\n\n return create_mesh_object(\n \"ConicalShade\",\n vertices,\n faces,\n [outer_material, inner_material],\n material_indices=material_indices,\n smooth_faces=smooth,\n )\n\n\ndef main():\n dark_green = make_material(\n \"DarkGreenEnamel\",\n (0.055, 0.145, 0.105, 1.0),\n metallic=0.72,\n roughness=0.24,\n )\n arm_metal = make_material(\n \"ArmMetal\",\n (0.105, 0.120, 0.105, 1.0),\n metallic=0.88,\n roughness=0.20,\n )\n shade_inner = make_material(\n \"ShadeReflectiveInterior\",\n (0.72, 0.68, 0.52, 1.0),\n metallic=0.78,\n roughness=0.25,\n )\n\n create_rounded_base(dark_green)\n\n create_cylinder_between(\n \"AngledArm\",\n (0.0, 0.038, 0.061),\n (0.0, -0.105, 0.540),\n 0.0125,\n arm_metal,\n segments=32,\n )\n\n create_shade(dark_green, shade_inner)\n\n\nmain()\n```",
+ "usage": {
+ "backend": "codex",
+ "model": "gpt-5.6-sol",
+ "input_tokens": 14622,
+ "output_tokens": 3893,
+ "cached_tokens": 9984,
+ "thoughts_tokens": 2070,
+ "tool_calls": 0,
+ "cost_usd": 0.20707199999999998,
+ "latency_ms": 68015
+ },
+ "tool_calls": 0,
+ "notes": "",
+ "duration_s": 68.02,
+ "transcript_dir": "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/stderr.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/stderr.txt
new file mode 100644
index 00000000..cdcd5692
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/stderr.txt
@@ -0,0 +1 @@
+Failed to write last message file "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/last_message.md": No such file or directory (os error 2)
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/stdout.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/stdout.txt
new file mode 100644
index 00000000..75e96ed3
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/gen/attempt0/stdout.txt
@@ -0,0 +1,4 @@
+{"type":"thread.started","thread_id":"01a02f1f-33e0-71e0-bd24-eedf584cd32e"}
+{"type":"turn.started"}
+{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"```python\nimport bpy\nimport math\nfrom mathutils import Vector\n\n\ndef make_material(name, color, metallic, roughness):\n material = bpy.data.materials.new(name=name)\n material.diffuse_color = color\n material.use_nodes = True\n\n shader = material.node_tree.nodes.get(\"Principled BSDF\")\n shader.inputs[\"Base Color\"].default_value = color\n shader.inputs[\"Metallic\"].default_value = metallic\n shader.inputs[\"Roughness\"].default_value = roughness\n return material\n\n\ndef create_mesh_object(name, vertices, faces, materials, material_indices=None,\n smooth_faces=None):\n mesh = bpy.data.meshes.new(name + \"Mesh\")\n mesh.from_pydata(vertices, [], faces)\n mesh.materials.clear()\n\n for material in materials:\n mesh.materials.append(material)\n\n mesh.update()\n\n obj = bpy.data.objects.new(name, mesh)\n bpy.context.scene.collection.objects.link(obj)\n\n if material_indices is not None:\n for polygon, index in zip(mesh.polygons, material_indices):\n polygon.material_index = index\n\n if smooth_faces is not None:\n for polygon, smooth in zip(mesh.polygons, smooth_faces):\n polygon.use_smooth = smooth\n\n return obj\n\n\ndef create_rounded_base(material, segments=64):\n profile = [\n (0.140, 0.000),\n (0.169, 0.005),\n (0.183, 0.016),\n (0.187, 0.031),\n (0.181, 0.045),\n (0.163, 0.057),\n (0.128, 0.065),\n (0.070, 0.069),\n ]\n\n vertices = []\n for radius, z in profile:\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n vertices.append((\n radius * math.cos(angle),\n radius * math.sin(angle),\n z,\n ))\n\n bottom_center = len(vertices)\n vertices.append((0.0, 0.0, 0.0))\n top_center = len(vertices)\n vertices.append((0.0, 0.0, 0.070))\n\n faces = []\n smooth = []\n\n for ring in range(len(profile) - 1):\n lower = ring * segments\n upper = (ring + 1) * segments\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((lower + i, lower + j, upper + j, upper + i))\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((bottom_center, j, i))\n smooth.append(False)\n\n last_ring = (len(profile) - 1) * segments\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((top_center, last_ring + i, last_ring + j))\n smooth.append(False)\n\n return create_mesh_object(\n \"RoundBase\",\n vertices,\n faces,\n [material],\n smooth_faces=smooth,\n )\n\n\ndef create_cylinder_between(name, start, end, radius, material, segments=32):\n start = Vector(start)\n end = Vector(end)\n axis = end - start\n direction = axis.normalized()\n\n reference = Vector((0.0, 0.0, 1.0))\n if abs(direction.dot(reference)) > 0.95:\n reference = Vector((0.0, 1.0, 0.0))\n\n basis_u = direction.cross(reference).normalized()\n basis_v = direction.cross(basis_u).normalized()\n\n vertices = []\n for point in (start, end):\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n offset = radius * (\n basis_u * math.cos(angle) + basis_v * math.sin(angle)\n )\n vertex = point + offset\n vertices.append(tuple(vertex))\n\n start_center = len(vertices)\n vertices.append(tuple(start))\n end_center = len(vertices)\n vertices.append(tuple(end))\n\n faces = []\n smooth = []\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((i, j, segments + j, segments + i))\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((start_center, j, i))\n smooth.append(False)\n faces.append((end_center, segments + i, segments + j))\n smooth.append(False)\n\n return create_mesh_object(\n name,\n vertices,\n faces,\n [material],\n smooth_faces=smooth,\n )\n\n\ndef create_shade(outer_material, inner_material, segments=64):\n center_x = 0.0\n center_y = -0.105\n top_z = 0.530\n bottom_z = 0.335\n\n outer_top_radius = 0.044\n outer_bottom_radius = 0.137\n inner_top_radius = 0.016\n inner_bottom_radius = 0.130\n\n vertices = []\n\n rings = [\n (outer_bottom_radius, bottom_z),\n (outer_top_radius, top_z),\n (inner_bottom_radius, bottom_z + 0.004),\n (inner_top_radius, top_z - 0.004),\n ]\n\n for radius, z in rings:\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n vertices.append((\n center_x + radius * math.cos(angle),\n center_y + radius * math.sin(angle),\n z,\n ))\n\n outer_bottom = 0\n outer_top = segments\n inner_bottom = segments * 2\n inner_top = segments * 3\n\n faces = []\n material_indices = []\n smooth = []\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n outer_bottom + i,\n outer_bottom + j,\n outer_top + j,\n outer_top + i,\n ))\n material_indices.append(0)\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n inner_bottom + i,\n inner_top + i,\n inner_top + j,\n inner_bottom + j,\n ))\n material_indices.append(1)\n smooth.append(True)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n outer_bottom + i,\n inner_bottom + i,\n inner_bottom + j,\n outer_bottom + j,\n ))\n material_indices.append(0)\n smooth.append(False)\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n outer_top + i,\n outer_top + j,\n inner_top + j,\n inner_top + i,\n ))\n material_indices.append(0)\n smooth.append(False)\n\n return create_mesh_object(\n \"ConicalShade\",\n vertices,\n faces,\n [outer_material, inner_material],\n material_indices=material_indices,\n smooth_faces=smooth,\n )\n\n\ndef main():\n dark_green = make_material(\n \"DarkGreenEnamel\",\n (0.055, 0.145, 0.105, 1.0),\n metallic=0.72,\n roughness=0.24,\n )\n arm_metal = make_material(\n \"ArmMetal\",\n (0.105, 0.120, 0.105, 1.0),\n metallic=0.88,\n roughness=0.20,\n )\n shade_inner = make_material(\n \"ShadeReflectiveInterior\",\n (0.72, 0.68, 0.52, 1.0),\n metallic=0.78,\n roughness=0.25,\n )\n\n create_rounded_base(dark_green)\n\n create_cylinder_between(\n \"AngledArm\",\n (0.0, 0.038, 0.061),\n (0.0, -0.105, 0.540),\n 0.0125,\n arm_metal,\n segments=32,\n )\n\n create_shade(dark_green, shade_inner)\n\n\nmain()\n```"}}
+{"type":"turn.completed","usage":{"input_tokens":14622,"cached_input_tokens":9984,"cache_write_input_tokens":0,"output_tokens":3893,"reasoning_output_tokens":2070}}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/cell.json
new file mode 100644
index 00000000..7d5551c1
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/cell.json
@@ -0,0 +1,31 @@
+{
+ "prompt_id": "cmp_easy_desk_lamp",
+ "tier": "easy",
+ "arm": "oneshot:gemini:gemini-3.7-flash",
+ "kind": "oneshot",
+ "target": "gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "build_failed",
+ "score": 0.0,
+ "score_std": 0.0,
+ "passed": false,
+ "build_ok": false,
+ "gate_errors": [
+ "lint:blender: Principled BSDF input 'Emission' does not exist in Blender 4.x/5.x (KeyError at runtime)"
+ ],
+ "tris": null,
+ "gen_cost_usd": 0.0284595,
+ "judge_cost_usd": 0.0,
+ "gen_seconds": 55.18,
+ "wall_s": 55.6,
+ "attempts": 1,
+ "tool_calls": 0,
+ "criteria": {},
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "",
+ "glb": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/eval/artifacts/object.glb",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash",
+ "error": "AttributeError: BMeshOpsModule: operator \"create_cylinder\" doesn't exist"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/eval
new file mode 160000
index 00000000..0834ccf1
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/eval
@@ -0,0 +1 @@
+Subproject commit 0834ccf199859080867c4a5c824d079ea2b16f32
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/prompt.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/prompt.md
new file mode 100644
index 00000000..ce4836b5
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/prompt.md
@@ -0,0 +1,26 @@
+Model this object in raw bpy: a desk lamp with a heavy round base, a single angled arm and a conical metal shade
+MUST HAVE: round base on the ground
+MUST HAVE: one straight arm leaning forward
+MUST HAVE: conical shade at the top of the arm
+MUST HAVE: shade opens downward
+
+You write ONE file, `src/model.py`, in raw Blender Python (bpy, Blender 5.x API).
+The harness runs it headless (`blender -b --factory-startup --python <wrapper> -- --script src/model.py`)
+inside an EMPTIED scene (no default cube / camera / light) and then exports the result itself.
+
+Frame, units and placement: Coordinate frame: Z is UP, -Y is the FRONT of the object (the face a viewer sees from the default front view), +X is the object's right. Units are meters. The object stands on the ground plane z=0 and its footprint is centred on the Z axis. Build at real-world size.
+
+Naming: one mesh object per part, `obj.name` set to a unique PascalCase part name (e.g. `Seat`,
+`LegFrontLeft`); no auto-suffixed duplicates like `Leg.001`. Link every object to
+`bpy.context.scene.collection`. Every visible mesh gets a material (Principled BSDF base colour /
+roughness / metallic).
+
+Do NOT: create cameras or lights, touch render/world settings, call `bpy.ops.render.*`,
+`bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the network, or
+import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Keep the total under
+500k triangles and finish in under 120 s.
+
+The script must build the whole object when executed top-to-bottom (call your `main()` at module
+level). Make sure the code runs without errors in headless Blender — there is no second chance.
+
+You have NO tools in this session: you cannot write files, run Blender or inspect anything — the code must appear in your reply. Reply with the COMPLETE contents of `src/model.py` as ONE ```python fenced code block and nothing else (no prose before or after, no partial snippets).
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/response.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/response.md
new file mode 100644
index 00000000..505da3d3
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/response.md
@@ -0,0 +1,577 @@
+```python
+"""
+Desk Lamp Model
+Mid-century modern style desk lamp with a heavy round base,
+a single angled arm leaning forward, and a conical metal shade opening downward.
+"""
+
+import math
+import bmesh
+import bpy
+from mathutils import Matrix, Vector
+
+
+def create_material(name, base_color, metallic=0.0, roughness=0.3, emission_color=None, emission_strength=0.0):
+ """Creates a Principled BSDF material."""
+ mat = bpy.data.materials.new(name=name)
+ mat.use_nodes = True
+ nodes = mat.node_tree.nodes
+ links = mat.node_tree.links
+
+ nodes.clear()
+ node_out = nodes.new(type="ShaderNodeOutputMaterial")
+ node_bsdf = nodes.new(type="ShaderNodeBsdfPrincipled")
+
+ # Set Principled BSDF socket values
+ node_bsdf.inputs["Base Color"].default_value = base_color
+ node_bsdf.inputs["Metallic"].default_value = metallic
+ node_bsdf.inputs["Roughness"].default_value = roughness
+
+ # Emission handling for light bulb
+ if emission_color is not None and emission_strength > 0:
+ if "Emission Color" in node_bsdf.inputs:
+ node_bsdf.inputs["Emission Color"].default_value = emission_color
+ node_bsdf.inputs["Emission Strength"].default_value = emission_strength
+ elif "Emission" in node_bsdf.inputs:
+ node_bsdf.inputs["Emission"].default_value = emission_color
+
+ links.new(node_bsdf.outputs["BSDF"], node_out.inputs["Surface"])
+ return mat
+
+
+def set_smooth(mesh):
+ """Sets smooth shading on all faces of a mesh."""
+ for poly in mesh.polygons:
+ poly.use_smooth = True
+
+
+def make_base(materials):
+ """Creates the heavy circular weighted base with a bottom felt pad."""
+ # 1. Base Body
+ bm = bmesh.new()
+ # Create stepped profile for heavy cast look
+ segments = 48
+ r_outer = 0.095
+ r_top = 0.088
+ h_total = 0.022
+ h_step = 0.005
+
+ # Bottom ring
+ v_bot_center = bm.verts.new((0, 0, 0.002))
+ v_bot_rim = [
+ bm.verts.new(
+ (
+ r_outer * math.cos(i * 2 * math.pi / segments),
+ r_outer * math.sin(i * 2 * math.pi / segments),
+ 0.002,
+ )
+ )
+ for i in range(segments)
+ ]
+ for i in range(segments):
+ bm.faces.new(
+ (
+ v_bot_center,
+ v_bot_rim[(i + 1) % segments],
+ v_bot_rim[i],
+ )
+ )
+
+ # Lower chamfer
+ v_mid_rim = [
+ bm.verts.new(
+ (
+ r_outer * math.cos(i * 2 * math.pi / segments),
+ r_outer * math.sin(i * 2 * math.pi / segments),
+ h_step,
+ )
+ )
+ for i in range(segments)
+ ]
+ for i in range(segments):
+ bm.faces.new(
+ (
+ v_bot_rim[i],
+ v_bot_rim[(i + 1) % segments],
+ v_mid_rim[(i + 1) % segments],
+ v_mid_rim[i],
+ )
+ )
+
+ # Top outer rim
+ v_top_outer = [
+ bm.verts.new(
+ (
+ r_outer * math.cos(i * 2 * math.pi / segments),
+ r_outer * math.sin(i * 2 * math.pi / segments),
+ h_total - 0.003,
+ )
+ )
+ for i in range(segments)
+ ]
+ for i in range(segments):
+ bm.faces.new(
+ (
+ v_mid_rim[i],
+ v_mid_rim[(i + 1) % segments],
+ v_top_outer[(i + 1) % segments],
+ v_top_outer[i],
+ )
+ )
+
+ # Top chamfer rim
+ v_top_inner = [
+ bm.verts.new(
+ (
+ r_top * math.cos(i * 2 * math.pi / segments),
+ r_top * math.sin(i * 2 * math.pi / segments),
+ h_total,
+ )
+ )
+ for i in range(segments)
+ ]
+ for i in range(segments):
+ bm.faces.new(
+ (
+ v_top_outer[i],
+ v_top_outer[(i + 1) % segments],
+ v_top_inner[(i + 1) % segments],
+ v_top_inner[i],
+ )
+ )
+
+ # Top flat cap
+ v_top_center = bm.verts.new((0, 0, h_total))
+ for i in range(segments):
+ bm.faces.new(
+ (
+ v_top_center,
+ v_top_inner[i],
+ v_top_inner[(i + 1) % segments],
+ )
+ )
+
+ bm.normal_update()
+ mesh = bpy.data.meshes.new("LampBase")
+ bm.to_mesh(mesh)
+ bm.free()
+ set_smooth(mesh)
+
+ base_obj = bpy.data.objects.new("LampBase", mesh)
+ base_obj.data.materials.append(materials["matte_black"])
+ bpy.context.scene.collection.objects.link(base_obj)
+
+ # 2. Bottom Felt Pad (visible edge resting on z=0)
+ bm_pad = bmesh.new()
+ bmesh.ops.create_cylinder(
+ bm_pad,
+ cap_ends=True,
+ radius=0.093,
+ depth=0.002,
+ segments=48,
+ matrix=Matrix.Translation((0, 0, 0.001)),
+ )
+ mesh_pad = bpy.data.meshes.new("BasePad")
+ bm_pad.to_mesh(mesh_pad)
+ bm_pad.free()
+ set_smooth(mesh_pad)
+
+ pad_obj = bpy.data.objects.new("BasePad", mesh_pad)
+ pad_obj.data.materials.append(materials["rubber"])
+ bpy.context.scene.collection.objects.link(pad_obj)
+
+ return h_total
+
+
+def make_switch(materials, z_pos):
+ """Creates a brass push/rotary switch button on the base."""
+ bm = bmesh.new()
+ # Switch bezel collar
+ bmesh.ops.create_cylinder(
+ bm,
+ cap_ends=True,
+ radius=0.006,
+ depth=0.004,
+ segments=24,
+ matrix=Matrix.Translation((0, -0.052, z_pos + 0.002)),
+ )
+ # Switch button toggle
+ bmesh.ops.create_cylinder(
+ bm,
+ cap_ends=True,
+ radius=0.0035,
+ depth=0.008,
+ segments=24,
+ matrix=Matrix.Translation((0, -0.052, z_pos + 0.006)),
+ )
+ bm.normal_update()
+ mesh = bpy.data.meshes.new("BaseSwitch")
+ bm.to_mesh(mesh)
+ bm.free()
+ set_smooth(mesh)
+
+ obj = bpy.data.objects.new("BaseSwitch", mesh)
+ obj.data.materials.append(materials["brass"])
+ bpy.context.scene.collection.objects.link(obj)
+
+
+def make_arm_and_mount(materials, base_top_z):
+ """Creates the base pivot mount, the single forward-angled arm, and the top joint."""
+ arm_base_pos = Vector((0.0, 0.035, base_top_z))
+
+ # 1. Base Arm Mount (Brass Collar & Bracket)
+ bm_mount = bmesh.new()
+ # Bottom mount cylinder
+ bmesh.ops.create_cylinder(
+ bm_mount,
+ cap_ends=True,
+ radius=0.014,
+ depth=0.012,
+ segments=32,
+ matrix=Matrix.Translation((0.0, 0.035, base_top_z + 0.006)),
+ )
+ # Pivot fork / bracket
+ bmesh.ops.create_cube(
+ bm_mount,
+ size=1.0,
+ matrix=Matrix.Translation((0.0, 0.035, base_top_z + 0.017))
+ @ Matrix.Scale(0.016, 4, (1, 0, 0))
+ @ Matrix.Scale(0.012, 4, (0, 1, 0))
+ @ Matrix.Scale(0.014, 4, (0, 0, 1)),
+ )
+ # Pivot horizontal pin
+ bmesh.ops.create_cylinder(
+ bm_mount,
+ cap_ends=True,
+ radius=0.004,
+ depth=0.022,
+ segments=16,
+ matrix=Matrix.Translation((0.0, 0.035, base_top_z + 0.018))
+ @ Matrix.Rotation(math.radians(90), 4, "Y"),
+ )
+ bm_mount.normal_update()
+ mesh_mount = bpy.data.meshes.new("ArmBaseMount")
+ bm_mount.to_mesh(mesh_mount)
+ bm_mount.free()
+ set_smooth(mesh_mount)
+
+ mount_obj = bpy.data.objects.new("ArmBaseMount", mesh_mount)
+ mount_obj.data.materials.append(materials["brass"])
+ bpy.context.scene.collection.objects.link(mount_obj)
+
+ # 2. Main Straight Arm (Leaning forward into -Y)
+ # Pivot origin at base mount
+ pivot_z = base_top_z + 0.018
+ arm_start = Vector((0.0, 0.035, pivot_z))
+ arm_length = 0.44
+ tilt_angle = math.radians(24.0) # Leaning forward towards -Y
+
+ # Direction vector: leans forward (-Y) and up (+Z)
+ dir_vec = Vector((0.0, -math.sin(tilt_angle), math.cos(tilt_angle)))
+ arm_end = arm_start + dir_vec * arm_length
+ arm_center = (arm_start + arm_end) * 0.5
+
+ # Rotation matrix aligning cylinder (initially along Z) with dir_vec
+ rot_arm = Matrix.Rotation(-tilt_angle, 4, "X")
+
+ bm_arm = bmesh.new()
+ bmesh.ops.create_cylinder(
+ bm_arm,
+ cap_ends=True,
+ radius=0.006,
+ depth=arm_length,
+ segments=28,
+ matrix=Matrix.Translation(arm_center) @ rot_arm,
+ )
+ bm_arm.normal_update()
+ mesh_arm = bpy.data.meshes.new("LampArm")
+ bm_arm.to_mesh(mesh_arm)
+ bm_arm.free()
+ set_smooth(mesh_arm)
+
+ arm_obj = bpy.data.objects.new("LampArm", mesh_arm)
+ arm_obj.data.materials.append(materials["matte_black"])
+ bpy.context.scene.collection.objects.link(arm_obj)
+
+ # 3. Top Joint (Brass Swivel Fitting)
+ bm_joint = bmesh.new()
+ # Spherical / cylindrical knuckle at top of the arm
+ bmesh.ops.create_uvsphere(
+ bm_joint,
+ u_segments=24,
+ v_segments=16,
+ radius=0.011,
+ matrix=Matrix.Translation(arm_end),
+ )
+ # Locking screw / wing knob on side (+X)
+ bmesh.ops.create_cylinder(
+ bm_joint,
+ cap_ends=True,
+ radius=0.0035,
+ depth=0.028,
+ segments=16,
+ matrix=Matrix.Translation(arm_end) @ Matrix.Rotation(math.radians(90), 4, "Y"),
+ )
+ bmesh.ops.create_cylinder(
+ bm_joint,
+ cap_ends=True,
+ radius=0.007,
+ depth=0.004,
+ segments=16,
+ matrix=Matrix.Translation(arm_end + Vector((0.014, 0, 0)))
+ @ Matrix.Rotation(math.radians(90), 4, "Y"),
+ )
+ bm_joint.normal_update()
+ mesh_joint = bpy.data.meshes.new("UpperJoint")
+ bm_joint.to_mesh(mesh_joint)
+ bm_joint.free()
+ set_smooth(mesh_joint)
+
+ joint_obj = bpy.data.objects.new("UpperJoint", mesh_joint)
+ joint_obj.data.materials.append(materials["brass"])
+ bpy.context.scene.collection.objects.link(joint_obj)
+
+ return arm_end
+
+
+def make_shade_and_bulb(materials, top_pos):
+ """Creates the conical metal shade opening downwards and the inner light bulb."""
+ # Shade orientation: tilted slightly forward from straight down (e.g. 15 degrees)
+ shade_tilt = math.radians(12.0)
+ rot_shade = Matrix.Rotation(shade_tilt, 4, "X")
+
+ # Downward axis vector for shade
+ shade_dir = Vector((0.0, -math.sin(shade_tilt), -math.cos(shade_tilt)))
+
+ # 1. Brass Shade Socket / Neck attached to joint
+ socket_length = 0.025
+ socket_pos = top_pos + shade_dir * (socket_length * 0.5)
+
+ bm_socket = bmesh.new()
+ bmesh.ops.create_cylinder(
+ bm_socket,
+ cap_ends=True,
+ radius=0.010,
+ depth=socket_length,
+ segments=24,
+ matrix=Matrix.Translation(socket_pos) @ rot_shade,
+ )
+ bm_socket.normal_update()
+ mesh_socket = bpy.data.meshes.new("ShadeSocket")
+ bm_socket.to_mesh(mesh_socket)
+ bm_socket.free()
+ set_smooth(mesh_socket)
+
+ socket_obj = bpy.data.objects.new("ShadeSocket", mesh_socket)
+ socket_obj.data.materials.append(materials["brass"])
+ bpy.context.scene.collection.objects.link(socket_obj)
+
+ # 2. Conical Shade
+ # Top of cone attaches at socket end
+ shade_top_center = top_pos + shade_dir * socket_length
+ shade_height = 0.135
+ r_top = 0.028
+ r_bottom = 0.092
+ segments = 48
+
+ # Local coordinates along shade axis (Z from 0 to -shade_height)
+ bm_shade = bmesh.new()
+
+ # Outer cone vertices
+ v_outer_top = [
+ bm_shade.verts.new(
+ (
+ r_top * math.cos(i * 2 * math.pi / segments),
+ r_top * math.sin(i * 2 * math.pi / segments),
+ 0.0,
+ )
+ )
+ for i in range(segments)
+ ]
+ v_outer_bot = [
+ bm_shade.verts.new(
+ (
+ r_bottom * math.cos(i * 2 * math.pi / segments),
+ r_bottom * math.sin(i * 2 * math.pi / segments),
+ -shade_height,
+ )
+ )
+ for i in range(segments)
+ ]
+
+ # Inner cone vertices (for realistic thickness and white interior)
+ r_top_in = r_top - 0.0025
+ r_bot_in = r_bottom - 0.0025
+ v_inner_top = [
+ bm_shade.verts.new(
+ (
+ r_top_in * math.cos(i * 2 * math.pi / segments),
+ r_top_in * math.sin(i * 2 * math.pi / segments),
+ -0.003,
+ )
+ )
+ for i in range(segments)
+ ]
+ v_inner_bot = [
+ bm_shade.verts.new(
+ (
+ r_bot_in * math.cos(i * 2 * math.pi / segments),
+ r_bot_in * math.sin(i * 2 * math.pi / segments),
+ -shade_height + 0.001,
+ )
+ )
+ for i in range(segments)
+ ]
+
+ # Outer wall faces
+ for i in range(segments):
+ bm_shade.faces.new(
+ (
+ v_outer_top[i],
+ v_outer_top[(i + 1) % segments],
+ v_outer_bot[(i + 1) % segments],
+ v_outer_bot[i],
+ )
+ )
+
+ # Bottom rim lip faces (joining outer and inner)
+ for i in range(segments):
+ bm_shade.faces.new(
+ (
+ v_outer_bot[i],
+ v_outer_bot[(i + 1) % segments],
+ v_inner_bot[(i + 1) % segments],
+ v_inner_bot[i],
+ )
+ )
+
+ # Inner wall faces
+ for i in range(segments):
+ bm_shade.faces.new(
+ (
+ v_inner_bot[i],
+ v_inner_bot[(i + 1) % segments],
+ v_inner_top[(i + 1) % segments],
+ v_inner_top[i],
+ )
+ )
+
+ # Top cap rim faces
+ for i in range(segments):
+ bm_shade.faces.new(
+ (
+ v_inner_top[i],
+ v_inner_top[(i + 1) % segments],
+ v_outer_top[(i + 1) % segments],
+ v_outer_top[i],
+ )
+ )
+
+ # Top inner flat cap
+ v_in_cap_center = bm_shade.verts.new((0, 0, -0.003))
+ for i in range(segments):
+ bm_shade.faces.new(
+ (
+ v_in_cap_center,
+ v_inner_top[(i + 1) % segments],
+ v_inner_top[i],
+ )
+ )
+
+ # Top outer flat cap
+ v_out_cap_center = bm_shade.verts.new((0, 0, 0.0))
+ for i in range(segments):
+ bm_shade.faces.new(
+ (
+ v_out_cap_center,
+ v_outer_top[i],
+ v_outer_top[(i + 1) % segments],
+ )
+ )
+
+ # Transform shade into world orientation
+ world_transform = Matrix.Translation(shade_top_center) @ rot_shade
+ bm_shade.transform(world_transform)
+ bm_shade.normal_update()
+
+ mesh_shade = bpy.data.meshes.new("ConicalShade")
+ bm_shade.to_mesh(mesh_shade)
+ bm_shade.free()
+ set_smooth(mesh_shade)
+
+ shade_obj = bpy.data.objects.new("ConicalShade", mesh_shade)
+ shade_obj.data.materials.append(materials["matte_black"])
+ bpy.context.scene.collection.objects.link(shade_obj)
+
+ # 3. Light Bulb tucked inside shade
+ bulb_pos = shade_top_center + shade_dir * 0.055
+ bm_bulb = bmesh.new()
+ bmesh.ops.create_uvsphere(
+ bm_bulb,
+ u_segments=24,
+ v_segments=16,
+ radius=0.022,
+ matrix=Matrix.Translation(bulb_pos)
+ @ Matrix.Scale(1.0, 4, (1, 0, 0))
+ @ Matrix.Scale(1.0, 4, (0, 1, 0))
+ @ Matrix.Scale(1.25, 4, (0, 0, 1))
+ @ rot_shade,
+ )
+ bm_bulb.normal_update()
+ mesh_bulb = bpy.data.meshes.new("LightBulb")
+ bm_bulb.to_mesh(mesh_bulb)
+ bm_bulb.free()
+ set_smooth(mesh_bulb)
+
+ bulb_obj = bpy.data.objects.new("LightBulb", mesh_bulb)
+ bulb_obj.data.materials.append(materials["bulb_emission"])
+ bpy.context.scene.collection.objects.link(bulb_obj)
+
+
+def main():
+ # Setup high quality materials
+ materials = {
+ "matte_black": create_material(
+ "MatteBlackMetal",
+ base_color=(0.04, 0.04, 0.045, 1.0),
+ metallic=0.15,
+ roughness=0.35,
+ ),
+ "brass": create_material(
+ "BrushedBrass",
+ base_color=(0.85, 0.65, 0.22, 1.0),
+ metallic=0.92,
+ roughness=0.25,
+ ),
+ "rubber": create_material(
+ "RubberFelt",
+ base_color=(0.02, 0.02, 0.02, 1.0),
+ metallic=0.0,
+ roughness=0.9,
+ ),
+ "bulb_emission": create_material(
+ "FrostedBulb",
+ base_color=(0.95, 0.93, 0.88, 1.0),
+ metallic=0.0,
+ roughness=0.15,
+ emission_color=(1.0, 0.96, 0.88, 1.0),
+ emission_strength=4.0,
+ ),
+ }
+
+ # 1. Base on ground plane (z=0, footprint centered)
+ base_top_z = make_base(materials)
+
+ # 2. Switch on top of base
+ make_switch(materials, base_top_z)
+
+ # 3. Arm leaning forward (-Y) with pivots
+ top_pos = make_arm_and_mount(materials, base_top_z)
+
+ # 4. Conical shade opening downward + inner bulb
+ make_shade_and_bulb(materials, top_pos)
+
+
+if __name__ == "__main__":
+ main()
+```
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/result.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/result.json
new file mode 100644
index 00000000..93215b59
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0/result.json
@@ -0,0 +1,19 @@
+{
+ "ok": true,
+ "text": "```python\n\"\"\"\nDesk Lamp Model\nMid-century modern style desk lamp with a heavy round base,\na single angled arm leaning forward, and a conical metal shade opening downward.\n\"\"\"\n\nimport math\nimport bmesh\nimport bpy\nfrom mathutils import Matrix, Vector\n\n\ndef create_material(name, base_color, metallic=0.0, roughness=0.3, emission_color=None, emission_strength=0.0):\n \"\"\"Creates a Principled BSDF material.\"\"\"\n mat = bpy.data.materials.new(name=name)\n mat.use_nodes = True\n nodes = mat.node_tree.nodes\n links = mat.node_tree.links\n\n nodes.clear()\n node_out = nodes.new(type=\"ShaderNodeOutputMaterial\")\n node_bsdf = nodes.new(type=\"ShaderNodeBsdfPrincipled\")\n\n # Set Principled BSDF socket values\n node_bsdf.inputs[\"Base Color\"].default_value = base_color\n node_bsdf.inputs[\"Metallic\"].default_value = metallic\n node_bsdf.inputs[\"Roughness\"].default_value = roughness\n\n # Emission handling for light bulb\n if emission_color is not None and emission_strength > 0:\n if \"Emission Color\" in node_bsdf.inputs:\n node_bsdf.inputs[\"Emission Color\"].default_value = emission_color\n node_bsdf.inputs[\"Emission Strength\"].default_value = emission_strength\n elif \"Emission\" in node_bsdf.inputs:\n node_bsdf.inputs[\"Emission\"].default_value = emission_color\n\n links.new(node_bsdf.outputs[\"BSDF\"], node_out.inputs[\"Surface\"])\n return mat\n\n\ndef set_smooth(mesh):\n \"\"\"Sets smooth shading on all faces of a mesh.\"\"\"\n for poly in mesh.polygons:\n poly.use_smooth = True\n\n\ndef make_base(materials):\n \"\"\"Creates the heavy circular weighted base with a bottom felt pad.\"\"\"\n # 1. Base Body\n bm = bmesh.new()\n # Create stepped profile for heavy cast look\n segments = 48\n r_outer = 0.095\n r_top = 0.088\n h_total = 0.022\n h_step = 0.005\n\n # Bottom ring\n v_bot_center = bm.verts.new((0, 0, 0.002))\n v_bot_rim = [\n bm.verts.new(\n (\n r_outer * math.cos(i * 2 * math.pi / segments),\n r_outer * math.sin(i * 2 * math.pi / segments),\n 0.002,\n )\n )\n for i in range(segments)\n ]\n for i in range(segments):\n bm.faces.new(\n (\n v_bot_center,\n v_bot_rim[(i + 1) % segments],\n v_bot_rim[i],\n )\n )\n\n # Lower chamfer\n v_mid_rim = [\n bm.verts.new(\n (\n r_outer * math.cos(i * 2 * math.pi / segments),\n r_outer * math.sin(i * 2 * math.pi / segments),\n h_step,\n )\n )\n for i in range(segments)\n ]\n for i in range(segments):\n bm.faces.new(\n (\n v_bot_rim[i],\n v_bot_rim[(i + 1) % segments],\n v_mid_rim[(i + 1) % segments],\n v_mid_rim[i],\n )\n )\n\n # Top outer rim\n v_top_outer = [\n bm.verts.new(\n (\n r_outer * math.cos(i * 2 * math.pi / segments),\n r_outer * math.sin(i * 2 * math.pi / segments),\n h_total - 0.003,\n )\n )\n for i in range(segments)\n ]\n for i in range(segments):\n bm.faces.new(\n (\n v_mid_rim[i],\n v_mid_rim[(i + 1) % segments],\n v_top_outer[(i + 1) % segments],\n v_top_outer[i],\n )\n )\n\n # Top chamfer rim\n v_top_inner = [\n bm.verts.new(\n (\n r_top * math.cos(i * 2 * math.pi / segments),\n r_top * math.sin(i * 2 * math.pi / segments),\n h_total,\n )\n )\n for i in range(segments)\n ]\n for i in range(segments):\n bm.faces.new(\n (\n v_top_outer[i],\n v_top_outer[(i + 1) % segments],\n v_top_inner[(i + 1) % segments],\n v_top_inner[i],\n )\n )\n\n # Top flat cap\n v_top_center = bm.verts.new((0, 0, h_total))\n for i in range(segments):\n bm.faces.new(\n (\n v_top_center,\n v_top_inner[i],\n v_top_inner[(i + 1) % segments],\n )\n )\n\n bm.normal_update()\n mesh = bpy.data.meshes.new(\"LampBase\")\n bm.to_mesh(mesh)\n bm.free()\n set_smooth(mesh)\n\n base_obj = bpy.data.objects.new(\"LampBase\", mesh)\n base_obj.data.materials.append(materials[\"matte_black\"])\n bpy.context.scene.collection.objects.link(base_obj)\n\n # 2. Bottom Felt Pad (visible edge resting on z=0)\n bm_pad = bmesh.new()\n bmesh.ops.create_cylinder(\n bm_pad,\n cap_ends=True,\n radius=0.093,\n depth=0.002,\n segments=48,\n matrix=Matrix.Translation((0, 0, 0.001)),\n )\n mesh_pad = bpy.data.meshes.new(\"BasePad\")\n bm_pad.to_mesh(mesh_pad)\n bm_pad.free()\n set_smooth(mesh_pad)\n\n pad_obj = bpy.data.objects.new(\"BasePad\", mesh_pad)\n pad_obj.data.materials.append(materials[\"rubber\"])\n bpy.context.scene.collection.objects.link(pad_obj)\n\n return h_total\n\n\ndef make_switch(materials, z_pos):\n \"\"\"Creates a brass push/rotary switch button on the base.\"\"\"\n bm = bmesh.new()\n # Switch bezel collar\n bmesh.ops.create_cylinder(\n bm,\n cap_ends=True,\n radius=0.006,\n depth=0.004,\n segments=24,\n matrix=Matrix.Translation((0, -0.052, z_pos + 0.002)),\n )\n # Switch button toggle\n bmesh.ops.create_cylinder(\n bm,\n cap_ends=True,\n radius=0.0035,\n depth=0.008,\n segments=24,\n matrix=Matrix.Translation((0, -0.052, z_pos + 0.006)),\n )\n bm.normal_update()\n mesh = bpy.data.meshes.new(\"BaseSwitch\")\n bm.to_mesh(mesh)\n bm.free()\n set_smooth(mesh)\n\n obj = bpy.data.objects.new(\"BaseSwitch\", mesh)\n obj.data.materials.append(materials[\"brass\"])\n bpy.context.scene.collection.objects.link(obj)\n\n\ndef make_arm_and_mount(materials, base_top_z):\n \"\"\"Creates the base pivot mount, the single forward-angled arm, and the top joint.\"\"\"\n arm_base_pos = Vector((0.0, 0.035, base_top_z))\n\n # 1. Base Arm Mount (Brass Collar & Bracket)\n bm_mount = bmesh.new()\n # Bottom mount cylinder\n bmesh.ops.create_cylinder(\n bm_mount,\n cap_ends=True,\n radius=0.014,\n depth=0.012,\n segments=32,\n matrix=Matrix.Translation((0.0, 0.035, base_top_z + 0.006)),\n )\n # Pivot fork / bracket\n bmesh.ops.create_cube(\n bm_mount,\n size=1.0,\n matrix=Matrix.Translation((0.0, 0.035, base_top_z + 0.017))\n @ Matrix.Scale(0.016, 4, (1, 0, 0))\n @ Matrix.Scale(0.012, 4, (0, 1, 0))\n @ Matrix.Scale(0.014, 4, (0, 0, 1)),\n )\n # Pivot horizontal pin\n bmesh.ops.create_cylinder(\n bm_mount,\n cap_ends=True,\n radius=0.004,\n depth=0.022,\n segments=16,\n matrix=Matrix.Translation((0.0, 0.035, base_top_z + 0.018))\n @ Matrix.Rotation(math.radians(90), 4, \"Y\"),\n )\n bm_mount.normal_update()\n mesh_mount = bpy.data.meshes.new(\"ArmBaseMount\")\n bm_mount.to_mesh(mesh_mount)\n bm_mount.free()\n set_smooth(mesh_mount)\n\n mount_obj = bpy.data.objects.new(\"ArmBaseMount\", mesh_mount)\n mount_obj.data.materials.append(materials[\"brass\"])\n bpy.context.scene.collection.objects.link(mount_obj)\n\n # 2. Main Straight Arm (Leaning forward into -Y)\n # Pivot origin at base mount\n pivot_z = base_top_z + 0.018\n arm_start = Vector((0.0, 0.035, pivot_z))\n arm_length = 0.44\n tilt_angle = math.radians(24.0) # Leaning forward towards -Y\n\n # Direction vector: leans forward (-Y) and up (+Z)\n dir_vec = Vector((0.0, -math.sin(tilt_angle), math.cos(tilt_angle)))\n arm_end = arm_start + dir_vec * arm_length\n arm_center = (arm_start + arm_end) * 0.5\n\n # Rotation matrix aligning cylinder (initially along Z) with dir_vec\n rot_arm = Matrix.Rotation(-tilt_angle, 4, \"X\")\n\n bm_arm = bmesh.new()\n bmesh.ops.create_cylinder(\n bm_arm,\n cap_ends=True,\n radius=0.006,\n depth=arm_length,\n segments=28,\n matrix=Matrix.Translation(arm_center) @ rot_arm,\n )\n bm_arm.normal_update()\n mesh_arm = bpy.data.meshes.new(\"LampArm\")\n bm_arm.to_mesh(mesh_arm)\n bm_arm.free()\n set_smooth(mesh_arm)\n\n arm_obj = bpy.data.objects.new(\"LampArm\", mesh_arm)\n arm_obj.data.materials.append(materials[\"matte_black\"])\n bpy.context.scene.collection.objects.link(arm_obj)\n\n # 3. Top Joint (Brass Swivel Fitting)\n bm_joint = bmesh.new()\n # Spherical / cylindrical knuckle at top of the arm\n bmesh.ops.create_uvsphere(\n bm_joint,\n u_segments=24,\n v_segments=16,\n radius=0.011,\n matrix=Matrix.Translation(arm_end),\n )\n # Locking screw / wing knob on side (+X)\n bmesh.ops.create_cylinder(\n bm_joint,\n cap_ends=True,\n radius=0.0035,\n depth=0.028,\n segments=16,\n matrix=Matrix.Translation(arm_end) @ Matrix.Rotation(math.radians(90), 4, \"Y\"),\n )\n bmesh.ops.create_cylinder(\n bm_joint,\n cap_ends=True,\n radius=0.007,\n depth=0.004,\n segments=16,\n matrix=Matrix.Translation(arm_end + Vector((0.014, 0, 0)))\n @ Matrix.Rotation(math.radians(90), 4, \"Y\"),\n )\n bm_joint.normal_update()\n mesh_joint = bpy.data.meshes.new(\"UpperJoint\")\n bm_joint.to_mesh(mesh_joint)\n bm_joint.free()\n set_smooth(mesh_joint)\n\n joint_obj = bpy.data.objects.new(\"UpperJoint\", mesh_joint)\n joint_obj.data.materials.append(materials[\"brass\"])\n bpy.context.scene.collection.objects.link(joint_obj)\n\n return arm_end\n\n\ndef make_shade_and_bulb(materials, top_pos):\n \"\"\"Creates the conical metal shade opening downwards and the inner light bulb.\"\"\"\n # Shade orientation: tilted slightly forward from straight down (e.g. 15 degrees)\n shade_tilt = math.radians(12.0)\n rot_shade = Matrix.Rotation(shade_tilt, 4, \"X\")\n\n # Downward axis vector for shade\n shade_dir = Vector((0.0, -math.sin(shade_tilt), -math.cos(shade_tilt)))\n\n # 1. Brass Shade Socket / Neck attached to joint\n socket_length = 0.025\n socket_pos = top_pos + shade_dir * (socket_length * 0.5)\n\n bm_socket = bmesh.new()\n bmesh.ops.create_cylinder(\n bm_socket,\n cap_ends=True,\n radius=0.010,\n depth=socket_length,\n segments=24,\n matrix=Matrix.Translation(socket_pos) @ rot_shade,\n )\n bm_socket.normal_update()\n mesh_socket = bpy.data.meshes.new(\"ShadeSocket\")\n bm_socket.to_mesh(mesh_socket)\n bm_socket.free()\n set_smooth(mesh_socket)\n\n socket_obj = bpy.data.objects.new(\"ShadeSocket\", mesh_socket)\n socket_obj.data.materials.append(materials[\"brass\"])\n bpy.context.scene.collection.objects.link(socket_obj)\n\n # 2. Conical Shade\n # Top of cone attaches at socket end\n shade_top_center = top_pos + shade_dir * socket_length\n shade_height = 0.135\n r_top = 0.028\n r_bottom = 0.092\n segments = 48\n\n # Local coordinates along shade axis (Z from 0 to -shade_height)\n bm_shade = bmesh.new()\n\n # Outer cone vertices\n v_outer_top = [\n bm_shade.verts.new(\n (\n r_top * math.cos(i * 2 * math.pi / segments),\n r_top * math.sin(i * 2 * math.pi / segments),\n 0.0,\n )\n )\n for i in range(segments)\n ]\n v_outer_bot = [\n bm_shade.verts.new(\n (\n r_bottom * math.cos(i * 2 * math.pi / segments),\n r_bottom * math.sin(i * 2 * math.pi / segments),\n -shade_height,\n )\n )\n for i in range(segments)\n ]\n\n # Inner cone vertices (for realistic thickness and white interior)\n r_top_in = r_top - 0.0025\n r_bot_in = r_bottom - 0.0025\n v_inner_top = [\n bm_shade.verts.new(\n (\n r_top_in * math.cos(i * 2 * math.pi / segments),\n r_top_in * math.sin(i * 2 * math.pi / segments),\n -0.003,\n )\n )\n for i in range(segments)\n ]\n v_inner_bot = [\n bm_shade.verts.new(\n (\n r_bot_in * math.cos(i * 2 * math.pi / segments),\n r_bot_in * math.sin(i * 2 * math.pi / segments),\n -shade_height + 0.001,\n )\n )\n for i in range(segments)\n ]\n\n # Outer wall faces\n for i in range(segments):\n bm_shade.faces.new(\n (\n v_outer_top[i],\n v_outer_top[(i + 1) % segments],\n v_outer_bot[(i + 1) % segments],\n v_outer_bot[i],\n )\n )\n\n # Bottom rim lip faces (joining outer and inner)\n for i in range(segments):\n bm_shade.faces.new(\n (\n v_outer_bot[i],\n v_outer_bot[(i + 1) % segments],\n v_inner_bot[(i + 1) % segments],\n v_inner_bot[i],\n )\n )\n\n # Inner wall faces\n for i in range(segments):\n bm_shade.faces.new(\n (\n v_inner_bot[i],\n v_inner_bot[(i + 1) % segments],\n v_inner_top[(i + 1) % segments],\n v_inner_top[i],\n )\n )\n\n # Top cap rim faces\n for i in range(segments):\n bm_shade.faces.new(\n (\n v_inner_top[i],\n v_inner_top[(i + 1) % segments],\n v_outer_top[(i + 1) % segments],\n v_outer_top[i],\n )\n )\n\n # Top inner flat cap\n v_in_cap_center = bm_shade.verts.new((0, 0, -0.003))\n for i in range(segments):\n bm_shade.faces.new(\n (\n v_in_cap_center,\n v_inner_top[(i + 1) % segments],\n v_inner_top[i],\n )\n )\n\n # Top outer flat cap\n v_out_cap_center = bm_shade.verts.new((0, 0, 0.0))\n for i in range(segments):\n bm_shade.faces.new(\n (\n v_out_cap_center,\n v_outer_top[i],\n v_outer_top[(i + 1) % segments],\n )\n )\n\n # Transform shade into world orientation\n world_transform = Matrix.Translation(shade_top_center) @ rot_shade\n bm_shade.transform(world_transform)\n bm_shade.normal_update()\n\n mesh_shade = bpy.data.meshes.new(\"ConicalShade\")\n bm_shade.to_mesh(mesh_shade)\n bm_shade.free()\n set_smooth(mesh_shade)\n\n shade_obj = bpy.data.objects.new(\"ConicalShade\", mesh_shade)\n shade_obj.data.materials.append(materials[\"matte_black\"])\n bpy.context.scene.collection.objects.link(shade_obj)\n\n # 3. Light Bulb tucked inside shade\n bulb_pos = shade_top_center + shade_dir * 0.055\n bm_bulb = bmesh.new()\n bmesh.ops.create_uvsphere(\n bm_bulb,\n u_segments=24,\n v_segments=16,\n radius=0.022,\n matrix=Matrix.Translation(bulb_pos)\n @ Matrix.Scale(1.0, 4, (1, 0, 0))\n @ Matrix.Scale(1.0, 4, (0, 1, 0))\n @ Matrix.Scale(1.25, 4, (0, 0, 1))\n @ rot_shade,\n )\n bm_bulb.normal_update()\n mesh_bulb = bpy.data.meshes.new(\"LightBulb\")\n bm_bulb.to_mesh(mesh_bulb)\n bm_bulb.free()\n set_smooth(mesh_bulb)\n\n bulb_obj = bpy.data.objects.new(\"LightBulb\", mesh_bulb)\n bulb_obj.data.materials.append(materials[\"bulb_emission\"])\n bpy.context.scene.collection.objects.link(bulb_obj)\n\n\ndef main():\n # Setup high quality materials\n materials = {\n \"matte_black\": create_material(\n \"MatteBlackMetal\",\n base_color=(0.04, 0.04, 0.045, 1.0),\n metallic=0.15,\n roughness=0.35,\n ),\n \"brass\": create_material(\n \"BrushedBrass\",\n base_color=(0.85, 0.65, 0.22, 1.0),\n metallic=0.92,\n roughness=0.25,\n ),\n \"rubber\": create_material(\n \"RubberFelt\",\n base_color=(0.02, 0.02, 0.02, 1.0),\n metallic=0.0,\n roughness=0.9,\n ),\n \"bulb_emission\": create_material(\n \"FrostedBulb\",\n base_color=(0.95, 0.93, 0.88, 1.0),\n metallic=0.0,\n roughness=0.15,\n emission_color=(1.0, 0.96, 0.88, 1.0),\n emission_strength=4.0,\n ),\n }\n\n # 1. Base on ground plane (z=0, footprint centered)\n base_top_z = make_base(materials)\n\n # 2. Switch on top of base\n make_switch(materials, base_top_z)\n\n # 3. Arm leaning forward (-Y) with pivots\n top_pos = make_arm_and_mount(materials, base_top_z)\n\n # 4. Conical shade opening downward + inner bulb\n make_shade_and_bulb(materials, top_pos)\n\n\nif __name__ == \"__main__\":\n main()\n```",
+ "usage": {
+ "backend": "gemini",
+ "model": "gemini-3.7-flash",
+ "input_tokens": 526,
+ "output_tokens": 5731,
+ "cached_tokens": 0,
+ "thoughts_tokens": 1753,
+ "tool_calls": 0,
+ "cost_usd": 0.0284595,
+ "latency_ms": 17845
+ },
+ "tool_calls": 0,
+ "notes": "",
+ "duration_s": 55.18,
+ "transcript_dir": "bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/gen/attempt0"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_api-agent_gemini_gemini-3.7-flash/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_api-agent_gemini_gemini-3.7-flash/eval
new file mode 160000
index 00000000..c5372685
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_api-agent_gemini_gemini-3.7-flash/eval
@@ -0,0 +1 @@
+Subproject commit c537268577793d592345e84d026a60f420f210a0
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_api-agent_gemini_gemini-3.7-flash/run b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_api-agent_gemini_gemini-3.7-flash/run
new file mode 160000
index 00000000..c2b0a00d
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_api-agent_gemini_gemini-3.7-flash/run
@@ -0,0 +1 @@
+Subproject commit c2b0a00db126a8025fc084962b54fdd23be037d7
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/cell.json
new file mode 100644
index 00000000..55c44dbf
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/cell.json
@@ -0,0 +1,29 @@
+{
+ "prompt_id": "cmp_easy_stool",
+ "tier": "easy",
+ "arm": "harness:gemini-cli:gemini-3.7-flash",
+ "kind": "harness",
+ "target": "gemini-cli:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "error",
+ "score": null,
+ "score_std": 0.0,
+ "passed": null,
+ "build_ok": false,
+ "gate_errors": [],
+ "tris": null,
+ "gen_cost_usd": 0.0,
+ "judge_cost_usd": 0.0,
+ "gen_seconds": 0.0,
+ "wall_s": 68.3,
+ "attempts": 0,
+ "tool_calls": 0,
+ "criteria": {},
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "",
+ "glb": "",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash",
+ "error": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\nevents, runtime=runtime,\n ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n template=template, example=example, temperature=temperature,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n max_output_tokens=max_output_tokens, finalise=finalise, event_stats=event_stats)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 108, in plan_with_usage\n resp = model.generate(req)\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 203, in generate\n return rotate_with_retries(\n self.pool,\n ...<10 lines>...\n label=f\"gemini {self.model}\",\n )\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/retry.py\", line 177, in rotate_with_retries\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/eval
new file mode 160000
index 00000000..0299b00a
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/eval
@@ -0,0 +1 @@
+Subproject commit 0299b00a3279cc745805883674f40750ad55eb8f
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/run b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/run
new file mode 160000
index 00000000..0299b00a
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash/run
@@ -0,0 +1 @@
+Subproject commit 0299b00a3279cc745805883674f40750ad55eb8f
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/cell.json
new file mode 100644
index 00000000..429f77a3
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/cell.json
@@ -0,0 +1,41 @@
+{
+ "prompt_id": "cmp_easy_stool",
+ "tier": "easy",
+ "arm": "oneshot:claude-code",
+ "kind": "oneshot",
+ "target": "claude-code",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "scored",
+ "score": 0.646,
+ "score_std": 0.052,
+ "passed": false,
+ "build_ok": true,
+ "gate_errors": [
+ "connectivity: 'LegFrontRight' and 'Seat' interpenetrate by ≈15.9 mm (5% of surface samples inside)",
+ "connectivity: 'LegFrontLeft' and 'Seat' interpenetrate by ≈15.2 mm (5% of surface samples inside)",
+ "connectivity: 'LegBack' and 'Seat' interpenetrate by ≈16.1 mm (5% of surface samples inside)"
+ ],
+ "tris": 2516,
+ "gen_cost_usd": 0.7319350000000001,
+ "judge_cost_usd": 0.09858,
+ "gen_seconds": 143.03,
+ "wall_s": 196.2,
+ "attempts": 1,
+ "tool_calls": 0,
+ "criteria": {
+ "intent_fidelity": 0.95,
+ "structure_plausibility": 0.85,
+ "geometry_detail": 0.7,
+ "proportions_scale": 0.95,
+ "assembly_fit": 0.4,
+ "materials": 0.7,
+ "craftsmanship_no_artifacts": 0.7
+ },
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/eval/artifacts/renders/r00/sheet.png",
+ "glb": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/eval/artifacts/object.glb",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code",
+ "error": ""
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/eval
new file mode 160000
index 00000000..5c65600d
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/eval
@@ -0,0 +1 @@
+Subproject commit 5c65600d3e1056316565478bdee6c5acb84f780d
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/argv.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/argv.json
new file mode 100644
index 00000000..e9d03e63
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/argv.json
@@ -0,0 +1 @@
+["claude", "-p", "<prompt 1896 chars>", "--output-format", "json", "--tools", "", "--max-turns", "1", "--no-session-persistence", "--strict-mcp-config"]
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/prompt.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/prompt.md
new file mode 100644
index 00000000..26df6c8c
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/prompt.md
@@ -0,0 +1,26 @@
+Model this object in raw bpy: a simple three-legged round wooden stool, 45 cm tall
+Dimensions (m): height=0.45
+MUST HAVE: round seat
+MUST HAVE: three splayed legs of equal length
+MUST HAVE: legs touch the ground
+
+You write ONE file, `src/model.py`, in raw Blender Python (bpy, Blender 5.x API).
+The harness runs it headless (`blender -b --factory-startup --python <wrapper> -- --script src/model.py`)
+inside an EMPTIED scene (no default cube / camera / light) and then exports the result itself.
+
+Frame, units and placement: Coordinate frame: Z is UP, -Y is the FRONT of the object (the face a viewer sees from the default front view), +X is the object's right. Units are meters. The object stands on the ground plane z=0 and its footprint is centred on the Z axis. Build at real-world size.
+
+Naming: one mesh object per part, `obj.name` set to a unique PascalCase part name (e.g. `Seat`,
+`LegFrontLeft`); no auto-suffixed duplicates like `Leg.001`. Link every object to
+`bpy.context.scene.collection`. Every visible mesh gets a material (Principled BSDF base colour /
+roughness / metallic).
+
+Do NOT: create cameras or lights, touch render/world settings, call `bpy.ops.render.*`,
+`bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the network, or
+import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Keep the total under
+500k triangles and finish in under 120 s.
+
+The script must build the whole object when executed top-to-bottom (call your `main()` at module
+level). Make sure the code runs without errors in headless Blender — there is no second chance.
+
+You have NO tools in this session: you cannot write files, run Blender or inspect anything — the code must appear in your reply. Reply with the COMPLETE contents of `src/model.py` as ONE ```python fenced code block and nothing else (no prose before or after, no partial snippets).
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/response.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/response.md
new file mode 100644
index 00000000..d4cf43f9
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/response.md
@@ -0,0 +1,202 @@
+```python
+import bpy
+import bmesh
+import math
+from mathutils import Vector
+
+# ---------------------------------------------------------------------------
+# Dimensions (meters)
+# ---------------------------------------------------------------------------
+STOOL_HEIGHT = 0.45 # overall height (top of seat rim)
+SEAT_RADIUS = 0.17 # round seat, 34 cm diameter
+SEAT_THICKNESS = 0.035
+SEAT_TOP_Z = STOOL_HEIGHT
+SEAT_BOT_Z = SEAT_TOP_Z - SEAT_THICKNESS
+SEAT_EDGE_ROUND = 0.012 # rounded top edge radius
+SEAT_BOT_CHAMFER = 0.004
+SEAT_DISH_DEPTH = 0.006 # gentle concave dish in the seat top
+
+LEG_TOP_Z = SEAT_BOT_Z + 0.017 # leg tops are embedded into the seat underside
+LEG_RADIAL_TOP = 0.085 # distance of leg axis from stool axis at the top
+LEG_RADIAL_BOT = 0.150 # ... and at the floor (splay outward)
+LEG_R_TOP = 0.019 # leg thickness radius at the top
+LEG_R_BOT = 0.0155 # slightly tapered toward the foot
+
+SEAT_SEGS = 64
+LEG_SEGS = 24
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+def make_material(name, color, roughness=0.5, metallic=0.0):
+ mat = bpy.data.materials.get(name)
+ if mat is None:
+ mat = bpy.data.materials.new(name)
+ mat.use_nodes = True
+ nt = mat.node_tree
+ bsdf = nt.nodes.get("Principled BSDF")
+ if bsdf is None:
+ bsdf = nt.nodes.new("ShaderNodeBsdfPrincipled")
+ out = nt.nodes.get("Material Output")
+ if out is None:
+ out = nt.nodes.new("ShaderNodeOutputMaterial")
+ nt.links.new(bsdf.outputs["BSDF"], out.inputs["Surface"])
+ bsdf.inputs["Base Color"].default_value = (color[0], color[1], color[2], 1.0)
+ bsdf.inputs["Roughness"].default_value = roughness
+ bsdf.inputs["Metallic"].default_value = metallic
+ mat.diffuse_color = (color[0], color[1], color[2], 1.0)
+ mat.roughness = roughness
+ mat.metallic = metallic
+ return mat
+
+
+def new_object(name, bm, mat):
+ """Turn a bmesh into a uniquely named mesh object linked to the scene collection."""
+ old = bpy.data.objects.get(name)
+ if old is not None:
+ bpy.data.objects.remove(old, do_unlink=True)
+ old_me = bpy.data.meshes.get(name)
+ if old_me is not None:
+ bpy.data.meshes.remove(old_me)
+ me = bpy.data.meshes.new(name)
+ bm.to_mesh(me)
+ bm.free()
+ me.update()
+ obj = bpy.data.objects.new(name, me)
+ obj.data.materials.append(mat)
+ bpy.context.scene.collection.objects.link(obj)
+ return obj
+
+
+def lathe(profile, segs):
+ """Surface of revolution around Z. profile = [(r, z), ...]; r == 0 -> center vertex."""
+ bm = bmesh.new()
+ rings = []
+ for (r, z) in profile:
+ if r <= 1e-9:
+ rings.append([bm.verts.new((0.0, 0.0, z))])
+ else:
+ ring = []
+ for i in range(segs):
+ a = 2.0 * math.pi * i / segs
+ ring.append(bm.verts.new((r * math.cos(a), r * math.sin(a), z)))
+ rings.append(ring)
+ bm.verts.ensure_lookup_table()
+ for k in range(len(rings) - 1):
+ a = rings[k]
+ b = rings[k + 1]
+ if len(a) == 1 and len(b) == 1:
+ continue
+ if len(a) == 1:
+ c = a[0]
+ for i in range(segs):
+ j = (i + 1) % segs
+ bm.faces.new((c, b[i], b[j]))
+ elif len(b) == 1:
+ c = b[0]
+ for i in range(segs):
+ j = (i + 1) % segs
+ bm.faces.new((a[i], a[j], c))
+ else:
+ for i in range(segs):
+ j = (i + 1) % segs
+ bm.faces.new((a[i], a[j], b[j], b[i]))
+ bmesh.ops.recalc_face_normals(bm, faces=bm.faces)
+ for f in bm.faces:
+ f.smooth = True
+ return bm
+
+
+# ---------------------------------------------------------------------------
+# Parts
+# ---------------------------------------------------------------------------
+def seat_profile():
+ R = SEAT_RADIUS
+ b = SEAT_EDGE_ROUND
+ c = SEAT_BOT_CHAMFER
+ d = SEAT_DISH_DEPTH
+ rd = R - b - 0.015 # radius of the dished area
+ prof = []
+ # gently dished top (center is lowest, rim of the dish is at full height)
+ n_dish = 6
+ for i in range(n_dish + 1):
+ r = rd * i / n_dish
+ z = SEAT_TOP_Z - d * (1.0 - (r / rd) ** 2)
+ prof.append((r, z))
+ # flat band out to where the rounded edge starts
+ prof.append((R - b, SEAT_TOP_Z))
+ # rounded top edge
+ for ang in (67.5, 45.0, 22.5, 0.0):
+ a = math.radians(ang)
+ prof.append((R - b + b * math.cos(a), SEAT_TOP_Z - b + b * math.sin(a)))
+ # vertical rim, small bottom chamfer, flat bottom
+ prof.append((R, SEAT_BOT_Z + c))
+ prof.append((R - c, SEAT_BOT_Z))
+ prof.append((0.0, SEAT_BOT_Z))
+ return prof
+
+
+def make_seat(mat):
+ bm = lathe(seat_profile(), SEAT_SEGS)
+ return new_object("Seat", bm, mat)
+
+
+def make_leg(name, phi, mat):
+ """Splayed tapered round leg. Horizontal cross-sections: flat foot at z=0, flat top."""
+ top_c = Vector((LEG_RADIAL_TOP * math.cos(phi), LEG_RADIAL_TOP * math.sin(phi), LEG_TOP_Z))
+ bot_c = Vector((LEG_RADIAL_BOT * math.cos(phi), LEG_RADIAL_BOT * math.sin(phi), 0.0))
+ bm = bmesh.new()
+ nrings = 6
+ rings = []
+ for k in range(nrings):
+ t = k / (nrings - 1)
+ c = bot_c.lerp(top_c, t)
+ r = LEG_R_BOT * (1.0 - t) + LEG_R_TOP * t
+ ring = []
+ for i in range(LEG_SEGS):
+ a = 2.0 * math.pi * i / LEG_SEGS
+ ring.append(bm.verts.new((c.x + r * math.cos(a), c.y + r * math.sin(a), c.z)))
+ rings.append(ring)
+ bm.verts.ensure_lookup_table()
+ # sides
+ for k in range(nrings - 1):
+ a = rings[k]
+ b = rings[k + 1]
+ for i in range(LEG_SEGS):
+ j = (i + 1) % LEG_SEGS
+ f = bm.faces.new((a[i], a[j], b[j], b[i]))
+ f.smooth = True
+ # flat caps (foot on the ground at z=0, top embedded in the seat)
+ fb = bm.faces.new(rings[0])
+ ft = bm.faces.new(rings[-1])
+ fb.smooth = False
+ ft.smooth = False
+ bmesh.ops.recalc_face_normals(bm, faces=bm.faces)
+ for f in (fb, ft):
+ for e in f.edges:
+ e.smooth = False
+ return new_object(name, bm, mat)
+
+
+# ---------------------------------------------------------------------------
+# Build
+# ---------------------------------------------------------------------------
+def main():
+ wood_seat = make_material("WoodSeat", (0.58, 0.38, 0.19), roughness=0.45, metallic=0.0)
+ wood_leg = make_material("WoodLeg", (0.46, 0.28, 0.13), roughness=0.5, metallic=0.0)
+
+ make_seat(wood_seat)
+
+ # Three legs, 120 degrees apart: one at the back (+Y), two at the front (-Y).
+ legs = {
+ "LegBack": 90.0,
+ "LegFrontLeft": 210.0,
+ "LegFrontRight": 330.0,
+ }
+ for name, deg in legs.items():
+ make_leg(name, math.radians(deg), wood_leg)
+
+
+main()
+```
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/result.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/result.json
new file mode 100644
index 00000000..48b1e066
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/result.json
@@ -0,0 +1,19 @@
+{
+ "ok": true,
+ "text": "```python\nimport bpy\nimport bmesh\nimport math\nfrom mathutils import Vector\n\n# ---------------------------------------------------------------------------\n# Dimensions (meters)\n# ---------------------------------------------------------------------------\nSTOOL_HEIGHT = 0.45 # overall height (top of seat rim)\nSEAT_RADIUS = 0.17 # round seat, 34 cm diameter\nSEAT_THICKNESS = 0.035\nSEAT_TOP_Z = STOOL_HEIGHT\nSEAT_BOT_Z = SEAT_TOP_Z - SEAT_THICKNESS\nSEAT_EDGE_ROUND = 0.012 # rounded top edge radius\nSEAT_BOT_CHAMFER = 0.004\nSEAT_DISH_DEPTH = 0.006 # gentle concave dish in the seat top\n\nLEG_TOP_Z = SEAT_BOT_Z + 0.017 # leg tops are embedded into the seat underside\nLEG_RADIAL_TOP = 0.085 # distance of leg axis from stool axis at the top\nLEG_RADIAL_BOT = 0.150 # ... and at the floor (splay outward)\nLEG_R_TOP = 0.019 # leg thickness radius at the top\nLEG_R_BOT = 0.0155 # slightly tapered toward the foot\n\nSEAT_SEGS = 64\nLEG_SEGS = 24\n\n\n# ---------------------------------------------------------------------------\n# Helpers\n# ---------------------------------------------------------------------------\ndef make_material(name, color, roughness=0.5, metallic=0.0):\n mat = bpy.data.materials.get(name)\n if mat is None:\n mat = bpy.data.materials.new(name)\n mat.use_nodes = True\n nt = mat.node_tree\n bsdf = nt.nodes.get(\"Principled BSDF\")\n if bsdf is None:\n bsdf = nt.nodes.new(\"ShaderNodeBsdfPrincipled\")\n out = nt.nodes.get(\"Material Output\")\n if out is None:\n out = nt.nodes.new(\"ShaderNodeOutputMaterial\")\n nt.links.new(bsdf.outputs[\"BSDF\"], out.inputs[\"Surface\"])\n bsdf.inputs[\"Base Color\"].default_value = (color[0], color[1], color[2], 1.0)\n bsdf.inputs[\"Roughness\"].default_value = roughness\n bsdf.inputs[\"Metallic\"].default_value = metallic\n mat.diffuse_color = (color[0], color[1], color[2], 1.0)\n mat.roughness = roughness\n mat.metallic = metallic\n return mat\n\n\ndef new_object(name, bm, mat):\n \"\"\"Turn a bmesh into a uniquely named mesh object linked to the scene collection.\"\"\"\n old = bpy.data.objects.get(name)\n if old is not None:\n bpy.data.objects.remove(old, do_unlink=True)\n old_me = bpy.data.meshes.get(name)\n if old_me is not None:\n bpy.data.meshes.remove(old_me)\n me = bpy.data.meshes.new(name)\n bm.to_mesh(me)\n bm.free()\n me.update()\n obj = bpy.data.objects.new(name, me)\n obj.data.materials.append(mat)\n bpy.context.scene.collection.objects.link(obj)\n return obj\n\n\ndef lathe(profile, segs):\n \"\"\"Surface of revolution around Z. profile = [(r, z), ...]; r == 0 -> center vertex.\"\"\"\n bm = bmesh.new()\n rings = []\n for (r, z) in profile:\n if r <= 1e-9:\n rings.append([bm.verts.new((0.0, 0.0, z))])\n else:\n ring = []\n for i in range(segs):\n a = 2.0 * math.pi * i / segs\n ring.append(bm.verts.new((r * math.cos(a), r * math.sin(a), z)))\n rings.append(ring)\n bm.verts.ensure_lookup_table()\n for k in range(len(rings) - 1):\n a = rings[k]\n b = rings[k + 1]\n if len(a) == 1 and len(b) == 1:\n continue\n if len(a) == 1:\n c = a[0]\n for i in range(segs):\n j = (i + 1) % segs\n bm.faces.new((c, b[i], b[j]))\n elif len(b) == 1:\n c = b[0]\n for i in range(segs):\n j = (i + 1) % segs\n bm.faces.new((a[i], a[j], c))\n else:\n for i in range(segs):\n j = (i + 1) % segs\n bm.faces.new((a[i], a[j], b[j], b[i]))\n bmesh.ops.recalc_face_normals(bm, faces=bm.faces)\n for f in bm.faces:\n f.smooth = True\n return bm\n\n\n# ---------------------------------------------------------------------------\n# Parts\n# ---------------------------------------------------------------------------\ndef seat_profile():\n R = SEAT_RADIUS\n b = SEAT_EDGE_ROUND\n c = SEAT_BOT_CHAMFER\n d = SEAT_DISH_DEPTH\n rd = R - b - 0.015 # radius of the dished area\n prof = []\n # gently dished top (center is lowest, rim of the dish is at full height)\n n_dish = 6\n for i in range(n_dish + 1):\n r = rd * i / n_dish\n z = SEAT_TOP_Z - d * (1.0 - (r / rd) ** 2)\n prof.append((r, z))\n # flat band out to where the rounded edge starts\n prof.append((R - b, SEAT_TOP_Z))\n # rounded top edge\n for ang in (67.5, 45.0, 22.5, 0.0):\n a = math.radians(ang)\n prof.append((R - b + b * math.cos(a), SEAT_TOP_Z - b + b * math.sin(a)))\n # vertical rim, small bottom chamfer, flat bottom\n prof.append((R, SEAT_BOT_Z + c))\n prof.append((R - c, SEAT_BOT_Z))\n prof.append((0.0, SEAT_BOT_Z))\n return prof\n\n\ndef make_seat(mat):\n bm = lathe(seat_profile(), SEAT_SEGS)\n return new_object(\"Seat\", bm, mat)\n\n\ndef make_leg(name, phi, mat):\n \"\"\"Splayed tapered round leg. Horizontal cross-sections: flat foot at z=0, flat top.\"\"\"\n top_c = Vector((LEG_RADIAL_TOP * math.cos(phi), LEG_RADIAL_TOP * math.sin(phi), LEG_TOP_Z))\n bot_c = Vector((LEG_RADIAL_BOT * math.cos(phi), LEG_RADIAL_BOT * math.sin(phi), 0.0))\n bm = bmesh.new()\n nrings = 6\n rings = []\n for k in range(nrings):\n t = k / (nrings - 1)\n c = bot_c.lerp(top_c, t)\n r = LEG_R_BOT * (1.0 - t) + LEG_R_TOP * t\n ring = []\n for i in range(LEG_SEGS):\n a = 2.0 * math.pi * i / LEG_SEGS\n ring.append(bm.verts.new((c.x + r * math.cos(a), c.y + r * math.sin(a), c.z)))\n rings.append(ring)\n bm.verts.ensure_lookup_table()\n # sides\n for k in range(nrings - 1):\n a = rings[k]\n b = rings[k + 1]\n for i in range(LEG_SEGS):\n j = (i + 1) % LEG_SEGS\n f = bm.faces.new((a[i], a[j], b[j], b[i]))\n f.smooth = True\n # flat caps (foot on the ground at z=0, top embedded in the seat)\n fb = bm.faces.new(rings[0])\n ft = bm.faces.new(rings[-1])\n fb.smooth = False\n ft.smooth = False\n bmesh.ops.recalc_face_normals(bm, faces=bm.faces)\n for f in (fb, ft):\n for e in f.edges:\n e.smooth = False\n return new_object(name, bm, mat)\n\n\n# ---------------------------------------------------------------------------\n# Build\n# ---------------------------------------------------------------------------\ndef main():\n wood_seat = make_material(\"WoodSeat\", (0.58, 0.38, 0.19), roughness=0.45, metallic=0.0)\n wood_leg = make_material(\"WoodLeg\", (0.46, 0.28, 0.13), roughness=0.5, metallic=0.0)\n\n make_seat(wood_seat)\n\n # Three legs, 120 degrees apart: one at the back (+Y), two at the front (-Y).\n legs = {\n \"LegBack\": 90.0,\n \"LegFrontLeft\": 210.0,\n \"LegFrontRight\": 330.0,\n }\n for name, deg in legs.items():\n make_leg(name, math.radians(deg), wood_leg)\n\n\nmain()\n```",
+ "usage": {
+ "backend": "claude-code",
+ "model": "claude-haiku-4-5-20251001",
+ "input_tokens": 2,
+ "output_tokens": 12831,
+ "cached_tokens": 0,
+ "thoughts_tokens": 9644,
+ "tool_calls": 0,
+ "cost_usd": 0.7319350000000001,
+ "latency_ms": 141589
+ },
+ "tool_calls": 0,
+ "notes": "",
+ "duration_s": 143.03,
+ "transcript_dir": "bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/stderr.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/stderr.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/stdout.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/stdout.txt
new file mode 100644
index 00000000..be5582a0
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/gen/attempt0/stdout.txt
@@ -0,0 +1 @@
+{"is_error":false,"duration_api_ms":142570,"num_turns":1,"stop_reason":"end_turn","session_id":"bfd80ab0-f178-4523-9542-8d0bdc6b18f4","total_cost_usd":0.7319350000000001,"usage":{"input_tokens":2,"cache_creation_input_tokens":4442,"cache_read_input_tokens":0,"output_tokens":12831,"output_tokens_details":{"thinking_tokens":9644},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":4442,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":12831,"cache_read_input_tokens":0,"cache_creation_input_tokens":4442,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":4442},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-haiku-4-5-20251001":{"inputTokens":1445,"outputTokens":16,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001525,"contextWindow":200000,"maxOutputTokens":32000,"canonicalModel":"claude-haiku-4-5","provider":"firstParty"},"claude-fable-5":{"inputTokens":2,"outputTokens":12831,"cacheReadInputTokens":0,"cacheCreationInputTokens":4442,"webSearchRequests":0,"costUSD":0.7304100000000001,"contextWindow":1000000,"maxOutputTokens":64000,"canonicalModel":"claude-fable-5","provider":"firstParty"}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","fast_mode_disabled_reason":"sdk_opt_in_required","subagent_stats":{"spawned":0,"requested":{"background":0,"foreground":0,"unset":0},"started_in_background":0,"max_depth":0,"spawned_by_subagents":0,"completed":0,"failed":0,"killed":{"parent":0,"user":0,"system":0},"refused":{"depth_limit":0,"concurrency_limit":0,"budget":0},"by_type":{}},"subtype":"success","api_error_status":null,"result":"```python\nimport bpy\nimport bmesh\nimport math\nfrom mathutils import Vector\n\n# ---------------------------------------------------------------------------\n# Dimensions (meters)\n# ---------------------------------------------------------------------------\nSTOOL_HEIGHT = 0.45 # overall height (top of seat rim)\nSEAT_RADIUS = 0.17 # round seat, 34 cm diameter\nSEAT_THICKNESS = 0.035\nSEAT_TOP_Z = STOOL_HEIGHT\nSEAT_BOT_Z = SEAT_TOP_Z - SEAT_THICKNESS\nSEAT_EDGE_ROUND = 0.012 # rounded top edge radius\nSEAT_BOT_CHAMFER = 0.004\nSEAT_DISH_DEPTH = 0.006 # gentle concave dish in the seat top\n\nLEG_TOP_Z = SEAT_BOT_Z + 0.017 # leg tops are embedded into the seat underside\nLEG_RADIAL_TOP = 0.085 # distance of leg axis from stool axis at the top\nLEG_RADIAL_BOT = 0.150 # ... and at the floor (splay outward)\nLEG_R_TOP = 0.019 # leg thickness radius at the top\nLEG_R_BOT = 0.0155 # slightly tapered toward the foot\n\nSEAT_SEGS = 64\nLEG_SEGS = 24\n\n\n# ---------------------------------------------------------------------------\n# Helpers\n# ---------------------------------------------------------------------------\ndef make_material(name, color, roughness=0.5, metallic=0.0):\n mat = bpy.data.materials.get(name)\n if mat is None:\n mat = bpy.data.materials.new(name)\n mat.use_nodes = True\n nt = mat.node_tree\n bsdf = nt.nodes.get(\"Principled BSDF\")\n if bsdf is None:\n bsdf = nt.nodes.new(\"ShaderNodeBsdfPrincipled\")\n out = nt.nodes.get(\"Material Output\")\n if out is None:\n out = nt.nodes.new(\"ShaderNodeOutputMaterial\")\n nt.links.new(bsdf.outputs[\"BSDF\"], out.inputs[\"Surface\"])\n bsdf.inputs[\"Base Color\"].default_value = (color[0], color[1], color[2], 1.0)\n bsdf.inputs[\"Roughness\"].default_value = roughness\n bsdf.inputs[\"Metallic\"].default_value = metallic\n mat.diffuse_color = (color[0], color[1], color[2], 1.0)\n mat.roughness = roughness\n mat.metallic = metallic\n return mat\n\n\ndef new_object(name, bm, mat):\n \"\"\"Turn a bmesh into a uniquely named mesh object linked to the scene collection.\"\"\"\n old = bpy.data.objects.get(name)\n if old is not None:\n bpy.data.objects.remove(old, do_unlink=True)\n old_me = bpy.data.meshes.get(name)\n if old_me is not None:\n bpy.data.meshes.remove(old_me)\n me = bpy.data.meshes.new(name)\n bm.to_mesh(me)\n bm.free()\n me.update()\n obj = bpy.data.objects.new(name, me)\n obj.data.materials.append(mat)\n bpy.context.scene.collection.objects.link(obj)\n return obj\n\n\ndef lathe(profile, segs):\n \"\"\"Surface of revolution around Z. profile = [(r, z), ...]; r == 0 -> center vertex.\"\"\"\n bm = bmesh.new()\n rings = []\n for (r, z) in profile:\n if r <= 1e-9:\n rings.append([bm.verts.new((0.0, 0.0, z))])\n else:\n ring = []\n for i in range(segs):\n a = 2.0 * math.pi * i / segs\n ring.append(bm.verts.new((r * math.cos(a), r * math.sin(a), z)))\n rings.append(ring)\n bm.verts.ensure_lookup_table()\n for k in range(len(rings) - 1):\n a = rings[k]\n b = rings[k + 1]\n if len(a) == 1 and len(b) == 1:\n continue\n if len(a) == 1:\n c = a[0]\n for i in range(segs):\n j = (i + 1) % segs\n bm.faces.new((c, b[i], b[j]))\n elif len(b) == 1:\n c = b[0]\n for i in range(segs):\n j = (i + 1) % segs\n bm.faces.new((a[i], a[j], c))\n else:\n for i in range(segs):\n j = (i + 1) % segs\n bm.faces.new((a[i], a[j], b[j], b[i]))\n bmesh.ops.recalc_face_normals(bm, faces=bm.faces)\n for f in bm.faces:\n f.smooth = True\n return bm\n\n\n# ---------------------------------------------------------------------------\n# Parts\n# ---------------------------------------------------------------------------\ndef seat_profile():\n R = SEAT_RADIUS\n b = SEAT_EDGE_ROUND\n c = SEAT_BOT_CHAMFER\n d = SEAT_DISH_DEPTH\n rd = R - b - 0.015 # radius of the dished area\n prof = []\n # gently dished top (center is lowest, rim of the dish is at full height)\n n_dish = 6\n for i in range(n_dish + 1):\n r = rd * i / n_dish\n z = SEAT_TOP_Z - d * (1.0 - (r / rd) ** 2)\n prof.append((r, z))\n # flat band out to where the rounded edge starts\n prof.append((R - b, SEAT_TOP_Z))\n # rounded top edge\n for ang in (67.5, 45.0, 22.5, 0.0):\n a = math.radians(ang)\n prof.append((R - b + b * math.cos(a), SEAT_TOP_Z - b + b * math.sin(a)))\n # vertical rim, small bottom chamfer, flat bottom\n prof.append((R, SEAT_BOT_Z + c))\n prof.append((R - c, SEAT_BOT_Z))\n prof.append((0.0, SEAT_BOT_Z))\n return prof\n\n\ndef make_seat(mat):\n bm = lathe(seat_profile(), SEAT_SEGS)\n return new_object(\"Seat\", bm, mat)\n\n\ndef make_leg(name, phi, mat):\n \"\"\"Splayed tapered round leg. Horizontal cross-sections: flat foot at z=0, flat top.\"\"\"\n top_c = Vector((LEG_RADIAL_TOP * math.cos(phi), LEG_RADIAL_TOP * math.sin(phi), LEG_TOP_Z))\n bot_c = Vector((LEG_RADIAL_BOT * math.cos(phi), LEG_RADIAL_BOT * math.sin(phi), 0.0))\n bm = bmesh.new()\n nrings = 6\n rings = []\n for k in range(nrings):\n t = k / (nrings - 1)\n c = bot_c.lerp(top_c, t)\n r = LEG_R_BOT * (1.0 - t) + LEG_R_TOP * t\n ring = []\n for i in range(LEG_SEGS):\n a = 2.0 * math.pi * i / LEG_SEGS\n ring.append(bm.verts.new((c.x + r * math.cos(a), c.y + r * math.sin(a), c.z)))\n rings.append(ring)\n bm.verts.ensure_lookup_table()\n # sides\n for k in range(nrings - 1):\n a = rings[k]\n b = rings[k + 1]\n for i in range(LEG_SEGS):\n j = (i + 1) % LEG_SEGS\n f = bm.faces.new((a[i], a[j], b[j], b[i]))\n f.smooth = True\n # flat caps (foot on the ground at z=0, top embedded in the seat)\n fb = bm.faces.new(rings[0])\n ft = bm.faces.new(rings[-1])\n fb.smooth = False\n ft.smooth = False\n bmesh.ops.recalc_face_normals(bm, faces=bm.faces)\n for f in (fb, ft):\n for e in f.edges:\n e.smooth = False\n return new_object(name, bm, mat)\n\n\n# ---------------------------------------------------------------------------\n# Build\n# ---------------------------------------------------------------------------\ndef main():\n wood_seat = make_material(\"WoodSeat\", (0.58, 0.38, 0.19), roughness=0.45, metallic=0.0)\n wood_leg = make_material(\"WoodLeg\", (0.46, 0.28, 0.13), roughness=0.5, metallic=0.0)\n\n make_seat(wood_seat)\n\n # Three legs, 120 degrees apart: one at the back (+Y), two at the front (-Y).\n legs = {\n \"LegBack\": 90.0,\n \"LegFrontLeft\": 210.0,\n \"LegFrontRight\": 330.0,\n }\n for name, deg in legs.items():\n make_leg(name, math.radians(deg), wood_leg)\n\n\nmain()\n```","ttft_ms":116175,"ttft_stream_ms":1827,"time_to_request_ms":34,"type":"result","duration_ms":141589,"uuid":"b25dbbff-88f1-427b-aaa7-a1d47b363298"}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/cell.json
new file mode 100644
index 00000000..119bf289
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/cell.json
@@ -0,0 +1,37 @@
+{
+ "prompt_id": "cmp_easy_stool",
+ "tier": "easy",
+ "arm": "oneshot:codex",
+ "kind": "oneshot",
+ "target": "codex",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "scored",
+ "score": 0.9045,
+ "score_std": 0.0405,
+ "passed": true,
+ "build_ok": true,
+ "gate_errors": [],
+ "tris": 1184,
+ "gen_cost_usd": 0.19687700000000002,
+ "judge_cost_usd": 0.089192,
+ "gen_seconds": 75.02,
+ "wall_s": 126.9,
+ "attempts": 1,
+ "tool_calls": 0,
+ "criteria": {
+ "intent_fidelity": 0.925,
+ "structure_plausibility": 0.95,
+ "geometry_detail": 0.825,
+ "proportions_scale": 0.975,
+ "assembly_fit": 0.925,
+ "materials": 0.7,
+ "craftsmanship_no_artifacts": 1.0
+ },
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/eval/artifacts/renders/r00/sheet.png",
+ "glb": "/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/eval/artifacts/object.glb",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex",
+ "error": ""
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/eval
new file mode 160000
index 00000000..5c65600d
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/eval
@@ -0,0 +1 @@
+Subproject commit 5c65600d3e1056316565478bdee6c5acb84f780d
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/argv.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/argv.json
new file mode 100644
index 00000000..f1034d66
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/argv.json
@@ -0,0 +1 @@
+["codex", "exec", "--json", "-C", "/tmp/cv3d_oneshot_codex_ytcvb97r", "--sandbox", "read-only", "--skip-git-repo-check", "--ephemeral", "--color", "never", "-o", "bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/last_message.md", "-"]
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/prompt.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/prompt.md
new file mode 100644
index 00000000..26df6c8c
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/prompt.md
@@ -0,0 +1,26 @@
+Model this object in raw bpy: a simple three-legged round wooden stool, 45 cm tall
+Dimensions (m): height=0.45
+MUST HAVE: round seat
+MUST HAVE: three splayed legs of equal length
+MUST HAVE: legs touch the ground
+
+You write ONE file, `src/model.py`, in raw Blender Python (bpy, Blender 5.x API).
+The harness runs it headless (`blender -b --factory-startup --python <wrapper> -- --script src/model.py`)
+inside an EMPTIED scene (no default cube / camera / light) and then exports the result itself.
+
+Frame, units and placement: Coordinate frame: Z is UP, -Y is the FRONT of the object (the face a viewer sees from the default front view), +X is the object's right. Units are meters. The object stands on the ground plane z=0 and its footprint is centred on the Z axis. Build at real-world size.
+
+Naming: one mesh object per part, `obj.name` set to a unique PascalCase part name (e.g. `Seat`,
+`LegFrontLeft`); no auto-suffixed duplicates like `Leg.001`. Link every object to
+`bpy.context.scene.collection`. Every visible mesh gets a material (Principled BSDF base colour /
+roughness / metallic).
+
+Do NOT: create cameras or lights, touch render/world settings, call `bpy.ops.render.*`,
+`bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the network, or
+import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Keep the total under
+500k triangles and finish in under 120 s.
+
+The script must build the whole object when executed top-to-bottom (call your `main()` at module
+level). Make sure the code runs without errors in headless Blender — there is no second chance.
+
+You have NO tools in this session: you cannot write files, run Blender or inspect anything — the code must appear in your reply. Reply with the COMPLETE contents of `src/model.py` as ONE ```python fenced code block and nothing else (no prose before or after, no partial snippets).
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/response.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/response.md
new file mode 100644
index 00000000..4fe424f7
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/response.md
@@ -0,0 +1,196 @@
+```python
+import bpy
+import math
+from mathutils import Vector
+
+
+def create_material(name, base_color, roughness):
+ material = bpy.data.materials.new(name=name)
+ material.use_nodes = True
+
+ principled = material.node_tree.nodes.get("Principled BSDF")
+ principled.inputs["Base Color"].default_value = (*base_color, 1.0)
+ principled.inputs["Roughness"].default_value = roughness
+ principled.inputs["Metallic"].default_value = 0.0
+
+ return material
+
+
+def create_seat(material):
+ segments = 64
+
+ # Rounded 30 cm diameter seat, with its top exactly at 45 cm.
+ profile = [
+ (0.136, 0.415),
+ (0.142, 0.417),
+ (0.147, 0.422),
+ (0.150, 0.429),
+ (0.150, 0.441),
+ (0.148, 0.446),
+ (0.142, 0.450),
+ ]
+
+ vertices = []
+ faces = []
+
+ for radius, z in profile:
+ for i in range(segments):
+ angle = 2.0 * math.pi * i / segments
+ vertices.append((
+ radius * math.cos(angle),
+ radius * math.sin(angle),
+ z,
+ ))
+
+ bottom_center = len(vertices)
+ vertices.append((0.0, 0.0, profile[0][1]))
+
+ top_center = len(vertices)
+ vertices.append((0.0, 0.0, profile[-1][1]))
+
+ for ring in range(len(profile) - 1):
+ lower = ring * segments
+ upper = (ring + 1) * segments
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((
+ lower + i,
+ lower + j,
+ upper + j,
+ upper + i,
+ ))
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((bottom_center, j, i))
+
+ top_ring = (len(profile) - 1) * segments
+ faces.append((top_center, top_ring + i, top_ring + j))
+
+ mesh = bpy.data.meshes.new("SeatMesh")
+ mesh.from_pydata(vertices, [], faces)
+ mesh.materials.append(material)
+ mesh.update()
+
+ obj = bpy.data.objects.new("Seat", mesh)
+ bpy.context.scene.collection.objects.link(obj)
+
+ side_face_count = (len(profile) - 1) * segments
+ for polygon in mesh.polygons[:side_face_count]:
+ polygon.use_smooth = True
+
+ return obj
+
+
+def create_splayed_leg(name, lower_center, upper_center, lower_radius,
+ upper_radius, material, segments=24):
+ lower_center = Vector(lower_center)
+ upper_center = Vector(upper_center)
+
+ axis = upper_center - lower_center
+ axis_direction = axis.normalized()
+
+ reference = Vector((0.0, 0.0, 1.0))
+ basis_u = axis_direction.cross(reference).normalized()
+ basis_v = axis_direction.cross(basis_u).normalized()
+
+ vertices = []
+
+ # The ring vertices are adjusted along the leg axis so both cut faces are
+ # exactly horizontal. Every lower vertex therefore touches z=0.
+ for center, radius in (
+ (lower_center, lower_radius),
+ (upper_center, upper_radius),
+ ):
+ for i in range(segments):
+ angle = 2.0 * math.pi * i / segments
+ radial = radius * (
+ math.cos(angle) * basis_u +
+ math.sin(angle) * basis_v
+ )
+ axial_adjustment = -radial.z / axis_direction.z
+ point = center + radial + axis_direction * axial_adjustment
+ vertices.append(tuple(point))
+
+ bottom_center_index = len(vertices)
+ vertices.append(tuple(lower_center))
+
+ top_center_index = len(vertices)
+ vertices.append(tuple(upper_center))
+
+ faces = []
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((i, j, segments + j, segments + i))
+
+ for i in range(segments):
+ j = (i + 1) % segments
+ faces.append((bottom_center_index, j, i))
+ faces.append((
+ top_center_index,
+ segments + i,
+ segments + j,
+ ))
+
+ mesh = bpy.data.meshes.new(name + "Mesh")
+ mesh.from_pydata(vertices, [], faces)
+ mesh.materials.append(material)
+ mesh.update()
+
+ obj = bpy.data.objects.new(name, mesh)
+ bpy.context.scene.collection.objects.link(obj)
+
+ for polygon in mesh.polygons[:segments]:
+ polygon.use_smooth = True
+
+ return obj
+
+
+def main():
+ seat_material = create_material(
+ "SeatWood",
+ (0.36, 0.155, 0.055),
+ 0.38,
+ )
+ leg_material = create_material(
+ "LegWood",
+ (0.29, 0.105, 0.032),
+ 0.43,
+ )
+
+ create_seat(seat_material)
+
+ foot_radius = 0.172
+ attachment_radius = 0.078
+ attachment_height = 0.423
+
+ leg_specs = (
+ ("LegFront", -90.0),
+ ("LegRearRight", 30.0),
+ ("LegRearLeft", 150.0),
+ )
+
+ for name, angle_degrees in leg_specs:
+ angle = math.radians(angle_degrees)
+ direction = Vector((math.cos(angle), math.sin(angle), 0.0))
+
+ lower = direction * foot_radius
+ lower.z = 0.0
+
+ upper = direction * attachment_radius
+ upper.z = attachment_height
+
+ create_splayed_leg(
+ name=name,
+ lower_center=lower,
+ upper_center=upper,
+ lower_radius=0.014,
+ upper_radius=0.019,
+ material=leg_material,
+ )
+
+
+main()
+```
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/result.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/result.json
new file mode 100644
index 00000000..c623228a
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/result.json
@@ -0,0 +1,19 @@
+{
+ "ok": true,
+ "text": "```python\nimport bpy\nimport math\nfrom mathutils import Vector\n\n\ndef create_material(name, base_color, roughness):\n material = bpy.data.materials.new(name=name)\n material.use_nodes = True\n\n principled = material.node_tree.nodes.get(\"Principled BSDF\")\n principled.inputs[\"Base Color\"].default_value = (*base_color, 1.0)\n principled.inputs[\"Roughness\"].default_value = roughness\n principled.inputs[\"Metallic\"].default_value = 0.0\n\n return material\n\n\ndef create_seat(material):\n segments = 64\n\n # Rounded 30 cm diameter seat, with its top exactly at 45 cm.\n profile = [\n (0.136, 0.415),\n (0.142, 0.417),\n (0.147, 0.422),\n (0.150, 0.429),\n (0.150, 0.441),\n (0.148, 0.446),\n (0.142, 0.450),\n ]\n\n vertices = []\n faces = []\n\n for radius, z in profile:\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n vertices.append((\n radius * math.cos(angle),\n radius * math.sin(angle),\n z,\n ))\n\n bottom_center = len(vertices)\n vertices.append((0.0, 0.0, profile[0][1]))\n\n top_center = len(vertices)\n vertices.append((0.0, 0.0, profile[-1][1]))\n\n for ring in range(len(profile) - 1):\n lower = ring * segments\n upper = (ring + 1) * segments\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n lower + i,\n lower + j,\n upper + j,\n upper + i,\n ))\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((bottom_center, j, i))\n\n top_ring = (len(profile) - 1) * segments\n faces.append((top_center, top_ring + i, top_ring + j))\n\n mesh = bpy.data.meshes.new(\"SeatMesh\")\n mesh.from_pydata(vertices, [], faces)\n mesh.materials.append(material)\n mesh.update()\n\n obj = bpy.data.objects.new(\"Seat\", mesh)\n bpy.context.scene.collection.objects.link(obj)\n\n side_face_count = (len(profile) - 1) * segments\n for polygon in mesh.polygons[:side_face_count]:\n polygon.use_smooth = True\n\n return obj\n\n\ndef create_splayed_leg(name, lower_center, upper_center, lower_radius,\n upper_radius, material, segments=24):\n lower_center = Vector(lower_center)\n upper_center = Vector(upper_center)\n\n axis = upper_center - lower_center\n axis_direction = axis.normalized()\n\n reference = Vector((0.0, 0.0, 1.0))\n basis_u = axis_direction.cross(reference).normalized()\n basis_v = axis_direction.cross(basis_u).normalized()\n\n vertices = []\n\n # The ring vertices are adjusted along the leg axis so both cut faces are\n # exactly horizontal. Every lower vertex therefore touches z=0.\n for center, radius in (\n (lower_center, lower_radius),\n (upper_center, upper_radius),\n ):\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n radial = radius * (\n math.cos(angle) * basis_u +\n math.sin(angle) * basis_v\n )\n axial_adjustment = -radial.z / axis_direction.z\n point = center + radial + axis_direction * axial_adjustment\n vertices.append(tuple(point))\n\n bottom_center_index = len(vertices)\n vertices.append(tuple(lower_center))\n\n top_center_index = len(vertices)\n vertices.append(tuple(upper_center))\n\n faces = []\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((i, j, segments + j, segments + i))\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((bottom_center_index, j, i))\n faces.append((\n top_center_index,\n segments + i,\n segments + j,\n ))\n\n mesh = bpy.data.meshes.new(name + \"Mesh\")\n mesh.from_pydata(vertices, [], faces)\n mesh.materials.append(material)\n mesh.update()\n\n obj = bpy.data.objects.new(name, mesh)\n bpy.context.scene.collection.objects.link(obj)\n\n for polygon in mesh.polygons[:segments]:\n polygon.use_smooth = True\n\n return obj\n\n\ndef main():\n seat_material = create_material(\n \"SeatWood\",\n (0.36, 0.155, 0.055),\n 0.38,\n )\n leg_material = create_material(\n \"LegWood\",\n (0.29, 0.105, 0.032),\n 0.43,\n )\n\n create_seat(seat_material)\n\n foot_radius = 0.172\n attachment_radius = 0.078\n attachment_height = 0.423\n\n leg_specs = (\n (\"LegFront\", -90.0),\n (\"LegRearRight\", 30.0),\n (\"LegRearLeft\", 150.0),\n )\n\n for name, angle_degrees in leg_specs:\n angle = math.radians(angle_degrees)\n direction = Vector((math.cos(angle), math.sin(angle), 0.0))\n\n lower = direction * foot_radius\n lower.z = 0.0\n\n upper = direction * attachment_radius\n upper.z = attachment_height\n\n create_splayed_leg(\n name=name,\n lower_center=lower,\n upper_center=upper,\n lower_radius=0.014,\n upper_radius=0.019,\n material=leg_material,\n )\n\n\nmain()\n```",
+ "usage": {
+ "backend": "codex",
+ "model": "gpt-5.6-sol",
+ "input_tokens": 14611,
+ "output_tokens": 3462,
+ "cached_tokens": 9984,
+ "thoughts_tokens": 2163,
+ "tool_calls": 0,
+ "cost_usd": 0.19687700000000002,
+ "latency_ms": 75017
+ },
+ "tool_calls": 0,
+ "notes": "",
+ "duration_s": 75.02,
+ "transcript_dir": "bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/stderr.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/stderr.txt
new file mode 100644
index 00000000..ce1d88c6
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/stderr.txt
@@ -0,0 +1 @@
+Failed to write last message file "bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/last_message.md": No such file or directory (os error 2)
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/stdout.txt b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/stdout.txt
new file mode 100644
index 00000000..cd178af4
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/gen/attempt0/stdout.txt
@@ -0,0 +1,4 @@
+{"type":"thread.started","thread_id":"01a02f1d-447d-7321-8cb4-40fe826ef177"}
+{"type":"turn.started"}
+{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"```python\nimport bpy\nimport math\nfrom mathutils import Vector\n\n\ndef create_material(name, base_color, roughness):\n material = bpy.data.materials.new(name=name)\n material.use_nodes = True\n\n principled = material.node_tree.nodes.get(\"Principled BSDF\")\n principled.inputs[\"Base Color\"].default_value = (*base_color, 1.0)\n principled.inputs[\"Roughness\"].default_value = roughness\n principled.inputs[\"Metallic\"].default_value = 0.0\n\n return material\n\n\ndef create_seat(material):\n segments = 64\n\n # Rounded 30 cm diameter seat, with its top exactly at 45 cm.\n profile = [\n (0.136, 0.415),\n (0.142, 0.417),\n (0.147, 0.422),\n (0.150, 0.429),\n (0.150, 0.441),\n (0.148, 0.446),\n (0.142, 0.450),\n ]\n\n vertices = []\n faces = []\n\n for radius, z in profile:\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n vertices.append((\n radius * math.cos(angle),\n radius * math.sin(angle),\n z,\n ))\n\n bottom_center = len(vertices)\n vertices.append((0.0, 0.0, profile[0][1]))\n\n top_center = len(vertices)\n vertices.append((0.0, 0.0, profile[-1][1]))\n\n for ring in range(len(profile) - 1):\n lower = ring * segments\n upper = (ring + 1) * segments\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((\n lower + i,\n lower + j,\n upper + j,\n upper + i,\n ))\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((bottom_center, j, i))\n\n top_ring = (len(profile) - 1) * segments\n faces.append((top_center, top_ring + i, top_ring + j))\n\n mesh = bpy.data.meshes.new(\"SeatMesh\")\n mesh.from_pydata(vertices, [], faces)\n mesh.materials.append(material)\n mesh.update()\n\n obj = bpy.data.objects.new(\"Seat\", mesh)\n bpy.context.scene.collection.objects.link(obj)\n\n side_face_count = (len(profile) - 1) * segments\n for polygon in mesh.polygons[:side_face_count]:\n polygon.use_smooth = True\n\n return obj\n\n\ndef create_splayed_leg(name, lower_center, upper_center, lower_radius,\n upper_radius, material, segments=24):\n lower_center = Vector(lower_center)\n upper_center = Vector(upper_center)\n\n axis = upper_center - lower_center\n axis_direction = axis.normalized()\n\n reference = Vector((0.0, 0.0, 1.0))\n basis_u = axis_direction.cross(reference).normalized()\n basis_v = axis_direction.cross(basis_u).normalized()\n\n vertices = []\n\n # The ring vertices are adjusted along the leg axis so both cut faces are\n # exactly horizontal. Every lower vertex therefore touches z=0.\n for center, radius in (\n (lower_center, lower_radius),\n (upper_center, upper_radius),\n ):\n for i in range(segments):\n angle = 2.0 * math.pi * i / segments\n radial = radius * (\n math.cos(angle) * basis_u +\n math.sin(angle) * basis_v\n )\n axial_adjustment = -radial.z / axis_direction.z\n point = center + radial + axis_direction * axial_adjustment\n vertices.append(tuple(point))\n\n bottom_center_index = len(vertices)\n vertices.append(tuple(lower_center))\n\n top_center_index = len(vertices)\n vertices.append(tuple(upper_center))\n\n faces = []\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((i, j, segments + j, segments + i))\n\n for i in range(segments):\n j = (i + 1) % segments\n faces.append((bottom_center_index, j, i))\n faces.append((\n top_center_index,\n segments + i,\n segments + j,\n ))\n\n mesh = bpy.data.meshes.new(name + \"Mesh\")\n mesh.from_pydata(vertices, [], faces)\n mesh.materials.append(material)\n mesh.update()\n\n obj = bpy.data.objects.new(name, mesh)\n bpy.context.scene.collection.objects.link(obj)\n\n for polygon in mesh.polygons[:segments]:\n polygon.use_smooth = True\n\n return obj\n\n\ndef main():\n seat_material = create_material(\n \"SeatWood\",\n (0.36, 0.155, 0.055),\n 0.38,\n )\n leg_material = create_material(\n \"LegWood\",\n (0.29, 0.105, 0.032),\n 0.43,\n )\n\n create_seat(seat_material)\n\n foot_radius = 0.172\n attachment_radius = 0.078\n attachment_height = 0.423\n\n leg_specs = (\n (\"LegFront\", -90.0),\n (\"LegRearRight\", 30.0),\n (\"LegRearLeft\", 150.0),\n )\n\n for name, angle_degrees in leg_specs:\n angle = math.radians(angle_degrees)\n direction = Vector((math.cos(angle), math.sin(angle), 0.0))\n\n lower = direction * foot_radius\n lower.z = 0.0\n\n upper = direction * attachment_radius\n upper.z = attachment_height\n\n create_splayed_leg(\n name=name,\n lower_center=lower,\n upper_center=upper,\n lower_radius=0.014,\n upper_radius=0.019,\n material=leg_material,\n )\n\n\nmain()\n```"}}
+{"type":"turn.completed","usage":{"input_tokens":14611,"cached_input_tokens":9984,"cache_write_input_tokens":0,"output_tokens":3462,"reasoning_output_tokens":2163}}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/cell.json b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/cell.json
new file mode 100644
index 00000000..025624a6
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/cell.json
@@ -0,0 +1,29 @@
+{
+ "prompt_id": "cmp_easy_stool",
+ "tier": "easy",
+ "arm": "oneshot:gemini:gemini-3.7-flash",
+ "kind": "oneshot",
+ "target": "gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "status": "no_code",
+ "score": 0.0,
+ "score_std": 0.0,
+ "passed": false,
+ "build_ok": false,
+ "gate_errors": [],
+ "tris": null,
+ "gen_cost_usd": 0.0,
+ "judge_cost_usd": 0.0,
+ "gen_seconds": 65.39,
+ "wall_s": 65.4,
+ "attempts": 1,
+ "tool_calls": 0,
+ "criteria": {},
+ "harness_status": "",
+ "harness_rounds": 0,
+ "harness_loop_score": null,
+ "sheet": "",
+ "glb": "",
+ "workspace": "bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash",
+ "error": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later."
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/eval b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/eval
new file mode 160000
index 00000000..c5372685
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/eval
@@ -0,0 +1 @@
+Subproject commit c537268577793d592345e84d026a60f420f210a0
diff --git a/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/gen/attempt0/prompt.md b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/gen/attempt0/prompt.md
new file mode 100644
index 00000000..26df6c8c
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash/gen/attempt0/prompt.md
@@ -0,0 +1,26 @@
+Model this object in raw bpy: a simple three-legged round wooden stool, 45 cm tall
+Dimensions (m): height=0.45
+MUST HAVE: round seat
+MUST HAVE: three splayed legs of equal length
+MUST HAVE: legs touch the ground
+
+You write ONE file, `src/model.py`, in raw Blender Python (bpy, Blender 5.x API).
+The harness runs it headless (`blender -b --factory-startup --python <wrapper> -- --script src/model.py`)
+inside an EMPTIED scene (no default cube / camera / light) and then exports the result itself.
+
+Frame, units and placement: Coordinate frame: Z is UP, -Y is the FRONT of the object (the face a viewer sees from the default front view), +X is the object's right. Units are meters. The object stands on the ground plane z=0 and its footprint is centred on the Z axis. Build at real-world size.
+
+Naming: one mesh object per part, `obj.name` set to a unique PascalCase part name (e.g. `Seat`,
+`LegFrontLeft`); no auto-suffixed duplicates like `Leg.001`. Link every object to
+`bpy.context.scene.collection`. Every visible mesh gets a material (Principled BSDF base colour /
+roughness / metallic).
+
+Do NOT: create cameras or lights, touch render/world settings, call `bpy.ops.render.*`,
+`bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the network, or
+import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Keep the total under
+500k triangles and finish in under 120 s.
+
+The script must build the whole object when executed top-to-bottom (call your `main()` at module
+level). Make sure the code runs without errors in headless Blender — there is no second chance.
+
+You have NO tools in this session: you cannot write files, run Blender or inspect anything — the code must appear in your reply. Reply with the COMPLETE contents of `src/model.py` as ONE ```python fenced code block and nothing else (no prose before or after, no partial snippets).
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/matrix.json b/harness/bench/out/compare_v1_live2/matrix.json
new file mode 100644
index 00000000..44cdb1f2
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/matrix.json
@@ -0,0 +1,157 @@
+{
+ "battery": {
+ "name": "compare_v1",
+ "track": "static_object",
+ "language": "blender",
+ "description": "8 static-object prompts (blender / bpy) used to compare the harness loop against raw one-shot generation by strong models under ONE fixed judge. Tiers: 2 easy, 3 medium, 3 hard. must_have items become the fixed judge's acceptance checklist for EVERY arm (no plan-derived items, so harness and one-shot are judged alike).\n",
+ "prompts": [
+ {
+ "id": "cmp_easy_stool",
+ "prompt": "a simple three-legged round wooden stool, 45 cm tall",
+ "tier": "easy",
+ "category": "furniture",
+ "must_have": [
+ "round seat",
+ "three splayed legs of equal length",
+ "legs touch the ground"
+ ],
+ "dimensions_m": {
+ "height": 0.45
+ },
+ "tags": []
+ },
+ {
+ "id": "cmp_easy_desk_lamp",
+ "prompt": "a desk lamp with a heavy round base, a single angled arm and a conical metal shade",
+ "tier": "easy",
+ "category": "lighting",
+ "must_have": [
+ "round base on the ground",
+ "one straight arm leaning forward",
+ "conical shade at the top of the arm",
+ "shade opens downward"
+ ],
+ "dimensions_m": null,
+ "tags": []
+ },
+ {
+ "id": "cmp_med_dining_chair",
+ "prompt": "a wooden dining chair with a gently curved backrest, a flat seat and four tapered legs",
+ "tier": "medium",
+ "category": "furniture",
+ "must_have": [
+ "four tapered legs touching the ground",
+ "flat seat slab",
+ "backrest visibly curved in plan view",
+ "back legs continue up into the backrest posts"
+ ],
+ "dimensions_m": null,
+ "tags": []
+ },
+ {
+ "id": "cmp_med_wheelbarrow",
+ "prompt": "a garden wheelbarrow with a sheet-metal tray, one front wheel, two straight handles and two rear legs",
+ "tier": "medium",
+ "category": "garden",
+ "must_have": [
+ "open tray with sloped front",
+ "single front wheel with a hub",
+ "two parallel handles at the back",
+ "two rear legs reaching the ground"
+ ],
+ "dimensions_m": null,
+ "tags": []
+ },
+ {
+ "id": "cmp_med_espresso_machine",
+ "prompt": "a home espresso machine with a boxy body, a portafilter group head at the front, a steam wand on the right side and a drip tray",
+ "tier": "medium",
+ "category": "appliances",
+ "must_have": [
+ "boxy body",
+ "group head with a portafilter handle at the front",
+ "steam wand on the right side",
+ "drip tray at the bottom front",
+ "cup warmer top surface"
+ ],
+ "dimensions_m": null,
+ "tags": []
+ },
+ {
+ "id": "cmp_hard_bicycle",
+ "prompt": "a vintage bicycle with a diamond frame, two spoked wheels, handlebars, saddle, pedals with a chainring and a rear rack",
+ "tier": "hard",
+ "category": "vehicles",
+ "must_have": [
+ "two equal wheels with spokes",
+ "diamond frame tubes connecting the wheels",
+ "saddle above the rear wheel",
+ "handlebars above the front wheel",
+ "chainring and pedals between the wheels",
+ "rear luggage rack"
+ ],
+ "dimensions_m": null,
+ "tags": []
+ },
+ {
+ "id": "cmp_hard_acoustic_guitar",
+ "prompt": "an acoustic guitar standing upright on a small floor stand, with a waisted body, a round sound hole, a fretted neck with a headstock and six tuning pegs",
+ "tier": "hard",
+ "category": "instruments",
+ "must_have": [
+ "waisted hourglass body",
+ "round sound hole",
+ "neck with frets",
+ "headstock with six tuning pegs",
+ "bridge on the body",
+ "stand holding the guitar upright on the ground"
+ ],
+ "dimensions_m": null,
+ "tags": []
+ },
+ {
+ "id": "cmp_hard_sailboat",
+ "prompt": "a small wooden sailboat on a display cradle, with a hull, a keel, one mast with a boom and a triangular mainsail, a rudder and a tiller",
+ "tier": "hard",
+ "category": "vehicles",
+ "must_have": [
+ "hull with a pointed bow",
+ "keel under the hull",
+ "single mast with a boom",
+ "triangular mainsail",
+ "rudder and tiller at the stern",
+ "cradle keeping the boat upright on the ground"
+ ],
+ "dimensions_m": null,
+ "tags": []
+ }
+ ]
+ },
+ "arms": [
+ "harness:api-agent:gemini:gemini-3.7-flash",
+ "harness:gemini-cli:gemini-3.7-flash",
+ "oneshot:claude-code",
+ "oneshot:codex",
+ "oneshot:gemini:gemini-3.7-flash"
+ ],
+ "options": {
+ "judge": "gemini:gemini-3.1-pro-preview",
+ "loop_judge": null,
+ "planner": null,
+ "rounds": 3,
+ "max_usd": 2.5,
+ "max_minutes": 45.0,
+ "parallel": 3,
+ "limit": 2,
+ "ids": [],
+ "resume": true,
+ "gen_timeout_s": 900.0,
+ "repair_attempts": 2,
+ "n_samples": 2,
+ "pairwise": true,
+ "redo_status": [
+ "no_code"
+ ]
+ },
+ "started_at": "2026-08-23T15:01:59.131769+00:00"
+}
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/report.html b/harness/bench/out/compare_v1_live2/report.html
new file mode 100644
index 00000000..4641fafc
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/report.html
@@ -0,0 +1 @@
+<!doctype html><meta charset='utf-8'><title>compare compare_v1</title><style>body{font-family:system-ui,sans-serif;margin:24px;background:#111;color:#eee}table{border-collapse:collapse;margin:12px 0}td,th{border:1px solid #444;padding:4px 8px;font-size:13px;vertical-align:top}th{background:#222}.g img{width:100%;display:block;border-radius:4px}.g td{min-width:260px;max-width:360px}.ok{color:#8f8}.bad{color:#f88}pre{white-space:pre-wrap;font-size:11px;color:#bbb}</style><h1>harness vs one-shot — compare_v1</h1><p>fixed judge gemini:gemini-3.1-pro-preview</p><table><tr><th>arm</th><th>n</th><th>mean</th><th>pass</th><th>build ok</th><th>$gen</th><th>min</th></tr><tr><td>oneshot:codex</td><td>2</td><td>0.786</td><td>50%</td><td>100%</td><td>0.20</td><td>2.1</td></tr><tr><td>oneshot:claude-code</td><td>2</td><td>0.673</td><td>0%</td><td>100%</td><td>0.95</td><td>4.0</td></tr><tr><td>oneshot:gemini:gemini-3.7-flash</td><td>2</td><td>0.000</td><td>0%</td><td>0%</td><td>0.01</td><td>0.9</td></tr></table><h2>pairwise (A wins / B wins / ties)</h2><table><tr><th>A (harness)</th><th>B (one-shot)</th><th>w/l/t</th><th>A win rate</th></tr></table><h2>gallery</h2><table class='g'><tr><th>prompt</th><th>oneshot:claude-code</th><th>oneshot:codex</th><th>oneshot:gemini:gemini-3.7-flash</th></tr><tr><td><b>cmp_easy_stool</b></td><td><img src='report_assets/cmp_easy_stool__oneshot_claude-code.png' loading='lazy'><b class='bad'>0.65</b> scored · tris 2516 · $0.73 · 3.3 min</td><td><img src='report_assets/cmp_easy_stool__oneshot_codex.png' loading='lazy'><b class='ok'>0.90</b> pass · tris 1184 · $0.20 · 2.1 min</td><td><b class='bad'>0.00</b> no_code · tris - · $0.00 · 0.9 min<pre>ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.</pre></td></tr><tr><td><b>cmp_easy_desk_lamp</b></td><td><img src='report_assets/cmp_easy_desk_lamp__oneshot_claude-code.png' loading='lazy'><b class='bad'>0.70</b> scored · tris 5824 · $1.17 · 4.8 min</td><td><img src='report_assets/cmp_easy_desk_lamp__oneshot_codex.png' loading='lazy'><b class='bad'>0.67</b> scored · tris 1664 · $0.21 · 2.0 min</td><td><b class='bad'>0.00</b> build_failed · tris - · $0.03 · 0.9 min<pre>AttributeError: BMeshOpsModule: operator &quot;create_cylinder&quot; doesn&#x27;t exist</pre></td></tr></table>
\ No newline at end of file
diff --git a/harness/bench/out/compare_v1_live2/report.md b/harness/bench/out/compare_v1_live2/report.md
new file mode 100644
index 00000000..60c509c3
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/report.md
@@ -0,0 +1,32 @@
+# harness vs one-shot — compare_v1
+
+fixed judge: **gemini:gemini-3.1-pro-preview** (rubric static_object_v1, n_samples=2, acceptance = must_have list) · harness loop judge: settings default · harness rounds ≤ 3, ≤ $2.5 · cells: 6
+
+Every arm's final `src/model.py` is re-built, re-rendered and judged by the same evaluator; a failed build scores 0. `$gen` for harness arms is the whole run (planner + generator + its loop judge); for one-shot arms it is the single call (subscription CLIs report 0 unless the CLI returns a cost).
+
+## arms
+
+| arm | kind | n | mean | median | pass | build ok | $gen | $judge | min | tool calls | errors |
+|---|---|---|---|---|---|---|---|---|---|---|---|
+| oneshot:codex | oneshot | 2 | 0.786 | 0.786 | 50% | 100% | 0.20 | 0.091 | 2.1 | 0 | 0 |
+| oneshot:claude-code | oneshot | 2 | 0.673 | 0.673 | 0% | 100% | 0.95 | 0.088 | 4.0 | 0 | 0 |
+| oneshot:gemini:gemini-3.7-flash | oneshot | 2 | 0.000 | 0.000 | 0% | 0% | 0.01 | 0.000 | 0.9 | 0 | 0 |
+
+## per prompt (score, ✓ = passed, ✗build = build failed)
+
+| prompt | tier | oneshot:claude-code | oneshot:codex | oneshot:gemini:gemini-3.7-flash |
+|---|---|---|---|---|
+| cmp_easy_stool | easy | 0.65 | 0.90✓ | 0.00 ✗build |
+| cmp_easy_desk_lamp | easy | 0.70 | 0.67 | 0.00 ✗build |
+
+## pairwise arena (PairwiseJudge, both orders; ties = orderings disagree or equal)
+
+| harness arm (A) | one-shot arm (B) | n | A wins | B wins | ties | A win rate | conf |
+|---|---|---|---|---|---|---|---|
+
+**Total cost**: generation $2.33 · fixed judge $0.36 · pairwise $0.00 (subscription CLIs report their own cost figure or 0).
+
+## errors
+
+- `cmp_easy_stool` / `oneshot:gemini:gemini-3.7-flash` (no_code): ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+- `cmp_easy_desk_lamp` / `oneshot:gemini:gemini-3.7-flash` (build_failed): AttributeError: BMeshOpsModule: operator "create_cylinder" doesn't exist
diff --git a/harness/bench/out/compare_v1_live2/results.jsonl b/harness/bench/out/compare_v1_live2/results.jsonl
new file mode 100644
index 00000000..8f5dfa13
--- /dev/null
+++ b/harness/bench/out/compare_v1_live2/results.jsonl
@@ -0,0 +1,9 @@
+{"prompt_id":"cmp_easy_stool","tier":"easy","arm":"oneshot:codex","kind":"oneshot","target":"codex","judge":"gemini:gemini-3.1-pro-preview","status":"scored","score":0.9045,"score_std":0.0405,"passed":true,"build_ok":true,"gate_errors":[],"tris":1184,"gen_cost_usd":0.19687700000000002,"judge_cost_usd":0.089192,"gen_seconds":75.02,"wall_s":126.9,"attempts":1,"tool_calls":0,"criteria":{"intent_fidelity":0.925,"structure_plausibility":0.95,"geometry_detail":0.825,"proportions_scale":0.975,"assembly_fit":0.925,"materials":0.7,"craftsmanship_no_artifacts":1.0},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/eval/artifacts/renders/r00/sheet.png","glb":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex/eval/artifacts/object.glb","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_codex","error":""}
+{"prompt_id":"cmp_easy_stool","tier":"easy","arm":"oneshot:claude-code","kind":"oneshot","target":"claude-code","judge":"gemini:gemini-3.1-pro-preview","status":"scored","score":0.646,"score_std":0.052,"passed":false,"build_ok":true,"gate_errors":["connectivity: 'LegFrontRight' and 'Seat' interpenetrate by ≈15.9 mm (5% of surface samples inside)","connectivity: 'LegFrontLeft' and 'Seat' interpenetrate by ≈15.2 mm (5% of surface samples inside)","connectivity: 'LegBack' and 'Seat' interpenetrate by ≈16.1 mm (5% of surface samples inside)"],"tris":2516,"gen_cost_usd":0.7319350000000001,"judge_cost_usd":0.09858,"gen_seconds":143.03,"wall_s":196.2,"attempts":1,"tool_calls":0,"criteria":{"intent_fidelity":0.95,"structure_plausibility":0.85,"geometry_detail":0.7,"proportions_scale":0.95,"assembly_fit":0.4,"materials":0.7,"craftsmanship_no_artifacts":0.7},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/eval/artifacts/renders/r00/sheet.png","glb":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code/eval/artifacts/object.glb","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_claude-code","error":""}
+{"prompt_id":"cmp_easy_desk_lamp","tier":"easy","arm":"oneshot:codex","kind":"oneshot","target":"codex","judge":"gemini:gemini-3.1-pro-preview","status":"scored","score":0.6675,"score_std":0.0105,"passed":false,"build_ok":true,"gate_errors":["connectivity: 'AngledArm' and 'RoundBase' interpenetrate by ≈11.7 mm (3% of surface samples inside)"],"tris":1664,"gen_cost_usd":0.20707199999999998,"judge_cost_usd":0.093532,"gen_seconds":68.02,"wall_s":121.6,"attempts":1,"tool_calls":0,"criteria":{"intent_fidelity":0.85,"structure_plausibility":0.75,"geometry_detail":0.675,"proportions_scale":0.75,"assembly_fit":0.5,"materials":0.8,"craftsmanship_no_artifacts":0.875},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/eval/artifacts/renders/r00/sheet.png","glb":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex/eval/artifacts/object.glb","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_codex","error":""}
+{"prompt_id":"cmp_easy_stool","tier":"easy","arm":"oneshot:gemini:gemini-3.7-flash","kind":"oneshot","target":"gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview","status":"no_code","score":0.0,"score_std":0.0,"passed":false,"build_ok":false,"gate_errors":[],"tris":null,"gen_cost_usd":0.0,"judge_cost_usd":0.0,"gen_seconds":56.96,"wall_s":57.0,"attempts":1,"tool_calls":0,"criteria":{},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"","glb":"","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash","error":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later."}
+{"prompt_id":"cmp_easy_desk_lamp","tier":"easy","arm":"oneshot:claude-code","kind":"oneshot","target":"claude-code","judge":"gemini:gemini-3.1-pro-preview","status":"scored","score":0.7,"score_std":0.012,"passed":false,"build_ok":true,"gate_errors":["connectivity: 'ShadeNeck' and 'ShadeJoint' interpenetrate by ≈15.5 mm (45% of surface samples inside)","connectivity: 'ShadeJoint' and 'Arm' interpenetrate by ≈14.4 mm (9% of surface samples inside)","connectivity: 'Arm' and 'ArmJoint' interpenetrate by ≈13.0 mm (8% of surface samples inside)","connectivity: 'ArmJoint' and 'BaseCollar' interpenetrate by ≈14.3 mm (41% of surface samples inside)"],"tris":5824,"gen_cost_usd":1.16671,"judge_cost_usd":0.0774098,"gen_seconds":241.05,"wall_s":290.0,"attempts":1,"tool_calls":0,"criteria":{"intent_fidelity":1.0,"structure_plausibility":0.9,"geometry_detail":0.8,"proportions_scale":1.0,"assembly_fit":0.5,"materials":0.9,"craftsmanship_no_artifacts":1.0},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/eval/artifacts/renders/r00/sheet.png","glb":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code/eval/artifacts/object.glb","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_claude-code","error":""}
+{"prompt_id":"cmp_easy_desk_lamp","tier":"easy","arm":"oneshot:gemini:gemini-3.7-flash","kind":"oneshot","target":"gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview","status":"build_failed","score":0.0,"score_std":0.0,"passed":false,"build_ok":false,"gate_errors":["lint:blender: Principled BSDF input 'Emission' does not exist in Blender 4.x/5.x (KeyError at runtime)"],"tris":null,"gen_cost_usd":0.0284595,"judge_cost_usd":0.0,"gen_seconds":55.18,"wall_s":55.6,"attempts":1,"tool_calls":0,"criteria":{},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"","glb":"/home/yipeng/3dcodeverse/harness/bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash/eval/artifacts/object.glb","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/oneshot_gemini_gemini-3.7-flash","error":"AttributeError: BMeshOpsModule: operator \"create_cylinder\" doesn't exist"}
+{"prompt_id":"cmp_easy_desk_lamp","tier":"easy","arm":"harness:api-agent:gemini:gemini-3.7-flash","kind":"harness","target":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview","status":"error","score":null,"score_std":0.0,"passed":null,"build_ok":false,"gate_errors":[],"tris":null,"gen_cost_usd":0.0,"judge_cost_usd":0.0,"gen_seconds":0.0,"wall_s":65.0,"attempts":0,"tool_calls":0,"criteria":{},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"","glb":"","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_desk_lamp/harness_api-agent_gemini_gemini-3.7-flash","error":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\nevents, runtime=runtime,\n ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n template=template, example=example, temperature=temperature,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n max_output_tokens=max_output_tokens, finalise=finalise, event_stats=event_stats)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 108, in plan_with_usage\n resp = model.generate(req)\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 203, in generate\n return rotate_with_retries(\n self.pool,\n ...<10 lines>...\n label=f\"gemini {self.model}\",\n )\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/retry.py\", line 177, in rotate_with_retries\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n"}
+{"prompt_id":"cmp_easy_stool","tier":"easy","arm":"oneshot:gemini:gemini-3.7-flash","kind":"oneshot","target":"gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview","status":"no_code","score":0.0,"score_std":0.0,"passed":false,"build_ok":false,"gate_errors":[],"tris":null,"gen_cost_usd":0.0,"judge_cost_usd":0.0,"gen_seconds":65.39,"wall_s":65.4,"attempts":1,"tool_calls":0,"criteria":{},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"","glb":"","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_stool/oneshot_gemini_gemini-3.7-flash","error":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later."}
+{"prompt_id":"cmp_easy_stool","tier":"easy","arm":"harness:gemini-cli:gemini-3.7-flash","kind":"harness","target":"gemini-cli:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview","status":"error","score":null,"score_std":0.0,"passed":null,"build_ok":false,"gate_errors":[],"tris":null,"gen_cost_usd":0.0,"judge_cost_usd":0.0,"gen_seconds":0.0,"wall_s":68.3,"attempts":0,"tool_calls":0,"criteria":{},"harness_status":"","harness_rounds":0,"harness_loop_score":null,"sheet":"","glb":"","workspace":"bench/out/compare_v1_live2/cells/cmp_easy_stool/harness_gemini-cli_gemini-3.7-flash","error":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\nevents, runtime=runtime,\n ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n template=template, example=example, temperature=temperature,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n max_output_tokens=max_output_tokens, finalise=finalise, event_stats=event_stats)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 108, in plan_with_usage\n resp = model.generate(req)\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 203, in generate\n return rotate_with_retries(\n self.pool,\n ...<10 lines>...\n label=f\"gemini {self.model}\",\n )\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/retry.py\", line 177, in rotate_with_retries\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n"}
diff --git a/harness/bench/out/static_v1_flash/report.md b/harness/bench/out/static_v1_flash/report.md
new file mode 100644
index 00000000..90a72b89
--- /dev/null
+++ b/harness/bench/out/static_v1_flash/report.md
@@ -0,0 +1,59 @@
+# bench report — static_objects_v1
+
+runs: 24 · generator: api-agent:gemini:gemini-3.7-flash · judge: gemini:gemini-3.1-pro-preview · rounds ≤ 3
+
+### overall
+
+| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors |
+|---|---|---|---|---|---|---|---|---|---|---|
+| all | 24 | 19 | 0.643 | 0.718 | 0.750 | +0.075 | 58% | 1.10 | 15.6 | 12 |
+
+### by tier
+
+| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors |
+|---|---|---|---|---|---|---|---|---|---|---|
+| easy | 8 | 7 | 0.751 | 0.861 | 0.850 | +0.110 | 100% | 0.56 | 9.3 | 1 |
+| medium | 8 | 5 | 0.545 | 0.643 | 0.600 | +0.098 | 20% | 1.18 | 16.1 | 7 |
+| hard | 8 | 7 | 0.606 | 0.630 | 0.600 | +0.024 | 43% | 1.56 | 21.4 | 4 |
+
+### by category
+
+| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors |
+|---|---|---|---|---|---|---|---|---|---|---|
+| appliances | 3 | 2 | 0.741 | 0.841 | 0.841 | +0.101 | 100% | 0.42 | 10.3 | 1 |
+| buildings | 3 | 3 | 0.667 | 0.726 | 0.746 | +0.060 | 67% | 1.14 | 12.9 | 1 |
+| furniture | 3 | 2 | 0.725 | 0.750 | 0.750 | +0.025 | 100% | 0.51 | 9.3 | 1 |
+| musical_instruments | 3 | 1 | 0.600 | 0.600 | 0.600 | +0.000 | 0% | 0.63 | 9.1 | 2 |
+| plants | 3 | 2 | 0.633 | 0.633 | 0.633 | +0.000 | 50% | 0.87 | 12.5 | 2 |
+| tools | 3 | 3 | 0.662 | 0.685 | 0.576 | +0.023 | 33% | 1.83 | 21.1 | 2 |
+| toys | 3 | 3 | 0.735 | 0.839 | 0.867 | +0.104 | 67% | 1.20 | 14.7 | 1 |
+| vehicles | 3 | 3 | 0.411 | 0.616 | 0.526 | +0.204 | 33% | 2.19 | 35.0 | 2 |
+
+### per prompt
+
+| id | tier | category | baseline | final | passed | rounds | $ | min | status |
+|---|---|---|---|---|---|---|---|---|---|
+| appl_easy_kettle | easy | appliances | 0.600 | 0.801 | yes | 2 | 0.94 | 22.6 | passed |
+| bld_easy_shed | easy | buildings | 0.833 | 0.833 | yes | 1 | 0.41 | 6.2 | passed |
+| furn_easy_stool | easy | furniture | 0.750 | 0.750 | yes | 1 | 0.30 | 4.5 | passed |
+| mus_easy_drum | easy | musical_instruments | - | - | - | 0 | 0.00 | 1.1 | error ⚠ |
+| plant_easy_cactus | easy | plants | 0.883 | 0.883 | yes | 1 | 0.44 | 10.0 | passed |
+| tool_easy_hammer | easy | tools | 0.944 | 0.944 | yes | 1 | 0.40 | 5.6 | passed |
+| toy_easy_blocks | easy | toys | 0.964 | 0.964 | yes | 1 | 0.29 | 4.1 | passed |
+| veh_easy_toy_car | easy | vehicles | 0.282 | 0.850 | yes | 3 | 1.68 | 20.8 | passed |
+| appl_med_toaster | medium | appliances | 0.881 | 0.881 | yes | 1 | 0.32 | 5.5 | passed |
+| bld_med_lighthouse | medium | buildings | 0.467 | 0.600 | no | 2 | 2.28 | 21.9 | budget ⚠ |
+| furn_med_dining_chair | medium | furniture | - | - | - | 0 | 0.00 | 0.9 | error ⚠ |
+| mus_med_acoustic_guitar | medium | musical_instruments | - | - | - | 0 | 0.00 | 1.5 | error ⚠ |
+| plant_med_palm | medium | plants | - | - | - | 0 | 0.00 | 1.4 | error ⚠ |
+| tool_med_hand_drill | medium | tools | 0.576 | 0.576 | no | 1 | 2.06 | 22.3 | budget ⚠ |
+| toy_med_rocking_horse | medium | toys | 0.374 | 0.685 | no | 2 | 2.62 | 28.8 | budget ⚠ |
+| veh_med_pickup | medium | vehicles | 0.426 | 0.471 | no | 2 | 2.14 | 46.5 | budget ⚠ |
+| appl_hard_espresso | hard | appliances | - | - | - | 0 | 0.00 | 2.7 | error ⚠ |
+| bld_hard_windmill | hard | buildings | 0.700 | 0.746 | yes | 2 | 0.73 | 10.7 | passed |
+| furn_hard_rolltop_desk | hard | furniture | 0.700 | 0.750 | yes | 3 | 1.22 | 22.5 | passed |
+| mus_hard_drum_kit | hard | musical_instruments | 0.600 | 0.600 | no | 3 | 1.89 | 24.7 | plateau |
+| plant_hard_bonsai | hard | plants | 0.384 | 0.384 | no | 1 | 2.18 | 26.1 | budget ⚠ |
+| tool_hard_bench_vise | hard | tools | 0.466 | 0.536 | no | 2 | 3.02 | 35.4 | budget ⚠ |
+| toy_hard_train | hard | toys | 0.867 | 0.867 | yes | 1 | 0.70 | 11.0 | passed |
+| veh_hard_tractor | hard | vehicles | 0.526 | 0.526 | no | 2 | 2.75 | 37.8 | budget ⚠ |
diff --git a/harness/bench/out/static_v1_flash/results.csv b/harness/bench/out/static_v1_flash/results.csv
new file mode 100644
index 00000000..acd19256
--- /dev/null
+++ b/harness/bench/out/static_v1_flash/results.csv
@@ -0,0 +1,136 @@
+id,tier,category,score_baseline,score_final,passed,rounds,cost_usd,minutes,status,errors,workspace,generator,judge
+furn_easy_stool,easy,furniture,0.75,0.75,True,1,0.30198767499999996,4.46,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/furn_easy_stool,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+furn_med_dining_chair,medium,furniture,,,,0,0.0,0.92,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+l=self.plan_model)
+ ^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ~~~~~~~~~~~~~~~~^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ plan = run_planner(ctx.spec, ctx.spec.backends.planner, self.plan_model, ctx.ws, model=self._planner_model,
+ events=ctx.events, budget=ctx.budget, runtime=ctx.runtime)
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ result, usage = plan_with_usage(spec, model_id, plan_model, ws, model=model, events=events, runtime=runtime)
+ ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ resp = model.generate(req)
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/furn_med_dining_chair,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+furn_hard_rolltop_desk,hard,furniture,0.7,0.75,True,3,1.2165176,22.52,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/furn_hard_rolltop_desk,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+veh_easy_toy_car,easy,vehicles,0.282,0.85,True,3,1.6814668499999998,20.78,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_easy_toy_car,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+veh_med_pickup,medium,vehicles,0.426,0.471,False,2,2.144166425,46.46,budget,cost $2.144 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_med_pickup,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+veh_hard_tractor,hard,vehicles,0.526,0.526,False,2,2.75278335,37.78,budget,cost $2.753 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_hard_tractor,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+tool_easy_hammer,easy,tools,0.944,0.944,True,1,0.40064702500000005,5.59,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_easy_hammer,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+tool_med_hand_drill,medium,tools,0.576,0.576,False,1,2.0616418,22.34,budget,cost $2.062 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_med_hand_drill,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+tool_hard_bench_vise,hard,tools,0.466,0.536,False,2,3.019153075,35.43,budget,cost $2.478 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_hard_bench_vise,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+appl_easy_kettle,easy,appliances,0.6,0.801,True,2,0.9444692499999997,22.57,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_easy_kettle,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+appl_med_toaster,medium,appliances,0.881,0.881,True,1,0.31813375,5.49,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_med_toaster,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+appl_hard_espresso,hard,appliances,,,,0,0.0,2.71,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+g/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in run
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ return []
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ try:
+
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ def __init__(self, message: str, usage: Usage | None = None):
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_hard_espresso,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+plant_easy_cactus,easy,plants,0.883,0.883,True,1,0.4371587499999999,9.99,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_easy_cactus,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+plant_med_palm,medium,plants,,,,0,0.0,1.36,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+g/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in run
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ return []
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ try:
+
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ def __init__(self, message: str, usage: Usage | None = None):
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_med_palm,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+plant_hard_bonsai,hard,plants,0.384,0.384,False,1,2.182076575,26.07,budget,cost $2.182 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_hard_bonsai,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+bld_easy_shed,easy,buildings,0.833,0.833,True,1,0.41163482500000015,6.23,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_easy_shed,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+bld_med_lighthouse,medium,buildings,0.467,0.6,False,2,2.275502575,21.92,budget,cost $2.276 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_med_lighthouse,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+bld_hard_windmill,hard,buildings,0.7,0.746,True,2,0.7304633250000001,10.7,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_hard_windmill,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+toy_easy_blocks,easy,toys,0.964,0.964,True,1,0.28989425,4.12,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_easy_blocks,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+toy_med_rocking_horse,medium,toys,0.374,0.685,False,2,2.6155671000000003,28.83,budget,cost $2.616 exceeds max_usd $2.00,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_med_rocking_horse,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+toy_hard_train,hard,toys,0.867,0.867,True,1,0.6989257500000001,11.04,passed,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_hard_train,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+mus_easy_drum,easy,musical_instruments,,,,0,0.0,1.05,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+g/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in run
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ return []
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ try:
+
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ def __init__(self, message: str, usage: Usage | None = None):
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_easy_drum,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+mus_med_acoustic_guitar,medium,musical_instruments,,,,0,0.0,1.53,error,"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+g/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in run
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ return []
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py"", line 91, in stage
+ result = fn()
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 147, in <lambda>
+ def round_files_hint(self, ctx: RunContext) -> list[str]:
+ ^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py"", line 230, in _plan_stage
+ try:
+
+ ctx.plan = runner.stage(""plan"", lambda: self._plan_stage(ctx), inputs={""spec"": plan_stage_inputs(spec), ""track"": self.track.value},
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 48, in plan
+ def __init__(self, message: str, usage: Usage | None = None):
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py"", line 70, in plan_with_usage
+ File ""/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py"", line 257, in generate
+ raise err from exc
+codeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.
+",/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_med_acoustic_guitar,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
+mus_hard_drum_kit,hard,musical_instruments,0.6,0.6,False,3,1.8923563749999999,24.67,plateau,,/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_hard_drum_kit,api-agent:gemini:gemini-3.7-flash,gemini:gemini-3.1-pro-preview
diff --git a/harness/bench/out/static_v1_flash/results.json b/harness/bench/out/static_v1_flash/results.json
new file mode 100644
index 00000000..bca0fa65
--- /dev/null
+++ b/harness/bench/out/static_v1_flash/results.json
@@ -0,0 +1,386 @@
+[
+ {
+ "id": "furn_easy_stool",
+ "tier": "easy",
+ "category": "furniture",
+ "score_baseline": 0.75,
+ "score_final": 0.75,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.30198767499999996,
+ "minutes": 4.46,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/furn_easy_stool",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "furn_med_dining_chair",
+ "tier": "medium",
+ "category": "furniture",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 0.92,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\nl=self.plan_model)\n ^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ~~~~~~~~~~~~~~~~^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n plan = run_planner(ctx.spec, ctx.spec.backends.planner, self.plan_model, ctx.ws, model=self._planner_model,\n events=ctx.events, budget=ctx.budget, runtime=ctx.runtime)\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n result, usage = plan_with_usage(spec, model_id, plan_model, ws, model=model, events=events, runtime=runtime)\n ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n resp = model.generate(req)\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/furn_med_dining_chair",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "furn_hard_rolltop_desk",
+ "tier": "hard",
+ "category": "furniture",
+ "score_baseline": 0.7,
+ "score_final": 0.75,
+ "passed": true,
+ "rounds": 3,
+ "cost_usd": 1.2165176,
+ "minutes": 22.52,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/furn_hard_rolltop_desk",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "veh_easy_toy_car",
+ "tier": "easy",
+ "category": "vehicles",
+ "score_baseline": 0.282,
+ "score_final": 0.85,
+ "passed": true,
+ "rounds": 3,
+ "cost_usd": 1.6814668499999998,
+ "minutes": 20.78,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_easy_toy_car",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "veh_med_pickup",
+ "tier": "medium",
+ "category": "vehicles",
+ "score_baseline": 0.426,
+ "score_final": 0.471,
+ "passed": false,
+ "rounds": 2,
+ "cost_usd": 2.144166425,
+ "minutes": 46.46,
+ "status": "budget",
+ "errors": "cost $2.144 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_med_pickup",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "veh_hard_tractor",
+ "tier": "hard",
+ "category": "vehicles",
+ "score_baseline": 0.526,
+ "score_final": 0.526,
+ "passed": false,
+ "rounds": 2,
+ "cost_usd": 2.75278335,
+ "minutes": 37.78,
+ "status": "budget",
+ "errors": "cost $2.753 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_hard_tractor",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "tool_easy_hammer",
+ "tier": "easy",
+ "category": "tools",
+ "score_baseline": 0.944,
+ "score_final": 0.944,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.40064702500000005,
+ "minutes": 5.59,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_easy_hammer",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "tool_med_hand_drill",
+ "tier": "medium",
+ "category": "tools",
+ "score_baseline": 0.576,
+ "score_final": 0.576,
+ "passed": false,
+ "rounds": 1,
+ "cost_usd": 2.0616418,
+ "minutes": 22.34,
+ "status": "budget",
+ "errors": "cost $2.062 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_med_hand_drill",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "tool_hard_bench_vise",
+ "tier": "hard",
+ "category": "tools",
+ "score_baseline": 0.466,
+ "score_final": 0.536,
+ "passed": false,
+ "rounds": 2,
+ "cost_usd": 3.019153075,
+ "minutes": 35.43,
+ "status": "budget",
+ "errors": "cost $2.478 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_hard_bench_vise",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "appl_easy_kettle",
+ "tier": "easy",
+ "category": "appliances",
+ "score_baseline": 0.6,
+ "score_final": 0.801,
+ "passed": true,
+ "rounds": 2,
+ "cost_usd": 0.9444692499999997,
+ "minutes": 22.57,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_easy_kettle",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "appl_med_toaster",
+ "tier": "medium",
+ "category": "appliances",
+ "score_baseline": 0.881,
+ "score_final": 0.881,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.31813375,
+ "minutes": 5.49,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_med_toaster",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "appl_hard_espresso",
+ "tier": "hard",
+ "category": "appliances",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 2.71,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_hard_espresso",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "plant_easy_cactus",
+ "tier": "easy",
+ "category": "plants",
+ "score_baseline": 0.883,
+ "score_final": 0.883,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.4371587499999999,
+ "minutes": 9.99,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_easy_cactus",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "plant_med_palm",
+ "tier": "medium",
+ "category": "plants",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 1.36,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_med_palm",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "plant_hard_bonsai",
+ "tier": "hard",
+ "category": "plants",
+ "score_baseline": 0.384,
+ "score_final": 0.384,
+ "passed": false,
+ "rounds": 1,
+ "cost_usd": 2.182076575,
+ "minutes": 26.07,
+ "status": "budget",
+ "errors": "cost $2.182 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_hard_bonsai",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "bld_easy_shed",
+ "tier": "easy",
+ "category": "buildings",
+ "score_baseline": 0.833,
+ "score_final": 0.833,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.41163482500000015,
+ "minutes": 6.23,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_easy_shed",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "bld_med_lighthouse",
+ "tier": "medium",
+ "category": "buildings",
+ "score_baseline": 0.467,
+ "score_final": 0.6,
+ "passed": false,
+ "rounds": 2,
+ "cost_usd": 2.275502575,
+ "minutes": 21.92,
+ "status": "budget",
+ "errors": "cost $2.276 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_med_lighthouse",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "bld_hard_windmill",
+ "tier": "hard",
+ "category": "buildings",
+ "score_baseline": 0.7,
+ "score_final": 0.746,
+ "passed": true,
+ "rounds": 2,
+ "cost_usd": 0.7304633250000001,
+ "minutes": 10.7,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_hard_windmill",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "toy_easy_blocks",
+ "tier": "easy",
+ "category": "toys",
+ "score_baseline": 0.964,
+ "score_final": 0.964,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.28989425,
+ "minutes": 4.12,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_easy_blocks",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "toy_med_rocking_horse",
+ "tier": "medium",
+ "category": "toys",
+ "score_baseline": 0.374,
+ "score_final": 0.685,
+ "passed": false,
+ "rounds": 2,
+ "cost_usd": 2.6155671000000003,
+ "minutes": 28.83,
+ "status": "budget",
+ "errors": "cost $2.616 exceeds max_usd $2.00",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_med_rocking_horse",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "toy_hard_train",
+ "tier": "hard",
+ "category": "toys",
+ "score_baseline": 0.867,
+ "score_final": 0.867,
+ "passed": true,
+ "rounds": 1,
+ "cost_usd": 0.6989257500000001,
+ "minutes": 11.04,
+ "status": "passed",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_hard_train",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "mus_easy_drum",
+ "tier": "easy",
+ "category": "musical_instruments",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 1.05,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_easy_drum",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "mus_med_acoustic_guitar",
+ "tier": "medium",
+ "category": "musical_instruments",
+ "score_baseline": null,
+ "score_final": null,
+ "passed": null,
+ "rounds": 0,
+ "cost_usd": 0.0,
+ "minutes": 1.53,
+ "status": "error",
+ "errors": "ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_med_acoustic_guitar",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ },
+ {
+ "id": "mus_hard_drum_kit",
+ "tier": "hard",
+ "category": "musical_instruments",
+ "score_baseline": 0.6,
+ "score_final": 0.6,
+ "passed": false,
+ "rounds": 3,
+ "cost_usd": 1.8923563749999999,
+ "minutes": 24.67,
+ "status": "plateau",
+ "errors": "",
+ "workspace": "/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_hard_drum_kit",
+ "generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview"
+ }
+]
\ No newline at end of file
diff --git a/harness/bench/out/static_v1_flash/results.jsonl b/harness/bench/out/static_v1_flash/results.jsonl
index c6180b1c..aa369a8c 100644
--- a/harness/bench/out/static_v1_flash/results.jsonl
+++ b/harness/bench/out/static_v1_flash/results.jsonl
@@ -3,3 +3,22 @@
{"id":"veh_easy_toy_car","tier":"easy","category":"vehicles","score_baseline":0.282,"score_final":0.85,"passed":true,"rounds":3,"cost_usd":1.6814668499999998,"minutes":20.78,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_easy_toy_car","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
{"id":"furn_hard_rolltop_desk","tier":"hard","category":"furniture","score_baseline":0.7,"score_final":0.75,"passed":true,"rounds":3,"cost_usd":1.2165176,"minutes":22.52,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/furn_hard_rolltop_desk","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
{"id":"tool_easy_hammer","tier":"easy","category":"tools","score_baseline":0.944,"score_final":0.944,"passed":true,"rounds":1,"cost_usd":0.40064702500000005,"minutes":5.59,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_easy_hammer","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"veh_hard_tractor","tier":"hard","category":"vehicles","score_baseline":0.526,"score_final":0.526,"passed":false,"rounds":2,"cost_usd":2.75278335,"minutes":37.78,"status":"budget","errors":"cost $2.753 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_hard_tractor","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"tool_med_hand_drill","tier":"medium","category":"tools","score_baseline":0.576,"score_final":0.576,"passed":false,"rounds":1,"cost_usd":2.0616418,"minutes":22.34,"status":"budget","errors":"cost $2.062 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_med_hand_drill","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"veh_med_pickup","tier":"medium","category":"vehicles","score_baseline":0.426,"score_final":0.471,"passed":false,"rounds":2,"cost_usd":2.144166425,"minutes":46.46,"status":"budget","errors":"cost $2.144 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/veh_med_pickup","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"appl_hard_espresso","tier":"hard","category":"appliances","score_baseline":null,"score_final":null,"passed":null,"rounds":0,"cost_usd":0.0,"minutes":2.71,"status":"error","errors":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_hard_espresso","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"appl_med_toaster","tier":"medium","category":"appliances","score_baseline":0.881,"score_final":0.881,"passed":true,"rounds":1,"cost_usd":0.31813375,"minutes":5.49,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_med_toaster","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"plant_med_palm","tier":"medium","category":"plants","score_baseline":null,"score_final":null,"passed":null,"rounds":0,"cost_usd":0.0,"minutes":1.36,"status":"error","errors":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_med_palm","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"plant_easy_cactus","tier":"easy","category":"plants","score_baseline":0.883,"score_final":0.883,"passed":true,"rounds":1,"cost_usd":0.4371587499999999,"minutes":9.99,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_easy_cactus","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"tool_hard_bench_vise","tier":"hard","category":"tools","score_baseline":0.466,"score_final":0.536,"passed":false,"rounds":2,"cost_usd":3.019153075,"minutes":35.43,"status":"budget","errors":"cost $2.478 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/tool_hard_bench_vise","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"appl_easy_kettle","tier":"easy","category":"appliances","score_baseline":0.6,"score_final":0.801,"passed":true,"rounds":2,"cost_usd":0.9444692499999997,"minutes":22.57,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/appl_easy_kettle","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"bld_easy_shed","tier":"easy","category":"buildings","score_baseline":0.833,"score_final":0.833,"passed":true,"rounds":1,"cost_usd":0.41163482500000015,"minutes":6.23,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_easy_shed","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"toy_easy_blocks","tier":"easy","category":"toys","score_baseline":0.964,"score_final":0.964,"passed":true,"rounds":1,"cost_usd":0.28989425,"minutes":4.12,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_easy_blocks","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"bld_hard_windmill","tier":"hard","category":"buildings","score_baseline":0.7,"score_final":0.746,"passed":true,"rounds":2,"cost_usd":0.7304633250000001,"minutes":10.7,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_hard_windmill","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"plant_hard_bonsai","tier":"hard","category":"plants","score_baseline":0.384,"score_final":0.384,"passed":false,"rounds":1,"cost_usd":2.182076575,"minutes":26.07,"status":"budget","errors":"cost $2.182 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/plant_hard_bonsai","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"mus_easy_drum","tier":"easy","category":"musical_instruments","score_baseline":null,"score_final":null,"passed":null,"rounds":0,"cost_usd":0.0,"minutes":1.05,"status":"error","errors":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_easy_drum","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"mus_med_acoustic_guitar","tier":"medium","category":"musical_instruments","score_baseline":null,"score_final":null,"passed":null,"rounds":0,"cost_usd":0.0,"minutes":1.53,"status":"error","errors":"ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\ng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in run\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n return []\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/orchestrator/runner.py\", line 91, in stage\n result = fn()\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 147, in <lambda>\n def round_files_hint(self, ctx: RunContext) -> list[str]:\n ^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/lifecycle.py\", line 230, in _plan_stage\n try:\n \n ctx.plan = runner.stage(\"plan\", lambda: self._plan_stage(ctx), inputs={\"spec\": plan_stage_inputs(spec), \"track\": self.track.value},\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 48, in plan\n def __init__(self, message: str, usage: Usage | None = None):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/yipeng/3dcodeverse/harness/codeverse/tracks/planner.py\", line 70, in plan_with_usage\n File \"/home/yipeng/3dcodeverse/harness/codeverse/models/gemini.py\", line 257, in generate\n raise err from exc\ncodeverse.models.base.ModelError: Gemini API error 503: This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\n","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_med_acoustic_guitar","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"bld_med_lighthouse","tier":"medium","category":"buildings","score_baseline":0.467,"score_final":0.6,"passed":false,"rounds":2,"cost_usd":2.275502575,"minutes":21.92,"status":"budget","errors":"cost $2.276 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/bld_med_lighthouse","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"toy_hard_train","tier":"hard","category":"toys","score_baseline":0.867,"score_final":0.867,"passed":true,"rounds":1,"cost_usd":0.6989257500000001,"minutes":11.04,"status":"passed","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_hard_train","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"toy_med_rocking_horse","tier":"medium","category":"toys","score_baseline":0.374,"score_final":0.685,"passed":false,"rounds":2,"cost_usd":2.6155671000000003,"minutes":28.83,"status":"budget","errors":"cost $2.616 exceeds max_usd $2.00","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/toy_med_rocking_horse","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
+{"id":"mus_hard_drum_kit","tier":"hard","category":"musical_instruments","score_baseline":0.6,"score_final":0.6,"passed":false,"rounds":3,"cost_usd":1.8923563749999999,"minutes":24.67,"status":"plateau","errors":"","workspace":"/home/yipeng/3dcodeverse/harness/bench/out/static_v1_flash/runs/mus_hard_drum_kit","generator":"api-agent:gemini:gemini-3.7-flash","judge":"gemini:gemini-3.1-pro-preview"}
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.py
index 62e47372..54263f9b 100644
--- a/harness/bench/run_bench.py
+++ b/harness/bench/run_bench.py
@@ -17,7 +17,7 @@ import csv
import json
import time
import traceback
-from collections.abc import Callable
+from collections.abc import Callable, Sequence
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import UTC, datetime
from pathlib import Path
@@ -26,6 +26,7 @@ from typing import Any
import yaml
from pydantic import BaseModel, Field
+from codeverse.config import get_settings
from codeverse.contracts.common import Backends, Budget, Language, Track
from codeverse.contracts.run import RunRecord
from codeverse.contracts.spec import Constraints, Spec
@@ -89,17 +90,26 @@ class BenchOptions(BaseModel):
resume: bool = True
-def spec_for(battery: Battery, item: BenchPrompt, opts: BenchOptions) -> Spec:
- backends = Backends(**{k: v for k, v in {"generator": opts.generator, "planner": opts.planner,
- "judge": opts.judge}.items() if v})
+def build_spec(
+ battery: Battery, item: BenchPrompt, *, backends: Backends, rounds: int, max_usd: float,
+ max_minutes: float, tag0: str, extra_tags: Sequence[str] = (),
+) -> Spec:
+ """Shared Spec core for bench drivers (``run_bench`` / ``compare_backends``).
+ Drivers resolve their own ``Backends`` and pass their leading tag."""
return Spec(
id=f"{battery.name}/{item.id}", track=battery.track, language=battery.language, prompt=item.prompt,
constraints=Constraints(must_have=list(item.must_have), dimensions_m=item.dimensions_m),
- budget=Budget(max_rounds=opts.rounds, max_usd=opts.max_usd, max_minutes=opts.max_minutes),
- backends=backends, tags=["bench", battery.name, item.tier, item.category, *item.tags],
+ budget=Budget(max_rounds=rounds, max_usd=max_usd, max_minutes=max_minutes),
+ backends=backends, tags=[tag0, battery.name, item.tier, item.category, *extra_tags, *item.tags],
)
+def spec_for(battery: Battery, item: BenchPrompt, opts: BenchOptions) -> Spec:
+ backends = get_settings().backends(generator=opts.generator, planner=opts.planner, judge=opts.judge)
+ return build_spec(battery, item, backends=backends, rounds=opts.rounds, max_usd=opts.max_usd,
+ max_minutes=opts.max_minutes, tag0="bench")
+
+
def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Workspace) -> BenchItemResult:
best = next((r for r in rec.rounds if r.index == rec.best_round), None)
return BenchItemResult(
@@ -121,33 +131,45 @@ def _load_done(results_jsonl: Path) -> dict[str, BenchItemResult]:
return done
+def select_prompts(
+ battery: Battery, *, ids: Sequence[str] = (), tiers: Sequence[str] = (), limit: int | None = None
+) -> list[BenchPrompt]:
+ """Filter a battery's prompts by id list, tier list, then head-``limit``
+ (single owner for the selection semantics of every bench driver)."""
+ items = list(battery.prompts)
+ if ids:
+ items = [p for p in items if p.id in set(ids)]
+ if tiers:
+ items = [p for p in items if p.tier in set(tiers)]
+ return items[:limit] if limit is not None else items
+
+
def _select(battery: Battery, opts: BenchOptions) -> list[BenchPrompt]:
- items = battery.prompts
- if opts.ids:
- items = [p for p in items if p.id in set(opts.ids)]
- if opts.tiers:
- items = [p for p in items if p.tier in set(opts.tiers)]
- if opts.limit is not None:
- items = items[: opts.limit]
- return items
+ """Deprecated: use :func:`select_prompts`."""
+ return select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit)
RunFn = Callable[[Spec, Workspace, bool], RunRecord]
-def _default_run(spec: Spec, ws: Workspace, resume: bool) -> RunRecord:
+def default_run_track(spec: Spec, ws: Workspace, resume: bool) -> RunRecord:
+ """Run the spec's track (the real thing; tests inject a fake ``run_fn``)."""
from codeverse.tracks import get_track
return get_track(spec.track).run(spec, ws, resume=resume)
+#: deprecated alias — use :func:`default_run_track`
+_default_run = default_run_track
+
+
def run_battery(
battery_path: Path | str, out_dir: Path | str, opts: BenchOptions | None = None, *,
run_fn: RunFn | None = None, on_result: Callable[[BenchItemResult], None] | None = None,
) -> list[BenchItemResult]:
"""Run (or resume) a battery; returns every result (previous + new)."""
opts = opts or BenchOptions()
- run_fn = run_fn or _default_run
+ run_fn = run_fn or default_run_track
battery = Battery.load(battery_path)
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
@@ -156,7 +178,7 @@ def run_battery(
"started_at": datetime.now(UTC).isoformat()}, indent=2))
results_jsonl = out / "results.jsonl"
done = _load_done(results_jsonl) if opts.resume else {}
- todo = [p for p in _select(battery, opts) if p.id not in done]
+ todo = [p for p in select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit) if p.id not in done]
def _one(item: BenchPrompt) -> BenchItemResult:
ws = Workspace(out / "runs" / item.id)
diff --git a/harness/codeverse/agents/api_agent.py b/harness/codeverse/agents/api_agent.py
index ea8e8717..fa53dc73 100644
--- a/harness/codeverse/agents/api_agent.py
+++ b/harness/codeverse/agents/api_agent.py
@@ -95,12 +95,10 @@ class _Loop:
def __init__(self, agent: ApiAgent, s: Session, chat: Any):
self.agent, self.s, self.chat = agent, s, chat
job = s.job
- extra = job.extra
- self.files = FileTools(s.ws, job.write_roots, allow_shell=bool(extra.get("allow_shell", True)))
+ self.files = FileTools(s.ws, job.write_roots, allow_shell=job.api.allow_shell)
self.spatial: SpatialTools | None = None
if job.spatial_tools:
- self.spatial = SpatialTools(s.ws, track=str(extra.get("track", "")), language=str(extra.get("language", "")),
- round_index=int(extra.get("round", 0) or 0))
+ self.spatial = SpatialTools(s.ws, track=job.track, language=job.language, round_index=job.round)
if not self.spatial.tools:
s.notes.append("spatial registry is empty; running with file tools only")
self.spatial = None
@@ -113,7 +111,7 @@ class _Loop:
self.errors: list[str] = []
self.messages: list[ChatMessage] = [ChatMessage.user(job.prompt)]
self.system = self._system_prompt()
- self.max_usd = float(extra.get("max_usd", 0) or 0)
+ self.max_usd = job.api.max_usd
self.t_deadline = time.monotonic() + job.timeout_s
def _system_prompt(self) -> str:
@@ -165,8 +163,8 @@ class _Loop:
def _generate(self, turn: int) -> ChatResponse | None:
req = ChatRequest(messages=self.messages, system=self.system, tools=self.specs,
- temperature=float(self.s.job.extra.get("temperature", 0.3)),
- thinking=self.s.job.extra.get("thinking", "low"), label=f"api-agent:{self.s.label}:t{turn}")
+ temperature=self.s.job.api.temperature,
+ thinking=self.s.job.api.thinking, label=f"api-agent:{self.s.label}:t{turn}")
delay = 2.0
for attempt in range(MODEL_RETRIES):
try:
diff --git a/harness/codeverse/agents/cli_common.py b/harness/codeverse/agents/cli_common.py
index acfcbee4..e16f5264 100644
--- a/harness/codeverse/agents/cli_common.py
+++ b/harness/codeverse/agents/cli_common.py
@@ -73,7 +73,7 @@ def default_mcp_command(ws: Workspace, *, language: str = "", track: str = "", r
def mcp_command_for(ws: Workspace, job: AgentJob) -> list[str]:
- """The 3dcv MCP command for this job: ``.mcp.json`` (materialised) > ``job.extra['mcp_command']`` > default."""
+ """The 3dcv MCP command for this job: ``.mcp.json`` (materialised) > ``job.mcp_command`` > default."""
mcp = ws.root / ".mcp.json"
if mcp.is_file():
try:
@@ -82,10 +82,9 @@ def mcp_command_for(ws: Workspace, job: AgentJob) -> list[str]:
srv = None
if srv and srv.get("command"):
return [srv["command"], *srv.get("args", [])]
- if job.extra.get("mcp_command"):
- return list(job.extra["mcp_command"])
- return default_mcp_command(ws, language=str(job.extra.get("language", "")), track=str(job.extra.get("track", "")),
- round_index=int(job.extra.get("round", 0) or 0))
+ if job.mcp_command:
+ return list(job.mcp_command)
+ return default_mcp_command(ws, language=job.language, track=job.track, round_index=job.round)
# --------------------------------------------------------------------------- session
@@ -117,7 +116,7 @@ def _register(ws: Workspace, label: str, hints: frozenset[str]) -> _LiveSession:
def _sibling_hints(ws: Workspace, me: _LiveSession) -> frozenset[str]:
- """Files claimed (``extra['files_hint']``) by OTHER sessions that overlapped ``me`` in time."""
+ """Files claimed (``job.files_hint``) by OTHER sessions that overlapped ``me`` in time."""
me.t_end = time.monotonic()
with _LIVE_LOCK:
entries = _LIVE.get(str(ws.root), [])
@@ -146,7 +145,7 @@ def attribute_changes(
) -> list[FileChange]:
"""The subset of a whole-worktree git diff that belongs to ONE session: inside its
``write_roots``, not harness-owned, and not a file another concurrent session
- declared as its target (``job.extra['files_hint']``) unless this session declared it too."""
+ declared as its target (``job.files_hint``) unless this session declared it too."""
roots = tuple(r.strip("/") for r in write_roots if r.strip("/"))
out: list[FileChange] = []
for f in files:
@@ -196,11 +195,11 @@ def begin_session(job: AgentJob, kind: str) -> Session:
raise FileNotFoundError(f"workspace does not exist: {ws.root}")
if not (ws.root / ".git").exists():
ws.create()
- round_index = int(job.extra.get("round", 0) or 0)
+ round_index = job.round
label, attempt = _session_label(ws, job.label or kind, round_index)
traj = Trajectory(ws.trajectory_dir(label, round_index))
traj.write_prompt(job.prompt, job.system_append)
- hints = frozenset(str(h) for h in (job.extra.get("files_hint") or []) if str(h).strip())
+ hints = frozenset(h for h in (str(x).strip() for x in job.files_hint) if h)
live = _register(ws, label, hints)
head_before = ws.commit(f"pre:{label}")
return Session(ws=ws, job=job, kind=kind, label=label, round_index=round_index,
diff --git a/harness/codeverse/agents/gemini_cli.py b/harness/codeverse/agents/gemini_cli.py
index a133453d..3b7ce9dc 100644
--- a/harness/codeverse/agents/gemini_cli.py
+++ b/harness/codeverse/agents/gemini_cli.py
@@ -41,14 +41,7 @@ from codeverse.agents.watchdog import CompletedProc, run_with_watchdog
from codeverse.config import get_settings
from codeverse.contracts.agent import AgentJob, AgentResult
from codeverse.contracts.common import Usage
-
-try:
- from codeverse.models.keypool import KeyPoolExhausted
-except ImportError: # pragma: no cover - models package is always present in a full install
-
- class KeyPoolExhausted(RuntimeError): # type: ignore[no-redef]
- """Stand-in so the retry path can catch the pool's exhaustion uniformly."""
-
+from codeverse.models.keypool import KeyPool, KeyPoolExhausted
log = logging.getLogger(__name__)
@@ -64,40 +57,15 @@ IDLE_GRACE_S = 300.0
RETRY_KEY_WAIT_S = 10.0
-class _LocalKeyRotor:
- """Fallback key rotation used only when ``codeverse.models.keypool`` is absent."""
+_POOLS: dict[tuple[str, ...], KeyPool] = {}
- def __init__(self, keys: list[str]):
- self._keys, self._i = list(keys), 0
-
- def acquire(self, *, exclude: set[str] | None = None, timeout_s: float | None = None) -> str:
- for _ in range(len(self._keys)):
- k = self._keys[self._i % len(self._keys)]
- self._i += 1
- if not exclude or k not in exclude or len(self._keys) == 1:
- return k
- return self._keys[0]
-
- def report(self, key: str, status: str) -> None: # parity with KeyPool.report
- return None
-
-_POOLS: dict[tuple[str, ...], Any] = {}
-
-
-def _key_pool(keys: list[str]):
+def _key_pool(keys: list[str]) -> KeyPool:
"""Process-wide pool per key set so cooldowns persist across agent runs."""
tkey = tuple(keys)
- if tkey in _POOLS:
- return _POOLS[tkey]
- try:
- from codeverse.models.keypool import KeyPool
-
- pool = KeyPool(keys)
- except ImportError:
- log.warning("codeverse.models.keypool unavailable; using local round-robin")
- pool = _LocalKeyRotor(keys)
- _POOLS[tkey] = pool
+ pool = _POOLS.get(tkey)
+ if pool is None:
+ pool = _POOLS[tkey] = KeyPool(keys)
return pool
@@ -160,7 +128,7 @@ def usage_from_stats(stats: dict[str, Any], model: str) -> Usage:
return u
-def _retry_key(pool: Any, used: set[str]) -> str | None:
+def _retry_key(pool: KeyPool, used: set[str]) -> str | None:
"""Key for a retry: a different healthy key if one frees up within
``RETRY_KEY_WAIT_S``, else the same key again (single-key pools, 5xx), else
``None`` when every key is throttled (429 cooldown) — the caller then stops."""
diff --git a/harness/codeverse/cli/_judge.py b/harness/codeverse/cli/_judge.py
index 88015952..813c39a5 100644
--- a/harness/codeverse/cli/_judge.py
+++ b/harness/codeverse/cli/_judge.py
@@ -18,20 +18,12 @@ from pathlib import Path
from typing import Any
from codeverse.cli import _common as C
-from codeverse.contracts.artifacts import GateReport, RenderSet, RenderView
-from codeverse.contracts.common import Track
+from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderView
+from codeverse.contracts.common import TRACK_INFO
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.run import RoundRecord, RunRecord
from codeverse.workspace import Workspace
-#: rubric per track when the round has no stored judgment (mirrors the tracks)
-TRACK_RUBRIC: dict[Track, str] = {
- Track.STATIC_OBJECT: "static_object_v1",
- Track.ARTICULATED_OBJECT: "articulated_v1",
- Track.SCENE: "scene_v1",
- Track.GRAPHICS: "shader_v1",
-}
-
def load_round(ws: Workspace, rec: RunRecord, index: int) -> RoundRecord | None:
"""rounds/rNN.json (written by the round loop) beats the copy in record.json."""
@@ -45,12 +37,13 @@ def load_round(ws: Workspace, rec: RunRecord, index: int) -> RoundRecord | None:
def rubric_for(rec: RunRecord, rnd: RoundRecord, override: str | None) -> str:
- """--rubric > the rubric the round was judged with > the track default."""
+ """--rubric > the rubric the round was judged with > the track default (TRACK_INFO)."""
if override:
return override
if rnd.judgment is not None and rnd.judgment.rubric:
return rnd.judgment.rubric
- return TRACK_RUBRIC.get(rec.spec.track, "static_object_v1")
+ info = TRACK_INFO.get(rec.spec.track)
+ return info.rubric if info is not None else "static_object_v1"
def plan_summary_for(ws: Workspace) -> str:
@@ -79,42 +72,16 @@ def previous_judgment(ws: Workspace, rec: RunRecord, index: int) -> Any:
return None
-def _gate(rnd: RoundRecord, name: str) -> GateReport | None:
- return next((g for g in rnd.gates if g.gate == name), None)
-
-
def extra_context_for(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> str:
- """Rebuild the track pipeline's ``judge_context`` from the stored artifacts."""
- track = rec.spec.track
- if track is Track.GRAPHICS:
- frame_stats_text = C.lazy("codeverse.tracks.graphics_steps", "frame_stats_text")
- renderer = ""
- if rnd.build is not None and isinstance(rnd.build.census, dict):
- renderer = str(rnd.build.census.get("renderer", ""))
- return f"FRAME METRICS (harness-measured, renderer {renderer or 'moderngl'}):\n{frame_stats_text(ws)}"
- if track is Track.ARTICULATED_OBJECT:
- lines = ["Articulation sheet: the pose_* views show the object at rest, each joint at its lower and upper limit."]
- sweep = _gate(rnd, "joint_sweep")
- if sweep is not None:
- errs = [f"- {f.target or 'joint'}: {f.message}" for f in sweep.errors]
- lines.append(f"Joint sweep: {'no penetrations' if not errs else str(len(errs)) + ' problems'}")
- lines.extend(errs[:10])
- motion = _gate(rnd, "motion_direction")
- if motion is not None:
- wrong = [f"- {f.target}: {f.message}" for f in motion.errors]
- lines.append("Motion direction (harness FK check): "
- + ("all planned directions realised" if not wrong else f"{len(wrong)} WRONG"))
- lines.extend(wrong[:6])
- return "\n".join(lines)
- if track is Track.SCENE and rec.plan is not None:
- plan = rec.plan
- lines = [f"Environment plan: {getattr(plan, 'environment', '')}",
- "Animation plan: " + "; ".join(getattr(plan, "animation", []) or [])]
- cameras = getattr(plan, "cameras", []) or []
- if cameras:
- lines.append("Cameras: " + "; ".join(f"{c.name} ({c.purpose})" for c in cameras))
- return "\n".join(lines)
- return ""
+ """The track pipeline's OWN ``judge_context`` over the stored artifacts —
+ ``judge_context(ws, plan, round_index, build, gates)`` needs no run context,
+ so the CLI never restates the per-track formatting."""
+ get_track = C.lazy("codeverse.tracks", "get_track")
+ build = rnd.build if rnd.build is not None else BuildResult(ok=False, language=rec.spec.language.value)
+ try:
+ return get_track(rec.spec.track).make_pipeline().judge_context(ws, rec.plan, rnd.index, build, list(rnd.gates))
+ except Exception as e: # noqa: BLE001 — extra context is optional judge input
+ return f"(judge context unavailable: {type(e).__name__}: {e})"
def clay_geometry_views(ws: Workspace, index: int) -> RenderSet | None:
@@ -138,7 +105,18 @@ def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:
sheet = rs.contact_sheet
if sheet and not Path(sheet).is_absolute():
sheet = str(ws.root / sheet)
- return rs.model_copy(update={"views": fixed, "contact_sheet": sheet})
+ out_dir = rs.out_dir
+ if out_dir and not Path(out_dir).is_absolute():
+ out_dir = str(ws.root / out_dir)
+ return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})
+
+
+def judged_subset(rs: RenderSet | None) -> RenderSet | None:
+ """The views the in-run judge actually saw: the per-view ``judge`` flags
+ stamped at render time; legacy rounds (no flags) keep every stored view."""
+ if rs is None or not any(v.judge is not None for v in rs.views):
+ return rs
+ return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})
def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:
@@ -151,7 +129,7 @@ def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:
except (ValueError, TypeError):
acceptance = []
return JudgeInput(
- spec=rec.spec, renders=resolve_paths(ws, rnd.renders), measurement=rnd.measurement, gates=rnd.gates,
+ spec=rec.spec, renders=judged_subset(resolve_paths(ws, rnd.renders)), measurement=rnd.measurement, gates=rnd.gates,
acceptance=acceptance, plan_summary=plan_summary_for(ws), round_index=rnd.index,
previous=previous_judgment(ws, rec, rnd.index), extra_context=extra_context_for(ws, rec, rnd),
geometry_views=clay_geometry_views(ws, rnd.index),
diff --git a/harness/codeverse/cli/doctor.py b/harness/codeverse/cli/doctor.py
index 179eef41..5a443601 100644
--- a/harness/codeverse/cli/doctor.py
+++ b/harness/codeverse/cli/doctor.py
@@ -93,8 +93,7 @@ _PROBE_JS = (
def check_gpu_probe(timeout_s: int = 120) -> list[Row]:
"""Launch headless Chrome through runtime_js/gpu_launch.cjs and report the WebGL renderer."""
- import os
- import signal
+ from codeverse.proc import run_subprocess
s = get_settings()
rj = s.runtime_js_dir()
@@ -103,21 +102,18 @@ def check_gpu_probe(timeout_s: int = 120) -> list[Row]:
return [("chrome webgl", "SKIP", f"{launcher} missing (spatial render package not installed yet)")]
cmd = [s.binaries.node, "-e", _PROBE_JS, str(launcher)]
try:
- proc = subprocess.Popen(cmd, cwd=rj, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
- start_new_session=True)
- out, errs = proc.communicate(timeout=timeout_s)
- except subprocess.TimeoutExpired:
- os.killpg(proc.pid, signal.SIGKILL)
- return [("chrome webgl", "FAIL", f"probe timed out after {timeout_s}s")]
+ proc = run_subprocess(cmd, cwd=rj, timeout_s=timeout_s)
except OSError as e:
return [("chrome webgl", "FAIL", str(e))]
- line = next((ln for ln in reversed(out.strip().splitlines()) if ln.startswith("{")), "")
+ if proc.timed_out:
+ return [("chrome webgl", "FAIL", f"probe timed out after {timeout_s}s")]
+ line = next((ln for ln in reversed(proc.stdout.strip().splitlines()) if ln.startswith("{")), "")
try:
res = json.loads(line) if line else {}
except ValueError:
res = {}
if "error" in res or proc.returncode != 0:
- detail = str(res.get("error") or (errs.strip().splitlines() or ["unknown error"])[-1])
+ detail = str(res.get("error") or (proc.stderr.strip().splitlines() or ["unknown error"])[-1])
return [("chrome webgl", "FAIL", detail[:160])]
renderer = str(res.get("renderer", "?"))
return [("chrome webgl", "OK" if res.get("gpu") else "WARN",
diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.py
index 3f140778..8aef9cd3 100644
--- a/harness/codeverse/cli/main.py
+++ b/harness/codeverse/cli/main.py
@@ -23,8 +23,9 @@ from codeverse.cli.doctor import doctor_app
from codeverse.cli.flywheel_cmd import flywheel_app
from codeverse.cli.texture_cmd import texture_app
from codeverse.cli.tools_cmd import tools
-from codeverse.contracts.common import Backends, Budget, Language, Track
-from codeverse.contracts.spec import Constraints, ReferenceImage, Spec
+from codeverse.config import get_settings
+from codeverse.contracts.common import TRACK_LANGUAGES, Budget, Language, Track
+from codeverse.contracts.spec import Constraints, ReferenceImage, RunOptions, Spec
app = typer.Typer(name="3dcv", help="3dcodeverse: LLMs write raw 3D code; the harness builds, judges, refines, records.",
pretty_exceptions_enable=False)
@@ -52,7 +53,8 @@ def _root(ctx: typer.Context, version: Annotated[bool, typer.Option("--version",
def make(
prompt: Annotated[str, typer.Argument(help="what to build")],
track: Annotated[Track, typer.Option("--track")] = Track.STATIC_OBJECT,
- language: Annotated[Language, typer.Option("--language")] = Language.BLENDER,
+ language: Annotated[Language | None, typer.Option("--language", help="default: the track's first language "
+ "(blender / urdf_blender / scene_threejs / glsl_shader)")] = None,
generator: Annotated[str | None, typer.Option(help="api-agent:gemini:gemini-3.7-flash | gemini-cli:... | claude-code:... | codex:... | agy:...")] = None,
planner: Annotated[str | None, typer.Option()] = None,
judge: Annotated[str | None, typer.Option()] = None,
@@ -69,22 +71,22 @@ def make(
must_not: Annotated[list[str] | None, typer.Option("--must-not")] = None,
style: Annotated[str, typer.Option("--style")] = "",
tag: Annotated[list[str] | None, typer.Option("--tag")] = None,
- texture: Annotated[bool, typer.Option("--texture", help="run the text-to-image texture pass after the rounds (adds the 'texture' tag the track hook reads)")] = False,
+ texture: Annotated[bool, typer.Option("--texture", help="run the text-to-image texture pass after the rounds (frozen on spec.options)")] = False,
seed: Annotated[int, typer.Option("--seed")] = 0,
force: Annotated[bool, typer.Option("--force", help="overwrite an existing run dir")] = False,
no_run: Annotated[bool, typer.Option("--no-run", help="only create the workspace + spec.json")] = False,
) -> None:
"""Create a run (workspace + spec.json) and execute the track pipeline."""
image, dim, must, must_not, tag = image or [], dim or [], must or [], must_not or [], tag or []
- if texture and "texture" not in tag:
- tag = [*tag, "texture"]
for p in image:
if not p.is_file():
raise C.CliError(f"reference image not found: {p}")
+ if language is None:
+ language = TRACK_LANGUAGES[track][0]
run_slug = C.make_slug(prompt, track.value, language.value, slug)
- ws = C.create_workspace(C.runs_root(runs_dir) / run_slug, force=force)
- backends = Backends(**{k: v for k, v in {"generator": generator, "planner": planner, "judge": judge,
- "captioner": captioner}.items() if v})
+ backends = get_settings().backends(generator=generator, planner=planner, judge=judge, captioner=captioner)
+ # validate the whole Spec BEFORE touching the filesystem: an invalid
+ # track/language combination must not leave an orphan run directory behind.
try:
spec = Spec(
id=run_slug, track=track, language=language, prompt=prompt,
@@ -92,10 +94,12 @@ def make(
constraints=Constraints(dimensions_m=C.parse_kv_floats(dim, "--dim") or None, must_have=must,
must_not=must_not, style=style),
budget=Budget(max_rounds=rounds, max_usd=max_usd, max_minutes=max_minutes),
- backends=backends, seed=seed, tags=tag,
+ backends=backends, options=RunOptions(candidates=candidates, texture=texture),
+ seed=seed, tags=tag,
)
except ValueError as e:
- raise C.CliError(str(e)) from e
+ raise C.CliError(f"invalid run spec: {e}") from e
+ ws = C.create_workspace(C.runs_root(runs_dir) / run_slug, force=force)
ws.write_json(ws.spec_path, spec)
ws.commit("spec")
console.print(kv_table("run", {"slug": run_slug, "workspace": ws.root, "track": track.value,
@@ -127,10 +131,24 @@ def resume(
slug: str,
runs_dir: RunsDirOpt = None,
candidates: Annotated[int | None, typer.Option("--candidates", min=1, help="best-of-N baseline width (only matters before round 0 ran)")] = None,
+ max_usd: Annotated[float | None, typer.Option("--max-usd", help="raise the budget cap before resuming (rewrites spec.json)")] = None,
+ max_minutes: Annotated[float | None, typer.Option("--max-minutes", help="raise the time cap before resuming")] = None,
+ rounds: Annotated[int | None, typer.Option("--rounds", min=0, help="new max refine rounds (rewrites spec.json)")] = None,
) -> None:
- """Resume an interrupted / partial run (or start a `--no-run` one)."""
+ """Resume an interrupted / partial run (or start a `--no-run` one).
+
+ ``--max-usd`` / ``--max-minutes`` / ``--rounds`` rewrite the spec's budget
+ first — the only way to continue a BUDGET-stopped run."""
ws = C.open_workspace(slug, runs_dir)
- _run_track(C.load_spec(ws), ws, resume=True, candidates=candidates)
+ spec = C.load_spec(ws)
+ raised = {k: v for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items() if v is not None}
+ if raised:
+ from codeverse.events import EventLog
+
+ spec = spec.model_copy(update={"budget": spec.budget.model_copy(update=raised)})
+ ws.write_json(ws.spec_path, spec)
+ EventLog(ws.events_path).emit("budget.raised", **raised)
+ _run_track(spec, ws, resume=True, candidates=candidates)
# --------------------------------------------------------------------------- status
diff --git a/harness/codeverse/fanout.py b/harness/codeverse/fanout.py
new file mode 100644
index 00000000..f111d18d
--- /dev/null
+++ b/harness/codeverse/fanout.py
@@ -0,0 +1,102 @@
+"""Bounded parallel fan-out with per-item timing and error capture.
+
+``fan_out`` never raises for a failing item: the result list holds either the
+item's return value or the exception, in input order, so callers decide what a
+partial failure means (a failed asset is skipped; a failed part task is
+retried). Exceptions that are ``BaseException`` but not ``Exception``
+(KeyboardInterrupt, SystemExit) propagate.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable, Iterable, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Any, TypeVar
+
+log = logging.getLogger(__name__)
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+@dataclass
+class FanOutReport:
+ """Timing/outcome per item, for events and logs."""
+
+ label: str
+ n_items: int
+ n_ok: int
+ n_failed: int
+ durations_s: list[float] = field(default_factory=list)
+ total_s: float = 0.0
+
+ def as_dict(self) -> dict[str, Any]:
+ return {
+ "label": self.label, "n_items": self.n_items, "n_ok": self.n_ok,
+ "n_failed": self.n_failed, "total_s": round(self.total_s, 2),
+ "max_item_s": round(max(self.durations_s), 2) if self.durations_s else 0.0,
+ }
+
+
+def fan_out(
+ items: Sequence[T] | Iterable[T],
+ fn: Callable[[T], R],
+ max_workers: int = 4,
+ *,
+ label: str = "fanout",
+ item_name: Callable[[T], str] | None = None,
+ on_done: Callable[[int, T, R | Exception, float], None] | None = None,
+) -> list[R | Exception]:
+ """Run ``fn`` over ``items`` in a thread pool; return results in input order.
+
+ Each result is the return value or the raised ``Exception``. ``on_done``
+ (if given) is called from the worker thread with ``(index, item, result,
+ seconds)`` — keep it cheap and thread-safe (e.g. an EventLog.emit).
+ """
+ items = list(items)
+ results: list[R | Exception] = [None] * len(items) # type: ignore[list-item]
+ if not items:
+ return results
+ workers = max(1, min(max_workers, len(items)))
+ t_all = time.time()
+ durations = [0.0] * len(items)
+
+ def _run(i: int) -> None:
+ item = items[i]
+ name = item_name(item) if item_name else str(i)
+ t0 = time.time()
+ try:
+ out: R | Exception = fn(item)
+ except Exception as e: # noqa: BLE001 — captured per item by design
+ out = e
+ log.warning("%s[%s] failed: %s: %s", label, name, type(e).__name__, e)
+ dt = time.time() - t0
+ durations[i] = dt
+ results[i] = out
+ log.info("%s[%s] done in %.1fs (%s)", label, name, dt, "error" if isinstance(out, Exception) else "ok")
+ if on_done is not None:
+ on_done(i, item, out, dt)
+
+ if workers == 1:
+ for i in range(len(items)):
+ _run(i)
+ else:
+ with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=label) as pool:
+ list(pool.map(_run, range(len(items))))
+
+ n_failed = sum(1 for r in results if isinstance(r, Exception))
+ report = FanOutReport(label, len(items), len(items) - n_failed, n_failed, durations, time.time() - t_all)
+ log.info("%s: %d/%d ok in %.1fs", label, report.n_ok, report.n_items, report.total_s)
+ return results
+
+
+def split_results(results: Sequence[R | Exception]) -> tuple[list[R], list[Exception]]:
+ """Separate successes from failures (order preserved within each list)."""
+ ok: list[R] = []
+ bad: list[Exception] = []
+ for r in results:
+ (bad if isinstance(r, Exception) else ok).append(r) # type: ignore[arg-type]
+ return ok, bad
diff --git a/harness/codeverse/flywheel/captions.py b/harness/codeverse/flywheel/captions.py
index 940d228d..f65235ae 100644
--- a/harness/codeverse/flywheel/captions.py
+++ b/harness/codeverse/flywheel/captions.py
@@ -23,10 +23,10 @@ from pathlib import Path
from pydantic import BaseModel, Field
from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart
-from codeverse.contracts.common import Language, Track
+from codeverse.contracts.common import ENTRY_FILE, Language, Track
from codeverse.contracts.run import RunRecord
from codeverse.flywheel import _git
-from codeverse.flywheel.sample import ENTRY_BY_LANGUAGE, best_round_record, code_files_for_round
+from codeverse.flywheel.sample import best_round_record, code_files_for_round
from codeverse.prompts import prompt_hash
from codeverse.workspace import Workspace
@@ -123,7 +123,7 @@ def _round_images(ws: Workspace, record: RunRecord) -> tuple[list[ImagePart], li
def _code_excerpt(ws: Workspace, record: RunRecord) -> str:
rnd = best_round_record(record)
files, _src = code_files_for_round(ws, rnd)
- entry = ENTRY_BY_LANGUAGE[record.spec.language]
+ entry = ENTRY_FILE[record.spec.language]
ordered = {k: files[k] for k in sorted(files, key=lambda k: (k != entry, k))}
text, _skipped = _git.decode_text_files(ordered, max_total=MAX_CODE_CHARS)
return "\n\n".join(f"### {p}\n{t}" for p, t in text.items())
diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.py
index 24c07c0d..9f693866 100644
--- a/harness/codeverse/flywheel/export.py
+++ b/harness/codeverse/flywheel/export.py
@@ -29,7 +29,7 @@ from typing import Any
from pydantic import BaseModel, Field
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.run import RunRecord
from codeverse.flywheel import sample as S
from codeverse.flywheel.quality import DuplicateGroup, mark_duplicates
@@ -98,7 +98,7 @@ def export_one(
dest.mkdir(parents=True)
entry, written = S.write_code_tree(dest, files, record.spec.language)
if entry not in written:
- raise S.SampleError(f"entry file {S.ENTRY_BY_LANGUAGE[record.spec.language]} missing at {code_source}")
+ raise S.SampleError(f"entry file {ENTRY_FILE[record.spec.language]} missing at {code_source}")
renders = S.copy_renders(ws, rnd, dest)
meshes = S.copy_link_meshes(ws, dest) if record.spec.language is Language.URDF_BLENDER else []
textured = S.copy_textured(ws, record, dest)
diff --git a/harness/codeverse/flywheel/sample.py b/harness/codeverse/flywheel/sample.py
index a1707401..d161611c 100644
--- a/harness/codeverse/flywheel/sample.py
+++ b/harness/codeverse/flywheel/sample.py
@@ -22,7 +22,14 @@ from typing import Any
from pydantic import BaseModel, Field
from codeverse import __version__
-from codeverse.contracts.common import Language, Track
+from codeverse.contracts.common import (
+ ENTRY_FILE,
+ LANGUAGE_LABEL,
+ TRACK_INFO,
+ Language,
+ Track,
+ code_file,
+)
from codeverse.contracts.run import RoundRecord, RunRecord
from codeverse.flywheel import _git
from codeverse.flywheel.dedupe import code_fingerprint
@@ -30,41 +37,11 @@ from codeverse.flywheel.quality import QualityTier, prompt_hash, quality_tier
from codeverse.flywheel.record import best_round_index, effective_judgment, round_summary
from codeverse.workspace import Workspace
-#: entry file inside src/ per language
-ENTRY_BY_LANGUAGE: dict[Language, str] = {
- Language.BLENDER: "src/model.py",
- Language.CADQUERY: "src/model.py",
- Language.URDF_BLENDER: "src/model.py",
- Language.THREEJS: "src/object.js",
- Language.SCENE_THREEJS: "src/scene.js",
- Language.GLSL_SHADER: "src/shader.frag",
- Language.OPENGL_PYTHON: "src/program.py",
-}
-#: top-level copy of the entry (``code.<ext>``)
-CODE_FILE_BY_LANGUAGE: dict[Language, str] = {
- Language.BLENDER: "code.py",
- Language.CADQUERY: "code.py",
- Language.URDF_BLENDER: "code.py",
- Language.THREEJS: "code.js",
- Language.SCENE_THREEJS: "code.js",
- Language.GLSL_SHADER: "code.frag",
- Language.OPENGL_PYTHON: "code.py",
-}
-LANGUAGE_LABEL: dict[Language, str] = {
- Language.BLENDER: "Blender Python",
- Language.CADQUERY: "CadQuery (Python)",
- Language.URDF_BLENDER: "URDF + Blender Python",
- Language.THREEJS: "Three.js (ESM)",
- Language.SCENE_THREEJS: "Three.js scene (multi-file ESM + GLSL)",
- Language.GLSL_SHADER: "GLSL fragment shader",
- Language.OPENGL_PYTHON: "OpenGL (moderngl Python + GLSL)",
-}
-TYPE_LABEL: dict[Track, str] = {
- Track.STATIC_OBJECT: "3D Objects",
- Track.ARTICULATED_OBJECT: "Articulated Objects",
- Track.SCENE: "3D Scenes",
- Track.GRAPHICS: "Procedural Graphics",
-}
+#: deprecated aliases — the registries in ``codeverse.contracts.common`` are the
+#: single source now; import ``ENTRY_FILE`` / ``code_file`` / ``TRACK_INFO`` instead
+ENTRY_BY_LANGUAGE: dict[Language, str] = ENTRY_FILE
+CODE_FILE_BY_LANGUAGE: dict[Language, str] = {lang: code_file(lang) for lang in Language}
+TYPE_LABEL: dict[Track, str] = {t: TRACK_INFO[t].label for t in Track}
MAX_GLB_BYTES = 20 * 1024 * 1024
SOURCE_NAME = "3dcodeverse"
SAMPLE_LICENSE = "CC-BY-4.0"
@@ -173,8 +150,8 @@ def write_code_tree(dest: Path, files: dict[str, bytes], language: Language) ->
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
written.append(rel)
- entry_src = ENTRY_BY_LANGUAGE[language]
- code_name = CODE_FILE_BY_LANGUAGE[language]
+ entry_src = ENTRY_FILE[language]
+ code_name = code_file(language)
if entry_src in files:
(dest / code_name).write_bytes(files[entry_src])
written.append(code_name)
@@ -296,7 +273,7 @@ def build_meta(
id=sample_id(record, key),
key=key,
name=name,
- type=TYPE_LABEL[spec.track],
+ type=TRACK_INFO[spec.track].label,
track=spec.track.value,
language=spec.language.value,
language_label=LANGUAGE_LABEL[spec.language],
diff --git a/harness/codeverse/judges/images.py b/harness/codeverse/judges/images.py
index 7f4521e1..e92747b5 100644
--- a/harness/codeverse/judges/images.py
+++ b/harness/codeverse/judges/images.py
@@ -15,12 +15,13 @@ import os
import threading
from pathlib import Path
-from PIL import Image, ImageDraw, ImageFont
+from PIL import Image, ImageDraw
from codeverse.config import get_settings
from codeverse.contracts.artifacts import RenderView
from codeverse.contracts.chat import ImagePart
from codeverse.conventions import OBJECT_VIEWS, SCENE_VIEWS, ViewPreset
+from codeverse.spatial.sheet import load_font
_PRESETS: dict[str, ViewPreset] = {v.name: v for v in (*OBJECT_VIEWS, *SCENE_VIEWS)}
@@ -64,15 +65,6 @@ def view_label(view: RenderView, index: int, total: int) -> str:
return " · ".join(bits)
-def _font(size: int) -> ImageFont.ImageFont | ImageFont.FreeTypeFont:
- for cand in ("DejaVuSans-Bold.ttf", "DejaVuSans.ttf", "Arial.ttf"):
- try:
- return ImageFont.truetype(cand, size)
- except OSError:
- continue
- return ImageFont.load_default()
-
-
def _cache_key(src: Path, max_px: int, label: str) -> str:
st = src.stat()
h = hashlib.sha1(f"{src.resolve()}|{st.st_mtime_ns}|{st.st_size}|{max_px}|{label}".encode())
@@ -111,7 +103,7 @@ def prepare_image(
def _with_label_strip(im: Image.Image, label: str) -> Image.Image:
w, h = im.size
strip_h = max(22, int(h * 0.055))
- font = _font(max(12, int(strip_h * 0.62)))
+ font = load_font(max(12, int(strip_h * 0.62)), bold=True)
canvas = Image.new("RGB", (w, h + strip_h), (18, 18, 22))
canvas.paste(im, (0, strip_h))
draw = ImageDraw.Draw(canvas)
diff --git a/harness/codeverse/judges/output_schema.py b/harness/codeverse/judges/output_schema.py
index 8c47e48a..79840f4a 100644
--- a/harness/codeverse/judges/output_schema.py
+++ b/harness/codeverse/judges/output_schema.py
@@ -9,14 +9,13 @@ JSON in plain text) but strict about content: a missing criterion is a
from __future__ import annotations
-import json
-import re
from typing import Any
from pydantic import BaseModel, Field, ValidationError, create_model
from codeverse.contracts.judgment import ImprovementItem, JudgeIssue
from codeverse.judges.rubrics import Rubric
+from codeverse.models.schema_utils import JsonParseError, parse_json_lenient
class JudgeParseError(ValueError):
@@ -103,33 +102,6 @@ def wire_schema(rubric: Rubric, acceptance_ids: list[str]) -> dict[str, Any]:
# --------------------------------------------------------------------------- parsing
-_FENCE = re.compile(r"```(?:json)?\s*(.*?)```", re.DOTALL)
-
-
-def extract_json(text: str) -> Any:
- """Parse JSON from a reply: raw, fenced, or the first balanced ``{...}`` block."""
- text = text.strip()
- for cand in (text, *(m.strip() for m in _FENCE.findall(text))):
- try:
- return json.loads(cand)
- except json.JSONDecodeError:
- continue
- start = text.find("{")
- if start >= 0:
- depth = 0
- for i, ch in enumerate(text[start:], start):
- if ch == "{":
- depth += 1
- elif ch == "}":
- depth -= 1
- if depth == 0:
- try:
- return json.loads(text[start : i + 1])
- except json.JSONDecodeError:
- break
- raise JudgeParseError(f"no JSON object in reply (first 200 chars): {text[:200]!r}")
-
-
def _records_to_dict(value: Any, key_names: tuple[str, ...]) -> dict[str, Any]:
"""Accept ``{id: {...}}`` or ``[{id: ..., ...}, ...]`` and return ``{id: {...}}``."""
if isinstance(value, dict):
@@ -193,7 +165,10 @@ def parse_judge_output(
criteria so the output always covers the full rubric.
"""
if isinstance(payload, str):
- payload = extract_json(payload)
+ try:
+ payload = parse_json_lenient(payload)
+ except JsonParseError as e:
+ raise JudgeParseError(f"no JSON object in reply: {e}") from e
if not isinstance(payload, dict):
raise JudgeParseError(f"reply is not a JSON object: {type(payload).__name__}")
data = dict(payload)
diff --git a/harness/codeverse/judges/pairwise.py b/harness/codeverse/judges/pairwise.py
index fae077e9..d678137d 100644
--- a/harness/codeverse/judges/pairwise.py
+++ b/harness/codeverse/judges/pairwise.py
@@ -1,11 +1,13 @@
"""``PairwiseJudge``: which of two candidates better satisfies the spec?
-Position bias is real, so the comparison runs TWICE with A/B swapped; the
-winner is declared only when both orderings agree, otherwise ``tie``. Each
-call sees the brief, the rubric's criteria (titles + descriptions) and ONE
-2×2 montage per candidate (the 4 most informative views — never a big sheet:
-VLM judges flip with many tiles). ``compare_many`` ranks N candidates by a
-round-robin of such comparisons (best-of-N).
+Position bias is real, so the comparison runs TWICE with A/B swapped (the two
+orderings run in parallel via ``codeverse.fanout``); the winner is declared
+only when both orderings agree, otherwise ``tie``. Each call sees the brief,
+the rubric's criteria (titles + descriptions) and ONE 2×2 montage per candidate
+(the 4 most informative views — never a big sheet: VLM judges flip with many
+tiles). ``compare_many`` ranks N candidates by a round-robin of such
+comparisons (best-of-N), fanning the pairs at 4 workers; results accumulate in
+input order either way.
"""
from __future__ import annotations
@@ -24,12 +26,14 @@ from codeverse.contracts.artifacts import RenderSet
from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart, TextPart
from codeverse.contracts.common import Usage
from codeverse.contracts.spec import Spec
+from codeverse.fanout import fan_out
from codeverse.judges.images import image_part, prepare_image
from codeverse.judges.montage import montage_label, plan_montages, render_montage
-from codeverse.judges.output_schema import JudgeParseError, extract_json
+from codeverse.judges.output_schema import JudgeParseError
from codeverse.judges.prompt_builder import MONTAGE_TILE_PX, brief_section
from codeverse.judges.rubrics import Rubric, load_rubric
from codeverse.models.base import ChatModel, ModelError
+from codeverse.models.schema_utils import JsonParseError, parse_json_lenient
log = logging.getLogger(__name__)
@@ -108,38 +112,62 @@ class PairwiseJudge:
# ------------------------------------------------------------------ API
def compare(
- self, spec: Spec, renders_a: RenderSet, renders_b: RenderSet, *, rubric: str | Rubric = "static_object_v1"
+ self, spec: Spec, renders_a: RenderSet, renders_b: RenderSet, *,
+ rubric: str | Rubric = "static_object_v1", label: str = ""
) -> PairwiseResult:
+ """``label`` (optional) tags the two orderings' requests (``pairwise[:label]:fwd/swap``)
+ so logs — and deterministic fakes — can tell concurrent orderings apart."""
rub = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)
usage = Usage()
verdicts: list[tuple[Winner, float, list[str]]] = []
orderings: list[dict] = []
errors: list[str] = []
- for swapped in (False, True):
- first, second = (renders_b, renders_a) if swapped else (renders_a, renders_b)
- req = self._request(spec, rub, first, second)
- try:
- resp = self.model.generate(req)
- except ModelError as e:
- errors.append(f"ModelError: {e}")
- continue
- usage = usage + resp.usage
- try:
- payload = resp.parsed if resp.parsed is not None else extract_json(resp.text)
- reply = PairwiseReply.model_validate(payload)
- except (JudgeParseError, ValueError) as e:
- errors.append(f"parse: {e}")
+ self.model # noqa: B018 — materialise the lazy chat model once, before the threads race
+ results = fan_out(
+ (False, True), lambda swapped: self._ordering(spec, rub, renders_a, renders_b, swapped, label),
+ max_workers=2, label="pairwise:orderings", item_name=lambda s: f"swapped={s}",
+ )
+ for res in results: # ordered accumulation: (fwd, swap)
+ if isinstance(res, Exception):
+ raise res # ModelError / parse failures are captured; anything else is a bug
+ u, verdict, ordering, error = res
+ usage = usage + u
+ if error:
+ errors.append(error)
continue
- winner = _map_winner(reply.winner, swapped)
- reasons = [_unswap_text(r, swapped) for r in reply.reasons]
- verdicts.append((winner, reply.confidence, reasons))
- orderings.append({"swapped": swapped, "winner": winner, "confidence": reply.confidence,
- "reasons": reasons,
- "criteria_won": {c.criterion: _map_winner(c.winner, swapped) for c in reply.criteria_won}})
+ assert verdict is not None and ordering is not None
+ verdicts.append(verdict)
+ orderings.append(ordering)
if not verdicts:
return PairwiseResult(winner="tie", confidence=0.0, usage=usage, error=" || ".join(errors))
return _combine(verdicts, orderings, usage, errors)
+ def _ordering(
+ self, spec: Spec, rub: Rubric, renders_a: RenderSet, renders_b: RenderSet, swapped: bool, label: str = ""
+ ) -> tuple[Usage, tuple[Winner, float, list[str]] | None, dict | None, str]:
+ """One A/B ordering → ``(usage, verdict, ordering, error)`` (never raises for
+ model/parse failures — the caller aggregates them as errors)."""
+ first, second = (renders_b, renders_a) if swapped else (renders_a, renders_b)
+ tag = ":".join(x for x in ("pairwise", label, "swap" if swapped else "fwd") if x)
+ req = self._request(spec, rub, first, second, label=tag)
+ try:
+ resp = self.model.generate(req)
+ except ModelError as e:
+ return Usage(), None, None, f"ModelError: {e}"
+ try:
+ payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
+ reply = PairwiseReply.model_validate(payload)
+ except JsonParseError as e:
+ return resp.usage, None, None, f"parse: no JSON object in reply: {e}"
+ except (JudgeParseError, ValueError) as e:
+ return resp.usage, None, None, f"parse: {e}"
+ winner = _map_winner(reply.winner, swapped)
+ reasons = [_unswap_text(r, swapped) for r in reply.reasons]
+ ordering = {"swapped": swapped, "winner": winner, "confidence": reply.confidence,
+ "reasons": reasons,
+ "criteria_won": {c.criterion: _map_winner(c.winner, swapped) for c in reply.criteria_won}}
+ return resp.usage, (winner, reply.confidence, reasons), ordering, ""
+
def compare_many(
self, spec: Spec, candidates: list[RenderSet], *, rubric: str | Rubric = "static_object_v1"
) -> RankingResult:
@@ -157,28 +185,38 @@ class PairwiseJudge:
pairs: list[dict] = []
usage = Usage()
errors: list[str] = []
- for i in range(n):
- for j in range(i + 1, n):
- res = self.compare(spec, candidates[i], candidates[j], rubric=rubric)
- usage = usage + res.usage
- if res.error:
- errors.append(f"{i}v{j}: {res.error}")
- if res.winner == "a":
- points[i] += 1.0
- conf_won[i].append(res.confidence)
- elif res.winner == "b":
- points[j] += 1.0
- conf_won[j].append(res.confidence)
- else:
- points[i] += 0.5
- points[j] += 0.5
- pairs.append({"a": i, "b": j, "winner": res.winner, "confidence": res.confidence, "error": res.error})
+ idx_pairs = [(i, j) for i in range(n) for j in range(i + 1, n)]
+ if idx_pairs:
+ self.model # noqa: B018 — materialise the lazy chat model once, before the threads race
+ results = fan_out(
+ idx_pairs, lambda ij: self.compare(spec, candidates[ij[0]], candidates[ij[1]], rubric=rubric,
+ label=f"{ij[0]}v{ij[1]}"),
+ max_workers=4, label="pairwise:pairs", item_name=lambda ij: f"{ij[0]}v{ij[1]}",
+ )
+ for (i, j), res in zip(idx_pairs, results, strict=True): # ordered accumulation
+ if isinstance(res, Exception):
+ raise res # compare() reports model/parse failures in-band; anything else is a bug
+ usage = usage + res.usage
+ if res.error:
+ errors.append(f"{i}v{j}: {res.error}")
+ if res.winner == "a":
+ points[i] += 1.0
+ conf_won[i].append(res.confidence)
+ elif res.winner == "b":
+ points[j] += 1.0
+ conf_won[j].append(res.confidence)
+ else:
+ points[i] += 0.5
+ points[j] += 0.5
+ pairs.append({"a": i, "b": j, "winner": res.winner, "confidence": res.confidence, "error": res.error})
confidence = {i: (round(statistics.fmean(v), 3) if v else 0.0) for i, v in conf_won.items()}
order = sorted(range(n), key=lambda i: (-points[i], -confidence[i], i))
return RankingResult(order=order, points=points, confidence=confidence, pairs=pairs, usage=usage, errors=errors)
# ------------------------------------------------------------------ prompt
- def _request(self, spec: Spec, rub: Rubric, first: RenderSet, second: RenderSet) -> ChatRequest:
+ def _request(
+ self, spec: Spec, rub: Rubric, first: RenderSet, second: RenderSet, *, label: str = "pairwise"
+ ) -> ChatRequest:
crit = "\n".join(f"- {c.id}: {c.description.strip()}" for c in rub.visual_criteria())
text = (
brief_section(spec)
@@ -193,7 +231,7 @@ class PairwiseJudge:
parts.append(TextPart(text="Compare A and B and return the JSON object."))
return ChatRequest(
messages=[ChatMessage(role="user", parts=parts)], system=_SYSTEM,
- response_schema=PairwiseReply.model_json_schema(), temperature=self.temperature, label="pairwise",
+ response_schema=PairwiseReply.model_json_schema(), temperature=self.temperature, label=label,
)
def _side_images(self, tag: str, rs: RenderSet) -> list[tuple[str, ImagePart]]:
diff --git a/harness/codeverse/judges/prompt_builder.py b/harness/codeverse/judges/prompt_builder.py
index deae162e..142bb27b 100644
--- a/harness/codeverse/judges/prompt_builder.py
+++ b/harness/codeverse/judges/prompt_builder.py
@@ -31,6 +31,7 @@ from codeverse.judges.montage import (
shuffle_montages,
)
from codeverse.judges.rubrics import Rubric
+from codeverse.spatial.measure import measure_summary_table
if TYPE_CHECKING: # pragma: no cover
from codeverse.judges.base import JudgeInput
@@ -126,36 +127,7 @@ def acceptance_section(items: list[AcceptanceItem]) -> str:
def measurement_section(m: Measurement | None) -> str:
if m is None:
return "MEASUREMENTS: (none available)"
- try:
- from codeverse.spatial.measure import measure_summary_table # type: ignore[attr-defined]
- except (ImportError, AttributeError):
- measure_summary_table = None
- if measure_summary_table is not None:
- return "MEASUREMENTS (harness, Y-up meters):\n" + _clip(str(measure_summary_table(m)), 6000)
- return "MEASUREMENTS (harness, Y-up meters):\n" + _local_measure_table(m)
-
-
-def _fmt3(v: tuple[float, float, float]) -> str:
- return f"({v[0]:.3f}, {v[1]:.3f}, {v[2]:.3f})"
-
-
-def _local_measure_table(m: Measurement) -> str:
- ex = m.extents
- lines = [
- f"overall extents x/y/z = {ex[0]:.3f} × {ex[1]:.3f} × {ex[2]:.3f} m; bbox min {_fmt3(m.bbox_min)} max {_fmt3(m.bbox_max)}; centre {_fmt3(m.center)}",
- f"triangles {m.tri_count}; meshes {m.n_meshes}; islands {m.n_islands}; materials {m.materials}; ground_gap {m.ground_gap_m:+.4f} m; footprint_offset {m.footprint_offset_m:.4f} m",
- ]
- if m.parts:
- lines.append("part | size x×y×z (m) | min y | tris | islands | watertight")
- for p in m.parts[:40]:
- sx, sy, sz = (p.bbox_max[i] - p.bbox_min[i] for i in range(3))
- wt = "-" if p.watertight is None else ("yes" if p.watertight else "no")
- lines.append(f"{p.name} | {sx:.3f}×{sy:.3f}×{sz:.3f} | {p.bbox_min[1]:.3f} | {p.tri_count} | {p.islands} | {wt}")
- if len(m.parts) > 40:
- lines.append(f"… {len(m.parts) - 40} more parts")
- for k, v in list(m.extra.items())[:8]:
- lines.append(f"{k}: {_clip(str(v), 200)}")
- return "\n".join(lines)
+ return "MEASUREMENTS (harness, Y-up meters):\n" + _clip(str(measure_summary_table(m)), 6000)
def gates_section(gates: list[GateReport], *, max_errors: int = 12, max_warns: int = 8) -> str:
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index b302b362..358c4d23 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -3,7 +3,8 @@
Per sample: one ``ChatRequest`` with a per-rubric JSON schema (criteria scored
0..1 with evidence, the rubric's binary defect checklist, issues, improvement
plan, acceptance verdicts); samples differ by montage/tile order (shuffle seed)
-so n-sample mean/std measures judge noise. Score/defect penalties/floors/caps
+so n-sample mean/std measures judge noise. With ``n_samples > 1`` the model
+calls run in parallel (``codeverse.fanout``); results accumulate in sample order. Score/defect penalties/floors/caps
/pass are computed in code (``scoring.py``). Images are ≤2×2 montages
(``montage.py``); tracks may pass a clay/normals ``geometry_views`` RenderSet. Retries: up to
``max_attempts`` per sample on ``ModelError`` / parse failure; if no sample
@@ -25,6 +26,7 @@ from codeverse.contracts.artifacts import RenderSet
from codeverse.contracts.chat import ChatRequest, ChatResponse
from codeverse.contracts.common import Usage
from codeverse.contracts.judgment import Judgment
+from codeverse.fanout import fan_out
from codeverse.judges.base import JudgeInput
from codeverse.judges.output_schema import (
JudgeOutput,
@@ -109,6 +111,7 @@ class VlmJudge:
usage = Usage()
samples: list[JudgeOutput] = []
errors: list[str] = []
+ reqs: list[ChatRequest] = []
for k in range(self.n_samples):
seed = None if (self.n_samples == 1 and k == 0) else (inp.round_index * 1000 + k)
# a missing render is a pipeline bug, not a judge glitch → JudgeImageError propagates
@@ -117,11 +120,24 @@ class VlmJudge:
detail_crops=self.detail_crops, max_px=self.max_px, cache_dir=self.cache_dir,
extra_images=extra_images, extra_text=extra_text,
)
- req = ChatRequest(
+ reqs.append(ChatRequest(
messages=messages, system=system, response_schema=schema, temperature=self.temperature,
thinking=self.thinking, label=f"{self.label}:{self.rubric.name}:s{k}",
+ ))
+ if len(reqs) == 1: # serial path: no pool, no thread hop
+ outcomes = [self._sample(reqs[0], acceptance_ids, measured)]
+ else:
+ self.model # noqa: B018 — materialise the lazy chat model once, before the threads race
+ results = fan_out(
+ reqs, lambda r: self._sample(r, acceptance_ids, measured),
+ max_workers=len(reqs), label=f"{self.label}:samples", item_name=lambda r: r.label,
)
- out, u, err = self._sample(req, acceptance_ids, measured)
+ # ordered accumulation; _sample never raises, but a crashed worker still counts as an error
+ outcomes = [
+ r if not isinstance(r, Exception) else (None, Usage(), f"{type(r).__name__}: {r}")
+ for r in results
+ ]
+ for out, u, err in outcomes:
usage = usage + u
if out is not None:
samples.append(out)
diff --git a/harness/codeverse/languages/_ast_lint.py b/harness/codeverse/languages/_ast_lint.py
new file mode 100644
index 00000000..eacaff98
--- /dev/null
+++ b/harness/codeverse/languages/_ast_lint.py
@@ -0,0 +1,63 @@
+"""Shared AST-lint primitives for the python authoring languages (blender, cadquery, opengl_python).
+
+Each language keeps its own collector and rules — only the language-neutral
+pieces live here: :func:`dotted` (dotted-name of an Attribute/Name chain), the
+:data:`BASE_FORBIDDEN_IMPORTS` floor every language's forbidden set must
+include (a drift test pins the superset), and :func:`check_imports` (the
+forbidden → ERROR / not-allowed → WARN routing, with the finding text owned by
+the caller via ``make_finding``).
+"""
+
+from __future__ import annotations
+
+import ast
+from collections.abc import Callable, Collection, Mapping
+
+from codeverse.contracts.artifacts import GateFinding
+
+#: modules no agent script may import in ANY python language (subprocesses,
+#: network, file-system side effects, dynamic import). Languages compose their
+#: supersets: blender adds ``pathlib`` (but allows ``os``/``sys`` — headless bpy
+#: scripts legitimately touch them and the sandbox confines the damage),
+#: cadquery adds ``os``/``sys``/``pathlib`` + the OCC/CQ tooling packages,
+#: opengl_python adds ``os``/``sys`` + clock/tempfile/io modules (determinism).
+BASE_FORBIDDEN_IMPORTS: frozenset[str] = frozenset({
+ "subprocess", "urllib", "requests", "socket", "http", "shutil", "ctypes", "pickle",
+ "multiprocessing", "threading", "webbrowser", "ftplib", "smtplib", "importlib",
+})
+
+
+def dotted(node: ast.AST) -> str:
+ """Dotted name of an Attribute/Name chain (e.g. ``pkg.mod.attr.call``); '' otherwise."""
+ parts: list[str] = []
+ while isinstance(node, ast.Attribute):
+ parts.append(node.attr)
+ node = node.value
+ if isinstance(node, ast.Name):
+ parts.append(node.id)
+ return ".".join(reversed(parts))
+ if parts: # method on an expression result, e.g. ``wp.box(1).rotate`` → ``<expr>.rotate``
+ return "<expr>." + ".".join(reversed(parts))
+ return ""
+
+
+def check_imports(
+ imports: Mapping[str, int],
+ *,
+ forbidden: Collection[str],
+ allowed: Collection[str],
+ make_finding: Callable[[str, str, int], GateFinding],
+) -> list[GateFinding]:
+ """Route each ``{top-level module: line}`` through the language's import policy.
+
+ ``make_finding(kind, module, line)`` builds the finding — ``kind`` is
+ ``"forbidden"`` (module in ``forbidden``) or ``"unexpected"`` (module in
+ neither set); the caller owns message, severity and hint text.
+ """
+ out: list[GateFinding] = []
+ for mod, line in imports.items():
+ if mod in forbidden:
+ out.append(make_finding("forbidden", mod, line))
+ elif mod not in allowed:
+ out.append(make_finding("unexpected", mod, line))
+ return out
diff --git a/harness/codeverse/languages/_common.py b/harness/codeverse/languages/_common.py
index ca7dd216..80b6eac8 100644
--- a/harness/codeverse/languages/_common.py
+++ b/harness/codeverse/languages/_common.py
@@ -4,11 +4,13 @@ Runtimes execute a *wrapper script* in a subprocess (Blender's python or the
harness python). The wrapper writes ``artifacts/build.json`` (+ ``census.json``)
and the runtime turns those into a :class:`BuildResult`. This module owns:
-* :func:`run_subprocess` — timeout, process-group kill, captured output;
-* :func:`tail` — bounded log tails for BuildResult;
-* :func:`read_json_file` / :func:`write_json_atomic`;
* :func:`compose_build_result` — wrapper json + process outcome → BuildResult,
- failing loud when the wrapper did not report.
+ failing loud when the wrapper did not report;
+* :func:`read_json_file` / :func:`remove_stale` / :func:`strip_blender_noise`.
+
+The subprocess + atomic-JSON primitives (:class:`ProcResult`,
+:func:`run_subprocess`, :func:`tail`, :func:`write_json_atomic`) live in
+:mod:`codeverse.proc` and are re-exported here for the runtime callers.
Wrappers themselves are standalone scripts (they never import ``codeverse``;
Blender's bundled python cannot see this package).
@@ -17,16 +19,17 @@ Blender's bundled python cannot see this package).
from __future__ import annotations
import json
-import os
-import signal
-import subprocess
-import time
from collections.abc import Callable, Mapping
-from dataclasses import dataclass
from pathlib import Path
from typing import Any
from codeverse.contracts.artifacts import BuildResult
+from codeverse.proc import ( # noqa: F401 — re-exported
+ ProcResult,
+ run_subprocess,
+ tail,
+ write_json_atomic,
+)
# lines Blender prints on every headless run that carry no signal for the agent
_BLENDER_NOISE_PREFIXES = (
@@ -53,70 +56,6 @@ class WrapperError(RuntimeError):
"""The wrapper process did not produce a readable ``build.json`` (harness bug or crash)."""
-@dataclass(frozen=True)
-class ProcResult:
- """Outcome of one subprocess run."""
-
- returncode: int
- stdout: str
- stderr: str
- timed_out: bool
- duration_ms: int
-
-
-def run_subprocess(
- cmd: list[str],
- *,
- cwd: Path | str,
- timeout_s: float,
- env: Mapping[str, str] | None = None,
- stdin_text: str | None = None,
-) -> ProcResult:
- """Run ``cmd`` in its own process group; kill the whole group on timeout.
-
- Never raises on non-zero exit — callers inspect :attr:`ProcResult.returncode`.
- """
- t0 = time.monotonic()
- proc = subprocess.Popen(
- cmd,
- cwd=str(cwd),
- env=dict(env) if env is not None else None,
- stdin=subprocess.PIPE if stdin_text is not None else subprocess.DEVNULL,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- start_new_session=True,
- )
- timed_out = False
- try:
- out, err = proc.communicate(input=stdin_text, timeout=timeout_s)
- except subprocess.TimeoutExpired:
- timed_out = True
- _kill_group(proc)
- out, err = proc.communicate()
- return ProcResult(
- returncode=proc.returncode if proc.returncode is not None else -1,
- stdout=out or "",
- stderr=err or "",
- timed_out=timed_out,
- duration_ms=int((time.monotonic() - t0) * 1000),
- )
-
-
-def _kill_group(proc: subprocess.Popen[str]) -> None:
- try:
- os.killpg(proc.pid, signal.SIGKILL)
- except (ProcessLookupError, PermissionError):
- proc.kill()
-
-
-def tail(text: str, *, max_lines: int = 40, max_chars: int = 4000) -> str:
- """Last ``max_lines`` lines of ``text``, capped at ``max_chars`` characters."""
- lines = text.splitlines()[-max_lines:]
- s = "\n".join(lines)
- return s[-max_chars:] if len(s) > max_chars else s
-
-
def strip_blender_noise(text: str) -> str:
"""Drop Blender's boilerplate stderr/stdout lines so the agent sees signal only."""
keep = [
@@ -126,14 +65,6 @@ def strip_blender_noise(text: str) -> str:
return "\n".join(keep)
-def write_json_atomic(path: Path, data: Any) -> None:
- """tmp + rename so readers never see a partial file."""
- path.parent.mkdir(parents=True, exist_ok=True)
- tmp = path.with_suffix(path.suffix + ".tmp")
- tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False, default=str))
- tmp.replace(path)
-
-
def read_json_file(path: Path) -> dict[str, Any]:
"""Parse a JSON object file; raise :class:`WrapperError` if missing/invalid."""
if not path.is_file():
diff --git a/harness/codeverse/languages/_gl_common.py b/harness/codeverse/languages/_gl_common.py
new file mode 100644
index 00000000..f7de9204
--- /dev/null
+++ b/harness/codeverse/languages/_gl_common.py
@@ -0,0 +1,201 @@
+"""Shared build plumbing for the two graphics languages (glsl_shader, opengl_python).
+
+``finish_build`` turns a :class:`GlResult` into the canonical artifact set —
+``artifacts/frames/*.png`` (written by the runner), ``artifacts/frames_sheet.png``,
+``artifacts/preview.gif``, ``artifacts/metrics.json`` (frame stats + the
+``gl_frames`` gate), ``artifacts/build.json`` — and a typed :class:`BuildResult`
+(``glb_path`` is always None: the deliverable is code + frames).
+``judge_times`` / ``preview_times`` derive the sampled times from the plan's duration.
+``make_host`` is the one place a runtime turns settings into a :class:`GlHost`.
+
+GLSL info-log parsing lives here too (:class:`GlslMessage`, :class:`LineMap`,
+:func:`parse_glsl_log`) so ``opengl_python`` does not depend on its sibling
+``glsl_shader`` package; ``glsl_shader/wrap.py`` composes sources and keeps
+``compose`` / ``Composed`` / ``first_error``, importing the types from here.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from codeverse.config import get_settings
+from codeverse.contracts.artifacts import BuildResult, GateReport
+from codeverse.contracts.plan import GraphicsPlan
+from codeverse.spatial.frame_stats import SequenceStats, frame_gate, sequence_stats
+from codeverse.spatial.gl_render import GlHost, GlResult, gif_times, write_contact_sheet, write_gif
+from codeverse.workspace import Workspace
+
+JUDGE_TIMES: tuple[float, ...] = (0.0, 1.0, 2.5, 4.0, 6.0)
+SHEET_NAME = "frames_sheet.png"
+GIF_NAME = "preview.gif"
+METRICS_NAME = "metrics.json"
+BUILD_JSON = "build.json"
+DEFAULT_RESOLUTION: tuple[int, int] = (1280, 720)
+MAX_PIXELS = 1920 * 1080
+
+
+def make_host(override: GlHost | None = None, timeout_s: float | None = None) -> GlHost:
+ """The runtime's :class:`GlHost`: the injected one (tests) or one built from settings."""
+ if override is not None:
+ return override
+ settings = get_settings()
+ return GlHost(gpu=settings.render.gpu, timeout_s=float(timeout_s or settings.limits.render_timeout_s))
+
+
+def load_plan(ws: Workspace) -> GraphicsPlan | None:
+ p = ws.plan_path
+ if not p.is_file():
+ return None
+ try:
+ return GraphicsPlan.model_validate(json.loads(p.read_text()))
+ except Exception: # noqa: BLE001 — not a graphics plan (or invalid) → defaults
+ return None
+
+
+def resolution_for(plan: GraphicsPlan | None, *, fallback: tuple[int, int] = DEFAULT_RESOLUTION) -> tuple[int, int]:
+ if plan is None:
+ return fallback
+ w, h = int(plan.resolution[0]), int(plan.resolution[1])
+ if w < 64 or h < 64 or w * h > MAX_PIXELS:
+ return fallback
+ return w, h
+
+
+def judge_times(duration_s: float | None) -> list[float]:
+ """t ∈ {0, 1, 2.5, 4, 6} clipped to the loop duration (always ≥ 3 samples)."""
+ d = float(duration_s or 8.0)
+ ts = [t for t in JUDGE_TIMES if t <= d + 1e-9]
+ if len(ts) < 3:
+ ts = [round(i * d / 4, 3) for i in range(5)]
+ return ts
+
+
+def preview_times(duration_s: float | None, n: int = 12) -> list[float]:
+ return gif_times(float(duration_s or 8.0), n)
+
+
+def finish_build(ws: Workspace, res: GlResult, *, language: str, error_file: str = "", error_line: int | None = None,
+ error_message: str | None = None, census: dict | None = None, motion_expected: bool = True) -> BuildResult:
+ """Write derived artifacts + build.json and return the BuildResult."""
+ art = ws.artifacts
+ art.mkdir(parents=True, exist_ok=True)
+ extras: dict[str, str] = {}
+ cen: dict = {"renderer": res.renderer, "gpu": res.gpu, "feedback": res.feedback, "n_frames": len(res.frames),
+ "gl_duration_ms": res.duration_ms, **(census or {})}
+ if res.ok and res.judge_frames:
+ frames = res.judge_frames
+ extras["frames"] = str(art / "frames")
+ sheet = write_contact_sheet(frames, art / SHEET_NAME)
+ extras["sheet"] = str(sheet)
+ gif = write_gif(res.frames, art / GIF_NAME)
+ if gif is not None:
+ extras["gif"] = str(gif)
+ seq = sequence_stats([(f.time, f.path) for f in frames], nan_counts=[(f.nan, f.inf) for f in frames])
+ gate = frame_gate(seq, motion_expected=motion_expected)
+ metrics = art / METRICS_NAME
+ metrics.write_text(json.dumps({"stats": seq.model_dump(mode="json"), "gate": gate.model_dump(mode="json")}, indent=1))
+ extras["metrics"] = str(metrics)
+ cen["frame_stats"] = {"mean_lum": seq.mean_lum, "mean_diff": seq.mean_diff, "static": seq.static, "any_nan": seq.any_nan,
+ "colourfulness": seq.mean_colourfulness, "edge_density": seq.mean_edge_density}
+ result = BuildResult(
+ ok=bool(res.ok and res.judge_frames), language=language, glb_path=None, extra_paths=extras,
+ stdout_tail=res.stdout_tail, stderr_tail=(res.traceback or res.stderr_tail)[-4000:],
+ error_type="" if res.ok else (res.error_type or "RenderError"),
+ error_message="" if res.ok else (error_message if error_message is not None else res.error_message)[:4000],
+ error_file=error_file if not res.ok else "", error_line=error_line if not res.ok else None,
+ duration_ms=res.duration_ms, census=cen,
+ )
+ if res.ok and not res.judge_frames:
+ result.error_type, result.error_message = "NoFrames", "the renderer produced no frames"
+ ws.write_json(art / BUILD_JSON, result)
+ return result
+
+
+def read_metrics(ws: Workspace) -> tuple[SequenceStats, GateReport] | None:
+ p = ws.artifacts / METRICS_NAME
+ if not p.is_file():
+ return None
+ data = json.loads(p.read_text())
+ return SequenceStats.model_validate(data["stats"]), GateReport.model_validate(data["gate"])
+
+
+_TB_LINE = re.compile(r'File "(?P<file>[^"]+)", line (?P<line>\d+)')
+
+
+def traceback_location(tb: str, program_path: Path) -> tuple[str, int | None]:
+ """Last traceback frame inside the agent's program → (rel path, line)."""
+ hits = [(m.group("file"), int(m.group("line"))) for m in _TB_LINE.finditer(tb or "")]
+ name = program_path.name
+ for f, line in reversed(hits):
+ if Path(f).name == name or f.endswith(str(program_path)):
+ return "src/" + name, line
+ return "", None
+
+
+# --------------------------------------------------------------------------- GLSL info logs
+# The composed-source line map (``wrap.compose`` builds one) and the driver-dialect
+# log parser live here so both GL runtimes can map compiler messages to agent files.
+
+
+@dataclass(frozen=True)
+class Segment:
+ file: str # "src/shader.frag" | "src/common.glsl" | "harness"
+ start: int # 1-based first line in the composed source
+ n_lines: int
+
+ @property
+ def end(self) -> int:
+ return self.start + self.n_lines - 1
+
+
+@dataclass
+class LineMap:
+ segments: list[Segment] = field(default_factory=list)
+
+ def locate(self, line: int) -> tuple[str, int]:
+ """Composed line → (file, line-in-file); harness lines map to ("harness", line)."""
+ for s in self.segments:
+ if s.start <= line <= s.end:
+ return s.file, line - s.start + 1
+ return "harness", line
+
+
+# Mesa: "0:12(5): error: `foo' undeclared" · NVIDIA: "0(12) : error C1008: ..." · ANGLE/ES: "ERROR: 0:12: 'foo' : undeclared"
+_MSG_PATTERNS = (
+ re.compile(r"^\s*(?P<src>\d+):(?P<line>\d+)\((?P<col>\d+)\):\s*(?P<kind>error|warning):\s*(?P<msg>.*)$"),
+ re.compile(r"^\s*(?P<src>\d+)\((?P<line>\d+)\)\s*:\s*(?P<kind>error|warning)\s*(?P<msg>.*)$"),
+ re.compile(r"^\s*(?P<kind>ERROR|WARNING):\s*(?P<src>\d+):(?P<line>\d+):\s*(?P<msg>.*)$"),
+)
+
+
+@dataclass(frozen=True)
+class GlslMessage:
+ kind: str # error | warning
+ line: int # composed line
+ message: str
+ file: str = ""
+ file_line: int = 0
+
+ def text(self) -> str:
+ loc = f"{self.file}:{self.file_line}" if self.file else f"line {self.line}"
+ return f"{loc}: {self.kind}: {self.message}"
+
+
+def parse_glsl_log(log: str, line_map: LineMap | None = None) -> list[GlslMessage]:
+ """Parse a GLSL info log (any driver dialect) into messages mapped to agent files."""
+ out: list[GlslMessage] = []
+ for raw in (log or "").splitlines():
+ for pat in _MSG_PATTERNS:
+ m = pat.match(raw)
+ if not m:
+ continue
+ line = int(m.group("line"))
+ kind = m.group("kind").lower()
+ msg = m.group("msg").strip()
+ f, fl = line_map.locate(line) if line_map else ("", 0)
+ out.append(GlslMessage(kind=kind, line=line, message=msg, file=f, file_line=fl))
+ break
+ return out
diff --git a/harness/codeverse/languages/blender/layout.py b/harness/codeverse/languages/blender/layout.py
index 3a0070a0..9077a338 100644
--- a/harness/codeverse/languages/blender/layout.py
+++ b/harness/codeverse/languages/blender/layout.py
@@ -18,11 +18,12 @@ import time
from pathlib import Path
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.conventions import to_snake
from codeverse.languages.blender.lint import GATE, lint_blender_source
from codeverse.workspace import Workspace
-ENTRY_REL = "src/model.py"
+ENTRY_REL = ENTRY_FILE[Language.BLENDER] # "src/model.py"
PARTS_DIR = "parts"
PARTS_PKG = "parts"
diff --git a/harness/codeverse/languages/blender/lint.py b/harness/codeverse/languages/blender/lint.py
index 7d5cb579..940132bd 100644
--- a/harness/codeverse/languages/blender/lint.py
+++ b/harness/codeverse/languages/blender/lint.py
@@ -16,19 +16,23 @@ import time
from pathlib import Path
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
+from codeverse.languages._ast_lint import ( # noqa: F401 — dotted re-exported
+ BASE_FORBIDDEN_IMPORTS,
+ check_imports,
+ dotted,
+)
GATE = "lint:blender"
ALLOWED_IMPORTS = {
"bpy", "bmesh", "mathutils", "math", "random", "numpy", "np", "itertools", "functools",
"collections", "typing", "dataclasses", "colorsys", "statistics", "operator", "enum", "copy",
+ # os/sys are deliberately ALLOWED here (unlike cadquery/opengl): headless bpy scripts
+ # touch them legitimately and the forbidden-call rules below catch the harmful uses.
"sys", "os", "__future__", "bpy_extras",
"parts", # the workspace's own src/parts/<snake>.py modules (multi-file layout)
}
-FORBIDDEN_IMPORTS = {
- "subprocess", "urllib", "requests", "socket", "http", "shutil", "ctypes", "pickle",
- "multiprocessing", "threading", "webbrowser", "ftplib", "smtplib", "importlib", "pathlib",
-}
+FORBIDDEN_IMPORTS = frozenset(BASE_FORBIDDEN_IMPORTS | {"pathlib"})
# dotted-name prefixes of calls that violate the contract (harness owns them)
FORBIDDEN_CALL_PREFIXES: tuple[tuple[str, str], ...] = (
("bpy.ops.render.", "rendering is done by the harness; delete all bpy.ops.render.* calls"),
@@ -66,20 +70,6 @@ KNOWN_BINDINGS = ("Vector", "Matrix", "Euler", "Quaternion", "bmesh", "math", "r
PASCAL_RE = re.compile(r"^[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)*(?:_\d+)?$")
-def dotted(node: ast.AST) -> str:
- """``bpy.ops.render.render`` for an Attribute/Name chain; '' otherwise."""
- parts: list[str] = []
- while isinstance(node, ast.Attribute):
- parts.append(node.attr)
- node = node.value
- if isinstance(node, ast.Name):
- parts.append(node.id)
- return ".".join(reversed(parts))
- if parts: # method on an expression result, e.g. ``wp.box(1).rotate`` → ``<expr>.rotate``
- return "<expr>." + ".".join(reversed(parts))
- return ""
-
-
class _Collector(ast.NodeVisitor):
"""One pass over the tree; rules read the collected facts."""
@@ -191,11 +181,13 @@ def _rules(c: _Collector, source: str, *, target: str, expect_names: bool, expec
E, W, I = Severity.ERROR, Severity.WARN, Severity.INFO # noqa: E741
if expect_bpy and "bpy" not in c.imports:
out.append(_f(E, f"{target} never imports bpy", 1, "start the file with `import bpy`"))
- for mod, line in c.imports.items():
- if mod in FORBIDDEN_IMPORTS:
- out.append(_f(E, f"forbidden import `{mod}`", line, "only bpy/bmesh/mathutils/math/random/numpy (+stdlib data helpers) are allowed"))
- elif mod not in ALLOWED_IMPORTS:
- out.append(_f(W, f"unexpected import `{mod}` (not available / not allowed in the build sandbox)", line, "use only bpy, bmesh, mathutils, math, random, numpy"))
+
+ def _import_finding(kind: str, mod: str, line: int) -> GateFinding:
+ if kind == "forbidden":
+ return _f(E, f"forbidden import `{mod}`", line, "only bpy/bmesh/mathutils/math/random/numpy (+stdlib data helpers) are allowed")
+ return _f(W, f"unexpected import `{mod}` (not available / not allowed in the build sandbox)", line, "use only bpy, bmesh, mathutils, math, random, numpy")
+
+ out.extend(check_imports(c.imports, forbidden=FORBIDDEN_IMPORTS, allowed=ALLOWED_IMPORTS, make_finding=_import_finding))
for name, call in c.calls:
for prefix, hint in FORBIDDEN_CALL_PREFIXES:
if name.startswith(prefix):
diff --git a/harness/codeverse/languages/cadquery/lint.py b/harness/codeverse/languages/cadquery/lint.py
index 74a58653..a970d578 100644
--- a/harness/codeverse/languages/cadquery/lint.py
+++ b/harness/codeverse/languages/cadquery/lint.py
@@ -13,12 +13,19 @@ import time
from pathlib import Path
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
+from codeverse.languages._ast_lint import ( # noqa: F401 — dotted re-exported
+ BASE_FORBIDDEN_IMPORTS,
+ check_imports,
+ dotted,
+)
GATE = "lint:cadquery"
ALLOWED_IMPORTS = {"cadquery", "cq", "math", "random", "numpy", "np", "itertools", "functools", "collections",
"typing", "dataclasses", "enum", "copy", "statistics", "operator", "__future__"}
-FORBIDDEN_IMPORTS = {"os", "sys", "subprocess", "shutil", "pathlib", "urllib", "requests", "socket", "http", "pickle",
- "ctypes", "multiprocessing", "threading", "importlib", "OCP", "OCC", "ocp_vscode", "cq_editor", "cq_warehouse", "build123d"}
+FORBIDDEN_IMPORTS = frozenset(BASE_FORBIDDEN_IMPORTS | {
+ "os", "sys", "pathlib", # no file-system / interpreter access (unlike blender, nothing legitimate needs them)
+ "OCP", "OCC", "ocp_vscode", "cq_editor", "cq_warehouse", "build123d", # raw cadquery only — no OCC/tooling layers
+})
FORBIDDEN_CALLS: tuple[tuple[str, str], ...] = (
("show_object", "show_object() exists only in CQ-editor; delete it — the harness renders for you"),
("show", "show() is for notebooks/CQ-editor; delete it"),
@@ -35,19 +42,6 @@ FORBIDDEN_METHODS = {"save": "no .save()/.export() on the assembly — the harne
PASCAL_RE = re.compile(r"^[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)*(?:_\d+)?$")
-def dotted(node: ast.AST) -> str:
- parts: list[str] = []
- while isinstance(node, ast.Attribute):
- parts.append(node.attr)
- node = node.value
- if isinstance(node, ast.Name):
- parts.append(node.id)
- return ".".join(reversed(parts))
- if parts: # method on an expression result, e.g. ``wp.box(1).rotate`` → ``<expr>.rotate``
- return "<expr>." + ".".join(reversed(parts))
- return ""
-
-
RADIAN_CONSTS = {"pi", "tau"}
TO_RADIANS_FUNCS = {"radians", "deg2rad"}
TO_DEGREES_FUNCS = {"degrees", "rad2deg"}
@@ -150,11 +144,13 @@ def _rules(c: _Collector, tree: ast.Module, source: str) -> list[GateFinding]:
out: list[GateFinding] = []
if "cadquery" not in c.imports:
out.append(_f(E, "model.py never imports cadquery", 1, "start with `import cadquery as cq`"))
- for mod, line in c.imports.items():
- if mod in FORBIDDEN_IMPORTS:
- out.append(_f(E, f"forbidden import `{mod}`", line, "only `cadquery`, `math`, `random`, `numpy` (+ stdlib data helpers) are allowed"))
- elif mod not in ALLOWED_IMPORTS:
- out.append(_f(W, f"unexpected import `{mod}` (may not exist in the build sandbox)", line, "stick to cadquery / math / random / numpy"))
+
+ def _import_finding(kind: str, mod: str, line: int) -> GateFinding:
+ if kind == "forbidden":
+ return _f(E, f"forbidden import `{mod}`", line, "only `cadquery`, `math`, `random`, `numpy` (+ stdlib data helpers) are allowed")
+ return _f(W, f"unexpected import `{mod}` (may not exist in the build sandbox)", line, "stick to cadquery / math / random / numpy")
+
+ out.extend(check_imports(c.imports, forbidden=FORBIDDEN_IMPORTS, allowed=ALLOWED_IMPORTS, make_finding=_import_finding))
top, guarded = _top_level_assigned(tree)
if "result" not in top:
hint = "assign `result = cq.Assembly(...)` (or a Workplane) at module level, NOT under `if __name__ == '__main__':`" if "result" in guarded \
diff --git a/harness/codeverse/languages/cadquery/runtime.py b/harness/codeverse/languages/cadquery/runtime.py
index f887bd04..6e7068bb 100644
--- a/harness/codeverse/languages/cadquery/runtime.py
+++ b/harness/codeverse/languages/cadquery/runtime.py
@@ -12,7 +12,7 @@ from pathlib import Path
from codeverse.config import Settings, get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import Plan, StaticPlan
from codeverse.conventions import MAX_TRIS_OBJECT
from codeverse.languages._common import compose_build_result, remove_stale, run_subprocess
@@ -37,7 +37,7 @@ class CadQueryRuntime:
"""LanguageRuntime for ``Language.CADQUERY``."""
language = Language.CADQUERY
- entry_globs: tuple[str, ...] = ("src/model.py",)
+ entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.CADQUERY],)
def __init__(self, *, python: str | None = None, settings: Settings | None = None, rlimit_gb: float = 8.0):
self._settings = settings or get_settings()
diff --git a/harness/codeverse/languages/glsl_shader/gl_build.py b/harness/codeverse/languages/glsl_shader/gl_build.py
index da26f27b..d06fe2e2 100644
--- a/harness/codeverse/languages/glsl_shader/gl_build.py
+++ b/harness/codeverse/languages/glsl_shader/gl_build.py
@@ -1,119 +1,7 @@
-"""Shared build plumbing for the two graphics languages (glsl_shader, opengl_python).
+"""Compat shim — everything here moved to :mod:`codeverse.languages._gl_common`.
-``finish_build`` turns a :class:`GlResult` into the canonical artifact set —
-``artifacts/frames/*.png`` (written by the runner), ``artifacts/frames_sheet.png``,
-``artifacts/preview.gif``, ``artifacts/metrics.json`` (frame stats + the
-``gl_frames`` gate), ``artifacts/build.json`` — and a typed :class:`BuildResult`
-(``glb_path`` is always None: the deliverable is code + frames).
-``judge_times`` / ``preview_times`` derive the sampled times from the plan's duration.
+TODO(follow-up wave): flip ``tracks/graphics.py`` / ``tracks/graphics_steps.py`` /
+``spatial/tools*.py`` / ``tests/graphics`` imports to ``_gl_common`` and delete this shim.
"""
-from __future__ import annotations
-
-import json
-import re
-from pathlib import Path
-
-from codeverse.contracts.artifacts import BuildResult, GateReport
-from codeverse.contracts.plan import GraphicsPlan
-from codeverse.spatial.frame_stats import SequenceStats, frame_gate, sequence_stats
-from codeverse.spatial.gl_render import GlResult, gif_times, write_contact_sheet, write_gif
-from codeverse.workspace import Workspace
-
-JUDGE_TIMES: tuple[float, ...] = (0.0, 1.0, 2.5, 4.0, 6.0)
-SHEET_NAME = "frames_sheet.png"
-GIF_NAME = "preview.gif"
-METRICS_NAME = "metrics.json"
-BUILD_JSON = "build.json"
-DEFAULT_RESOLUTION: tuple[int, int] = (1280, 720)
-MAX_PIXELS = 1920 * 1080
-
-
-def load_plan(ws: Workspace) -> GraphicsPlan | None:
- p = ws.plan_path
- if not p.is_file():
- return None
- try:
- return GraphicsPlan.model_validate(json.loads(p.read_text()))
- except Exception: # noqa: BLE001 — not a graphics plan (or invalid) → defaults
- return None
-
-
-def resolution_for(plan: GraphicsPlan | None, *, fallback: tuple[int, int] = DEFAULT_RESOLUTION) -> tuple[int, int]:
- if plan is None:
- return fallback
- w, h = int(plan.resolution[0]), int(plan.resolution[1])
- if w < 64 or h < 64 or w * h > MAX_PIXELS:
- return fallback
- return w, h
-
-
-def judge_times(duration_s: float | None) -> list[float]:
- """t ∈ {0, 1, 2.5, 4, 6} clipped to the loop duration (always ≥ 3 samples)."""
- d = float(duration_s or 8.0)
- ts = [t for t in JUDGE_TIMES if t <= d + 1e-9]
- if len(ts) < 3:
- ts = [round(i * d / 4, 3) for i in range(5)]
- return ts
-
-
-def preview_times(duration_s: float | None, n: int = 12) -> list[float]:
- return gif_times(float(duration_s or 8.0), n)
-
-
-def finish_build(ws: Workspace, res: GlResult, *, language: str, error_file: str = "", error_line: int | None = None,
- error_message: str | None = None, census: dict | None = None, motion_expected: bool = True) -> BuildResult:
- """Write derived artifacts + build.json and return the BuildResult."""
- art = ws.artifacts
- art.mkdir(parents=True, exist_ok=True)
- extras: dict[str, str] = {}
- cen: dict = {"renderer": res.renderer, "gpu": res.gpu, "feedback": res.feedback, "n_frames": len(res.frames),
- "gl_duration_ms": res.duration_ms, **(census or {})}
- if res.ok and res.judge_frames:
- frames = res.judge_frames
- extras["frames"] = str(art / "frames")
- sheet = write_contact_sheet(frames, art / SHEET_NAME)
- extras["sheet"] = str(sheet)
- gif = write_gif(res.frames, art / GIF_NAME)
- if gif is not None:
- extras["gif"] = str(gif)
- seq = sequence_stats([(f.time, f.path) for f in frames], nan_counts=[(f.nan, f.inf) for f in frames])
- gate = frame_gate(seq, motion_expected=motion_expected)
- metrics = art / METRICS_NAME
- metrics.write_text(json.dumps({"stats": seq.model_dump(mode="json"), "gate": gate.model_dump(mode="json")}, indent=1))
- extras["metrics"] = str(metrics)
- cen["frame_stats"] = {"mean_lum": seq.mean_lum, "mean_diff": seq.mean_diff, "static": seq.static, "any_nan": seq.any_nan,
- "colourfulness": seq.mean_colourfulness, "edge_density": seq.mean_edge_density}
- result = BuildResult(
- ok=bool(res.ok and res.judge_frames), language=language, glb_path=None, extra_paths=extras,
- stdout_tail=res.stdout_tail, stderr_tail=(res.traceback or res.stderr_tail)[-4000:],
- error_type="" if res.ok else (res.error_type or "RenderError"),
- error_message="" if res.ok else (error_message if error_message is not None else res.error_message)[:4000],
- error_file=error_file if not res.ok else "", error_line=error_line if not res.ok else None,
- duration_ms=res.duration_ms, census=cen,
- )
- if res.ok and not res.judge_frames:
- result.error_type, result.error_message = "NoFrames", "the renderer produced no frames"
- ws.write_json(art / BUILD_JSON, result)
- return result
-
-
-def read_metrics(ws: Workspace) -> tuple[SequenceStats, GateReport] | None:
- p = ws.artifacts / METRICS_NAME
- if not p.is_file():
- return None
- data = json.loads(p.read_text())
- return SequenceStats.model_validate(data["stats"]), GateReport.model_validate(data["gate"])
-
-
-_TB_LINE = re.compile(r'File "(?P<file>[^"]+)", line (?P<line>\d+)')
-
-
-def traceback_location(tb: str, program_path: Path) -> tuple[str, int | None]:
- """Last traceback frame inside the agent's program → (rel path, line)."""
- hits = [(m.group("file"), int(m.group("line"))) for m in _TB_LINE.finditer(tb or "")]
- name = program_path.name
- for f, line in reversed(hits):
- if Path(f).name == name or f.endswith(str(program_path)):
- return "src/" + name, line
- return "", None
+from codeverse.languages._gl_common import * # noqa: F401,F403
diff --git a/harness/codeverse/languages/glsl_shader/runtime.py b/harness/codeverse/languages/glsl_shader/runtime.py
index 40f0999e..ddb13104 100644
--- a/harness/codeverse/languages/glsl_shader/runtime.py
+++ b/harness/codeverse/languages/glsl_shader/runtime.py
@@ -11,20 +11,21 @@ from __future__ import annotations
from pathlib import Path
-from codeverse.config import get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import Plan
-from codeverse.languages.glsl_shader.gl_build import (
+from codeverse.languages._gl_common import (
finish_build,
judge_times,
load_plan,
+ make_host,
+ parse_glsl_log,
preview_times,
resolution_for,
)
from codeverse.languages.glsl_shader.lint import BUFFER_A, COMMON, SHADER, lint_workspace
from codeverse.languages.glsl_shader.skeleton import write_skeleton
-from codeverse.languages.glsl_shader.wrap import Composed, compose, first_error, parse_glsl_log
+from codeverse.languages.glsl_shader.wrap import Composed, compose, first_error
from codeverse.prompts import PROMPTS_DIR, load_text
from codeverse.spatial.gl_render import GlHost, GlResult
from codeverse.workspace import Workspace
@@ -37,7 +38,7 @@ Optional src/common.glsl (helpers, pasted in first) and src/buffer_a.frag (one f
class GlslShaderRuntime:
language = Language.GLSL_SHADER
- entry_globs: tuple[str, ...] = ("src/shader.frag", "src/common.glsl", "src/buffer_a.frag")
+ entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.GLSL_SHADER], COMMON, BUFFER_A)
def __init__(self, *, host: GlHost | None = None):
self._host = host
@@ -61,10 +62,7 @@ class GlslShaderRuntime:
# ------------------------------------------------------------------ build
def host(self, timeout_s: float | None = None) -> GlHost:
- if self._host is not None:
- return self._host
- settings = get_settings()
- return GlHost(gpu=settings.render.gpu, timeout_s=float(timeout_s or settings.limits.render_timeout_s))
+ return make_host(self._host, timeout_s)
def compose_sources(self, ws: Workspace) -> tuple[Composed, Composed | None]:
shader = (ws.root / SHADER).read_text(errors="replace")
diff --git a/harness/codeverse/languages/glsl_shader/wrap.py b/harness/codeverse/languages/glsl_shader/wrap.py
index ebd4d2ff..cbe93a37 100644
--- a/harness/codeverse/languages/glsl_shader/wrap.py
+++ b/harness/codeverse/languages/glsl_shader/wrap.py
@@ -11,7 +11,14 @@ to ``src/shader.frag`` / ``src/common.glsl`` line numbers.
from __future__ import annotations
import re
-from dataclasses import dataclass, field
+from dataclasses import dataclass
+
+from codeverse.languages._gl_common import ( # noqa: F401 — re-exported
+ GlslMessage,
+ LineMap,
+ Segment,
+ parse_glsl_log,
+)
UNIFORM_NAMES: tuple[str, ...] = ("u_time", "u_resolution", "u_mouse", "u_frame", "u_prev", "u_noise", "u_buffer_a")
@@ -46,29 +53,6 @@ _PLAIN_MAIN = re.compile(r"\bvoid\s+main\s*\(\s*(void)?\s*\)")
_OUT_DECL = re.compile(r"\bout\s+vec4\s+\w+\s*;")
-@dataclass(frozen=True)
-class Segment:
- file: str # "src/shader.frag" | "src/common.glsl" | "harness"
- start: int # 1-based first line in the composed source
- n_lines: int
-
- @property
- def end(self) -> int:
- return self.start + self.n_lines - 1
-
-
-@dataclass
-class LineMap:
- segments: list[Segment] = field(default_factory=list)
-
- def locate(self, line: int) -> tuple[str, int]:
- """Composed line → (file, line-in-file); harness lines map to ("harness", line)."""
- for s in self.segments:
- if s.start <= line <= s.end:
- return s.file, line - s.start + 1
- return "harness", line
-
-
@dataclass
class Composed:
source: str
@@ -125,44 +109,8 @@ def compose(shader_src: str, common_src: str | None = None, *, shader_file: str
# --------------------------------------------------------------------------- compiler messages
-# Mesa: "0:12(5): error: `foo' undeclared" · NVIDIA: "0(12) : error C1008: ..." · ANGLE/ES: "ERROR: 0:12: 'foo' : undeclared"
-_MSG_PATTERNS = (
- re.compile(r"^\s*(?P<src>\d+):(?P<line>\d+)\((?P<col>\d+)\):\s*(?P<kind>error|warning):\s*(?P<msg>.*)$"),
- re.compile(r"^\s*(?P<src>\d+)\((?P<line>\d+)\)\s*:\s*(?P<kind>error|warning)\s*(?P<msg>.*)$"),
- re.compile(r"^\s*(?P<kind>ERROR|WARNING):\s*(?P<src>\d+):(?P<line>\d+):\s*(?P<msg>.*)$"),
-)
-
-
-@dataclass(frozen=True)
-class GlslMessage:
- kind: str # error | warning
- line: int # composed line
- message: str
- file: str = ""
- file_line: int = 0
-
- def text(self) -> str:
- loc = f"{self.file}:{self.file_line}" if self.file else f"line {self.line}"
- return f"{loc}: {self.kind}: {self.message}"
-
-
-def parse_glsl_log(log: str, line_map: LineMap | None = None) -> list[GlslMessage]:
- """Parse a GLSL info log (any driver dialect) into messages mapped to agent files."""
- out: list[GlslMessage] = []
- for raw in (log or "").splitlines():
- for pat in _MSG_PATTERNS:
- m = pat.match(raw)
- if not m:
- continue
- line = int(m.group("line"))
- kind = m.group("kind").lower()
- msg = m.group("msg").strip()
- f, fl = line_map.locate(line) if line_map else ("", 0)
- out.append(GlslMessage(kind=kind, line=line, message=msg, file=f, file_line=fl))
- break
- return out
-
-
+# GlslMessage / LineMap / parse_glsl_log live in ``_gl_common`` (shared with opengl_python)
+# and are re-exported above; only source composition + first_error remain here.
def first_error(messages: list[GlslMessage]) -> GlslMessage | None:
for m in messages:
if m.kind == "error":
diff --git a/harness/codeverse/languages/opengl_python/lint.py b/harness/codeverse/languages/opengl_python/lint.py
index 9a154658..2938a751 100644
--- a/harness/codeverse/languages/opengl_python/lint.py
+++ b/harness/codeverse/languages/opengl_python/lint.py
@@ -18,6 +18,7 @@ import ast
import re
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
+from codeverse.languages._ast_lint import BASE_FORBIDDEN_IMPORTS, check_imports
from codeverse.workspace import Workspace
GATE = "lint:opengl_python"
@@ -27,8 +28,10 @@ ALLOWED_MODULES: frozenset[str] = frozenset({
"functools", "collections", "colorsys", "__future__", "enum",
})
WINDOW_LIBS = {"glfw", "pygame", "pyglet", "PyQt5", "PyQt6", "PySide2", "PySide6", "tkinter", "moderngl_window", "sdl2", "wx", "OpenGL"}
-DANGEROUS_MODULES = {"os", "sys", "subprocess", "socket", "requests", "urllib", "http", "shutil", "ctypes", "importlib",
- "multiprocessing", "threading", "signal", "pickle", "tempfile", "io", "time", "datetime"}
+DANGEROUS_MODULES = frozenset(BASE_FORBIDDEN_IMPORTS | {
+ "os", "sys", # no interpreter / file-system access
+ "signal", "tempfile", "io", "time", "datetime", # frames must be a pure, deterministic function of t
+})
FORBIDDEN_CALLS = {"open", "exec", "eval", "compile", "__import__", "input", "exit", "quit", "breakpoint"}
MAX_CHARS = 120_000
_VERSION_RE = re.compile(r"#\s*version\s+(\d+)(?:\s+(\w+))?")
@@ -56,13 +59,18 @@ class _Visitor(ast.NodeVisitor):
if root in WINDOW_LIBS:
self.findings.append(_finding(Severity.ERROR, "window_lib", f"imports `{name}` (window / input library)",
"the harness creates the context and framebuffer; draw into the fbo it passes — no window", line))
- elif root in DANGEROUS_MODULES:
- self.findings.append(_finding(Severity.ERROR, "forbidden_import", f"imports `{name}` (not allowed in a rendering program)",
- "allowed: moderngl, numpy, math, random, struct, array, pathlib, typing, dataclasses, itertools, "
- "functools, collections, colorsys", line))
- elif root not in ALLOWED_MODULES:
- self.findings.append(_finding(Severity.ERROR, "forbidden_import", f"imports `{name}` which is outside the allow-list",
- "use only moderngl + numpy + stdlib math/random/struct/array/pathlib", line))
+ return
+
+ def _import_finding(kind: str, _mod: str, ln: int) -> GateFinding:
+ if kind == "forbidden":
+ return _finding(Severity.ERROR, "forbidden_import", f"imports `{name}` (not allowed in a rendering program)",
+ "allowed: moderngl, numpy, math, random, struct, array, pathlib, typing, dataclasses, itertools, "
+ "functools, collections, colorsys", ln)
+ return _finding(Severity.ERROR, "forbidden_import", f"imports `{name}` which is outside the allow-list",
+ "use only moderngl + numpy + stdlib math/random/struct/array/pathlib", ln)
+
+ self.findings.extend(check_imports({root: line}, forbidden=DANGEROUS_MODULES, allowed=ALLOWED_MODULES,
+ make_finding=_import_finding))
def visit_Import(self, node: ast.Import) -> None:
for a in node.names:
diff --git a/harness/codeverse/languages/opengl_python/runtime.py b/harness/codeverse/languages/opengl_python/runtime.py
index d2a8d9a1..2862dba3 100644
--- a/harness/codeverse/languages/opengl_python/runtime.py
+++ b/harness/codeverse/languages/opengl_python/runtime.py
@@ -12,19 +12,19 @@ from __future__ import annotations
from pathlib import Path
-from codeverse.config import get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import Plan
-from codeverse.languages.glsl_shader.gl_build import (
+from codeverse.languages._gl_common import (
finish_build,
judge_times,
load_plan,
+ make_host,
+ parse_glsl_log,
preview_times,
resolution_for,
traceback_location,
)
-from codeverse.languages.glsl_shader.wrap import parse_glsl_log
from codeverse.languages.opengl_python.lint import PROGRAM, lint_workspace
from codeverse.languages.opengl_python.skeleton import write_skeleton
from codeverse.prompts import PROMPTS_DIR, load_text
@@ -39,7 +39,7 @@ no file IO (except src/*.glsl next to program.py). Imports: moderngl, numpy, ma
class OpenGLPythonRuntime:
language = Language.OPENGL_PYTHON
- entry_globs: tuple[str, ...] = ("src/program.py", "src/*.glsl")
+ entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.OPENGL_PYTHON], "src/*.glsl")
def __init__(self, *, host: GlHost | None = None):
self._host = host
@@ -63,10 +63,7 @@ class OpenGLPythonRuntime:
# ------------------------------------------------------------------ build
def host(self, timeout_s: float | None = None) -> GlHost:
- if self._host is not None:
- return self._host
- settings = get_settings()
- return GlHost(gpu=settings.render.gpu, timeout_s=float(timeout_s or settings.limits.render_timeout_s))
+ return make_host(self._host, timeout_s)
def build(self, ws: Workspace, *, timeout_s: int | None = None, times: list[float] | None = None,
preview: bool = True, width: int | None = None, height: int | None = None) -> BuildResult:
diff --git a/harness/codeverse/languages/scene_threejs/assemble.py b/harness/codeverse/languages/scene_threejs/assemble.py
index 33677c69..ddf34476 100644
--- a/harness/codeverse/languages/scene_threejs/assemble.py
+++ b/harness/codeverse/languages/scene_threejs/assemble.py
@@ -3,10 +3,14 @@
Zone agents write ``src/zones/<snake>.js`` (``build(ctx) → Group``) and an env
agent writes ``src/env.js``; this module (code, not an LLM) then:
-1. probes every zone module in the browser host (``probe_zone_modules``):
- does it import, does ``build(ctx)`` return an Object3D, what is its bbox;
-2. derives cameras from the measured zone bboxes (``derive_cameras``):
- eye on the sun side (``SUN_AZIMUTH_DEG`` from env.js), overview + per-zone;
+1. probes every zone module in the browser host (``probe_zone_modules``),
+ passing the sun azimuth (``SUN_AZIMUTH_DEG`` from env.js): does it import,
+ does ``build(ctx)`` return an Object3D, what is its bbox — and the driver
+ (``probe_scene.mjs --sun-azimuth``, camera math owned by
+ ``runtime_js/lib/orbit.mjs``) fits overview + per-zone cameras, eye on the
+ sun side, from the measured bboxes;
+2. converts those fitted specs into ``CameraPlan``s (``cameras_from_specs``;
+ the only camera math left in python is the no-measurable-zones fallback);
3. writes ``src/scene.js`` importing only the healthy zones, naming groups,
preloading ``public/assets/*.glb`` into ``ctx.assets`` and fanning
``update(t, dt)`` out to env + zones (``assemble``).
@@ -15,7 +19,6 @@ agent writes ``src/env.js``; this module (code, not an LLM) then:
from __future__ import annotations
import json
-import math
import re
from pathlib import Path
from typing import Any
@@ -41,6 +44,22 @@ class ZoneProbe(BaseModel):
triangles: int = 0
+class ZoneProbeReport(BaseModel):
+ """``probe_zone_modules`` outcome. Unpacks as ``probes, census, other = ...``
+ (historical 3-tuple); ``camera_specs`` carries the driver-fitted cameras
+ (``{azimuth, overview, zones:[{group, position, lookAt, fov}]}``)."""
+
+ probes: dict[str, ZoneProbe] = Field(default_factory=dict)
+ census: dict[str, Any] = Field(default_factory=dict)
+ other_errors: list[str] = Field(default_factory=list)
+ camera_specs: dict[str, Any] = Field(default_factory=dict)
+
+ def __iter__(self): # type: ignore[override]
+ yield self.probes
+ yield self.census
+ yield self.other_errors
+
+
class AssembleResult(BaseModel):
scene_path: str
zones_included: list[str] = Field(default_factory=list)
@@ -123,21 +142,24 @@ export async function createScene({{ renderer, loaders }}) {{
"""
-def probe_zone_modules(ws: Workspace, *, timeout_s: float = 90.0) -> tuple[dict[str, ZoneProbe], dict[str, Any], list[str]]:
+def probe_zone_modules(ws: Workspace, *, timeout_s: float = 90.0, sun_azimuth_deg: float | None = None) -> ZoneProbeReport:
"""Probe env + every zone module independently in the browser host.
- Returns (zone probes by snake name, census of the probe scene, other console errors).
+ Returns a :class:`ZoneProbeReport` (unpacks as the historical 3-tuple of
+ zone probes by snake name, census, other console errors). When
+ ``sun_azimuth_deg`` is given the driver also fits overview + per-group
+ cameras from the measured bboxes (``camera_specs``).
"""
from codeverse.spatial.render_scene import run_scene_script
zones = zone_files(ws)
probe_path = ws.root / PROBE_REL
probe_path.write_text(_probe_module(zones, glb_assets(ws)))
+ args = ["--ws", str(ws.root), "--scene", PROBE_REL, "--timeout-ms", str(int(timeout_s * 1000))]
+ if sun_azimuth_deg is not None:
+ args += ["--sun-azimuth", str(sun_azimuth_deg)]
try:
- res = run_scene_script(
- "probe_scene.mjs", ["--ws", str(ws.root), "--scene", PROBE_REL, "--timeout-ms", str(int(timeout_s * 1000))],
- timeout_s=timeout_s + 20,
- )
+ res = run_scene_script("probe_scene.mjs", args, timeout_s=timeout_s + 20)
finally:
probe_path.unlink(missing_ok=True)
s = res.summary
@@ -165,68 +187,35 @@ def probe_zone_modules(ws: Workspace, *, timeout_s: float = 90.0) -> tuple[dict[
else:
probes[name] = ZoneProbe(name=name, file=f"src/zones/{p.name}", ok=True, bbox=g.get("bbox"),
meshes=g.get("meshes", 0), triangles=g.get("triangles", 0))
- return probes, census, other
+ return ZoneProbeReport(probes=probes, census=census, other_errors=other, camera_specs=s.get("fitted_cameras") or {})
# --------------------------------------------------------------------------- cameras
-def _dir(az_deg: float, el_deg: float) -> tuple[float, float, float]:
- az, el = math.radians(az_deg), math.radians(el_deg)
- return (math.sin(az) * math.cos(el), math.sin(el), math.cos(az) * math.cos(el))
-
-
-def _union(boxes: list[dict[str, Any]]) -> dict[str, Any] | None:
- boxes = [b for b in boxes if b and b.get("min") and b.get("max")]
- if not boxes:
- return None
- mn = [min(b["min"][i] for b in boxes) for i in range(3)]
- mx = [max(b["max"][i] for b in boxes) for i in range(3)]
- return {"min": mn, "max": mx, "size": [mx[i] - mn[i] for i in range(3)]}
-
-
-def fit_overview(bbox: dict[str, Any], *, azimuth: float, elevation: float = 30.0, fov: float = 50.0, aspect: float = 16 / 9,
- floor: float | None = None, name: str = "overview") -> CameraPlan:
- """Eye on the sun side, fitted to the bbox bounding sphere (ground-aware)."""
- size = bbox["size"]
- center = [bbox["min"][i] + size[i] / 2 for i in range(3)]
- radius = max(0.5, math.hypot(*size) / 2)
- fov_v = math.radians(fov)
- fov_h = 2 * math.atan(math.tan(fov_v / 2) * aspect)
- dist = radius / math.sin(min(fov_v, fov_h) / 2) * 0.95
- d = _dir(azimuth, elevation)
- eye = [center[i] + d[i] * dist for i in range(3)]
- fl = bbox["min"][1] if floor is None else floor
- eye[1] = max(eye[1], fl + 0.5)
- return CameraPlan(name=name, position=tuple(round(v, 3) for v in eye), look_at=tuple(round(v, 3) for v in center), fov=fov,
- purpose="overview fitted to measured content bounds, sun side")
-
-
-def fit_zone_camera(name: str, bbox: dict[str, Any], *, azimuth: float, floor: float | None = None, fov: float = 50.0) -> CameraPlan:
- """Eye-level shot standing just outside the zone footprint on the sun side."""
- size = bbox["size"]
- center = [bbox["min"][i] + size[i] / 2 for i in range(3)]
- dx, _, dz = _dir(azimuth, 0.0)
- hx, hz = size[0] / 2, size[2] / 2
- tx = hx / abs(dx) if abs(dx) > 1e-6 else math.inf
- tz = hz / abs(dz) if abs(dz) > 1e-6 else math.inf
- back = min(tx, tz) + max(2.0, 0.25 * max(size[0], size[2]))
- fl = bbox["min"][1] if floor is None else floor
- eye_h = fl + 1.7 + min(4.0, 0.15 * size[1])
- eye = (center[0] + dx * back, eye_h, center[2] + dz * back)
- look = (center[0], min(center[1], fl + 0.4 * size[1] + 0.5), center[2])
- return CameraPlan(name=f"{to_snake(name)}_view", position=tuple(round(v, 3) for v in eye), look_at=tuple(round(v, 3) for v in look),
- fov=fov, purpose=f"zone {to_pascal(name)} from the sun side")
-
-
-def derive_cameras(probes: dict[str, ZoneProbe], *, sun_azimuth_deg: float, ground_y: float | None = None, max_cameras: int = 6) -> list[CameraPlan]:
- """Overview + per-zone cameras (largest footprints first), eye on the sun side."""
+_FALLBACK_CAMERA = CameraPlan(name="overview", position=(30.0, 18.0, 30.0), look_at=(0.0, 1.0, 0.0), fov=50.0,
+ purpose="fallback: no measurable zones")
+
+
+def _plan_from_spec(spec: dict[str, Any], *, name: str, purpose: str) -> CameraPlan:
+ return CameraPlan(name=name, position=tuple(spec["position"]), look_at=tuple(spec["lookAt"]),
+ fov=float(spec.get("fov", 50.0)), purpose=purpose)
+
+
+def cameras_from_specs(specs: dict[str, Any], probes: dict[str, ZoneProbe], *, max_cameras: int = 6) -> list[CameraPlan]:
+ """Convert driver-fitted camera specs (``probe_scene.mjs --sun-azimuth``;
+ math owned by ``runtime_js/lib/orbit.mjs``) into CameraPlans: overview
+ first, then per-zone cameras, largest footprints first. Falls back to a
+ fixed overview when nothing was measurable."""
healthy = [p for p in probes.values() if p.ok and p.bbox]
- if not healthy:
- return [CameraPlan(name="overview", position=(30.0, 18.0, 30.0), look_at=(0.0, 1.0, 0.0), fov=50.0, purpose="fallback: no measurable zones")]
- union = _union([p.bbox for p in healthy])
- cams = [fit_overview(union, azimuth=sun_azimuth_deg, floor=ground_y)]
+ overview = (specs or {}).get("overview")
+ if not healthy or not overview:
+ return [_FALLBACK_CAMERA.model_copy()]
+ cams = [_plan_from_spec(overview, name="overview", purpose="overview fitted to measured content bounds, sun side")]
+ by_zone = {str(z.get("group", "")).removeprefix("__zone__"): z for z in (specs or {}).get("zones", [])}
by_area = sorted(healthy, key=lambda p: -(p.bbox["size"][0] * p.bbox["size"][2]))
for p in by_area[: max_cameras - 1]:
- cams.append(fit_zone_camera(p.name, p.bbox, azimuth=sun_azimuth_deg, floor=ground_y))
+ spec = by_zone.get(p.name)
+ if spec:
+ cams.append(_plan_from_spec(spec, name=f"{to_snake(p.name)}_view", purpose=f"zone {to_pascal(p.name)} from the sun side"))
return cams
@@ -297,9 +286,11 @@ def assemble(ws: Workspace, plan: ScenePlan | None = None, *, cameras: str = "de
warnings.append("src/env.js missing: scene assembled without env (flat ground at y=0, no lights unless zones add them)")
probes: dict[str, ZoneProbe] = {}
census: dict[str, Any] = {}
+ camera_specs: dict[str, Any] = {}
if probe:
- probes, census, other = probe_zone_modules(ws, timeout_s=timeout_s)
- for o in other:
+ report = probe_zone_modules(ws, timeout_s=timeout_s, sun_azimuth_deg=sun_azimuth(ws))
+ probes, census, camera_specs = report.probes, report.census, report.camera_specs
+ for o in report.other_errors:
if _PREFIX in o and "env failed" in o:
env_ok = False
warnings.append(f"env.js failed in probe; assembled without it: {o[:300]}")
@@ -311,11 +302,10 @@ def assemble(ws: Workspace, plan: ScenePlan | None = None, *, cameras: str = "de
failed = {n: p.error for n, p in probes.items() if not p.ok}
if not healthy:
warnings.append("no zone module survived probing; scene.js assembled with zero zones")
- ground_y = census.get("ground_y") if census else None
if cameras == "plan" and plan is not None and plan.cameras:
cams = list(plan.cameras[:6])
else:
- cams = derive_cameras(probes, sun_azimuth_deg=sun_azimuth(ws), ground_y=ground_y)
+ cams = cameras_from_specs(camera_specs, probes)
text = render_scene_js(healthy, cams, glb_assets(ws), env_ok=env_ok)
out = ws.src / "scene.js"
out.parent.mkdir(parents=True, exist_ok=True)
diff --git a/harness/codeverse/languages/scene_threejs/runtime.py b/harness/codeverse/languages/scene_threejs/runtime.py
index c374fd4f..4fb7e43c 100644
--- a/harness/codeverse/languages/scene_threejs/runtime.py
+++ b/harness/codeverse/languages/scene_threejs/runtime.py
@@ -1,9 +1,10 @@
"""``SceneThreeJsRuntime``: the LanguageRuntime for multi-file three.js scenes.
-build(ws) = ``probe_scene`` (module loads, shape, cameras, update, census) +
-``check_shaders`` (static GLSL audits + GPU compile preflight). A scene has no
-GLB deliverable — ``BuildResult.glb_path`` stays None; the census, probe and
-preflight JSONs are in ``extra_paths``.
+build(ws) = ONE ``probe_scene.mjs --compile`` run (single browser boot) that
+yields both gates: ``scene_probe`` (module loads, shape, cameras, update,
+census) and ``shader_preflight`` (static GLSL audits + GPU compile preflight).
+A scene has no GLB deliverable — ``BuildResult.glb_path`` stays None; the
+census, probe and preflight JSONs are in ``extra_paths``.
"""
from __future__ import annotations
@@ -13,7 +14,7 @@ import time
from pathlib import Path
from codeverse.contracts.artifacts import BuildResult, GateReport, Severity
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import Plan
from codeverse.languages.scene_threejs.lint import lint as _lint
from codeverse.languages.scene_threejs.skeleton import write_skeleton
@@ -25,7 +26,7 @@ _PROMPTS = _HERE.parent.parent / "prompts" / "scene_threejs"
class SceneThreeJsRuntime:
language = Language.SCENE_THREEJS
- entry_globs: tuple[str, ...] = ("src/scene.js", "src/zones/*.js", "src/assets/*.js", "src/env.js", "src/shaders/*.js")
+ entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.SCENE_THREEJS], "src/zones/*.js", "src/assets/*.js", "src/env.js", "src/shaders/*.js")
def skeleton(self, ws: Workspace, plan: Plan | None) -> list[Path]:
return write_skeleton(ws, plan)
@@ -34,16 +35,17 @@ class SceneThreeJsRuntime:
return _lint(ws)
def build(self, ws: Workspace, *, timeout_s: int | None = None) -> BuildResult:
- """Probe + shader preflight; ok iff the module loads and no shader errors."""
+ """Probe + shader preflight (one browser boot); ok iff the module loads
+ and no shader errors. External contract unchanged: the same two
+ GateReports land under ``artifacts/gates/``."""
from codeverse.config import get_settings
- from codeverse.spatial.probes import check_shaders, probe_scene
t0 = time.time()
- tmo = float(timeout_s or get_settings().limits.build_timeout_s)
- probe, census = probe_scene(ws, timeout_s=min(tmo, 120.0))
- shaders = check_shaders(ws, timeout_s=min(tmo, 120.0)) if probe.passed or _module_loaded(probe) else GateReport(
- gate="shader_preflight", passed=False, findings=[])
+ tmo = min(float(timeout_s or get_settings().limits.build_timeout_s), 120.0)
+ probe, shaders, census = _probe_and_preflight(ws, timeout_s=tmo)
ws.artifacts.mkdir(parents=True, exist_ok=True)
+ if census:
+ (ws.artifacts / "census.json").write_text(json.dumps(census, indent=1))
gates_dir = ws.artifacts / "gates"
gates_dir.mkdir(exist_ok=True)
(gates_dir / "scene_probe.json").write_text(probe.model_dump_json(indent=1))
@@ -80,9 +82,42 @@ class SceneThreeJsRuntime:
return _PROMPTS / "cookbook.md"
-def _module_loaded(probe: GateReport) -> bool:
- """Run the shader preflight even when the probe failed for non-boot reasons."""
- return not any(f.data.get("stage") for f in probe.findings if f.severity == Severity.ERROR)
+def _probe_and_preflight(ws: Workspace, *, timeout_s: float) -> tuple[GateReport, GateReport, dict]:
+ """One ``probe_scene.mjs --compile`` run → (scene_probe, shader_preflight, census).
+
+ The driver boots the scene once and runs both stages on the same page;
+ when the scene never boots the preflight is skipped, matching the old
+ two-call behaviour (a failed-empty shader gate).
+ """
+ from codeverse.contracts.artifacts import GateFinding
+ from codeverse.spatial.probes import PROBE_GATE, SHADER_GATE, probe_report, shader_report
+ from codeverse.spatial.render_scene import SceneRenderError, run_scene_script
+
+ t0 = time.time()
+ args = [
+ "--ws", str(ws.root), "--compile",
+ "--out", str(ws.artifacts / "scene_probe.json"),
+ "--shaders-out", str(ws.artifacts / "shader_preflight.json"),
+ "--timeout-ms", str(int(timeout_s * 1000)),
+ ]
+ try:
+ res = run_scene_script("probe_scene.mjs", args, timeout_s=timeout_s + 20)
+ except SceneRenderError as e:
+ dur = int((time.time() - t0) * 1000)
+ finding = GateFinding(gate=PROBE_GATE, severity=Severity.ERROR, target="src/scene.js",
+ message=f"scene probe could not run: {e}"[:1500],
+ fix_hint="this is a harness/driver failure, not your code; retry or report",
+ data={"harness_failure": True})
+ return (GateReport(gate=PROBE_GATE, passed=False, findings=[finding], duration_ms=dur),
+ GateReport(gate=SHADER_GATE, passed=False, findings=[]), {})
+ dur = int((time.time() - t0) * 1000)
+ probe, census = probe_report(res.summary, duration_ms=dur)
+ rep = res.summary.get("shader_report") or {}
+ if not rep or rep.get("skipped"):
+ shaders = GateReport(gate=SHADER_GATE, passed=False, findings=[])
+ else:
+ shaders = shader_report(rep, duration_ms=int(rep.get("duration_ms") or 0))
+ return probe, shaders, census
def _summary_text(probe: GateReport, shaders: GateReport) -> str:
diff --git a/harness/codeverse/languages/threejs/runtime.py b/harness/codeverse/languages/threejs/runtime.py
index 32a698da..d6f8ec23 100644
--- a/harness/codeverse/languages/threejs/runtime.py
+++ b/harness/codeverse/languages/threejs/runtime.py
@@ -9,7 +9,7 @@ from typing import Any
from codeverse.config import get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import Plan
from codeverse.conventions import to_snake
from codeverse.languages.threejs.contract import CONTRACT_FALLBACK
@@ -31,7 +31,7 @@ class ThreeJsRuntime:
"""LanguageRuntime for ``Language.THREEJS`` (raw ESM three.js, exported via node)."""
language = Language.THREEJS
- entry_globs: tuple[str, ...] = ("src/object.js", "src/parts/*.js")
+ entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.THREEJS], "src/parts/*.js")
# ------------------------------------------------------------------ contract
def contract_doc(self) -> str:
diff --git a/harness/codeverse/languages/urdf/runtime.py b/harness/codeverse/languages/urdf/runtime.py
index c687f500..7be701d1 100644
--- a/harness/codeverse/languages/urdf/runtime.py
+++ b/harness/codeverse/languages/urdf/runtime.py
@@ -14,23 +14,21 @@ the gate layer (``sweep_findings``).
from __future__ import annotations
-import contextlib
import json
import os
import shutil
-import signal
-import subprocess
import time
from pathlib import Path
from typing import Any
from codeverse.config import get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import ArticulatedPlan, Plan
from codeverse.languages.urdf.consistency import check_fk_consistency
from codeverse.languages.urdf.lint import lint_workspace
from codeverse.languages.urdf.skeleton import write_skeleton
+from codeverse.proc import run_subprocess
from codeverse.spatial.joints import (
UrdfError,
load_urdf,
@@ -53,7 +51,7 @@ def _tail(s: str, n: int = 3000) -> str:
class UrdfBlenderRuntime:
language = Language.URDF_BLENDER
- entry_globs = ("src/model.py", "src/robot.urdf")
+ entry_globs = (ENTRY_FILE[Language.URDF_BLENDER], "src/robot.urdf")
# ------------------------------------------------------------ protocol
def skeleton(self, ws: Workspace, plan: Plan) -> list[Path]:
@@ -138,7 +136,7 @@ class UrdfBlenderRuntime:
fk_findings = check_fk_consistency(robot, census.get("links") or {}, tol_m=FK_TOL_M)
census["fk_check"] = [f.model_dump(mode="json") for f in fk_findings]
if fk_findings:
- msg = "\n".join(f"- {f.message}\n FIX: {f.fix_hint}" for f in fk_findings)
+ msg = "\n".join(f"- {f.as_line()}" for f in fk_findings)
return fail("FkInconsistent", f"URDF frames do not reproduce the authored geometry:\n{msg}", census=census,
extra_paths=extra)
@@ -167,7 +165,7 @@ class UrdfBlenderRuntime:
res.error_file = "src/model.py"
res.error_message = (f"links interpenetrate by {report.summary.rest_max_penetration_m*1000:.1f} mm at the rest pose "
f"(max {REST_PENETRATION_MAX_M*1000:.0f} mm):\n" +
- "\n".join(f"- {f.message}\n FIX: {f.fix_hint}" for f in worst[:8]))
+ "\n".join(f"- {f.as_line()}" for f in worst[:8]))
return res
@@ -177,14 +175,5 @@ def _run_blender(blender: str, ws: Workspace, art: Path, timeout_s: int, rlimit_
"--out", str(art), "--rlimit-gb", str(rlimit_gb)]
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "HOME": os.environ.get("HOME", str(ws.root)),
"PYTHONDONTWRITEBYTECODE": "1", "BLENDER_USER_CONFIG": str(art / ".blender_config")}
- p = subprocess.Popen(cmd, cwd=ws.root, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
- start_new_session=True)
- timed_out = False
- try:
- out, err = p.communicate(timeout=timeout_s)
- except subprocess.TimeoutExpired:
- timed_out = True
- with contextlib.suppress(ProcessLookupError):
- os.killpg(p.pid, signal.SIGKILL)
- out, err = p.communicate()
- return {"returncode": p.returncode, "stdout": out or "", "stderr": err or "", "timed_out": timed_out}
+ r = run_subprocess(cmd, cwd=ws.root, env=env, timeout_s=timeout_s)
+ return {"returncode": r.returncode, "stdout": r.stdout, "stderr": r.stderr, "timed_out": r.timed_out}
diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.py
index 3bacb831..95fd580d 100644
--- a/harness/codeverse/models/gemini.py
+++ b/harness/codeverse/models/gemini.py
@@ -39,7 +39,7 @@ from codeverse.models.gemini_convert import (
from codeverse.models.keypool import KeyPool, KeyPoolExhausted, Outcome
from codeverse.models.parts import Stopwatch
from codeverse.models.pricing import estimate_cost
-from codeverse.models.retry import backoff_delay
+from codeverse.models.retry import rotate_with_retries
from codeverse.models.schema_utils import JsonParseError, parse_json_lenient
log = logging.getLogger(__name__)
@@ -188,93 +188,32 @@ class GeminiModel:
def generate(self, request: ChatRequest) -> ChatResponse:
contents = to_contents(request.messages)
warnings: list[str] = []
- failed_keys: set[str] = set() # keys that 429'd or looked dead during this call
- dead_keys: set[str] = set()
- last_err: ModelError | None = None
- attempt = 0
- config = self._config(request, warnings)
- while attempt < self.max_attempts:
- attempt += 1
- # never go back to a key that looked dead this call; throttled keys are
- # excluded while an untried one remains, else acquire() waits for a cooldown
- exclude = dead_keys | (failed_keys if len(failed_keys) < len(self.pool) else set())
- try:
- key = self.pool.acquire(exclude=exclude)
- except KeyPoolExhausted as exc:
- raise classify_exception(exc) from exc
- try:
- resp = self._once(key, contents, config, request, warnings)
- self.pool.report(
- key, "ok", tokens=resp.usage.input_tokens + resp.usage.output_tokens
- )
- self._bench(dead_keys)
- return resp
- except Exception as exc: # noqa: BLE001 - classified below
- err = classify_exception(exc)
- last_err = err
- if self._thinking_ok and err.status == 400 and "thinking" in str(err).lower():
- # model rejects ThinkingConfig: retry without it, once, for free
- with self._lock:
- self._thinking_ok = False
- warnings.append(
- f"{self.model} rejected thinking config; retrying without it: {err}"
- )
- config = self._config(request, warnings)
- attempt -= 1
- continue
- outcome = failure_outcome(err)
- if outcome == "dead":
- # key-scoped: move on at once (no budget, no sleep); the key is only
- # benched once another key proves the request itself is fine
- dead_keys.add(key)
- failed_keys.add(key)
- self.pool.report(key, "error")
- if len(dead_keys) < len(self.pool):
- log.warning(
- "gemini %s key …%s looks dead (%s); rotating", self.model, key[-4:], err
- )
- attempt -= 1
- continue
- raise err from exc # every key failed the same way: not the keys' fault
- self.pool.report(
- key, outcome, retry_after_s=_retry_after_s(exc) if outcome == "429" else None
- )
- if outcome == "429":
- failed_keys.add(key)
- if len(failed_keys) < len(self.pool):
- # an untried key remains: rotation is free, only a courtesy pause
- attempt -= 1
- log.warning(
- "gemini %s key …%s throttled (%s); rotating to a fresh key",
- self.model,
- key[-4:],
- err,
- )
- self._sleep(0.5)
- continue
- if not err.retryable or attempt >= self.max_attempts:
- self._bench(dead_keys)
- raise err from exc
- delay = backoff_delay(attempt, base_delay=self.base_delay, max_delay=self.max_delay)
- log.warning(
- "gemini %s attempt %d/%d failed (%s); retrying in %.1fs",
- self.model,
- attempt,
- self.max_attempts,
- err,
- delay,
- )
- self._sleep(delay)
- assert last_err is not None
- self._bench(dead_keys)
- raise last_err
-
- def _bench(self, dead_keys: set[str]) -> None:
- """Mark keys that failed with key-scoped errors dead — called once the call
- got past the auth layer on some other key (success, 429, 5xx, bad JSON …)."""
- for key in dead_keys:
- self.pool.report(key, "dead")
- dead_keys.clear()
+ state = {"config": self._config(request, warnings)}
+
+ def downgrade_thinking(err: ModelError) -> bool:
+ """Free-retry hook: model rejects ThinkingConfig → retry without it, once."""
+ if not (self._thinking_ok and err.status == 400 and "thinking" in str(err).lower()):
+ return False
+ with self._lock:
+ self._thinking_ok = False
+ warnings.append(f"{self.model} rejected thinking config; retrying without it: {err}")
+ state["config"] = self._config(request, warnings)
+ return True
+
+ return rotate_with_retries(
+ self.pool,
+ lambda key: self._once(key, contents, state["config"], request, warnings),
+ classify=classify_exception,
+ outcome_of=failure_outcome,
+ max_attempts=self.max_attempts,
+ base_delay=self.base_delay,
+ max_delay=self.max_delay,
+ sleep=self._sleep,
+ on_free_retry=downgrade_thinking,
+ retry_after=_retry_after_s,
+ tokens_of=lambda r: r.usage.input_tokens + r.usage.output_tokens,
+ label=f"gemini {self.model}",
+ )
def _config(self, request: ChatRequest, warnings: list[str]) -> types.GenerateContentConfig:
return build_config(
diff --git a/harness/codeverse/models/gemini_image.py b/harness/codeverse/models/gemini_image.py
index c580e17f..1af91063 100644
--- a/harness/codeverse/models/gemini_image.py
+++ b/harness/codeverse/models/gemini_image.py
@@ -46,10 +46,10 @@ from codeverse.models.gemini import (
shared_pool,
)
from codeverse.models.gemini_convert import FATAL_FINISH, parse_usage
-from codeverse.models.keypool import KeyPool, KeyPoolExhausted
+from codeverse.models.keypool import KeyPool
from codeverse.models.parts import Stopwatch
from codeverse.models.pricing import estimate_cost
-from codeverse.models.retry import backoff_delay
+from codeverse.models.retry import rotate_with_retries
log = logging.getLogger(__name__)
@@ -237,64 +237,19 @@ class GeminiImageModel:
seed=seed,
http_options=types.HttpOptions(timeout=int(self.timeout_s * 1000)),
)
- failed_keys: set[str] = set() # keys that 429'd or looked dead during this call
- dead_keys: set[str] = set()
- last: ModelError | None = None
- attempt = 0
- while attempt < self.max_attempts:
- attempt += 1
- # never go back to a key that looked dead this call; throttled keys are
- # excluded while an untried one remains, else acquire() waits for a cooldown
- exclude = dead_keys | (failed_keys if len(failed_keys) < len(self.pool) else set())
- try:
- key = self.pool.acquire(exclude=exclude)
- except KeyPoolExhausted as exc:
- raise classify_exception(exc) from exc
- try:
- images, usage = self._call(key, model, contents, config, size)
- self.pool.report(key, "ok", tokens=usage.input_tokens + usage.output_tokens)
- self._bench(dead_keys)
- return images, usage
- except Exception as exc: # noqa: BLE001 — classified below
- err = classify_exception(exc)
- last = err
- outcome = failure_outcome(err)
- if outcome == "dead":
- # key-scoped: move on at once (no budget, no sleep); the key is only
- # benched once another key proves the request itself is fine
- dead_keys.add(key)
- failed_keys.add(key)
- self.pool.report(key, "error")
- if len(dead_keys) < len(self.pool):
- log.warning("image %s key …%s looks dead (%s); rotating", model, key[-4:], err)
- attempt -= 1
- continue
- raise err from exc # every key failed the same way: not the keys' fault
- self.pool.report(key, outcome, retry_after_s=_retry_after_s(exc) if outcome == "429" else None)
- if outcome == "429":
- failed_keys.add(key)
- if len(failed_keys) < len(self.pool):
- # an untried key remains: rotation is free, only a courtesy pause
- attempt -= 1
- log.warning("image %s key …%s throttled (%s); rotating to a fresh key", model, key[-4:], err)
- self._sleep(0.5)
- continue
- if not err.retryable or attempt >= self.max_attempts:
- self._bench(dead_keys)
- raise err from exc
- delay = backoff_delay(attempt, base_delay=self.base_delay, max_delay=self.max_delay)
- log.warning("image %s attempt %d/%d failed (%s); retrying in %.1fs", model, attempt, self.max_attempts, err, delay)
- self._sleep(delay)
- assert last is not None
- self._bench(dead_keys)
- raise last
-
- def _bench(self, dead_keys: set[str]) -> None:
- """Mark keys that failed with key-scoped errors dead — called once the call
- as a whole succeeded or failed for a non-key reason."""
- for key in dead_keys:
- self.pool.report(key, "dead")
- dead_keys.clear()
+ return rotate_with_retries(
+ self.pool,
+ lambda key: self._call(key, model, contents, config, size),
+ classify=classify_exception,
+ outcome_of=failure_outcome,
+ max_attempts=self.max_attempts,
+ base_delay=self.base_delay,
+ max_delay=self.max_delay,
+ sleep=self._sleep,
+ retry_after=_retry_after_s,
+ tokens_of=lambda r: r[1].input_tokens + r[1].output_tokens,
+ label=f"image {model}",
+ )
def _call(
self, key: str, model: str, contents: list[types.Content], config: types.GenerateContentConfig, size: int
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index 07d12535..097f68d0 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -1,15 +1,30 @@
-"""Generic retry helper with exponential backoff + full jitter.
+"""Generic retry helpers: exponential backoff + full jitter, and the shared
+key-rotation state machine used by the Gemini backends.
-Provider modules use it for 429/5xx/timeouts; it is deliberately tiny and has
-no provider knowledge. ``sleep`` is injectable so unit tests run instantly.
+* :func:`with_retries` — plain bounded retry; no provider knowledge.
+* :func:`rotate_with_retries` — the dead-key / 429-rotation / backoff machine
+ over a :class:`~codeverse.models.keypool.KeyPool`. Provider specifics
+ (exception classification, outcome mapping, retry-after extraction) are
+ injected as hooks, so this module stays provider-neutral.
+
+``sleep`` is injectable everywhere so unit tests run instantly.
"""
from __future__ import annotations
+import logging
import random
import time
from collections.abc import Callable
-from typing import TypeVar
+from typing import TYPE_CHECKING, TypeVar
+
+from codeverse.models.keypool import KeyPoolExhausted
+
+if TYPE_CHECKING: # pragma: no cover
+ from codeverse.models.base import ModelError
+ from codeverse.models.keypool import KeyPool, Outcome
+
+log = logging.getLogger(__name__)
T = TypeVar("T")
@@ -63,3 +78,128 @@ def with_retries(
# unreachable, but keeps type-checkers happy
assert last is not None
raise last
+
+
+def rotate_with_retries(
+ pool: KeyPool,
+ call: Callable[[str], T],
+ *,
+ classify: Callable[[BaseException], ModelError],
+ outcome_of: Callable[[ModelError], Outcome],
+ max_attempts: int = 6,
+ base_delay: float = 1.0,
+ max_delay: float = 30.0,
+ storm_attempts: int = 10,
+ storm_max_delay: float = 120.0,
+ sleep: Callable[[float], None] = time.sleep,
+ on_free_retry: Callable[[ModelError], bool] | None = None,
+ retry_after: Callable[[BaseException], float | None] | None = None,
+ tokens_of: Callable[[T], int] | None = None,
+ label: str = "model",
+) -> T:
+ """Run ``call(key)`` against a rotating :class:`KeyPool` until it succeeds.
+
+ The state machine (shared by ``GeminiModel`` / ``GeminiImageModel``):
+
+ * success → ``pool.report(key, "ok", tokens=tokens_of(result))``; keys that
+ looked dead during this call are benched (another key proved the request
+ itself is fine) and the result is returned.
+ * ``on_free_retry(err)`` returning ``True`` → retry at once, for free
+ (no report, no sleep) — e.g. a thinking-config downgrade.
+ * ``outcome_of(err) == "dead"`` (key-scoped auth failure) → rotate at once
+ for free; the key is only benched once a sibling key proves the request
+ is fine. Every key failing the same way raises without benching.
+ * ``"429"`` → cool the key down (honouring ``retry_after``); rotation is
+ FREE while an untried key remains (0.5 s courtesy pause), otherwise the
+ 429 counts against ``max_attempts`` with exponential backoff.
+ * a **capacity storm** (HTTP 503/529 — model-wide, key rotation cannot
+ help) gets its own patience budget: up to ``storm_attempts`` waits with
+ exponential backoff capped at ``storm_max_delay`` (~8 min total by
+ default) that do NOT consume ``max_attempts``. Observed 2026-08-23: a
+ multi-minute gemini-3.7-flash "high demand" outage killed 8 bench runs
+ under the plain 6-attempt budget.
+ * other retryable errors → exponential backoff + jitter until
+ ``max_attempts``; non-retryable errors raise immediately.
+ """
+ failed_keys: set[str] = set() # keys that 429'd or looked dead during this call
+ dead_keys: set[str] = set()
+
+ def bench() -> None:
+ # mark keys that failed with key-scoped errors dead — called once the call
+ # got past the auth layer on some other key (success, 429, 5xx, bad JSON …)
+ for k in dead_keys:
+ pool.report(k, "dead")
+ dead_keys.clear()
+
+ last_err: ModelError | None = None
+ attempt = 0
+ storm = 0
+ while attempt < max_attempts:
+ attempt += 1
+ # never go back to a key that looked dead this call; throttled keys are
+ # excluded while an untried one remains, else acquire() waits for a cooldown
+ exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())
+ try:
+ key = pool.acquire(exclude=exclude)
+ except KeyPoolExhausted as exc:
+ raise classify(exc) from exc
+ try:
+ result = call(key)
+ pool.report(key, "ok", tokens=tokens_of(result) if tokens_of is not None else 0)
+ bench()
+ return result
+ except Exception as exc: # noqa: BLE001 - classification is delegated
+ err = classify(exc)
+ last_err = err
+ if on_free_retry is not None and on_free_retry(err):
+ attempt -= 1
+ continue
+ outcome = outcome_of(err)
+ if outcome == "dead":
+ # key-scoped: move on at once (no budget, no sleep); the key is only
+ # benched once another key proves the request itself is fine
+ dead_keys.add(key)
+ failed_keys.add(key)
+ pool.report(key, "error")
+ if len(dead_keys) < len(pool):
+ log.warning("%s key …%s looks dead (%s); rotating", label, key[-4:], err)
+ attempt -= 1
+ continue
+ raise err from exc # every key failed the same way: not the keys' fault
+ pool.report(
+ key,
+ outcome,
+ retry_after_s=retry_after(exc) if (outcome == "429" and retry_after) else None,
+ )
+ if err.retryable and err.status in (503, 529) and storm < storm_attempts:
+ # capacity storm: model-wide, so waiting (on a rotated key) is the
+ # only cure — paid from its own budget, not max_attempts
+ storm += 1
+ delay = min(storm_max_delay, base_delay * (2 ** min(storm, 8)))
+ delay *= 0.75 + 0.5 * random.random()
+ log.warning("%s capacity storm %d/%d (%s); waiting %.0fs", label, storm, storm_attempts, err, delay)
+ sleep(delay)
+ attempt -= 1
+ continue
+ if outcome == "429":
+ failed_keys.add(key)
+ if len(failed_keys) < len(pool):
+ # an untried key remains: rotation is free, only a courtesy pause
+ attempt -= 1
+ log.warning(
+ "%s key …%s throttled (%s); rotating to a fresh key", label, key[-4:], err
+ )
+ sleep(0.5)
+ continue
+ if not err.retryable or attempt >= max_attempts:
+ bench()
+ raise err from exc
+ delay = backoff_delay(attempt, base_delay=base_delay, max_delay=max_delay)
+ log.warning(
+ "%s attempt %d/%d failed (%s); retrying in %.1fs",
+ label, attempt, max_attempts, err, delay,
+ )
+ sleep(delay)
+ assert last_err is not None
+ bench()
+ raise last_err
diff --git a/harness/codeverse/models/schema_utils.py b/harness/codeverse/models/schema_utils.py
index e52acf5c..cbf549c2 100644
--- a/harness/codeverse/models/schema_utils.py
+++ b/harness/codeverse/models/schema_utils.py
@@ -274,10 +274,28 @@ class JsonParseError(ValueError):
"""The model's text does not contain a parseable JSON value."""
+def _first_balanced(text: str, opener: str = "{", closer: str = "}") -> str | None:
+ """The first depth-balanced ``{...}`` span (naive counting; braces inside JSON
+ strings can fool it, which is why it is only one candidate among several)."""
+ start = text.find(opener)
+ if start < 0:
+ return None
+ depth = 0
+ for i, ch in enumerate(text[start:], start):
+ if ch == opener:
+ depth += 1
+ elif ch == closer:
+ depth -= 1
+ if depth == 0:
+ return text[start : i + 1]
+ return None
+
+
def parse_json_lenient(text: str) -> Any:
"""Parse JSON from model text tolerating code fences, leading prose and
- trailing chatter. Tries: whole text → fenced block → outermost {...} / [...]
- span. Raises ``JsonParseError`` when nothing parses."""
+ trailing chatter. Tries: whole text → fenced block → first balanced {...}
+ → outermost {...} / [...] span. Raises ``JsonParseError`` when nothing
+ parses."""
if text is None:
raise JsonParseError("empty response")
s = text.strip()
@@ -286,6 +304,9 @@ def parse_json_lenient(text: str) -> Any:
candidates: list[str] = [s]
for m in _FENCE_RE.finditer(s):
candidates.append(m.group(1).strip())
+ balanced = _first_balanced(s)
+ if balanced is not None:
+ candidates.append(balanced)
for opener, closer in (("{", "}"), ("[", "]")):
i, j = s.find(opener), s.rfind(closer)
if i != -1 and j > i:
diff --git a/harness/codeverse/orchestrator/fanout.py b/harness/codeverse/orchestrator/fanout.py
index f111d18d..dfe6aac7 100644
--- a/harness/codeverse/orchestrator/fanout.py
+++ b/harness/codeverse/orchestrator/fanout.py
@@ -1,102 +1,7 @@
-"""Bounded parallel fan-out with per-item timing and error capture.
+"""Re-export shim: the fan-out helper moved to ``codeverse.fanout`` so packages
+outside the orchestrator (judges, texturing) can use it without a cross-package
+dependency. Import from ``codeverse.fanout`` in new code."""
-``fan_out`` never raises for a failing item: the result list holds either the
-item's return value or the exception, in input order, so callers decide what a
-partial failure means (a failed asset is skipped; a failed part task is
-retried). Exceptions that are ``BaseException`` but not ``Exception``
-(KeyboardInterrupt, SystemExit) propagate.
-"""
+from codeverse.fanout import FanOutReport, fan_out, split_results
-from __future__ import annotations
-
-import logging
-import time
-from collections.abc import Callable, Iterable, Sequence
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass, field
-from typing import Any, TypeVar
-
-log = logging.getLogger(__name__)
-
-T = TypeVar("T")
-R = TypeVar("R")
-
-
-@dataclass
-class FanOutReport:
- """Timing/outcome per item, for events and logs."""
-
- label: str
- n_items: int
- n_ok: int
- n_failed: int
- durations_s: list[float] = field(default_factory=list)
- total_s: float = 0.0
-
- def as_dict(self) -> dict[str, Any]:
- return {
- "label": self.label, "n_items": self.n_items, "n_ok": self.n_ok,
- "n_failed": self.n_failed, "total_s": round(self.total_s, 2),
- "max_item_s": round(max(self.durations_s), 2) if self.durations_s else 0.0,
- }
-
-
-def fan_out(
- items: Sequence[T] | Iterable[T],
- fn: Callable[[T], R],
- max_workers: int = 4,
- *,
- label: str = "fanout",
- item_name: Callable[[T], str] | None = None,
- on_done: Callable[[int, T, R | Exception, float], None] | None = None,
-) -> list[R | Exception]:
- """Run ``fn`` over ``items`` in a thread pool; return results in input order.
-
- Each result is the return value or the raised ``Exception``. ``on_done``
- (if given) is called from the worker thread with ``(index, item, result,
- seconds)`` — keep it cheap and thread-safe (e.g. an EventLog.emit).
- """
- items = list(items)
- results: list[R | Exception] = [None] * len(items) # type: ignore[list-item]
- if not items:
- return results
- workers = max(1, min(max_workers, len(items)))
- t_all = time.time()
- durations = [0.0] * len(items)
-
- def _run(i: int) -> None:
- item = items[i]
- name = item_name(item) if item_name else str(i)
- t0 = time.time()
- try:
- out: R | Exception = fn(item)
- except Exception as e: # noqa: BLE001 — captured per item by design
- out = e
- log.warning("%s[%s] failed: %s: %s", label, name, type(e).__name__, e)
- dt = time.time() - t0
- durations[i] = dt
- results[i] = out
- log.info("%s[%s] done in %.1fs (%s)", label, name, dt, "error" if isinstance(out, Exception) else "ok")
- if on_done is not None:
- on_done(i, item, out, dt)
-
- if workers == 1:
- for i in range(len(items)):
- _run(i)
- else:
- with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=label) as pool:
- list(pool.map(_run, range(len(items))))
-
- n_failed = sum(1 for r in results if isinstance(r, Exception))
- report = FanOutReport(label, len(items), len(items) - n_failed, n_failed, durations, time.time() - t_all)
- log.info("%s: %d/%d ok in %.1fs", label, report.n_ok, report.n_items, report.total_s)
- return results
-
-
-def split_results(results: Sequence[R | Exception]) -> tuple[list[R], list[Exception]]:
- """Separate successes from failures (order preserved within each list)."""
- ok: list[R] = []
- bad: list[Exception] = []
- for r in results:
- (bad if isinstance(r, Exception) else ok).append(r) # type: ignore[arg-type]
- return ok, bad
+__all__ = ["FanOutReport", "fan_out", "split_results"]
diff --git a/harness/codeverse/orchestrator/rounds.py b/harness/codeverse/orchestrator/rounds.py
index 7ce5777f..ee457ac3 100644
--- a/harness/codeverse/orchestrator/rounds.py
+++ b/harness/codeverse/orchestrator/rounds.py
@@ -22,6 +22,7 @@ from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem, Plan
from codeverse.contracts.run import RoundRecord
from codeverse.conventions import to_snake
+from codeverse.judges.metrics import best_index
log = logging.getLogger(__name__)
@@ -89,31 +90,14 @@ class StopPolicy:
class BestSelector:
"""Best round = highest score, then fewer gate errors, then earlier.
- Delegates to ``codeverse.judges.metrics.best_index`` when that exists so the
- judge package owns the ranking rule; falls back to the local rule.
- """
+ ``codeverse.judges.metrics.best_index`` owns the ranking rule; rounds
+ without any score fall back to the last successfully built one."""
def pick(self, rounds: Sequence[RoundRecord]) -> int | None:
- if not rounds:
- return None
- try:
- from codeverse.judges.metrics import best_index # type: ignore[import-not-found]
- except Exception: # noqa: BLE001 — optional dependency during bootstrap
- best_index = None
- scored = [(i, r) for i, r in enumerate(rounds) if r.score is not None]
- if best_index is not None and scored:
- try:
- k = best_index([(float(r.score), gate_error_count(r)) for _, r in scored]) # type: ignore[arg-type]
- return scored[int(k)][0]
- except Exception: # noqa: BLE001 — never let a ranking helper kill a run
- log.warning("judges.metrics.best_index failed; using local rule", exc_info=True)
- return self._local_pick(rounds)
-
- @staticmethod
- def _local_pick(rounds: Sequence[RoundRecord]) -> int | None:
scored = [(i, r) for i, r in enumerate(rounds) if r.score is not None]
if scored:
- return max(scored, key=lambda ir: (ir[1].score, -gate_error_count(ir[1]), -ir[0]))[0]
+ k = best_index([(float(r.score), gate_error_count(r)) for _, r in scored]) # type: ignore[arg-type]
+ return scored[int(k)][0]
built = [i for i, r in enumerate(rounds) if r.build is not None and r.build.ok]
return built[-1] if built else None
@@ -190,8 +174,7 @@ def build_refine_instructions(
for g in gates:
for f in g.errors:
- text = f.message if not f.fix_hint else f"{f.message} FIX: {f.fix_hint}"
- _add(RefineTask(target=f.target or "overall", kind=f"gate:{g.gate}", instruction=text,
+ _add(RefineTask(target=f.target or "overall", kind=f"gate:{g.gate}", instruction=f.as_line(),
priority=0, source="gate"))
for t in extra:
_add(t.model_copy())
@@ -249,6 +232,25 @@ def plan_parallel_groups(tasks: Sequence[RefineTask]) -> list[TaskGroup]:
return out
+def plan_refine_groups(
+ tasks: Sequence[RefineTask],
+ *,
+ allow_fanout: bool = True,
+ parallel_min_tasks: int = 2,
+) -> tuple[list[TaskGroup], bool]:
+ """The ONE grouping rule of the refine scaffold (``BaseTrack.refine_tasks``).
+
+ Groups by file ownership via :func:`plan_parallel_groups`; the groups run in
+ parallel only when fan-out is allowed for the track, at least
+ ``parallel_min_tasks`` file-disjoint groups exist and every group owns
+ files. Otherwise everything collapses into ONE whole-artifact group."""
+ groups = plan_parallel_groups(tasks)
+ parallel = allow_fanout and len(groups) >= parallel_min_tasks and all(g.files for g in groups)
+ if not parallel:
+ groups = [TaskGroup(tasks=list(tasks), files=sorted({f for t in tasks for f in t.files}))]
+ return groups, parallel
+
+
def compact_instructions(tasks: Sequence[RefineTask], max_lines: int = 6) -> list[str]:
"""The instruction lines handed to ONE generation task: grouped by target
(instances ``Leg_0``/``Leg_1`` fold into ``Leg``), ordered by priority, capped
diff --git a/harness/codeverse/spatial/connectivity.py b/harness/codeverse/spatial/connectivity.py
index 63d25d89..554a157a 100644
--- a/harness/codeverse/spatial/connectivity.py
+++ b/harness/codeverse/spatial/connectivity.py
@@ -2,10 +2,12 @@
Mechanism
---------
-1. Load one world-space mesh per top-level node (``measure.part_meshes``).
+1. Load one world-space mesh per top-level node (``measure.cached_parts``,
+ memoized on the file's stat so repeat gates on one GLB parse it once).
2. For every part pair whose AABBs come within ``gap_m`` of each other compute
- the exact minimum surface distance (python-fcl via ``trimesh.collision``;
- sampled closest-point fallback when fcl is missing) and the closest points.
+ the exact minimum surface distance (``fcl.distance`` on per-part BVHs that
+ are built once and shared across all pairs; sampled closest-point fallback
+ when fcl is missing) and the closest points.
3. Contact graph: edge when distance ≤ ``gap_m``. The *support component* is
the connected component holding the ground-touching parts (lowest point at
y ≤ gap) — or the largest component when nothing touches the ground.
@@ -29,6 +31,7 @@ import itertools
import time
from dataclasses import dataclass
from pathlib import Path
+from typing import Any
import numpy as np
import trimesh
@@ -36,7 +39,8 @@ import trimesh
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
from codeverse.conventions import CONTACT_GAP_M, Frame
from codeverse.spatial.contract import frame_label, glb_vec_to_plan, language_frame
-from codeverse.spatial.measure import GlbLoadError, load_scene, part_meshes
+from codeverse.spatial.joints_collide import fcl_collision_object
+from codeverse.spatial.measure import GlbLoadError, cached_parts
GATE = "connectivity"
#: parts smaller than this (max extent) are ignored for floating checks (INFO only)
@@ -67,6 +71,19 @@ class PairDistance:
return tuple(float(pb - pa) for pa, pb in zip(self.point_a, self.point_b, strict=True)) # type: ignore[return-value]
+class _FclObjects:
+ """Lazily-memoized ``fcl.CollisionObject`` per part (None entries when fcl is missing)."""
+
+ def __init__(self, parts: dict[str, trimesh.Trimesh]):
+ self._parts = parts
+ self._objs: dict[str, Any] = {}
+
+ def get(self, name: str) -> Any:
+ if name not in self._objs:
+ self._objs[name] = fcl_collision_object(self._parts[name])
+ return self._objs[name]
+
+
def _aabb_gap(a: trimesh.Trimesh, b: trimesh.Trimesh) -> float:
"""Axis-aligned lower bound of the distance between two meshes."""
lo = np.maximum(a.bounds[0] - b.bounds[1], b.bounds[0] - a.bounds[1])
@@ -82,16 +99,31 @@ def _fcl_available() -> bool:
return False
-def pair_distance(name_a: str, a: trimesh.Trimesh, name_b: str, b: trimesh.Trimesh) -> PairDistance:
- """Exact (fcl) or sampled minimum distance between two meshes with closest points."""
+def pair_distance(
+ name_a: str,
+ a: trimesh.Trimesh,
+ name_b: str,
+ b: trimesh.Trimesh,
+ *,
+ obj_a: Any = None,
+ obj_b: Any = None,
+) -> PairDistance:
+ """Exact (fcl) or sampled minimum distance between two meshes with closest points.
+
+ ``obj_a``/``obj_b`` accept prebuilt ``fcl.CollisionObject``\\ s (see
+ :func:`codeverse.spatial.joints_collide.fcl_collision_object`) so a caller
+ testing many pairs builds each part's BVH once instead of once per pair.
+ """
if _fcl_available():
try:
- from trimesh.collision import CollisionManager
-
- cm = CollisionManager()
- cm.add_object("a", a)
- d, _name, data = cm.min_distance_single(b, return_name=True, return_data=True)
- pa, pb = data.point("a"), data.point("__external")
+ import fcl
+
+ oa = obj_a if obj_a is not None else fcl_collision_object(a)
+ ob = obj_b if obj_b is not None else fcl_collision_object(b)
+ req = fcl.DistanceRequest(enable_nearest_points=True)
+ res = fcl.DistanceResult()
+ d = fcl.distance(oa, ob, req, res)
+ pa, pb = res.nearest_points
return PairDistance(name_a, name_b, float(max(d, 0.0)), tuple(map(float, pa)), tuple(map(float, pb)))
except Exception:
pass # fall through to the sampled estimate
@@ -192,7 +224,7 @@ def check_connectivity(
up = "z" if language_frame(language) is Frame.Z_UP_NEG_Y_FRONT else "y"
findings: list[GateFinding] = []
try:
- parts = {k: v for k, v in part_meshes(load_scene(glb)).items() if v is not None and len(v.faces)}
+ parts = {k: v for k, v in cached_parts(glb).items() if v is not None and len(v.faces)}
except GlbLoadError as e:
findings.append(GateFinding(gate=GATE, severity=Severity.ERROR, message=str(e), fix_hint="run `build` first"))
return GateReport(gate=GATE, passed=False, findings=findings, duration_ms=int((time.time() - t0) * 1000))
@@ -211,11 +243,12 @@ def check_connectivity(
# ---- pairwise distances (AABB prefilter) + penetration
edges: set[tuple[str, str]] = set()
exact: dict[tuple[str, str], PairDistance] = {}
+ objs = _FclObjects(parts) # one BVH per part, built lazily, shared across pairs
for a, b in itertools.combinations(big, 2):
lower = _aabb_gap(parts[a], parts[b])
if lower > gap_m:
continue
- pd = pair_distance(a, parts[a], b, parts[b])
+ pd = pair_distance(a, parts[a], b, parts[b], obj_a=objs.get(a), obj_b=objs.get(b))
exact[(a, b)] = pd
if pd.distance <= gap_m:
edges.add((a, b))
@@ -244,7 +277,7 @@ def check_connectivity(
for n in big:
if n in support:
continue
- near = _nearest_supported(n, parts, support, exact)
+ near = _nearest_supported(n, parts, support, exact, objs)
if near is None:
findings.append(GateFinding(gate=GATE, severity=Severity.ERROR, target=n,
message=f"part '{n}' is floating: touches nothing", fix_hint=f"attach '{n}' to a neighbouring part"))
@@ -277,11 +310,13 @@ def _nearest_supported(
parts: dict[str, trimesh.Trimesh],
support: set[str],
exact: dict[tuple[str, str], PairDistance],
+ objs: _FclObjects,
n_candidates: int = 3,
) -> PairDistance | None:
"""Exact distance from floating part ``n`` to its nearest supported part.
- Candidates are ranked by AABB lower bound; the closest few get an exact test.
+ Candidates are ranked by AABB lower bound; the closest few get an exact test
+ (reusing the per-part BVHs in ``objs``).
"""
if not support:
return None
@@ -290,7 +325,7 @@ def _nearest_supported(
for o in ranked:
pd = exact.get((n, o)) or exact.get((o, n))
if pd is None:
- pd = pair_distance(n, parts[n], o, parts[o])
+ pd = pair_distance(n, parts[n], o, parts[o], obj_a=objs.get(n), obj_b=objs.get(o))
exact[(n, o)] = pd
if best is None or pd.distance < best.distance:
best = pd
diff --git a/harness/codeverse/spatial/gl_render.py b/harness/codeverse/spatial/gl_render.py
index 8b4db1a2..dcca9c01 100644
--- a/harness/codeverse/spatial/gl_render.py
+++ b/harness/codeverse/spatial/gl_render.py
@@ -29,7 +29,7 @@ from typing import Any
from pydantic import BaseModel, Field
-from codeverse.languages._common import run_subprocess
+from codeverse.proc import run_subprocess
RUNNER = Path(__file__).resolve().parent.parent / "languages" / "opengl_python" / "wrappers" / "run_gl.py"
@@ -193,7 +193,7 @@ def write_contact_sheet(frames: Sequence[GlFrame], out: Path, *, cols: int = 3,
from PIL import Image, ImageDraw
- from codeverse.spatial.sheet import LABEL_BG, LABEL_FG, LABEL_H, PAD, _font
+ from codeverse.spatial.sheet import BG, LABEL_BG, LABEL_FG, LABEL_H, PAD, load_font
frames = list(frames)
if not frames:
@@ -203,9 +203,9 @@ def write_contact_sheet(frames: Sequence[GlFrame], out: Path, *, cols: int = 3,
cols = max(1, min(cols, len(frames)))
rows = math.ceil(len(frames) / cols)
cell_h = tile_h + LABEL_H
- sheet = Image.new("RGB", (cols * (tile_w + PAD) + PAD, rows * (cell_h + PAD) + PAD), (250, 250, 250))
+ sheet = Image.new("RGB", (cols * (tile_w + PAD) + PAD, rows * (cell_h + PAD) + PAD), BG)
draw = ImageDraw.Draw(sheet)
- font = _font(15)
+ font = load_font(15)
for i, f in enumerate(frames):
x = PAD + (i % cols) * (tile_w + PAD)
y = PAD + (i // cols) * (cell_h + PAD)
diff --git a/harness/codeverse/spatial/joints_collide.py b/harness/codeverse/spatial/joints_collide.py
index 009dde53..d7108855 100644
--- a/harness/codeverse/spatial/joints_collide.py
+++ b/harness/codeverse/spatial/joints_collide.py
@@ -34,6 +34,26 @@ def backend_name() -> str:
return "fcl" if _fcl is not None else "trimesh"
+def fcl_collision_object(mesh: trimesh.Trimesh):
+ """``fcl.CollisionObject`` for ``mesh`` (identity transform; None when
+ python-fcl is unavailable). Convex meshes get the cheap ``fcl.Convex``
+ (same choice trimesh's CollisionManager makes — building a BVH for a
+ 300k-tri convex part costs ~0.7 s); everything else gets a BVH. Shared by
+ :class:`LinkBody` and the connectivity gate so each mesh's collision
+ geometry is built exactly once."""
+ if _fcl is None:
+ return None
+ if mesh.is_convex:
+ fs = np.concatenate((3 * np.ones((len(mesh.faces), 1), dtype=np.int64), mesh.faces), axis=1)
+ return _fcl.CollisionObject(_fcl.Convex(np.asarray(mesh.vertices, dtype=float), len(fs), fs.flatten()),
+ _fcl.Transform())
+ model = _fcl.BVHModel()
+ model.beginModel(len(mesh.vertices), len(mesh.faces))
+ model.addSubModel(np.asarray(mesh.vertices, dtype=float), np.asarray(mesh.faces, dtype=np.int64))
+ model.endModel()
+ return _fcl.CollisionObject(model, _fcl.Transform())
+
+
def _sample_points(mesh: trimesh.Trimesh) -> np.ndarray:
pts = np.asarray(mesh.vertices)
if len(pts) > _MAX_SAMPLE_POINTS:
@@ -77,13 +97,7 @@ class LinkBody:
self.watertight = all(bool(i.is_watertight) for i in self.islands)
self._island_pq = [trimesh.proximity.ProximityQuery(i) for i in self.islands]
self._pq = trimesh.proximity.ProximityQuery(mesh) # unsigned distances, LOCAL frame
- self.obj = None
- if _fcl is not None:
- model = _fcl.BVHModel()
- model.beginModel(len(mesh.vertices), len(mesh.faces))
- model.addSubModel(np.asarray(mesh.vertices, dtype=float), np.asarray(mesh.faces, dtype=np.int64))
- model.endModel()
- self.obj = _fcl.CollisionObject(model, _fcl.Transform())
+ self.obj = fcl_collision_object(mesh)
self.T = np.eye(4)
self.T_inv = np.eye(4)
diff --git a/harness/codeverse/spatial/joints_export.py b/harness/codeverse/spatial/joints_export.py
index 9b17d74f..0f100129 100644
--- a/harness/codeverse/spatial/joints_export.py
+++ b/harness/codeverse/spatial/joints_export.py
@@ -12,8 +12,6 @@ Non-root link nodes carry ``extras.joint`` and the scene carries
from __future__ import annotations
import json
-import math
-import subprocess
import sys
import time
from collections.abc import Callable, Sequence
@@ -25,8 +23,11 @@ import trimesh
from codeverse.contracts.artifacts import RenderSet, RenderView
from codeverse.conventions import OBJECT_VIEWS_QUICK, ViewPreset
+from codeverse.proc import run_subprocess
from codeverse.spatial.joints_model import Joint, Robot, UrdfError, fk
from codeverse.spatial.joints_poses import limit_poses
+from codeverse.spatial.render import render_glb
+from codeverse.spatial.sheet import contact_sheet
ARTICULATION_SHEET_NAME = "articulation_sheet.png"
#: URDF (Z-up, -Y front) → glTF (Y-up, +Z front): (x, y, z) → (x, z, -y)
@@ -101,20 +102,12 @@ def urdf_to_glb(robot: Robot, out_glb: Path | str, pose: dict[str, float] | None
# ------------------------------------------------------------------ rendering
-def _default_renderer() -> Renderer:
- """``codeverse.spatial.render.render_glb`` when available, else the Blender fallback."""
- try:
- from codeverse.spatial.render import render_glb # package C1
-
- return render_glb
- except Exception:
- return blender_render_glb
-
-
def blender_render_glb(glb: Path, out_dir: Path, *, views: Sequence[ViewPreset] | None = None, width: int = 512,
height: int = 512, sheet: bool = False, timeout_s: int = 180, **_: Any) -> RenderSet:
- """Minimal headless-Blender (Workbench) renderer used when the GPU renderer
- (package C1) is not importable. Same signature subset as ``render_glb``."""
+ """Minimal headless-Blender (Workbench) renderer with the same signature
+ subset as ``render_glb``. Only used when a caller *explicitly* passes
+ ``renderer=blender_render_glb`` (e.g. to render without node/Chrome) — it is
+ never auto-selected."""
from codeverse.config import get_settings
blender = get_settings().resolve_blender()
@@ -130,7 +123,9 @@ def blender_render_glb(glb: Path, out_dir: Path, *, views: Sequence[ViewPreset]
spec_path.write_text(json.dumps(spec))
t0 = time.time()
cmd = [blender, "-b", "--factory-startup", "--python", str(script), "--", str(spec_path)]
- proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout_s, start_new_session=True)
+ proc = run_subprocess(cmd, cwd=out_dir, timeout_s=timeout_s)
+ if proc.timed_out:
+ raise RuntimeError(f"blender fallback render timed out after {timeout_s}s: {proc.stderr[-2000:]}")
if proc.returncode != 0:
raise RuntimeError(f"blender fallback render failed: {proc.stderr[-2000:] or proc.stdout[-2000:]}")
rs = RenderSet(renderer="blender-workbench", duration_ms=int((time.time() - t0) * 1000))
@@ -140,33 +135,10 @@ def blender_render_glb(glb: Path, out_dir: Path, *, views: Sequence[ViewPreset]
raise RuntimeError(f"blender fallback render produced no {p.name}; stdout tail: {proc.stdout[-800:]}")
rs.views.append(RenderView(name=v.name, path=str(p), width=width, height=height))
if sheet:
- rs.contact_sheet = str(make_sheet([(v.name, Path(v.path)) for v in rs.views], out_dir / "sheet.png"))
+ rs.contact_sheet = str(contact_sheet([(v.name, Path(v.path)) for v in rs.views], out_dir / "sheet.png"))
return rs
-def make_sheet(images: list[tuple[str, Path]], out: Path, *, cols: int = 4, tile: int = 384) -> Path:
- """Labelled grid of PNGs (PIL only; stand-in for ``codeverse.spatial.sheet.contact_sheet``)."""
- from PIL import Image, ImageDraw
-
- if not images:
- raise ValueError("make_sheet: no images")
- cols = max(1, min(cols, len(images)))
- rows = math.ceil(len(images) / cols)
- label_h = 22
- sheet = Image.new("RGB", (cols * tile, rows * (tile + label_h)), "white")
- draw = ImageDraw.Draw(sheet)
- for i, (label, path) in enumerate(images):
- im = Image.open(path).convert("RGB")
- im.thumbnail((tile, tile))
- x, y = (i % cols) * tile, (i // cols) * (tile + label_h)
- sheet.paste(im, (x + (tile - im.width) // 2, y + label_h + (tile - im.height) // 2))
- draw.rectangle([x, y, x + tile - 1, y + label_h - 1], fill=(30, 30, 30))
- draw.text((x + 6, y + 4), label[:60], fill="white")
- out.parent.mkdir(parents=True, exist_ok=True)
- sheet.save(out)
- return out
-
-
def render_poses(
robot: Robot,
out_dir: Path | str,
@@ -185,7 +157,7 @@ def render_poses(
out_dir.mkdir(parents=True, exist_ok=True)
poses = poses if poses is not None else limit_poses(robot)
views = tuple(views or OBJECT_VIEWS_QUICK[:3])
- render = renderer or _default_renderer()
+ render = renderer or render_glb
results: list[tuple[str, RenderSet]] = []
tiles: list[tuple[str, Path]] = []
for label, q in poses:
@@ -196,7 +168,7 @@ def render_poses(
for v in rs.views:
if sheet_view is None or v.name == sheet_view:
tiles.append((f"{label} · {v.name}", Path(v.path)))
- make_sheet(tiles, out_dir / ARTICULATION_SHEET_NAME, cols=len(views) if sheet_view is None else 4)
+ contact_sheet(tiles, out_dir / ARTICULATION_SHEET_NAME, cols=len(views) if sheet_view is None else 4)
return results
diff --git a/harness/codeverse/spatial/measure.py b/harness/codeverse/spatial/measure.py
index 40a38352..f3e9f51c 100644
--- a/harness/codeverse/spatial/measure.py
+++ b/harness/codeverse/spatial/measure.py
@@ -7,12 +7,17 @@ rows, islands, ground gap, footprint offset, materials). Degenerate input
``part_meshes`` is the shared loader other spatial modules use: it returns one
world-space ``trimesh.Trimesh`` per top-level node (all child meshes merged).
+``cached_parts`` is the memoized front door for gates that re-read the same
+GLB (keyed on path + size + mtime_ns; ``load_scene`` itself is never cached —
+texturing mutates scenes).
"""
from __future__ import annotations
import re
+import threading
from collections import OrderedDict
+from dataclasses import dataclass
from pathlib import Path
from typing import Any
@@ -24,6 +29,7 @@ from codeverse.contracts.artifacts import Measurement, PartMeasure
__all__ = [
"load_scene",
"part_meshes",
+ "cached_parts",
"measure_glb",
"measure_summary_table",
"instance_groups",
@@ -149,6 +155,59 @@ def part_meshes(scene: trimesh.Scene) -> OrderedDict[str, trimesh.Trimesh | None
return out
+# --------------------------------------------------------------------- parse memo
+#: LRU entries: parsed parts + finished Measurement for one (path, size, mtime_ns)
+_LRU_MAX = 4
+_lru_lock = threading.Lock()
+
+
+@dataclass
+class _CacheEntry:
+ parts: OrderedDict[str, trimesh.Trimesh | None] | None = None
+ measurement: Measurement | None = None
+
+
+_lru: OrderedDict[tuple[str, int, int], _CacheEntry] = OrderedDict()
+
+
+def _stat_key(p: Path) -> tuple[str, int, int] | None:
+ """(resolved path, size, mtime_ns) — None when the file cannot be stat'ed."""
+ try:
+ st = p.stat()
+ except OSError:
+ return None
+ return (str(p.resolve()), st.st_size, st.st_mtime_ns)
+
+
+def _cache_entry(key: tuple[str, int, int]) -> _CacheEntry:
+ """The (possibly fresh) LRU entry for ``key``; evicts the oldest past ``_LRU_MAX``."""
+ with _lru_lock:
+ e = _lru.get(key)
+ if e is None:
+ e = _lru[key] = _CacheEntry()
+ _lru.move_to_end(key)
+ while len(_lru) > _LRU_MAX:
+ _lru.popitem(last=False)
+ return e
+
+
+def cached_parts(glb: Path | str) -> OrderedDict[str, trimesh.Trimesh | None]:
+ """``part_meshes(load_scene(glb))`` memoized on the file's stat (LRU of 4).
+
+ Rewriting the file (new size or mtime_ns) invalidates the entry. The
+ returned dict is a fresh copy per call, but the meshes are SHARED between
+ callers — treat them as read-only (copy before transforming).
+ """
+ p = Path(glb)
+ key = _stat_key(p)
+ if key is None:
+ return part_meshes(load_scene(p)) # raises GlbLoadError for a missing file
+ entry = _cache_entry(key)
+ if entry.parts is None:
+ entry.parts = part_meshes(load_scene(p))
+ return OrderedDict(entry.parts)
+
+
def merged_mesh(parts: dict[str, trimesh.Trimesh | None]) -> trimesh.Trimesh | None:
"""All part meshes concatenated (None if nothing has faces)."""
meshes = [m for m in parts.values() if m is not None]
@@ -194,10 +253,25 @@ def measure_glb(glb: Path | str) -> Measurement:
Raises :class:`GlbLoadError` only when the file is unreadable; an empty or
degenerate model yields a zero-extent ``Measurement`` with findings in
- ``extra["findings"]``.
+ ``extra["findings"]``. Memoized on the file's stat (same LRU as
+ :func:`cached_parts`); every call returns its own deep copy.
"""
- scene = load_scene(glb)
- parts = part_meshes(scene)
+ p = Path(glb)
+ key = _stat_key(p)
+ entry = _cache_entry(key) if key is not None else None
+ if entry is not None and entry.measurement is not None:
+ return entry.measurement.model_copy(deep=True)
+ scene = load_scene(p)
+ parts = entry.parts if entry is not None and entry.parts is not None else part_meshes(scene)
+ m = _measure(scene, parts)
+ if entry is not None:
+ if entry.parts is None:
+ entry.parts = parts
+ entry.measurement = m
+ return m.model_copy(deep=True)
+
+
+def _measure(scene: trimesh.Scene, parts: OrderedDict[str, trimesh.Trimesh | None]) -> Measurement:
findings: list[str] = []
rows: list[PartMeasure] = []
for name, mesh in parts.items():
diff --git a/harness/codeverse/spatial/node.py b/harness/codeverse/spatial/node.py
index d9e5fbc1..5bfc85ad 100644
--- a/harness/codeverse/spatial/node.py
+++ b/harness/codeverse/spatial/node.py
@@ -14,14 +14,12 @@ import contextlib
import json
import os
import resource
-import signal
-import subprocess
-import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from codeverse.config import get_settings
+from codeverse.proc import run_subprocess
TAIL_CHARS = 4000
@@ -136,39 +134,22 @@ def run_node(
if env_extra:
env.update(env_extra)
- t0 = time.time()
try:
- proc = subprocess.Popen(
- cmd,
- cwd=str(cwd) if cwd else None,
- env=env,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- start_new_session=True,
- preexec_fn=_limit_memory(mem_limit_gb),
- )
+ proc = run_subprocess(cmd, cwd=cwd or Path.cwd(), timeout_s=timeout_s, env=env,
+ preexec_fn=_limit_memory(mem_limit_gb))
except FileNotFoundError as e:
raise NodeError(f"node binary not found ({node_bin}): {e}") from e
- timed_out = False
- try:
- out, err = proc.communicate(timeout=timeout_s)
- except subprocess.TimeoutExpired:
- timed_out = True
- _kill_group(proc)
- out, err = proc.communicate()
- duration_ms = int((time.time() - t0) * 1000)
result = NodeResult(
- rc=proc.returncode if not timed_out else -9,
- stdout=out or "",
- stderr=err or "",
- last_json=parse_last_json(out or ""),
- duration_ms=duration_ms,
- timed_out=timed_out,
+ rc=proc.returncode if not proc.timed_out else -9,
+ stdout=proc.stdout,
+ stderr=proc.stderr,
+ last_json=parse_last_json(proc.stdout),
+ duration_ms=proc.duration_ms,
+ timed_out=proc.timed_out,
cmd=cmd,
)
- if timed_out:
+ if result.timed_out:
raise NodeError(f"node script {script.name} timed out after {timeout_s:.0f}s\n{result.stderr_tail}", result)
if check and result.rc != 0:
msg = f"node script {script.name} exited {result.rc}"
@@ -177,12 +158,3 @@ def run_node(
msg += ": " + (e.get("message", "") if isinstance(e, dict) else str(e))
raise NodeError(msg + "\n" + result.stderr_tail, result)
return result
-
-
-def _kill_group(proc: subprocess.Popen) -> None:
- try:
- os.killpg(proc.pid, signal.SIGKILL)
- except ProcessLookupError:
- pass
- except PermissionError:
- proc.kill()
diff --git a/harness/codeverse/spatial/probes.py b/harness/codeverse/spatial/probes.py
index 2b02baf8..a30f474f 100644
--- a/harness/codeverse/spatial/probes.py
+++ b/harness/codeverse/spatial/probes.py
@@ -87,7 +87,20 @@ def probe_scene(ws: Workspace, *, timeout_s: float = 60.0, write_census: bool =
hint="this is a harness/driver failure, not your code; retry or report", harness_failure=True))
return _result(GateReport(gate=gate, passed=False, findings=findings, duration_ms=int((time.time() - t0) * 1000)), census,
driver_failure=f"scene probe could not run: {e}"[:800])
- s = res.summary
+ report, census = probe_report(res.summary, duration_ms=int((time.time() - t0) * 1000))
+ if write_census and census:
+ ws.artifacts.mkdir(parents=True, exist_ok=True)
+ (ws.artifacts / "census.json").write_text(json.dumps(census, indent=1))
+ return _result(report, census)
+
+
+def probe_report(summary: dict[str, Any], *, duration_ms: int = 0) -> tuple[GateReport, dict[str, Any]]:
+ """Interpret a ``probe_scene.mjs`` summary into the ``scene_probe`` GateReport
+ (+ census). Pure over the driver JSON, so the combined single-boot build
+ (``probe_scene.mjs --compile``, scene_threejs runtime) reuses it."""
+ gate = PROBE_GATE
+ findings: list[GateFinding] = []
+ s = summary
boot = s.get("boot") or {}
if not boot.get("ok"):
stage = boot.get("stage", "?")
@@ -132,11 +145,8 @@ def probe_scene(ws: Workspace, *, timeout_s: float = 60.0, write_census: bool =
if cam0 and cam0.get("dark_frac", 0) > 0.85:
findings.append(_f(gate, Severity.WARN, f"camera '{cam0.get('name')}' frame is {cam0['dark_frac']:.0%} black — no lights or nothing in view?",
target=str(cam0.get("name")), hint="add a DirectionalLight + HemisphereLight in env, aim the camera at the content"))
- if write_census and census:
- ws.artifacts.mkdir(parents=True, exist_ok=True)
- (ws.artifacts / "census.json").write_text(json.dumps(census, indent=1))
passed = not any(f.severity == Severity.ERROR for f in findings)
- return _result(GateReport(gate=gate, passed=passed, findings=findings, duration_ms=int((time.time() - t0) * 1000)), census)
+ return GateReport(gate=gate, passed=passed, findings=findings, duration_ms=duration_ms), census
def _census_findings(c: dict[str, Any]) -> list[GateFinding]:
@@ -179,7 +189,16 @@ def check_shaders(ws: Workspace, *, module: str | None = None, timeout_s: float
findings.append(_f(gate, Severity.ERROR, f"shader preflight could not run: {e}", target="src/scene.js",
hint="harness/driver failure; retry or report", harness_failure=True))
return GateReport(gate=gate, passed=False, findings=findings, duration_ms=int((time.time() - t0) * 1000))
- rep = res.summary
+ return shader_report(res.summary, duration_ms=int((time.time() - t0) * 1000))
+
+
+def shader_report(report: dict[str, Any], *, duration_ms: int = 0) -> GateReport:
+ """Interpret a shader-preflight report (``check_shaders.mjs`` summary, or the
+ ``shader_report`` block of ``probe_scene.mjs --compile``) into the
+ ``shader_preflight`` GateReport. Pure over the driver JSON."""
+ gate = SHADER_GATE
+ findings: list[GateFinding] = []
+ rep = report
for e in rep.get("errors", []):
target = f"{e.get('file', '?')}:{e['line']}" if e.get("line") else str(e.get("file", "?"))
findings.append(_f(gate, Severity.ERROR, e.get("message", "shader error"), target=target,
@@ -193,7 +212,7 @@ def check_shaders(ws: Workspace, *, module: str | None = None, timeout_s: float
findings.append(_f(gate, Severity.INFO, f"compiled {comp.get('programs')} programs ({comp.get('custom_materials')} custom) in {comp.get('ms')} ms",
target="scene", **comp))
passed = bool(rep.get("ok")) and not any(f.severity == Severity.ERROR for f in findings)
- return GateReport(gate=gate, passed=passed, findings=findings, duration_ms=int((time.time() - t0) * 1000))
+ return GateReport(gate=gate, passed=passed, findings=findings, duration_ms=duration_ms)
def shader_presence(
diff --git a/harness/codeverse/spatial/render.py b/harness/codeverse/spatial/render.py
index 35247a82..31d7f794 100644
--- a/harness/codeverse/spatial/render.py
+++ b/harness/codeverse/spatial/render.py
@@ -24,6 +24,7 @@ from codeverse.contracts.artifacts import RenderSet, RenderView
from codeverse.contracts.plan import CameraPlan
from codeverse.conventions import OBJECT_VIEWS, ViewPreset
from codeverse.spatial.node import NodeError, run_node, runtime_js_dir
+from codeverse.spatial.render_scene import render_scene as _render_scene_impl
from codeverse.spatial.sheet import contact_sheet
from codeverse.workspace import Workspace
@@ -244,15 +245,8 @@ def render_scene(
height: int = 576,
sheet: bool = True,
) -> RenderSet:
- """Scene renders: delegates to ``codeverse.spatial.render_scene`` (package C2)."""
- try:
- from codeverse.spatial.render_scene import render_scene as _impl
- except ImportError as e: # pragma: no cover - until C2 lands
- raise NotImplementedError(
- "render_scene is implemented by codeverse.spatial.render_scene (runtime_js/render_scene.mjs); "
- f"module not importable: {e}"
- ) from e
- return _impl(ws, Path(out_dir), cameras=cameras, orbit=orbit, times=times, width=width, height=height, sheet=sheet)
+ """Scene renders: delegates to ``codeverse.spatial.render_scene`` (narrower kwarg surface)."""
+ return _render_scene_impl(ws, Path(out_dir), cameras=cameras, orbit=orbit, times=times, width=width, height=height, sheet=sheet)
__all__ = ["render_glb", "render_turntable", "render_scene", "RenderError", "MODES", "BACKGROUNDS"]
diff --git a/harness/codeverse/spatial/render_scene.py b/harness/codeverse/spatial/render_scene.py
index 36c25df3..caeb87f3 100644
--- a/harness/codeverse/spatial/render_scene.py
+++ b/harness/codeverse/spatial/render_scene.py
@@ -7,18 +7,15 @@ times, deterministic instruments (console/shader errors, fps, census, camera
checks incl. luminance + content coverage) and a labelled contact sheet built
from the JUDGE subset of views. Returns a ``RenderSet`` with every view; the
full instrument payload is left at ``out_dir/metrics.json`` (``metrics_path_for``
-finds it again from a RenderSet) and ``views.json`` marks ``judge: true`` on the
-views ``select_judge_views`` keeps.
+finds it again from a RenderSet). Each ``RenderView`` (and each entry of the
+driver's ``views.json``) carries ``judge: true|false`` — stamped once from
+``select_judge_views`` at render time.
"""
from __future__ import annotations
-import contextlib
import json
import os
-import signal
-import subprocess
-import time
from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import Path
@@ -28,6 +25,8 @@ from codeverse.config import get_settings
from codeverse.contracts.artifacts import RenderSet, RenderView
from codeverse.contracts.plan import BBox, CameraPlan
from codeverse.conventions import SCENE_VIEWS, ViewPreset
+from codeverse.spatial.node import NodeError, run_node
+from codeverse.spatial.sheet import contact_sheet
from codeverse.workspace import Workspace
#: views the judge sees (contact sheet + individual images are built from these)
@@ -54,12 +53,10 @@ def runtime_js_dir() -> Path:
def run_scene_script(script: str, args: Sequence[str], *, timeout_s: float, cwd: Path | None = None) -> NodeResult:
- """Run ``runtime_js/<script>`` with node; kill the whole process group on timeout.
-
- Goes through ``codeverse.spatial.node.run_node`` (package C1) when it is
- importable, else a local equivalent. The driver prints a JSON summary as
- its LAST stdout line; exit code 2/3 means the driver itself failed →
- ``SceneRenderError``.
+ """Run ``runtime_js/<script>`` via ``codeverse.spatial.node.run_node`` (which
+ delegates to ``codeverse.proc.run_subprocess``: own process group, group kill
+ on timeout). The driver prints a JSON summary as its LAST stdout line; a
+ timeout or exit code 2/3 means the driver itself failed → ``SceneRenderError``.
"""
rt = runtime_js_dir()
path = rt / script
@@ -70,55 +67,18 @@ def run_scene_script(script: str, args: Sequence[str], *, timeout_s: float, cwd:
if gpu and "CV3D_RENDER_GPU" not in os.environ:
env_extra["CV3D_RENDER_GPU"] = gpu
try:
- from codeverse.spatial.node import NodeError, run_node
- except ImportError: # pragma: no cover - C1 not present
- rc, out, err, dur = _run_node_local(path, args, timeout_s=timeout_s, cwd=cwd or rt, env_extra=env_extra)
- else:
- try:
- r = run_node(path, list(map(str, args)), cwd=cwd or rt, timeout_s=timeout_s, env_extra=env_extra, check=False)
- except NodeError as e:
- raise SceneRenderError(f"{script} could not run: {e}") from e
- if r.timed_out:
- raise SceneRenderError(f"{script} timed out after {timeout_s:.0f}s\n{r.stderr_tail}")
- rc, out, err, dur = r.rc, r.stdout, r.stderr, r.duration_ms
- summary = _last_json_line(out)
- if rc in (2, 3) or (rc != 0 and not summary):
+ r = run_node(path, list(map(str, args)), cwd=cwd or rt, timeout_s=timeout_s, env_extra=env_extra, check=False)
+ except NodeError as e:
+ if e.result is not None and e.result.timed_out:
+ raise SceneRenderError(f"{script} timed out after {timeout_s:.0f}s\n{e.result.stderr_tail}") from e
+ raise SceneRenderError(f"{script} could not run: {e}") from e
+ summary = r.last_json or {}
+ if r.rc in (2, 3) or (r.rc != 0 and not summary):
raise SceneRenderError(
- f"{script} failed (exit {rc}): {summary.get('error') if summary else ''}\n"
- f"stderr tail: {(err or '')[-2000:]}\nstdout tail: {(out or '')[-1000:]}"
+ f"{script} failed (exit {r.rc}): {summary.get('error') if summary else ''}\n"
+ f"stderr tail: {r.stderr_tail}\nstdout tail: {r.stdout[-1000:]}"
)
- return NodeResult(rc, out, err, summary, dur)
-
-
-def _run_node_local(path: Path, args: Sequence[str], *, timeout_s: float, cwd: Path, env_extra: dict[str, str]) -> tuple[int, str, str, int]:
- node = get_settings().binaries.node or "node"
- env = dict(os.environ)
- env.setdefault("NODE_PATH", str(path.parent / "node_modules"))
- env.update(env_extra)
- t0 = time.time()
- proc = subprocess.Popen(
- [node, str(path), *map(str, args)], cwd=str(cwd), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- text=True, start_new_session=True,
- )
- try:
- out, err = proc.communicate(timeout=timeout_s)
- except subprocess.TimeoutExpired:
- with contextlib.suppress(ProcessLookupError):
- os.killpg(proc.pid, signal.SIGKILL)
- out, err = proc.communicate()
- raise SceneRenderError(f"{path.name} timed out after {timeout_s:.0f}s\n{(err or '')[-2000:]}") from None
- return proc.returncode, out, err, int((time.time() - t0) * 1000)
-
-
-def _last_json_line(stdout: str) -> dict[str, Any]:
- for line in reversed((stdout or "").splitlines()):
- line = line.strip()
- if line.startswith("{") and line.endswith("}"):
- try:
- return json.loads(line)
- except json.JSONDecodeError:
- continue
- return {}
+ return NodeResult(r.rc, r.stdout, r.stderr, summary, r.duration_ms)
def _camera_json(cams: Sequence[CameraPlan]) -> str:
@@ -148,38 +108,10 @@ def plan_bounds(ws: Workspace) -> BBox | None:
def _contact_sheet(images: list[tuple[str, Path]], out: Path, cols: int, tile: int) -> Path | None:
- """Use ``codeverse.spatial.sheet.contact_sheet`` when present, else a local PIL grid."""
+ """``sheet.contact_sheet`` with an empty-list guard (it raises on no images)."""
if not images:
return None
- try:
- from codeverse.spatial.sheet import contact_sheet # package C1
-
- return Path(contact_sheet(images, out, cols=cols, tile=tile))
- except ImportError:
- return _local_contact_sheet(images, out, cols=cols, tile=tile)
-
-
-def _local_contact_sheet(images: list[tuple[str, Path]], out: Path, *, cols: int = 4, tile: int = 384) -> Path:
- from PIL import Image, ImageDraw
-
- cols = max(1, min(cols, len(images)))
- rows = (len(images) + cols - 1) // cols
- th = int(tile * 9 / 16) + 18
- sheet = Image.new("RGB", (cols * tile, rows * th), (24, 24, 24))
- draw = ImageDraw.Draw(sheet)
- for i, (label, p) in enumerate(images):
- x, y = (i % cols) * tile, (i // cols) * th
- try:
- im = Image.open(p).convert("RGB")
- im.thumbnail((tile, th - 18))
- sheet.paste(im, (x + (tile - im.width) // 2, y + 18))
- except OSError:
- draw.rectangle([x, y + 18, x + tile, y + th], fill=(60, 0, 0))
- draw.rectangle([x, y, x + tile, y + 18], fill=(0, 0, 0))
- draw.text((x + 4, y + 3), label[: tile // 7], fill=(255, 255, 255))
- out.parent.mkdir(parents=True, exist_ok=True)
- sheet.save(out)
- return out
+ return Path(contact_sheet(images, out, cols=cols, tile=tile))
def render_scene(
@@ -259,11 +191,16 @@ def render_scene(
rs = RenderSet(
views=views, renderer=str(metrics.get("renderer") or res.summary.get("renderer") or ""),
duration_ms=res.duration_ms, console_errors=errors, fps=float(fps) if fps is not None else None,
+ out_dir=str(out_dir),
)
- judge_names = {(v.name, v.time_s) for v in select_judge_views(rs, max_n=sheet_max_views or len(views), orbit_names=[v.name for v in orbit_views]).views}
- _mark_judge_views(out_dir / "views.json", judge_names)
+ # stamp the judge subset ONCE (select_judge_views stays a pure function)
+ chosen = select_judge_views(rs, max_n=sheet_max_views or len(views), orbit_names=[v.name for v in orbit_views])
+ keep = {id(v) for v in chosen.views} # same objects, so identity is exact
+ for v in rs.views:
+ v.judge = id(v) in keep
+ _mark_judge_views(out_dir / "views.json", rs.views)
if sheet and views:
- labelled = [(f"{v.name} t={v.time_s:g}", Path(v.path)) for v in views if (v.name, v.time_s) in judge_names]
+ labelled = [(f"{v.name} t={v.time_s:g}", Path(v.path)) for v in rs.views if v.judge]
sheet_path = _contact_sheet(labelled, out_dir / "sheet.png", settings.render.sheet_cols, settings.render.sheet_tile)
rs.contact_sheet = str(sheet_path) if sheet_path else None
return rs
@@ -294,16 +231,17 @@ def select_judge_views(rs: RenderSet, max_n: int = JUDGE_MAX_VIEWS, *, orbit_nam
return rs.model_copy(update={"views": [v for v in views if id(v) in keep]}) # disk order, judge subset
-def _mark_judge_views(views_json: Path, judge: set[tuple[str, float | None]]) -> None:
- """Stamp ``judge: true|false`` on each entry of the driver's views.json."""
+def _mark_judge_views(views_json: Path, views: Sequence[RenderView]) -> None:
+ """Serialise each view's stamped ``judge`` flag into the driver's views.json."""
if not views_json.is_file():
return
try:
entries = json.loads(views_json.read_text())
except (OSError, ValueError):
return
+ flags = {(v.name, v.time_s): bool(v.judge) for v in views}
for e in entries:
- e["judge"] = (e.get("name"), e.get("time_s")) in judge
+ e["judge"] = flags.get((e.get("name"), e.get("time_s")), False)
views_json.write_text(json.dumps(entries, indent=1))
@@ -357,7 +295,12 @@ def frame_table(source: RenderSet | dict[str, Any] | Path | str) -> str:
def metrics_path_for(rs: RenderSet) -> Path | None:
- """``metrics.json`` next to a RenderSet's sheet / views (None when not a scene render)."""
+ """``metrics.json`` of a RenderSet: the stamped ``out_dir`` when present, else
+ guessed as a sibling of the sheet / views (rounds recorded before stamping)."""
+ if rs.out_dir:
+ p = Path(rs.out_dir) / "metrics.json"
+ if p.is_file():
+ return p
for cand in (rs.contact_sheet, *(v.path for v in rs.views)):
if cand:
p = Path(cand).parent / "metrics.json"
diff --git a/harness/codeverse/spatial/sections.py b/harness/codeverse/spatial/sections.py
index b39ebde7..6466afa6 100644
--- a/harness/codeverse/spatial/sections.py
+++ b/harness/codeverse/spatial/sections.py
@@ -18,10 +18,11 @@ from pathlib import Path
import numpy as np
import trimesh
-from PIL import Image, ImageDraw, ImageFont
+from PIL import Image, ImageDraw
-from codeverse.spatial.measure import GlbLoadError, load_scene, merged_mesh, part_meshes
+from codeverse.spatial.measure import GlbLoadError, cached_parts, merged_mesh
from codeverse.spatial.registry import Observation
+from codeverse.spatial.sheet import load_font
_AXIS_INDEX = {"x": 0, "y": 1, "z": 2}
#: which two axes are drawn (horizontal, vertical) for a slicing axis
@@ -44,13 +45,6 @@ class SectionData:
bbox_max: np.ndarray | None = None
-def _font(size: int = 13) -> ImageFont.ImageFont:
- try:
- return ImageFont.truetype("DejaVuSans.ttf", size)
- except Exception:
- return ImageFont.load_default()
-
-
def _section_loops(mesh: trimesh.Trimesh, axis: str, at: float) -> list[np.ndarray]:
"""Closed 2D loops (in plane axes) of ``mesh`` cut at ``axis = at``."""
normal = np.zeros(3)
@@ -120,7 +114,7 @@ def draw_section(data: SectionData, out_png: Path, *, size: int = 512, title: st
hi, vi = _AXIS_INDEX[h_ax], _AXIS_INDEX[v_ax]
img = Image.new("RGB", (size, size), (250, 250, 250))
d = ImageDraw.Draw(img, "RGBA")
- font, small = _font(13), _font(11)
+ font, small = load_font(13), load_font(11)
margin = 36
if data.bbox_min is None:
d.text((margin, margin), "empty model", fill=(0, 0, 0), font=font)
@@ -186,7 +180,7 @@ def _resolve_at(parts: dict[str, trimesh.Trimesh], axis: str, at: float, absolut
def _load_parts(glb: Path | str, parts: Sequence[str] | None) -> dict[str, trimesh.Trimesh]:
- all_parts = {k: v for k, v in part_meshes(load_scene(glb)).items() if v is not None and len(v.faces)}
+ all_parts = {k: v for k, v in cached_parts(glb).items() if v is not None and len(v.faces)}
if parts:
sel = {k: v for k, v in all_parts.items() if k in set(parts)}
missing = [p for p in parts if p not in all_parts]
diff --git a/harness/codeverse/spatial/sheet.py b/harness/codeverse/spatial/sheet.py
index 3d217dbd..95052ac1 100644
--- a/harness/codeverse/spatial/sheet.py
+++ b/harness/codeverse/spatial/sheet.py
@@ -2,6 +2,11 @@
Used for judge inputs and agent observations: one image, every view labelled
under its tile, fixed tile size so the VLM sees consistent scale.
+
+This module is also the one home for shared PIL drawing bits: ``load_font``
+and the sheet geometry/palette constants (``LABEL_H``, ``PAD``, ``BG``,
+``LABEL_BG``, ``LABEL_FG``) are public and reused by ``spatial.gl_render``,
+``spatial.sections`` and ``judges.images``.
"""
from __future__ import annotations
@@ -12,15 +17,25 @@ from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
+#: shared sheet geometry / palette (public — imported by other drawing modules)
LABEL_H = 28
PAD = 6
BG = (250, 250, 250)
LABEL_BG = (34, 34, 38)
LABEL_FG = (240, 240, 240)
+_FONTS = ("DejaVuSans.ttf", "DejaVuSansMono.ttf", "LiberationSans-Regular.ttf", "Arial.ttf")
+_FONTS_BOLD = ("DejaVuSans-Bold.ttf", "LiberationSans-Bold.ttf", "Arialbd.ttf")
+
-def _font(size: int = 15) -> ImageFont.ImageFont | ImageFont.FreeTypeFont:
- for name in ("DejaVuSans.ttf", "DejaVuSansMono.ttf", "LiberationSans-Regular.ttf", "Arial.ttf"):
+def load_font(size: int = 15, bold: bool = False) -> ImageFont.ImageFont | ImageFont.FreeTypeFont:
+ """First available truetype font at ``size`` (PIL default as last resort).
+
+ The candidate list covers the fonts present on typical Linux/CI hosts;
+ ``bold=True`` prefers the bold faces and falls back to the regular ones.
+ """
+ names = (*_FONTS_BOLD, *_FONTS) if bold else _FONTS
+ for name in names:
try:
return ImageFont.truetype(name, size)
except OSError:
@@ -58,7 +73,7 @@ def contact_sheet(
cell_h = tile + (LABEL_H if label else 0)
sheet = Image.new("RGB", (cols * (tile + PAD) + PAD, rows * (cell_h + PAD) + PAD), BG)
draw = ImageDraw.Draw(sheet)
- font = _font()
+ font = load_font()
for i, (text, path) in enumerate(images):
x = PAD + (i % cols) * (tile + PAD)
y = PAD + (i // cols) * (cell_h + PAD)
diff --git a/harness/codeverse/spatial/tools_graphics.py b/harness/codeverse/spatial/tools_graphics.py
index b44b14ec..f6fcbbda 100644
--- a/harness/codeverse/spatial/tools_graphics.py
+++ b/harness/codeverse/spatial/tools_graphics.py
@@ -90,7 +90,7 @@ def _run_build(ctx: ToolContext, *, times: list[float], preview: bool, width: in
def _stats_text(ctx: ToolContext) -> tuple[str, dict]:
- read_metrics = lazy("codeverse.languages.glsl_shader.gl_build", "read_metrics")
+ read_metrics = lazy("codeverse.languages._gl_common", "read_metrics")
m = read_metrics(ctx.workspace)
if m is None:
return "(no frame metrics)", {}
diff --git a/harness/codeverse/spatial/tools_texture.py b/harness/codeverse/spatial/tools_texture.py
index 9fda4747..1f6aedaa 100644
--- a/harness/codeverse/spatial/tools_texture.py
+++ b/harness/codeverse/spatial/tools_texture.py
@@ -4,7 +4,8 @@ generated textures). Registered for the object tracks, every language.
The tools read ``spec.json`` / ``plan.json`` from the workspace; the image model
and judge come from the spec's backends (planner for the material plan, judge for
-the ship gate) unless ``ctx.extra`` injects ``image_model`` / ``judge`` (tests).
+the ship gate) unless ``ctx.extra['texture_services']`` injects a
+``codeverse.texturing.run.TextureServices`` bundle of fakes (tests).
"""
from __future__ import annotations
@@ -58,11 +59,9 @@ def texture_pass_tool(ctx: ToolContext, args: TexturePassArgs) -> Observation:
texture_pass = lazy("codeverse.texturing.run", "texture_pass")
except ToolUnavailable as e:
return unavailable_obs("texture_pass", e)
- judge = ctx.extra.get("judge", args.judge)
+ services = ctx.extra.get("texture_services") # the ONE injection point (TextureServices)
rep = texture_pass(ctx.workspace, spec, plan, model_id=args.model or spec.backends.planner,
- image_model=ctx.extra.get("image_model"), judge=judge, glb_in=glb, size=args.size,
- plan_model=ctx.extra.get("plan_model"), render=ctx.extra.get("render"),
- cache_dir=ctx.extra.get("cache_dir"))
+ judge=args.judge, glb_in=glb, size=args.size, services=services)
s = rep.summary()
root = ctx.workspace.root
lines = [
diff --git a/harness/codeverse/texturing/run.py b/harness/codeverse/texturing/run.py
index 599d76a6..cf968f6f 100644
--- a/harness/codeverse/texturing/run.py
+++ b/harness/codeverse/texturing/run.py
@@ -16,13 +16,14 @@ import json
import logging
import time
from collections.abc import Sequence
+from dataclasses import dataclass
from pathlib import Path
from typing import Any
from pydantic import BaseModel, Field
from codeverse.contracts.artifacts import Measurement, RenderSet
-from codeverse.contracts.common import Track, Usage
+from codeverse.contracts.common import TRACK_INFO, Usage
from codeverse.contracts.plan import StaticPlan
from codeverse.contracts.spec import Spec
from codeverse.conventions import OBJECT_VIEWS_QUICK, ViewPreset
@@ -38,10 +39,19 @@ log = logging.getLogger(__name__)
TEXTURED_GLB = "object_textured.glb"
TEXTURES_DIR = "textures"
REPORT_NAME = "texturing.json"
-RUBRIC_BY_TRACK: dict[Track, str] = {
- Track.STATIC_OBJECT: "static_object_v1",
- Track.ARTICULATED_OBJECT: "articulated_v1",
-}
+
+
+@dataclass
+class TextureServices:
+ """The injectable dependencies of :func:`texture_pass` in one bundle (tests and
+ the spatial tool pass fakes through a single ``ctx.extra['texture_services']``
+ key). Every field defaults to None = "build/use the real thing"."""
+
+ image_model: Any | None = None
+ plan_model: Any | None = None
+ render: Any | None = None
+ cache_dir: Path | None = None
+ judge_obj: Any | None = None
class TextureReport(BaseModel):
@@ -103,7 +113,8 @@ def texture_pass(
*,
model_id: str,
image_model: Any | None = None,
- judge: bool | Any = True,
+ judge: bool = True,
+ judge_obj: Any | None = None,
judge_model_id: str | None = None,
rubric: str | None = None,
glb_in: Path | None = None,
@@ -112,12 +123,27 @@ def texture_pass(
plan_model: Any | None = None,
render: Any | None = None,
cache_dir: Path | None = None,
+ services: TextureServices | None = None,
events: EventLog | None = None,
update_record: bool = True,
) -> TextureReport:
- """Run the whole pass on ``ws``. ``judge`` may be ``True`` (build a VlmJudge on
- ``judge_model_id or spec.backends.judge``), ``False`` (skip the gate → ship iff
- something was textured and no seam failed) or a judge object (tests)."""
+ """Run the whole pass on ``ws``.
+
+ ``judge=False`` skips the before/after VLM gate (ship iff something was
+ textured and no seam failed); ``judge_obj`` replaces the constructed
+ ``VlmJudge`` (tests). ``services`` bundles the five injectable dependencies
+ (image_model / plan_model / render / cache_dir / judge_obj); explicit
+ keyword arguments win over the bundle."""
+ if not isinstance(judge, bool): # tolerate the retired bool|judge union for one wave
+ log.warning("texture_pass(judge=<object>) is deprecated; pass judge_obj=... instead")
+ judge_obj = judge_obj if judge_obj is not None else judge
+ judge = True
+ if services is not None:
+ image_model = image_model if image_model is not None else services.image_model
+ plan_model = plan_model if plan_model is not None else services.plan_model
+ render = render if render is not None else services.render
+ cache_dir = cache_dir if cache_dir is not None else services.cache_dir
+ judge_obj = judge_obj if judge_obj is not None else services.judge_obj
t0 = time.time()
events = events or EventLog(ws.events_path)
glb_in = Path(glb_in) if glb_in else ws.artifacts / "object.glb"
@@ -171,12 +197,12 @@ def texture_pass(
return _finish(ws, report, t0, events, update_record)
# 5. gate
- if judge is False:
+ if not judge:
report.shipped = True
notes.append("judge gate skipped (--no-judge): shipped on seam gate only")
return _finish(ws, report, t0, events, update_record)
- judge_obj = judge if judge is not True else _make_judge(spec, rubric, judge_model_id)
- gate = judge_gate(spec, plan, glb_in, glb_out, tex_dir / "gate", judge=judge_obj, measurement=_measure(glb_in),
+ gate_judge = judge_obj if judge_obj is not None else _make_judge(spec, rubric, judge_model_id)
+ gate = judge_gate(spec, plan, glb_in, glb_out, tex_dir / "gate", judge=gate_judge, measurement=_measure(glb_in),
views=views, render=render)
report.gate, report.shipped, report.delta = gate, gate.shipped, gate.delta
report.usage = report.usage + gate.usage
@@ -189,7 +215,14 @@ def texture_pass(
def _make_judge(spec: Spec, rubric: str | None, judge_model_id: str | None) -> Any:
from codeverse.judges.vlm_judge import VlmJudge
- name = rubric or RUBRIC_BY_TRACK.get(spec.track, "static_object_v1")
+ info = TRACK_INFO.get(spec.track)
+ if rubric:
+ name = rubric
+ elif info is not None:
+ name = info.rubric
+ else:
+ name = "static_object_v1"
+ log.warning("no TRACK_INFO row for track %s; texture gate falls back to %s", spec.track, name)
return VlmJudge(rubric=name, model_id=judge_model_id or spec.backends.judge, n_samples=1, label="texture_gate")
diff --git a/harness/codeverse/texturing/scene_pack.py b/harness/codeverse/texturing/scene_pack.py
index 7807c52a..71c2ea92 100644
--- a/harness/codeverse/texturing/scene_pack.py
+++ b/harness/codeverse/texturing/scene_pack.py
@@ -10,7 +10,7 @@ generated BEFORE zone generation so scene code can load them.
* ``texture_pack_prompt(manifest)``: the snippet the scene track injects into zone /
env prompts (URL, tile size, exact three.js loading lines).
-Served URL: the workspace root is served at ``/`` (``runtime_js/lib/_compat.mjs``),
+Served URL: the workspace root is served at ``/`` (``runtime_js/lib/host_coverage.mjs``),
so ``public/textures/<name>.png`` is reachable as ``/public/textures/<name>.png``.
"""
diff --git a/harness/codeverse/tracks/articulated_object.py b/harness/codeverse/tracks/articulated_object.py
index 7272abe2..9dfb874e 100644
--- a/harness/codeverse/tracks/articulated_object.py
+++ b/harness/codeverse/tracks/articulated_object.py
@@ -21,9 +21,10 @@ from codeverse.contracts.artifacts import (
RenderView,
Severity,
)
-from codeverse.contracts.common import Track
+from codeverse.contracts.common import TRACK_INFO, Track
from codeverse.contracts.plan import ArticulatedPlan, Plan
from codeverse.tracks.common import RunContext
+from codeverse.tracks.motion import MOTION_GATE
from codeverse.tracks.static_object import ObjectPipeline, StaticObjectTrack
from codeverse.workspace import Workspace
@@ -40,11 +41,9 @@ class ArticulatedPipeline(ObjectPipeline):
out_dir = ctx.ws.renders_dir(round_index) / "poses"
report, views = ctx.services.joint_sweep(ctx.ws, ctx.plan, out_dir)
ctx.extra["pose_views"] = views
- ctx.extra["sweep_report"] = report
out.append(report)
motion = self._motion_gate(ctx)
if motion is not None:
- ctx.extra["motion_report"] = motion
out.append(motion)
return out
@@ -73,9 +72,9 @@ class ArticulatedPipeline(ObjectPipeline):
joints = "; ".join(f"{j.name} ({j.type} {j.parent}→{j.child}, [{j.lower:.2f},{j.upper:.2f}])" for j in plan.joints)
return f"{base} Root link {plan.root_link}. Joints: {joints}."
- def judge_context(self, ctx: RunContext, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
- report: GateReport | None = ctx.extra.pop("sweep_report", None)
- motion: GateReport | None = ctx.extra.pop("motion_report", None)
+ def judge_context(self, ws: Workspace, plan: Plan | None, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
+ report = next((g for g in gates if g.gate == SWEEP_GATE), None)
+ motion = next((g for g in gates if g.gate == MOTION_GATE), None)
lines = ["Articulation sheet: the pose_* views show the object at rest, each joint at its lower and upper limit."]
if report is not None:
errs = [f"- {f.target or 'joint'}: {f.message}" for f in report.errors]
@@ -90,11 +89,11 @@ class ArticulatedPipeline(ObjectPipeline):
class ArticulatedObjectTrack(StaticObjectTrack):
track = Track.ARTICULATED_OBJECT
- rubric = "articulated_v1"
+ rubric = TRACK_INFO[Track.ARTICULATED_OBJECT].rubric
plan_model = ArticulatedPlan
generate_template = "tracks/generate_articulated.j2"
- def make_pipeline(self, ctx: RunContext) -> ArticulatedPipeline:
+ def make_pipeline(self) -> ArticulatedPipeline:
return ArticulatedPipeline()
def system_prompt(self, ctx: RunContext) -> str:
diff --git a/harness/codeverse/tracks/candidates.py b/harness/codeverse/tracks/candidates.py
index c17c19d5..2a14fbef 100644
--- a/harness/codeverse/tracks/candidates.py
+++ b/harness/codeverse/tracks/candidates.py
@@ -204,7 +204,7 @@ def quick_judge(track: Any, ctx: RunContext, pipeline: RoundPipeline, build: Any
judge = track.make_judge(ctx, n_samples=1)
inp = JudgeInput(spec=ctx.spec, renders=renders, measurement=measurement, gates=gates,
acceptance=list(getattr(ctx.plan, "acceptance", []) or []), plan_summary=pipeline.plan_summary(ctx),
- round_index=0, extra_context=pipeline.judge_context(ctx, 0, build, gates))
+ round_index=0, extra_context=pipeline.judge_context(ctx.ws, ctx.plan, 0, build, gates))
judgment = judge.judge(inp)
except Exception as e: # noqa: BLE001 — judge outage: candidate ranked by gates only
log.warning("quick judge failed for %s: %s", ctx.ws.root, e)
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index fd1b4678..ff87c0d7 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -205,21 +205,26 @@ def run_agent_task(
"""CodingAgent path. ``ok`` = the agent changed files (envelope-vs-disk truth)."""
before = ws.head()
timeout = task.timeout_s or (settings.limits.agent_timeout_s if settings is not None else 1800)
- # job.extra is honoured by every CodingAgent: round → trajectory dir + ToolContext,
+ # typed job context honoured by every CodingAgent: round → trajectory dir + ToolContext,
# language/track → spatial tool filtering, files_hint → per-session attribution of
# files_changed when tasks run concurrently in ONE workspace (see agents/cli_common).
- extra: dict[str, Any] = {"round": task.round, "kind": task.kind, "files_hint": list(task.files_hint)}
+ language = track = ""
spec_path = ws.spec_path
if spec_path.is_file():
try:
spec_d = ws.read_json(spec_path)
- extra["language"] = spec_d.get("language", "")
- extra["track"] = spec_d.get("track", "")
+ language = str(spec_d.get("language", ""))
+ track = str(spec_d.get("track", ""))
except Exception: # noqa: BLE001 — best effort context only
pass
+ # extra mirrors the typed fields until every agent backend reads job.round/… directly
+ extra: dict[str, Any] = {"round": task.round, "kind": task.kind, "files_hint": list(task.files_hint),
+ "language": language, "track": track}
job = AgentJob(workspace=str(ws.root), prompt=task.prompt, system_append=task.system,
model=getattr(agent, "model", ""), label=task.label, timeout_s=timeout,
- spatial_tools=True, write_roots=task.write_roots, extra=extra)
+ spatial_tools=True, write_roots=task.write_roots,
+ round=task.round, kind=task.kind, language=language, track=track,
+ files_hint=list(task.files_hint), extra=extra)
res: AgentResult = agent.run(job)
usage = res.usage
if budget is not None:
diff --git a/harness/codeverse/tracks/graphics.py b/harness/codeverse/tracks/graphics.py
index bde74c48..e5db92f3 100644
--- a/harness/codeverse/tracks/graphics.py
+++ b/harness/codeverse/tracks/graphics.py
@@ -2,50 +2,46 @@
Languages: glsl_shader (Shadertoy-style fragment shader) · opengl_python (raw
moderngl program). Reuses ``BaseTrack`` (lifecycle) and ``run_round`` (steps)
-unchanged; the track-specific pieces are the planner template, the prompt
-context (no 3D frame), the ``gl_frames`` gate (frame statistics from the build)
-and the render step (the sampled frames + contact sheet as the RenderSet the
-``shader_v1`` judge sees). Refinement is always one whole-program task.
-
-The shared prompt helpers look the language up in ``conventions.LANGUAGE_FRAME``
-(repair prompts); graphics languages have no 3D frame, so they are registered
-here as Y-up/+Z-front (GL clip space) until conventions.py carries them.
+unchanged; the track-specific pieces are the planner hooks (template / example /
+acceptance in ``graphics_steps``), the prompt context (no 3D frame), the
+``gl_frames`` gate (frame statistics from the build) and the render step (the
+sampled frames + contact sheet as the RenderSet the ``shader_v1`` judge sees).
+Refinement is always one whole-program task.
"""
from __future__ import annotations
import logging
-from collections.abc import Sequence
+from typing import Any
from codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet
-from codeverse.contracts.common import Language, Track
+from codeverse.contracts.common import TRACK_INFO, Language, Track
from codeverse.contracts.plan import GraphicsPlan, Plan
from codeverse.contracts.run import RoundRecord
-from codeverse.conventions import LANGUAGE_FRAME, Frame
-from codeverse.events import EventLog
+from codeverse.contracts.spec import Spec
from codeverse.languages.glsl_shader.gl_build import read_metrics
-from codeverse.orchestrator.rounds import TaskGroup, build_refine_instructions
+from codeverse.orchestrator.rounds import TaskGroup
from codeverse.prompts import render
from codeverse.tracks.common import RunContext
from codeverse.tracks.generation import GenerationTask
from codeverse.tracks.graphics_steps import (
- expected_files,
+ PLAN_MAX_OUTPUT_TOKENS,
+ PLAN_TEMPERATURE,
+ PLAN_TEMPLATE,
+ ensure_graphics_acceptance,
frame_stats_text,
frames_render_set,
+ graphics_event_stats,
+ graphics_expected_files,
graphics_prompt_context,
- plan_graphics,
)
+from codeverse.tracks.graphics_steps import plan_example as graphics_plan_example
from codeverse.tracks.lifecycle import BaseTrack
-from codeverse.tracks.repair import format_error_report
-from codeverse.tracks.static_object import current_files, judge_digest, skeleton_files
-from codeverse.tracks.steps import failed_acceptance
+from codeverse.tracks.prompting import current_files, judge_digest, skeleton_files
from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
-for _lang in (Language.GLSL_SHADER, Language.OPENGL_PYTHON):
- LANGUAGE_FRAME.setdefault(_lang.value, Frame.Y_UP_POS_Z_FRONT)
-
class GraphicsPipeline:
"""No measurement; gates = gl_frames (from artifacts/metrics.json); render = the frames."""
@@ -71,37 +67,36 @@ class GraphicsPipeline:
return (f"{plan.title}: {plan.summary} Style: {plan.style}. Passes: {passes}. Key visuals: {visuals}. "
f"Motion: {plan.motion or '(none planned)'}. {plan.resolution[0]}x{plan.resolution[1]}, loop {plan.duration_s:g}s.")
- def judge_context(self, ctx: RunContext, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
+ def judge_context(self, ws: Workspace, plan: Plan | None, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
renderer = build.census.get("renderer", "") if isinstance(build.census, dict) else ""
- return f"FRAME METRICS (harness-measured, renderer {renderer or 'moderngl'}):\n{frame_stats_text(ctx.ws)}"
+ return f"FRAME METRICS (harness-measured, renderer {renderer or 'moderngl'}):\n{frame_stats_text(ws)}"
class GraphicsTrack(BaseTrack):
track = Track.GRAPHICS
- rubric = "shader_v1"
+ rubric = TRACK_INFO[Track.GRAPHICS].rubric
plan_model = GraphicsPlan
generate_template = "tracks/generate_graphics.j2"
refine_template = "tracks/refine_graphics.j2"
-
- def make_pipeline(self, ctx: RunContext) -> GraphicsPipeline:
+ # planner hooks: own template/example/acceptance, T=0.5, 16k tokens (no 3D frame)
+ plan_template = PLAN_TEMPLATE
+ plan_temperature = PLAN_TEMPERATURE
+ plan_max_output_tokens = PLAN_MAX_OUTPUT_TOKENS
+ # refinement is always ONE whole-program task
+ allow_refine_fanout = False
+
+ def make_pipeline(self) -> GraphicsPipeline:
return GraphicsPipeline()
- # ------------------------------------------------------------------ planning (own template / acceptance)
- def plan(self, spec, ws: Workspace) -> Plan:
- ws.create()
- events = EventLog(ws.events_path)
- runtime = self._runtime or self.services.runtime(spec.language)
- return plan_graphics(spec, spec.backends.planner, ws, model=self._planner_model, events=events, runtime=runtime)
+ # ------------------------------------------------------------------ planner hooks
+ def plan_example(self, spec: Spec) -> dict[str, Any]:
+ return graphics_plan_example()
- def _plan_stage(self, ctx: RunContext) -> Plan:
- from codeverse.contracts.run import RunStatus
+ def finalise_plan(self, plan_obj: Any, spec: Spec) -> Any:
+ return ensure_graphics_acceptance(plan_obj, spec)
- ctx.state.status = RunStatus.PLANNING
- ctx.state.save(ctx.ws)
- plan = plan_graphics(ctx.spec, ctx.spec.backends.planner, ctx.ws, model=self._planner_model, events=ctx.events,
- budget=ctx.budget, runtime=ctx.runtime)
- self._save_spent(ctx)
- return plan
+ def plan_event_stats(self, plan_obj: Any) -> dict[str, Any]:
+ return graphics_event_stats(plan_obj)
# ------------------------------------------------------------------ baseline
def system_prompt(self, ctx: RunContext) -> str:
@@ -111,7 +106,7 @@ class GraphicsTrack(BaseTrack):
f"Render and LOOK at your frames before finishing.")
def baseline_tasks(self, ctx: RunContext) -> list[GenerationTask]:
- files = expected_files(ctx)
+ files = graphics_expected_files(ctx)
prompt = render(self.generate_template, **graphics_prompt_context(
ctx, skeleton_files=skeleton_files(ctx) if ctx.single_shot else {}, previous_error=""))
ctx.record_prompt("generate", prompt)
@@ -119,33 +114,18 @@ class GraphicsTrack(BaseTrack):
kind="baseline", temperature=0.6, thinking="medium")]
def round_files_hint(self, ctx: RunContext) -> list[str]:
- return expected_files(ctx)
-
- # ------------------------------------------------------------------ refine
- def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord]) -> tuple[list[GenerationTask], list[str]]:
- index = len(history)
- if last.build is None or not last.build.ok:
- return [self._rebuild_task(ctx, last, index)], ["rebuild: previous round did not build"]
- tasks = build_refine_instructions(last.judgment, last.gates, failed_acceptance(ctx, last.judgment), ctx.plan,
- file_for_target=None, max_tasks=ctx.policy.max_refine_tasks)
- if not tasks:
- return [], []
- group = TaskGroup(tasks=tasks, files=expected_files(ctx))
+ return graphics_expected_files(ctx)
+
+ def generate_context(self, ctx: RunContext, **extra: Any) -> dict[str, Any]:
+ return graphics_prompt_context(ctx, **extra)
+
+ # ------------------------------------------------------------------ refine (scaffold hook; never fans out)
+ def _refine_task(self, ctx: RunContext, group: TaskGroup, last: RoundRecord, index: int, *, parallel: bool) -> GenerationTask:
+ files = graphics_expected_files(ctx)
prompt = render(self.refine_template, **graphics_prompt_context(
- ctx, round_index=index, tasks=[t.line() for t in tasks], targets=group.targets, files=group.files,
+ ctx, round_index=index, tasks=[t.line() for t in group.tasks], targets=group.targets, files=files,
judge_summary=judge_digest(last), frame_notes=frame_stats_text(ctx.ws),
- current_files=current_files(ctx, group.files) if ctx.single_shot else {}))
+ current_files=current_files(ctx, files) if ctx.single_shot else {}))
ctx.record_prompt("refine", prompt)
- ctx.events.emit("refine.planned", round=index, n_tasks=len(tasks), n_groups=1, parallel=False, targets=[group.targets])
- task = GenerationTask(label="refine", prompt=prompt, system=self.system_prompt(ctx), files_hint=group.files, round=index,
+ return GenerationTask(label="refine", prompt=prompt, system=self.system_prompt(ctx), files_hint=files, round=index,
kind="refine", temperature=0.5, thinking="medium")
- return [task], [t.line() for t in tasks]
-
- def _rebuild_task(self, ctx: RunContext, last: RoundRecord, index: int) -> GenerationTask:
- files = expected_files(ctx)
- lint = next((g for g in last.gates if g.gate.startswith("lint")), GateReport(gate="lint", passed=True))
- report = format_error_report(last.build, lint, ctx.cookbook_text) if last.build else "build did not run"
- prompt = render(self.generate_template, **graphics_prompt_context(
- ctx, skeleton_files=current_files(ctx, files) if ctx.single_shot else {}, previous_error=report))
- return GenerationTask(label="rebuild", prompt=prompt, system=self.system_prompt(ctx), files_hint=files, round=index,
- kind="rebuild", temperature=0.7, thinking="high")
diff --git a/harness/codeverse/tracks/graphics_steps.py b/harness/codeverse/tracks/graphics_steps.py
index e2a5f67b..d385c403 100644
--- a/harness/codeverse/tracks/graphics_steps.py
+++ b/harness/codeverse/tracks/graphics_steps.py
@@ -1,32 +1,29 @@
-"""Graphics-track helpers: planner (GraphicsPlan), prompt context, frame RenderSet.
+"""Graphics-track helpers: planner hooks (GraphicsPlan), prompt context, frame RenderSet.
-The shared planner / prompt helpers assume a 3D frame (``LANGUAGE_FRAME``,
-``ensure_acceptance`` adds a ground-contact item); shaders have neither, so the
-graphics track owns these three pieces and reuses everything else from
+The shared planner assumes a 3D frame in its acceptance fixups
+(``ensure_acceptance`` adds a ground-contact item); shaders have none, so the
+graphics track supplies its own template / example / acceptance hooks to the
+ONE planner loop in ``tracks/planner.py`` and reuses everything else from
``tracks/lifecycle.py`` + ``tracks/steps.py`` unchanged.
"""
from __future__ import annotations
-import json
import logging
import shutil
from collections.abc import Sequence
from pathlib import Path
from typing import Any
-from pydantic import ValidationError
-
-from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderView
-from codeverse.contracts.chat import ChatMessage, ChatRequest
-from codeverse.contracts.common import Language, Usage
+from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderView, Severity
+from codeverse.contracts.common import Language
from codeverse.contracts.plan import AcceptanceItem, GraphicsPlan
from codeverse.contracts.spec import Spec
from codeverse.languages.glsl_shader.gl_build import SHEET_NAME, read_metrics
-from codeverse.prompts import prompt_hash, render
-from codeverse.tracks.common import RunContext, language_contract
+from codeverse.tracks.common import RunContext
from codeverse.tracks.generation import SINGLE_SHOT_FORMAT
-from codeverse.tracks.planner import PlanningError, _parse_json
+from codeverse.tracks.planner import add_acceptance_item, build_system_prompt
+from codeverse.tracks.planner import plan as run_planner
from codeverse.tracks.prompting import (
AGENT_OUTPUT_RULES,
acceptance_lines,
@@ -38,6 +35,8 @@ from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
PLAN_TEMPLATE = "tracks/plan_graphics.j2"
+PLAN_TEMPERATURE = 0.5
+PLAN_MAX_OUTPUT_TOKENS = 16000
EXPECTED_FILES: dict[Language, list[str]] = {
Language.GLSL_SHADER: ["src/shader.frag", "src/common.glsl"],
Language.OPENGL_PYTHON: ["src/program.py"],
@@ -68,87 +67,41 @@ def plan_example() -> dict[str, Any]:
def build_plan_system_prompt(spec: Spec, *, runtime: Any | None = None) -> str:
- return render(PLAN_TEMPLATE, track=spec.track.value, language=spec.language.value,
- contract=language_contract(spec.language, runtime)[:6000],
- example_json=json.dumps(plan_example(), indent=1),
- schema_fields=", ".join(GraphicsPlan.model_json_schema().get("properties", {}).keys()))
-
-
-def build_plan_user_prompt(spec: Spec) -> str:
- return "\n".join([f"REQUEST: {spec.prompt}", "", "CONSTRAINTS:", constraints_text(spec), "",
- "Return the plan as JSON matching the schema."])
+ """The graphics plan system prompt = the shared builder with the graphics
+ template + worked example (``plan_graphics.j2`` references no 3D frame)."""
+ return build_system_prompt(spec, GraphicsPlan, runtime=runtime, template=PLAN_TEMPLATE, example=plan_example())
def ensure_graphics_acceptance(plan: GraphicsPlan, spec: Spec) -> GraphicsPlan:
"""Spec must_have / must_not → visual items; planned motion → a probe item (never 'ground contact')."""
items: list[AcceptanceItem] = list(plan.acceptance)
- have = {a.text.strip().lower() for a in items}
- ids = {a.id for a in items}
-
- def _add(prefix: str, text: str, how: str) -> None:
- if text.strip().lower() in have:
- return
- n = 1
- while f"{prefix}{n}" in ids:
- n += 1
- ids.add(f"{prefix}{n}")
- items.append(AcceptanceItem(id=f"{prefix}{n}", text=text, how=how, priority="must")) # type: ignore[arg-type]
-
for m in spec.constraints.must_have:
- _add("must", f"Includes: {m}", "visual")
+ add_acceptance_item(items, "must", f"Includes: {m}", "visual")
for m in spec.constraints.must_not:
- _add("not", f"Does NOT include: {m}", "visual")
+ add_acceptance_item(items, "not", f"Does NOT include: {m}", "visual")
if plan.motion.strip() and not any("static" in a.text.lower() or "motion" in a.text.lower() or "change over time" in a.text.lower() for a in items):
- _add("motion", "Frames change over time as planned (not a static image)", "probe")
+ add_acceptance_item(items, "motion", "Frames change over time as planned (not a static image)", "probe")
plan.acceptance = items
return plan
+def graphics_event_stats(plan: GraphicsPlan) -> dict[str, Any]:
+ """``plan.done`` payload for graphics (passes, not parts/zones)."""
+ return {"n_passes": len(plan.passes), "n_acceptance": len(plan.acceptance)}
+
+
def plan_graphics(spec: Spec, model_id: str, ws: Workspace, *, model: Any | None = None, events: Any | None = None,
budget: Any | None = None, runtime: Any | None = None) -> GraphicsPlan:
- """Structured planner call → validated GraphicsPlan (one re-ask with the validation errors)."""
- if model is None:
- from codeverse.models import get_chat_model
-
- model = get_chat_model(model_id)
- system = build_plan_system_prompt(spec, runtime=runtime)
- messages = [ChatMessage.user(build_plan_user_prompt(spec))]
- schema = GraphicsPlan.model_json_schema()
- usage = Usage()
- last_error = ""
- for attempt in range(2):
- req = ChatRequest(messages=messages, system=system, response_schema=schema, temperature=0.5, thinking="medium",
- max_output_tokens=16000, label=f"planner{'-retry' if attempt else ''}")
- resp = model.generate(req)
- usage = usage + resp.usage
- raw = resp.parsed if resp.parsed is not None else _parse_json(resp.text)
- try:
- if not isinstance(raw, dict):
- raise ValueError(f"planner returned {type(raw).__name__}, expected a JSON object")
- plan = GraphicsPlan.model_validate(raw)
- except (ValidationError, ValueError) as e:
- last_error = str(e)[:4000]
- if events is not None:
- events.emit("plan.invalid", attempt=attempt, error=last_error[:500])
- messages = messages + [ChatMessage.assistant(json.dumps(raw)[:20000] if raw is not None else (resp.text or "")[:20000]),
- ChatMessage.user("Your plan failed validation. Fix EXACTLY these problems and return the full corrected "
- f"plan JSON again (same schema):\n{last_error}")]
- continue
- plan = ensure_graphics_acceptance(plan, spec)
- ws.write_json(ws.plan_path, plan)
- if budget is not None:
- budget.charge(usage)
- if events is not None:
- events.emit("plan.done", model=model_id, attempt=attempt, n_passes=len(plan.passes), n_acceptance=len(plan.acceptance),
- cost_usd=round(usage.cost_usd, 4), prompt_hash=prompt_hash(system))
- return plan
- if budget is not None:
- budget.charge(usage)
- raise PlanningError(f"graphics plan did not validate after re-ask: {last_error}")
+ """Structured planner call → validated GraphicsPlan: the ONE planner loop
+ (``tracks/planner.plan``) parameterised with the graphics hooks."""
+ return run_planner(spec, model_id, GraphicsPlan, ws, model=model, events=events, budget=budget, runtime=runtime,
+ template=PLAN_TEMPLATE, example=plan_example(), temperature=PLAN_TEMPERATURE,
+ max_output_tokens=PLAN_MAX_OUTPUT_TOKENS,
+ finalise=lambda p: ensure_graphics_acceptance(p, spec), event_stats=graphics_event_stats)
# ----------------------------------------------------------------------------- prompt context
-def expected_files(ctx: RunContext) -> list[str]:
+def graphics_expected_files(ctx: RunContext) -> list[str]:
return list(EXPECTED_FILES.get(ctx.language, ["src/shader.frag"]))
@@ -174,7 +127,7 @@ def graphics_prompt_context(ctx: RunContext, **extra: Any) -> dict[str, Any]:
"passes_table": passes_table(plan), "motion": plan.motion if plan else "", "key_visuals": list(plan.key_visuals) if plan else [],
"uniforms": ", ".join(plan.uniforms) if plan and plan.uniforms else "u_time, u_resolution",
"acceptance": acceptance_lines(plan), "entry_files": ", ".join(getattr(ctx.runtime, "entry_globs", ()) or ()),
- "expected_files": expected_files(ctx), "reference_note": reference_note(ctx),
+ "expected_files": graphics_expected_files(ctx), "reference_note": reference_note(ctx),
"judge_times": "0, 1, 2.5, 4, 6 s",
}
d.update(extra)
@@ -220,7 +173,7 @@ def frame_stats_text(ws: Workspace) -> str:
stats, gate = m
lines = stats.summary_lines()
for f in gate.findings:
- lines.append(f"- GATE gl_frames [{f.severity.value}] {f.message}" + (f" FIX: {f.fix_hint}" if f.fix_hint and f.severity.value != "info" else ""))
+ lines.append(f"- GATE gl_frames {f.as_line(with_severity=True, with_hint=f.severity is not Severity.INFO)}")
return "\n".join(lines)
diff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.py
index 8354c366..006eea31 100644
--- a/harness/codeverse/tracks/lifecycle.py
+++ b/harness/codeverse/tracks/lifecycle.py
@@ -1,9 +1,10 @@
"""BaseTrack: the resumable plan → prepare → rounds → finalise lifecycle.
-Concrete tracks (static / articulated / scene) override the hooks
-(``plan_model``, ``make_pipeline``, ``prepare``, ``baseline_tasks``,
-``refine_tasks``). All bookkeeping — workspace, events, budget, run state,
-stage cache, best tracking, stop policy, record — lives here once.
+Concrete tracks (static / articulated / scene / graphics) override the hooks
+(``plan_model``, ``make_pipeline``, ``prepare``, ``baseline_tasks``, the planner
+knobs ``plan_template``/``plan_example``/``finalise_plan`` and the refine
+scaffold's ``_refine_task``). All bookkeeping — workspace, events, budget, run
+state, stage cache, best tracking, stop policy, record — lives here once.
"""
from __future__ import annotations
@@ -18,15 +19,25 @@ from pathlib import Path
from typing import Any
from codeverse.config import Settings, get_settings
+from codeverse.contracts.artifacts import GateReport
from codeverse.contracts.common import Track, Usage
from codeverse.contracts.plan import Plan
from codeverse.contracts.run import RoundRecord, RunRecord, RunStatus
from codeverse.contracts.spec import Spec
from codeverse.events import EventLog
from codeverse.orchestrator.budget import BudgetExceeded, BudgetGuard
-from codeverse.orchestrator.rounds import BestSelector, RoundPolicy, StopPolicy, StopReason
+from codeverse.orchestrator.rounds import (
+ BestSelector,
+ RoundPolicy,
+ StopPolicy,
+ StopReason,
+ TaskGroup,
+ build_refine_instructions,
+ plan_refine_groups,
+)
from codeverse.orchestrator.runner import StageRunner
from codeverse.orchestrator.state import RunState
+from codeverse.prompts import render
from codeverse.tracks.candidates import choose_best_round, run_best_of_n
from codeverse.tracks.common import (
RunContext,
@@ -36,10 +47,15 @@ from codeverse.tracks.common import (
load_prompt_or,
)
from codeverse.tracks.generation import GenerationTask, is_single_shot, single_shot_model_id
+from codeverse.tracks.planner import default_event_stats, ensure_acceptance, normalise_names
from codeverse.tracks.planner import plan as run_planner
+from codeverse.tracks.planner import plan_example as default_plan_example
+from codeverse.tracks.prompting import base_prompt_context, current_files, expected_files
+from codeverse.tracks.repair import format_error_report
from codeverse.tracks.steps import (
RoundFailed,
RoundPipeline,
+ failed_acceptance,
load_round_records,
rejudge_round,
run_round,
@@ -76,11 +92,21 @@ def plan_stage_inputs(spec: Spec) -> dict[str, Any]:
class BaseTrack:
- """Shared lifecycle. Subclasses set ``track``, ``rubric``, ``plan_model``."""
+ """Shared lifecycle. Subclasses set ``track``, ``rubric``, ``plan_model``
+ and parameterise the ONE planner loop / refine scaffold via the hook
+ attributes below (``plan_template``, ``plan_example``, ``_refine_task`` …)."""
track: Track
rubric: str
plan_model: type[Plan]
+ generate_template: str = ""
+ refine_template: str = ""
+ #: planner knobs (GraphicsTrack: own template/example, T=0.5, 16k tokens)
+ plan_template: str | None = None # None → tracks/plan_<track>.j2
+ plan_temperature: float = 0.4
+ plan_max_output_tokens: int = 24000
+ #: refine fan-out (graphics is always ONE whole-program task)
+ allow_refine_fanout: bool = True
def __init__(
self,
@@ -106,7 +132,7 @@ class BaseTrack:
self._n_candidates = n_candidates
# ------------------------------------------------------------------ hooks
- def make_pipeline(self, ctx: RunContext) -> RoundPipeline:
+ def make_pipeline(self) -> RoundPipeline:
raise NotImplementedError
def prepare(self, ctx: RunContext, runner: StageRunner) -> None:
@@ -116,19 +142,78 @@ class BaseTrack:
def baseline_tasks(self, ctx: RunContext) -> list[GenerationTask]:
raise NotImplementedError
- def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord]) -> tuple[list[GenerationTask], list[str]]:
+ def system_prompt(self, ctx: RunContext) -> str:
raise NotImplementedError
def round_files_hint(self, ctx: RunContext) -> list[str]:
return []
+ # ---- planner hooks (tracks/planner.py runs the one loop)
+ def plan_example(self, spec: Spec) -> dict[str, Any]:
+ """Worked example JSON for the plan system prompt."""
+ return default_plan_example(spec.track)
+
+ def finalise_plan(self, plan_obj: Any, spec: Spec) -> Any:
+ """Post-validation fixup (names + deterministic acceptance items)."""
+ return ensure_acceptance(normalise_names(plan_obj), spec)
+
+ def plan_event_stats(self, plan_obj: Any) -> dict[str, Any]:
+ """Extra ``plan.done`` payload (n_parts/n_zones… per track)."""
+ return default_event_stats(plan_obj)
+
+ # ---- refine hooks (the scaffold below is shared; tracks fill in the task)
+ def refine_file_for_target(self, ctx: RunContext) -> Any:
+ """``target → [files]`` mapper for refine tasks (None = whole-object)."""
+ return None
+
+ def extra_refine_tasks(self, ctx: RunContext, last: RoundRecord) -> Sequence[Any]:
+ """Harness-derived tasks prepended to the judge's (e.g. reference IoU)."""
+ return ()
+
+ def generate_context(self, ctx: RunContext, **extra: Any) -> dict[str, Any]:
+ """Template context for ``generate_template`` (baseline / rebuild prompts)."""
+ extra.setdefault("expected_files", expected_files(ctx))
+ return base_prompt_context(ctx, **extra)
+
+ def _refine_task(self, ctx: RunContext, group: TaskGroup, last: RoundRecord, index: int, *, parallel: bool) -> GenerationTask:
+ raise NotImplementedError
+
+ # ------------------------------------------------------------------ refine scaffold
+ def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord]) -> tuple[list[GenerationTask], list[str]]:
+ """index → rebuild guard → instructions → groups → per-track ``_refine_task``."""
+ index = len(history)
+ if last.build is None or not last.build.ok:
+ return [self._rebuild_task(ctx, last, index)], ["rebuild: previous round did not build"]
+ tasks = build_refine_instructions(last.judgment, last.gates, failed_acceptance(ctx, last.judgment), ctx.plan,
+ file_for_target=self.refine_file_for_target(ctx),
+ max_tasks=ctx.policy.max_refine_tasks,
+ extra=self.extra_refine_tasks(ctx, last))
+ if not tasks:
+ return [], []
+ groups, parallel = plan_refine_groups(tasks, allow_fanout=self.allow_refine_fanout,
+ parallel_min_tasks=ctx.policy.parallel_min_tasks)
+ gen_tasks = [self._refine_task(ctx, g, last, index, parallel=parallel) for g in groups]
+ ctx.events.emit("refine.planned", round=index, n_tasks=len(tasks), n_groups=len(groups), parallel=parallel,
+ targets=[g.targets for g in groups])
+ return gen_tasks, [t.line() for t in tasks]
+
+ def _rebuild_task(self, ctx: RunContext, last: RoundRecord, index: int) -> GenerationTask:
+ """Regenerate after a failed build: the generate prompt + the error report."""
+ files = self.round_files_hint(ctx)
+ lint = next((g for g in last.gates if g.gate.startswith("lint")), GateReport(gate="lint", passed=True))
+ report = format_error_report(last.build, lint, ctx.cookbook_text) if last.build else "build did not run"
+ prompt = render(self.generate_template, **self.generate_context(
+ ctx, skeleton_files=current_files(ctx, files) if ctx.single_shot else {}, previous_error=report))
+ return GenerationTask(label="rebuild", prompt=prompt, system=self.system_prompt(ctx), files_hint=files, round=index,
+ kind="rebuild", temperature=0.7, thinking="high")
+
# ------------------------------------------------------------------ public API
def plan(self, spec: Spec, ws: Workspace) -> Plan:
ws.create()
events = EventLog(ws.events_path)
runtime = self._runtime or self.services.runtime(spec.language)
return run_planner(spec, spec.backends.planner, self.plan_model, ws, model=self._planner_model,
- events=events, runtime=runtime)
+ events=events, runtime=runtime, **self._plan_kwargs(spec))
def run(self, spec: Spec, ws: Workspace, *, resume: bool = False) -> RunRecord:
ws.create()
@@ -174,7 +259,7 @@ class BaseTrack:
spent = state.extra.get("spent_usage")
if spent:
budget.spent = Usage.model_validate(spent)
- n_cand = self._resolve_candidates(state, settings)
+ n_cand = self._resolve_candidates(spec, state, settings)
policy = (self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds)).with_candidates(n_cand)
# reference images: static objects are scored with reference_v1 (adds the measured silhouette
# criterion); other tracks keep their rubric but the judge still sees the references.
@@ -190,9 +275,12 @@ class BaseTrack:
ctx.record_prompt(name, text)
return ctx
- def _resolve_candidates(self, state: RunState, settings: Settings) -> int:
- """Best-of-N width: constructor (CLI --candidates) > persisted run state (resume) > settings default."""
+ def _resolve_candidates(self, spec: Spec, state: RunState, settings: Settings) -> int:
+ """Best-of-N width: constructor (CLI --candidates) > ``spec.options.candidates``
+ > persisted run state (legacy resume) > settings default."""
n = self._n_candidates
+ if n is None:
+ n = spec.options.candidates
if n is None:
n = state.extra.get("n_candidates")
if n is None:
@@ -227,11 +315,18 @@ class BaseTrack:
def _plan_stage(self, ctx: RunContext) -> Plan:
ctx.state.status = RunStatus.PLANNING
ctx.state.save(ctx.ws)
- plan = run_planner(ctx.spec, ctx.spec.backends.planner, self.plan_model, ctx.ws, model=self._planner_model,
- events=ctx.events, budget=ctx.budget, runtime=ctx.runtime)
- self._save_spent(ctx)
+ try:
+ plan = run_planner(ctx.spec, ctx.spec.backends.planner, self.plan_model, ctx.ws, model=self._planner_model,
+ events=ctx.events, budget=ctx.budget, runtime=ctx.runtime, **self._plan_kwargs(ctx.spec))
+ finally:
+ self._save_spent(ctx) # charged on success AND PlanningError
return plan
+ def _plan_kwargs(self, spec: Spec) -> dict[str, Any]:
+ return {"template": self.plan_template, "example": self.plan_example(spec),
+ "temperature": self.plan_temperature, "max_output_tokens": self.plan_max_output_tokens,
+ "finalise": lambda p: self.finalise_plan(p, spec), "event_stats": self.plan_event_stats}
+
# ------------------------------------------------------------------ stages
def stage_skeleton(self, ctx: RunContext, runner: StageRunner) -> list[str]:
def _do() -> list[str]:
@@ -268,7 +363,7 @@ class BaseTrack:
# ------------------------------------------------------------------ round loop
def _round_loop(self, ctx: RunContext, rounds: list[RoundRecord]) -> StopReason:
self.ensure_materialized(ctx)
- pipeline = self.make_pipeline(ctx)
+ pipeline = self.make_pipeline()
stop_policy = StopPolicy(ctx.policy)
selector = BestSelector()
rejudged: set[int] = set()
@@ -346,7 +441,7 @@ class BaseTrack:
ctx.events.emit("finalise.rebuild", round=best, ok=build.ok)
except Exception as e: # noqa: BLE001 — the best round already built once; report, don't fail
ctx.events.emit("finalise.rebuild_failed", error=f"{type(e).__name__}: {e}")
- if rounds and ("texture" in ctx.spec.tags or ctx.extra.get("texture")):
+ if rounds and (ctx.spec.options.texture or "texture" in ctx.spec.tags or ctx.extra.get("texture")):
self._texture_pass(ctx)
ctx.state.status, ctx.state.stop_reason, ctx.state.error = status, stop_reason, error
ctx.state.save(ctx.ws)
diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.py
index 03ee5931..008ce81c 100644
--- a/harness/codeverse/tracks/planner.py
+++ b/harness/codeverse/tracks/planner.py
@@ -13,13 +13,14 @@ from __future__ import annotations
import json
import logging
+from collections.abc import Callable
from typing import Any, TypeVar
from pydantic import BaseModel, ValidationError
from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart
from codeverse.contracts.common import Language, Track, Usage
-from codeverse.contracts.plan import AcceptanceItem, ArticulatedPlan, ScenePlan, StaticPlan
+from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.spec import Spec
from codeverse.conventions import LANGUAGE_FRAME, frame_doc, to_pascal
from codeverse.prompts import prompt_hash, render
@@ -39,25 +40,62 @@ PLAN_TEMPLATES: dict[Track, str] = {
class PlanningError(RuntimeError):
- """The planner could not produce a valid plan after one re-ask."""
+ """The planner could not produce a valid plan after one re-ask.
+
+ Carries the ``usage`` already spent so callers can charge the budget even
+ when planning fails (a failed re-ask is still paid for)."""
+
+ def __init__(self, message: str, usage: Usage | None = None):
+ super().__init__(message)
+ self.usage = usage or Usage()
+
+
+#: hook types (BaseTrack subclasses parameterise the ONE planner loop with these)
+FinalisePlan = Callable[[Any], Any]
+EventStats = Callable[[Any], dict[str, Any]]
+
+
+def default_event_stats(plan_obj: Any) -> dict[str, Any]:
+ return {"n_parts": len(getattr(plan_obj, "parts", []) or []),
+ "n_zones": len(getattr(plan_obj, "zones", []) or []),
+ "n_acceptance": len(plan_obj.acceptance)}
def plan(spec: Spec, model_id: str, plan_model: type[P], ws: Workspace, *, model: Any | None = None,
- events: Any | None = None, budget: Any | None = None, runtime: Any | None = None) -> P:
- """Plan and write ``ws.plan_path``. ``model`` may be injected (tests)."""
- result, usage = plan_with_usage(spec, model_id, plan_model, ws, model=model, events=events, runtime=runtime)
+ events: Any | None = None, budget: Any | None = None, runtime: Any | None = None,
+ template: str | None = None, example: dict[str, Any] | None = None, temperature: float = 0.4,
+ max_output_tokens: int = 24000, finalise: FinalisePlan | None = None,
+ event_stats: EventStats | None = None) -> P:
+ """Plan and write ``ws.plan_path``. ``model`` may be injected (tests).
+
+ The budget is charged on success AND on ``PlanningError`` — a failed
+ re-ask is still paid for."""
+ try:
+ result, usage = plan_with_usage(spec, model_id, plan_model, ws, model=model, events=events, runtime=runtime,
+ template=template, example=example, temperature=temperature,
+ max_output_tokens=max_output_tokens, finalise=finalise, event_stats=event_stats)
+ except PlanningError as e:
+ if budget is not None:
+ budget.charge(e.usage)
+ raise
if budget is not None:
budget.charge(usage)
return result
def plan_with_usage(spec: Spec, model_id: str, plan_model: type[P], ws: Workspace, *, model: Any | None = None,
- events: Any | None = None, runtime: Any | None = None) -> tuple[P, Usage]:
+ events: Any | None = None, runtime: Any | None = None, template: str | None = None,
+ example: dict[str, Any] | None = None, temperature: float = 0.4, max_output_tokens: int = 24000,
+ finalise: FinalisePlan | None = None, event_stats: EventStats | None = None) -> tuple[P, Usage]:
+ """One structured planner call + one re-ask, parameterised by the track hooks:
+ ``template``/``example`` (system prompt), ``finalise`` (post-validation fixup,
+ default = ``ensure_acceptance(normalise_names(...))``) and ``event_stats``
+ (extra ``plan.done`` payload)."""
if model is None:
from codeverse.models import get_chat_model
model = get_chat_model(model_id)
- system = build_system_prompt(spec, plan_model, runtime=runtime)
+ system = build_system_prompt(spec, plan_model, runtime=runtime, template=template, example=example)
user = build_user_prompt(spec)
images = [ImagePart(path=r.path, label=f"{r.role}: {r.note}".strip(": ")) for r in spec.references]
messages = [ChatMessage.user(user, images=images or None)]
@@ -65,8 +103,8 @@ def plan_with_usage(spec: Spec, model_id: str, plan_model: type[P], ws: Workspac
usage = Usage()
last_error = ""
for attempt in range(2):
- req = ChatRequest(messages=messages, system=system, response_schema=schema, temperature=0.4,
- thinking="medium", max_output_tokens=24000, label=f"planner{'-retry' if attempt else ''}")
+ req = ChatRequest(messages=messages, system=system, response_schema=schema, temperature=temperature,
+ thinking="medium", max_output_tokens=max_output_tokens, label=f"planner{'-retry' if attempt else ''}")
resp = model.generate(req)
usage = usage + resp.usage
raw = resp.parsed if resp.parsed is not None else _parse_json(resp.text)
@@ -84,19 +122,20 @@ def plan_with_usage(spec: Spec, model_id: str, plan_model: type[P], ws: Workspac
f"plan JSON again (same schema):\n{last_error}"),
]
continue
- result = ensure_acceptance(normalise_names(result), spec)
+ result = finalise(result) if finalise is not None else ensure_acceptance(normalise_names(result), spec)
ws.write_json(ws.plan_path, result)
if events is not None:
- events.emit("plan.done", model=model_id, attempt=attempt, n_parts=len(getattr(result, "parts", []) or []),
- n_zones=len(getattr(result, "zones", []) or []), n_acceptance=len(result.acceptance),
- cost_usd=round(usage.cost_usd, 4), prompt_hash=prompt_hash(system))
+ stats = (event_stats or default_event_stats)(result)
+ events.emit("plan.done", model=model_id, **stats,
+ cost_usd=round(usage.cost_usd, 4), prompt_hash=prompt_hash(system), attempt=attempt)
return result, usage
- raise PlanningError(f"plan did not validate after re-ask: {last_error}")
+ raise PlanningError(f"plan did not validate after re-ask: {last_error}", usage)
# ----------------------------------------------------------------------------- prompts
-def build_system_prompt(spec: Spec, plan_model: type[BaseModel], *, runtime: Any | None = None) -> str:
- template = PLAN_TEMPLATES[spec.track]
+def build_system_prompt(spec: Spec, plan_model: type[BaseModel], *, runtime: Any | None = None,
+ template: str | None = None, example: dict[str, Any] | None = None) -> str:
+ template = template or PLAN_TEMPLATES[spec.track]
lang: Language = spec.language
return render(
template,
@@ -104,7 +143,7 @@ def build_system_prompt(spec: Spec, plan_model: type[BaseModel], *, runtime: Any
language=lang.value,
frame_doc=frame_doc(LANGUAGE_FRAME[lang.value]),
contract=language_contract(lang, runtime)[:6000],
- example_json=json.dumps(plan_example(spec.track), indent=1),
+ example_json=json.dumps(example if example is not None else plan_example(spec.track), indent=1),
schema_fields=", ".join(plan_model.model_json_schema().get("properties", {}).keys()),
)
@@ -136,33 +175,33 @@ def _parse_json(text: str) -> Any:
# ----------------------------------------------------------------------------- deterministic acceptance
-def ensure_acceptance(plan_obj: P, spec: Spec) -> P:
- """Append acceptance items derived from the spec constraints when missing."""
- items: list[AcceptanceItem] = list(plan_obj.acceptance)
- have = {a.text.strip().lower() for a in items}
+def add_acceptance_item(items: list[AcceptanceItem], prefix: str, text: str, how: str) -> None:
+ """Append a framework-derived *must* acceptance item unless an item with the
+ same text exists; the id is ``<prefix><N>`` with N chosen to be unique."""
+ if text.strip().lower() in {a.text.strip().lower() for a in items}:
+ return
ids = {a.id for a in items}
+ n = 1
+ while f"{prefix}{n}" in ids:
+ n += 1
+ items.append(AcceptanceItem(id=f"{prefix}{n}", text=text, how=how, priority="must")) # type: ignore[arg-type]
- def _add(prefix: str, text: str, how: str) -> None:
- if text.strip().lower() in have:
- return
- n = 1
- while f"{prefix}{n}" in ids:
- n += 1
- ids.add(f"{prefix}{n}")
- items.append(AcceptanceItem(id=f"{prefix}{n}", text=text, how=how, priority="must")) # type: ignore[arg-type]
+def ensure_acceptance(plan_obj: P, spec: Spec) -> P:
+ """Append acceptance items derived from the spec constraints when missing."""
+ items: list[AcceptanceItem] = list(plan_obj.acceptance)
c = spec.constraints
if c.dimensions_m:
dims = ", ".join(f"{k} = {v:.3f} m" for k, v in c.dimensions_m.items())
- _add("dim", f"Overall dimensions match the request within 5%: {dims}", "measure")
+ add_acceptance_item(items, "dim", f"Overall dimensions match the request within 5%: {dims}", "measure")
if c.max_triangles:
- _add("tri", f"Triangle count ≤ {c.max_triangles}", "measure")
+ add_acceptance_item(items, "tri", f"Triangle count ≤ {c.max_triangles}", "measure")
for m in c.must_have:
- _add("must", f"Includes: {m}", "visual")
+ add_acceptance_item(items, "must", f"Includes: {m}", "visual")
for m in c.must_not:
- _add("not", f"Does NOT include: {m}", "visual")
+ add_acceptance_item(items, "not", f"Does NOT include: {m}", "visual")
if spec.track is not Track.SCENE and not any(a.how == "measure" for a in items):
- _add("ground", "Object stands on the ground plane (lowest point at up=0) with its footprint centred", "measure")
+ add_acceptance_item(items, "ground", "Object stands on the ground plane (lowest point at up=0) with its footprint centred", "measure")
plan_obj.acceptance = items
return plan_obj
@@ -221,10 +260,6 @@ def plan_example(track: Track) -> dict[str, Any]:
return base
-def plan_model_for(track: Track) -> type[StaticPlan] | type[ArticulatedPlan] | type[ScenePlan]:
- return {Track.STATIC_OBJECT: StaticPlan, Track.ARTICULATED_OBJECT: ArticulatedPlan, Track.SCENE: ScenePlan}[track]
-
-
def normalise_names(plan_obj: P) -> P:
"""PascalCase part/link/zone/asset names in place — prompts ask for it; code guarantees it.
Normalisation keeps the snake key, so validated uniqueness/tree properties are preserved."""
@@ -242,5 +277,5 @@ def normalise_names(plan_obj: P) -> P:
return plan_obj
-__all__ = ["PlanningError", "plan", "plan_with_usage", "plan_model_for", "plan_example", "ensure_acceptance",
- "normalise_names", "build_system_prompt", "build_user_prompt", "load_prompt_or"]
+__all__ = ["PlanningError", "plan", "plan_with_usage", "plan_example", "ensure_acceptance", "add_acceptance_item",
+ "default_event_stats", "normalise_names", "build_system_prompt", "build_user_prompt", "load_prompt_or"]
diff --git a/harness/codeverse/tracks/prompting.py b/harness/codeverse/tracks/prompting.py
index 19e30e3e..7e3589a5 100644
--- a/harness/codeverse/tracks/prompting.py
+++ b/harness/codeverse/tracks/prompting.py
@@ -16,12 +16,18 @@ from typing import Any
from codeverse.contracts.chat import ImagePart
from codeverse.contracts.common import Language
from codeverse.contracts.plan import Plan, StaticPlan
+from codeverse.contracts.run import RoundRecord
from codeverse.conventions import LANGUAGE_FRAME, Frame, frame_doc, to_snake
from codeverse.tracks.common import RunContext
from codeverse.tracks.generation import SINGLE_SHOT_FORMAT
log = logging.getLogger(__name__)
+MAX_SKELETON_CHARS = 14_000
+
+#: the files a scene refine task falls back to when the group has no file ownership
+SCENE_FILES: tuple[str, ...] = ("src/scene.js", "src/env.js")
+
def parts_table(plan: Plan) -> str:
"""Markdown table of parts with exact bboxes (meters, 3 decimals)."""
@@ -154,6 +160,88 @@ def reference_note(ctx: RunContext) -> str:
return "\n".join(lines)
+# ----------------------------------------------------------------------------- files + round digests
+def expected_files(ctx: RunContext) -> list[str]:
+ """Files the generator is expected to produce for this language + plan."""
+ lang = ctx.language
+ parts = getattr(ctx.plan, "parts", None) or []
+ if lang is Language.THREEJS:
+ return ["src/object.js"] + [f"src/parts/{to_snake(p.name)}.js" for p in parts]
+ if lang is Language.URDF_BLENDER:
+ return ["src/model.py", "src/robot.urdf"]
+ files = ["src/model.py"]
+ custom = getattr(ctx.runtime, "file_for_part", None) # blender: src/parts/<snake>.py per part
+ if callable(custom):
+ for p in parts:
+ try:
+ rel = custom(p.name)
+ except Exception as e: # noqa: BLE001
+ log.warning("runtime.file_for_part failed for %s: %s", p.name, e)
+ continue
+ if rel and str(rel) not in files:
+ files.append(str(rel))
+ return files
+
+
+def skeleton_files(ctx: RunContext, max_chars: int = MAX_SKELETON_CHARS) -> dict[str, str]:
+ """Current src/ files (the skeleton), trimmed, for single-shot prompts."""
+ return current_files(ctx, [str(p.relative_to(ctx.ws.root)) for p in sorted(ctx.ws.src.rglob("*")) if p.is_file()], max_chars)
+
+
+def current_files(ctx: RunContext, rels: Sequence[str], max_chars: int = MAX_SKELETON_CHARS) -> dict[str, str]:
+ out: dict[str, str] = {}
+ total = 0
+ for rel in rels:
+ p = ctx.ws.root / rel
+ if not p.is_file():
+ continue
+ text = p.read_text(errors="replace")
+ room = max_chars - total
+ if room <= 0:
+ break
+ if len(text) > room:
+ text = text[:room] + "\n# ... truncated ...\n"
+ out[rel] = text
+ total += len(text)
+ return out
+
+
+def judge_digest(last: RoundRecord, max_issues: int = 8) -> str:
+ j = last.judgment
+ if j is None:
+ return "(no judgment for the previous round)"
+ lines = [f"Previous score {j.overall:.2f} ({'passed' if j.passed else 'not passed'}). {j.summary}".strip()]
+ for k, v in sorted(j.scores.items(), key=lambda kv: kv[1])[:6]:
+ lines.append(f"- {k}: {v:.2f}")
+ for i in j.issues[:max_issues]:
+ lines.append(f"- [{i.severity}/{i.kind}] {i.target}: {i.detail}" + (f" (seen in {i.evidence})" if i.evidence else ""))
+ return "\n".join(lines)
+
+
+def measurement_vs_plan(last: RoundRecord, plan: Plan | None, language: Language = Language.THREEJS) -> str:
+ """Exact numbers (in the plan's frame): measured overall/part bboxes vs planned ones."""
+ m = last.measurement
+ if m is None or plan is None or not hasattr(plan, "overall_bbox"):
+ return ""
+ pe = plan.overall_bbox.extents
+ me = glb_to_plan_frame(m.extents, language, extents=True)
+ lines = [f"Measured overall extents {me[0]:.3f}×{me[1]:.3f}×{me[2]:.3f} m vs plan "
+ f"{pe[0]:.3f}×{pe[1]:.3f}×{pe[2]:.3f} m; ground gap {m.ground_gap_m:+.3f} m; footprint offset {m.footprint_offset_m:.3f} m; "
+ f"{m.tri_count} tris, {m.n_meshes} meshes, {m.n_islands} islands."]
+ planned = {to_snake(p.name): p for p in getattr(plan, "parts", [])}
+ for pm in m.parts[:24]:
+ p = planned.get(to_snake(pm.name))
+ if p is None:
+ continue
+ ext = glb_to_plan_frame([b - a for a, b in zip(pm.bbox_min, pm.bbox_max, strict=True)], language, extents=True)
+ cen = glb_to_plan_frame([(a + b) / 2 for a, b in zip(pm.bbox_min, pm.bbox_max, strict=True)], language)
+ lines.append(f"- {p.name}: measured centre ({cen[0]:.3f}, {cen[1]:.3f}, {cen[2]:.3f}) extents ({ext[0]:.3f}, {ext[1]:.3f}, {ext[2]:.3f})"
+ f" | plan centre ({p.bbox.center[0]:.3f}, {p.bbox.center[1]:.3f}, {p.bbox.center[2]:.3f}) extents "
+ f"({p.bbox.extents[0]:.3f}, {p.bbox.extents[1]:.3f}, {p.bbox.extents[2]:.3f})")
+ gate_lines = [f"- {f.as_line(with_gate=True)}" for g in last.gates for f in g.errors][:12]
+ return "\n".join(lines + gate_lines)
+
+
AGENT_OUTPUT_RULES = """HOW TO FINISH (agent mode): edit files under src/ only (and public/ for compiled assets).
Before you finish you MUST run the `build` tool and fix every error it reports; then run `measure`
(objects) or `scene_probe` (scenes) once and compare the numbers with the plan. Do not write
diff --git a/harness/codeverse/tracks/repair.py b/harness/codeverse/tracks/repair.py
index 73160f74..31f0299f 100644
--- a/harness/codeverse/tracks/repair.py
+++ b/harness/codeverse/tracks/repair.py
@@ -66,10 +66,7 @@ def format_error_report(build: BuildResult, lint: GateReport, cookbook: str = ""
lines.append("```")
if lint.errors:
lines.append(f"LINT ERRORS ({lint.gate}):")
- for f in lint.errors:
- tgt = f" [{f.target}]" if f.target else ""
- hint = f" → FIX: {f.fix_hint}" if f.fix_hint else ""
- lines.append(f"- {f.message}{tgt}{hint}")
+ lines.extend(f"- {f.as_line(with_target=True)}" for f in lint.errors)
section = relevant_cookbook_section(cookbook, " ".join([build.error_type, build.error_message, build.stderr_tail[-2000:]]))
if section:
lines.append("\nRelevant cookbook section:\n" + section)
diff --git a/harness/codeverse/tracks/scene.py b/harness/codeverse/tracks/scene.py
index 9201fb08..de2ea602 100644
--- a/harness/codeverse/tracks/scene.py
+++ b/harness/codeverse/tracks/scene.py
@@ -10,7 +10,6 @@ zone / asset / env / camera, in parallel when file-disjoint.
from __future__ import annotations
import logging
-from collections.abc import Sequence
from typing import Any
from codeverse.contracts.artifacts import (
@@ -21,33 +20,34 @@ from codeverse.contracts.artifacts import (
RenderSet,
Severity,
)
-from codeverse.contracts.common import Track
-from codeverse.contracts.plan import ScenePlan, ZonePlan
+from codeverse.contracts.common import TRACK_INFO, Track
+from codeverse.contracts.plan import Plan, ScenePlan, ZonePlan
from codeverse.contracts.run import RoundRecord
from codeverse.conventions import to_snake
from codeverse.orchestrator.fanout import fan_out
-from codeverse.orchestrator.rounds import (
- TaskGroup,
- build_refine_instructions,
- compact_instructions,
- plan_parallel_groups,
-)
+from codeverse.orchestrator.rounds import TaskGroup, compact_instructions
from codeverse.orchestrator.runner import StageRunner
from codeverse.prompts import render
+from codeverse.spatial.render_scene import JUDGE_MAX_VIEWS
from codeverse.tracks.common import RunContext, ServiceUnavailable
from codeverse.tracks.generation import GenerationResult, GenerationTask, generate
from codeverse.tracks.lifecycle import BaseTrack
-from codeverse.tracks.prompting import base_prompt_context, bbox_line, file_for_target_factory
+from codeverse.tracks.prompting import (
+ SCENE_FILES,
+ base_prompt_context,
+ bbox_line,
+ current_files,
+ file_for_target_factory,
+ judge_digest,
+)
from codeverse.tracks.repair import format_error_report
from codeverse.tracks.scene_assets import AssetResult, asset_api_summary, run_asset_stage
-from codeverse.tracks.static_object import current_files, judge_digest
-from codeverse.tracks.steps import failed_acceptance
+from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
SCENE_TIMES: tuple[float, float] = (0.0, 1.5)
MAX_CONSOLE_ERRORS = 8
-JUDGE_MAX_VIEWS = 10
class ScenePipeline:
@@ -92,7 +92,11 @@ class ScenePipeline:
return out
def judge_views(self, ctx: RunContext, renders: RenderSet) -> RenderSet:
- """The ≤ 10 views the judge sees (authored@t0 first); the full set stays on disk."""
+ """The ≤ 10 views the judge sees (authored@t0 first); the full set stays on disk.
+ Prefers the per-view ``judge`` flags stamped at render time; legacy render
+ sets (no flags) fall back to ``select_judge_views``."""
+ if any(v.judge is not None for v in renders.views):
+ return renders.model_copy(update={"views": [v for v in renders.views if v.judge]})
return ctx.services.select_judge_views(renders, max_n=JUDGE_MAX_VIEWS)
def plan_summary(self, ctx: RunContext) -> str:
@@ -101,8 +105,9 @@ class ScenePipeline:
assets = ", ".join(a.name for a in plan.assets)
return f"{plan.title}: {plan.summary} Setting: {plan.setting}. Zones: {zones}. Assets: {assets}. Cameras: {', '.join(c.name for c in plan.cameras)}."
- def judge_context(self, ctx: RunContext, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
- plan: ScenePlan = ctx.plan # type: ignore[assignment]
+ def judge_context(self, ws: Workspace, plan: Plan | None, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
+ if not isinstance(plan, ScenePlan):
+ return ""
lines = [f"Environment plan: {plan.environment}", "Animation plan: " + "; ".join(plan.animation)]
lines.append("Cameras: " + "; ".join(f"{c.name} ({c.purpose})" for c in plan.cameras))
return "\n".join(lines)
@@ -110,10 +115,10 @@ class ScenePipeline:
class SceneTrack(BaseTrack):
track = Track.SCENE
- rubric = "scene_v1"
+ rubric = TRACK_INFO[Track.SCENE].rubric
plan_model = ScenePlan
- def make_pipeline(self, ctx: RunContext) -> ScenePipeline:
+ def make_pipeline(self) -> ScenePipeline:
return ScenePipeline()
# ------------------------------------------------------------------ stages
@@ -196,36 +201,22 @@ class SceneTrack(BaseTrack):
return [] # generation happened in the stages; round 0 = build → render → judge
def round_files_hint(self, ctx: RunContext) -> list[str]:
- return ["src/scene.js", "src/env.js"]
-
- def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord]) -> tuple[list[GenerationTask], list[str]]:
- index = len(history)
- if last.build is None or not last.build.ok:
- # no rebuild path meant a probe/console failure that lint does not flag
- # yielded zero tasks and the run ended as 'plateau' with budget left
- return [self._rebuild_task(ctx, last, index)], ["rebuild: previous round did not build"]
- fft = file_for_target_factory(ctx)
- tasks = build_refine_instructions(last.judgment, last.gates, failed_acceptance(ctx, last.judgment), ctx.plan,
- file_for_target=fft, max_tasks=ctx.policy.max_refine_tasks)
- if not tasks:
- return [], []
- groups = plan_parallel_groups(tasks)
- parallel = len(groups) >= ctx.policy.parallel_min_tasks and all(g.files for g in groups)
- if not parallel:
- groups = [TaskGroup(tasks=tasks, files=sorted({f for t in tasks for f in t.files}))]
- gen = []
- for g in groups:
- files = g.files or ["src/scene.js", "src/env.js"]
- lines = compact_instructions(g.tasks, max_lines=ctx.policy.max_instructions_per_task)
- prompt = render("tracks/scene_refine.j2", **self._ctx(ctx, round_index=index, tasks=lines,
- targets=g.targets, files=files, edit_only_these=parallel,
- judge_summary=judge_digest(last),
- current_files=current_files(ctx, files) if ctx.single_shot else {}))
- ctx.record_prompt("scene_refine", prompt)
- gen.append(GenerationTask(label=f"refine_{g.label}" if parallel else "refine", prompt=prompt, system=self.system_prompt(ctx),
- files_hint=files, round=index, kind="refine", temperature=0.4))
- ctx.events.emit("refine.planned", round=index, n_tasks=len(tasks), n_groups=len(groups), parallel=parallel)
- return gen, [t.line() for t in tasks]
+ return list(SCENE_FILES)
+
+ # ------------------------------------------------------------------ refine (scaffold hooks)
+ def refine_file_for_target(self, ctx: RunContext) -> Any:
+ return file_for_target_factory(ctx)
+
+ def _refine_task(self, ctx: RunContext, group: TaskGroup, last: RoundRecord, index: int, *, parallel: bool) -> GenerationTask:
+ files = group.files or list(SCENE_FILES)
+ lines = compact_instructions(group.tasks, max_lines=ctx.policy.max_instructions_per_task)
+ prompt = render("tracks/scene_refine.j2", **self._ctx(ctx, round_index=index, tasks=lines,
+ targets=group.targets, files=files, edit_only_these=parallel,
+ judge_summary=judge_digest(last),
+ current_files=current_files(ctx, files) if ctx.single_shot else {}))
+ ctx.record_prompt("scene_refine", prompt)
+ return GenerationTask(label=f"refine_{group.label}" if parallel else "refine", prompt=prompt, system=self.system_prompt(ctx),
+ files_hint=files, round=index, kind="refine", temperature=0.4)
def _rebuild_task(self, ctx: RunContext, last: RoundRecord, index: int) -> GenerationTask:
"""One repair task carrying the structured error report (build + lint + census
@@ -233,8 +224,7 @@ class SceneTrack(BaseTrack):
lint = next((g for g in last.gates if g.gate.startswith("lint")), GateReport(gate="lint", passed=True))
report = format_error_report(last.build, lint, ctx.cookbook_text) if last.build else "build did not run"
census = census_gate_report(last.build) if last.build else None
- lines = [report] + [f"- {f.message}" + (f" FIX: {f.fix_hint}" if f.fix_hint else "")
- for f in (census.errors if census else [])][:8]
+ lines = [report] + [f"- {f.as_line()}" for f in (census.errors if census else [])][:8]
files: list[str] = []
if last.build is not None and last.build.error_file:
rel = last.build.error_file.removeprefix(str(ctx.ws.root)).lstrip("/")
diff --git a/harness/codeverse/tracks/static_object.py b/harness/codeverse/tracks/static_object.py
index 481cd9a2..4c795704 100644
--- a/harness/codeverse/tracks/static_object.py
+++ b/harness/codeverse/tracks/static_object.py
@@ -10,37 +10,35 @@ from __future__ import annotations
import logging
from collections.abc import Sequence
from pathlib import Path
+from typing import Any
from codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet
-from codeverse.contracts.common import Language, Track
+from codeverse.contracts.common import TRACK_INFO, Track
from codeverse.contracts.plan import Plan, StaticPlan
from codeverse.contracts.run import RoundRecord
-from codeverse.conventions import BBOX_TOLERANCE_M, OBJECT_VIEWS, to_snake
-from codeverse.orchestrator.rounds import (
- RefineTask,
- TaskGroup,
- build_refine_instructions,
- compact_instructions,
- plan_parallel_groups,
-)
+from codeverse.conventions import BBOX_TOLERANCE_M, OBJECT_VIEWS
+from codeverse.orchestrator.rounds import RefineTask, TaskGroup, compact_instructions
from codeverse.prompts import render
from codeverse.tracks.common import RunContext
from codeverse.tracks.generation import GenerationTask
from codeverse.tracks.lifecycle import BaseTrack
from codeverse.tracks.prompting import (
+ MAX_SKELETON_CHARS, # noqa: F401 — re-exported: this was their import path
base_prompt_context,
+ current_files,
+ expected_files,
file_for_target_factory,
glb_to_plan_frame,
+ judge_digest,
+ measurement_vs_plan,
reference_images,
+ skeleton_files,
)
from codeverse.tracks.reference import reference_refine_tasks, silhouette_gate
-from codeverse.tracks.repair import format_error_report
-from codeverse.tracks.steps import failed_acceptance
+from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
-MAX_SKELETON_CHARS = 14_000
-
#: the 4 clay views judged for holes/intersections (subset of OBJECT_VIEWS by name)
GEOMETRY_VIEW_NAMES: tuple[str, ...] = ("front_right_34", "back_left_34", "top", "low_front_left")
GEOMETRY_VIEWS = tuple(v for v in OBJECT_VIEWS if v.name in GEOMETRY_VIEW_NAMES)
@@ -102,18 +100,18 @@ class ObjectPipeline:
e = glb_to_plan_frame(plan.overall_bbox.extents, ctx.language, extents=True)
return f"{plan.object_name}: {plan.summary} Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m (W×H×D). Parts: {parts}."
- def judge_context(self, ctx: RunContext, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
+ def judge_context(self, ws: Workspace, plan: Plan | None, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
return ""
class StaticObjectTrack(BaseTrack):
track = Track.STATIC_OBJECT
- rubric = "static_object_v1"
+ rubric = TRACK_INFO[Track.STATIC_OBJECT].rubric
plan_model = StaticPlan
generate_template = "tracks/generate_static.j2"
refine_template = "tracks/refine_object.j2"
- def make_pipeline(self, ctx: RunContext) -> ObjectPipeline:
+ def make_pipeline(self) -> ObjectPipeline:
return ObjectPipeline()
# ------------------------------------------------------------------ baseline
@@ -132,25 +130,12 @@ class StaticObjectTrack(BaseTrack):
def round_files_hint(self, ctx: RunContext) -> list[str]:
return expected_files(ctx)
- # ------------------------------------------------------------------ refine
- def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord]) -> tuple[list[GenerationTask], list[str]]:
- index = len(history)
- if last.build is None or not last.build.ok:
- return [self._rebuild_task(ctx, last, index)], ["rebuild: previous round did not build"]
- fft = file_for_target_factory(ctx)
- tasks = build_refine_instructions(last.judgment, last.gates, failed_acceptance(ctx, last.judgment), ctx.plan,
- file_for_target=fft, max_tasks=ctx.policy.max_refine_tasks,
- extra=reference_refine_tasks(ctx, last))
- if not tasks:
- return [], []
- groups = plan_parallel_groups(tasks)
- parallel = len(groups) >= ctx.policy.parallel_min_tasks and all(g.files for g in groups)
- if not parallel:
- groups = [TaskGroup(tasks=tasks, files=sorted({f for t in tasks for f in t.files}))]
- gen_tasks = [self._refine_task(ctx, g, last, index, parallel=parallel) for g in groups]
- ctx.events.emit("refine.planned", round=index, n_tasks=len(tasks), n_groups=len(groups), parallel=parallel,
- targets=[g.targets for g in groups])
- return gen_tasks, [t.line() for t in tasks]
+ # ------------------------------------------------------------------ refine (scaffold hooks)
+ def refine_file_for_target(self, ctx: RunContext) -> Any:
+ return file_for_target_factory(ctx)
+
+ def extra_refine_tasks(self, ctx: RunContext, last: RoundRecord) -> Sequence[RefineTask]:
+ return reference_refine_tasks(ctx, last)
def _refine_task(self, ctx: RunContext, group: TaskGroup, last: RoundRecord, index: int, *, parallel: bool) -> GenerationTask:
files = group.files if parallel else (group.files or expected_files(ctx))
@@ -164,101 +149,8 @@ class StaticObjectTrack(BaseTrack):
files_hint=files, round=index, kind="refine", temperature=0.4, thinking="medium",
images=reference_images(ctx))
- def _rebuild_task(self, ctx: RunContext, last: RoundRecord, index: int) -> GenerationTask:
- files = expected_files(ctx)
- lint = next((g for g in last.gates if g.gate.startswith("lint")), GateReport(gate="lint", passed=True))
- report = format_error_report(last.build, lint, ctx.cookbook_text) if last.build else "build did not run"
- prompt = render(self.generate_template, **base_prompt_context(
- ctx, expected_files=files, skeleton_files=current_files(ctx, files) if ctx.single_shot else {},
- previous_error=report))
- return GenerationTask(label="rebuild", prompt=prompt, system=self.system_prompt(ctx), files_hint=files, round=index,
- kind="rebuild", temperature=0.7, thinking="high")
# ----------------------------------------------------------------------------- helpers
-def expected_files(ctx: RunContext) -> list[str]:
- """Files the generator is expected to produce for this language + plan."""
- lang = ctx.language
- parts = getattr(ctx.plan, "parts", None) or []
- if lang is Language.THREEJS:
- return ["src/object.js"] + [f"src/parts/{to_snake(p.name)}.js" for p in parts]
- if lang is Language.URDF_BLENDER:
- return ["src/model.py", "src/robot.urdf"]
- if lang is Language.SCENE_THREEJS:
- return ["src/scene.js", "src/env.js"]
- files = ["src/model.py"]
- custom = getattr(ctx.runtime, "file_for_part", None) # blender: src/parts/<snake>.py per part
- if callable(custom):
- for p in parts:
- try:
- rel = custom(p.name)
- except Exception as e: # noqa: BLE001
- log.warning("runtime.file_for_part failed for %s: %s", p.name, e)
- continue
- if rel and str(rel) not in files:
- files.append(str(rel))
- return files
-
-
-def skeleton_files(ctx: RunContext, max_chars: int = MAX_SKELETON_CHARS) -> dict[str, str]:
- """Current src/ files (the skeleton), trimmed, for single-shot prompts."""
- return current_files(ctx, [str(p.relative_to(ctx.ws.root)) for p in sorted(ctx.ws.src.rglob("*")) if p.is_file()], max_chars)
-
-
-def current_files(ctx: RunContext, rels: Sequence[str], max_chars: int = MAX_SKELETON_CHARS) -> dict[str, str]:
- out: dict[str, str] = {}
- total = 0
- for rel in rels:
- p = ctx.ws.root / rel
- if not p.is_file():
- continue
- text = p.read_text(errors="replace")
- room = max_chars - total
- if room <= 0:
- break
- if len(text) > room:
- text = text[:room] + "\n# ... truncated ...\n"
- out[rel] = text
- total += len(text)
- return out
-
-
-def judge_digest(last: RoundRecord, max_issues: int = 8) -> str:
- j = last.judgment
- if j is None:
- return "(no judgment for the previous round)"
- lines = [f"Previous score {j.overall:.2f} ({'passed' if j.passed else 'not passed'}). {j.summary}".strip()]
- for k, v in sorted(j.scores.items(), key=lambda kv: kv[1])[:6]:
- lines.append(f"- {k}: {v:.2f}")
- for i in j.issues[:max_issues]:
- lines.append(f"- [{i.severity}/{i.kind}] {i.target}: {i.detail}" + (f" (seen in {i.evidence})" if i.evidence else ""))
- return "\n".join(lines)
-
-
-def measurement_vs_plan(last: RoundRecord, plan: Plan | None, language: Language = Language.THREEJS) -> str:
- """Exact numbers (in the plan's frame): measured overall/part bboxes vs planned ones."""
- m = last.measurement
- if m is None or plan is None or not hasattr(plan, "overall_bbox"):
- return ""
- pe = plan.overall_bbox.extents
- me = glb_to_plan_frame(m.extents, language, extents=True)
- lines = [f"Measured overall extents {me[0]:.3f}×{me[1]:.3f}×{me[2]:.3f} m vs plan "
- f"{pe[0]:.3f}×{pe[1]:.3f}×{pe[2]:.3f} m; ground gap {m.ground_gap_m:+.3f} m; footprint offset {m.footprint_offset_m:.3f} m; "
- f"{m.tri_count} tris, {m.n_meshes} meshes, {m.n_islands} islands."]
- planned = {to_snake(p.name): p for p in getattr(plan, "parts", [])}
- for pm in m.parts[:24]:
- p = planned.get(to_snake(pm.name))
- if p is None:
- continue
- ext = glb_to_plan_frame([b - a for a, b in zip(pm.bbox_min, pm.bbox_max, strict=True)], language, extents=True)
- cen = glb_to_plan_frame([(a + b) / 2 for a, b in zip(pm.bbox_min, pm.bbox_max, strict=True)], language)
- lines.append(f"- {p.name}: measured centre ({cen[0]:.3f}, {cen[1]:.3f}, {cen[2]:.3f}) extents ({ext[0]:.3f}, {ext[1]:.3f}, {ext[2]:.3f})"
- f" | plan centre ({p.bbox.center[0]:.3f}, {p.bbox.center[1]:.3f}, {p.bbox.center[2]:.3f}) extents "
- f"({p.bbox.extents[0]:.3f}, {p.bbox.extents[1]:.3f}, {p.bbox.extents[2]:.3f})")
- gate_lines = [f"- GATE {g.gate}: {f.message}" + (f" FIX: {f.fix_hint}" if f.fix_hint else "")
- for g in last.gates for f in g.errors][:12]
- return "\n".join(lines + gate_lines)
-
-
def refine_task_lines(tasks: Sequence[RefineTask]) -> list[str]:
return [t.line() for t in tasks]
diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.py
index 94d5eb21..f2da07cb 100644
--- a/harness/codeverse/tracks/steps.py
+++ b/harness/codeverse/tracks/steps.py
@@ -18,12 +18,13 @@ from typing import Any, Protocol
from codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet
from codeverse.contracts.common import Usage
from codeverse.contracts.judgment import Judgment
-from codeverse.contracts.plan import AcceptanceItem
+from codeverse.contracts.plan import AcceptanceItem, Plan
from codeverse.contracts.run import RoundRecord
from codeverse.orchestrator.fanout import fan_out
from codeverse.tracks.common import RunContext
from codeverse.tracks.generation import GenerationResult, GenerationTask, generate
from codeverse.tracks.repair import RepairOutcome, build_with_repair
+from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
@@ -35,7 +36,10 @@ class RoundPipeline(Protocol):
def gates(self, ctx: RunContext, round_index: int, build: BuildResult, measurement: Measurement | None) -> list[GateReport]: ...
def render(self, ctx: RunContext, round_index: int, build: BuildResult, measurement: Measurement | None) -> RenderSet: ...
def plan_summary(self, ctx: RunContext) -> str: ...
- def judge_context(self, ctx: RunContext, round_index: int, build: BuildResult, gates: list[GateReport]) -> str: ...
+ def judge_context(self, ws: Workspace, plan: Plan | None, round_index: int, build: BuildResult, gates: list[GateReport]) -> str:
+ """Track-specific judge context. Deliberately ``RunContext``-free so
+ ``3dcv judge`` can rebuild the in-run context from stored artifacts."""
+ ...
class RoundFailed(RuntimeError):
@@ -190,7 +194,7 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildRes
inp = JudgeInput(
spec=ctx.spec, renders=renders, measurement=rec.measurement, gates=gates,
acceptance=list(getattr(ctx.plan, "acceptance", []) or []), plan_summary=pipeline.plan_summary(ctx),
- round_index=index, previous=previous, extra_context=pipeline.judge_context(ctx, index, build, gates),
+ round_index=index, previous=previous, extra_context=pipeline.judge_context(ctx.ws, ctx.plan, index, build, gates),
geometry_views=geometry,
)
t0 = time.time()
diff --git a/harness/runtime_js/README.md b/harness/runtime_js/README.md
index 47bb46c1..92a0ce6d 100644
--- a/harness/runtime_js/README.md
+++ b/harness/runtime_js/README.md
@@ -5,9 +5,14 @@ never imports from this directory**; it only imports `three` (and
`three/addons/...`), which the harness resolves for it.
```
-gpu_launch.cjs launchBrowser({gpu:'auto'|'on'|'off'}) → {browser, gpu, renderer}; rendererInfo(browser)
+gpu_launch.cjs launchBrowser({gpu:'auto'|'on'|'off'}) → {browser, gpu, renderer, shared, release()};
+ rendererInfo(browser). Callers must release(), never browser.close(): connect-first
+ reuse shares one browser per backend via browser_daemon.cjs (endpoint file under
+ CV3D_CACHE_DIR, connect ~5 ms vs ~0.55 s launch; CV3D_BROWSER_REUSE=off disables).
WSL2 hardware WebGL (ANGLE gl-egl + Mesa d3d12 env), UNMASKED_RENDERER probe,
negative verdict cached 20 min in ~/.cache/codeverse/gpu_probe.json, SwiftShader fallback
+browser_daemon.cjs detached keeper of the shared browser (one per gpu|cpu backend): advertises its
+ ws endpoint, reaps after ~90 s idle (endpoint-file mtime heartbeat + open-page count)
serve.cjs serveDirs({root, mounts, routes}) loopback static server (MIME table, CORS);
runtime_js is always mounted at /__runtime/; importMapHtml() maps 'three' +
'three/addons/' onto it — nothing is ever fetched from the internet
@@ -35,6 +40,19 @@ lib/syntax_check.mjs `node --input-type=module --check` per file to locate ESM
lib/browser/*.js page-side ESM (served through /__runtime/): camera_fit.js (azimuth/elevation →
tight bbox fit), studio.js (RoomEnvironment PMREM + key/fill/rim, ACES, sRGB,
shadow catcher, render modes), render_rig.js (load GLB, isolate/explode/anim, views)
+
+probe_scene.mjs scene build gate: boot src/scene.js, census, update(t,dt), first-camera checks;
+ --compile folds the full shader preflight into the SAME boot (shader_report);
+ --sun-azimuth returns harness-fitted overview + per-group camera specs (lib/orbit.mjs)
+check_shaders.mjs standalone shader preflight (static GLSL audits + GPU compile, file:line mapped)
+render_scene.mjs authored cameras + orbit rig renders at times, metrics.json/views.json
+lib/host_env.mjs ESM adapters over gpu_launch.cjs + serve.cjs (launchBrowser, serveWorkspace)
+lib/host_page.mjs node-side page driver: serve ws, launch, boot scene, collect errors, releaseBrowser
+lib/scene_host.mjs page-side host (window.__c3v): boot/renderAt/census/compileAll/fps/cameraChecks
+lib/orbit.mjs THE camera-fit owner: fitOverviewCamera (corner-frustum), fitZoneCamera (eye level),
+ fitOrbitCameras, framingBox
+lib/shader_report.mjs shared shader-preflight report builder (static + compile stages)
+lib/glsl_audit.mjs static GLSL audits + compiler-line → file:line mapping
```
Conventions (from `codeverse/conventions.py`): Y up, +Z front, meters; azimuth 0 = front,
diff --git a/harness/runtime_js/browser_daemon.cjs b/harness/runtime_js/browser_daemon.cjs
new file mode 100644
index 00000000..617c7fb2
--- /dev/null
+++ b/harness/runtime_js/browser_daemon.cjs
@@ -0,0 +1,100 @@
+#!/usr/bin/env node
+// Detached keeper of ONE shared headless-Chrome per backend ('gpu'|'cpu').
+//
+// node browser_daemon.cjs --backend gpu|cpu
+//
+// Spawned (detached, stdio ignored) by gpu_launch.cjs when no shared browser
+// is advertised. Launches the browser (same verified GPU/CPU paths as a
+// direct launch), advertises its ws endpoint in CACHE_DIR/browser_<backend>.json
+// and reaps it when idle: clients touch that file's mtime on connect/release,
+// so a stale heartbeat (~90 s) with no open pages means nobody is using it.
+// A very stale heartbeat (10 min) reaps even with leaked pages (crashed
+// client). On browser death or a superseded endpoint file the daemon exits.
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { _internal } = require('./gpu_launch.cjs');
+
+const IDLE_REAP_MS = 90 * 1000;
+const HARD_REAP_MS = 10 * 60 * 1000;
+const POLL_MS = 15 * 1000;
+
+function arg(name, dflt) {
+ const i = process.argv.indexOf(`--${name}`);
+ return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : dflt;
+}
+
+async function main() {
+ const backend = arg('backend', 'cpu');
+ if (!['gpu', 'cpu'].includes(backend)) throw new Error(`--backend must be gpu|cpu, got ${backend}`);
+ const puppeteer = _internal.loadPuppeteer();
+ const epPath = _internal.endpointPath(backend);
+ const lockPath = path.join(_internal.CACHE_DIR, `browser_${backend}.lock`);
+ fs.mkdirSync(_internal.CACHE_DIR, { recursive: true });
+
+ let browser = null;
+ let gpu = false;
+ let renderer = '';
+ try {
+ if (backend === 'gpu') {
+ const attempt = await _internal.tryGpu(puppeteer);
+ if (!attempt.browser) throw new Error(attempt.renderer);
+ browser = attempt.browser;
+ gpu = true;
+ renderer = attempt.renderer;
+ } else {
+ const h = await _internal.launchCpu(puppeteer);
+ browser = h.browser;
+ renderer = h.renderer;
+ }
+ } catch (e) {
+ fs.writeFileSync(_internal.daemonFailPath(backend), JSON.stringify({ error: String(e.message || e), at: Date.now() }));
+ fs.rmSync(lockPath, { force: true });
+ process.exit(1);
+ }
+
+ const record = { ws: browser.wsEndpoint(), pid: browser.process() ? browser.process().pid : null, gpu, renderer, daemon_pid: process.pid, at: Date.now() };
+ fs.writeFileSync(epPath, JSON.stringify(record, null, 2));
+ fs.rmSync(lockPath, { force: true });
+
+ const bail = async (code) => {
+ const cur = (() => { try { return JSON.parse(fs.readFileSync(epPath, 'utf8')); } catch (_e) { return null; } })();
+ if (cur && cur.ws === record.ws) fs.rmSync(epPath, { force: true });
+ try { await browser.close(); } catch (_e) { /* already dead */ }
+ process.exit(code);
+ };
+
+ browser.on('disconnected', () => {
+ // our own connection died with the browser process
+ try { const cur = JSON.parse(fs.readFileSync(epPath, 'utf8')); if (cur.ws === record.ws) fs.rmSync(epPath, { force: true }); } catch (_e) { /* ignore */ }
+ process.exit(0);
+ });
+
+ setInterval(async () => {
+ let st = null;
+ try {
+ st = fs.statSync(epPath);
+ } catch (_e) {
+ return bail(0); // endpoint deleted (poisoned verdict or manual cleanup)
+ }
+ const cur = (() => { try { return JSON.parse(fs.readFileSync(epPath, 'utf8')); } catch (_e) { return null; } })();
+ if (!cur || cur.ws !== record.ws) return bail(0); // superseded by another daemon
+ const idle = Date.now() - st.mtimeMs;
+ if (idle < IDLE_REAP_MS) return;
+ let pages = 2;
+ try {
+ pages = (await browser.pages()).length;
+ } catch (_e) {
+ return bail(0);
+ }
+ // >1 page = a client is mid-render (its heartbeat only ticks on connect/
+ // release); reap anyway once VERY stale — that is a crashed client's leak.
+ if (pages <= 1 || idle > HARD_REAP_MS) return bail(0);
+ }, POLL_MS);
+}
+
+main().catch((e) => {
+ process.stderr.write(String((e && e.stack) || e) + '\n');
+ process.exit(1);
+});
diff --git a/harness/runtime_js/check_shaders.mjs b/harness/runtime_js/check_shaders.mjs
index 5abe1a47..ecc99dec 100644
--- a/harness/runtime_js/check_shaders.mjs
+++ b/harness/runtime_js/check_shaders.mjs
@@ -1,6 +1,7 @@
#!/usr/bin/env node
/**
- * Shader compile preflight for a scene_threejs workspace.
+ * Shader compile preflight for a scene_threejs workspace (standalone tool;
+ * the scene build runs the same stages through `probe_scene.mjs --compile`).
*
* node check_shaders.mjs --ws <ws> [--module src/shaders/x.js] [--gpu auto] [--out report.json]
*
@@ -11,81 +12,34 @@
* 2. Compile preflight: boot the scene, force-compile every material
* (renderer.compile + one frame), collect GPU info logs and map the
* offending source line back to file:line.
+ * Both stages live in lib/shader_report.mjs.
* Output (last stdout line): {ok, errors:[{file,line,message,...}], warnings:[...]}.
* Exit 0 clean, 1 errors, 2 could not run.
*/
-import fs from 'node:fs';
import path from 'node:path';
import { armWatchdog, fail, finish, parseCli, writeJson } from './lib/cli.mjs';
-import { auditFile, fixHintFor, hasGlsl, locateSourceLine } from './lib/glsl_audit.mjs';
-import { errorSummary, openHost } from './lib/host_page.mjs';
+import { openHost } from './lib/host_page.mjs';
+import { compileIntoReport, staticShaderReport } from './lib/shader_report.mjs';
const args = parseCli({
ws: {}, module: {}, out: {}, gpu: { default: process.env.CV3D_RENDER_GPU || 'auto' },
'timeout-ms': { default: '90000' }, 'no-compile': { type: 'boolean', default: false },
});
-function listSources(root, sub = 'src') {
- const out = [];
- const walk = (dir) => {
- if (!fs.existsSync(dir)) return;
- for (const ent of fs.readdirSync(dir, { withFileTypes: true })) {
- const p = path.join(dir, ent.name);
- if (ent.isDirectory()) { if (ent.name !== 'node_modules') walk(p); continue; }
- if (/\.(m?js)$/.test(ent.name)) out.push({ file: path.relative(root, p).split(path.sep).join('/'), text: fs.readFileSync(p, 'utf8') });
- }
- };
- walk(path.join(root, sub));
- return out;
-}
-
async function main() {
if (!args.ws) throw new Error('--ws is required');
const ws = path.resolve(args.ws);
const watchdog = armWatchdog(parseInt(args['timeout-ms'], 10));
const t0 = Date.now();
- let files = listSources(ws);
- const sceneUsesFog = files.some((f) => /new\s+THREE\.(Fog|FogExp2)\b|\.fog\s*=/.test(f.text));
- const auditTargets = args.module ? files.filter((f) => f.file === args.module.replace(/^\.?\//, '')) : files;
- if (args.module && !auditTargets.length) throw new Error(`--module ${args.module} not found under ${ws}`);
- const errors = [], warnings = [];
- for (const f of auditTargets) {
- for (const fd of auditFile(f.file, f.text, { sceneUsesFog })) {
- (fd.severity === 'error' ? errors : warnings).push({ file: fd.file, line: fd.line, kind: fd.kind, message: fd.message, fix_hint: fd.severity === 'error' ? fd.message : '' });
- }
- }
- const report = { ok: false, ws, module: args.module || null, static: { files: auditTargets.length, glsl_files: auditTargets.filter((f) => hasGlsl(f.text)).length }, compile: null, errors, warnings };
+ const { report, files } = staticShaderReport(ws, args.module || null);
+ const { errors, warnings } = report;
if (!args['no-compile']) {
let host = null;
try {
host = await openHost(ws, { width: 256, height: 144, gpu: args.gpu });
- const { page, boot } = host;
- if (!boot.ok) {
- errors.push({ file: 'src/scene.js', line: null, kind: 'boot', message: `scene did not boot at stage '${boot.stage}': ${boot.error}`.slice(0, 1200), fix_hint: 'fix the import/runtime error first; then re-run check_shaders' });
- } else {
- const comp = await page.evaluate((spec) => window.__c3v.compileAll(spec || undefined), null);
- const shaderErrors = comp.shader_errors.map(({ _key, ...e }) => e);
- for (const e of shaderErrors) {
- const loc = locateSourceLine(files, e.source_line);
- errors.push({
- file: loc.file || (e.material ? `(material ${e.material})` : '(assembled shader)'),
- line: loc.line, kind: 'compile', stage: e.stage, material: e.material || '', material_type: e.material_type || '',
- message: `${e.stage} shader: ${e.message}` + (e.source_line ? ` — at: ${e.source_line}` : '') + (loc.candidates > 1 ? ` (${loc.candidates} identical lines; first shown)` : ''),
- assembled_line: e.assembled_line, source_line: e.source_line, context: e.context, fix_hint: fixHintFor(e.message),
- });
- }
- for (const a of comp.material_audit) {
- (a.severity === 'error' ? errors : warnings).push({ file: '(runtime material)', line: null, kind: a.kind, material: a.material, message: a.message, fix_hint: a.severity === 'error' ? a.message : '' });
- }
- report.compile = { ms: comp.compile_ms, programs: comp.programs, custom_materials: comp.custom_materials, renderer: boot.renderer, gpu: host.gpu };
- }
- const es = errorSummary(host.errors, boot);
- for (const c of es.shader_console) {
- if (!errors.some((e) => e.kind === 'compile')) errors.push({ file: '(console)', line: null, kind: 'shader_console', message: c, fix_hint: 'search your GLSL for the quoted ERROR line' });
- }
- for (const c of es.console_errors) if (!boot.error || !c.startsWith('boot[')) warnings.push({ file: '(console)', line: null, kind: 'console', message: c });
+ await compileIntoReport(report, files, host);
await host.close();
} catch (e) {
if (host) await host.close().catch(() => {});
diff --git a/harness/runtime_js/gpu_launch.cjs b/harness/runtime_js/gpu_launch.cjs
index 77aa0b0c..315e6f03 100644
--- a/harness/runtime_js/gpu_launch.cjs
+++ b/harness/runtime_js/gpu_launch.cjs
@@ -7,11 +7,23 @@
// is not repeated on every render.
//
// const { launchBrowser, rendererInfo } = require('./gpu_launch.cjs');
-// const { browser, gpu, renderer } = await launchBrowser({ gpu: 'auto' });
+// const handle = await launchBrowser({ gpu: 'auto' });
+// ... handle.browser / handle.gpu / handle.renderer ...
+// await handle.release(); // NOT browser.close(): the browser may be shared
//
// gpu: 'auto' (probe, cached) | 'on' (require hardware; throw otherwise) | 'off' (CPU).
+//
+// Browser reuse (connect-first): a launch costs ~0.55 s, a puppeteer.connect
+// ~5 ms. A detached `browser_daemon.cjs` owns one long-lived browser per
+// backend ('gpu'|'cpu') and advertises its ws endpoint in a file under
+// CACHE_DIR; launchBrowser connects to it when present, spawns the daemon when
+// absent, and falls back to a plain owned launch when the daemon cannot help.
+// `release()` disconnects from a shared browser (the daemon reaps it after
+// ~90 s idle) and closes an owned one. `CV3D_BROWSER_REUSE=off` disables
+// sharing entirely.
'use strict';
+const { spawn } = require('child_process');
const fs = require('fs');
const os = require('os');
const path = require('path');
@@ -20,6 +32,8 @@ const CACHE_DIR = process.env.CV3D_CACHE_DIR || path.join(os.homedir(), '.cache'
const CACHE_PATH = path.join(CACHE_DIR, 'gpu_probe.json');
const NEGATIVE_TTL_MS = 20 * 60 * 1000; // how long a "no GPU" verdict is trusted
const PROTOCOL_TIMEOUT_MS = 15 * 60 * 1000; // big frame batches exceed puppeteer's 180 s default
+const DAEMON_WAIT_MS = 25000; // max wait for a spawned daemon to advertise its endpoint
+const SPAWN_LOCK_STALE_MS = 30000;
const BASE_ARGS = ['--no-sandbox', '--disable-setuid-sandbox', '--enable-webgl', '--ignore-gpu-blocklist'];
const GPU_ARGS = [...BASE_ARGS, '--enable-gpu-rasterization', '--use-angle=gl-egl'];
@@ -72,6 +86,10 @@ function loadPuppeteer(puppeteer) {
return puppeteer || require('puppeteer');
}
+function owned(browser, gpu, renderer) {
+ return { browser, gpu, renderer, shared: false, release: () => browser.close().catch(() => {}) };
+}
+
async function launchCpu(puppeteer) {
const env = { ...process.env };
for (const k of Object.keys(GPU_ENV)) delete env[k];
@@ -87,7 +105,7 @@ async function launchCpu(puppeteer) {
} catch (_e) {
/* keep the default label */
}
- return { browser, gpu: false, renderer };
+ return owned(browser, false, renderer);
}
async function tryGpu(puppeteer) {
@@ -111,37 +129,154 @@ async function tryGpu(puppeteer) {
}
}
+// --------------------------------------------------------------------- sharing
+function endpointPath(backend) {
+ return path.join(CACHE_DIR, `browser_${backend}.json`);
+}
+
+function daemonFailPath(backend) {
+ return path.join(CACHE_DIR, `browser_${backend}.failed.json`);
+}
+
+function reuseEnabled() {
+ return !/^(off|0|false)$/i.test(process.env.CV3D_BROWSER_REUSE || '');
+}
+
+function readJson(p) {
+ try {
+ return JSON.parse(fs.readFileSync(p, 'utf8'));
+ } catch (_e) {
+ return null;
+ }
+}
+
+/** Touch the endpoint file: its mtime is the daemon's idle-reap heartbeat. */
+function heartbeat(backend) {
+ try {
+ const now = new Date();
+ fs.utimesSync(endpointPath(backend), now, now);
+ } catch (_e) { /* endpoint may be gone; connect error handling covers it */ }
+}
+
+/** Connect to the shared browser advertised for `backend`; null when absent/poisoned. */
+async function connectShared(puppeteer, backend) {
+ const info = readJson(endpointPath(backend));
+ if (!info || !info.ws) return null;
+ try {
+ const browser = await puppeteer.connect({ browserWSEndpoint: info.ws, protocolTimeout: PROTOCOL_TIMEOUT_MS });
+ heartbeat(backend);
+ return {
+ browser,
+ gpu: !!info.gpu,
+ renderer: String(info.renderer || ''),
+ shared: true,
+ release: async () => { heartbeat(backend); try { await browser.disconnect(); } catch (_e) { /* already gone */ } },
+ };
+ } catch (_e) {
+ // poisoned: the advertised browser is gone/unresponsive — forget it
+ try { fs.rmSync(endpointPath(backend), { force: true }); } catch (_e2) { /* ignore */ }
+ return null;
+ }
+}
+
+/** Spawn the detached daemon for `backend` (single flight via a lock file). */
+function spawnDaemon(backend) {
+ fs.mkdirSync(CACHE_DIR, { recursive: true });
+ const lock = path.join(CACHE_DIR, `browser_${backend}.lock`);
+ try {
+ const st = fs.statSync(lock);
+ if (Date.now() - st.mtimeMs < SPAWN_LOCK_STALE_MS) return; // someone else is spawning
+ fs.rmSync(lock, { force: true });
+ } catch (_e) { /* no lock */ }
+ try {
+ fs.writeFileSync(lock, String(process.pid), { flag: 'wx' });
+ } catch (_e) {
+ return; // lost the race
+ }
+ try { fs.rmSync(daemonFailPath(backend), { force: true }); } catch (_e) { /* ignore */ }
+ const child = spawn(process.execPath, [path.join(__dirname, 'browser_daemon.cjs'), '--backend', backend],
+ { detached: true, stdio: 'ignore', env: process.env });
+ child.unref();
+}
+
/**
- * Launch headless Chrome on the best available WebGL backend.
+ * Get a shared browser for `backend` ('gpu'|'cpu'): connect to the advertised
+ * endpoint, spawning the daemon first when there is none.
+ * @returns handle, or {failed: string} when the daemon reported a launch
+ * failure (meaningful for 'gpu'), or null on timeout/no daemon.
+ */
+async function sharedBrowser(puppeteer, backend) {
+ let handle = await connectShared(puppeteer, backend);
+ if (handle) return handle;
+ spawnDaemon(backend);
+ const deadline = Date.now() + DAEMON_WAIT_MS;
+ while (Date.now() < deadline) {
+ await new Promise((r) => setTimeout(r, 50));
+ handle = await connectShared(puppeteer, backend);
+ if (handle) return handle;
+ const failed = readJson(daemonFailPath(backend));
+ if (failed) return { failed: String(failed.error || 'daemon launch failed') };
+ }
+ return null;
+}
+
+async function cpuBrowser(puppeteer) {
+ if (reuseEnabled()) {
+ const h = await sharedBrowser(puppeteer, 'cpu');
+ if (h && h.browser) return h;
+ }
+ return launchCpu(puppeteer);
+}
+
+/** GPU browser or {browser:null, renderer:<why>} — shared first, owned fallback. */
+async function gpuBrowser(puppeteer) {
+ if (reuseEnabled()) {
+ const h = await sharedBrowser(puppeteer, 'gpu');
+ if (h && h.browser) return h;
+ if (h && h.failed) return { browser: null, renderer: h.failed };
+ // daemon never answered: fall through to an owned launch
+ }
+ const attempt = await tryGpu(puppeteer);
+ if (!attempt.browser) return { browser: null, renderer: attempt.renderer };
+ return owned(attempt.browser, true, attempt.renderer);
+}
+
+/**
+ * Launch (or connect to) headless Chrome on the best available WebGL backend.
* @param {{gpu?: 'auto'|'on'|'off', puppeteer?: object}} opts
- * @returns {Promise<{browser: object, gpu: boolean, renderer: string}>}
+ * @returns {Promise<{browser: object, gpu: boolean, renderer: string, shared: boolean, release: () => Promise<void>}>}
*/
async function launchBrowser(opts = {}) {
const mode = String(opts.gpu || process.env.CV3D_RENDER_GPU || 'auto').toLowerCase();
const puppeteer = loadPuppeteer(opts.puppeteer);
if (!['auto', 'on', 'off'].includes(mode)) throw new Error(`gpu must be auto|on|off, got ${mode}`);
- if (mode === 'off') return launchCpu(puppeteer);
+ if (mode === 'off') return cpuBrowser(puppeteer);
if (mode === 'on') {
- const attempt = await tryGpu(puppeteer);
+ const attempt = await gpuBrowser(puppeteer);
if (!attempt.browser) throw new Error(`gpu=on but no hardware WebGL: ${attempt.renderer}`);
- return { browser: attempt.browser, gpu: true, renderer: attempt.renderer };
+ return attempt;
}
// auto: a fresh negative verdict skips the slow GPU attempt; a positive one is
- // always re-verified live (cheap, and the GPU can go away).
+ // re-verified by the GPU attempt itself (shared endpoint or launch), since the
+ // GPU can go away.
const cached = readCache();
const fresh = cached && typeof cached.at === 'number' && Date.now() - cached.at < NEGATIVE_TTL_MS;
- if (cached && cached.use_gpu === false && fresh) return launchCpu(puppeteer);
+ if (cached && cached.use_gpu === false && fresh) return cpuBrowser(puppeteer);
- const attempt = await tryGpu(puppeteer);
+ const attempt = await gpuBrowser(puppeteer);
if (attempt.browser) {
writeCache({ use_gpu: true, renderer: attempt.renderer });
- return { browser: attempt.browser, gpu: true, renderer: attempt.renderer };
+ return attempt;
}
writeCache({ use_gpu: false, renderer: attempt.renderer });
- return launchCpu(puppeteer);
+ return cpuBrowser(puppeteer);
}
-module.exports = { launchBrowser, rendererInfo, GPU_ARGS, CPU_ARGS, GPU_ENV, CACHE_PATH };
+module.exports = {
+ launchBrowser, rendererInfo, GPU_ARGS, CPU_ARGS, GPU_ENV, CACHE_PATH,
+ // internals shared with browser_daemon.cjs (not a public surface)
+ _internal: { launchCpu, tryGpu, loadPuppeteer, endpointPath, daemonFailPath, CACHE_DIR, SOFTWARE_RE },
+};
diff --git a/harness/runtime_js/lib/_compat.mjs b/harness/runtime_js/lib/_compat.mjs
deleted file mode 100644
index 6cf5108e..00000000
--- a/harness/runtime_js/lib/_compat.mjs
+++ /dev/null
@@ -1,186 +0,0 @@
-/**
- * Thin node-side fallbacks for what package C1 provides
- * (`runtime_js/gpu_launch.cjs` → launchBrowser, `runtime_js/serve.cjs` →
- * static server with import-map injection). Each helper first tries the C1
- * module and uses it when it exposes the expected function; otherwise a local
- * implementation with the same behaviour is used so this package runs alone.
- */
-
-import fs from 'node:fs';
-import http from 'node:http';
-import path from 'node:path';
-import { createRequire } from 'node:module';
-import { fileURLToPath } from 'node:url';
-
-const require_ = createRequire(import.meta.url);
-export const RUNTIME_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-
-const BASE_ARGS = ['--no-sandbox', '--disable-setuid-sandbox', '--enable-webgl', '--ignore-gpu-blocklist'];
-const GPU_ARGS = [...BASE_ARGS, '--enable-gpu-rasterization', '--use-angle=gl-egl'];
-const CPU_ARGS = [...BASE_ARGS, '--enable-unsafe-swiftshader', '--use-angle=swiftshader'];
-const GPU_ENV = {
- GALLIUM_DRIVER: 'd3d12',
- MESA_LOADER_DRIVER_OVERRIDE: 'd3d12',
- MESA_D3D12_DEFAULT_ADAPTER_NAME: 'NVIDIA',
-};
-const SOFTWARE_RE = /swiftshader|llvmpipe|softpipe|software|basic render/i;
-
-function tryRequire(rel) {
- try {
- return require_(path.join(RUNTIME_ROOT, rel));
- } catch (e) {
- if (e && e.code === 'MODULE_NOT_FOUND' && String(e.message).includes(rel.replace('./', ''))) return null;
- throw e;
- }
-}
-
-async function probeRenderer(browser) {
- const page = await browser.newPage();
- try {
- return await page.evaluate(() => {
- const c = document.createElement('canvas');
- const gl = c.getContext('webgl2') || c.getContext('webgl');
- if (!gl) return null;
- const ext = gl.getExtension('WEBGL_debug_renderer_info');
- return String(ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : gl.getParameter(gl.RENDERER));
- });
- } finally {
- await page.close();
- }
-}
-
-async function launchWith(puppeteer, args, env) {
- const saved = {};
- for (const k of Object.keys(GPU_ENV)) {
- saved[k] = process.env[k];
- if (env[k] !== undefined) process.env[k] = env[k];
- else delete process.env[k];
- }
- try {
- return await puppeteer.launch({ headless: true, args, protocolTimeout: 600000 });
- } finally {
- for (const k of Object.keys(saved)) {
- if (saved[k] === undefined) delete process.env[k];
- else process.env[k] = saved[k];
- }
- }
-}
-
-/**
- * Local launcher: GPU attempt (WSL2 d3d12 + gl-egl) verified via the
- * unmasked renderer string, else SwiftShader CPU.
- * @param {{gpu?: 'auto'|'on'|'off'}} opts
- * @returns {Promise<{browser: object, gpu: boolean, renderer: string}>}
- */
-async function localLaunchBrowser({ gpu = 'auto' } = {}) {
- const puppeteer = require_('puppeteer');
- if (gpu !== 'off') {
- let browser = null;
- try {
- browser = await launchWith(puppeteer, GPU_ARGS, GPU_ENV);
- const renderer = await probeRenderer(browser);
- if (renderer && !SOFTWARE_RE.test(renderer)) return { browser, gpu: true, renderer };
- await browser.close().catch(() => {});
- if (gpu === 'on') throw new Error(`gpu=on but WebGL renderer is software/none: ${renderer}`);
- } catch (e) {
- if (browser) await browser.close().catch(() => {});
- if (gpu === 'on') throw e;
- }
- }
- const browser = await launchWith(puppeteer, CPU_ARGS, {});
- const renderer = (await probeRenderer(browser)) || 'SwiftShader (CPU)';
- return { browser, gpu: false, renderer };
-}
-
-/** Launch a headless browser; C1's gpu_launch.cjs when present. */
-export async function launchBrowser(opts = {}) {
- const mod = tryRequire('./gpu_launch.cjs');
- if (mod && typeof mod.launchBrowser === 'function') {
- const res = await mod.launchBrowser(opts);
- if (res && res.browser) return { browser: res.browser, gpu: !!res.gpu, renderer: String(res.renderer || '') };
- }
- return localLaunchBrowser(opts);
-}
-
-const MIME = {
- '.js': 'text/javascript', '.mjs': 'text/javascript', '.cjs': 'text/javascript', '.json': 'application/json',
- '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp', '.hdr': 'application/octet-stream',
- '.glb': 'model/gltf-binary', '.gltf': 'model/gltf+json', '.html': 'text/html', '.css': 'text/css', '.wasm': 'application/wasm',
- '.ktx2': 'image/ktx2', '.svg': 'image/svg+xml', '.mp3': 'audio/mpeg', '.ogg': 'audio/ogg', '.txt': 'text/plain',
-};
-
-const LOCAL_MOUNT = '/__rt/';
-
-/** URL prefix under which runtime_js is served ('/__runtime/' with C1's serve.cjs). */
-export function runtimeMount() {
- const mod = tryRequire('./serve.cjs');
- return mod && typeof mod.RUNTIME_MOUNT === 'string' ? mod.RUNTIME_MOUNT : LOCAL_MOUNT;
-}
-
-/** Import map that resolves 'three' and 'three/addons/*' to the local runtime copy. */
-export function importMapHtml() {
- const mod = tryRequire('./serve.cjs');
- if (mod && typeof mod.importMapHtml === 'function') return mod.importMapHtml();
- const m = LOCAL_MOUNT;
- const map = {
- imports: {
- three: `${m}node_modules/three/build/three.module.js`,
- 'three/addons/': `${m}node_modules/three/examples/jsm/`,
- 'three/examples/jsm/': `${m}node_modules/three/examples/jsm/`,
- },
- };
- return `<script type="importmap">${JSON.stringify(map)}</script>`;
-}
-
-function safeJoin(root, rel) {
- const abs = path.resolve(root, '.' + rel);
- if (abs !== root && !abs.startsWith(root + path.sep)) return null;
- return abs;
-}
-
-/**
- * Serve a workspace on 127.0.0.1:0 with these mounts:
- * /src/** /public/** → <ws>/src, <ws>/public
- * /assets/** → <ws>/public/assets (contract: '/assets/x.glb')
- * <runtimeMount()>/** → runtime_js (node_modules/three, lib/*.mjs)
- * /__host.html → generated host page (import map + scene_host.mjs)
- * Uses C1's serve.cjs when present (same mounts); returns { server, base, close() }.
- */
-export async function serveWorkspace(wsRoot, { runtimeRoot = RUNTIME_ROOT, hostHtml = '' } = {}) {
- wsRoot = path.resolve(wsRoot);
- runtimeRoot = path.resolve(runtimeRoot);
- const c1 = tryRequire('./serve.cjs');
- if (c1 && typeof c1.serveDirs === 'function') {
- const srv = await c1.serveDirs({
- root: wsRoot,
- mounts: { '/assets/': path.join(wsRoot, 'public', 'assets') },
- routes: { '/__host.html': { body: hostHtml, type: 'text/html; charset=utf-8' } },
- });
- return { server: srv.server, base: srv.base, close: srv.close };
- }
- const server = http.createServer((req, res) => {
- const rel = decodeURIComponent((req.url || '/').split('?')[0]);
- res.setHeader('Access-Control-Allow-Origin', '*');
- res.setHeader('Cache-Control', 'no-store');
- if (rel === '/favicon.ico') { res.statusCode = 204; res.end(); return; }
- if (rel === '/__host.html') {
- res.setHeader('Content-Type', 'text/html');
- res.end(hostHtml);
- return;
- }
- let abs = null;
- if (rel.startsWith(LOCAL_MOUNT)) abs = safeJoin(runtimeRoot, rel.slice(LOCAL_MOUNT.length - 1));
- else if (rel.startsWith('/assets/')) abs = safeJoin(path.join(wsRoot, 'public'), rel);
- else abs = safeJoin(wsRoot, rel);
- if (!abs || !fs.existsSync(abs) || !fs.statSync(abs).isFile()) {
- res.statusCode = 404;
- res.end('not found: ' + rel);
- return;
- }
- res.setHeader('Content-Type', MIME[path.extname(abs).toLowerCase()] || 'application/octet-stream');
- fs.createReadStream(abs).pipe(res);
- });
- await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
- const base = `http://127.0.0.1:${server.address().port}`;
- return { server, base, close: () => new Promise((r) => server.close(() => r())) };
-}
diff --git a/harness/runtime_js/lib/host_env.mjs b/harness/runtime_js/lib/host_env.mjs
new file mode 100644
index 00000000..c64a3502
--- /dev/null
+++ b/harness/runtime_js/lib/host_env.mjs
@@ -0,0 +1,53 @@
+/**
+ * Node-side environment for the scene host drivers: browser launch and
+ * workspace serving. Thin ESM adapters over the two CJS runtime modules —
+ * `gpu_launch.cjs` (GPU-verified headless Chrome) and `serve.cjs` (loopback
+ * static server + import map). Single consumer: `lib/host_page.mjs`.
+ */
+
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+
+const require_ = createRequire(import.meta.url);
+export const RUNTIME_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+const gpuLaunch = require_(path.join(RUNTIME_ROOT, 'gpu_launch.cjs'));
+const serve = require_(path.join(RUNTIME_ROOT, 'serve.cjs'));
+
+/**
+ * Launch (or connect to) a headless browser; see gpu_launch.cjs.
+ * @param {{gpu?: 'auto'|'on'|'off'}} opts
+ * @returns {Promise<{browser: object, gpu: boolean, renderer: string, release: () => Promise<void>}>}
+ */
+export async function launchBrowser(opts = {}) {
+ return gpuLaunch.launchBrowser(opts);
+}
+
+/** URL prefix under which runtime_js is served ('/__runtime/'). */
+export function runtimeMount() {
+ return serve.RUNTIME_MOUNT;
+}
+
+/** Import map that resolves 'three' and 'three/addons/*' to the local runtime copy. */
+export function importMapHtml() {
+ return serve.importMapHtml();
+}
+
+/**
+ * Serve a workspace on 127.0.0.1:0 with these mounts:
+ * /src/** /public/** → <ws>/src, <ws>/public
+ * /assets/** → <ws>/public/assets (contract: '/assets/x.glb')
+ * <runtimeMount()>/** → runtime_js (node_modules/three, lib/*.mjs)
+ * /__host.html → generated host page (import map + scene_host.mjs)
+ * @returns {Promise<{server, base: string, close: () => Promise<void>}>}
+ */
+export async function serveWorkspace(wsRoot, { hostHtml = '' } = {}) {
+ wsRoot = path.resolve(wsRoot);
+ const srv = await serve.serveDirs({
+ root: wsRoot,
+ mounts: { '/assets/': path.join(wsRoot, 'public', 'assets') },
+ routes: { '/__host.html': { body: hostHtml, type: 'text/html; charset=utf-8' } },
+ });
+ return { server: srv.server, base: srv.base, close: srv.close };
+}
diff --git a/harness/runtime_js/lib/host_page.mjs b/harness/runtime_js/lib/host_page.mjs
index 9b60ec30..096db40f 100644
--- a/harness/runtime_js/lib/host_page.mjs
+++ b/harness/runtime_js/lib/host_page.mjs
@@ -6,7 +6,7 @@
import fs from 'node:fs';
import path from 'node:path';
-import { importMapHtml, launchBrowser, runtimeMount, serveWorkspace } from './_compat.mjs';
+import { importMapHtml, launchBrowser, runtimeMount, serveWorkspace } from './host_env.mjs';
const SHADER_NOISE_RE = /shader|program not valid|glsl|WebGL|compile|THREE\.WebGLProgram/i;
const MAX_CONSOLE = 60;
@@ -63,9 +63,10 @@ export async function openHost(wsRoot, { width = 1024, height = 576, gpu = 'auto
const srv = await serveWorkspace(wsRoot, { hostHtml: hostHtml() });
const errors = { console: [], page: [], shader_console: [], warnings: [] };
let launched = null;
+ let page = null;
try {
launched = await launchBrowser({ gpu });
- const page = await launched.browser.newPage();
+ page = await launched.browser.newPage();
await page.setViewport({ width, height, deviceScaleFactor: 1 });
page.on('console', (m) => {
const type = m.type();
@@ -103,17 +104,28 @@ export async function openHost(wsRoot, { width = 1024, height = 576, gpu = 'auto
{ sceneUrl: `/${sceneRel}`, width, height, logDepth, createSceneTimeoutMs },
);
const close = async () => {
- try { await launched.browser.close(); } catch (e) { /* ignore */ }
+ // page first, then release: a SHARED browser (connect-first reuse) must get
+ // its pages closed by us and be disconnected, never closed.
+ try { await page.close(); } catch (e) { /* ignore */ }
+ try { await releaseBrowser(launched); } catch (e) { /* ignore */ }
await srv.close();
};
return { page, browser: launched.browser, base: srv.base, gpu: launched.gpu, renderer: launched.renderer, errors, boot, close };
} catch (e) {
- if (launched && launched.browser) await launched.browser.close().catch(() => {});
+ if (page) await page.close().catch(() => {});
+ if (launched && launched.browser) await releaseBrowser(launched).catch(() => {});
await srv.close();
throw e;
}
}
+/** Release a launchBrowser handle: release() when present (shared-aware), else close(). */
+export async function releaseBrowser(launched) {
+ if (!launched || !launched.browser) return;
+ if (typeof launched.release === 'function') await launched.release();
+ else await launched.browser.close().catch(() => {});
+}
+
/** Flatten the collected errors into the summary shape used by all drivers. */
export function errorSummary(errors, boot) {
const consoleErrors = [...errors.console, ...errors.page.map((p) => `pageerror: ${p}`)];
diff --git a/harness/runtime_js/lib/orbit.mjs b/harness/runtime_js/lib/orbit.mjs
index cc86c1ae..77c1f7e8 100644
--- a/harness/runtime_js/lib/orbit.mjs
+++ b/harness/runtime_js/lib/orbit.mjs
@@ -87,45 +87,74 @@ export function fitDistance(bbox, center, dir, tanH, tanV, margin) {
return dist;
}
+function boxGeom(bbox, groundY) {
+ const size = bbox.max.map((v, i) => v - bbox.min[i]);
+ const center = bbox.min.map((v, i) => v + size[i] / 2);
+ const floor = groundY === null || groundY === undefined ? bbox.min[1] : groundY;
+ return { size, center, floor };
+}
+
+function round3(spec) {
+ spec.position = spec.position.map((x) => +x.toFixed(3));
+ spec.lookAt = spec.lookAt.map((x) => +x.toFixed(3));
+ return spec;
+}
+
+/**
+ * Eye-level camera standing just outside the bbox footprint on the given
+ * azimuth side, ~1.6 m above ground, looking into the content. THE owner of
+ * the zone-camera fit (assemble.py consumes it through probe_scene.mjs).
+ * @returns {{position:number[], lookAt:number[], fov:number}}
+ */
+export function fitZoneCamera(bbox, { azimuth = 45, floor = null, fov = 50 } = {}) {
+ const g = boxGeom(bbox, floor);
+ const [dx, , dz] = orbitDirection(azimuth, 0);
+ // distance from the centre to the bbox's xz boundary along (dx, dz)
+ const hx = g.size[0] / 2, hz = g.size[2] / 2;
+ const tx = Math.abs(dx) > 1e-6 ? hx / Math.abs(dx) : Infinity;
+ const tz = Math.abs(dz) > 1e-6 ? hz / Math.abs(dz) : Infinity;
+ const back = Math.min(tx, tz) + Math.max(2, 0.12 * Math.max(g.size[0], g.size[2]));
+ const eyeH = Math.max(g.floor + 0.5, g.floor + 1.6 + Math.min(4, 0.1 * g.size[1]));
+ return round3({
+ position: [g.center[0] + dx * back, eyeH, g.center[2] + dz * back],
+ lookAt: [g.center[0], Math.min(g.center[1], g.floor + 0.35 * g.size[1] + 0.5), g.center[2]],
+ fov,
+ });
+}
+
/**
- * Build camera specs for `views` around `bbox` ({min:[..], max:[..]}).
+ * Overview camera on the given azimuth side, fitted so the bbox's eight
+ * corners fill the frame (exact per-corner frustum fit, ground-aware — never
+ * a bounding-sphere fit). THE owner of the overview fit.
+ * @returns {{position:number[], lookAt:number[], fov:number}}
+ */
+export function fitOverviewCamera(bbox, { azimuth = 45, elevation = 30, fov = 50, aspect = 16 / 9, groundY = null, margin = 1.05 } = {}) {
+ const g = boxGeom(bbox, groundY);
+ const el = Math.max(3, Math.min(89, elevation));
+ const d = orbitDirection(azimuth, el);
+ const tanV = Math.tan((fov * DEG) / 2);
+ const tanH = tanV * aspect;
+ const minDist = Math.max(1.5, 0.35 * Math.hypot(g.size[0], g.size[1], g.size[2]));
+ const dist = Math.max(minDist, fitDistance(bbox, g.center, d, tanH, tanV, margin));
+ const eye = g.center.map((c, i) => c + d[i] * dist);
+ if (eye[1] < g.floor + 0.5) eye[1] = g.floor + 0.5;
+ return round3({ position: eye, lookAt: g.center.slice(), fov });
+}
+
+/**
+ * Build camera specs for `views` around `bbox` ({min:[..], max:[..]}):
+ * fitOverviewCamera above 20°, fitZoneCamera at eye level.
* @returns {Array<{name, position, lookAt, fov, kind:'orbit'}>}
*/
export function fitOrbitCameras(bbox, views, { fov = 50, aspect = 16 / 9, groundY = null, margin = 1.05, noFog = true } = {}) {
if (!bbox || !bbox.min || !bbox.max) return [];
- const size = bbox.max.map((v, i) => v - bbox.min[i]);
- const center = bbox.min.map((v, i) => v + size[i] / 2);
- const tanV = Math.tan((fov * DEG) / 2);
- const tanH = tanV * aspect;
- const minDist = Math.max(1.5, 0.35 * Math.hypot(size[0], size[1], size[2]));
- const floor = groundY === null || groundY === undefined ? bbox.min[1] : groundY;
const out = [];
- const horizDist = (dx, dz) => {
- // distance from the centre to the bbox's xz boundary along (dx, dz)
- const hx = size[0] / 2, hz = size[2] / 2;
- const tx = Math.abs(dx) > 1e-6 ? hx / Math.abs(dx) : Infinity;
- const tz = Math.abs(dz) > 1e-6 ? hz / Math.abs(dz) : Infinity;
- return Math.min(tx, tz);
- };
for (const v of views) {
const el = Math.max(3, Math.min(89, v.elevation));
- const d = orbitDirection(v.azimuth, el);
- let eye, lookAt;
- if (el <= 20) {
- // eye-level: a person standing just outside the content footprint, looking in
- const dh = orbitDirection(v.azimuth, 0);
- const edge = horizDist(dh[0], dh[2]);
- const back = edge + Math.max(2, 0.12 * Math.max(size[0], size[2]));
- const eyeH = floor + 1.6 + Math.min(4, 0.1 * size[1]);
- eye = [center[0] + dh[0] * back, eyeH, center[2] + dh[2] * back];
- lookAt = [center[0], Math.min(center[1], floor + 0.35 * size[1] + 0.5), center[2]];
- } else {
- const dist = Math.max(minDist, fitDistance(bbox, center, d, tanH, tanV, margin));
- eye = center.map((c, i) => c + d[i] * dist);
- lookAt = center.slice();
- }
- if (eye[1] < floor + 0.5) eye[1] = floor + 0.5;
- out.push({ name: v.name, position: eye.map((x) => +x.toFixed(3)), lookAt: lookAt.map((x) => +x.toFixed(3)), fov, kind: 'orbit', noFog: !!noFog && el > 20 });
+ const fit = el <= 20
+ ? fitZoneCamera(bbox, { azimuth: v.azimuth, floor: groundY, fov })
+ : fitOverviewCamera(bbox, { azimuth: v.azimuth, elevation: el, fov, aspect, groundY, margin });
+ out.push({ name: v.name, ...fit, kind: 'orbit', noFog: !!noFog && el > 20 });
}
return out;
}
diff --git a/harness/runtime_js/lib/shader_report.mjs b/harness/runtime_js/lib/shader_report.mjs
new file mode 100644
index 00000000..b6da4c48
--- /dev/null
+++ b/harness/runtime_js/lib/shader_report.mjs
@@ -0,0 +1,95 @@
+/**
+ * Shader preflight report builder, shared by the standalone tool
+ * (`check_shaders.mjs`) and the combined build probe (`probe_scene.mjs
+ * --compile`). Two node-side stages over one report shape:
+ *
+ * staticShaderReport(ws, module) — walk src/**, run the static GLSL audits
+ * compileIntoReport(report, ...) — on an already-booted host page, force-
+ * compile every material and fold GPU errors (file:line mapped), the
+ * runtime material audit and console noise into the same report.
+ *
+ * Report shape (identical to the historical check_shaders.mjs output):
+ * {ok, ws, module, static:{files, glsl_files}, compile, errors:[...], warnings:[...]}
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { auditFile, fixHintFor, hasGlsl, locateSourceLine } from './glsl_audit.mjs';
+import { errorSummary } from './host_page.mjs';
+
+/** All .js/.mjs sources under <root>/<sub> as [{file, text}] (posix rel paths). */
+export function listSources(root, sub = 'src') {
+ const out = [];
+ const walk = (dir) => {
+ if (!fs.existsSync(dir)) return;
+ for (const ent of fs.readdirSync(dir, { withFileTypes: true })) {
+ const p = path.join(dir, ent.name);
+ if (ent.isDirectory()) { if (ent.name !== 'node_modules') walk(p); continue; }
+ if (/\.(m?js)$/.test(ent.name)) out.push({ file: path.relative(root, p).split(path.sep).join('/'), text: fs.readFileSync(p, 'utf8') });
+ }
+ };
+ walk(path.join(root, sub));
+ return out;
+}
+
+/**
+ * Static-audit stage: build the report skeleton with the node-side GLSL
+ * findings. `module` (optional) restricts the audit to one source file.
+ * @returns {{report: object, files: Array<{file, text}>}}
+ */
+export function staticShaderReport(ws, module = null) {
+ const files = listSources(ws);
+ const sceneUsesFog = files.some((f) => /new\s+THREE\.(Fog|FogExp2)\b|\.fog\s*=/.test(f.text));
+ const auditTargets = module ? files.filter((f) => f.file === module.replace(/^\.?\//, '')) : files;
+ if (module && !auditTargets.length) throw new Error(`--module ${module} not found under ${ws}`);
+ const errors = [], warnings = [];
+ for (const f of auditTargets) {
+ for (const fd of auditFile(f.file, f.text, { sceneUsesFog })) {
+ (fd.severity === 'error' ? errors : warnings).push({ file: fd.file, line: fd.line, kind: fd.kind, message: fd.message, fix_hint: fd.severity === 'error' ? fd.message : '' });
+ }
+ }
+ const report = {
+ ok: false, ws, module: module || null,
+ static: { files: auditTargets.length, glsl_files: auditTargets.filter((f) => hasGlsl(f.text)).length },
+ compile: null, errors, warnings,
+ };
+ return { report, files };
+}
+
+/**
+ * Compile stage on an already-booted host page: force-compile every material
+ * (`window.__c3v.compileAll`), map compiler errors back to file:line, audit
+ * runtime materials, fold console noise. Mutates and returns `report`.
+ * @param {object} report from staticShaderReport
+ * @param {Array} files from staticShaderReport
+ * @param {{page, boot, gpu, errors}} host openHost result (or equivalent)
+ */
+export async function compileIntoReport(report, files, host) {
+ const { page, boot } = host;
+ const { errors, warnings } = report;
+ if (!boot.ok) {
+ errors.push({ file: 'src/scene.js', line: null, kind: 'boot', message: `scene did not boot at stage '${boot.stage}': ${boot.error}`.slice(0, 1200), fix_hint: 'fix the import/runtime error first; then re-run check_shaders' });
+ } else {
+ const comp = await page.evaluate((spec) => window.__c3v.compileAll(spec || undefined), null);
+ const shaderErrors = comp.shader_errors.map(({ _key, ...e }) => e);
+ for (const e of shaderErrors) {
+ const loc = locateSourceLine(files, e.source_line);
+ errors.push({
+ file: loc.file || (e.material ? `(material ${e.material})` : '(assembled shader)'),
+ line: loc.line, kind: 'compile', stage: e.stage, material: e.material || '', material_type: e.material_type || '',
+ message: `${e.stage} shader: ${e.message}` + (e.source_line ? ` — at: ${e.source_line}` : '') + (loc.candidates > 1 ? ` (${loc.candidates} identical lines; first shown)` : ''),
+ assembled_line: e.assembled_line, source_line: e.source_line, context: e.context, fix_hint: fixHintFor(e.message),
+ });
+ }
+ for (const a of comp.material_audit) {
+ (a.severity === 'error' ? errors : warnings).push({ file: '(runtime material)', line: null, kind: a.kind, material: a.material, message: a.message, fix_hint: a.severity === 'error' ? a.message : '' });
+ }
+ report.compile = { ms: comp.compile_ms, programs: comp.programs, custom_materials: comp.custom_materials, renderer: boot.renderer, gpu: host.gpu };
+ }
+ const es = errorSummary(host.errors, boot);
+ for (const c of es.shader_console) {
+ if (!errors.some((e) => e.kind === 'compile')) errors.push({ file: '(console)', line: null, kind: 'shader_console', message: c, fix_hint: 'search your GLSL for the quoted ERROR line' });
+ }
+ for (const c of es.console_errors) if (!boot.error || !c.startsWith('boot[')) warnings.push({ file: '(console)', line: null, kind: 'console', message: c });
+ return report;
+}
diff --git a/harness/runtime_js/probe_scene.mjs b/harness/runtime_js/probe_scene.mjs
index 3da3ea74..1ecbae46 100644
--- a/harness/runtime_js/probe_scene.mjs
+++ b/harness/runtime_js/probe_scene.mjs
@@ -6,18 +6,46 @@
* console/page/shader errors. No screenshots.
*
* node probe_scene.mjs --ws <ws> [--scene src/scene.js] [--gpu auto] [--timeout-ms 60000] [--out probe.json]
+ * [--compile] [--shaders-out report.json] [--sun-azimuth 60]
+ *
+ * `--compile` additionally runs the full shader preflight (static GLSL audits +
+ * force-compile of every material, lib/shader_report.mjs) on the SAME booted
+ * page — one browser boot instead of two for a scene build. The report
+ * (identical schema to check_shaders.mjs) lands in the summary as
+ * `shader_report` and, when `--shaders-out` is given, in that file.
+ * `--sun-azimuth <deg>` returns harness-fitted camera specs (lib/orbit.mjs —
+ * the single owner of camera-fit math) from the measured census: an overview
+ * of the content bounds plus one eye-level camera per top-level group, all
+ * standing on the sun side — `fitted_cameras: {azimuth, overview, zones}`.
+ * The scene assembler (assemble.py) converts these into its CameraPlans.
* Last stdout line = JSON {ok, boot, census, console_errors, shader_errors, ...}.
*/
import path from 'node:path';
import { armWatchdog, fail, finish, parseCli, writeJson } from './lib/cli.mjs';
import { createTimeoutMs, errorSummary, openHost } from './lib/host_page.mjs';
+import { fitOverviewCamera, fitZoneCamera, framingBox } from './lib/orbit.mjs';
+import { compileIntoReport, staticShaderReport } from './lib/shader_report.mjs';
const args = parseCli({
ws: {}, out: {}, gpu: { default: process.env.CV3D_RENDER_GPU || 'auto' }, 'timeout-ms': { default: '60000' },
'create-timeout-ms': { default: '' }, 'update-steps': { default: '10' }, scene: { default: 'src/scene.js' },
+ compile: { type: 'boolean', default: false }, 'shaders-out': { default: '' }, 'sun-azimuth': { default: '' },
});
+/** Harness-fitted camera specs from the census: overview + one per group. */
+function fitCameras(census, azimuth) {
+ const groundY = census.ground_y;
+ const box = framingBox(census, null);
+ const overview = box ? fitOverviewCamera(box, { azimuth, elevation: 30, groundY }) : null;
+ const zones = [];
+ for (const g of census.groups || []) {
+ if (!g.bbox || g.kind === 'sky' || g.kind === 'empty' || g.kind === 'light') continue;
+ zones.push({ group: g.name, ...fitZoneCamera(g.bbox, { azimuth, floor: groundY }) });
+ }
+ return { azimuth, overview, zones };
+}
+
async function main() {
if (!args.ws) throw new Error('--ws is required');
const timeoutMs = parseInt(args['timeout-ms'], 10);
@@ -59,10 +87,29 @@ async function main() {
}
const frame = await page.evaluate(() => { const c = window.__c3v.cameras()[0]; return window.__c3v.cameraChecks(c); });
result.first_camera = frame;
+ const sunAz = parseFloat(args['sun-azimuth']);
+ if (Number.isFinite(sunAz)) result.fitted_cameras = fitCameras(result.census, sunAz);
}
+ // Probe truth is snapshotted BEFORE any forced compile: the probe gate sees
+ // exactly what a standalone probe would; the preflight below sees everything.
result.shader_errors = (await page.evaluate(() => window.__c3v.shaderErrors())).map(({ _key, ...e }) => e);
Object.assign(result, errorSummary(host.errors, boot));
result.ok = boot.ok && result.update_ok === true && result.console_errors.length === 0 && result.shader_errors.length === 0;
+
+ if (args.compile) {
+ const ts = Date.now();
+ if (boot.ok) {
+ const { report, files } = staticShaderReport(path.resolve(args.ws), null);
+ await compileIntoReport(report, files, host);
+ report.ok = report.errors.length === 0;
+ report.duration_ms = Date.now() - ts;
+ result.shader_report = report;
+ } else {
+ result.shader_report = { ok: false, skipped: 'scene did not boot', errors: [], warnings: [], compile: null, duration_ms: Date.now() - ts };
+ }
+ if (args['shaders-out']) writeJson(path.resolve(args['shaders-out']), result.shader_report);
+ }
+
result.duration_ms = Date.now() - t0;
clearTimeout(watchdog);
await host.close();
diff --git a/harness/runtime_js/render_glb.mjs b/harness/runtime_js/render_glb.mjs
index 02c3212b..e298b815 100644
--- a/harness/runtime_js/render_glb.mjs
+++ b/harness/runtime_js/render_glb.mjs
@@ -95,18 +95,18 @@ async function main() {
const srv = await serveDirs({ root: path.dirname(cfg.glb), routes: { '/__render.html': { body: pageHtml(cfg) } } });
const consoleErrors = [];
- let browser = null;
+ let launched = null;
+ let page = null;
let gpu = false;
let launchRenderer = '';
const timing = {};
try {
- const launched = await launchBrowser({ gpu: cfg.gpu });
- browser = launched.browser;
+ launched = await launchBrowser({ gpu: cfg.gpu });
gpu = launched.gpu;
launchRenderer = launched.renderer;
timing.launch_ms = Date.now() - t0;
- const page = await browser.newPage();
+ page = await launched.browser.newPage();
await page.setViewport({ width: cfg.width, height: cfg.height, deviceScaleFactor: 1 });
page.on('console', (msg) => {
if (msg.type() === 'error' || msg.type() === 'warning') consoleErrors.push(`${msg.type()}: ${msg.text()}`);
@@ -141,7 +141,12 @@ async function main() {
fs.writeFileSync(path.join(cfg.out, 'views.json'), JSON.stringify(record, null, 2));
emit(record);
} finally {
- if (browser) await browser.close().catch(() => {});
+ // page first, then release: a shared browser is disconnected, never closed
+ if (page) await page.close().catch(() => {});
+ if (launched && launched.browser) {
+ if (typeof launched.release === 'function') await Promise.resolve(launched.release()).catch(() => {});
+ else await launched.browser.close().catch(() => {});
+ }
await srv.close();
}
}
diff --git a/harness/tests/compare_bench/test_bench_shared.py b/harness/tests/compare_bench/test_bench_shared.py
new file mode 100644
index 00000000..c5d31c38
--- /dev/null
+++ b/harness/tests/compare_bench/test_bench_shared.py
@@ -0,0 +1,52 @@
+"""Shared bench plumbing: select_prompts / build_spec / settings-routed backends."""
+
+from __future__ import annotations
+
+import bench.compare_backends as cb
+import bench.run_bench as rb
+from bench.run_bench import Battery, BenchOptions, build_spec, select_prompts, spec_for
+from tests.compare_bench.conftest import BATTERY
+
+
+def _battery() -> Battery:
+ return Battery.load(BATTERY)
+
+
+def test_select_prompts_ids_tiers_limit():
+ b = _battery()
+ assert select_prompts(b) == b.prompts
+ only = select_prompts(b, ids=[b.prompts[0].id])
+ assert [p.id for p in only] == [b.prompts[0].id]
+ tiers = {p.tier for p in b.prompts}
+ tier = sorted(tiers)[0]
+ assert all(p.tier == tier for p in select_prompts(b, tiers=[tier]))
+ assert len(select_prompts(b, limit=1)) == 1
+ assert select_prompts(b, ids=["nope"]) == []
+
+
+def test_select_prompts_is_single_owner_for_both_drivers():
+ """compare_backends deleted its private copy; both drivers filter identically."""
+ b = _battery()
+ opts = rb.BenchOptions(ids=[b.prompts[0].id])
+ assert rb._select(b, opts) == select_prompts(b, ids=opts.ids)
+ assert cb.select_prompts is select_prompts and cb.default_run_track is rb.default_run_track
+
+
+def test_build_spec_tags_and_budget():
+ b = _battery()
+ item = b.prompts[0]
+ spec = build_spec(b, item, backends=rb.get_settings().backends(), rounds=2, max_usd=1.5,
+ max_minutes=10, tag0="compare", extra_tags=("harness",))
+ assert spec.tags[:4] == ["compare", b.name, item.tier, item.category] and "harness" in spec.tags
+ assert spec.budget.max_rounds == 2 and spec.budget.max_usd == 1.5
+ assert spec.id == f"{b.name}/{item.id}"
+
+
+def test_spec_for_routes_through_settings_backends():
+ b = _battery()
+ s = rb.get_settings()
+ spec = spec_for(b, b.prompts[0], BenchOptions(generator="gemini-cli:gemini-3.7-flash"))
+ assert spec.backends.generator == "gemini-cli:gemini-3.7-flash"
+ # unset roles fall through to the settings defaults, not the Backends() literals
+ assert spec.backends.planner == s.default_planner and spec.backends.judge == s.default_judge
+ assert spec.backends.captioner == s.default_captioner
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 7485e6e3..1d738871 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -169,11 +169,11 @@ def test_judge_wraps_value_error_as_cli_error(runs_dir: Path, monkeypatch):
def test_judge_rubric_map_includes_graphics():
- from codeverse.cli._judge import TRACK_RUBRIC, rubric_for
- from codeverse.contracts.common import Track
+ from codeverse.cli._judge import rubric_for
+ from codeverse.contracts.common import TRACK_INFO, Track
from codeverse.contracts.run import RoundRecord
- assert TRACK_RUBRIC[Track.GRAPHICS] == "shader_v1"
+ assert TRACK_INFO[Track.GRAPHICS].rubric == "shader_v1"
class _R: # minimal record stub
class spec:
@@ -185,13 +185,61 @@ def test_judge_rubric_map_includes_graphics():
# --------------------------------------------------------------------------- make --texture / status extras / render graphics
-def test_make_texture_flag_adds_tag(tmp_path: Path):
+def test_make_texture_flag_sets_spec_options(tmp_path: Path):
runs = tmp_path / "runs"
- r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run", "--texture"])
+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run", "--texture", "--candidates", "3"])
assert r.exit_code == 0, r.output
ws = next(d for d in runs.iterdir() if d.is_dir())
spec = Spec.model_validate_json((ws / "spec.json").read_text())
- assert "texture" in spec.tags
+ # run-shape options are frozen on the spec (no more magic 'texture' tag)
+ assert spec.options.texture is True and spec.options.candidates == 3
+ assert "texture" not in spec.tags
+
+
+def test_make_language_defaults_to_the_tracks_first(tmp_path: Path):
+ runs = tmp_path / "runs"
+ r = runner.invoke(app, ["make", "a harbour at night", "--track", "scene", "--runs-dir", str(runs), "--no-run"])
+ assert r.exit_code == 0, r.output
+ ws = next(d for d in runs.iterdir() if d.is_dir())
+ spec = Spec.model_validate_json((ws / "spec.json").read_text())
+ assert spec.language.value == "scene_threejs"
+
+
+def test_make_invalid_combo_leaves_no_orphan_workspace(tmp_path: Path):
+ runs = tmp_path / "runs"
+ r = runner.invoke(app, ["make", "x", "--track", "scene", "--language", "blender",
+ "--runs-dir", str(runs), "--no-run", "--slug", "bad"])
+ assert r.exit_code == 1
+ assert not (runs / "bad").exists(), "invalid spec must not leave an orphan run dir"
+
+
+def test_resume_budget_flags_rewrite_spec_and_emit_event(tmp_path: Path, monkeypatch):
+ runs = tmp_path / "runs"
+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run", "--max-usd", "1.0", "--rounds", "1"])
+ assert r.exit_code == 0, r.output
+ ws = next(d for d in runs.iterdir() if d.is_dir())
+
+ seen = {}
+
+ def fake_get_track(track, **options):
+ class _T:
+ def run(self, spec, ws_, *, resume=False):
+ seen["spec"] = spec
+ raise KeyboardInterrupt # stop before any real work
+
+ return _T()
+
+ import codeverse.tracks as tracks_pkg
+
+ monkeypatch.setattr(tracks_pkg, "get_track", fake_get_track)
+ r2 = runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs), "--max-usd", "4.5", "--rounds", "3"])
+ assert r2.exit_code == 130
+ spec = Spec.model_validate_json((ws / "spec.json").read_text())
+ assert spec.budget.max_usd == 4.5 and spec.budget.max_rounds == 3 and spec.budget.max_minutes == 60.0
+ assert seen["spec"].budget.max_usd == 4.5, "the resumed run must see the raised budget"
+ events = [json.loads(line) for line in (ws / "events.jsonl").read_text().splitlines()]
+ raised = [e for e in events if e.get("event") == "budget.raised"]
+ assert raised and raised[0]["max_usd"] == 4.5 and raised[0]["max_rounds"] == 3
def test_status_shows_candidates_and_texturing(runs_dir: Path):
diff --git a/harness/tests/judges/conftest.py b/harness/tests/judges/conftest.py
index 29ad6bdf..a14d218c 100644
--- a/harness/tests/judges/conftest.py
+++ b/harness/tests/judges/conftest.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import threading
from pathlib import Path
from typing import Any
@@ -110,16 +111,24 @@ def good_reply(rubric: Rubric, acceptance_ids: list[str], score: float = 0.8, *,
class FakeChatModel:
- """ChatModel stand-in: pops replies from a queue (dict → parsed; str → text; Exception → raised)."""
+ """ChatModel stand-in: pops replies from a queue (dict → parsed; str → text;
+ Exception → raised). Thread-safe: judges fan samples/orderings across threads.
+
+ ``by_label`` routes deterministically under concurrency: each key is a substring
+ of ``request.label`` with its own reply queue (longest matching key wins);
+ unmatched requests fall back to the shared queue / ``default``."""
provider = "fake"
model = "fake-1"
- def __init__(self, replies: list[Any] | None = None, *, default: Any = None, cost: float = 0.001):
+ def __init__(self, replies: list[Any] | None = None, *, default: Any = None, cost: float = 0.001,
+ by_label: dict[str, list[Any]] | None = None):
self.replies = list(replies or [])
self.default = default
self.cost = cost
+ self.by_label = {k: list(v) for k, v in (by_label or {}).items()}
self.requests: list[ChatRequest] = []
+ self._lock = threading.Lock()
@property
def id(self) -> str:
@@ -128,9 +137,16 @@ class FakeChatModel:
def supports_vision(self) -> bool:
return True
+ def _next_reply(self, request: ChatRequest) -> Any:
+ for key in sorted(self.by_label, key=len, reverse=True):
+ if key in (request.label or "") and self.by_label[key]:
+ return self.by_label[key].pop(0)
+ return self.replies.pop(0) if self.replies else self.default
+
def generate(self, request: ChatRequest) -> ChatResponse:
- self.requests.append(request)
- reply = self.replies.pop(0) if self.replies else self.default
+ with self._lock:
+ self.requests.append(request)
+ reply = self._next_reply(request)
if callable(reply) and not isinstance(reply, type):
reply = reply(request)
if isinstance(reply, Exception):
diff --git a/harness/tests/judges/test_defects.py b/harness/tests/judges/test_defects.py
index 21e50f58..7f7c4aa3 100644
--- a/harness/tests/judges/test_defects.py
+++ b/harness/tests/judges/test_defects.py
@@ -85,12 +85,13 @@ def test_penalties_and_caps_computed_in_code(judge_input, cache_dir):
def test_defect_majority_vote_ties_count_as_present(judge_input, cache_dir):
- model = FakeChatModel([_reply(0.9, ["render_artifacts"]), _reply(0.9), _reply(0.9)])
+ model = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)],
+ ":s2": [_reply(0.9)]})
j = VlmJudge("static_object_v1", chat_model=model, n_samples=3, cache_dir=cache_dir).judge(judge_input)
raw = json.loads(j.raw)
assert raw["defects"]["render_artifacts"] is False and raw["defect_votes"]["render_artifacts"] == [True, False, False]
assert j.overall == pytest.approx(0.9)
- model2 = FakeChatModel([_reply(0.9, ["render_artifacts"]), _reply(0.9)])
+ model2 = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)]})
j2 = VlmJudge("static_object_v1", chat_model=model2, n_samples=2, cache_dir=cache_dir).judge(judge_input)
assert json.loads(j2.raw)["defects"]["render_artifacts"] is True and j2.overall == pytest.approx(0.85)
# per-criterion std is reported for n-sample runs
diff --git a/harness/tests/judges/test_output_schema.py b/harness/tests/judges/test_output_schema.py
index b1dd5d91..cc331f61 100644
--- a/harness/tests/judges/test_output_schema.py
+++ b/harness/tests/judges/test_output_schema.py
@@ -4,7 +4,6 @@ import pytest
from codeverse.judges.output_schema import (
JudgeParseError,
- extract_json,
parse_judge_output,
wire_schema,
)
@@ -87,8 +86,8 @@ def test_measured_injection():
parse_judge_output(rep, ref, []) # measured criterion missing
-def test_extract_json_variants():
- assert extract_json('{"a": 1}') == {"a": 1}
- assert extract_json('blah {"a": {"b": 2}} trailing') == {"a": {"b": 2}}
- with pytest.raises(JudgeParseError):
- extract_json("no json here")
+def test_text_reply_without_json_raises_judge_parse_error():
+ """String payloads go through the shared lenient parser; its JsonParseError is
+ re-raised as JudgeParseError so callers keep one exception surface."""
+ with pytest.raises(JudgeParseError, match="no JSON object"):
+ parse_judge_output("no json here", R, IDS)
diff --git a/harness/tests/judges/test_pairwise.py b/harness/tests/judges/test_pairwise.py
index 542379c1..824895e0 100644
--- a/harness/tests/judges/test_pairwise.py
+++ b/harness/tests/judges/test_pairwise.py
@@ -1,3 +1,5 @@
+import pytest
+
from codeverse.judges.pairwise import PairwiseJudge
from codeverse.models.base import ModelError
from tests.judges.conftest import FakeChatModel, image_parts, make_renders, make_spec
@@ -10,15 +12,16 @@ def _reply(w, conf=0.8):
def test_agreement_across_swap(tmp_path, cache_dir):
ra, rb = make_renders(tmp_path / "a"), make_renders(tmp_path / "b")
# ordering 1: A=ra,B=rb → "A" wins; ordering 2 (swapped): A=rb,B=ra → "B" wins → both say ra
- model = FakeChatModel([_reply("A", 0.9), _reply("B", 0.7)])
+ model = FakeChatModel(by_label={":fwd": [_reply("A", 0.9)], ":swap": [_reply("B", 0.7)]})
res = PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare(make_spec(), ra, rb)
assert res.winner == "a" and res.confidence == 0.8 and len(res.orderings) == 2
assert res.usage.cost_usd > 0
- labels = [p.label for p in image_parts(model.requests[0])]
+ reqs = {r.label: r for r in model.requests} # orderings run in parallel: look up by label
+ labels = [p.label for p in image_parts(reqs["pairwise:fwd"])]
assert len(labels) == 2 # one 2×2 montage per side
assert labels[0].startswith("CANDIDATE A — MONTAGE 1/1 — SHADED views: top-left = front_right_34") and labels[1].startswith("CANDIDATE B — MONTAGE")
# swapped ordering puts rb first as "A"
- labels2 = [p.label for p in image_parts(model.requests[1])]
+ labels2 = [p.label for p in image_parts(reqs["pairwise:swap"])]
assert labels2[0].startswith("CANDIDATE A — MONTAGE")
@@ -42,7 +45,8 @@ def test_tie_and_errors(tmp_path, cache_dir):
def test_single_ordering_success_halves_confidence(tmp_path, cache_dir):
ra, rb = make_renders(tmp_path / "a"), make_renders(tmp_path / "b")
- model = FakeChatModel([ModelError("x", retryable=True), _reply("A", 0.8)]) # swapped ordering: A=rb → b wins
+ model = FakeChatModel(by_label={":fwd": [ModelError("x", retryable=True)],
+ ":swap": [_reply("A", 0.8)]}) # swapped ordering: A=rb → b wins
res = PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare(make_spec(), ra, rb)
assert res.winner == "b" and res.confidence == 0.4
@@ -57,10 +61,10 @@ def test_compare_many_round_robin(tmp_path, cache_dir):
from codeverse.judges.pairwise import RankingResult
cands = [make_renders(tmp_path / f"c{i}") for i in range(3)]
# pairs (0,1), (0,2), (1,2); each compare = 2 orderings. Make 2 beat everyone, 0 beat 1.
- replies = [_reply("A", 0.9), _reply("B", 0.8), # 0 v 1 → a (0 wins)
- _reply("B", 0.9), _reply("A", 0.9), # 0 v 2 → b (2 wins)
- _reply("B", 0.7), _reply("A", 0.7)] # 1 v 2 → b (2 wins)
- model = FakeChatModel(replies)
+ replies = {"0v1:fwd": [_reply("A", 0.9)], "0v1:swap": [_reply("B", 0.8)], # 0 v 1 → a (0 wins)
+ "0v2:fwd": [_reply("B", 0.9)], "0v2:swap": [_reply("A", 0.9)], # 0 v 2 → b (2 wins)
+ "1v2:fwd": [_reply("B", 0.7)], "1v2:swap": [_reply("A", 0.7)]} # 1 v 2 → b (2 wins)
+ model = FakeChatModel(by_label=replies)
rank: RankingResult = PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare_many(make_spec(), cands)
assert rank.order == [2, 0, 1] and rank.best == 2
assert rank.points == {0: 1.0, 1: 0.0, 2: 2.0}
@@ -77,3 +81,30 @@ def test_compare_many_ties_and_errors(tmp_path, cache_dir):
rank = PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare_many(make_spec(), cands)
assert rank.points == {0: 0.5, 1: 0.5} and rank.order == [0, 1]
assert rank.errors and "orderings disagree" in rank.errors[0]
+
+
+def test_parallel_orderings_accumulate_in_order(tmp_path, cache_dir):
+ """The two orderings run concurrently but land in (fwd, swap) order."""
+ ra, rb = make_renders(tmp_path / "a"), make_renders(tmp_path / "b")
+ model = FakeChatModel(by_label={":fwd": [_reply("A", 0.9)], ":swap": [_reply("B", 0.7)]})
+ res = PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare(make_spec(), ra, rb)
+ assert [o["swapped"] for o in res.orderings] == [False, True]
+ assert [o["confidence"] for o in res.orderings] == [0.9, 0.7]
+ # usage sums across both parallel orderings
+ assert res.usage.cost_usd == pytest.approx(0.002) and res.usage.input_tokens == 2000
+
+
+def test_compare_many_parallel_is_deterministic(tmp_path, cache_dir):
+ """Fanned round-robin gives the same ranking / pair order on every run."""
+ cands = [make_renders(tmp_path / f"c{i}") for i in range(3)]
+ script = {"0v1:fwd": [_reply("A", 0.9)], "0v1:swap": [_reply("B", 0.8)],
+ "0v2:fwd": [_reply("B", 0.9)], "0v2:swap": [_reply("A", 0.9)],
+ "1v2:fwd": [_reply("B", 0.7)], "1v2:swap": [_reply("A", 0.7)]}
+ outcomes = []
+ for _ in range(3):
+ model = FakeChatModel(by_label={k: list(v) for k, v in script.items()})
+ rank = PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare_many(make_spec(), cands)
+ outcomes.append((tuple(rank.order), tuple((p["a"], p["b"], p["winner"]) for p in rank.pairs)))
+ assert len(set(outcomes)) == 1
+ assert outcomes[0][0] == (2, 0, 1)
+ assert outcomes[0][1] == ((0, 1, "a"), (0, 2, "b"), (1, 2, "b")) # pairs in input order
diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.py
index c2ed9fff..90951b0e 100644
--- a/harness/tests/judges/test_vlm_judge.py
+++ b/harness/tests/judges/test_vlm_judge.py
@@ -47,7 +47,9 @@ def test_overall_is_code_computed_weighted_mean(judge_input, cache_dir):
def test_n_samples_mean_std_and_shuffle(judge_input, cache_dir):
- model = FakeChatModel([good_reply(R, IDS, 0.7), good_reply(R, IDS, 0.9), good_reply(R, IDS, 0.8)])
+ # samples fan out in parallel: route replies by the request's :s<k> label
+ model = FakeChatModel(by_label={":s0": [good_reply(R, IDS, 0.7)], ":s1": [good_reply(R, IDS, 0.9)],
+ ":s2": [good_reply(R, IDS, 0.8)]})
j = _judge(model, n_samples=3, cache_dir=cache_dir).judge(judge_input)
assert j.n_samples == 3 and j.overall == pytest.approx(0.8) and j.score_std == pytest.approx(0.0816, abs=1e-3)
assert j.usage.cost_usd == pytest.approx(0.003) and j.usage.input_tokens == 3000
diff --git a/harness/tests/languages/__init__.py b/harness/tests/languages/__init__.py
new file mode 100644
index 00000000..a10d15f4
--- /dev/null
+++ b/harness/tests/languages/__init__.py
@@ -0,0 +1 @@
+"""Languages-layer shared-helper tests (offline)."""
diff --git a/harness/tests/languages/test_ast_lint.py b/harness/tests/languages/test_ast_lint.py
new file mode 100644
index 00000000..525e1ce9
--- /dev/null
+++ b/harness/tests/languages/test_ast_lint.py
@@ -0,0 +1,80 @@
+"""Shared AST-lint primitives + the per-language forbidden-import supersets (drift guard)."""
+
+from __future__ import annotations
+
+import ast
+
+from codeverse.contracts.artifacts import GateFinding, Severity
+from codeverse.languages._ast_lint import BASE_FORBIDDEN_IMPORTS, check_imports, dotted
+from codeverse.languages.blender import lint as blender_lint
+from codeverse.languages.cadquery import lint as cadquery_lint
+from codeverse.languages.opengl_python import lint as opengl_lint
+
+# ----------------------------------------------------------------- superset (drift fails here)
+
+
+def test_base_forbidden_imports_shape() -> None:
+ assert len(BASE_FORBIDDEN_IMPORTS) == 14
+ # blender legitimately allows os/sys — the base set must never grow them
+ assert "os" not in BASE_FORBIDDEN_IMPORTS and "sys" not in BASE_FORBIDDEN_IMPORTS
+ assert {"subprocess", "webbrowser", "ftplib", "smtplib", "importlib"} <= BASE_FORBIDDEN_IMPORTS
+
+
+def test_language_forbidden_sets_are_supersets_of_base() -> None:
+ assert BASE_FORBIDDEN_IMPORTS <= blender_lint.FORBIDDEN_IMPORTS
+ assert BASE_FORBIDDEN_IMPORTS <= cadquery_lint.FORBIDDEN_IMPORTS
+ assert BASE_FORBIDDEN_IMPORTS <= opengl_lint.DANGEROUS_MODULES
+
+
+def test_language_specific_extras_survive() -> None:
+ assert "pathlib" in blender_lint.FORBIDDEN_IMPORTS
+ assert {"os", "sys", "OCP", "build123d"} <= cadquery_lint.FORBIDDEN_IMPORTS
+ assert {"os", "sys", "time", "datetime", "io", "tempfile", "signal"} <= opengl_lint.DANGEROUS_MODULES
+ # blender's documented allowance: os/sys stay importable (call rules catch abuse)
+ assert {"os", "sys"} <= blender_lint.ALLOWED_IMPORTS
+ assert not ({"os", "sys"} & blender_lint.FORBIDDEN_IMPORTS)
+
+
+def test_base_modules_error_in_every_language_lint() -> None:
+ src_bl = "import bpy\nimport webbrowser\nobj = bpy.context.object\n"
+ rep = blender_lint.lint_blender_source(src_bl, expect_names=False)
+ assert any(f.severity == Severity.ERROR and "webbrowser" in f.message for f in rep.findings)
+
+ src_cq = "import cadquery as cq\nimport webbrowser\nresult = cq.Workplane('XY').box(1, 1, 1)\n"
+ rep = cadquery_lint.lint_cadquery_source(src_cq)
+ assert any(f.severity == Severity.ERROR and "webbrowser" in f.message for f in rep.findings)
+
+ findings = opengl_lint.lint_source(
+ "import moderngl\nimport webbrowser\n"
+ "def setup(ctx, width, height):\n return None\n"
+ "def render(ctx, state, t, frame, fbo):\n pass\n"
+ )
+ assert any(f.severity == Severity.ERROR and "webbrowser" in f.message for f in findings)
+
+
+# ----------------------------------------------------------------- check_imports / dotted
+
+
+def _mk(kind: str, mod: str, line: int) -> GateFinding:
+ return GateFinding(gate="lint:test", severity=Severity.ERROR if kind == "forbidden" else Severity.WARN,
+ message=f"{kind}:{mod}", data={"line": line})
+
+
+def test_check_imports_routes_forbidden_unexpected_allowed() -> None:
+ out = check_imports({"subprocess": 1, "mystery": 2, "math": 3},
+ forbidden=frozenset({"subprocess"}), allowed=frozenset({"math"}), make_finding=_mk)
+ assert [(f.message, f.severity) for f in out] == [
+ ("forbidden:subprocess", Severity.ERROR), ("unexpected:mystery", Severity.WARN)]
+ assert [f.data["line"] for f in out] == [1, 2]
+
+
+def test_dotted_chains() -> None:
+ def expr(src: str) -> ast.AST:
+ return ast.parse(src, mode="eval").body
+
+ assert dotted(expr("bpy.ops.render.render")) == "bpy.ops.render.render"
+ assert dotted(expr("name")) == "name"
+ assert dotted(ast.parse("wp.box(1).rotate", mode="eval").body) == "<expr>.rotate"
+ assert dotted(expr("(a + b)")) == ""
+ # the same function is what the language lints re-export
+ assert blender_lint.dotted is dotted and cadquery_lint.dotted is dotted
diff --git a/harness/tests/languages/test_gl_common_entry.py b/harness/tests/languages/test_gl_common_entry.py
new file mode 100644
index 00000000..7f501025
--- /dev/null
+++ b/harness/tests/languages/test_gl_common_entry.py
@@ -0,0 +1,42 @@
+"""_gl_common consolidation + ENTRY_FILE adoption across the language runtimes (offline)."""
+
+from __future__ import annotations
+
+from codeverse.contracts.common import ENTRY_FILE, Language
+from codeverse.languages import _gl_common
+from codeverse.languages.glsl_shader import gl_build, wrap
+from codeverse.spatial.gl_render import GlHost
+
+
+def test_gl_build_shim_reexports_gl_common() -> None:
+ for name in ("finish_build", "judge_times", "load_plan", "preview_times", "resolution_for",
+ "traceback_location", "read_metrics", "SHEET_NAME", "JUDGE_TIMES"):
+ assert getattr(gl_build, name) is getattr(_gl_common, name), name
+
+
+def test_wrap_reexports_moved_glsl_log_types() -> None:
+ assert wrap.parse_glsl_log is _gl_common.parse_glsl_log
+ assert wrap.GlslMessage is _gl_common.GlslMessage
+ assert wrap.LineMap is _gl_common.LineMap and wrap.Segment is _gl_common.Segment
+
+
+def test_make_host_prefers_injected_host() -> None:
+ injected = GlHost(gpu="off", timeout_s=1.0)
+ assert _gl_common.make_host(injected, 99.0) is injected
+ built = _gl_common.make_host(None, 7.5)
+ assert isinstance(built, GlHost) and built is not injected
+
+
+def test_runtimes_lead_with_the_entry_file_table() -> None:
+ from codeverse.languages.blender.layout import ENTRY_REL
+ from codeverse.languages.blender.runtime import BlenderRuntime
+ from codeverse.languages.cadquery.runtime import CadQueryRuntime
+ from codeverse.languages.glsl_shader.runtime import GlslShaderRuntime
+ from codeverse.languages.opengl_python.runtime import OpenGLPythonRuntime
+ from codeverse.languages.threejs.runtime import ThreeJsRuntime
+ from codeverse.languages.urdf.runtime import UrdfBlenderRuntime
+
+ assert ENTRY_FILE[Language.BLENDER] == ENTRY_REL
+ for rt in (BlenderRuntime, CadQueryRuntime, GlslShaderRuntime, OpenGLPythonRuntime,
+ ThreeJsRuntime, UrdfBlenderRuntime):
+ assert rt.entry_globs[0] == ENTRY_FILE[rt.language], rt.__name__
diff --git a/harness/tests/models/test_gemini.py b/harness/tests/models/test_gemini.py
index 5ef24262..1e74b7e6 100644
--- a/harness/tests/models/test_gemini.py
+++ b/harness/tests/models/test_gemini.py
@@ -288,16 +288,26 @@ def test_every_key_dead_raises_without_benching():
def test_5xx_retries_then_gives_up_with_retryable_error():
- err = genai_errors.APIError(503, {"error": {"message": "high demand", "status": "UNAVAILABLE"}})
+ # 500 = plain 5xx (no storm budget): the classic 6-attempt backoff applies
+ err = genai_errors.APIError(500, {"error": {"message": "internal", "status": "INTERNAL"}})
slept: list[float] = []
m, log, _ = make_model([err] * 6)
m._sleep = slept.append
with pytest.raises(ModelError) as ei:
m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
- assert ei.value.retryable and ei.value.status == 503
+ assert ei.value.retryable and ei.value.status == 500
assert len(log) == 6 and len(slept) == 5 and slept[1] > slept[0] / 2
+def test_503_storm_survives_beyond_the_attempt_budget():
+ # 503 = capacity storm: its own patience budget on top of max_attempts
+ err = genai_errors.APIError(503, {"error": {"message": "high demand", "status": "UNAVAILABLE"}})
+ m, log, _ = make_model([err] * 8 + [text_response("recovered")])
+ m._sleep = lambda d: None
+ resp = m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
+ assert resp.text == "recovered" and len(log) == 9
+
+
def test_400_is_not_retried():
err = genai_errors.APIError(
400, {"error": {"message": "Invalid argument", "status": "INVALID_ARGUMENT"}}
diff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.py
index 7a542f0b..e62b4d2e 100644
--- a/harness/tests/models/test_retry.py
+++ b/harness/tests/models/test_retry.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import pytest
import pytest
from codeverse.models.retry import backoff_delay, with_retries
@@ -59,3 +60,197 @@ def test_exhausted_raises_last():
with pytest.raises(TimeoutError):
with_retries(fn, is_retryable=lambda e: True, attempts=3, sleep=lambda s: None)
+
+
+# --------------------------------------------------------------------------- rotate_with_retries
+from codeverse.models.base import ModelError # noqa: E402
+from codeverse.models.keypool import KeyPool, KeyPoolExhausted, Outcome # noqa: E402
+from codeverse.models.retry import rotate_with_retries # noqa: E402
+
+
+def _classify(exc: BaseException) -> ModelError:
+ if isinstance(exc, ModelError):
+ return exc
+ if isinstance(exc, KeyPoolExhausted):
+ return ModelError(f"pool exhausted: {exc}", retryable=False, status=429)
+ return ModelError(str(exc), retryable=False)
+
+
+def _outcome(err: ModelError) -> Outcome:
+ if err.status == 429:
+ return "429"
+ if (err.status or 0) >= 500:
+ return "5xx"
+ if err.status in (401, 403):
+ return "dead"
+ return "error" if err.status else "ok"
+
+
+def _rotate(pool, call, **kw):
+ kw.setdefault("classify", _classify)
+ kw.setdefault("outcome_of", _outcome)
+ kw.setdefault("sleep", lambda s: None)
+ return rotate_with_retries(pool, call, **kw)
+
+
+def test_rotate_success_reports_ok_with_tokens():
+ pool = KeyPool(["k1", "k2"])
+ out = _rotate(pool, lambda key: f"ok:{key}", tokens_of=lambda r: 42)
+ assert out == "ok:k1"
+ st = pool.stats()
+ assert st["ok"] == 1 and st["429"] == 0 and st["dead"] == 0
+
+
+def test_rotate_dead_key_is_free_and_benched_after_sibling_success():
+ pool = KeyPool(["k1", "k2"])
+ calls: list[str] = []
+
+ def call(key: str) -> str:
+ calls.append(key)
+ if key == "k1":
+ raise ModelError("suspended", retryable=False, status=403)
+ return key
+
+ assert _rotate(pool, call, max_attempts=1) == "k2" # dead rotation burnt no budget
+ assert calls == ["k1", "k2"]
+ st = pool.stats()
+ assert st["dead"] == 1 and st["n_dead"] == 1 # k1 benched only after k2 proved the request
+
+
+def test_rotate_every_key_dead_raises_without_benching():
+ pool = KeyPool(["k1", "k2", "k3"])
+ calls: list[str] = []
+
+ def call(key: str) -> str:
+ calls.append(key)
+ raise ModelError("suspended", retryable=False, status=403)
+
+ with pytest.raises(ModelError) as ei:
+ _rotate(pool, call)
+ assert ei.value.status == 403 and calls == ["k1", "k2", "k3"]
+ assert pool.stats()["dead"] == 0 # the request, not the keys, is suspect
+
+
+def test_rotate_429_is_free_while_untried_keys_remain():
+ pool = KeyPool(["k1", "k2", "k3"], cooldown_s=30)
+ slept: list[float] = []
+ calls: list[str] = []
+
+ def call(key: str) -> str:
+ calls.append(key)
+ if key in ("k1", "k2"):
+ raise ModelError("quota", retryable=True, status=429)
+ return key
+
+ out = _rotate(pool, call, max_attempts=1, sleep=slept.append)
+ assert out == "k3" and calls == ["k1", "k2", "k3"]
+ assert slept == [0.5, 0.5] # courtesy pauses only, no backoff
+ assert pool.stats()["n_cooling"] == 2
+
+
+def test_rotate_on_free_retry_hook_consumes_no_budget():
+ pool = KeyPool(["k1"])
+ state = {"fixed": False}
+ calls: list[int] = []
+
+ def call(key: str) -> str:
+ calls.append(1)
+ if not state["fixed"]:
+ raise ModelError("thinking not supported", retryable=False, status=400)
+ return "ok"
+
+ def free(err: ModelError) -> bool:
+ if "thinking" in str(err) and not state["fixed"]:
+ state["fixed"] = True
+ return True
+ return False
+
+ assert _rotate(pool, call, max_attempts=1, on_free_retry=free) == "ok"
+ assert len(calls) == 2
+
+
+def test_rotate_retryable_backoff_then_exhausted_raises_last():
+ pool = KeyPool(["k1"])
+ slept: list[float] = []
+
+ def call(key: str) -> str:
+ raise ModelError("unavailable", retryable=True, status=500)
+
+ with pytest.raises(ModelError) as ei:
+ _rotate(pool, call, max_attempts=3, base_delay=1.0, max_delay=4.0, sleep=slept.append)
+ assert ei.value.status == 500 and len(slept) == 2 # backoff between the 3 attempts
+
+
+def test_rotate_nonretryable_raises_immediately():
+ pool = KeyPool(["k1", "k2"])
+ calls: list[str] = []
+
+ def call(key: str) -> str:
+ calls.append(key)
+ raise ModelError("bad request", retryable=False, status=400)
+
+ with pytest.raises(ModelError):
+ _rotate(pool, call, max_attempts=5)
+ assert calls == ["k1"]
+
+
+def test_rotate_retry_after_hint_reaches_the_pool():
+ clock = {"t": 0.0}
+ pool = KeyPool(["k1", "k2"], cooldown_s=30, clock=lambda: clock["t"], sleep=lambda s: clock.__setitem__("t", clock["t"] + s))
+ seen: list[BaseException] = []
+
+ def call(key: str) -> str:
+ if key == "k1":
+ raise ModelError("quota retryDelay 7s", retryable=True, status=429)
+ return key
+
+ out = _rotate(pool, call, max_attempts=1, retry_after=lambda e: seen.append(e) or 7.0)
+ assert out == "k2" and len(seen) == 1
+ cooling = {k["key"]: k["cooldown_s"] for k in pool.stats()["keys"]}
+ assert 0 < cooling["…k1"] <= 7.0
+
+
+def test_503_storm_has_its_own_patience_budget():
+ """A model-wide 503 storm must not burn the normal attempt budget."""
+ from codeverse.models.base import ModelError
+ from codeverse.models.keypool import KeyPool
+ from codeverse.models.retry import rotate_with_retries
+
+ pool = KeyPool(["k1", "k2"], rpm_per_key=10_000)
+ calls = {"n": 0}
+ naps: list[float] = []
+
+ def call(key):
+ calls["n"] += 1
+ if calls["n"] <= 8: # longer than max_attempts=3 could survive
+ raise RuntimeError("503 storm")
+ return "ok"
+
+ def classify(exc):
+ return ModelError(str(exc), retryable=True, status=503)
+
+ out = rotate_with_retries(
+ pool, call, classify=classify, outcome_of=lambda e: "5xx",
+ max_attempts=3, base_delay=0.01, storm_attempts=10, storm_max_delay=0.05,
+ sleep=naps.append,
+ )
+ assert out == "ok" and calls["n"] == 9
+ assert len(naps) == 8 and all(d <= 0.05 * 1.25 for d in naps)
+
+
+def test_503_storm_budget_exhausts_then_normal_budget_applies():
+ from codeverse.models.base import ModelError
+ from codeverse.models.keypool import KeyPool
+ from codeverse.models.retry import rotate_with_retries
+
+ pool = KeyPool(["k1"], rpm_per_key=10_000)
+
+ def call(key):
+ raise RuntimeError("503 forever")
+
+ with pytest.raises(ModelError):
+ rotate_with_retries(
+ pool, call, classify=lambda e: ModelError(str(e), retryable=True, status=503),
+ outcome_of=lambda e: "5xx", max_attempts=2, base_delay=0.0,
+ storm_attempts=3, storm_max_delay=0.0, sleep=lambda d: None,
+ )
diff --git a/harness/tests/models/test_schema_utils.py b/harness/tests/models/test_schema_utils.py
index bf9d4935..a7da5d90 100644
--- a/harness/tests/models/test_schema_utils.py
+++ b/harness/tests/models/test_schema_utils.py
@@ -209,3 +209,18 @@ def test_parse_json_lenient_array_and_failure():
with pytest.raises(JsonParseError):
parse_json_lenient("")
assert json.dumps(parse_json_lenient('{"n": {"x": [1]}}')) == '{"n": {"x": [1]}}'
+
+
+def test_parse_json_lenient_extract_variants():
+ """Ported from the judges' retired ``extract_json`` (one parser, one home)."""
+ assert parse_json_lenient('{"a": 1}') == {"a": 1}
+ assert parse_json_lenient('blah {"a": {"b": 2}} trailing') == {"a": {"b": 2}}
+ with pytest.raises(JsonParseError):
+ parse_json_lenient("no json here")
+
+
+def test_parse_json_lenient_first_balanced_brace_wins_over_outer_span():
+ # trailing chatter contains a second, unrelated brace pair: the depth-counted
+ # first balanced candidate parses where the outermost {...} span cannot
+ assert parse_json_lenient('{"a": 1} and later {broken')['a'] == 1
+ assert parse_json_lenient('note {"a": 1} ps: see {figure 2}') == {"a": 1}
diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
new file mode 100644
index 00000000..bad72204
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
@@ -0,0 +1,172 @@
+"""Quality-wave batch 2: refine scaffold, planner hooks, RunOptions wiring, judge-view flags."""
+
+from __future__ import annotations
+
+from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderView
+from codeverse.contracts.common import Language, Track
+from codeverse.contracts.plan import ScenePlan
+from codeverse.contracts.run import RunStatus
+from codeverse.contracts.spec import RunOptions
+from codeverse.events import EventLog
+from codeverse.orchestrator.rounds import RefineTask, plan_refine_groups
+from codeverse.orchestrator.state import RunState
+from codeverse.tracks.planner import plan_example
+from codeverse.tracks.scene import ScenePipeline, SceneTrack
+from codeverse.tracks.static_object import StaticObjectTrack
+from codeverse.workspace import Workspace
+from tests.orchestrator_tracks.conftest import make_spec
+from tests.orchestrator_tracks.fakes import (
+ FakeAgent,
+ FakeChatModel,
+ FakeJudge,
+ FakeRuntime,
+ FakeServices,
+)
+
+
+def _planner(plan_dict):
+ return FakeChatModel(lambda req: plan_dict)
+
+
+def _writer(job, ws):
+ return {"src/object.js": f"// {job.label} r{job.round}\nexport function build(THREE) {{ return new THREE.Group(); }}\n"}
+
+
+# --------------------------------------------------------------------- plan_refine_groups
+def _task(target: str, files: list[str] | None = None, priority: int = 2) -> RefineTask:
+ return RefineTask(target=target, kind="geometry", instruction=f"fix {target}", priority=priority, files=files or [])
+
+
+def test_plan_refine_groups_fans_out_only_when_allowed_and_disjoint():
+ tasks = [_task("Seat", ["src/parts/seat.js"]), _task("Leg", ["src/parts/leg.js"])]
+ groups, parallel = plan_refine_groups(tasks, allow_fanout=True, parallel_min_tasks=2)
+ assert parallel and len(groups) == 2
+
+ groups, parallel = plan_refine_groups(tasks, allow_fanout=False, parallel_min_tasks=2)
+ assert not parallel and len(groups) == 1
+ assert groups[0].files == ["src/parts/leg.js", "src/parts/seat.js"] and len(groups[0].tasks) == 2
+
+ # unknown file ownership collapses into one group even when fan-out is allowed
+ groups, parallel = plan_refine_groups([_task("Seat", ["src/parts/seat.js"]), _task("overall")], allow_fanout=True)
+ assert not parallel and len(groups) == 1
+
+
+# --------------------------------------------------------------------- RunOptions wiring (F29)
+def test_spec_options_candidates_flow_into_the_policy(tmp_path, chair_plan, settings):
+ spec = make_spec(options=RunOptions(candidates=2))
+ ws = Workspace(tmp_path / "runs" / "r").create()
+ track = StaticObjectTrack(services=FakeServices(), settings=settings, runtime=FakeRuntime(Language.THREEJS))
+ ctx = track.build_context(spec, ws, EventLog(ws.events_path), RunState())
+ assert ctx.policy.n_candidates == 2 and ctx.state.extra["n_candidates"] == 2
+
+
+def test_constructor_candidates_beat_spec_options(tmp_path, settings):
+ spec = make_spec(options=RunOptions(candidates=2))
+ ws = Workspace(tmp_path / "runs" / "r").create()
+ track = StaticObjectTrack(services=FakeServices(), settings=settings, runtime=FakeRuntime(Language.THREEJS), n_candidates=3)
+ ctx = track.build_context(spec, ws, EventLog(ws.events_path), RunState())
+ assert ctx.policy.n_candidates == 3
+
+
+def test_legacy_run_state_width_survives_resume(tmp_path, settings):
+ spec = make_spec() # no options.candidates
+ ws = Workspace(tmp_path / "runs" / "r").create()
+ state = RunState()
+ state.extra["n_candidates"] = 4 # persisted by an older run
+ track = StaticObjectTrack(services=FakeServices(), settings=settings, runtime=FakeRuntime(Language.THREEJS))
+ ctx = track.build_context(spec, ws, EventLog(ws.events_path), state)
+ assert ctx.policy.n_candidates == 4
+
+
+def test_options_texture_triggers_texture_pass(tmp_path, chair_plan, settings, monkeypatch):
+ calls = {}
+
+ def fake_texture_pass(ws, spec, plan, **kw):
+ from types import SimpleNamespace
+
+ from codeverse.contracts.common import Usage
+
+ calls["ws"] = str(ws.root)
+ return SimpleNamespace(usage=Usage(backend="fake", cost_usd=0.01), summary=lambda: {"shipped": True})
+
+ import codeverse.texturing.run as trun
+
+ monkeypatch.setattr(trun, "texture_pass", fake_texture_pass)
+ spec = make_spec(max_rounds=0, options=RunOptions(texture=True))
+ ws = Workspace(tmp_path / "runs" / "r")
+ track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.9,)), agent=FakeAgent(_writer),
+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,
+ runtime=FakeRuntime(Language.THREEJS))
+ rec = track.run(spec, ws)
+ assert rec.status in (RunStatus.PASSED, RunStatus.PLATEAU)
+ assert calls, "options.texture must run the texture pass without the legacy tag"
+ assert rec.extra.get("texturing") == {"shipped": True}
+
+
+# --------------------------------------------------------------------- judge-view flags (F30 consumption)
+def _rs(flags: list[bool | None]) -> RenderSet:
+ views = [RenderView(name=f"v{i}", path=f"/tmp/v{i}.png", judge=f) for i, f in enumerate(flags)]
+ return RenderSet(views=views, renderer="fake")
+
+
+def test_scene_judge_views_prefers_stamped_flags():
+ from types import SimpleNamespace
+
+ pipe = ScenePipeline()
+ ctx = SimpleNamespace(services=FakeServices())
+ rs = _rs([True, False, True])
+ out = pipe.judge_views(ctx, rs)
+ assert [v.name for v in out.views] == ["v0", "v2"]
+ # legacy render set (no flags) falls back to select_judge_views (identity here: few views)
+ legacy = _rs([None, None])
+ out2 = pipe.judge_views(ctx, legacy)
+ assert len(out2.views) == 2
+
+
+def test_cli_judge_reconstructs_the_judged_subset(tmp_path):
+ from codeverse.cli._judge import judged_subset, resolve_paths
+
+ rs = _rs([True, False, None])
+ sub = judged_subset(rs)
+ assert [v.name for v in sub.views] == ["v0"]
+ assert judged_subset(_rs([None, None])).views == _rs([None, None]).views
+ # resolve_paths absolutises the stored out_dir alongside views/sheet
+ ws = Workspace(tmp_path / "ws").create()
+ rel = RenderSet(views=[RenderView(name="a", path="artifacts/renders/r00/a.png")],
+ contact_sheet="artifacts/renders/r00/sheet.png", out_dir="artifacts/renders/r00")
+ fixed = resolve_paths(ws, rel)
+ assert fixed.out_dir == str(ws.root / "artifacts/renders/r00")
+ assert fixed.views[0].path == str(ws.root / "artifacts/renders/r00/a.png")
+
+
+# --------------------------------------------------------------------- judge_context is ctx-free (F3)
+def test_judge_context_needs_no_run_context(tmp_path):
+ plan = ScenePlan.model_validate(plan_example(Track.SCENE))
+ ws = Workspace(tmp_path / "ws").create()
+ build = BuildResult(ok=True, language="scene_threejs")
+ text = ScenePipeline().judge_context(ws, plan, 0, build, [])
+ assert "Environment plan:" in text and "Cameras:" in text
+ assert ScenePipeline().judge_context(ws, None, 0, build, []) == ""
+
+
+def test_graphics_planner_hooks_charge_budget_on_planning_error(tmp_ws):
+ import pytest
+
+ from codeverse.contracts.spec import Budget
+ from codeverse.orchestrator.budget import BudgetGuard
+ from codeverse.tracks.graphics_steps import plan_graphics
+ from codeverse.tracks.planner import PlanningError
+
+ spec = make_spec(Track.GRAPHICS, Language.GLSL_SHADER)
+ budget = BudgetGuard(Budget(max_usd=5.0, max_minutes=10))
+ always_bad = FakeChatModel(lambda req: {"title": "x"})
+ with pytest.raises(PlanningError):
+ plan_graphics(spec, "fake:planner", tmp_ws, model=always_bad, budget=budget)
+ assert budget.spent.cost_usd > 0, "a failed re-ask is still paid for"
+
+
+def test_scene_track_uses_track_info_rubric():
+ from codeverse.contracts.common import TRACK_INFO
+
+ assert SceneTrack.rubric == TRACK_INFO[Track.SCENE].rubric == "scene_v1"
+ assert StaticObjectTrack.rubric == "static_object_v1"
diff --git a/harness/tests/scene_gates/test_judge_views.py b/harness/tests/scene_gates/test_judge_views.py
index 37189497..e4ae4180 100644
--- a/harness/tests/scene_gates/test_judge_views.py
+++ b/harness/tests/scene_gates/test_judge_views.py
@@ -63,3 +63,38 @@ def test_metrics_path_for_locates_sibling_file(tmp_path: Path):
assert metrics_path_for(rs) == out / "metrics.json"
assert metrics_path_for(RenderSet(contact_sheet=str(out / "sheet.png"))) == out / "metrics.json"
assert metrics_path_for(RenderSet()) is None
+
+
+def test_metrics_path_for_prefers_stamped_out_dir(tmp_path: Path):
+ out = tmp_path / "r02"
+ out.mkdir()
+ (out / "metrics.json").write_text("{}")
+ elsewhere = tmp_path / "elsewhere"
+ elsewhere.mkdir()
+ rs = RenderSet(out_dir=str(out), views=[RenderView(name="a", path=str(elsewhere / "a.png"))])
+ assert metrics_path_for(rs) == out / "metrics.json"
+ # stale out_dir (no metrics.json) → sibling-guessing fallback still works
+ (elsewhere / "metrics.json").write_text("{}")
+ rs2 = RenderSet(out_dir=str(tmp_path / "gone"), views=[RenderView(name="a", path=str(elsewhere / "a.png"))])
+ assert metrics_path_for(rs2) == elsewhere / "metrics.json"
+
+
+def test_mark_judge_views_serialises_stamped_flags(tmp_path: Path):
+ import json
+
+ from codeverse.spatial.render_scene import _mark_judge_views
+
+ views_json = tmp_path / "views.json"
+ views_json.write_text(json.dumps([
+ {"name": "a", "time_s": 0.0, "path": "a.png"},
+ {"name": "b", "time_s": 0.0, "path": "b.png"},
+ {"name": "a", "time_s": 1.5, "path": "a2.png"},
+ ]))
+ views = [
+ RenderView(name="a", path="a.png", time_s=0.0, judge=True),
+ RenderView(name="b", path="b.png", time_s=0.0, judge=False),
+ RenderView(name="a", path="a2.png", time_s=1.5, judge=True),
+ ]
+ _mark_judge_views(views_json, views)
+ flags = {(e["name"], e["time_s"]): e["judge"] for e in json.loads(views_json.read_text())}
+ assert flags == {("a", 0.0): True, ("b", 0.0): False, ("a", 1.5): True}
diff --git a/harness/tests/scene_gates/test_live_frames.py b/harness/tests/scene_gates/test_live_frames.py
index 6567ac3d..e89f4f3f 100644
--- a/harness/tests/scene_gates/test_live_frames.py
+++ b/harness/tests/scene_gates/test_live_frames.py
@@ -53,11 +53,15 @@ def test_example_scene_passes_frame_gate_and_judge_subset(starter_ws: Workspace)
assert m["framing_bbox"]["size"][0] < m["census"]["bbox"]["size"][0] # content, not the sky dome / ground
gate = frame_gate_from_renders(rs)
assert gate.gate == "scene_frames" and gate.passed, [f.message for f in gate.findings]
- # judge subset: ≤ 10 views, marked in views.json, sheet built from them only
- judge = select_judge_views(rs)
- assert len(judge.views) == JUDGE_MAX_VIEWS
- flags = {(v["name"], v["time_s"]) for v in json.loads((out / "views.json").read_text()) if v.get("judge")}
- assert flags == {(v.name, v.time_s) for v in judge.views}
+ # judge subset: stamped ONCE at render time (RenderView.judge + views.json), sheet built from it only
+ assert rs.out_dir == str(out)
+ assert all(v.judge is not None for v in rs.views)
+ stamped = {(v.name, v.time_s) for v in rs.views if v.judge}
+ assert len(stamped) == JUDGE_MAX_VIEWS
+ assert stamped == {(v.name, v.time_s) for v in select_judge_views(rs).views}
+ entries = json.loads((out / "views.json").read_text())
+ assert all("judge" in e for e in entries)
+ assert {(v["name"], v["time_s"]) for v in entries if v["judge"]} == stamped
sheet = Image.open(rs.contact_sheet)
tile_h = Image.open(rs.views[0].path).size[1]
assert sheet.size[1] < 5 * tile_h # 10 tiles in 4 columns → 3 rows, not 5
diff --git a/harness/tests/scene_runtime/test_assemble.py b/harness/tests/scene_runtime/test_assemble.py
index acf70f88..24409d7a 100644
--- a/harness/tests/scene_runtime/test_assemble.py
+++ b/harness/tests/scene_runtime/test_assemble.py
@@ -10,8 +10,7 @@ from codeverse.contracts.plan import CameraPlan
from codeverse.languages.scene_threejs.assemble import (
ZoneProbe,
assemble,
- derive_cameras,
- fit_zone_camera,
+ cameras_from_specs,
probe_zone_modules,
render_scene_js,
sun_azimuth,
@@ -22,32 +21,41 @@ from tests.scene_runtime.conftest import needs_browser, needs_node
BOX_A = {"min": [-10, 0, -10], "max": [10, 6, 10], "size": [20, 6, 20]}
BOX_B = {"min": [20, 0, -5], "max": [30, 3, 5], "size": [10, 3, 10]}
+# what probe_scene.mjs --sun-azimuth 90 emits for the two-zone layout above
+# (camera-fit math is owned by runtime_js/lib/orbit.mjs; tested in test_orbit.py)
+SPECS = {
+ "azimuth": 90,
+ "overview": {"position": [55.0, 22.0, 0.0], "lookAt": [10.0, 3.0, 0.0], "fov": 50},
+ "zones": [
+ {"group": "__zone__harbour", "position": [14.4, 2.2, 0.0], "lookAt": [0.0, 2.6, 0.0], "fov": 50},
+ {"group": "__zone__market", "position": [32.2, 1.9, 0.0], "lookAt": [25.0, 1.55, 0.0], "fov": 50},
+ ],
+}
-def test_derive_cameras_sun_side_and_order():
+
+def test_cameras_from_specs_sun_side_and_order():
probes = {
"harbour": ZoneProbe(name="harbour", file="src/zones/harbour.js", ok=True, bbox=BOX_A),
"market": ZoneProbe(name="market", file="src/zones/market.js", ok=True, bbox=BOX_B),
"broken": ZoneProbe(name="broken", file="src/zones/broken.js", ok=False, error="boom"),
}
- cams = derive_cameras(probes, sun_azimuth_deg=90, ground_y=0.0)
+ cams = cameras_from_specs(SPECS, probes)
assert [c.name for c in cams] == ["overview", "harbour_view", "market_view"]
ov = cams[0]
- # sun at azimuth 90 → eye on +X side of the union centre, above ground, looking at the centre
- assert ov.position[0] > ov.look_at[0] and ov.position[1] > 0.5
+ assert ov.position == (55.0, 22.0, 0.0) and ov.look_at == (10.0, 3.0, 0.0)
+ assert "sun side" in ov.purpose
hv = cams[1]
- assert hv.position[0] > 10 and 1.5 < hv.position[1] < 4.5 and abs(hv.position[2]) < 1e-6
- assert math.dist(hv.position, hv.look_at) > 10
-
-
-def test_derive_cameras_fallback_without_zones():
- cams = derive_cameras({}, sun_azimuth_deg=45)
- assert len(cams) == 1 and cams[0].name == "overview"
+ assert hv.position[0] > 10 and math.dist(hv.position, hv.look_at) > 10
+ assert cams[2].purpose == "zone Market from the sun side"
-def test_fit_zone_camera_respects_floor():
- c = fit_zone_camera("pier", BOX_B, azimuth=0, floor=2.0)
- assert c.position[1] >= 3.7
- assert c.name == "pier_view"
+def test_cameras_from_specs_fallback_without_zones_or_specs():
+ cams = cameras_from_specs({}, {})
+ assert len(cams) == 1 and cams[0].name == "overview" and "fallback" in cams[0].purpose
+ # measurable zones but no driver specs (e.g. probe ran without --sun-azimuth)
+ probes = {"harbour": ZoneProbe(name="harbour", file="src/zones/harbour.js", ok=True, bbox=BOX_A)}
+ cams2 = cameras_from_specs({}, probes)
+ assert len(cams2) == 1 and "fallback" in cams2[0].purpose
def test_render_scene_js_contents():
@@ -80,13 +88,19 @@ def test_assemble_without_probe_writes_all_zones(starter_ws):
def test_probe_zone_modules_reports_failures_and_bboxes(starter_ws):
(starter_ws.src / "zones" / "broken.js").write_text("export function build(ctx) { throw new Error('kaboom'); }\n")
(starter_ws.src / "zones" / "nobuild.js").write_text("export const x = 1;\n")
- probes, census, other = probe_zone_modules(starter_ws)
+ report = probe_zone_modules(starter_ws, sun_azimuth_deg=60.0)
+ probes, census, other = report # historical 3-tuple unpacking still works
assert probes["meadow"].ok and probes["meadow"].bbox and probes["meadow"].meshes > 5
assert probes["pondside"].ok
assert not probes["broken"].ok and "kaboom" in probes["broken"].error
assert not probes["nobuild"].ok and "no export build" in probes["nobuild"].error
assert census.get("ground_y") is not None
assert not (starter_ws.root / "src" / "_c3v_assemble_probe.js").exists()
+ # driver-fitted cameras: an overview plus one eye-level spec per measured zone group
+ specs = report.camera_specs
+ assert specs["azimuth"] == 60.0 and specs["overview"]["position"][1] > 0.5
+ zone_groups = {z["group"] for z in specs["zones"]}
+ assert {"__zone__meadow", "__zone__pondside"} <= zone_groups
@pytest.mark.node
diff --git a/harness/tests/scene_runtime/test_orbit.py b/harness/tests/scene_runtime/test_orbit.py
index 09e72098..8bde0406 100644
--- a/harness/tests/scene_runtime/test_orbit.py
+++ b/harness/tests/scene_runtime/test_orbit.py
@@ -59,3 +59,38 @@ def test_orbit_cameras_are_above_ground_and_fitted():
def test_empty_bbox_returns_no_cameras():
assert fit(None) == []
+
+
+def test_fit_zone_camera_respects_floor_and_sun_side():
+ """Single owner of the zone fit (assemble.py consumes it via probe_scene.mjs)."""
+ body = """
+import { fitZoneCamera } from './lib/orbit.mjs';
+const box = { min: [20, 0, -5], max: [30, 3, 5], size: [10, 3, 10] };
+console.log(JSON.stringify({
+ floored: fitZoneCamera(box, { azimuth: 0, floor: 2.0 }),
+ sun_x: fitZoneCamera(box, { azimuth: 90 }),
+}));
+"""
+ out = run_node_json(body)
+ assert out["floored"]["position"][1] >= 3.5 # floor 2.0 + eye height
+ assert out["floored"]["lookAt"][1] <= out["floored"]["position"][1]
+ # azimuth 90 → eye on the +X side, outside the footprint (x > 30)
+ assert out["sun_x"]["position"][0] > 30
+ assert out["sun_x"]["lookAt"] == [25, 1.5, 0]
+
+
+def test_fit_overview_camera_corner_fit_and_ground_aware():
+ body = """
+import { fitOverviewCamera, fitDistance, orbitDirection } from './lib/orbit.mjs';
+const box = { min: [-40, -1, -40], max: [40, 9, 40], size: [80, 10, 80] };
+const cam = fitOverviewCamera(box, { azimuth: 90, elevation: 30, groundY: 0.0 });
+const d = orbitDirection(90, 30);
+console.log(JSON.stringify({ cam, d }));
+"""
+ out = run_node_json(body)
+ cam = out["cam"]
+ # eye on the +X sun side, above ground, looking at the box centre
+ assert cam["position"][0] > 40 and cam["position"][1] > 0.5
+ assert cam["lookAt"] == [0, 4, 0]
+ dist = math.dist(cam["position"], cam["lookAt"])
+ assert 80 < dist < 200 # corner-frustum fit, not a loose sphere fit
diff --git a/harness/tests/scene_runtime/test_runtime.py b/harness/tests/scene_runtime/test_runtime.py
index 0a492566..727ab30c 100644
--- a/harness/tests/scene_runtime/test_runtime.py
+++ b/harness/tests/scene_runtime/test_runtime.py
@@ -65,3 +65,53 @@ def test_build_fails_on_import_error(starter_ws):
(starter_ws.src / "scene.js").write_text("import { x } from './nope.js';\nexport function createScene() {}\n")
res = SceneThreeJsRuntime().build(starter_ws)
assert not res.ok and res.error_file == "src/scene.js" and "nope.js" in res.error_message
+
+
+def test_build_interprets_combined_summary_offline(ws, monkeypatch):
+ """The single-boot build (probe_scene.mjs --compile) still yields the SAME two
+ GateReports: scene_probe from the probe summary, shader_preflight from the
+ embedded shader_report; a non-booting scene keeps the failed-empty shader gate."""
+ import codeverse.languages.scene_threejs.runtime as rt_mod
+ from codeverse.spatial.render_scene import NodeResult
+
+ calls: list[list[str]] = []
+
+ def fake_run(script, args, **kw):
+ calls.append([script, *args])
+ return NodeResult(0, "", "", {
+ "ok": True,
+ "boot": {"ok": True, "stage": "ready", "cameras": [{"name": "a"}], "camera_problems": []},
+ "update_ok": True, "census": {"totals": {"meshes": 3, "lights": 1}},
+ "console_errors": [], "console_warnings": [], "shader_errors": [],
+ "shader_report": {"ok": False, "errors": [
+ {"file": "src/shaders/x.js", "line": 7, "kind": "compile", "message": "fragment shader: bad", "fix_hint": "fix it"},
+ ], "warnings": [], "compile": {"ms": 3, "programs": 4, "custom_materials": 1}, "duration_ms": 9},
+ }, 5)
+
+ import codeverse.spatial.render_scene as rs_mod
+ monkeypatch.setattr(rs_mod, "run_scene_script", fake_run)
+ res = rt_mod.SceneThreeJsRuntime().build(ws)
+ assert len(calls) == 1 and calls[0][0] == "probe_scene.mjs" and "--compile" in calls[0]
+ assert not res.ok # shader gate failed
+ assert res.error_file == "src/shaders/x.js" and res.error_line == 7
+ import json as _json
+ probe = _json.loads((ws.artifacts / "gates" / "scene_probe.json").read_text())
+ shaders = _json.loads((ws.artifacts / "gates" / "shader_preflight.json").read_text())
+ assert probe["gate"] == "scene_probe" and probe["passed"]
+ assert shaders["gate"] == "shader_preflight" and not shaders["passed"]
+ assert shaders["findings"][0]["target"] == "src/shaders/x.js:7"
+ assert (ws.artifacts / "census.json").is_file()
+
+ # boot failure → shader gate failed-empty (old two-call behaviour preserved)
+ def fake_run_noboot(script, args, **kw):
+ return NodeResult(1, "", "", {
+ "ok": False, "boot": {"ok": False, "stage": "import", "error": "SyntaxError: x"},
+ "console_errors": [], "console_warnings": [], "shader_errors": [],
+ "shader_report": {"ok": False, "skipped": "scene did not boot", "errors": [], "warnings": []},
+ }, 5)
+
+ monkeypatch.setattr(rs_mod, "run_scene_script", fake_run_noboot)
+ res2 = rt_mod.SceneThreeJsRuntime().build(ws)
+ assert not res2.ok
+ shaders2 = _json.loads((ws.artifacts / "gates" / "shader_preflight.json").read_text())
+ assert not shaders2["passed"] and shaders2["findings"] == []
diff --git a/harness/tests/spatial_tools/test_measure.py b/harness/tests/spatial_tools/test_measure.py
index 200b4379..c98f7fd7 100644
--- a/harness/tests/spatial_tools/test_measure.py
+++ b/harness/tests/spatial_tools/test_measure.py
@@ -78,3 +78,37 @@ def test_cyclic_scene_graph_does_not_hang(tmp_path: Path) -> None:
scene.graph.update(frame_to="root", frame_from="world") # cycle: root → wrapper → world → root
assert "world" in _subtree_nodes(scene, "root") and len(_subtree_nodes(scene, "root")) == 3
assert [k for k, v in part_meshes(scene).items() if v is not None]
+
+
+# --------------------------------------------------------------------- parse memo (F24)
+def test_cached_parts_identity_and_invalidation(stool_glb: Path) -> None:
+ from codeverse.spatial.measure import cached_parts
+
+ p1 = cached_parts(stool_glb)
+ p2 = cached_parts(stool_glb)
+ assert p1 is not p2 # fresh dict per call
+ assert all(p1[k] is p2[k] for k in p1) # same stat → shared meshes
+ # rewriting the file (new mtime_ns/size) invalidates the entry
+ import os
+ os.utime(stool_glb, ns=(1, 1))
+ p3 = cached_parts(stool_glb)
+ assert set(p3) == set(p1) and all(p3[k] is not p1[k] for k in p3)
+
+
+def test_measure_glb_memoized_and_isolated(stool_glb: Path) -> None:
+ m1 = measure_glb(stool_glb)
+ m2 = measure_glb(stool_glb)
+ assert m1 is not m2 and m1 == m2
+ m1.extra["findings"].append("mutated") # a caller's edit never leaks back
+ assert measure_glb(stool_glb).extra["findings"] == []
+
+
+def test_load_scene_is_never_cached(stool_glb: Path) -> None:
+ """Texturing mutates scenes in place — load_scene must hand out fresh objects."""
+ from codeverse.spatial.measure import load_scene
+
+ s1 = load_scene(stool_glb)
+ s2 = load_scene(stool_glb)
+ assert s1 is not s2
+ g1 = s1.geometry[next(iter(s1.geometry))]
+ assert g1 is not s2.geometry[next(iter(s2.geometry))]
diff --git a/harness/tests/texturing/test_run.py b/harness/tests/texturing/test_run.py
index c041c682..25b55366 100644
--- a/harness/tests/texturing/test_run.py
+++ b/harness/tests/texturing/test_run.py
@@ -27,7 +27,7 @@ def test_texture_pass_ships_and_records(tmp_path, chair_glb, chair_spec, chair_p
ws = _ws(tmp_path, chair_glb, chair_spec, chair_plan)
judge = FakeJudge([(0.70, {"materials": 0.6, "intent_fidelity": 0.8}), (0.73, {"materials": 0.75, "intent_fidelity": 0.8})])
img = FakeImageModel(usd_per_image=0.05)
- rep = texture_pass(ws, chair_spec, chair_plan, model_id="", image_model=img, judge=judge, render=fake_render,
+ rep = texture_pass(ws, chair_spec, chair_plan, model_id="", image_model=img, judge_obj=judge, render=fake_render,
cache_dir=tmp_path / "cache")
assert isinstance(rep, TextureReport) and rep.shipped and rep.delta == 0.03
assert rep.plan.source == "default" and len(rep.textures.paths()) == 2
@@ -50,8 +50,8 @@ def test_texture_pass_ships_and_records(tmp_path, chair_glb, chair_spec, chair_p
def test_texture_pass_not_shipped_keeps_report(tmp_path, chair_glb, chair_spec, chair_plan):
ws = _ws(tmp_path, chair_glb, chair_spec, chair_plan)
judge = FakeJudge([(0.70, {"materials": 0.6}), (0.60, {"materials": 0.6})])
- rep = texture_pass(ws, chair_spec, chair_plan, model_id="", image_model=FakeImageModel(), judge=judge, render=fake_render,
- cache_dir=tmp_path / "cache")
+ rep = texture_pass(ws, chair_spec, chair_plan, model_id="", image_model=FakeImageModel(), judge_obj=judge,
+ render=fake_render, cache_dir=tmp_path / "cache")
assert not rep.shipped and rep.gate is not None and rep.glb_out.endswith("object_textured.glb")
assert json.loads(ws.record_path.read_text())["extra"]["texturing"]["shipped"] is False
@@ -66,3 +66,21 @@ def test_texture_pass_no_judge_and_all_failed(tmp_path, chair_glb, chair_spec, c
rep2 = texture_pass(ws2, chair_spec, chair_plan, model_id="", image_model=FakeImageModel(fail_on=("seamless",)), judge=False,
render=fake_render, cache_dir=tmp_path / "cache2")
assert not rep2.shipped and rep2.apply is None and rep2.glb_out == "" and rep2.textures.failed()
+
+
+def test_texture_services_bundle_and_deprecated_judge_object(tmp_path, chair_glb, chair_spec, chair_plan):
+ from codeverse.texturing.run import TextureServices
+
+ # services bundles the five injection params (explicit kwargs would win)
+ ws = _ws(tmp_path, chair_glb, chair_spec, chair_plan)
+ judge = FakeJudge([(0.70, {"materials": 0.6}), (0.73, {"materials": 0.75})])
+ services = TextureServices(image_model=FakeImageModel(), judge_obj=judge, render=fake_render,
+ cache_dir=tmp_path / "cache")
+ rep = texture_pass(ws, chair_spec, chair_plan, model_id="", services=services)
+ assert rep.shipped and rep.gate is not None
+ # the retired bool|judge union still works for one wave (warns, routes to judge_obj)
+ ws2 = _ws(tmp_path / "b", chair_glb, chair_spec, chair_plan)
+ judge2 = FakeJudge([(0.70, {"materials": 0.6}), (0.73, {"materials": 0.75})])
+ rep2 = texture_pass(ws2, chair_spec, chair_plan, model_id="", image_model=FakeImageModel(), judge=judge2,
+ render=fake_render, cache_dir=tmp_path / "cache2")
+ assert rep2.shipped and rep2.gate is not None
diff --git a/harness/tests/texturing/test_tools_cli.py b/harness/tests/texturing/test_tools_cli.py
index 232bd9a3..5df0cfd8 100644
--- a/harness/tests/texturing/test_tools_cli.py
+++ b/harness/tests/texturing/test_tools_cli.py
@@ -9,6 +9,7 @@ from typer.testing import CliRunner
from codeverse.spatial.registry import ToolContext, get_tool, list_tools
from codeverse.texturing.generate import FakeImageModel
+from codeverse.texturing.run import TextureServices
from codeverse.workspace import Workspace
from tests.texturing.conftest import FakeJudge, fake_render
@@ -32,8 +33,10 @@ def test_tools_registered_for_object_tracks():
def test_texture_pass_tool_runs_with_injected_fakes(tmp_path, chair_glb, chair_spec, chair_plan):
ws = _ws(tmp_path, chair_glb, chair_spec, chair_plan)
judge = FakeJudge([(0.7, {"materials": 0.5}), (0.72, {"materials": 0.7})])
+ services = TextureServices(image_model=FakeImageModel(), judge_obj=judge, render=fake_render,
+ cache_dir=tmp_path / "c")
ctx = ToolContext(workspace=ws, track="static_object", language="blender",
- extra={"image_model": FakeImageModel(), "judge": judge, "render": fake_render, "cache_dir": tmp_path / "c"})
+ extra={"texture_services": services})
obs = get_tool("texture_pass").call(ctx, {"model": ""})
assert obs.ok, obs.text
assert "SHIPPED" in obs.text and obs.numbers["shipped"] is True and obs.images
diff --git a/harness/tests/threejs_render/test_runtime_js.py b/harness/tests/threejs_render/test_runtime_js.py
index 2d60e0d6..4b43764c 100644
--- a/harness/tests/threejs_render/test_runtime_js.py
+++ b/harness/tests/threejs_render/test_runtime_js.py
@@ -74,17 +74,66 @@ def test_gpu_launch_probe(tmp_path: Path):
const {{ launchBrowser, rendererInfo }} = require({json.dumps(str(runtime_js_dir() / 'gpu_launch.cjs'))});
(async () => {{
const t0 = Date.now();
- const {{ browser, gpu, renderer }} = await launchBrowser({{ gpu: process.env.T_GPU || 'auto' }});
+ const handle = await launchBrowser({{ gpu: process.env.T_GPU || 'auto' }});
+ const {{ browser, gpu, renderer }} = handle;
const probe = await rendererInfo(browser);
- await browser.close();
- console.log(JSON.stringify({{ gpu, renderer, probe, ms: Date.now() - t0 }}));
+ await handle.release(); // never browser.close(): the browser may be shared
+ console.log(JSON.stringify({{ gpu, renderer, probe, shared: handle.shared, ms: Date.now() - t0 }}));
}})().catch((e) => {{ console.error(e); process.exit(1); }});
""")
- auto = run_node(script, [], timeout_s=120).last_json
+ env = {"CV3D_CACHE_DIR": str(tmp_path / "cache")}
+ auto = run_node(script, [], timeout_s=120, env_extra=env).last_json
assert auto["renderer"] and auto["probe"]
if auto["gpu"]:
assert "swiftshader" not in auto["renderer"].lower()
- off = run_node(script, [], timeout_s=120, env_extra={"T_GPU": "off"}).last_json
+ off = run_node(script, [], timeout_s=120, env_extra={**env, "T_GPU": "off"}).last_json
assert off["gpu"] is False and "swiftshader" in off["probe"].lower()
with pytest.raises(NodeError):
- run_node(script, [], timeout_s=60, env_extra={"T_GPU": "bogus"})
+ run_node(script, [], timeout_s=60, env_extra={**env, "T_GPU": "bogus"})
+ _reap_daemons(tmp_path / "cache")
+
+
+def _reap_daemons(cache_dir: Path) -> None:
+ """Kill any shared browser advertised under an ephemeral test cache dir."""
+ import contextlib
+ import os
+ import signal
+
+ for ep in cache_dir.glob("browser_*.json"):
+ if ep.name.endswith(".failed.json"):
+ continue
+ info = json.loads(ep.read_text())
+ for key in ("pid", "daemon_pid"):
+ if info.get(key):
+ with contextlib.suppress(OSError):
+ os.kill(int(info[key]), signal.SIGTERM)
+
+
+def test_gpu_launch_browser_reuse(tmp_path: Path):
+ """F21 phase 2: second launchBrowser connects to the daemon's shared browser
+ (~ms, shared=true); release() disconnects and the browser survives; with
+ CV3D_BROWSER_REUSE=off every launch is owned."""
+ script = tmp_path / "r.cjs"
+ script.write_text(f"""
+const {{ launchBrowser }} = require({json.dumps(str(runtime_js_dir() / 'gpu_launch.cjs'))});
+(async () => {{
+ const a = await launchBrowser({{ gpu: 'off' }});
+ const p = await a.browser.newPage();
+ await p.close();
+ await a.release();
+ const t0 = Date.now();
+ const b = await launchBrowser({{ gpu: 'off' }});
+ const reconnect_ms = Date.now() - t0;
+ const alive = b.browser.connected !== false;
+ await b.release();
+ console.log(JSON.stringify({{ a_shared: a.shared, b_shared: b.shared, reconnect_ms, alive }}));
+}})().catch((e) => {{ console.error(e); process.exit(1); }});
+""")
+ env = {"CV3D_CACHE_DIR": str(tmp_path / "cache")}
+ out = run_node(script, [], timeout_s=120, env_extra=env).last_json
+ assert out["a_shared"] is True and out["b_shared"] is True and out["alive"]
+ assert out["reconnect_ms"] < 1000 # connect, not a fresh ~550ms+ launch
+ assert (tmp_path / "cache" / "browser_cpu.json").is_file()
+ off = run_node(script, [], timeout_s=120, env_extra={**env, "CV3D_BROWSER_REUSE": "off"}).last_json
+ assert off["a_shared"] is False and off["b_shared"] is False
+ _reap_daemons(tmp_path / "cache")
diff --git a/harness/tests/urdf_joints/test_export.py b/harness/tests/urdf_joints/test_export.py
index 3655879e..8c6e1017 100644
--- a/harness/tests/urdf_joints/test_export.py
+++ b/harness/tests/urdf_joints/test_export.py
@@ -1,4 +1,4 @@
-"""urdf_to_glb hierarchy/extras and the articulation sheet composer."""
+"""urdf_to_glb hierarchy/extras and the articulation sheet (built by ``sheet.contact_sheet``)."""
from __future__ import annotations
@@ -17,7 +17,7 @@ from codeverse.spatial.joints import (
render_poses,
urdf_to_glb,
)
-from codeverse.spatial.joints_export import make_sheet
+from codeverse.spatial.sheet import LABEL_H, PAD, contact_sheet
from tests.urdf_joints.conftest import write_mesh_robot
@@ -53,7 +53,7 @@ def test_urdf_to_glb_hierarchy_and_extras(tmp_path):
assert _gltf_json(glb2)["scenes"][0]["extras"]["pose"] == {"hinge": 1.57}
-def test_make_sheet_and_render_poses_with_fake_renderer(tmp_path):
+def test_render_poses_with_fake_renderer_builds_sheet(tmp_path):
urdf, meshes = write_mesh_robot(tmp_path)
r = load_urdf(urdf, meshes)
calls = []
@@ -75,15 +75,18 @@ def test_make_sheet_and_render_poses_with_fake_renderer(tmp_path):
assert sheet.is_file() and Image.open(sheet).size[0] > 32
-def test_make_sheet_labels(tmp_path):
+def test_articulation_sheet_geometry_and_missing_tile(tmp_path):
+ """The articulation sheet is ``sheet.contact_sheet``: labelled grid, and a
+ missing pose PNG becomes a grey labelled tile instead of an OSError."""
imgs = []
for i in range(3):
p = tmp_path / f"{i}.png"
Image.new("RGB", (64, 48), (i * 50, 0, 0)).save(p)
imgs.append((f"img{i}", p))
- out = make_sheet(imgs, tmp_path / "s.png", cols=2, tile=64)
+ imgs.append(("missing", tmp_path / "nope.png")) # never written
+ out = contact_sheet(imgs, tmp_path / "s.png", cols=2, tile=64)
im = Image.open(out)
- assert im.size == (128, 2 * (64 + 22))
+ assert im.size == (2 * (64 + PAD) + PAD, 2 * (64 + LABEL_H + PAD) + PAD)
def test_multi_material_link_keeps_materials(tmp_path):