Official Codebase of 3DCodeVerse
Python
candidates: best-of-N was paying for N and keeping the first one blind on graphics
gaoypeng4db61bdfdd67
diff --git a/harness/codeverse/tracks/candidates.py b/harness/codeverse/tracks/candidates.pyindex 2a14fbef..cd2a0da7 100644--- a/harness/codeverse/tracks/candidates.py+++ b/harness/codeverse/tracks/candidates.py@@ -166,7 +166,8 @@ def run_candidate(track: Any, sub: RunContext, k: int, tasks: Sequence[Generatioif outcome.build.ok:measurement = pipeline.measure(sub, outcome.build)gates.extend(pipeline.gates(sub, 0, outcome.build, measurement))- renders = quick_render(sub, outcome.build.glb_path)+ renders = quick_render(sub, outcome.build.glb_path, pipeline=pipeline,+ build=outcome.build, measurement=measurement)if renders is not None and renders.views:rec.sheet = renders.contact_sheet or ""judgment = quick_judge(track, sub, pipeline, outcome.build, gates, measurement, renders)@@ -184,12 +185,34 @@ def run_candidate(track: Any, sub: RunContext, k: int, tasks: Sequence[Generatioreturn rec, renders-def quick_render(ctx: RunContext, glb_path: str | None) -> RenderSet | None:- if not glb_path:- return None+def quick_render(ctx: RunContext, glb_path: str | None, *, pipeline: RoundPipeline | None = None,+ build: Any | None = None, measurement: Any | None = None) -> RenderSet | None:+ """Cheap renders for ranking a candidate — by the route THIS track actually has.++ An object has a GLB and gets the reduced-view, reduced-resolution rig, which is the+ whole point of a "quick" render. A shader has no GLB at all, and returning None there+ meant the candidate was never judged, scored None, and best-of-N fell through to index+ 0 — so every extra candidate was generated, paid for and discarded unlooked-at.++ Measured 2026-08-25 across the teaser battery: static_object and articulated_object+ candidates all carried real scores, and all SIX graphics runs recorded+ ``scores: {c0: null, c1: null}`` with ``selected: 0``, two of them at n=3. No+ ``candidate.judge_failed`` event fired anywhere — nothing errored, the judge was simply+ never reached.++ So when there is no GLB, fall back to ``pipeline.render``, which is the renderer the+ track uses for its real rounds (frames for graphics, authored cameras for a scene).+ That costs more than the quick rig; it costs less than paying for N generations and+ keeping the first one blind.+ """out_dir = ctx.ws.renders_dir(0) / "quick"try:- return ctx.services.render_object(Path(glb_path), out_dir, views=OBJECT_VIEWS_QUICK, width=QUICK_PX, height=QUICK_PX)+ if glb_path:+ return ctx.services.render_object(Path(glb_path), out_dir, views=OBJECT_VIEWS_QUICK,+ width=QUICK_PX, height=QUICK_PX)+ if pipeline is None or build is None:+ return None+ return pipeline.render(ctx, 0, build, measurement)except Exception as e: # noqa: BLE001 — a candidate without renders is ranked by gates onlylog.warning("quick render failed for %s: %s", ctx.ws.root, e)ctx.events.emit("candidate.render_failed", workspace=str(ctx.ws.root), error=f"{type(e).__name__}: {e}")diff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 95c25d25..697e5694 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1038,6 +1038,18 @@ plumbing through several layers and deserves its own measured change.## 23. The key pool is per-PROCESS, so N batteries multiply the quota by N+> **Count processes with `pool_budget()`, never with a hand-rolled `pgrep`.** A gate like+> `pgrep -af 'bin/3dcv make' | grep -c 'codex:'` **counts itself**: the pattern text is in+> the checking shell's own command line, so it sees phantom runs. Measured 2026-08-25 on an+> idle box — zero `3dcv` processes running, the naive gate returned **3**, and even the+> bracket trick `pgrep -f '[b]in/3dcv make'` returned **2**, because the wrapper shell's+> argv also carries the string. Used in `while [ $(gate) -ge 3 ]; do sleep 60; done` that+> blocks forever on nothing. `codeverse.models.health.sibling_processes()` reads `/proc`+> and excludes its own pid; `pool_budget()` wraps it with the in-flight arithmetic:+> ```+> python3 -c "from codeverse.models.health import pool_budget; print(pool_budget())"+> ```+§20 measured the concurrency knee at 64 in-flight and shipped it as the default. Thatnumber was measured with **one process and nothing else running**, and the limiter itconfigures is per-process by construction:diff --git a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.pyindex f6435ed0..3e3ecbbd 100644--- a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py+++ b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py@@ -380,3 +380,67 @@ def test_default_motion_checks_on_real_urdf(tmp_path):assert "WRONG" in bad.errors[0].message and '<axis xyz="-0 -0 -1"/>' in bad.errors[0].fix_hintplan.joints[0].motion = "rotates" # ambiguous → no gateassert default_motion_checks(robot(tmp_path / "none", 1), plan) is None+++# ------------------------------------------------- best-of-N on a track with no GLB+def test_a_track_without_a_glb_still_gets_candidate_renders():+ """`quick_render` hardcoded the GLB rig, so tracks that produce no GLB got None —+ the candidate was never judged, scored None, and best-of-N fell through to index 0.++ Measured 2026-08-25 across the teaser battery: static_object and articulated_object+ candidates all carried real scores; all SIX graphics runs recorded+ `scores: {c0: null, c1: null}` with `selected: 0`, two of them at n=3. No+ `candidate.judge_failed` event fired anywhere — nothing errored, the judge was simply+ never reached, and every extra candidate was generated, paid for and discarded blind.+ """+ from pathlib import Path++ from codeverse.tracks.candidates import quick_render++ class _Pipeline:+ def __init__(self): self.calls = []+ def render(self, ctx, round_index, build, measurement):+ self.calls.append((round_index, build, measurement))+ return "frames-renderset"++ class _WS:+ def renders_dir(self, i): return Path("/tmp/unused")++ class _Ctx:+ ws = _WS()++ pipe = _Pipeline()+ got = quick_render(_Ctx(), None, pipeline=pipe, build="BUILD", measurement="MEAS")+ assert got == "frames-renderset", "no GLB must fall back to the track's own renderer"+ assert pipe.calls == [(0, "BUILD", "MEAS")]++ # and with nothing to fall back to, it still degrades quietly rather than raising+ assert quick_render(_Ctx(), None) is None+++def test_an_object_candidate_still_uses_the_cheap_rig(tmp_path):+ """The fallback must not make the object path more expensive: a GLB still goes through+ the reduced-view, reduced-resolution quick rig, not pipeline.render."""+ from codeverse.tracks.candidates import OBJECT_VIEWS_QUICK, QUICK_PX, quick_render++ seen = {}++ class _Services:+ def render_object(self, glb, out_dir, *, views, width, height):+ seen.update(glb=glb, views=views, width=width, height=height)+ return "object-renderset"++ class _WS:+ def renders_dir(self, i): return tmp_path++ class _Ctx:+ ws = _WS()+ services = _Services()++ class _Pipeline:+ def render(self, *a, **k): # must not be reached+ raise AssertionError("an object candidate must not use pipeline.render")++ got = quick_render(_Ctx(), "/x/object.glb", pipeline=_Pipeline(), build="B", measurement="M")+ assert got == "object-renderset"+ assert seen["views"] == OBJECT_VIEWS_QUICK and seen["width"] == QUICK_PX