Skip to content

Official Codebase of 3DCodeVerse

Python

candidates: best-of-N was paying for N and keeping the first one blind on graphics

gaoypeng4db61bdfdd67
diff --git a/harness/codeverse/tracks/candidates.py b/harness/codeverse/tracks/candidates.py
index 2a14fbef..cd2a0da7 100644
--- a/harness/codeverse/tracks/candidates.py
+++ b/harness/codeverse/tracks/candidates.py
@@ -166,7 +166,8 @@ def run_candidate(track: Any, sub: RunContext, k: int, tasks: Sequence[Generatio
if outcome.build.ok:
measurement = pipeline.measure(sub, outcome.build)
gates.extend(pipeline.gates(sub, 0, outcome.build, measurement))
- renders = quick_render(sub, outcome.build.glb_path)
+ renders = quick_render(sub, outcome.build.glb_path, pipeline=pipeline,
+ build=outcome.build, measurement=measurement)
if renders is not None and renders.views:
rec.sheet = renders.contact_sheet or ""
judgment = quick_judge(track, sub, pipeline, outcome.build, gates, measurement, renders)
@@ -184,12 +185,34 @@ def run_candidate(track: Any, sub: RunContext, k: int, tasks: Sequence[Generatio
return rec, renders
-def quick_render(ctx: RunContext, glb_path: str | None) -> RenderSet | None:
- if not glb_path:
- return None
+def quick_render(ctx: RunContext, glb_path: str | None, *, pipeline: RoundPipeline | None = None,
+ build: Any | None = None, measurement: Any | None = None) -> RenderSet | None:
+ """Cheap renders for ranking a candidate — by the route THIS track actually has.
+
+ An object has a GLB and gets the reduced-view, reduced-resolution rig, which is the
+ whole point of a "quick" render. A shader has no GLB at all, and returning None there
+ meant the candidate was never judged, scored None, and best-of-N fell through to index
+ 0 — so every extra candidate was generated, paid for and discarded unlooked-at.
+
+ Measured 2026-08-25 across the teaser battery: static_object and articulated_object
+ candidates all carried real scores, and all SIX graphics runs recorded
+ ``scores: {c0: null, c1: null}`` with ``selected: 0``, two of them at n=3. No
+ ``candidate.judge_failed`` event fired anywhere — nothing errored, the judge was simply
+ never reached.
+
+ So when there is no GLB, fall back to ``pipeline.render``, which is the renderer the
+ track uses for its real rounds (frames for graphics, authored cameras for a scene).
+ That costs more than the quick rig; it costs less than paying for N generations and
+ keeping the first one blind.
+ """
out_dir = ctx.ws.renders_dir(0) / "quick"
try:
- return ctx.services.render_object(Path(glb_path), out_dir, views=OBJECT_VIEWS_QUICK, width=QUICK_PX, height=QUICK_PX)
+ if glb_path:
+ return ctx.services.render_object(Path(glb_path), out_dir, views=OBJECT_VIEWS_QUICK,
+ width=QUICK_PX, height=QUICK_PX)
+ if pipeline is None or build is None:
+ return None
+ return pipeline.render(ctx, 0, build, measurement)
except Exception as e: # noqa: BLE001 — a candidate without renders is ranked by gates only
log.warning("quick render failed for %s: %s", ctx.ws.root, e)
ctx.events.emit("candidate.render_failed", workspace=str(ctx.ws.root), error=f"{type(e).__name__}: {e}")
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 95c25d25..697e5694 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1038,6 +1038,18 @@ plumbing through several layers and deserves its own measured change.
## 23. The key pool is per-PROCESS, so N batteries multiply the quota by N
+> **Count processes with `pool_budget()`, never with a hand-rolled `pgrep`.** A gate like
+> `pgrep -af 'bin/3dcv make' | grep -c 'codex:'` **counts itself**: the pattern text is in
+> the checking shell's own command line, so it sees phantom runs. Measured 2026-08-25 on an
+> idle box — zero `3dcv` processes running, the naive gate returned **3**, and even the
+> bracket trick `pgrep -f '[b]in/3dcv make'` returned **2**, because the wrapper shell's
+> argv also carries the string. Used in `while [ $(gate) -ge 3 ]; do sleep 60; done` that
+> blocks forever on nothing. `codeverse.models.health.sibling_processes()` reads `/proc`
+> and excludes its own pid; `pool_budget()` wraps it with the in-flight arithmetic:
+> ```
+> python3 -c "from codeverse.models.health import pool_budget; print(pool_budget())"
+> ```
+
§20 measured the concurrency knee at 64 in-flight and shipped it as the default. That
number was measured with **one process and nothing else running**, and the limiter it
configures is per-process by construction:
diff --git a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
index f6435ed0..3e3ecbbd 100644
--- a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
+++ b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
@@ -380,3 +380,67 @@ def test_default_motion_checks_on_real_urdf(tmp_path):
assert "WRONG" in bad.errors[0].message and '<axis xyz="-0 -0 -1"/>' in bad.errors[0].fix_hint
plan.joints[0].motion = "rotates" # ambiguous → no gate
assert default_motion_checks(robot(tmp_path / "none", 1), plan) is None
+
+
+# ------------------------------------------------- best-of-N on a track with no GLB
+def test_a_track_without_a_glb_still_gets_candidate_renders():
+ """`quick_render` hardcoded the GLB rig, so tracks that produce no GLB got None —
+ the candidate was never judged, scored None, and best-of-N fell through to index 0.
+
+ Measured 2026-08-25 across the teaser battery: static_object and articulated_object
+ candidates all carried real scores; all SIX graphics runs recorded
+ `scores: {c0: null, c1: null}` with `selected: 0`, two of them at n=3. No
+ `candidate.judge_failed` event fired anywhere — nothing errored, the judge was simply
+ never reached, and every extra candidate was generated, paid for and discarded blind.
+ """
+ from pathlib import Path
+
+ from codeverse.tracks.candidates import quick_render
+
+ class _Pipeline:
+ def __init__(self): self.calls = []
+ def render(self, ctx, round_index, build, measurement):
+ self.calls.append((round_index, build, measurement))
+ return "frames-renderset"
+
+ class _WS:
+ def renders_dir(self, i): return Path("/tmp/unused")
+
+ class _Ctx:
+ ws = _WS()
+
+ pipe = _Pipeline()
+ got = quick_render(_Ctx(), None, pipeline=pipe, build="BUILD", measurement="MEAS")
+ assert got == "frames-renderset", "no GLB must fall back to the track's own renderer"
+ assert pipe.calls == [(0, "BUILD", "MEAS")]
+
+ # and with nothing to fall back to, it still degrades quietly rather than raising
+ assert quick_render(_Ctx(), None) is None
+
+
+def test_an_object_candidate_still_uses_the_cheap_rig(tmp_path):
+ """The fallback must not make the object path more expensive: a GLB still goes through
+ the reduced-view, reduced-resolution quick rig, not pipeline.render."""
+ from codeverse.tracks.candidates import OBJECT_VIEWS_QUICK, QUICK_PX, quick_render
+
+ seen = {}
+
+ class _Services:
+ def render_object(self, glb, out_dir, *, views, width, height):
+ seen.update(glb=glb, views=views, width=width, height=height)
+ return "object-renderset"
+
+ class _WS:
+ def renders_dir(self, i): return tmp_path
+
+ class _Ctx:
+ ws = _WS()
+ services = _Services()
+
+ class _Pipeline:
+ def render(self, *a, **k): # must not be reached
+ raise AssertionError("an object candidate must not use pipeline.render")
+
+ got = quick_render(_Ctx(), "/x/object.glb", pipeline=_Pipeline(), build="B", measurement="M")
+ assert got == "object-renderset"
+ assert seen["views"] == OBJECT_VIEWS_QUICK and seen["width"] == QUICK_PX