Official Codebase of 3DCodeVerse
Python
bench: the shader-traps readout counts the shader_preflight gate — it read a key the raw report never had
gaoypeng82115b60d613
diff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.pyindex 6f03dc85..32da9edd 100644--- a/eval/bench/skill_targets.py+++ b/eval/bench/skill_targets.py@@ -176,16 +176,21 @@ def _min_authored_changed_frac(d: Path) -> float | None:return min((float(r.changed_frac) for r in rows), default=None)-def _shader_preflight_findings(run: Run) -> float | None:- """``artifacts/shader_preflight.json`` — written beside the run, not into the record."""- p = run.dir / "artifacts" / "shader_preflight.json"- if not p.is_file():- return None- try:- rep = json.loads(p.read_text())- except (OSError, json.JSONDecodeError):+def _shader_preflight_findings(run: Run, rd: dict) -> float | None:+ """WARN + ERROR findings of the build's shader_preflight gate: the round's own copy since+ 2026-09-22 (``BuildResult.gates``), else the last build's ``artifacts/gates/shader_preflight.json``.+ The raw driver report beside it has ``errors`` / ``warnings`` and no ``findings``: reading+ that counted 0 on every run before 2026-09-22."""+ gates = [g for g in rd.get("gates") or [] if isinstance(g, dict) and g.get("gate") == "shader_preflight"]+ if not gates:+ p = run.dir / "artifacts" / "gates" / "shader_preflight.json"+ try:+ gates = [json.loads(p.read_text())] if p.is_file() else []+ except (OSError, json.JSONDecodeError):+ return None+ if not gates:return None- return float(sum(1 for f in rep.get("findings") or []+ return float(sum(1 for g in gates for f in g.get("findings") or []if str(f.get("severity", "")).lower() in ("warn", "error")))@@ -203,7 +208,8 @@ def measure(target: Target, run: Run, *, which: str = "last",return Nonereturn sum(0.0 if b.get("ok") else 1.0 for b in builds) / len(builds)if target.source == SRC_ARTIFACT:- return None if run.gated_round(which) is None else _shader_preflight_findings(run)+ rd = run.gated_round(which)+ return None if rd is None else _shader_preflight_findings(run, rd)key = f"{target.metric}:{run.path}:{which}"if cache is not None and key in cache:diff --git a/eval/tests/test_skill_targets_readout.py b/eval/tests/test_skill_targets_readout.pyindex e30c08e3..3f363b4a 100644--- a/eval/tests/test_skill_targets_readout.py+++ b/eval/tests/test_skill_targets_readout.py@@ -143,3 +143,23 @@ def test_a_retried_cell_pairs_on_the_attempt_that_reached_a_gate(tmp_path):(cell / leaf / "record.json").write_text(json.dumps(rec))rows = {r["skill"]: r for r in ab_rows(ab_arms(root), list(TARGETS), which="last", cache=None)}assert rows["c3d-part-contact"]["n"] == 1+++def test_the_shader_preflight_row_reads_the_gate_not_the_raw_driver_json(tmp_path):+ """The raw driver report has errors / warnings and no findings; the row read it and+ counted 0 on every run. It reads the round's shader_preflight gate, else the last+ build's gate file."""+ from bench.skill_targets import battery_rows, load_runs++ warn = _finding("shader_preflight", "ShaderMaterial 'Neon' ignores scene.fog")+ root = _battery(tmp_path, {+ "in_round": _record("scene_threejs", [{"gate": "shader_preflight", "findings": [warn, warn]}]),+ "on_disk": _record("scene_threejs", [{"gate": "scene_placement", "findings": []}]),+ })+ gate_dir = root / "runs" / "on_disk" / "artifacts" / "gates"+ gate_dir.mkdir(parents=True)+ (gate_dir / "shader_preflight.json").write_text(json.dumps({"gate": "shader_preflight", "findings": [warn]}))+ (gate_dir.parent / "shader_preflight.json").write_text(json.dumps({"errors": [], "warnings": ["x"]}))+ rows = {r["skill"]: r for r in battery_rows(load_runs(root), list(TARGETS), which="last", cache=None)}+ assert rows["c3d-threejs-shader-traps"]["n"] == 2+ assert rows["c3d-threejs-shader-traps"]["mean"] == 1.5 # 2 in the round, 1 in the gate file