Skip to content

Official Codebase of 3DCodeVerse

Python

bench: the shader-traps readout counts the shader_preflight gate — it read a key the raw report never had

gaoypeng82115b60d613
diff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.py
index 6f03dc85..32da9edd 100644
--- a/eval/bench/skill_targets.py
+++ b/eval/bench/skill_targets.py
@@ -176,16 +176,21 @@ def _min_authored_changed_frac(d: Path) -> float | None:
return min((float(r.changed_frac) for r in rows), default=None)
-def _shader_preflight_findings(run: Run) -> float | None:
- """``artifacts/shader_preflight.json`` — written beside the run, not into the record."""
- p = run.dir / "artifacts" / "shader_preflight.json"
- if not p.is_file():
- return None
- try:
- rep = json.loads(p.read_text())
- except (OSError, json.JSONDecodeError):
+def _shader_preflight_findings(run: Run, rd: dict) -> float | None:
+ """WARN + ERROR findings of the build's shader_preflight gate: the round's own copy since
+ 2026-09-22 (``BuildResult.gates``), else the last build's ``artifacts/gates/shader_preflight.json``.
+ The raw driver report beside it has ``errors`` / ``warnings`` and no ``findings``: reading
+ that counted 0 on every run before 2026-09-22."""
+ gates = [g for g in rd.get("gates") or [] if isinstance(g, dict) and g.get("gate") == "shader_preflight"]
+ if not gates:
+ p = run.dir / "artifacts" / "gates" / "shader_preflight.json"
+ try:
+ gates = [json.loads(p.read_text())] if p.is_file() else []
+ except (OSError, json.JSONDecodeError):
+ return None
+ if not gates:
return None
- return float(sum(1 for f in rep.get("findings") or []
+ return float(sum(1 for g in gates for f in g.get("findings") or []
if str(f.get("severity", "")).lower() in ("warn", "error")))
@@ -203,7 +208,8 @@ def measure(target: Target, run: Run, *, which: str = "last",
return None
return sum(0.0 if b.get("ok") else 1.0 for b in builds) / len(builds)
if target.source == SRC_ARTIFACT:
- return None if run.gated_round(which) is None else _shader_preflight_findings(run)
+ rd = run.gated_round(which)
+ return None if rd is None else _shader_preflight_findings(run, rd)
key = f"{target.metric}:{run.path}:{which}"
if cache is not None and key in cache:
diff --git a/eval/tests/test_skill_targets_readout.py b/eval/tests/test_skill_targets_readout.py
index e30c08e3..3f363b4a 100644
--- a/eval/tests/test_skill_targets_readout.py
+++ b/eval/tests/test_skill_targets_readout.py
@@ -143,3 +143,23 @@ def test_a_retried_cell_pairs_on_the_attempt_that_reached_a_gate(tmp_path):
(cell / leaf / "record.json").write_text(json.dumps(rec))
rows = {r["skill"]: r for r in ab_rows(ab_arms(root), list(TARGETS), which="last", cache=None)}
assert rows["c3d-part-contact"]["n"] == 1
+
+
+def test_the_shader_preflight_row_reads_the_gate_not_the_raw_driver_json(tmp_path):
+ """The raw driver report has errors / warnings and no findings; the row read it and
+ counted 0 on every run. It reads the round's shader_preflight gate, else the last
+ build's gate file."""
+ from bench.skill_targets import battery_rows, load_runs
+
+ warn = _finding("shader_preflight", "ShaderMaterial 'Neon' ignores scene.fog")
+ root = _battery(tmp_path, {
+ "in_round": _record("scene_threejs", [{"gate": "shader_preflight", "findings": [warn, warn]}]),
+ "on_disk": _record("scene_threejs", [{"gate": "scene_placement", "findings": []}]),
+ })
+ gate_dir = root / "runs" / "on_disk" / "artifacts" / "gates"
+ gate_dir.mkdir(parents=True)
+ (gate_dir / "shader_preflight.json").write_text(json.dumps({"gate": "shader_preflight", "findings": [warn]}))
+ (gate_dir.parent / "shader_preflight.json").write_text(json.dumps({"errors": [], "warnings": ["x"]}))
+ rows = {r["skill"]: r for r in battery_rows(load_runs(root), list(TARGETS), which="last", cache=None)}
+ assert rows["c3d-threejs-shader-traps"]["n"] == 2
+ assert rows["c3d-threejs-shader-traps"]["mean"] == 1.5 # 2 in the round, 1 in the gate file