Official Codebase of 3DCodeVerse
Python
skill readout: eval counts a finding by the kind the harness routes on; an old record by its old shape
gaoypeng172a32446bd8
diff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.pyindex 90b8ca0a..edc2feed 100644--- a/eval/bench/skill_targets.py+++ b/eval/bench/skill_targets.py@@ -28,6 +28,7 @@ import statisticsimport sysfrom dataclasses import dataclassfrom pathlib import Path+from types import SimpleNamespacefrom typing import Anyfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):@@ -45,7 +46,7 @@ from codeverse3d.addons.skill_targets import ( # noqa: E402Target,)from codeverse3d.proc import write_text_atomic # noqa: E402-from codeverse3d.skills.registry import finding_kind # noqa: E402+from codeverse3d.skills.registry import UNTYPED, finding_kind, kind_matches # noqa: E402CACHE_NAME = ".skill_targets_cache.json"_FRAME_T = re.compile(r"_t(\d+(?:\.\d+)?)\.png$")@@ -120,6 +121,36 @@ def ab_arms(root: Path) -> dict[str, Path] | None:# --------------------------------------------------------------------------- metrics+#: How a record written before its gate typed ``data.kind`` (the contract gate until+#: 2026-09-24, connectivity until 2026-08-30) names a finding the targets count: the+#: harness routes on the typed kind only, and these message shapes are the old gates'.+_UNTYPED_SHAPES: tuple[tuple[str, re.Pattern[str], str], ...] = tuple(+ (gate, re.compile(pattern), kind) for gate, pattern, kind in (+ ("connectivity", r"interpenetrate", "connectivity/penetration"),+ ("contract", r"\(each instance\)|\(all instances\)", "contract/instance_bbox"),+ ("contract", r"part '.*' bbox deviates", "contract/part_bbox"),+ ("contract", r"overall bbox deviates", "contract/overall_bbox"),+ ("contract", r"footprint centre", "contract/footprint_offset"),+ ("contract", r"floats .* the ground", "contract/ground_gap"),+ ("contract", r"missing from the GLB", "contract/missing_part"),+ ("contract", r"instance\(s\), plan asks for", "contract/instance_count"),+ ("joint_sweep", r"overlap by|overlap in \d+ of the sampled poses|more overlapping pair", "joint_sweep/penetration"),+ ("joint_sweep", r"apart at rest|nothing physically connects", "joint_sweep/unattached"),+ ("scene_frames", r"too dark", "scene_frames/dark_frame"),+ ("scene_frames", r"flat frame", "scene_frames/flat_frame"),+ ("scene_frames", r"BELOW the ground|inside / touching geometry", "scene_frames/camera_in_geometry"),+ ))+++def _kind_of(gate: str, f: dict, severity: str) -> str | None:+ """``skills.registry.finding_kind``, with the old message shapes for an untyped record."""+ kind = finding_kind(SimpleNamespace(gate=str(f.get("gate") or gate), severity=severity, data=f.get("data") or {}))+ if kind is None or not kind.endswith(f"/{UNTYPED}"):+ return kind+ msg = str(f.get("message", ""))+ return next((k for g, pat, k in _UNTYPED_SHAPES if kind == f"{g}/{UNTYPED}" and pat.search(msg)), kind)++def count_kinds(rd: dict, kinds: tuple[str, ...], severities: tuple[str, ...]) -> int:n = 0for g in rd.get("gates") or []:@@ -127,7 +158,8 @@ def count_kinds(rd: dict, kinds: tuple[str, ...], severities: tuple[str, ...]) -sev = str(f.get("severity", "warn")).lower()if severities and sev not in severities:continue- if finding_kind(str(g.get("gate", "")), str(f.get("message", "")), sev) in kinds:+ kind = _kind_of(str(g.get("gate", "")), f, sev)+ if kind is not None and kind_matches(kind, kinds):n += 1return ndiff --git a/eval/tests/test_skill_targets_readout.py b/eval/tests/test_skill_targets_readout.pyindex 9a89cf3a..52ae48c9 100644--- a/eval/tests/test_skill_targets_readout.py+++ b/eval/tests/test_skill_targets_readout.py@@ -146,3 +146,16 @@ def test_a_glb_row_measures_the_measured_rounds_own_glb(tmp_path, monkeypatch):run = st.load_runs(root)[0]assert st.measure(target, run, which="first") == 10.0assert st.measure(target, run, which="last") == 11.0+++def test_a_typed_finding_counts_by_its_kind_and_an_old_record_by_its_shape():+ """N52: the harness routes on ``data.kind``; the readout counts the same kind, and reads a+ record written before its gate typed the finding by the old message shape."""+ from bench.skill_targets import count_kinds++ typed = {"gate": "contract", "severity": "warn", "message": "reworded", "data": {"kind": "part_bbox"}}+ camera = {"gate": "scene_frames", "severity": "warn", "message": "x", "data": {"kind": "camera_below_high_ground"}}+ rd = {"gates": [{"gate": "contract", "findings": [typed, *CONTRACT]},+ {"gate": "scene_frames", "findings": [camera]}]}+ assert count_kinds(rd, ("contract/part_bbox",), ()) == 2+ assert count_kinds(rd, ("scene_frames/camera_*",), ()) == 1diff --git a/harness/codeverse3d/skills/registry.py b/harness/codeverse3d/skills/registry.pyindex 43faf05a..890bfd0a 100644--- a/harness/codeverse3d/skills/registry.py+++ b/harness/codeverse3d/skills/registry.py@@ -62,6 +62,11 @@ def finding_kind(finding: Any) -> str | None:return f"{family}/{kind if isinstance(kind, str) and kind else UNTYPED}"+def kind_matches(kind: str, patterns: tuple[str, ...]) -> bool:+ """``kind`` is one of ``patterns``: an exact kind, or a family ending in ``*`` (``gl_frames/*``)."""+ return any(kind.startswith(p[:-1]) if p.endswith("*") else kind == p for p in patterns)++def finding_kinds(findings: object) -> list[str]:"""Distinct kinds of an iterable of ``GateFinding`` (or of ``GateReport``s), in order."""out: list[str] = []@@ -112,12 +117,7 @@ class Route:def matches_finding(self, kinds: list[str]) -> str | None:"""The first routed finding kind this row answers, or None."""- for pat in self.findings:- family = pat[:-1] if pat.endswith("*") else ""- for k in kinds:- if (family and k.startswith(family)) or k == pat:- return k- return None+ return next((k for pat in self.findings for k in kinds if kind_matches(k, (pat,))), None)ROUTES: tuple[Route, ...] = (