Skip to content

Official Codebase of 3DCodeVerse

Python

skill readout: eval counts a finding by the kind the harness routes on; an old record by its old shape

gaoypeng172a32446bd8
diff --git a/eval/bench/skill_targets.py b/eval/bench/skill_targets.py
index 90b8ca0a..edc2feed 100644
--- a/eval/bench/skill_targets.py
+++ b/eval/bench/skill_targets.py
@@ -28,6 +28,7 @@ import statistics
import sys
from dataclasses import dataclass
from pathlib import Path
+from types import SimpleNamespace
from typing import Any
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
@@ -45,7 +46,7 @@ from codeverse3d.addons.skill_targets import ( # noqa: E402
Target,
)
from codeverse3d.proc import write_text_atomic # noqa: E402
-from codeverse3d.skills.registry import finding_kind # noqa: E402
+from codeverse3d.skills.registry import UNTYPED, finding_kind, kind_matches # noqa: E402
CACHE_NAME = ".skill_targets_cache.json"
_FRAME_T = re.compile(r"_t(\d+(?:\.\d+)?)\.png$")
@@ -120,6 +121,36 @@ def ab_arms(root: Path) -> dict[str, Path] | None:
# --------------------------------------------------------------------------- metrics
+#: How a record written before its gate typed ``data.kind`` (the contract gate until
+#: 2026-09-24, connectivity until 2026-08-30) names a finding the targets count: the
+#: harness routes on the typed kind only, and these message shapes are the old gates'.
+_UNTYPED_SHAPES: tuple[tuple[str, re.Pattern[str], str], ...] = tuple(
+ (gate, re.compile(pattern), kind) for gate, pattern, kind in (
+ ("connectivity", r"interpenetrate", "connectivity/penetration"),
+ ("contract", r"\(each instance\)|\(all instances\)", "contract/instance_bbox"),
+ ("contract", r"part '.*' bbox deviates", "contract/part_bbox"),
+ ("contract", r"overall bbox deviates", "contract/overall_bbox"),
+ ("contract", r"footprint centre", "contract/footprint_offset"),
+ ("contract", r"floats .* the ground", "contract/ground_gap"),
+ ("contract", r"missing from the GLB", "contract/missing_part"),
+ ("contract", r"instance\(s\), plan asks for", "contract/instance_count"),
+ ("joint_sweep", r"overlap by|overlap in \d+ of the sampled poses|more overlapping pair", "joint_sweep/penetration"),
+ ("joint_sweep", r"apart at rest|nothing physically connects", "joint_sweep/unattached"),
+ ("scene_frames", r"too dark", "scene_frames/dark_frame"),
+ ("scene_frames", r"flat frame", "scene_frames/flat_frame"),
+ ("scene_frames", r"BELOW the ground|inside / touching geometry", "scene_frames/camera_in_geometry"),
+ ))
+
+
+def _kind_of(gate: str, f: dict, severity: str) -> str | None:
+ """``skills.registry.finding_kind``, with the old message shapes for an untyped record."""
+ kind = finding_kind(SimpleNamespace(gate=str(f.get("gate") or gate), severity=severity, data=f.get("data") or {}))
+ if kind is None or not kind.endswith(f"/{UNTYPED}"):
+ return kind
+ msg = str(f.get("message", ""))
+ return next((k for g, pat, k in _UNTYPED_SHAPES if kind == f"{g}/{UNTYPED}" and pat.search(msg)), kind)
+
+
def count_kinds(rd: dict, kinds: tuple[str, ...], severities: tuple[str, ...]) -> int:
n = 0
for g in rd.get("gates") or []:
@@ -127,7 +158,8 @@ def count_kinds(rd: dict, kinds: tuple[str, ...], severities: tuple[str, ...]) -
sev = str(f.get("severity", "warn")).lower()
if severities and sev not in severities:
continue
- if finding_kind(str(g.get("gate", "")), str(f.get("message", "")), sev) in kinds:
+ kind = _kind_of(str(g.get("gate", "")), f, sev)
+ if kind is not None and kind_matches(kind, kinds):
n += 1
return n
diff --git a/eval/tests/test_skill_targets_readout.py b/eval/tests/test_skill_targets_readout.py
index 9a89cf3a..52ae48c9 100644
--- a/eval/tests/test_skill_targets_readout.py
+++ b/eval/tests/test_skill_targets_readout.py
@@ -146,3 +146,16 @@ def test_a_glb_row_measures_the_measured_rounds_own_glb(tmp_path, monkeypatch):
run = st.load_runs(root)[0]
assert st.measure(target, run, which="first") == 10.0
assert st.measure(target, run, which="last") == 11.0
+
+
+def test_a_typed_finding_counts_by_its_kind_and_an_old_record_by_its_shape():
+ """N52: the harness routes on ``data.kind``; the readout counts the same kind, and reads a
+ record written before its gate typed the finding by the old message shape."""
+ from bench.skill_targets import count_kinds
+
+ typed = {"gate": "contract", "severity": "warn", "message": "reworded", "data": {"kind": "part_bbox"}}
+ camera = {"gate": "scene_frames", "severity": "warn", "message": "x", "data": {"kind": "camera_below_high_ground"}}
+ rd = {"gates": [{"gate": "contract", "findings": [typed, *CONTRACT]},
+ {"gate": "scene_frames", "findings": [camera]}]}
+ assert count_kinds(rd, ("contract/part_bbox",), ()) == 2
+ assert count_kinds(rd, ("scene_frames/camera_*",), ()) == 1
diff --git a/harness/codeverse3d/skills/registry.py b/harness/codeverse3d/skills/registry.py
index 43faf05a..890bfd0a 100644
--- a/harness/codeverse3d/skills/registry.py
+++ b/harness/codeverse3d/skills/registry.py
@@ -62,6 +62,11 @@ def finding_kind(finding: Any) -> str | None:
return f"{family}/{kind if isinstance(kind, str) and kind else UNTYPED}"
+def kind_matches(kind: str, patterns: tuple[str, ...]) -> bool:
+ """``kind`` is one of ``patterns``: an exact kind, or a family ending in ``*`` (``gl_frames/*``)."""
+ return any(kind.startswith(p[:-1]) if p.endswith("*") else kind == p for p in patterns)
+
+
def finding_kinds(findings: object) -> list[str]:
"""Distinct kinds of an iterable of ``GateFinding`` (or of ``GateReport``s), in order."""
out: list[str] = []
@@ -112,12 +117,7 @@ class Route:
def matches_finding(self, kinds: list[str]) -> str | None:
"""The first routed finding kind this row answers, or None."""
- for pat in self.findings:
- family = pat[:-1] if pat.endswith("*") else ""
- for k in kinds:
- if (family and k.startswith(family)) or k == pat:
- return k
- return None
+ return next((k for pat in self.findings for k in kinds if kind_matches(k, (pat,))), None)
ROUTES: tuple[Route, ...] = (