Official Codebase of 3DCodeVerse
Python
judge: a checklist defect a passed gate measured absent neither penalises nor caps
gaoypeng83d58ceaf738
diff --git a/harness/codeverse/judges/caps.py b/harness/codeverse/judges/caps.pyindex 9cdf6b2e..4b6a74dd 100644--- a/harness/codeverse/judges/caps.py+++ b/harness/codeverse/judges/caps.py@@ -53,7 +53,56 @@ def missing_must_items(acceptance_items: list[AcceptanceItem], acceptance_results: dict[str, bool]) -> list[str]:"""Ids of ``must`` acceptance items that are not verified (missing counts as failed)."""- return [a.id for a in acceptance_items if a.priority == "must" and not acceptance_results.get(a.id, False)]+ return [+ a.id+ for a in acceptance_items+ if a.priority == "must" and not acceptance_results.get(a.id, False)+ ]+++def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) -> bool:+ """True when a deterministic gate MEASURED the thing this checklist defect claims, and found nothing.++ A checklist defect is a VLM's reading of a picture. Some of them — a floating part, an+ interpenetration — are also what a gate measures on the exported mesh, and the rubric+ already says which: a ``when="gate"`` cap rule with the same id and a ``kinds`` list.+ When every gate that rule watches ran and passed with no ERROR finding of those kinds,+ the measurement contradicts the perception, and the perception must not cap.++ Measured 2026-08-26 on a plan-pinned pair (fancy_v1 gas_street_lamp): the connectivity+ gate said "all 9 parts connected, gap <= 2 mm"; the judge read the dark seam under the+ pedestal as "floating in mid-air, a clear daylight gap", marked floating_part present, and+ ``defect:floating_part`` capped the run at 0.6 (uncapped 0.72) against 0.96 for a sibling+ the eye cannot tell apart. CLAUDE.md law 3: gates decide geometry, the VLM is perception.++ A defect with no matching gate rule (wrong_object, missing_named_part …) is never+ overridden: nothing measured it. A gate that did not run overrides nothing either.+ """+ rule = next(+ (r for r in rubric.caps if r.when == "gate" and r.id == defect_id and r.kinds), None+ )+ if rule is None:+ return False+ watched = [g for g in gates if fnmatch.fnmatch(g.gate, rule.gate)]+ if not watched:+ return False+ for g in watched:+ if not g.passed:+ return False+ for f in g.findings:+ if f.severity == Severity.ERROR and any(+ k in (f.message or "").lower() for k in rule.kinds+ ):+ return False+ return True+++def veto_measured_defects(+ rubric: Rubric, defects: dict[str, bool], gates: list[GateReport]+) -> tuple[dict[str, bool], list[str]]:+ """``defects`` with every measured-absent one switched off, plus the ids switched off."""+ overridden = [did for did, on in defects.items() if on and measured_absent(rubric, did, gates)]+ return {did: (on and did not in overridden) for did, on in defects.items()}, overriddendef apply_caps(@@ -76,7 +125,14 @@ def apply_caps("""applied: list[CapApplied] = []for rule in rubric.caps:- evidence = _rule_evidence(rule, gates, acceptance_results, acceptance_items or [], console_errors or [], views or [])+ evidence = _rule_evidence(+ rule,+ gates,+ acceptance_results,+ acceptance_items or [],+ console_errors or [],+ views or [],+ )if evidence is not None:applied.append(CapApplied(rule=rule.id, cap=rule.cap, evidence=evidence))for did, present in (defects_present or {}).items():@@ -87,7 +143,13 @@ def apply_caps(except KeyError:continueif item.cap is not None:- applied.append(CapApplied(rule=f"defect:{did}", cap=item.cap, evidence=f"judge checklist: {item.text[:120]}"))+ applied.append(+ CapApplied(+ rule=f"defect:{did}",+ cap=item.cap,+ evidence=f"judge checklist: {item.text[:120]}",+ )+ )capped = overallfor a in applied:capped = min(capped, a.cap)diff --git a/harness/codeverse/judges/scoring.py b/harness/codeverse/judges/scoring.pyindex ab572ed6..89bc093a 100644--- a/harness/codeverse/judges/scoring.py+++ b/harness/codeverse/judges/scoring.py@@ -25,7 +25,7 @@ from codeverse.contracts.artifacts import GateReport, RenderViewfrom codeverse.contracts.common import Usagefrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItem-from codeverse.judges.caps import CapResult, apply_caps, missing_must_items+from codeverse.judges.caps import CapResult, apply_caps, missing_must_items, veto_measured_defectsfrom codeverse.judges.output_schema import JudgeOutputfrom codeverse.judges.rubrics import Rubric@@ -37,14 +37,20 @@ class ScoreBreakdown(BaseModel):per_criterion_mean: dict[str, float]per_criterion_std: dict[str, float]per_sample_overall: list[float]- overall_uncapped: float = Field(description="rubric-weighted mean of the per-criterion means, before defects/caps")- overall_after_defects: float = Field(default=0.0, description="overall_uncapped minus defect penalties")+ overall_uncapped: float = Field(+ description="rubric-weighted mean of the per-criterion means, before defects/caps"+ )+ overall_after_defects: float = Field(+ default=0.0, description="overall_uncapped minus defect penalties"+ )overall: floatfloors_hit: list[dict[str, Any]] = Field(default_factory=list)caps: CapResultmust_missing: list[str] = Field(default_factory=list)acceptance_votes: dict[str, list[bool]] = Field(default_factory=dict)- defects: dict[str, bool] = Field(default_factory=dict, description="checklist id → present (majority vote)")+ defects: dict[str, bool] = Field(+ default_factory=dict, description="checklist id → present (majority vote)"+ )defect_votes: dict[str, list[bool]] = Field(default_factory=dict)defect_penalty: float = 0.0n_requested: int@@ -95,7 +101,8 @@ def aggregate_samples(overall_uncapped = rubric.weighted_overall(mean_scores)floors = [- {"criterion": cid, "score": round(sc, 4), "floor": fl} for cid, sc, fl in rubric.floors_hit(mean_scores)+ {"criterion": cid, "score": round(sc, 4), "floor": fl}+ for cid, sc, fl in rubric.floors_hit(mean_scores)]votes: dict[str, list[bool]] = {}for s in samples:@@ -111,17 +118,38 @@ def aggregate_samples(for did, flag in s.defects.items():d_votes.setdefault(did, []).append(bool(flag))defects = {did: (sum(v) * 2 >= len(v) and bool(v)) for did, v in d_votes.items()}+ # a checklist defect that a passed gate measured as absent neither penalises nor caps+ # (judges/caps.measured_absent); it is still named in the verdict tail+ defects, overridden = veto_measured_defects(rubric, defects, gates)penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)after_defects = max(0.0, overall_uncapped - penalty)- caps = apply_caps(rubric, after_defects, gates, acceptance, acceptance_items, console_errors=console_errors,- views=views, defects_present=defects)+ caps = apply_caps(+ rubric,+ after_defects,+ gates,+ acceptance,+ acceptance_items,+ console_errors=console_errors,+ views=views,+ defects_present=defects,+ )overall = caps.overallpassed = overall >= rubric.pass_threshold and not floors and not must_missingrep = _representative(samples, overalls)summary = rep.summary.strip()- tail = _verdict_tail(rubric, overall, passed, floors, caps, must_missing, defects=defects, penalty=penalty)+ tail = _verdict_tail(+ rubric,+ overall,+ passed,+ floors,+ caps,+ must_missing,+ defects=defects,+ penalty=penalty,+ overridden=overridden,+ )if tail:summary = (summary + " " if summary else "") + tail@@ -164,23 +192,45 @@ def aggregate_samples(def _verdict_tail(- rubric: Rubric, overall: float, passed: bool, floors: list[dict[str, Any]], caps: CapResult, must_missing: list[str],- *, defects: dict[str, bool] | None = None, penalty: float = 0.0,+ rubric: Rubric,+ overall: float,+ passed: bool,+ floors: list[dict[str, Any]],+ caps: CapResult,+ must_missing: list[str],+ *,+ defects: dict[str, bool] | None = None,+ penalty: float = 0.0,+ overridden: list[str] | None = None,) -> str:- bits = [f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"]+ bits = [+ f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"+ ]present = [d for d, on in (defects or {}).items() if on]if present:bits.append(f"defects (-{penalty:.2f}): " + ", ".join(present))+ if overridden:+ bits.append(+ "checklist claims contradicted by a passed gate (no penalty, no cap): "+ + ", ".join(overridden)+ )if caps.caps_applied:- bits.append("caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied))+ bits.append(+ "caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied)+ )if floors:- bits.append("floors: " + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors))+ bits.append(+ "floors: "+ + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors)+ )if must_missing:bits.append("must items unverified: " + ", ".join(must_missing))return " | ".join(bits) + "]"-def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int) -> Judgment:+def degraded_judgment(+ rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int+) -> Judgment:"""A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""breakdown = {"status": "degraded",diff --git a/harness/tests/judges/test_caps.py b/harness/tests/judges/test_caps.pyindex ab413eba..1ae599ae 100644--- a/harness/tests/judges/test_caps.py+++ b/harness/tests/judges/test_caps.py@@ -58,3 +58,39 @@ def test_scene_console_and_shader_caps():assert res.overall == 0.6 and res.caps_applied[0].rule == "console_error"res2 = apply_caps(S, 0.9, [_gate("shader_probe", Severity.ERROR, "GLSL compile failed in water.js", kind="shader")], {}, [])assert res2.overall == 0.5+++# ------------------------------------------- a passed gate vetoes the checklist's claim+def _connectivity(passed: bool, *msgs: tuple[str, str]):+ from codeverse.contracts.artifacts import GateFinding, GateReport, Severity+ sev = {"error": Severity.ERROR, "warn": Severity.WARN, "info": Severity.INFO}+ return GateReport(gate="connectivity", passed=passed,+ findings=[GateFinding(gate="connectivity", severity=sev[s], target="", message=m) for s, m in msgs])+++def test_a_floating_claim_the_connectivity_gate_measured_absent_neither_penalises_nor_caps():+ """Measured 2026-08-26 (fancy_v1 gas_street_lamp, plan-pinned pair): gate said+ 'all 9 parts connected, gap <= 2 mm'; the judge marked floating_part present on a dark+ seam and defect:floating_part capped the run at 0.6 (uncapped 0.72) against 0.96 for a+ sibling the eye cannot tell apart. Gates decide geometry (law 3)."""+ from codeverse.judges.caps import measured_absent, veto_measured_defects+ from codeverse.judges.rubrics import load_rubric++ rubric = load_rubric("static_object_v1")+ passed = [_connectivity(True, ("info", "all 9 parts are connected (9 contacts, gap <= 2 mm)"),+ ("warn", "part 'GasBurner' contains 7 tiny disconnected island(s)"))]+ assert measured_absent(rubric, "floating_part", passed)+ kept, overridden = veto_measured_defects(rubric, {"floating_part": True, "wrong_object": True}, passed)+ assert overridden == ["floating_part"]+ assert kept == {"floating_part": False, "wrong_object": True}, "only what a gate measures can be vetoed"+++def test_a_real_floating_part_still_caps():+ from codeverse.judges.caps import measured_absent+ from codeverse.judges.rubrics import load_rubric++ rubric = load_rubric("static_object_v1")+ failed = [_connectivity(False, ("error", "part 'Seat' is floating 12 mm above 'Leg'"))]+ assert not measured_absent(rubric, "floating_part", failed)+ assert not measured_absent(rubric, "floating_part", []), "a gate that did not run measured nothing"+ assert not measured_absent(rubric, "wrong_object", [_connectivity(True)]), "no gate measures 'wrong object'"