Skip to content

Official Codebase of 3DCodeVerse

Python

judge: a checklist defect a passed gate measured absent neither penalises nor caps

gaoypeng83d58ceaf738
diff --git a/harness/codeverse/judges/caps.py b/harness/codeverse/judges/caps.py
index 9cdf6b2e..4b6a74dd 100644
--- a/harness/codeverse/judges/caps.py
+++ b/harness/codeverse/judges/caps.py
@@ -53,7 +53,56 @@ def missing_must_items(
acceptance_items: list[AcceptanceItem], acceptance_results: dict[str, bool]
) -> list[str]:
"""Ids of ``must`` acceptance items that are not verified (missing counts as failed)."""
- return [a.id for a in acceptance_items if a.priority == "must" and not acceptance_results.get(a.id, False)]
+ return [
+ a.id
+ for a in acceptance_items
+ if a.priority == "must" and not acceptance_results.get(a.id, False)
+ ]
+
+
+def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) -> bool:
+ """True when a deterministic gate MEASURED the thing this checklist defect claims, and found nothing.
+
+ A checklist defect is a VLM's reading of a picture. Some of them — a floating part, an
+ interpenetration — are also what a gate measures on the exported mesh, and the rubric
+ already says which: a ``when="gate"`` cap rule with the same id and a ``kinds`` list.
+ When every gate that rule watches ran and passed with no ERROR finding of those kinds,
+ the measurement contradicts the perception, and the perception must not cap.
+
+ Measured 2026-08-26 on a plan-pinned pair (fancy_v1 gas_street_lamp): the connectivity
+ gate said "all 9 parts connected, gap <= 2 mm"; the judge read the dark seam under the
+ pedestal as "floating in mid-air, a clear daylight gap", marked floating_part present, and
+ ``defect:floating_part`` capped the run at 0.6 (uncapped 0.72) against 0.96 for a sibling
+ the eye cannot tell apart. CLAUDE.md law 3: gates decide geometry, the VLM is perception.
+
+ A defect with no matching gate rule (wrong_object, missing_named_part …) is never
+ overridden: nothing measured it. A gate that did not run overrides nothing either.
+ """
+ rule = next(
+ (r for r in rubric.caps if r.when == "gate" and r.id == defect_id and r.kinds), None
+ )
+ if rule is None:
+ return False
+ watched = [g for g in gates if fnmatch.fnmatch(g.gate, rule.gate)]
+ if not watched:
+ return False
+ for g in watched:
+ if not g.passed:
+ return False
+ for f in g.findings:
+ if f.severity == Severity.ERROR and any(
+ k in (f.message or "").lower() for k in rule.kinds
+ ):
+ return False
+ return True
+
+
+def veto_measured_defects(
+ rubric: Rubric, defects: dict[str, bool], gates: list[GateReport]
+) -> tuple[dict[str, bool], list[str]]:
+ """``defects`` with every measured-absent one switched off, plus the ids switched off."""
+ overridden = [did for did, on in defects.items() if on and measured_absent(rubric, did, gates)]
+ return {did: (on and did not in overridden) for did, on in defects.items()}, overridden
def apply_caps(
@@ -76,7 +125,14 @@ def apply_caps(
"""
applied: list[CapApplied] = []
for rule in rubric.caps:
- evidence = _rule_evidence(rule, gates, acceptance_results, acceptance_items or [], console_errors or [], views or [])
+ evidence = _rule_evidence(
+ rule,
+ gates,
+ acceptance_results,
+ acceptance_items or [],
+ console_errors or [],
+ views or [],
+ )
if evidence is not None:
applied.append(CapApplied(rule=rule.id, cap=rule.cap, evidence=evidence))
for did, present in (defects_present or {}).items():
@@ -87,7 +143,13 @@ def apply_caps(
except KeyError:
continue
if item.cap is not None:
- applied.append(CapApplied(rule=f"defect:{did}", cap=item.cap, evidence=f"judge checklist: {item.text[:120]}"))
+ applied.append(
+ CapApplied(
+ rule=f"defect:{did}",
+ cap=item.cap,
+ evidence=f"judge checklist: {item.text[:120]}",
+ )
+ )
capped = overall
for a in applied:
capped = min(capped, a.cap)
diff --git a/harness/codeverse/judges/scoring.py b/harness/codeverse/judges/scoring.py
index ab572ed6..89bc093a 100644
--- a/harness/codeverse/judges/scoring.py
+++ b/harness/codeverse/judges/scoring.py
@@ -25,7 +25,7 @@ from codeverse.contracts.artifacts import GateReport, RenderView
from codeverse.contracts.common import Usage
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
-from codeverse.judges.caps import CapResult, apply_caps, missing_must_items
+from codeverse.judges.caps import CapResult, apply_caps, missing_must_items, veto_measured_defects
from codeverse.judges.output_schema import JudgeOutput
from codeverse.judges.rubrics import Rubric
@@ -37,14 +37,20 @@ class ScoreBreakdown(BaseModel):
per_criterion_mean: dict[str, float]
per_criterion_std: dict[str, float]
per_sample_overall: list[float]
- overall_uncapped: float = Field(description="rubric-weighted mean of the per-criterion means, before defects/caps")
- overall_after_defects: float = Field(default=0.0, description="overall_uncapped minus defect penalties")
+ overall_uncapped: float = Field(
+ description="rubric-weighted mean of the per-criterion means, before defects/caps"
+ )
+ overall_after_defects: float = Field(
+ default=0.0, description="overall_uncapped minus defect penalties"
+ )
overall: float
floors_hit: list[dict[str, Any]] = Field(default_factory=list)
caps: CapResult
must_missing: list[str] = Field(default_factory=list)
acceptance_votes: dict[str, list[bool]] = Field(default_factory=dict)
- defects: dict[str, bool] = Field(default_factory=dict, description="checklist id → present (majority vote)")
+ defects: dict[str, bool] = Field(
+ default_factory=dict, description="checklist id → present (majority vote)"
+ )
defect_votes: dict[str, list[bool]] = Field(default_factory=dict)
defect_penalty: float = 0.0
n_requested: int
@@ -95,7 +101,8 @@ def aggregate_samples(
overall_uncapped = rubric.weighted_overall(mean_scores)
floors = [
- {"criterion": cid, "score": round(sc, 4), "floor": fl} for cid, sc, fl in rubric.floors_hit(mean_scores)
+ {"criterion": cid, "score": round(sc, 4), "floor": fl}
+ for cid, sc, fl in rubric.floors_hit(mean_scores)
]
votes: dict[str, list[bool]] = {}
for s in samples:
@@ -111,17 +118,38 @@ def aggregate_samples(
for did, flag in s.defects.items():
d_votes.setdefault(did, []).append(bool(flag))
defects = {did: (sum(v) * 2 >= len(v) and bool(v)) for did, v in d_votes.items()}
+ # a checklist defect that a passed gate measured as absent neither penalises nor caps
+ # (judges/caps.measured_absent); it is still named in the verdict tail
+ defects, overridden = veto_measured_defects(rubric, defects, gates)
penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)
after_defects = max(0.0, overall_uncapped - penalty)
- caps = apply_caps(rubric, after_defects, gates, acceptance, acceptance_items, console_errors=console_errors,
- views=views, defects_present=defects)
+ caps = apply_caps(
+ rubric,
+ after_defects,
+ gates,
+ acceptance,
+ acceptance_items,
+ console_errors=console_errors,
+ views=views,
+ defects_present=defects,
+ )
overall = caps.overall
passed = overall >= rubric.pass_threshold and not floors and not must_missing
rep = _representative(samples, overalls)
summary = rep.summary.strip()
- tail = _verdict_tail(rubric, overall, passed, floors, caps, must_missing, defects=defects, penalty=penalty)
+ tail = _verdict_tail(
+ rubric,
+ overall,
+ passed,
+ floors,
+ caps,
+ must_missing,
+ defects=defects,
+ penalty=penalty,
+ overridden=overridden,
+ )
if tail:
summary = (summary + " " if summary else "") + tail
@@ -164,23 +192,45 @@ def aggregate_samples(
def _verdict_tail(
- rubric: Rubric, overall: float, passed: bool, floors: list[dict[str, Any]], caps: CapResult, must_missing: list[str],
- *, defects: dict[str, bool] | None = None, penalty: float = 0.0,
+ rubric: Rubric,
+ overall: float,
+ passed: bool,
+ floors: list[dict[str, Any]],
+ caps: CapResult,
+ must_missing: list[str],
+ *,
+ defects: dict[str, bool] | None = None,
+ penalty: float = 0.0,
+ overridden: list[str] | None = None,
) -> str:
- bits = [f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"]
+ bits = [
+ f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"
+ ]
present = [d for d, on in (defects or {}).items() if on]
if present:
bits.append(f"defects (-{penalty:.2f}): " + ", ".join(present))
+ if overridden:
+ bits.append(
+ "checklist claims contradicted by a passed gate (no penalty, no cap): "
+ + ", ".join(overridden)
+ )
if caps.caps_applied:
- bits.append("caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied))
+ bits.append(
+ "caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied)
+ )
if floors:
- bits.append("floors: " + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors))
+ bits.append(
+ "floors: "
+ + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors)
+ )
if must_missing:
bits.append("must items unverified: " + ", ".join(must_missing))
return " | ".join(bits) + "]"
-def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int) -> Judgment:
+def degraded_judgment(
+ rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int
+) -> Judgment:
"""A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""
breakdown = {
"status": "degraded",
diff --git a/harness/tests/judges/test_caps.py b/harness/tests/judges/test_caps.py
index ab413eba..1ae599ae 100644
--- a/harness/tests/judges/test_caps.py
+++ b/harness/tests/judges/test_caps.py
@@ -58,3 +58,39 @@ def test_scene_console_and_shader_caps():
assert res.overall == 0.6 and res.caps_applied[0].rule == "console_error"
res2 = apply_caps(S, 0.9, [_gate("shader_probe", Severity.ERROR, "GLSL compile failed in water.js", kind="shader")], {}, [])
assert res2.overall == 0.5
+
+
+# ------------------------------------------- a passed gate vetoes the checklist's claim
+def _connectivity(passed: bool, *msgs: tuple[str, str]):
+ from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
+ sev = {"error": Severity.ERROR, "warn": Severity.WARN, "info": Severity.INFO}
+ return GateReport(gate="connectivity", passed=passed,
+ findings=[GateFinding(gate="connectivity", severity=sev[s], target="", message=m) for s, m in msgs])
+
+
+def test_a_floating_claim_the_connectivity_gate_measured_absent_neither_penalises_nor_caps():
+ """Measured 2026-08-26 (fancy_v1 gas_street_lamp, plan-pinned pair): gate said
+ 'all 9 parts connected, gap <= 2 mm'; the judge marked floating_part present on a dark
+ seam and defect:floating_part capped the run at 0.6 (uncapped 0.72) against 0.96 for a
+ sibling the eye cannot tell apart. Gates decide geometry (law 3)."""
+ from codeverse.judges.caps import measured_absent, veto_measured_defects
+ from codeverse.judges.rubrics import load_rubric
+
+ rubric = load_rubric("static_object_v1")
+ passed = [_connectivity(True, ("info", "all 9 parts are connected (9 contacts, gap <= 2 mm)"),
+ ("warn", "part 'GasBurner' contains 7 tiny disconnected island(s)"))]
+ assert measured_absent(rubric, "floating_part", passed)
+ kept, overridden = veto_measured_defects(rubric, {"floating_part": True, "wrong_object": True}, passed)
+ assert overridden == ["floating_part"]
+ assert kept == {"floating_part": False, "wrong_object": True}, "only what a gate measures can be vetoed"
+
+
+def test_a_real_floating_part_still_caps():
+ from codeverse.judges.caps import measured_absent
+ from codeverse.judges.rubrics import load_rubric
+
+ rubric = load_rubric("static_object_v1")
+ failed = [_connectivity(False, ("error", "part 'Seat' is floating 12 mm above 'Leg'"))]
+ assert not measured_absent(rubric, "floating_part", failed)
+ assert not measured_absent(rubric, "floating_part", []), "a gate that did not run measured nothing"
+ assert not measured_absent(rubric, "wrong_object", [_connectivity(True)]), "no gate measures 'wrong object'"