Skip to content

Official Codebase of 3DCodeVerse

Python

judges: the measured-absent veto reaches interpenetration, and the acceptance cap has a gradient

gaoypengd8ef22247390
diff --git a/harness/bench/rejudge_offline.py b/harness/bench/rejudge_offline.py
index e2483592..bedf3015 100644
--- a/harness/bench/rejudge_offline.py
+++ b/harness/bench/rejudge_offline.py
@@ -11,8 +11,13 @@ Two uses:
* **identity** — run it on an unchanged tree: every verdict must reproduce to 1e-9.
That is the acceptance test for this tool, and the guard that says the corpus is
- still readable. (A verdict scored under an older rubric hash is reported, not
- failed: the rubric moved, the tool did not.)
+ still readable. Only verdicts stamped with today's ``rubrics.SCORING_VERSION`` are
+ held to it; one scored under an older version (or an older rubric hash) is DRIFT —
+ reported, not failed: the scoring moved, the tool did not. Which means the guard
+ is DORMANT on today's corpus: every verdict on disk was written before the stamp
+ (version 0), so nothing is held until a version-``SCORING_VERSION`` run is recorded.
+ An empty held set is said out loud and exits 2 — a green line that checked nothing
+ is what a guard must never print — unless ``--allow-empty-identity`` is passed.
* **impact** — change ``rubrics.py`` or a rubric YAML, run it again: every moved
verdict, with its before/after caps and defects, the shift in pass rate, in the
0.600 spike, in σ and in pearson(gate errors, overall). That is how a scoring
@@ -24,7 +29,7 @@ Cost: seconds of CPU. It never writes into a run.
CLI::
python bench/rejudge_offline.py bench/out --rubric static_object_v1 --out scratch/replay
- python bench/rejudge_offline.py bench/out --rubric static_object_v1 --identity # exit 1 on any drift
+ python bench/rejudge_offline.py bench/out --rubric static_object_v1 --identity # exit 1 on any drift, 2 when nothing was held
"""
from __future__ import annotations
@@ -38,10 +43,16 @@ from pathlib import Path
from pydantic import BaseModel, Field
-from codeverse.contracts.artifacts import GateReport, Judgment, Severity
+from codeverse.contracts.artifacts import GateReport, Severity
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.run import RoundRecord
-from codeverse.judges.rubrics import JudgeOutput, Rubric, aggregate_samples, load_rubric
+from codeverse.judges.rubrics import (
+ SCORING_VERSION,
+ JudgeOutput,
+ Rubric,
+ aggregate_samples,
+ load_rubric,
+)
from codeverse.proc import read_json_or_none
TOL = 1e-9
@@ -53,6 +64,8 @@ class ReplayRow(BaseModel):
kind: str
rubric_hash_stored: str
rubric_hash_now: str
+ scoring_version_stored: int = Field(default=0, description="raw.scoring_version the run wrote; 0 = before the stamp")
+ scoring_version_now: int = SCORING_VERSION
gate_errors: int
stored: float
replay: float
@@ -73,6 +86,7 @@ class ReplayReport(BaseModel):
n_errors: int
n_moved: int
n_rubric_drift: int = Field(description="verdicts whose stored rubric hash differs from today's — drift, not a tool fault")
+ n_scoring_drift: int = Field(default=0, description="verdicts stamped with an older SCORING_VERSION (or none) — drift, not a tool fault")
mean_delta: float
median_delta: float
max_abs_delta: float
@@ -88,6 +102,8 @@ class ReplayReport(BaseModel):
cap_counts_after: dict[str, int]
defect_counts_before: dict[str, int]
defect_counts_after: dict[str, int]
+ overridden_counts: dict[str, int] = Field(default_factory=dict,
+ description="defect id → verdicts where the measured-absent veto switched it off on replay")
rows: list[ReplayRow]
def to_markdown(self) -> str:
@@ -96,8 +112,10 @@ class ReplayReport(BaseModel):
lines = [
f"# offline replay — {self.rubric}",
"",
- f"n={self.n} verdicts ({self.n_errors} unreadable, {self.n_rubric_drift} scored under another rubric hash)",
+ f"n={self.n} verdicts ({self.n_errors} unreadable, {self.n_rubric_drift} scored under another rubric hash, "
+ f"{self.n_scoring_drift} under an older scoring version than {SCORING_VERSION})",
f"moved: **{self.n_moved}** mean Δ {self.mean_delta:+.4f} median Δ {self.median_delta:+.4f} max |Δ| {self.max_abs_delta:.4f}",
+ "vetoed on replay (measured absent): " + (", ".join(f"{k} {v}" for k, v in sorted(self.overridden_counts.items())) or "none"),
f"pass rate {self.pass_rate_before:.1%} → {self.pass_rate_after:.1%} "
f"scores pinned at 0.600: {self.spike_600_before} → {self.spike_600_after} "
f"σ {self.sd_before:.3f} → {self.sd_after:.3f} pearson(gate errors, overall) {f(self.pearson_errors_before)} → {f(self.pearson_errors_after)}",
@@ -111,9 +129,9 @@ class ReplayReport(BaseModel):
lines.append(f"| {k} | {self.defect_counts_before.get(k, 0)} | {self.defect_counts_after.get(k, 0)} |")
if moved:
lines += ["", f"## moved verdicts ({len(moved)})", "",
- "| run | round | gate err | stored | replay | Δ | caps before → after | overridden |", "|---|---|---|---|---|---|---|---|"]
+ "| run | round | sv | gate err | stored | replay | Δ | caps before → after | overridden |", "|---|---|---|---|---|---|---|---|---|"]
for r in sorted(moved, key=lambda r: -abs(r.delta))[:200]:
- lines.append(f"| {r.run} | r{r.round:02d} | {r.gate_errors} | {r.stored:.3f} | {r.replay:.3f} | {r.delta:+.3f} | "
+ lines.append(f"| {r.run} | r{r.round:02d} | {r.scoring_version_stored} | {r.gate_errors} | {r.stored:.3f} | {r.replay:.3f} | {r.delta:+.3f} | "
f"{', '.join(r.caps_before) or '-'} → {', '.join(r.caps_after) or '-'} | {', '.join(r.overridden) or '-'} |")
return "\n".join(lines)
@@ -179,32 +197,18 @@ def replay_round(path: Path, rubric: Rubric | None = None) -> ReplayRow | None:
return ReplayRow(
run=_run_label(run_dir), round=rec.index, kind=rec.kind,
rubric_hash_stored=raw.get("rubric_hash", ""), rubric_hash_now=raw_now.get("rubric_hash", ""),
+ scoring_version_stored=int(raw.get("scoring_version", 0)), scoring_version_now=int(raw_now.get("scoring_version", 0)),
gate_errors=gate_errors,
stored=stored.overall, replay=now.overall, delta=round(now.overall - stored.overall, 6),
stored_passed=stored.passed, replay_passed=now.passed,
caps_before=[c["rule"] for c in (raw.get("caps") or {}).get("caps_applied", [])],
- caps_after=[c.rule for c in raw_now_caps(now)],
+ caps_after=[c["rule"] for c in (raw_now.get("caps") or {}).get("caps_applied", [])],
defects_before=[d for d, on in (raw.get("defects") or {}).items() if on],
defects_after=[d for d, on in (raw_now.get("defects") or {}).items() if on],
- overridden=_overridden(now),
+ overridden=list(raw_now.get("overridden") or []),
)
-def raw_now_caps(j: Judgment):
- from codeverse.judges.rubrics import CapApplied
-
- return [CapApplied.model_validate(c) for c in (json.loads(j.raw).get("caps") or {}).get("caps_applied", [])]
-
-
-def _overridden(j: Judgment) -> list[str]:
- # the verdict tail names them: "… | measured absent: floating_part"
- tail = j.summary.rsplit("[verdict:", 1)[-1]
- if "measured absent" not in tail:
- return []
- seg = tail.split("measured absent:", 1)[1].split("|")[0].split("]")[0]
- return [s.strip() for s in seg.split(",") if s.strip()]
-
-
def _pearson(xs: list[float], ys: list[float]) -> float | None:
if len(xs) < 3 or len(set(xs)) < 2 or len(set(ys)) < 2:
return None
@@ -246,6 +250,7 @@ def replay_corpus(roots: list[Path], *, rubric_name: str, rubric: Rubric | None
rubric=rubric_name, n=len(ok), n_errors=n_err,
n_moved=sum(1 for d in deltas if abs(d) > TOL),
n_rubric_drift=sum(1 for r in ok if r.rubric_hash_stored and r.rubric_hash_stored != r.rubric_hash_now),
+ n_scoring_drift=sum(1 for r in ok if r.scoring_version_stored != SCORING_VERSION),
mean_delta=round(statistics.fmean(deltas), 6) if deltas else 0.0,
median_delta=round(statistics.median(deltas), 6) if deltas else 0.0,
max_abs_delta=round(max((abs(d) for d in deltas), default=0.0), 6),
@@ -258,6 +263,7 @@ def replay_corpus(roots: list[Path], *, rubric_name: str, rubric: Rubric | None
pearson_errors_before=_pearson(errs, before), pearson_errors_after=_pearson(errs, after),
cap_counts_before=dict(caps_b), cap_counts_after=dict(caps_a),
defect_counts_before=dict(def_b), defect_counts_after=dict(def_a),
+ overridden_counts=dict(Counter(d for r in ok for d in r.overridden)),
rows=rows,
)
@@ -272,7 +278,10 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument("--kinds", default="", help="comma list of round kinds to keep (default all, incl. candidate)")
ap.add_argument("--out", type=Path, default=None, help="directory for replay.json + replay.md")
ap.add_argument("--identity", action="store_true",
- help="assert nothing moved: exit 1 on any |Δ| > 1e-9 outside rubric drift")
+ help="assert nothing moved: exit 1 on any |Δ| > 1e-9 outside rubric drift; "
+ "exit 2 when no verdict carries today's scoring version (the guard is dormant)")
+ ap.add_argument("--allow-empty-identity", action="store_true",
+ help="with --identity: exit 0 even when no verdict was held (the warning still prints)")
a = ap.parse_args(argv)
rubric = load_rubric(str(a.rubric_file)) if a.rubric_file else None
kinds = {k for k in a.kinds.split(",") if k} or None
@@ -283,14 +292,25 @@ def main(argv: list[str] | None = None) -> int:
(a.out / "replay.md").write_text(rep.to_markdown())
print(rep.to_markdown().split("\n## moved")[0])
if a.identity:
- drift = {r.rubric_hash_stored for r in rep.rows if r.rubric_hash_stored and r.rubric_hash_stored != r.rubric_hash_now}
- bad = [r for r in rep.rows if not r.error and abs(r.delta) > TOL and r.rubric_hash_stored == r.rubric_hash_now]
+ current = [r for r in rep.rows if not r.error
+ and r.scoring_version_stored == SCORING_VERSION and r.rubric_hash_stored == r.rubric_hash_now]
+ bad = [r for r in current if abs(r.delta) > TOL]
if bad:
- print(f"\nIDENTITY FAILED: {len(bad)} verdict(s) under today's rubric hash do not reproduce:", file=sys.stderr)
+ print(f"\nIDENTITY FAILED: {len(bad)} verdict(s) under scoring version {SCORING_VERSION} "
+ "and today's rubric hash do not reproduce:", file=sys.stderr)
for r in bad[:20]:
print(f" {r.run} r{r.round:02d}: stored {r.stored:.4f} replay {r.replay:.4f}", file=sys.stderr)
return 1
- print(f"\nidentity OK: {rep.n} verdicts reproduce" + (f" ({len(drift)} older rubric hash(es) skipped)" if drift else ""))
+ if not current:
+ # "identity OK: 0 of 419" once printed green over a corpus written entirely
+ # before the version stamp — a guard that holds nothing has checked nothing.
+ print(f"\nWARNING: identity guard is DORMANT: no verdict on disk carries scoring version "
+ f"{SCORING_VERSION} under today's rubric hash; nothing was checked "
+ f"({rep.n} verdicts, {rep.n_scoring_drift} under an older scoring version, "
+ f"{rep.n_rubric_drift} under another rubric hash)", file=sys.stderr)
+ return 0 if a.allow_empty_identity else 2
+ print(f"\nidentity OK: {len(current)} of {rep.n} verdicts held to it reproduce "
+ f"({rep.n_scoring_drift} under an older scoring version, {rep.n_rubric_drift} under another rubric hash: drift, not failure)")
return 0
diff --git a/harness/codeverse/judges/rubrics.py b/harness/codeverse/judges/rubrics.py
index 0329f9e5..b8f1a981 100644
--- a/harness/codeverse/judges/rubrics.py
+++ b/harness/codeverse/judges/rubrics.py
@@ -31,6 +31,7 @@ import yaml
from pydantic import (
BaseModel,
Field,
+ PrivateAttr,
ValidationError,
create_model,
field_validator,
@@ -118,11 +119,20 @@ class CapRule(BaseModel):
pattern on the gate name (``"build*"``), ``severity`` the minimum severity,
``kinds`` a list of tokens any of which must appear in ``finding.data.kind``
/ ``finding.data.code`` or, as a fallback, in the lower-cased message.
- ``when="acceptance"`` fires when any ``must`` acceptance item is not verified.
+ ``when="acceptance"`` fires when any ``must`` acceptance item is not verified;
+ with ``graded`` the cap is ``cap + (1 - cap) · verified/total`` over the must
+ items instead of the flat ``cap``.
``when="console"`` fires when the render set reports console errors (scenes).
``when="missing_views"`` fires when NO render view has a name/mode containing
any of ``kinds`` (e.g. ``kinds: [pose_, articulation_sheet]`` = the articulated
rubric requires posed views).
+
+ ``measures`` names the checklist defects this gate rule is the MEASUREMENT of
+ (default: the rule's own id). ``measured_absent`` reads it: a passed gate the
+ rule watches vetoes those checklist claims. Audited 2026-08-30 over 420
+ static_object verdicts: ``penetration_error`` had never vetoed ``interpenetration``
+ because the match was on ``id`` alone — 237 interpenetration flags, 69 with a gate
+ ERROR behind them, 0 vetoes.
"""
id: str
@@ -131,8 +141,21 @@ class CapRule(BaseModel):
gate: str = Field(default="*", description="fnmatch pattern on GateFinding.gate")
severity: Severity = Severity.ERROR
kinds: list[str] = Field(default_factory=list)
+ measures: list[str] = Field(
+ default_factory=list, description="checklist defect ids a passed gate under this rule vetoes; default [id]"
+ )
+ graded: bool = Field(
+ default=False,
+ description="acceptance rules only: cap + (1 - cap) * verified_must / total_must instead of the flat cap",
+ )
note: str = ""
+ @model_validator(mode="after")
+ def _default_measures(self) -> CapRule:
+ if not self.measures:
+ self.measures = [self.id]
+ return self
+
class Rubric(BaseModel):
name: str
@@ -201,8 +224,17 @@ class Rubric(BaseModel):
out.append((c.id, float(scores[c.id]), c.floor))
return out
+ _source: dict[str, Any] | None = PrivateAttr(default=None)
+
def content_hash(self) -> str:
- payload = json.dumps(self.model_dump(mode="json"), sort_keys=True)
+ """A hash of what the YAML DECLARES, not of the model's defaults.
+
+ Hashing ``model_dump`` moved the hash of every rubric — scene_v1, shader_v1/v2
+ included — when ``CapRule`` gained ``measures`` and ``graded`` (2026-08-30),
+ although their files had not changed a byte; a schema field must never re-key
+ recorded verdicts. A rubric built without a source dict (tests) hashes its dump."""
+ src = self._source if self._source is not None else self.model_dump(mode="json")
+ payload = json.dumps(src, sort_keys=True, default=str)
return hashlib.sha256(payload.encode()).hexdigest()[:12]
@@ -212,9 +244,11 @@ def list_rubrics() -> list[str]:
def rubric_from_dict(data: dict, *, name_hint: str = "") -> Rubric:
try:
- return Rubric(**data)
+ rubric = Rubric(**data)
except Exception as e: # pydantic ValidationError / TypeError
raise RubricError(f"invalid rubric {name_hint or data.get('name')!r}: {e}") from e
+ rubric._source = json.loads(json.dumps(data, default=str))
+ return rubric
@lru_cache(maxsize=32)
@@ -484,9 +518,10 @@ def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) ->
A checklist defect is a VLM's reading of a picture. Some of them — a floating part, an
interpenetration — are also what a gate measures on the exported mesh, and the rubric
- already says which: a ``when="gate"`` cap rule with the same id and a ``kinds`` list.
- When every gate that rule watches ran and passed with no ERROR finding of those kinds,
- the measurement contradicts the perception, and the perception must not cap.
+ already says which: a ``when="gate"`` cap rule whose ``measures`` names the defect and
+ whose ``kinds`` name the findings. When every gate that rule watches ran and passed
+ with no ERROR finding of those kinds, the measurement contradicts the perception, and
+ the perception must not cap.
Measured 2026-08-26 on a plan-pinned pair (fancy_v1 gas_street_lamp): the connectivity
gate said "all 9 parts connected, gap <= 2 mm"; the judge read the dark seam under the
@@ -494,24 +529,48 @@ def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) ->
``defect:floating_part`` capped the run at 0.6 (uncapped 0.72) against 0.96 for a sibling
the eye cannot tell apart. CLAUDE.md law 3: gates decide geometry, the VLM is perception.
+ Audited 2026-08-30 (420 static_object verdicts): the veto had fired 10 times. It matched
+ ``r.id == defect_id`` — ``penetration_error`` never reached ``interpenetration`` — and the
+ rules watched ``gate: "*"``, so a failing contract gate (53 of the 62 blocked cases)
+ switched off a veto the connectivity gate had earned. Now ``measures`` carries the
+ defect ids and the object rubrics watch ``connectivity`` alone.
+
A defect with no matching gate rule (wrong_object, missing_named_part …) is never
overridden: nothing measured it. A gate that did not run overrides nothing either.
+ When several rules measure one defect, every one of them must be clean.
"""
- rule = next(
- (r for r in rubric.caps if r.when == "gate" and r.id == defect_id and r.kinds), None
- )
- if rule is None:
+ rules = [r for r in rubric.caps if r.when == "gate" and defect_id in r.measures and r.kinds]
+ if not rules:
return False
- watched = [g for g in gates if fnmatch(g.gate, rule.gate)]
+ return all(_rule_gates_clean(r, gates) for r in rules)
+
+
+#: A penetration the gate rated WARN but measured at least this deep is NOT "measured
+#: absent": the rubric excuses "weld overlaps of a few mm", and the veto that trusted every
+#: WARN was switching off 163 interpenetration claims of which 79 sat on a 5 mm+ overlap
+#: (median 4.8 mm) — and, since 2026-08-30, a stile 17 mm through a seat is a WARN too.
+#: From here up the judge's reading stands and the picture decides.
+VETO_PENETRATION_DEPTH_M = 0.005
+
+
+def _rule_gates_clean(rule: CapRule, gates: list[GateReport]) -> bool:
+ """Every gate ``rule`` watches ran, passed, and has no finding of its ``kinds`` that is
+ an ERROR — or a penetration WARN measured at ``VETO_PENETRATION_DEPTH_M`` or deeper.
+ A report is watched when its name OR one of its findings' ``gate`` matches the rule,
+ the same predicate the cap side (``_finding_matches``) uses."""
+ watched = [g for g in gates if fnmatch(g.gate, rule.gate) or any(fnmatch(f.gate, rule.gate) for f in g.findings)]
if not watched:
return False
for g in watched:
if not g.passed:
return False
for f in g.findings:
- if f.severity == Severity.ERROR and any(
- k in (f.message or "").lower() for k in rule.kinds
- ):
+ if not any(k.lower() in _finding_tokens(f) for k in rule.kinds):
+ continue
+ if f.severity == Severity.ERROR:
+ return False
+ depth = f.data.get("depth_m") if isinstance(f.data, dict) else None
+ if f.severity == Severity.WARN and isinstance(depth, (int, float)) and depth >= VETO_PENETRATION_DEPTH_M:
return False
return True
@@ -544,7 +603,7 @@ def apply_caps(
"""
applied: list[CapApplied] = []
for rule in rubric.caps:
- evidence = _rule_evidence(
+ hit = _rule_hit(
rule,
gates,
acceptance_results,
@@ -552,8 +611,8 @@ def apply_caps(
console_errors or [],
views or [],
)
- if evidence is not None:
- applied.append(CapApplied(rule=rule.id, cap=rule.cap, evidence=evidence))
+ if hit is not None:
+ applied.append(hit)
for did, present in (defects_present or {}).items():
if not present:
continue
@@ -575,41 +634,71 @@ def apply_caps(
return CapResult(overall=max(0.0, min(1.0, capped)), caps_applied=applied)
-def _rule_evidence(
+def _rule_hit(
rule: CapRule,
gates: list[GateReport],
acceptance_results: dict[str, bool],
acceptance_items: list[AcceptanceItem],
console_errors: list[str],
views: list[RenderView],
-) -> str | None:
+) -> CapApplied | None:
+ """The ledger line ``rule`` earns on this evidence, or None when it does not fire."""
if rule.when == "missing_views":
tokens = [k.lower() for k in rule.kinds]
if any(any(t in f"{v.name} {v.mode}".lower() for t in tokens) for v in views):
return None
- return f"no render view matching {rule.kinds} among {len(views)} view(s)"
+ return CapApplied(rule=rule.id, cap=rule.cap,
+ evidence=f"no render view matching {rule.kinds} among {len(views)} view(s)")
if rule.when == "acceptance":
missing = missing_must_items(acceptance_items, acceptance_results)
- if missing:
- return f"must items not verified: {', '.join(missing[:6])}"
- return None
+ if not missing:
+ return None
+ total = sum(1 for a in acceptance_items if a.priority == "must")
+ verified = total - len(missing)
+ # A flat cap was the DECISIVE cap on 121 of 424 static_object verdicts (28.6 %,
+ # audited 2026-08-30): one unverified must item out of ten pinned the score to
+ # 0.600 exactly as ten out of ten did. Graded, the cap keeps its floor at ``cap``
+ # (0 of n verified) and climbs linearly to 1.0; pass/fail is untouched — the
+ # verdict still fails on ``must_missing``.
+ cap = rule.cap + (1.0 - rule.cap) * verified / total if rule.graded and total else rule.cap
+ return CapApplied(rule=rule.id, cap=cap,
+ evidence=f"{verified} of {total} must items verified; not verified: {', '.join(missing[:6])}")
if rule.when == "console":
if console_errors:
- return f"{len(console_errors)} console error(s); first: {console_errors[0][:160]}"
+ return CapApplied(rule=rule.id, cap=rule.cap,
+ evidence=f"{len(console_errors)} console error(s); first: {console_errors[0][:160]}")
return None
for report in gates:
for f in report.findings:
if _finding_matches(rule, report, f):
target = f" [{f.target}]" if f.target else ""
- return f"{report.gate}{target}: {f.message[:200]}"
+ return CapApplied(rule=rule.id, cap=rule.cap, evidence=f"{report.gate}{target}: {f.message[:200]}")
return None
# ===================================================================== scoring
+SCORING_VERSION = 2
+"""The arithmetic downstream of the model, stamped into every ``ScoreBreakdown``.
+
+A stored verdict is reproducible from its samples only under the version that wrote
+it (``bench/rejudge_offline.py --identity`` compares nothing else); an older stamp is
+drift to report, not a fault.
+
+0 — before 2026-08-26 (no stamp on disk): majority vote, penalties, the cap ladder.
+1 — 2026-08-26, 7a9b6d3: ``veto_measured_defects`` — a checklist defect a passed gate
+ measured absent neither penalises nor caps. (Unstamped; 41 corpus verdicts predate
+ it and move on replay.)
+2 — 2026-08-30: ``CapRule.measures`` (the veto reaches ``interpenetration``), the object
+ rubrics' floating/penetration rules watch ``connectivity`` alone, a defect vote tie
+ is absent, ``missing_must_acceptance`` is graded, ``overridden`` + this stamp in raw.
+"""
+
+
class ScoreBreakdown(BaseModel):
"""Everything the pass/fail decision used (serialised into ``Judgment.raw``)."""
status: str = Field(default="ok", description="ok | degraded")
+ scoring_version: int = Field(default=0, description="SCORING_VERSION that produced this breakdown")
per_criterion_mean: dict[str, float]
per_criterion_std: dict[str, float]
per_sample_overall: list[float]
@@ -628,12 +717,14 @@ class ScoreBreakdown(BaseModel):
default_factory=dict, description="checklist id → present (majority vote)"
)
defect_votes: dict[str, list[bool]] = Field(default_factory=dict)
+ overridden: list[str] = Field(default_factory=list,
+ description="checklist defects the vote marked present that a passed gate measured absent (veto_measured_defects)")
defect_penalty: float = 0.0
n_requested: int
n_used: int
sample_errors: list[str] = Field(default_factory=list)
tie_broken: list[str] = Field(default_factory=list,
- description="defect / acceptance ids an exact vote tie handed to the representative sample")
+ description="ids an exact vote tie decided: acceptance → the representative sample, defect → absent")
rubric_hash: str = ""
judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "
"role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")
@@ -650,7 +741,14 @@ def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeO
def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:
- """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie)."""
+ """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie).
+
+ Acceptance ties take the representative sample (the owner's policy: a must-item tie
+ decides pass/fail, and the old ties→False failed a run on one dissent). Defect ties
+ take ``False``: the rubric puts the burden of proof on the defect ("present only when an
+ image or a gate finding shows it"), so a split vote has not met it. 419 of 420 corpus
+ verdicts are n=1 — this moves nothing on disk (2026-08-30).
+ """
if not votes:
return False, False
yes = sum(1 for v in votes if v)
@@ -716,7 +814,7 @@ def aggregate_samples(
d_votes.setdefault(did, []).append(bool(flag))
defects: dict[str, bool] = {}
for did, v in d_votes.items():
- defects[did], tied = _vote(v, bool(rep.defects.get(did, False)))
+ defects[did], tied = _vote(v, False)
if tied:
tie_broken.append(did)
# a checklist defect that a passed gate measured as absent neither penalises nor caps
@@ -754,6 +852,7 @@ def aggregate_samples(
summary = (summary + " " if summary else "") + tail
breakdown = ScoreBreakdown(
+ scoring_version=SCORING_VERSION,
per_criterion_mean=mean_scores,
per_criterion_std=std_scores,
per_sample_overall=[round(o, 4) for o in overalls],
@@ -766,6 +865,7 @@ def aggregate_samples(
acceptance_votes=votes,
defects=defects,
defect_votes=d_votes,
+ overridden=overridden,
defect_penalty=penalty,
n_requested=n_requested or len(samples),
n_used=len(samples),
@@ -836,6 +936,7 @@ def degraded_judgment(
"""A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""
breakdown = {
"status": "degraded",
+ "scoring_version": SCORING_VERSION,
"error": error,
"n_requested": n_requested,
"n_used": 0,
diff --git a/harness/codeverse/judges/rubrics/articulated_v1.yaml b/harness/codeverse/judges/rubrics/articulated_v1.yaml
index d4ad301b..d8bdac69 100644
--- a/harness/codeverse/judges/rubrics/articulated_v1.yaml
+++ b/harness/codeverse/judges/rubrics/articulated_v1.yaml
@@ -189,6 +189,8 @@ caps:
- island
- touches nothing
cap: 0.6
+ note: 'stays on gate "*" (2026-08-30): connectivity reports rest-pose floating and joint_sweep
+ "touches nothing", so unlike the object rubrics no single gate owns this measurement'
- id: penetration_error
when: gate
gate: '*'
@@ -198,9 +200,18 @@ caps:
- intersect
- overlap
cap: 0.7
+ measures:
+ - interpenetration
+ - pose_clips_body
+ note: 'stays on gate "*": connectivity can report a rest-pose interpenetration and joint_sweep
+ collides every link pair in every sampled pose, the rest pose included
+ (spatial/joints_sweep.sweep_collisions). The veto therefore needs EVERY gate clean —
+ conservative, and sound for both the rest-pose and the posed checklist claims'
- id: missing_must_acceptance
when: acceptance
cap: 0.6
+ graded: true
+ note: 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object_v1); pass/fail unchanged
- id: unattached_link
when: gate
gate: '*'
diff --git a/harness/codeverse/judges/rubrics/asset_v1.yaml b/harness/codeverse/judges/rubrics/asset_v1.yaml
index 9aaf8a70..41c55eb9 100644
--- a/harness/codeverse/judges/rubrics/asset_v1.yaml
+++ b/harness/codeverse/judges/rubrics/asset_v1.yaml
@@ -122,21 +122,27 @@ caps:
gate: "build*"
severity: error
cap: 0.0
+ # `connectivity` is the only gate on this track that emits floating / penetration
+ # ERRORs (2026-08-30); watching it alone keeps the cap and frees the veto from
+ # unrelated failing gates (judges/rubrics.measured_absent).
- id: floating_part
when: gate
- gate: "*"
+ gate: "connectivity"
severity: error
kinds: [floating, no_contact, disconnected, unsupported, island]
cap: 0.6
- id: penetration_error
when: gate
- gate: "*"
+ gate: "connectivity"
severity: error
kinds: [penetrat, intersect, overlap, collision]
cap: 0.7
+ measures: [interpenetration]
- id: missing_must_acceptance
when: acceptance
cap: 0.6
+ graded: true
+ note: 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object_v1); pass/fail unchanged
# Binary checklist (judge answers, code scores: penalty subtracted, cap applied).
defects:
diff --git a/harness/codeverse/judges/rubrics/reference_v1.yaml b/harness/codeverse/judges/rubrics/reference_v1.yaml
index 764dd3bf..72a15d7b 100644
--- a/harness/codeverse/judges/rubrics/reference_v1.yaml
+++ b/harness/codeverse/judges/rubrics/reference_v1.yaml
@@ -104,21 +104,27 @@ caps:
gate: "build*"
severity: error
cap: 0.0
+ # `connectivity` is the only gate on this track that emits floating / penetration
+ # ERRORs (2026-08-30); watching it alone keeps the cap and frees the veto from
+ # unrelated failing gates (judges/rubrics.measured_absent).
- id: floating_part
when: gate
- gate: "*"
+ gate: "connectivity"
severity: error
kinds: [floating, no_contact, disconnected, unsupported, island]
cap: 0.6
- id: penetration_error
when: gate
- gate: "*"
+ gate: "connectivity"
severity: error
kinds: [penetrat, intersect, overlap, collision]
cap: 0.7
+ measures: [interpenetration]
- id: missing_must_acceptance
when: acceptance
cap: 0.6
+ graded: true
+ note: 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object_v1); pass/fail unchanged
# Binary checklist (judge answers, code scores: penalty subtracted, cap applied).
defects:
diff --git a/harness/codeverse/judges/rubrics/static_object_v1.yaml b/harness/codeverse/judges/rubrics/static_object_v1.yaml
index 03db0a5e..846caaa2 100644
--- a/harness/codeverse/judges/rubrics/static_object_v1.yaml
+++ b/harness/codeverse/judges/rubrics/static_object_v1.yaml
@@ -153,18 +153,23 @@ caps:
severity: error
cap: 0.0
note: the code did not build or export; nothing to score
+ # Both watch `connectivity` alone (2026-08-30): over 356 static_object records it is the
+ # only gate that emits floating / penetration ERRORs (540 findings), so the cap loses
+ # nothing — and the veto (judges/rubrics.measured_absent) stops being switched off by a
+ # failing contract gate, which had blocked it in 53 of 62 cases.
- id: floating_part
when: gate
- gate: "*"
+ gate: "connectivity"
severity: error
kinds: [floating, unattached, no_contact, disconnected, unsupported, island]
cap: 0.6
- id: penetration_error
when: gate
- gate: "*"
+ gate: "connectivity"
severity: error
kinds: [penetrat, intersect, overlap, collision]
cap: 0.7
+ measures: [interpenetration]
- id: contract_violation
when: gate
gate: "contract*"
@@ -174,6 +179,11 @@ caps:
- id: missing_must_acceptance
when: acceptance
cap: 0.6
+ graded: true
+ note: >
+ 0.6 + 0.4 * verified/total over the must items (2026-08-30): the flat 0.6 was the
+ decisive cap on 121 of 424 verdicts (28.6 %), one unverified item scoring like ten.
+ The verdict still FAILS on any unverified must item; only the score keeps its gradient.
# Binary checklist: answered by the judge, scored in code (penalty subtracted, cap applied).
defects:
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index 51f4f23b..86fbe692 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -103,8 +103,9 @@ class VlmJudge:
self.model_id = model_id or get_settings().default_judge
self.n_samples = max(1, int(n_samples))
if self.n_samples % 2 == 0:
- log.warning("judge n_samples=%d is even: exact vote ties on defects / acceptance items are decided by the "
- "representative sample (rubrics.aggregate_samples); an odd n gives a true majority", self.n_samples)
+ log.warning("judge n_samples=%d is even: an exact vote tie on a defect reads as absent and a tie on an "
+ "acceptance item follows the representative sample (rubrics.aggregate_samples); an odd n "
+ "gives a true majority", self.n_samples)
self.temperature = temperature
self.thinking = thinking
self.max_attempts = max(1, int(max_attempts))
diff --git a/harness/tests/judges/test_caps.py b/harness/tests/judges/test_caps.py
index 769a9a03..dd609e06 100644
--- a/harness/tests/judges/test_caps.py
+++ b/harness/tests/judges/test_caps.py
@@ -109,3 +109,94 @@ def test_scene_placement_gate_measures_the_floating_asset_claim():
res = apply_caps(rubric, 0.9, failed, {}, [], defects_present={})
assert res.overall == 0.6, "the older floating_part rule (gate *, cap 0.6) fires too; the tighter cap wins"
assert any(c.rule == "floating_or_sunken_asset" for c in res.caps_applied)
+
+
+# ------------------------------------------- 2026-08-30: the veto reaches interpenetration
+def test_a_passed_connectivity_gate_vetoes_the_interpenetration_claim():
+ """Audited 2026-08-30 over 420 static_object verdicts: 237 interpenetration flags, 110 of
+ them citing only WARNs the rubric says to ignore, and this veto had fired for it 0 times —
+ the cap rule is ``penetration_error``, the defect ``interpenetration``, and the match was
+ on id. ``CapRule.measures`` now carries the defect id."""
+ from codeverse.judges.rubrics import measured_absent, veto_measured_defects
+
+ passed = [_connectivity(True, ("info", "all 9 parts are connected (9 contacts, gap <= 2 mm)"),
+ ("warn", "'Seat' and 'Leg' overlap by 1.4 mm (weld)"))]
+ assert measured_absent(R, "interpenetration", passed)
+ kept, overridden = veto_measured_defects(R, {"interpenetration": True, "floating_part": True, "wrong_object": True}, passed)
+ assert overridden == ["interpenetration", "floating_part"]
+ assert kept == {"interpenetration": False, "floating_part": False, "wrong_object": True}
+ failed = [_connectivity(False, ("error", "'Leg' and 'Seat' interpenetrate by ≈15 mm (5% of surface samples inside)"))]
+ assert not measured_absent(R, "interpenetration", failed), "a measured penetration is not vetoed"
+ assert not measured_absent(R, "interpenetration", []), "a gate that did not run measured nothing"
+
+
+def test_a_failing_contract_gate_does_not_disable_the_connectivity_veto():
+ """53 of the 62 blocked cases (2026-08-30): the rules watched gate "*", so a contract ERROR
+ about the bbox switched off a veto the connectivity gate had earned."""
+ from codeverse.judges.rubrics import measured_absent
+
+ contract = GateReport(gate="contract", passed=False, findings=[GateFinding(
+ gate="contract", severity=Severity.ERROR, target="bbox", message="height 0.31 m vs plan 0.45 m (31% off)")])
+ gates = [contract, _connectivity(True, ("info", "all 4 parts are connected (4 contacts)"))]
+ assert measured_absent(R, "floating_part", gates)
+ assert measured_absent(R, "interpenetration", gates)
+ res = apply_caps(R, 0.9, gates, {}, [], defects_present={"floating_part": False})
+ assert res.overall == 0.75 and [c.rule for c in res.caps_applied] == ["contract_violation"], "the contract cap itself still applies"
+
+
+def test_graded_acceptance_cap_keeps_the_gradient_but_not_the_pass():
+ """The flat 0.6 was the decisive cap on 121 of 424 verdicts (28.6 %, 2026-08-30): one
+ unverified must item out of ten scored exactly like ten out of ten."""
+ import pytest
+
+ from codeverse.judges.rubrics import (
+ AcceptanceVerdict,
+ CriterionScore,
+ JudgeOutput,
+ aggregate_samples,
+ )
+
+ ten = [AcceptanceItem(id=f"M{i}", text="x", priority="must") for i in range(10)]
+ nine_ok = {a.id: True for a in ten} | {"M3": False}
+ res = apply_caps(R, 0.99, [], nine_ok, ten)
+ assert res.overall == pytest.approx(0.96) and res.caps_applied[0].rule == "missing_must_acceptance"
+ assert res.caps_applied[0].evidence.startswith("9 of 10 must items verified; not verified: M3")
+ assert apply_caps(R, 0.99, [], {}, ten[:2]).overall == pytest.approx(0.6), "0 of 2 verified is the floor"
+ assert apply_caps(R, 0.99, [], {"M0": True}, ten[:2]).overall == pytest.approx(0.8)
+ assert apply_caps(load_rubric("scene_v1"), 0.99, [], nine_ok, ten).overall == 0.6, "an ungraded rule keeps its flat cap"
+ # pass/fail is the owner's policy and unchanged: the verdict still fails on must_missing
+ sample = JudgeOutput(criteria={c.id: CriterionScore(score=0.99, evidence="") for c in R.criteria},
+ acceptance={k: AcceptanceVerdict(verified=v) for k, v in nine_ok.items()},
+ defects={d.id: False for d in R.defects})
+ j = aggregate_samples(R, [sample], gates=[], acceptance_items=ten)
+ assert j.overall == pytest.approx(0.96) and not j.passed and "must items unverified: M3" in j.summary
+
+
+def test_a_defect_vote_tie_is_absent_and_an_acceptance_tie_still_follows_the_representative():
+ """Burden of proof on the defect (2026-08-30): the rubric marks an item present only when an
+ image or a gate finding shows it, so a split vote has not shown it. Acceptance ties keep the
+ owner's policy (representative sample). 419/420 corpus verdicts are n=1: nothing moved."""
+ import json
+
+ import pytest
+
+ from codeverse.judges.rubrics import (
+ AcceptanceVerdict,
+ CriterionScore,
+ JudgeOutput,
+ aggregate_samples,
+ )
+
+ def sample(defect_on: bool, acc_ok: bool) -> JudgeOutput:
+ return JudgeOutput(criteria={c.id: CriterionScore(score=0.9, evidence="") for c in R.criteria},
+ acceptance={"M1": AcceptanceVerdict(verified=acc_ok)},
+ defects={d.id: (d.id == "render_artifacts" and defect_on) for d in R.defects})
+
+ item = [AcceptanceItem(id="M1", text="x", priority="must")]
+ j = aggregate_samples(R, [sample(True, True), sample(False, True)], gates=[], acceptance_items=item)
+ raw = json.loads(j.raw)
+ assert raw["defect_votes"]["render_artifacts"] == [True, False] and raw["defects"]["render_artifacts"] is False
+ assert raw["tie_broken"] == ["render_artifacts"] and j.overall == pytest.approx(0.9)
+ j2 = aggregate_samples(R, [sample(False, True), sample(False, False)], gates=[], acceptance_items=item)
+ assert j2.acceptance_results["M1"] is True and json.loads(j2.raw)["tie_broken"] == ["M1"]
+ assert raw["scoring_version"] == 2 and raw["overridden"] == []
diff --git a/harness/tests/judges/test_defects.py b/harness/tests/judges/test_defects.py
index 5bb486e0..9c5fabd6 100644
--- a/harness/tests/judges/test_defects.py
+++ b/harness/tests/judges/test_defects.py
@@ -101,15 +101,22 @@ def test_defect_majority_vote_with_odd_n(judge_input, cache_dir):
assert set(raw["per_criterion_std"]) == set(R.weights)
-def test_defect_vote_ties_follow_the_representative_sample(judge_input, cache_dir, caplog):
- """Exact defect-vote ties follow the representative sample."""
+def test_defect_vote_ties_are_absent(judge_input, cache_dir, caplog):
+ """An exact defect-vote tie is ABSENT (2026-08-30; was: the representative sample, D36).
+
+ At n=2 the representative is whichever sample's overall sits nearer the mean — a
+ float comparison, not evidence — so a 1-1 tie was a coin flip on a defect that caps
+ the run at 0.7. The rubric already puts the burden on the defect ("mark an item
+ present only when an image or a gate finding shows it"); a tie has not met it.
+ Acceptance ties still follow the representative (the test below): a must-item tie
+ decides pass/fail, and that policy is the owner's."""
flagged_first = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)]})
with caplog.at_level("WARNING", logger="codeverse.judges.vlm_judge"):
j = VlmJudge("static_object_v1", chat_model=flagged_first, n_samples=2, cache_dir=cache_dir).judge(judge_input)
assert "n_samples=2 is even" in caplog.text
raw = json.loads(j.raw)
assert raw["defect_votes"]["render_artifacts"] == [True, False]
- assert raw["defects"]["render_artifacts"] is True and j.overall == pytest.approx(0.85)
+ assert raw["defects"]["render_artifacts"] is False and j.overall == pytest.approx(0.9)
assert raw["tie_broken"] == ["render_artifacts"]
clean_first = FakeChatModel(by_label={":s0": [_reply(0.9)], ":s1": [_reply(0.9, ["render_artifacts"])]})
diff --git a/harness/tests/judges/test_detail_anchors.py b/harness/tests/judges/test_detail_anchors.py
index 471d6c7e..7a806070 100644
--- a/harness/tests/judges/test_detail_anchors.py
+++ b/harness/tests/judges/test_detail_anchors.py
@@ -13,8 +13,9 @@ from __future__ import annotations
import pytest
+from codeverse.contracts.plan import AcceptanceItem
from codeverse.judges.prompt_builder import build_system_prompt
-from codeverse.judges.rubrics import load_rubric
+from codeverse.judges.rubrics import _rule_hit, load_rubric
#: rubric -> the id of its geometry-detail criterion
DETAIL_CRITERION = {
@@ -88,6 +89,29 @@ def test_physical_plausibility_caps_are_untouched() -> None:
assert defects["wrong_object"].cap == 0.25
[email protected]("rubric", ["static_object_v1", "articulated_v1"])
+def test_acceptance_cap_is_graded_with_the_floor_pinned(rubric: str) -> None:
+ """The acceptance cap is GRADED since 2026-08-30 (DECISIONS D46 b): 0.6 + 0.4·verified/total.
+
+ The first version of the guard above pinned ``.cap`` alone and so waved that change
+ through without noticing. This one pins both halves: the flag, so a rubric edit cannot
+ silently fall back to the flat cap, and the floor, so 0 of n verified still scores 0.6.
+ """
+ rule = next(c for c in load_rubric(rubric).caps if c.id == "missing_must_acceptance")
+ assert rule.when == "acceptance" and rule.cap == 0.6 and rule.graded is True
+ items = [AcceptanceItem(id=f"M{i}", text=f"must {i}", priority="must") for i in range(10)]
+
+ def cap_at(verified: int) -> float:
+ results = {a.id: i < verified for i, a in enumerate(items)}
+ hit = _rule_hit(rule, [], results, items, [], [])
+ assert hit is not None
+ return hit.cap
+
+ assert cap_at(0) == pytest.approx(0.6)
+ assert cap_at(9) == pytest.approx(0.96)
+ assert _rule_hit(rule, [], {a.id: True for a in items}, items, [], []) is None
+
+
def test_craftsmanship_top_anchor_is_detail_scale() -> None:
for rubric in ("static_object_v1", "articulated_v1"):
top = load_rubric(rubric).criterion("craftsmanship_no_artifacts").anchors["1.0"].lower()
diff --git a/harness/tests/judges/test_replay.py b/harness/tests/judges/test_replay.py
new file mode 100644
index 00000000..b055ba36
--- /dev/null
+++ b/harness/tests/judges/test_replay.py
@@ -0,0 +1,136 @@
+"""``bench/rejudge_offline.replay_round`` reproduces a stored verdict from its samples.
+
+The tool's own acceptance test: a Judgment built by ``aggregate_samples`` from two synthetic
+samples and a gate set, written as ``rounds/r00.json`` + ``plan.json``, must replay to the
+same ``overall`` within 1e-9, carry the ``scoring_version`` that wrote it, and expose the
+veto's ``overridden`` list from ``raw`` (the 4ddde32 tool parsed the verdict tail for a
+phrase it never contained and reported no vetoes at all).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+import bench.rejudge_offline as ro
+from codeverse.contracts.artifacts import (
+ GateFinding,
+ GateReport,
+ Judgment,
+ RenderSet,
+ RenderView,
+ Severity,
+)
+from codeverse.contracts.plan import AcceptanceItem
+from codeverse.contracts.run import RoundRecord
+from codeverse.judges.rubrics import (
+ SCORING_VERSION,
+ AcceptanceVerdict,
+ CriterionScore,
+ JudgeOutput,
+ aggregate_samples,
+ load_rubric,
+)
+
+R = load_rubric("static_object_v1")
+ITEMS = [AcceptanceItem(id="M1", text="four legs", priority="must"),
+ AcceptanceItem(id="M2", text="0.45 m tall", priority="must", how="measure"),
+ AcceptanceItem(id="S1", text="bevelled edges", priority="should")]
+# a failing contract gate next to a passed connectivity gate: the veto must read connectivity alone
+GATES = [
+ GateReport(gate="contract", passed=False, findings=[GateFinding(
+ gate="contract", severity=Severity.ERROR, target="bbox", message="height 0.31 m vs plan 0.45 m")]),
+ GateReport(gate="connectivity", passed=True, findings=[
+ GateFinding(gate="connectivity", severity=Severity.INFO, target="", message="all 5 parts are connected (5 contacts)"),
+ GateFinding(gate="connectivity", severity=Severity.WARN, target="", message="'Seat' and 'Leg_0' overlap by 1.2 mm (weld)")]),
+]
+
+
+def _sample(score: float, *defects: str) -> JudgeOutput:
+ return JudgeOutput(
+ criteria={c.id: CriterionScore(score=score, evidence="v") for c in R.criteria}, summary="a chair",
+ acceptance={"M1": AcceptanceVerdict(verified=True), "M2": AcceptanceVerdict(verified=False),
+ "S1": AcceptanceVerdict(verified=True)},
+ defects={d.id: d.id in defects for d in R.defects},
+ )
+
+
+def _write_round(root: Path, judgment: Judgment | None) -> Path:
+ run = root / "run"
+ (run / "rounds").mkdir(parents=True)
+ (run / "plan.json").write_text(json.dumps({"acceptance": [a.model_dump() for a in ITEMS]}))
+ rec = RoundRecord(index=0, kind="baseline", gates=GATES,
+ renders=RenderSet(views=[RenderView(name="front", path="front.png")]), judgment=judgment)
+ path = run / "rounds" / "r00.json"
+ path.write_text(rec.model_dump_json())
+ return path
+
+
+def test_replay_reproduces_a_stored_verdict_and_reports_its_scoring_version(tmp_path):
+ samples = [_sample(0.9, "interpenetration", "render_artifacts"), _sample(0.8, "interpenetration", "render_artifacts")]
+ j = aggregate_samples(R, samples, gates=GATES, acceptance_items=ITEMS, judge_backend="fake:judge")
+ # uncapped 0.85, render_artifacts -0.05 → 0.80; interpenetration vetoed by the passed connectivity
+ # gate; caps: contract 0.75, graded acceptance 0.6 + 0.4·1/2 = 0.8 → 0.75
+ assert j.overall == pytest.approx(0.75) and not j.passed
+ raw = json.loads(j.raw)
+ assert raw["overridden"] == ["interpenetration"] and raw["scoring_version"] == SCORING_VERSION
+
+ row = ro.replay_round(_write_round(tmp_path, j))
+ assert row is not None
+ assert row.stored == j.overall and abs(row.replay - j.overall) <= 1e-9 and abs(row.delta) <= 1e-9
+ assert row.scoring_version_stored == SCORING_VERSION == row.scoring_version_now
+ assert row.rubric_hash_stored == row.rubric_hash_now == R.content_hash()
+ assert row.overridden == ["interpenetration"]
+ assert row.caps_before == row.caps_after == ["contract_violation", "missing_must_acceptance"]
+ assert row.defects_before == row.defects_after == ["render_artifacts"]
+ assert row.gate_errors == 1 and row.stored_passed is False and row.replay_passed is False
+
+
+def test_identity_holds_same_version_verdicts_and_reports_older_ones_as_drift(tmp_path, capsys):
+ j = aggregate_samples(R, [_sample(0.9)], gates=GATES, acceptance_items=ITEMS)
+
+ def stored_as(version: int, overall: float) -> Judgment:
+ raw = json.loads(j.raw) | {"scoring_version": version, "overall": overall}
+ return j.model_copy(update={"overall": overall, "raw": json.dumps(raw)})
+
+ off = round(j.overall + 0.05, 4)
+ _write_round(tmp_path / "now", stored_as(SCORING_VERSION, off))
+ assert ro.main([str(tmp_path / "now"), "--rubric", R.name, "--identity"]) == 1
+ assert "IDENTITY FAILED: 1 verdict(s)" in capsys.readouterr().err
+
+ # a corpus with NOTHING at today's version is drift all the way down: the guard is dormant,
+ # says so, and refuses the green exit unless told the empty set is expected
+ _write_round(tmp_path / "old", stored_as(1, off))
+ assert ro.main([str(tmp_path / "old"), "--rubric", R.name, "--identity"]) == 2
+ err = capsys.readouterr().err
+ assert "identity guard is DORMANT" in err and f"scoring version {SCORING_VERSION}" in err and "nothing was checked" in err
+ assert ro.main([str(tmp_path / "old"), "--rubric", R.name, "--identity", "--allow-empty-identity"]) == 0
+ assert "identity guard is DORMANT" in capsys.readouterr().err
+ rep = ro.replay_corpus([tmp_path / "old"], rubric_name=R.name)
+ assert rep.n == 1 and rep.n_moved == 1 and rep.n_scoring_drift == 1
+ assert rep.rows[0].scoring_version_stored == 1 and rep.rows[0].delta == pytest.approx(-0.05)
+
+ _write_round(tmp_path / "same", stored_as(SCORING_VERSION, j.overall))
+ assert ro.main([str(tmp_path / "same"), "--rubric", R.name, "--identity"]) == 0
+ assert "identity OK: 1 of 1" in capsys.readouterr().out
+
+
+def test_identity_is_dormant_on_a_pre_stamp_corpus(tmp_path, capsys):
+ """Every verdict on disk today is version 0 (written before the stamp): the guard must not print green."""
+ j = aggregate_samples(R, [_sample(0.9)], gates=GATES, acceptance_items=ITEMS)
+ raw = json.loads(j.raw)
+ raw.pop("scoring_version", None)
+ _write_round(tmp_path, j.model_copy(update={"raw": json.dumps(raw)}))
+ assert ro.replay_corpus([tmp_path], rubric_name=R.name).rows[0].scoring_version_stored == 0
+
+ assert ro.main([str(tmp_path), "--rubric", R.name, "--identity"]) == 2
+ out = capsys.readouterr()
+ assert "identity OK" not in out.out
+ assert "WARNING: identity guard is DORMANT" in out.err and "1 under an older scoring version" in out.err
+ assert ro.main([str(tmp_path), "--rubric", R.name, "--identity", "--allow-empty-identity"]) == 0
+
+
+def test_a_round_without_a_judged_sample_is_skipped(tmp_path):
+ assert ro.replay_round(_write_round(tmp_path, None)) is None
diff --git a/harness/tests/judges/test_rubrics.py b/harness/tests/judges/test_rubrics.py
index 610a725b..fe29b7e9 100644
--- a/harness/tests/judges/test_rubrics.py
+++ b/harness/tests/judges/test_rubrics.py
@@ -56,3 +56,44 @@ def test_unknown_and_invalid_rubric():
def test_content_hash_stable():
assert load_rubric("scene_v1").content_hash() == load_rubric("scene_v1").content_hash()
assert load_rubric("scene_v1").content_hash() != load_rubric("asset_v1").content_hash()
+
+
+def test_cap_rule_measures_defaults_to_its_own_id_and_the_object_rubrics_name_interpenetration():
+ """2026-08-30: ``penetration_error`` had never vetoed ``interpenetration`` because the veto
+ matched on id; the object rubrics now say which checklist defect each gate rule measures."""
+ from codeverse.judges.rubrics import CapRule
+
+ assert CapRule(id="floating_part", cap=0.6).measures == ["floating_part"]
+ assert CapRule(id="penetration_error", cap=0.7, measures=["interpenetration"]).measures == ["interpenetration"]
+ for name in ("static_object_v1", "reference_v1", "asset_v1", "articulated_v1"):
+ r = load_rubric(name)
+ rule = next(c for c in r.caps if c.id == "penetration_error")
+ assert "interpenetration" in rule.measures, name
+ assert next(c for c in r.caps if c.id == "missing_must_acceptance").graded, name
+ for name in ("static_object_v1", "reference_v1", "asset_v1"):
+ r = load_rubric(name)
+ assert {c.gate for c in r.caps if c.id in ("floating_part", "penetration_error")} == {"connectivity"}, name
+ # articulated keeps gate "*": connectivity AND joint_sweep both report floating / penetration there
+ a = load_rubric("articulated_v1")
+ assert {c.gate for c in a.caps if c.id in ("floating_part", "penetration_error")} == {"*"}
+ for name in ("scene_v1", "shader_v1", "shader_v2"): # not object rubrics: the flat cap stays
+ assert not next(c for c in load_rubric(name).caps if c.id == "missing_must_acceptance").graded, name
+
+
+def test_cap_rules_are_not_part_of_the_judge_prompt():
+ """The 2026-08-30 cap changes move ``content_hash`` but not ``judge_prompt_hash``: cap rules
+ are scored in code and never rendered to the judge (prompt_builder._rubric_block)."""
+ import yaml
+
+ from codeverse.judges.prompt_builder import judge_prompt_hash
+ from codeverse.judges.rubrics import RUBRICS_DIR, rubric_from_dict
+
+ r = load_rubric("static_object_v1")
+ data = yaml.safe_load((RUBRICS_DIR / "static_object_v1.yaml").read_text())
+ data["caps"] = []
+ stripped = rubric_from_dict(data)
+ assert stripped.content_hash() != r.content_hash(), "content_hash hashes what the YAML declares"
+ # …and only that: a schema default the YAML never wrote does not re-key recorded verdicts
+ assert r.content_hash() == rubric_from_dict(yaml.safe_load((RUBRICS_DIR / "static_object_v1.yaml").read_text())).content_hash()
+ assert judge_prompt_hash(stripped) == judge_prompt_hash(r)
+ assert "interpenetration" in judge_prompt_hash.__globals__["build_system_prompt"](r), "the checklist defect IS rendered"