Official Codebase of 3DCodeVerse
Python
bench: replay every stored verdict through today's scoring code, no model calls
gaoypenga8b11aa4de1f
diff --git a/harness/bench/rejudge_offline.py b/harness/bench/rejudge_offline.pynew file mode 100644index 00000000..e2483592--- /dev/null+++ b/harness/bench/rejudge_offline.py@@ -0,0 +1,298 @@+"""Re-aggregate every stored judge verdict through today's scoring code — no model calls.++Every static-object verdict on disk carries its raw samples (``Judgment.raw`` →+``samples``), and ``rounds/rNN.json`` carries the gates, renders and console errors+that round was scored against. So the whole scoring pipeline downstream of the model+— majority vote, the measured-absent veto, defect penalties, the cap ladder, pass —+can be replayed from disk through :func:`codeverse.judges.rubrics.aggregate_samples`+and diffed against what the run recorded.++Two uses:++* **identity** — run it on an unchanged tree: every verdict must reproduce to 1e-9.+ That is the acceptance test for this tool, and the guard that says the corpus is+ still readable. (A verdict scored under an older rubric hash is reported, not+ failed: the rubric moved, the tool did not.)+* **impact** — change ``rubrics.py`` or a rubric YAML, run it again: every moved+ verdict, with its before/after caps and defects, the shift in pass rate, in the+ 0.600 spike, in σ and in pearson(gate errors, overall). That is how a scoring+ change is measured BEFORE a single dollar is spent re-judging (docs/EVAL.md §8:+ a battery cannot resolve a 0.03 change; 420 stored rounds can).++Cost: seconds of CPU. It never writes into a run.++CLI::++ python bench/rejudge_offline.py bench/out --rubric static_object_v1 --out scratch/replay+ python bench/rejudge_offline.py bench/out --rubric static_object_v1 --identity # exit 1 on any drift+"""++from __future__ import annotations++import argparse+import json+import statistics+import sys+from collections import Counter+from pathlib import Path++from pydantic import BaseModel, Field++from codeverse.contracts.artifacts import GateReport, Judgment, Severity+from codeverse.contracts.plan import AcceptanceItem+from codeverse.contracts.run import RoundRecord+from codeverse.judges.rubrics import JudgeOutput, Rubric, aggregate_samples, load_rubric+from codeverse.proc import read_json_or_none++TOL = 1e-9+++class ReplayRow(BaseModel):+ run: str+ round: int+ kind: str+ rubric_hash_stored: str+ rubric_hash_now: str+ gate_errors: int+ stored: float+ replay: float+ delta: float+ stored_passed: bool+ replay_passed: bool+ caps_before: list[str]+ caps_after: list[str]+ defects_before: list[str]+ defects_after: list[str]+ overridden: list[str] = Field(default_factory=list, description="defects the measured-absent veto switched off on replay")+ error: str = ""+++class ReplayReport(BaseModel):+ rubric: str+ n: int+ n_errors: int+ n_moved: int+ n_rubric_drift: int = Field(description="verdicts whose stored rubric hash differs from today's — drift, not a tool fault")+ mean_delta: float+ median_delta: float+ max_abs_delta: float+ pass_rate_before: float+ pass_rate_after: float+ spike_600_before: int+ spike_600_after: int+ sd_before: float+ sd_after: float+ pearson_errors_before: float | None+ pearson_errors_after: float | None+ cap_counts_before: dict[str, int]+ cap_counts_after: dict[str, int]+ defect_counts_before: dict[str, int]+ defect_counts_after: dict[str, int]+ rows: list[ReplayRow]++ def to_markdown(self) -> str:+ f = lambda v: "n/a" if v is None else f"{v:+.3f}" # noqa: E731+ moved = [r for r in self.rows if abs(r.delta) > TOL]+ lines = [+ f"# offline replay — {self.rubric}",+ "",+ f"n={self.n} verdicts ({self.n_errors} unreadable, {self.n_rubric_drift} scored under another rubric hash)",+ f"moved: **{self.n_moved}** mean Δ {self.mean_delta:+.4f} median Δ {self.median_delta:+.4f} max |Δ| {self.max_abs_delta:.4f}",+ f"pass rate {self.pass_rate_before:.1%} → {self.pass_rate_after:.1%} "+ f"scores pinned at 0.600: {self.spike_600_before} → {self.spike_600_after} "+ f"σ {self.sd_before:.3f} → {self.sd_after:.3f} pearson(gate errors, overall) {f(self.pearson_errors_before)} → {f(self.pearson_errors_after)}",+ "",+ "| cap rule | before | after |", "|---|---|---|",+ ]+ for k in sorted(set(self.cap_counts_before) | set(self.cap_counts_after)):+ lines.append(f"| {k} | {self.cap_counts_before.get(k, 0)} | {self.cap_counts_after.get(k, 0)} |")+ lines += ["", "| defect | before | after |", "|---|---|---|"]+ for k in sorted(set(self.defect_counts_before) | set(self.defect_counts_after)):+ lines.append(f"| {k} | {self.defect_counts_before.get(k, 0)} | {self.defect_counts_after.get(k, 0)} |")+ if moved:+ lines += ["", f"## moved verdicts ({len(moved)})", "",+ "| run | round | gate err | stored | replay | Δ | caps before → after | overridden |", "|---|---|---|---|---|---|---|---|"]+ for r in sorted(moved, key=lambda r: -abs(r.delta))[:200]:+ lines.append(f"| {r.run} | r{r.round:02d} | {r.gate_errors} | {r.stored:.3f} | {r.replay:.3f} | {r.delta:+.3f} | "+ f"{', '.join(r.caps_before) or '-'} → {', '.join(r.caps_after) or '-'} | {', '.join(r.overridden) or '-'} |")+ return "\n".join(lines)+++# --------------------------------------------------------------------------- discovery+def round_files(roots: list[Path]) -> list[Path]:+ """Every ``rounds/rNN.json`` under the roots, once each (symlinked layouts deduped)."""+ seen: set[Path] = set()+ out: list[Path] = []+ for root in roots:+ for p in sorted(Path(root).rglob("rounds/r[0-9][0-9].json")):+ real = p.resolve()+ if real in seen:+ continue+ seen.add(real)+ out.append(p)+ return out+++def _run_label(run_dir: Path) -> str:+ parts = run_dir.resolve().parts+ tail = [q for q in parts[-4:] if q not in ("runs", "cells")]+ return "/".join(tail[-3:]) if tail and tail[-1] in ("run",) else "/".join(tail[-2:])+++def _acceptance_items(run_dir: Path) -> list[AcceptanceItem]:+ """The plan's acceptance list; a candidate sub-workspace inherits its parent's plan."""+ d = run_dir+ for _ in range(4):+ plan = read_json_or_none(d / "plan.json")+ if plan:+ return [AcceptanceItem.model_validate(a) for a in (plan.get("acceptance") or [])]+ d = d.parent+ return []+++# --------------------------------------------------------------------------- replay+def replay_round(path: Path, rubric: Rubric | None = None) -> ReplayRow | None:+ """Replay one stored round; ``None`` when the round carries no judged samples."""+ rec = RoundRecord.model_validate_json(path.read_text())+ stored = rec.judgment+ if stored is None or not stored.raw:+ return None+ raw = json.loads(stored.raw)+ samples = [JudgeOutput.model_validate(s) for s in raw.get("samples") or []]+ if not samples:+ return None+ run_dir = path.parent.parent+ rub = rubric or load_rubric(stored.rubric)+ if rub.name != stored.rubric:+ return None+ gates: list[GateReport] = list(rec.gates)+ views = list(rec.renders.views) if rec.renders else []+ console = list(rec.renders.console_errors) if rec.renders else []+ now = aggregate_samples(+ rub, samples, gates=gates, acceptance_items=_acceptance_items(run_dir),+ console_errors=console, views=views, judge_backend=stored.judge_backend,+ n_requested=raw.get("n_requested"), sample_errors=raw.get("sample_errors"),+ judge_prompt_hash=raw.get("judge_prompt_hash", ""),+ )+ raw_now = json.loads(now.raw)+ gate_errors = sum(1 for g in gates for f in g.findings if f.severity == Severity.ERROR)+ return ReplayRow(+ run=_run_label(run_dir), round=rec.index, kind=rec.kind,+ rubric_hash_stored=raw.get("rubric_hash", ""), rubric_hash_now=raw_now.get("rubric_hash", ""),+ gate_errors=gate_errors,+ stored=stored.overall, replay=now.overall, delta=round(now.overall - stored.overall, 6),+ stored_passed=stored.passed, replay_passed=now.passed,+ caps_before=[c["rule"] for c in (raw.get("caps") or {}).get("caps_applied", [])],+ caps_after=[c.rule for c in raw_now_caps(now)],+ defects_before=[d for d, on in (raw.get("defects") or {}).items() if on],+ defects_after=[d for d, on in (raw_now.get("defects") or {}).items() if on],+ overridden=_overridden(now),+ )+++def raw_now_caps(j: Judgment):+ from codeverse.judges.rubrics import CapApplied++ return [CapApplied.model_validate(c) for c in (json.loads(j.raw).get("caps") or {}).get("caps_applied", [])]+++def _overridden(j: Judgment) -> list[str]:+ # the verdict tail names them: "… | measured absent: floating_part"+ tail = j.summary.rsplit("[verdict:", 1)[-1]+ if "measured absent" not in tail:+ return []+ seg = tail.split("measured absent:", 1)[1].split("|")[0].split("]")[0]+ return [s.strip() for s in seg.split(",") if s.strip()]+++def _pearson(xs: list[float], ys: list[float]) -> float | None:+ if len(xs) < 3 or len(set(xs)) < 2 or len(set(ys)) < 2:+ return None+ return round(statistics.correlation(xs, ys), 4)+++def replay_corpus(roots: list[Path], *, rubric_name: str, rubric: Rubric | None = None,+ kinds: set[str] | None = None) -> ReplayReport:+ rows: list[ReplayRow] = []+ n_err = 0+ for p in round_files(roots):+ try:+ row = replay_round(p, rubric)+ except Exception as e: # noqa: BLE001 — one corrupt round must not stop the corpus+ n_err += 1+ rows.append(ReplayRow(run=str(p), round=-1, kind="?", rubric_hash_stored="", rubric_hash_now="",+ gate_errors=0, stored=0.0, replay=0.0, delta=0.0, stored_passed=False,+ replay_passed=False, caps_before=[], caps_after=[], defects_before=[],+ defects_after=[], error=f"{type(e).__name__}: {e}"))+ continue+ if row is None:+ continue+ rec_rubric = json.loads(p.read_text()).get("judgment", {}).get("rubric")+ if rec_rubric != rubric_name:+ continue+ if kinds and row.kind not in kinds:+ continue+ rows.append(row)+ ok = [r for r in rows if not r.error]+ deltas = [r.delta for r in ok]+ before = [r.stored for r in ok]+ after = [r.replay for r in ok]+ errs = [float(r.gate_errors) for r in ok]+ caps_b = Counter(c for r in ok for c in r.caps_before)+ caps_a = Counter(c for r in ok for c in r.caps_after)+ def_b = Counter(d for r in ok for d in r.defects_before)+ def_a = Counter(d for r in ok for d in r.defects_after)+ return ReplayReport(+ rubric=rubric_name, n=len(ok), n_errors=n_err,+ n_moved=sum(1 for d in deltas if abs(d) > TOL),+ n_rubric_drift=sum(1 for r in ok if r.rubric_hash_stored and r.rubric_hash_stored != r.rubric_hash_now),+ mean_delta=round(statistics.fmean(deltas), 6) if deltas else 0.0,+ median_delta=round(statistics.median(deltas), 6) if deltas else 0.0,+ max_abs_delta=round(max((abs(d) for d in deltas), default=0.0), 6),+ pass_rate_before=(sum(r.stored_passed for r in ok) / len(ok)) if ok else 0.0,+ pass_rate_after=(sum(r.replay_passed for r in ok) / len(ok)) if ok else 0.0,+ spike_600_before=sum(1 for v in before if abs(v - 0.6) < 1e-6),+ spike_600_after=sum(1 for v in after if abs(v - 0.6) < 1e-6),+ sd_before=round(statistics.pstdev(before), 4) if len(before) > 1 else 0.0,+ sd_after=round(statistics.pstdev(after), 4) if len(after) > 1 else 0.0,+ pearson_errors_before=_pearson(errs, before), pearson_errors_after=_pearson(errs, after),+ cap_counts_before=dict(caps_b), cap_counts_after=dict(caps_a),+ defect_counts_before=dict(def_b), defect_counts_after=dict(def_a),+ rows=rows,+ )+++# --------------------------------------------------------------------------- cli+def main(argv: list[str] | None = None) -> int:+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])+ ap.add_argument("roots", nargs="+", type=Path)+ ap.add_argument("--rubric", default="static_object_v1", help="rubric NAME the stored verdicts must carry")+ ap.add_argument("--rubric-file", type=Path, default=None,+ help="score with THIS yaml instead of the installed rubric of the same name (a variant under test)")+ ap.add_argument("--kinds", default="", help="comma list of round kinds to keep (default all, incl. candidate)")+ ap.add_argument("--out", type=Path, default=None, help="directory for replay.json + replay.md")+ ap.add_argument("--identity", action="store_true",+ help="assert nothing moved: exit 1 on any |Δ| > 1e-9 outside rubric drift")+ a = ap.parse_args(argv)+ rubric = load_rubric(str(a.rubric_file)) if a.rubric_file else None+ kinds = {k for k in a.kinds.split(",") if k} or None+ rep = replay_corpus(a.roots, rubric_name=a.rubric, rubric=rubric, kinds=kinds)+ if a.out:+ a.out.mkdir(parents=True, exist_ok=True)+ (a.out / "replay.json").write_text(rep.model_dump_json(indent=1))+ (a.out / "replay.md").write_text(rep.to_markdown())+ print(rep.to_markdown().split("\n## moved")[0])+ if a.identity:+ drift = {r.rubric_hash_stored for r in rep.rows if r.rubric_hash_stored and r.rubric_hash_stored != r.rubric_hash_now}+ bad = [r for r in rep.rows if not r.error and abs(r.delta) > TOL and r.rubric_hash_stored == r.rubric_hash_now]+ if bad:+ print(f"\nIDENTITY FAILED: {len(bad)} verdict(s) under today's rubric hash do not reproduce:", file=sys.stderr)+ for r in bad[:20]:+ print(f" {r.run} r{r.round:02d}: stored {r.stored:.4f} replay {r.replay:.4f}", file=sys.stderr)+ return 1+ print(f"\nidentity OK: {rep.n} verdicts reproduce" + (f" ({len(drift)} older rubric hash(es) skipped)" if drift else ""))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())