Official Codebase of 3DCodeVerse
Python
judge: a fixed-order re-judge mode, so σ can mean what the loop assumes it means
gaoypeng04b8cd68a6fd
diff --git a/harness/codeverse/judges/calibration.py b/harness/codeverse/judges/calibration.pyindex 95bcc75a..926f2304 100644--- a/harness/codeverse/judges/calibration.py+++ b/harness/codeverse/judges/calibration.py@@ -109,6 +109,7 @@ class CalibrationRow(BaseModel):class CalibrationTable(BaseModel):model_id: strn_samples: int+ fixed_order: bool = Falserows: list[CalibrationRow]pearson_errors_vs_score: float | None = Nonespearman_errors_vs_score: float | None = None@@ -143,7 +144,8 @@ class CalibrationTable(BaseModel):fmt = lambda v: "n/a" if v is None else f"{v:+.3f}" # noqa: E731lines.append("")lines.append(- f"model {self.model_id}, n_samples {self.n_samples}: pearson(gate errors, new score) = {fmt(self.pearson_errors_vs_score)}; "+ f"model {self.model_id}, n_samples {self.n_samples}{' (fixed montage order: σ is the re-judge noise)' if self.fixed_order else ''}: "+ f"pearson(gate errors, new score) = {fmt(self.pearson_errors_vs_score)}; "f"spearman = {fmt(self.spearman_errors_vs_score)}; pearson(stored, new) = {fmt(self.pearson_stored_vs_new)}; "f"mean overall std {self.mean_std:.3f}; mean per-criterion std {self.mean_criterion_std:.3f}; cost ${self.total_cost_usd:.3f}")@@ -266,11 +268,14 @@ def calibrate(thinking: str = "low",max_workers: int = 4,chat_model: Any = None,+ fixed_order: bool = False,) -> CalibrationTable:"""Re-judge every round of ``run_dirs`` with ``n_samples`` and tabulate separation/noise.Writes ``calibration_<model>.json`` / ``.md`` into ``out_dir`` (default: cwd) andnever touches the run directories. ``chat_model`` injects a ChatModel (tests).+ ``fixed_order`` sends every sample the same montage order, so ``mean_std`` becomes the+ model's own re-judge σ instead of its order-permutation σ (see ``VlmJudge.fixed_order``)."""from codeverse.config import get_settings@@ -286,12 +291,13 @@ def calibrate(for i, c in enumerate(cases):geometry[i] = render_geometry_views(c, out, geometry_mode) if (geometry_mode and c.is_best) else Nonejudges = {c.rubric: VlmJudge(c.rubric, model_id=model_id, n_samples=n_samples, thinking=thinking, # type: ignore[arg-type]- cache_dir=out / "cache", label="calibrate", chat_model=chat_model) for c in cases}+ cache_dir=out / "cache", label="calibrate", chat_model=chat_model,+ fixed_order=fixed_order) for c in cases}with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool:rows = list(pool.map(lambda ic: _judge_case(ic[1], judges[ic[1].rubric], geometry[ic[0]], out), enumerate(cases)))ok = [r for r in rows if not r.error]table = CalibrationTable(- model_id=model_id, n_samples=n_samples, rows=rows,+ model_id=model_id, n_samples=n_samples, fixed_order=fixed_order, rows=rows,pearson_errors_vs_score=pearson([float(r.gate_errors) for r in ok], [r.mean for r in ok]),spearman_errors_vs_score=spearman([float(r.gate_errors) for r in ok], [r.mean for r in ok]),pearson_stored_vs_new=pearson([r.stored_overall for r in ok if r.stored_overall is not None],@@ -316,12 +322,14 @@ def main(argv: list[str] | None = None) -> int:ap.add_argument("--rounds", default="", help="comma-separated round indices (default all)")ap.add_argument("--thinking", default="low")ap.add_argument("--workers", type=int, default=4)+ ap.add_argument("--fixed-order", action="store_true",+ help="same montage order for every sample: measures the model's re-judge σ, not order robustness")a = ap.parse_args(argv)logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")rounds = [int(x) for x in a.rounds.split(",") if x.strip()] or Nonetable = calibrate(a.run_dirs, model_id=a.model, n_samples=a.n, out_dir=a.out,geometry_mode=None if a.geometry == "none" else a.geometry, rounds=rounds,- thinking=a.thinking, max_workers=a.workers)+ thinking=a.thinking, max_workers=a.workers, fixed_order=a.fixed_order)print(table.to_markdown())return 0diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.pyindex 41bcb3d1..51f4f23b 100644--- a/harness/codeverse/judges/vlm_judge.py+++ b/harness/codeverse/judges/vlm_judge.py@@ -90,8 +90,16 @@ class VlmJudge:cache_dir: Path | None = None,label: str = "judge",sample_budget_s: float | None = None,+ fixed_order: bool = False,):self.rubric: Rubric = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)+ # Every sample of one call normally sees the montages and tiles in a different order+ # (shuffle_seed below), so score_std is a view-ORDER robustness figure — the number+ # cost/routing.JUDGE_NOISE was tabulated from. fixed_order sends the identical+ # prompt n times: the re-judge σ of the model itself, which is what the loop's+ # stop thresholds and the pairwise margin are actually keyed to (audit 2026-08-30;+ # brilliana measured 0.013 vs 0.035 between the two on 512 calls).+ self.fixed_order = bool(fixed_order)self.model_id = model_id or get_settings().default_judgeself.n_samples = max(1, int(n_samples))if self.n_samples % 2 == 0:@@ -139,7 +147,7 @@ class VlmJudge:errors: list[str] = []reqs: list[ChatRequest] = []for k in range(self.n_samples):- seed = None if (self.n_samples == 1 and k == 0) else (inp.round_index * 1000 + k)+ seed = None if (self.fixed_order or (self.n_samples == 1 and k == 0)) else (inp.round_index * 1000 + k)# a missing render is a pipeline bug, not a judge glitch → JudgeImageError propagatessystem, messages = build_judge_messages(inp, self.rubric, shuffle_seed=seed, geometry_views=geometry_views, max_montages=self.max_montages,diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.pyindex 8180a14f..26627f0c 100644--- a/harness/tests/judges/test_vlm_judge.py+++ b/harness/tests/judges/test_vlm_judge.py@@ -60,6 +60,19 @@ def test_n_samples_mean_std_and_shuffle(judge_input, cache_dir):assert j.summary.startswith("A recognisable chair.")+def test_fixed_order_sends_the_identical_prompt_to_every_sample(judge_input, cache_dir):+ """The σ the loop keys its stop thresholds to must be the model's re-judge noise; the+ default per-sample shuffle measures view-order robustness instead (JUDGE_NOISE was+ tabulated from that). fixed_order is the mode the calibration battery uses."""+ model = FakeChatModel(by_label={":s0": [good_reply(R, IDS, 0.7)], ":s1": [good_reply(R, IDS, 0.9)],+ ":s2": [good_reply(R, IDS, 0.8)]})+ j = _judge(model, n_samples=3, cache_dir=cache_dir, fixed_order=True).judge(judge_input)+ assert j.n_samples == 3+ orders = [[p.label for p in image_parts(r)] for r in model.requests]+ assert len({tuple(o) for o in orders}) == 1, "every sample must see the montages in the same order"+ assert len({r.system for r in model.requests}) == 1++def test_floor_fails_even_if_mean_high(judge_input, cache_dir):model = FakeChatModel([good_reply(R, IDS, 0.95, overrides={"intent_fidelity": 0.2})])j = _judge(model, cache_dir=cache_dir).judge(judge_input)