Official Codebase of 3DCodeVerse
Python
bench: ab_gate_rates.py — the same A/B run's deterministic readouts, paired per prompt, because the judged score is not the thing a contact skill targets
gaoypeng528269a79011
diff --git a/harness/bench/ab_gate_rates.py b/harness/bench/ab_gate_rates.pynew file mode 100644index 00000000..33fbab9a--- /dev/null+++ b/harness/bench/ab_gate_rates.py@@ -0,0 +1,77 @@+"""Paired, per-prompt DETERMINISTIC readouts of an ab_plan run: what the gates measured.++The judged score carries the planner's variance (docs/EVAL.md 8.1); these do not carry the+judge's. They are what cv3d-part-contact and cv3d-bbox-contract are FOR, so they are the+primary readout and the score is the second.+"""+from __future__ import annotations++import json+import re+import sys+from pathlib import Path++if len(sys.argv) < 2:+ raise SystemExit("usage: python bench/ab_gate_rates.py bench/out/<ab-run>")+OUT = Path(sys.argv[1])+DEPTH = re.compile(r"interpenetrate by ≈([\d.]+) mm")++def cell_stats(record: Path) -> dict | None:+ rec = json.loads(record.read_text())+ rounds = rec.get("rounds") or []+ if not rounds:+ return None+ last = rounds[-1]+ pairs = floats = islands = contract_n = 0+ worst = 0.0+ for g in last.get("gates") or []:+ for f in g.get("findings") or []:+ sev, msg = str(f.get("severity", "")), str(f.get("message", ""))+ if sev == "info":+ continue+ if g.get("gate") == "connectivity":+ m = DEPTH.search(msg)+ if m:+ pairs += 1+ worst = max(worst, float(m.group(1)))+ elif "is floating" in msg:+ floats += 1+ elif "tiny disconnected island" in msg:+ islands += 1+ elif g.get("gate") == "contract":+ contract_n += 1+ return {"pairs": pairs, "worst_depth_mm": worst, "floating": floats,+ "islands": islands, "contract_findings": contract_n,+ "score": rec.get("final_score"), "status": rec.get("status")}++rows: dict[str, dict[str, dict]] = {}+for arm in ("control", "variant"):+ for rec in sorted((OUT / "arms" / arm / "cells").rglob("record.json")):+ prompt = rec.relative_to(OUT / "arms" / arm / "cells").parts[0]+ s = cell_stats(rec)+ if s:+ rows.setdefault(prompt, {})[arm] = s++keys = ("pairs", "worst_depth_mm", "floating", "islands", "contract_findings")+paired = {k: [] for k in keys}+print(f"{'prompt':<28} {'pairs c/v':>10} {'worst mm c/v':>14} {'float c/v':>10} {'islands c/v':>12} {'contract c/v':>13}")+for prompt, arms in sorted(rows.items()):+ if len(arms) != 2:+ print(f"{prompt:<28} unpaired ({sorted(arms)})")+ continue+ c, v = arms["control"], arms["variant"]+ for k in keys:+ paired[k].append(v[k] - c[k])+ print(f"{prompt:<28} {c['pairs']:>4}/{v['pairs']:<5} {c['worst_depth_mm']:>6.1f}/{v['worst_depth_mm']:<7.1f}"+ f" {c['floating']:>4}/{v['floating']:<5} {c['islands']:>5}/{v['islands']:<6} {c['contract_findings']:>6}/{v['contract_findings']:<6}")+print()+n = len(paired["pairs"])+print(f"paired prompts with a gate report on both sides: {n}")+for k in keys:+ d = paired[k]+ if not d:+ continue+ mean = sum(d) / len(d)+ down = sum(1 for x in d if x < 0)+ up = sum(1 for x in d if x > 0)+ print(f" {k:<20} mean delta (variant - control) {mean:+.2f} better on {down}, worse on {up}, tied {len(d)-down-up}")diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex f05f6d85..49cf240f 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -174,6 +174,8 @@ bench/ run_bench.py, report.py (reuses flywheel gallery), compare*.jsonl journals — a truncated last line never costs the paid rows),ab_plan.py (paired control/variant A/B for plan + brief switches; pinsboth children to the cap the §23 admission check reserved), _ab_report.py,+ ab_gate_rates.py (the same run's DETERMINISTIC readouts, paired per+ prompt: penetrating pairs, worst depth, floating parts, contract findings),_oneshot.py, _fixed_eval.py, cost_report.py,concurrency_probe.py (in-flight knee sweep), complexity_report.py,prompts/{static_objects_v1 (24), articulated_v1 (12), scenes_v1 (12), compare_v1 (8)}.yaml