Official Codebase of 3DCodeVerse
Python
bench: the p-value D52 cites is computed in the repo, from the rows in the repo
ziyao2bf09afa1cb7
diff --git a/harness/bench/data/plan_stage/trigger_narrow.jsonl b/harness/bench/data/plan_stage/trigger_narrow.jsonlindex ae4af1bb..a23c8932 100644--- a/harness/bench/data/plan_stage/trigger_narrow.jsonl+++ b/harness/bench/data/plan_stage/trigger_narrow.jsonl@@ -34,3 +34,44 @@{"tree": "trigger_narrow", "prompt": "art_med_tool_chest", "rep": 13, "ok": true, "error": "", "seconds": 27.5, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0218, "n_parts": 8, "attempt": 0}{"tree": "trigger_narrow", "prompt": "art_med_tool_chest", "rep": 14, "ok": true, "error": "", "seconds": 34.6, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0236, "n_parts": 9, "attempt": 0}{"tree": "trigger_narrow", "prompt": "art_med_tool_chest", "rep": 15, "ok": true, "error": "", "seconds": 27.1, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0277, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_tool_chest", "rep": 16, "ok": true, "error": "", "seconds": 236.8, "invalid_reasks": 1, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0694, "n_parts": 14, "attempt": 3}+{"tree": "trigger_narrow", "prompt": "art_med_tool_chest", "rep": 17, "ok": true, "error": "", "seconds": 27.6, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0339, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_tool_chest", "rep": 18, "ok": true, "error": "", "seconds": 28.4, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0315, "n_parts": 8, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_tool_chest", "rep": 19, "ok": true, "error": "", "seconds": 56.3, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0248, "n_parts": 10, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 0, "ok": true, "error": "", "seconds": 45.3, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0348, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 1, "ok": true, "error": "", "seconds": 31.7, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0244, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 2, "ok": true, "error": "", "seconds": 45.8, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0448, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 3, "ok": true, "error": "", "seconds": 75.2, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0419, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 4, "ok": true, "error": "", "seconds": 46.1, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0327, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 5, "ok": true, "error": "", "seconds": 33.4, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0287, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 6, "ok": true, "error": "", "seconds": 32.3, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0381, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 7, "ok": true, "error": "", "seconds": 31.1, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0386, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 8, "ok": true, "error": "", "seconds": 24.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0246, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 9, "ok": true, "error": "", "seconds": 38.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.027, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 10, "ok": true, "error": "", "seconds": 51.9, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0472, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 11, "ok": true, "error": "", "seconds": 19.3, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0181, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 12, "ok": true, "error": "", "seconds": 39.9, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0458, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 13, "ok": true, "error": "", "seconds": 29.5, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0349, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 14, "ok": true, "error": "", "seconds": 39.7, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.027, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 15, "ok": true, "error": "", "seconds": 26.6, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.036, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 16, "ok": true, "error": "", "seconds": 146.9, "invalid_reasks": 1, "geometry_reasks": 0, "restarts": 1, "missing": ["UpperLeg1"], "cost_usd": 0.0613, "n_parts": 9, "attempt": 1}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 17, "ok": true, "error": "", "seconds": 19.6, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0267, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 18, "ok": true, "error": "", "seconds": 31.4, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0266, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_camera_tripod", "rep": 19, "ok": true, "error": "", "seconds": 19.2, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0273, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 0, "ok": true, "error": "", "seconds": 106.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0296, "n_parts": 8, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 1, "ok": true, "error": "", "seconds": 60.9, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0259, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 2, "ok": true, "error": "", "seconds": 139.1, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0334, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 3, "ok": true, "error": "", "seconds": 118.0, "invalid_reasks": 1, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0529, "n_parts": 9, "attempt": 1}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 4, "ok": true, "error": "", "seconds": 26.1, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0333, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 5, "ok": true, "error": "", "seconds": 121.5, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0278, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 6, "ok": true, "error": "", "seconds": 36.8, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0326, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 7, "ok": true, "error": "", "seconds": 82.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0278, "n_parts": 8, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 8, "ok": true, "error": "", "seconds": 107.5, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0333, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 9, "ok": true, "error": "", "seconds": 62.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0453, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 10, "ok": true, "error": "", "seconds": 32.2, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0333, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 11, "ok": true, "error": "", "seconds": 66.1, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0317, "n_parts": 8, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 12, "ok": true, "error": "", "seconds": 33.4, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0374, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 13, "ok": true, "error": "", "seconds": 53.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0413, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 14, "ok": true, "error": "", "seconds": 70.7, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0314, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 15, "ok": true, "error": "", "seconds": 112.5, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0431, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_med_dutch_door", "rep": 16, "ok": true, "error": "", "seconds": 129.9, "invalid_reasks": 2, "geometry_reasks": 0, "restarts": 1, "missing": ["LowerLeaf", "UpperLeaf", "SlideBolt", "SuffolkLatch", "LowerStrapHingeBottom", "LowerStrapHingeTop", "UpperStrapHingeBottom", "UpperStrapHingeTop"], "cost_usd": 0.1213, "n_parts": 9, "attempt": 2}diff --git a/harness/bench/plan_stage_report.py b/harness/bench/plan_stage_report.pyindex 02a52c44..44300404 100644--- a/harness/bench/plan_stage_report.py+++ b/harness/bench/plan_stage_report.py@@ -33,6 +33,25 @@ def wilson(hits: int, n: int, z: float = 1.96) -> tuple[float, float]:return (max(0.0, centre - half), min(1.0, centre + half))+def fisher_exact(a: int, b: int, c: int, d: int) -> float:+ """Two-sided Fisher exact p for the 2x2 table ((a, b), (c, d)), no scipy.++ The failure counts here are single digits out of a few hundred, where the normal+ approximation is not usable and the exact sum is cheap: every table with the same+ margins, keeping those no more likely than the observed one. The paper cites this+ number, so it is computed in the repo from the rows in the repo.+ """+ n = a + b + c + d+ row1, col1 = a + b, a + c+ lo, hi = max(0, col1 - (n - row1)), min(row1, col1)++ def prob(x: int) -> float:+ return (math.comb(row1, x) * math.comb(n - row1, col1 - x)) / math.comb(n, col1)++ observed = prob(a)+ return min(1.0, sum(prob(x) for x in range(lo, hi + 1) if prob(x) <= observed * (1 + 1e-9)))++def load(path: Path) -> list[dict]:return [json.loads(x) for x in path.read_text().splitlines() if x.strip()] if path.is_file() else []@@ -64,6 +83,14 @@ def report(arms: dict[str, list[dict]]) -> str:failures[f"{name}: {r['error'].split('Value error, ')[-1][:90]}"] += 1if failures:lines += ["", "validation failures:"] + [f" {n}x {msg}" for msg, n in failures.most_common()]+ if len(arms) == 2:+ (n1, rows1), (n2, rows2) = arms.items()+ c1, c2 = Counter(outcome(r) for r in rows1), Counter(outcome(r) for r in rows2)+ j1, j2 = len(rows1) - c1["provider"], len(rows2) - c2["provider"]+ p = fisher_exact(c1["planning_error"], j1 - c1["planning_error"],+ c2["planning_error"], j2 - c2["planning_error"])+ lines += ["", f"Fisher exact two-sided p = {p:.4f} ({n1} {c1['planning_error']}/{j1} vs "+ f"{n2} {c2['planning_error']}/{j2})"]fired = {name: [r for r in rows if r.get("restarts")] for name, rows in arms.items()}if any(fired.values()):lines += ["", "restarts:"] + [diff --git a/harness/tests/compare_bench/test_plan_stage.py b/harness/tests/compare_bench/test_plan_stage.pynew file mode 100644index 00000000..cf8c5913--- /dev/null+++ b/harness/tests/compare_bench/test_plan_stage.py@@ -0,0 +1,38 @@+"""The loss-event channel's arithmetic: the p-value a paper cites is computed here.++``bench/data/plan_stage/*.jsonl`` carries the 560 recorded plan calls behind D52; this+pins the statistic that turns them into 0.0067, and the report that prints it."""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from bench.plan_stage_report import fisher_exact, report++DATA = Path(__file__).resolve().parents[2] / "bench" / "data" / "plan_stage"+++def test_fisher_exact_reproduces_the_number_d52_cites() -> None:+ assert round(fisher_exact(13, 264, 2, 274), 4) == 0.0067++[email protected]("table,expected", [+ ((0, 100, 0, 100), 1.0), # nothing happened in either arm+ ((5, 95, 5, 95), 1.0), # identical arms+ ((10, 90, 0, 100), 0.00154), # a difference this size at n=100 is not noise+ ((1, 9, 8, 2), 0.0055), # the textbook 2x2, small and lopsided+])+def test_fisher_exact_against_known_tables(table, expected) -> None:+ assert fisher_exact(*table) == pytest.approx(expected, abs=5e-5)+++def test_the_recorded_arms_still_read_out_the_way_d52_says() -> None:+ arms = {p.stem: [json.loads(x) for x in p.read_text().splitlines() if x.strip()]+ for p in sorted(DATA.glob("restart_o*.jsonl"))}+ assert set(arms) == {"restart_off", "restart_on"} and all(len(v) == 280 for v in arms.values())+ text = report(arms)+ assert "Fisher exact two-sided p = 0.0067" in text+ assert "fired on 40/280 calls, recovered 39" in text