Official Codebase of 3DCodeVerse
Python
eval tests: delete duplicates and argparse checks; fold twins into parametrize and the corpus contract
gaoypenged00a1b5cc7d
diff --git a/eval/tests/test_ab_plan.py b/eval/tests/test_ab_plan.pyindex fe3c8a47..42a42806 100644--- a/eval/tests/test_ab_plan.py+++ b/eval/tests/test_ab_plan.py@@ -112,7 +112,8 @@ def test_outage_cells_are_excluded_from_every_arm_rate():by = {s.arm: s for s in arm_stats(clear + unlucky)}c, v = by[CONTROL], by[VARIANT]assert c.mean_score == v.mean_score and c.mean_minutes == v.mean_minutes and c.mean_gen_usd == v.mean_gen_usd- assert v.infra_failed == 1 and v.n == 5 and v.n_scored == 4+ assert c.build_ok_rate == v.build_ok_rate+ assert v.infra_failed == 1 and v.n == 5 and v.n_scored == 4 and v.n_evaluated == 4md = render_summary(pair_up(clear + unlucky), clear + unlucky, title="t", variant_env={"X": "1"},generator="g", judge="j", rounds=2)assert "n_infra_failed: 1" in md and "--redo-status infra_failed" in md and "variant env: `X=1`" in md@@ -415,11 +416,3 @@ def test_a_pinned_plan_that_dies_in_a_storm_records_the_pair_and_continues(tmp_passert {r.arm for r in first} == {CONTROL, VARIANT} and all(r.status == "infra_failed" for r in first)assert all("pinned plan" in r.error for r in first)assert [c[0] for c in fake.calls] == [ids[1], ids[1]], "no cell was spent on the dead pair; the next prompt ran"---def test_max_usd_flag_was_deleted(capsys):- """The money ceiling left the harness on 2026-08-28: the flag must be rejected,- not silently parsed into nothing."""- with pytest.raises(SystemExit):- main(["--prompts", "p.yaml", "--out", "o", "--max-usd=2.5"])- assert "--max-usd" in capsys.readouterr().errdiff --git a/eval/tests/test_articulated_compare.py b/eval/tests/test_articulated_compare.pyindex 00a30230..3996c970 100644--- a/eval/tests/test_articulated_compare.py+++ b/eval/tests/test_articulated_compare.py@@ -110,7 +110,6 @@ def test_rubric_for_is_the_one_track_to_rubric_mapping():import bench._compare_report as reportfrom codeverse3d.contracts.common import TRACK_INFO- assert not hasattr(report, "RUBRIC_BY_TRACK")lang = {Track.STATIC_OBJECT: Language.BLENDER, Track.ARTICULATED_OBJECT: Language.URDF_BLENDER,Track.SCENE: Language.SCENE_THREEJS, Track.GRAPHICS: Language.GLSL_SHADER}for t in Track:diff --git a/eval/tests/test_bench.py b/eval/tests/test_bench.pyindex 69bda67b..260cd903 100644--- a/eval/tests/test_bench.py+++ b/eval/tests/test_bench.py@@ -20,7 +20,7 @@ BATTERIES = sorted((REPO / "bench" / "prompts").glob("*.yaml"))def test_all_batteries_are_valid():"""One corpus contract is clearer than 27 identical pytest cases."""assert BATTERIES, "no checked-in benchmark batteries found"- expected = {"static_objects_v1": 24, "articulated_v1": 12, "scenes_v1": 12}+ expected = {"static_objects_v1": 24, "articulated_v1": 12, "scenes_v1": 12, "compare_v1": 8}for path in BATTERIES:b = Battery.load(path)ids = [p.id for p in b.prompts]@@ -191,9 +191,6 @@ def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):assert calls == [], "a plain resume still skips every recorded row"run_battery(battery, out, BenchOptions(parallel=1, limit=1, redo_status=["infra_failed"]), run_fn=counting)assert len(calls) == 1, "--redo-status infra_failed re-runs what the weather lost"- # ...and it is the ONLY way in. A `resume` switch that merely dropped the recorded rows- # skipped the archive below and silently resumed the old workspace, spec and clock.- assert "resume" not in BenchOptions.model_fieldsdef test_a_redo_starts_from_a_fresh_workspace(tmp_path):diff --git a/eval/tests/test_compare_matrix.py b/eval/tests/test_compare_matrix.pyindex e077276a..204e69df 100644--- a/eval/tests/test_compare_matrix.py+++ b/eval/tests/test_compare_matrix.py@@ -47,9 +47,7 @@ ARMS = "harness:gemini-cli:gemini-3.6-flash,oneshot:claude-code,oneshot+repair:cdef test_battery_compare_v1_is_valid():b = Battery.load(BATTERY)assert b.name == "compare_v1" and b.track.value == "static_object" and b.language.value == "blender"- assert len(b.prompts) == 8 and len({p.id for p in b.prompts}) == 8- assert all(len(p.must_have) >= 3 for p in b.prompts)- assert {p.tier for p in b.prompts} == {"easy", "medium", "hard"}+ assert all(len(p.must_have) >= 3 for p in b.prompts) # ids, tiers, count: test_bench's corpus contractdef test_parse_arms():@@ -259,15 +257,6 @@ def test_every_cell_and_its_harness_run_open_a_ledger(tmp_path: Path):assert [r.label for r in load_ledger(oneshot)] == ["judge:static_object_v1:r00:s0"]-def test_max_usd_flag_was_deleted(capsys):- """The money ceiling left the harness on 2026-08-28: the flag must be rejected,- not silently parsed into nothing."""- with pytest.raises(SystemExit):- main(["--prompts", "x.yaml", "--arms", "harness:gemini-cli:m", "--out", "o",- "--max-usd", "2.5"])- assert "--max-usd" in capsys.readouterr().err--def test_no_resume_starts_the_harness_arm_fresh(tmp_path: Path):"""`--no-resume` regenerates every other arm, so the harness arm must not resume.diff --git a/eval/tests/test_complexity_report.py b/eval/tests/test_complexity_report.pyindex 4882eb2f..ecdb1daa 100644--- a/eval/tests/test_complexity_report.py+++ b/eval/tests/test_complexity_report.py@@ -62,7 +62,6 @@ def test_collect_and_report_over_a_battery(tmp_path: Path) -> None:rows, skipped = CR.collect([battery])assert skipped == 0 and len(rows) == 3assert {r["slug"]: r["minutes"] for r in rows} == {"low": 5, "mid": 20, "high": 40}- assert {r["slug"] for r in rows} == {"low", "mid", "high"}assert rows[0]["plan_parts"] == 2 and rows[0]["n_materials"] == 3# the whole point of the study: a monotone fall shows up as a strong negative rxs, ys = CR._pairs(rows, "index", "overall")diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.pyindex 4fb63373..6f1d5a08 100644--- a/eval/tests/test_infra_failures.py+++ b/eval/tests/test_infra_failures.py@@ -19,6 +19,9 @@ OUTAGES = ["ModelError: Gemini API error 503: This model is currently experiencing high demand.","ModelError: Gemini request timed out: The read operation timed out","ModelError: Gemini API error 529: is overloaded",+ "Gemini API error 504: Deadline expired before operation could complete.",+ "ModelError: Gemini stream exceeded its attempt budget after 293 chunks",+ "ModelError: structured output unavailable (finish_reason=PROHIBITED_CONTENT; raise max_output_tokens if truncated)","KeyPoolExhausted: every key is cooling down","APIError: 500 Internal server error","ConnectionError: Connection reset by peer",@@ -78,21 +81,6 @@ def test_structured_exceptions_beat_string_matching():assert is_infra_failure(capability_failure) is False-def test_outage_cells_leave_every_rate_alone():- """The bug in one assertion: an arm hit by downtime must score the same as one- that ran in the clear, and the loss must be reported rather than hidden."""- clear = [CellResult(prompt_id=f"p{i}", arm="A", kind="oneshot", status="scored",- score=0.8, passed=True, build_ok=True, wall_s=60.0) for i in range(4)]- unlucky = [*[c.model_copy(update={"arm": "B"}) for c in clear],- CellResult(prompt_id="p9", arm="B", kind="oneshot", status="infra_failed",- score=None, passed=None, build_ok=False, wall_s=3600.0)]- a, b = {s.arm: s for s in arm_stats([*clear, *unlucky])}["A"], {s.arm: s for s in arm_stats([*clear, *unlucky])}["B"]- assert a.mean_score == b.mean_score, "downtime changed the score"- assert a.build_ok_rate == b.build_ok_rate, "downtime changed the build rate"- assert a.mean_minutes == b.mean_minutes, "an hour spent retrying a 503 is not model latency"- assert b.infra_failed == 1 and b.n_evaluated == 4 and b.n == 5, "the loss must stay visible"--def test_both_failure_paths_classify_the_same_way(tmp_path):"""CQ-1, the asymmetry itself: ONE error hits both arms, both must reach one verdict.@@ -147,7 +135,6 @@ def test_both_failure_paths_classify_the_same_way(tmp_path):def test_budget_exhaustion_is_scoreless_but_still_counts_against_build_rate():"""The 50-minutes-for-nothing case: no score to average, but the arm did miss."""- from bench._compare_report import CellResult, arm_statsrows = [CellResult(prompt_id="p1", arm="A", kind="harness", status="scored",score=0.9, passed=True, build_ok=True),@@ -300,15 +287,6 @@ def test_a_classifier_crash_still_records_the_cell(tmp_path):assert (Path(r.workspace) / "cell.json").is_file()--def test_stream_attempt_budget_and_504_are_infra():- from bench._infra import is_infra_failure-- assert is_infra_failure(RuntimeError("ModelError: Gemini stream exceeded its attempt budget after 293 chunks"))- assert is_infra_failure(RuntimeError("Gemini API error 504: Deadline expired before operation could complete."))- assert is_infra_failure(RuntimeError("ModelError: structured output unavailable (finish_reason=PROHIBITED_CONTENT; raise max_output_tokens if truncated)"))--def test_the_ab_viewer_refuses_to_call_a_winner_it_cannot_support():"""B − A per brief — the pairs the page shows — and the 95 % t-interval of its mean."""from bench.ab_view import Run, verdictdiff --git a/eval/tests/test_session_stats.py b/eval/tests/test_session_stats.pyindex a0f5a7f1..c50bfa95 100644--- a/eval/tests/test_session_stats.py+++ b/eval/tests/test_session_stats.py@@ -271,13 +271,3 @@ def test_the_depth_comparison_is_restricted_to_one_pose() -> None:assert s["sweep_findings"] == 2 and s["sweep_distinct"] == 2assert [(p, sw, cn) for _, p, sw, cn in s["both"]] == [("c|d", 0.0044, 0.0044)], \"only the rest-worst pair is comparable, and there the two probes agree"---def test_a_finding_whose_message_changes_still_counts() -> None:- """The whole point of reading `target` / `data`: a wording change in- `aggregate_findings` used to silently zero this report."""- from bench.penetration_thresholds import sweep_pairs-- f = _sweep("x|y", 0.01, poses=[{}], worst_pose={})- f["message"] = "completely different wording"- assert sweep_pairs({"gate": "joint_sweep", "findings": [f]}, rest_only=False) == {("x", "y"): 0.01}diff --git a/eval/tests/test_v2_batteries.py b/eval/tests/test_v2_batteries.pyindex 8e21ca42..db7fb05d 100644--- a/eval/tests/test_v2_batteries.py+++ b/eval/tests/test_v2_batteries.py@@ -153,10 +153,10 @@ def test_graphics_v2_opengl_prompts_carry_language_override():assert spec.language == want, f"{prompt.id}: spec language {spec.language} != {want}"[email protected]("name", ["compare_v2", "compare_v3"])-def test_compare_is_a_subset_of_static_v2(name: str):+def test_compare_is_a_subset_of_static_v2():+ """compare_v3 holds compare_v2 verbatim (the test below), so v3 covers both."""static = {p.id.split("_", 1)[1]: p for p in _load("static_objects_v2").prompts}- for p in _load(name).prompts:+ for p in _load("compare_v3").prompts:key = p.id.split("_", 1)[1] # cmp_hard_violin -> hard_violin == prefix-stripped static keysrc = static.get(key)assert src is not None, f"{p.id}: no matching static_objects_v2 prompt"