Skip to content

Official Codebase of 3DCodeVerse

Python

eval tests: delete duplicates and argparse checks; fold twins into parametrize and the corpus contract

gaoypenged00a1b5cc7d
diff --git a/eval/tests/test_ab_plan.py b/eval/tests/test_ab_plan.py
index fe3c8a47..42a42806 100644
--- a/eval/tests/test_ab_plan.py
+++ b/eval/tests/test_ab_plan.py
@@ -112,7 +112,8 @@ def test_outage_cells_are_excluded_from_every_arm_rate():
by = {s.arm: s for s in arm_stats(clear + unlucky)}
c, v = by[CONTROL], by[VARIANT]
assert c.mean_score == v.mean_score and c.mean_minutes == v.mean_minutes and c.mean_gen_usd == v.mean_gen_usd
- assert v.infra_failed == 1 and v.n == 5 and v.n_scored == 4
+ assert c.build_ok_rate == v.build_ok_rate
+ assert v.infra_failed == 1 and v.n == 5 and v.n_scored == 4 and v.n_evaluated == 4
md = render_summary(pair_up(clear + unlucky), clear + unlucky, title="t", variant_env={"X": "1"},
generator="g", judge="j", rounds=2)
assert "n_infra_failed: 1" in md and "--redo-status infra_failed" in md and "variant env: `X=1`" in md
@@ -415,11 +416,3 @@ def test_a_pinned_plan_that_dies_in_a_storm_records_the_pair_and_continues(tmp_p
assert {r.arm for r in first} == {CONTROL, VARIANT} and all(r.status == "infra_failed" for r in first)
assert all("pinned plan" in r.error for r in first)
assert [c[0] for c in fake.calls] == [ids[1], ids[1]], "no cell was spent on the dead pair; the next prompt ran"
-
-
-def test_max_usd_flag_was_deleted(capsys):
- """The money ceiling left the harness on 2026-08-28: the flag must be rejected,
- not silently parsed into nothing."""
- with pytest.raises(SystemExit):
- main(["--prompts", "p.yaml", "--out", "o", "--max-usd=2.5"])
- assert "--max-usd" in capsys.readouterr().err
diff --git a/eval/tests/test_articulated_compare.py b/eval/tests/test_articulated_compare.py
index 00a30230..3996c970 100644
--- a/eval/tests/test_articulated_compare.py
+++ b/eval/tests/test_articulated_compare.py
@@ -110,7 +110,6 @@ def test_rubric_for_is_the_one_track_to_rubric_mapping():
import bench._compare_report as report
from codeverse3d.contracts.common import TRACK_INFO
- assert not hasattr(report, "RUBRIC_BY_TRACK")
lang = {Track.STATIC_OBJECT: Language.BLENDER, Track.ARTICULATED_OBJECT: Language.URDF_BLENDER,
Track.SCENE: Language.SCENE_THREEJS, Track.GRAPHICS: Language.GLSL_SHADER}
for t in Track:
diff --git a/eval/tests/test_bench.py b/eval/tests/test_bench.py
index 69bda67b..260cd903 100644
--- a/eval/tests/test_bench.py
+++ b/eval/tests/test_bench.py
@@ -20,7 +20,7 @@ BATTERIES = sorted((REPO / "bench" / "prompts").glob("*.yaml"))
def test_all_batteries_are_valid():
"""One corpus contract is clearer than 27 identical pytest cases."""
assert BATTERIES, "no checked-in benchmark batteries found"
- expected = {"static_objects_v1": 24, "articulated_v1": 12, "scenes_v1": 12}
+ expected = {"static_objects_v1": 24, "articulated_v1": 12, "scenes_v1": 12, "compare_v1": 8}
for path in BATTERIES:
b = Battery.load(path)
ids = [p.id for p in b.prompts]
@@ -191,9 +191,6 @@ def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):
assert calls == [], "a plain resume still skips every recorded row"
run_battery(battery, out, BenchOptions(parallel=1, limit=1, redo_status=["infra_failed"]), run_fn=counting)
assert len(calls) == 1, "--redo-status infra_failed re-runs what the weather lost"
- # ...and it is the ONLY way in. A `resume` switch that merely dropped the recorded rows
- # skipped the archive below and silently resumed the old workspace, spec and clock.
- assert "resume" not in BenchOptions.model_fields
def test_a_redo_starts_from_a_fresh_workspace(tmp_path):
diff --git a/eval/tests/test_compare_matrix.py b/eval/tests/test_compare_matrix.py
index e077276a..204e69df 100644
--- a/eval/tests/test_compare_matrix.py
+++ b/eval/tests/test_compare_matrix.py
@@ -47,9 +47,7 @@ ARMS = "harness:gemini-cli:gemini-3.6-flash,oneshot:claude-code,oneshot+repair:c
def test_battery_compare_v1_is_valid():
b = Battery.load(BATTERY)
assert b.name == "compare_v1" and b.track.value == "static_object" and b.language.value == "blender"
- assert len(b.prompts) == 8 and len({p.id for p in b.prompts}) == 8
- assert all(len(p.must_have) >= 3 for p in b.prompts)
- assert {p.tier for p in b.prompts} == {"easy", "medium", "hard"}
+ assert all(len(p.must_have) >= 3 for p in b.prompts) # ids, tiers, count: test_bench's corpus contract
def test_parse_arms():
@@ -259,15 +257,6 @@ def test_every_cell_and_its_harness_run_open_a_ledger(tmp_path: Path):
assert [r.label for r in load_ledger(oneshot)] == ["judge:static_object_v1:r00:s0"]
-def test_max_usd_flag_was_deleted(capsys):
- """The money ceiling left the harness on 2026-08-28: the flag must be rejected,
- not silently parsed into nothing."""
- with pytest.raises(SystemExit):
- main(["--prompts", "x.yaml", "--arms", "harness:gemini-cli:m", "--out", "o",
- "--max-usd", "2.5"])
- assert "--max-usd" in capsys.readouterr().err
-
-
def test_no_resume_starts_the_harness_arm_fresh(tmp_path: Path):
"""`--no-resume` regenerates every other arm, so the harness arm must not resume.
diff --git a/eval/tests/test_complexity_report.py b/eval/tests/test_complexity_report.py
index 4882eb2f..ecdb1daa 100644
--- a/eval/tests/test_complexity_report.py
+++ b/eval/tests/test_complexity_report.py
@@ -62,7 +62,6 @@ def test_collect_and_report_over_a_battery(tmp_path: Path) -> None:
rows, skipped = CR.collect([battery])
assert skipped == 0 and len(rows) == 3
assert {r["slug"]: r["minutes"] for r in rows} == {"low": 5, "mid": 20, "high": 40}
- assert {r["slug"] for r in rows} == {"low", "mid", "high"}
assert rows[0]["plan_parts"] == 2 and rows[0]["n_materials"] == 3
# the whole point of the study: a monotone fall shows up as a strong negative r
xs, ys = CR._pairs(rows, "index", "overall")
diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.py
index 4fb63373..6f1d5a08 100644
--- a/eval/tests/test_infra_failures.py
+++ b/eval/tests/test_infra_failures.py
@@ -19,6 +19,9 @@ OUTAGES = [
"ModelError: Gemini API error 503: This model is currently experiencing high demand.",
"ModelError: Gemini request timed out: The read operation timed out",
"ModelError: Gemini API error 529: is overloaded",
+ "Gemini API error 504: Deadline expired before operation could complete.",
+ "ModelError: Gemini stream exceeded its attempt budget after 293 chunks",
+ "ModelError: structured output unavailable (finish_reason=PROHIBITED_CONTENT; raise max_output_tokens if truncated)",
"KeyPoolExhausted: every key is cooling down",
"APIError: 500 Internal server error",
"ConnectionError: Connection reset by peer",
@@ -78,21 +81,6 @@ def test_structured_exceptions_beat_string_matching():
assert is_infra_failure(capability_failure) is False
-def test_outage_cells_leave_every_rate_alone():
- """The bug in one assertion: an arm hit by downtime must score the same as one
- that ran in the clear, and the loss must be reported rather than hidden."""
- clear = [CellResult(prompt_id=f"p{i}", arm="A", kind="oneshot", status="scored",
- score=0.8, passed=True, build_ok=True, wall_s=60.0) for i in range(4)]
- unlucky = [*[c.model_copy(update={"arm": "B"}) for c in clear],
- CellResult(prompt_id="p9", arm="B", kind="oneshot", status="infra_failed",
- score=None, passed=None, build_ok=False, wall_s=3600.0)]
- a, b = {s.arm: s for s in arm_stats([*clear, *unlucky])}["A"], {s.arm: s for s in arm_stats([*clear, *unlucky])}["B"]
- assert a.mean_score == b.mean_score, "downtime changed the score"
- assert a.build_ok_rate == b.build_ok_rate, "downtime changed the build rate"
- assert a.mean_minutes == b.mean_minutes, "an hour spent retrying a 503 is not model latency"
- assert b.infra_failed == 1 and b.n_evaluated == 4 and b.n == 5, "the loss must stay visible"
-
-
def test_both_failure_paths_classify_the_same_way(tmp_path):
"""CQ-1, the asymmetry itself: ONE error hits both arms, both must reach one verdict.
@@ -147,7 +135,6 @@ def test_both_failure_paths_classify_the_same_way(tmp_path):
def test_budget_exhaustion_is_scoreless_but_still_counts_against_build_rate():
"""The 50-minutes-for-nothing case: no score to average, but the arm did miss."""
- from bench._compare_report import CellResult, arm_stats
rows = [CellResult(prompt_id="p1", arm="A", kind="harness", status="scored",
score=0.9, passed=True, build_ok=True),
@@ -300,15 +287,6 @@ def test_a_classifier_crash_still_records_the_cell(tmp_path):
assert (Path(r.workspace) / "cell.json").is_file()
-
-def test_stream_attempt_budget_and_504_are_infra():
- from bench._infra import is_infra_failure
-
- assert is_infra_failure(RuntimeError("ModelError: Gemini stream exceeded its attempt budget after 293 chunks"))
- assert is_infra_failure(RuntimeError("Gemini API error 504: Deadline expired before operation could complete."))
- assert is_infra_failure(RuntimeError("ModelError: structured output unavailable (finish_reason=PROHIBITED_CONTENT; raise max_output_tokens if truncated)"))
-
-
def test_the_ab_viewer_refuses_to_call_a_winner_it_cannot_support():
"""B − A per brief — the pairs the page shows — and the 95 % t-interval of its mean."""
from bench.ab_view import Run, verdict
diff --git a/eval/tests/test_session_stats.py b/eval/tests/test_session_stats.py
index a0f5a7f1..c50bfa95 100644
--- a/eval/tests/test_session_stats.py
+++ b/eval/tests/test_session_stats.py
@@ -271,13 +271,3 @@ def test_the_depth_comparison_is_restricted_to_one_pose() -> None:
assert s["sweep_findings"] == 2 and s["sweep_distinct"] == 2
assert [(p, sw, cn) for _, p, sw, cn in s["both"]] == [("c|d", 0.0044, 0.0044)], \
"only the rest-worst pair is comparable, and there the two probes agree"
-
-
-def test_a_finding_whose_message_changes_still_counts() -> None:
- """The whole point of reading `target` / `data`: a wording change in
- `aggregate_findings` used to silently zero this report."""
- from bench.penetration_thresholds import sweep_pairs
-
- f = _sweep("x|y", 0.01, poses=[{}], worst_pose={})
- f["message"] = "completely different wording"
- assert sweep_pairs({"gate": "joint_sweep", "findings": [f]}, rest_only=False) == {("x", "y"): 0.01}
diff --git a/eval/tests/test_v2_batteries.py b/eval/tests/test_v2_batteries.py
index 8e21ca42..db7fb05d 100644
--- a/eval/tests/test_v2_batteries.py
+++ b/eval/tests/test_v2_batteries.py
@@ -153,10 +153,10 @@ def test_graphics_v2_opengl_prompts_carry_language_override():
assert spec.language == want, f"{prompt.id}: spec language {spec.language} != {want}"
[email protected]("name", ["compare_v2", "compare_v3"])
-def test_compare_is_a_subset_of_static_v2(name: str):
+def test_compare_is_a_subset_of_static_v2():
+ """compare_v3 holds compare_v2 verbatim (the test below), so v3 covers both."""
static = {p.id.split("_", 1)[1]: p for p in _load("static_objects_v2").prompts}
- for p in _load(name).prompts:
+ for p in _load("compare_v3").prompts:
key = p.id.split("_", 1)[1] # cmp_hard_violin -> hard_violin == prefix-stripped static key
src = static.get(key)
assert src is not None, f"{p.id}: no matching static_objects_v2 prompt"