Official Codebase of 3DCodeVerse
Python
bench: an outage is decided from the harness's typed transient/quota first; plan_stage_report and a judge-down run use the same rule
gaoypengb04d9d7de4a3
diff --git a/eval/bench/_infra.py b/eval/bench/_infra.pyindex c2d72a44..7a9d0328 100644--- a/eval/bench/_infra.py+++ b/eval/bench/_infra.py@@ -22,6 +22,9 @@ from __future__ import annotationsimport loggingfrom collections.abc import Iterator+from codeverse3d.contracts.run import RunStatus+from codeverse3d.models.parts import retryable_status+log = logging.getLogger(__name__)#: substrings that identify a provider-side outage in a *stringified* error. Kept@@ -36,6 +39,7 @@ INFRA_MARKERS: tuple[str, ...] = ("read operation timed out","deadline expired", # Gemini 504"finish_reason=prohibited_content", # the provider's content filter tripped mid-JSON on a furniture plan (2026-08-28): provider behaviour, redone+ "prompt blocked", # the same filter on the harness's own prompt (BlockedReason.OTHER): plan_stage_report's "provider" since D52"exceeded its attempt budget", # models/gemini.py streaming: the provider held the socket past the attempt budget (2026-08-28, pro planner)"deadline exceeded","connection reset",@@ -57,31 +61,39 @@ INFRA_MARKERS: tuple[str, ...] = ("error creating webgl context",)-#: HTTP statuses that mean "the provider could not serve this", not "the model was bad".-#: 408 is the adapters' own code for a request timeout (gemini.py:117, openai.py:40,-#: anthropic.py:43) — the exception path must agree with the "request timed out" marker-#: the string path already had.-INFRA_STATUSES: frozenset[int] = frozenset({408, 429, 500, 502, 503, 504, 529})--def _has_marker(text: str) -> bool:return any(marker in text.lower() for marker in INFRA_MARKERS)def _looks_infra(exc: BaseException) -> bool:- """The outage rules, applied to ONE exception. Structured first — a ``ModelError`` /- ``KeyPoolExhausted`` carries its status, and trusting that beats guessing from prose —- with the string markers as the fallback. ONE copy: this ran as two hand-kept copies- (head vs each wrapped cause) until 2026-08-28, which is how a rule reaches one path- and not the other."""+ """The outage rules, applied to ONE exception. Typed first: the harness's own verdict+ (``RoundFailed`` / ``GenerationResult`` ``.transient`` — a provider storm, a dropped socket+ — and ``.quota`` — the vendor's usage limit, "not ours") decided from the CLI's own record,+ which the bench must not re-guess from the message (N76: a gemini-cli 503 storm that killed a+ baseline twice matched no marker and was scored as an error). Then the HTTP status, from the+ harness's one table (``models.parts.retryable_status``: 408/409/429/5xx), and a spent key+ pool. The string markers are the fallback, for exceptions nobody typed and for the error+ TEXT of old records. ONE copy: this ran as two hand-kept copies (head vs each wrapped+ cause) until 2026-08-28, which is how a rule reaches one path and not the other."""+ if getattr(exc, "transient", False) is True or getattr(exc, "quota", False) is True:+ return Trueif type(exc).__name__ == "KeyPoolExhausted" or isinstance(exc, TimeoutError):return Truestatus = getattr(exc, "status", None)- if isinstance(status, int) and status in INFRA_STATUSES:+ if isinstance(status, int) and retryable_status(status):return Truereturn _has_marker(f"{type(exc).__name__}: {exc}")+def judge_outage(status: RunStatus, picked_score: float | None) -> bool:+ """A harness run that stopped because its in-loop judge never answered, with no round+ judged and picked before it: the judge's outage, not the arm's result — dropped as+ ``infra_failed`` and re-run fresh by ``--redo-status``. A run that had picked a round+ before the outage is scored on that pick (owner, 2026-09-23). THE rule for run_bench+ and compare_backends (N76)."""+ return status is RunStatus.JUDGE_UNAVAILABLE and picked_score is None++def _chain(err: BaseException, limit: int = 32) -> Iterator[BaseException]:"""``err`` and every exception it was raised from, each yielded once.diff --git a/eval/bench/ab_plan.py b/eval/bench/ab_plan.pyindex a895bf83..124fc654 100644--- a/eval/bench/ab_plan.py+++ b/eval/bench/ab_plan.py@@ -499,7 +499,7 @@ def run_ab(# pair is recorded the way a dead cell is (docs/EVAL.md §7 — an outage is# not a score) so `--redo-status infra_failed` picks it up, and the loop# goes on to the next prompt.- status = "infra_failed" if is_infra_failure(str(e)) else "error"+ status = "infra_failed" if is_infra_failure(e) else "error" # the exception: its typed flags and statuslog.warning("%s: pinned plan failed (%s) — pair recorded %s, continuing",item.id,diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex 0ebf58c2..d94ce3c4 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -58,7 +58,7 @@ from datetime import UTC # noqa: E402from bench._compare_report import CellResult, PairRow, build_compare_report # noqa: E402from bench._fixed_eval import EvalOutcome, FixedEvaluator, rubric_for # noqa: E402-from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402+from bench._infra import is_budget_exhaustion, is_infra_failure, judge_outage # noqa: E402from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402from bench._oneshot import ( # noqa: E402OneShotBackend,@@ -295,7 +295,7 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptionres.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), summary.picked_scoreres.harness_stop_reason = summary.stop_reasonres.harness_aborted_rounds = len(rec.extra.get("aborted_rounds") or [])- if rec.status is RunStatus.JUDGE_UNAVAILABLE and summary.picked_score is None:+ if judge_outage(rec.status, summary.picked_score):# The run stopped because its judge never answered (p3_graphics_v2 mushroom_forest, 2026-09-23:# two 900 s pro read timeouts after round 0). Scoring what it had then would grade a one-round# run for someone else's outage — the asymmetry §7 ends for the one-shot arms — so the cell isdiff --git a/eval/bench/plan_stage_report.py b/eval/bench/plan_stage_report.pyindex cc859fb5..0bf3ab9b 100644--- a/eval/bench/plan_stage_report.py+++ b/eval/bench/plan_stage_report.py@@ -18,6 +18,7 @@ from pathlib import Pathfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from bench._infra import is_infra_failure # noqa: E402from bench._jsonl import read_jsonl # noqa: E402#: what a validation failure was ABOUT, read off the error text. The mechanism under test@@ -33,19 +34,16 @@ FAILURE_CLASSES: dict[str, str] = {def outcome(row: dict) -> str:"""``valid`` | ``planning_error`` (the code under test lost the call) | ``provider``.- ``provider`` is dropped from the denominator, so it is deliberately narrow: only the- model-side failures the harness cannot help. A harness-side death — a budget ceiling,- an unexpected exception — is a loss of the run and must NOT be hidden here, so anything- that is neither a ``PlanningError`` nor a known provider failure counts as a loss.+ ``provider`` is dropped from the denominator by the bench's one outage rule+ (``_infra.is_infra_failure``; N76 — this kept its own word list, which dropped a model's+ MAX_TOKENS or HTTP 400 as weather and counted a spent key pool as a loss). A harness-side+ death — a budget ceiling, an unexpected exception — is a loss of the run and must NOT be+ hidden here, so anything that is neither a ``PlanningError`` nor an outage counts as a loss."""if row["ok"]:return "valid"err = row.get("error", "")- if "PlanningError" in err:- return "planning_error"- if any(w in err for w in ("ModelError", "BlockedReason", "429", "503", "Deadline", "RESOURCE_EXHAUSTED")):- return "provider"- return "planning_error"+ return "provider" if "PlanningError" not in err and is_infra_failure(err) else "planning_error"def failure_class(row: dict) -> str:diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.pyindex 4bec2a44..850d476e 100644--- a/eval/bench/run_bench.py+++ b/eval/bench/run_bench.py@@ -45,7 +45,7 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolif str(_p) not in sys.path: # this tree's codeverse3d (harness/) + the `bench` package (eval/)sys.path.insert(0, str(_p))-from bench._infra import is_infra_failure # noqa: E402+from bench._infra import is_infra_failure, judge_outage # noqa: E402from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402from codeverse3d.addons import select # noqa: E402from codeverse3d.config import get_settings # noqa: E402@@ -211,11 +211,13 @@ def result_from_record(item: BenchPrompt, rec: RunRecord, ws: Workspace) -> Bencerror = rec.errorif unjudged and not error:error = f"no verdict in any of {len(rec.rounds)} round(s) — the judge was skipped every time"+ # the judge's outage is not the arm's result: dropped and --redo-status-able, as compare_backends does+ status = "infra_failed" if judge_outage(rec.status, s.picked_score) else s.stop_reasonreturn BenchItemResult(id=item.id, tier=item.tier, category=item.category, score_baseline=s.baseline_score,score_picked=s.picked_score, picked_round=s.picked_round,rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd, minutes=round(rec.minutes or 0.0, 2),- status=s.stop_reason, errors=error, workspace=str(ws.root),+ status=status, errors=error, workspace=str(ws.root),generator=rec.spec.backends.generator, judge=rec.spec.backends.judge,)diff --git a/eval/tests/test_bench.py b/eval/tests/test_bench.pyindex 260cd903..c827eeab 100644--- a/eval/tests/test_bench.py+++ b/eval/tests/test_bench.py@@ -166,16 +166,30 @@ def test_a_provider_outage_is_not_model_latency_and_not_an_error():assert BenchItemResult.model_validate({"id": "old", "tier": "easy", "score_final": 0.7, "passed": True}).score_picked == 0.7-def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):[email protected]("outage", ["api_503", "cli_storm", "cli_quota", "judge_down"])+def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path, outage):"""The status has to be recorded in the first place — the reporter can only honour- what run_battery wrote."""+ what run_battery wrote. The harness's typed verdict (``RoundFailed.transient`` / ``.quota``)+ decides before any wording (N76: a gemini-cli storm's message matches no marker), and a run+ whose judge never answered is dropped the way compare_backends drops it."""from codeverse3d.models.base import ModelError+ from codeverse3d.tracks.steps import RoundFailedbattery = REPO / "bench" / "prompts" / "static_objects_v1.yaml"out = tmp_path / "bench_out"def storm(spec, ws, resume):- raise ModelError("Gemini API error 503: The model is overloaded.", retryable=True, status=503)+ if outage == "judge_down":+ rec = _fake_run_fn({})(spec, ws, resume)+ rec.status = RunStatus.JUDGE_UNAVAILABLE+ for r in rec.rounds:+ r.judgment = None+ ws.write_json(ws.record_path, rec)+ return rec+ raise {"api_503": ModelError("Gemini API error 503: The model is overloaded.", retryable=True, status=503),+ "cli_storm": RoundFailed("baseline: gemini-cli exit 1: [API Error: got status: 503 UNAVAILABLE]",+ transient=True),+ "cli_quota": RoundFailed("baseline: codex: You've hit your usage limit", quota=True)}[outage]res = run_battery(battery, out, BenchOptions(parallel=1, limit=1), run_fn=storm)assert [r.status for r in res] == ["infra_failed"]diff --git a/eval/tests/test_plan_stage.py b/eval/tests/test_plan_stage.pyindex d62d72de..1648fec7 100644--- a/eval/tests/test_plan_stage.py+++ b/eval/tests/test_plan_stage.py@@ -61,6 +61,11 @@ def test_a_harness_side_death_is_a_loss_not_provider_weather() -> None:assert outcome({"ok": False, "error": "ModelError: Gemini prompt blocked: BlockedReason.OTHER"}) == "provider"assert outcome({"ok": False, "error": "KeyError: 'parts'"}) == "planning_error"+ # one outage rule with the bench (N76): a model's own bad answer is a loss, a spent key pool is weather+ for err, want in (("ModelError: Gemini returned no JSON (finish_reason=MAX_TOKENS)", "planning_error"),+ ("ModelError: HTTP 400 invalid argument", "planning_error"),+ ("KeyPoolExhausted: all 15 keys cooling", "provider")):+ assert outcome({"ok": False, "error": err}) == want, errdef test_a_killed_arm_file_still_reports(tmp_path: Path, capsys) -> None: