Skip to content

Official Codebase of 3DCodeVerse

Python

bench: an outage is decided from the harness's typed transient/quota first; plan_stage_report and a judge-down run use the same rule

gaoypengb04d9d7de4a3
diff --git a/eval/bench/_infra.py b/eval/bench/_infra.py
index c2d72a44..7a9d0328 100644
--- a/eval/bench/_infra.py
+++ b/eval/bench/_infra.py
@@ -22,6 +22,9 @@ from __future__ import annotations
import logging
from collections.abc import Iterator
+from codeverse3d.contracts.run import RunStatus
+from codeverse3d.models.parts import retryable_status
+
log = logging.getLogger(__name__)
#: substrings that identify a provider-side outage in a *stringified* error. Kept
@@ -36,6 +39,7 @@ INFRA_MARKERS: tuple[str, ...] = (
"read operation timed out",
"deadline expired", # Gemini 504
"finish_reason=prohibited_content", # the provider's content filter tripped mid-JSON on a furniture plan (2026-08-28): provider behaviour, redone
+ "prompt blocked", # the same filter on the harness's own prompt (BlockedReason.OTHER): plan_stage_report's "provider" since D52
"exceeded its attempt budget", # models/gemini.py streaming: the provider held the socket past the attempt budget (2026-08-28, pro planner)
"deadline exceeded",
"connection reset",
@@ -57,31 +61,39 @@ INFRA_MARKERS: tuple[str, ...] = (
"error creating webgl context",
)
-#: HTTP statuses that mean "the provider could not serve this", not "the model was bad".
-#: 408 is the adapters' own code for a request timeout (gemini.py:117, openai.py:40,
-#: anthropic.py:43) — the exception path must agree with the "request timed out" marker
-#: the string path already had.
-INFRA_STATUSES: frozenset[int] = frozenset({408, 429, 500, 502, 503, 504, 529})
-
-
def _has_marker(text: str) -> bool:
return any(marker in text.lower() for marker in INFRA_MARKERS)
def _looks_infra(exc: BaseException) -> bool:
- """The outage rules, applied to ONE exception. Structured first — a ``ModelError`` /
- ``KeyPoolExhausted`` carries its status, and trusting that beats guessing from prose —
- with the string markers as the fallback. ONE copy: this ran as two hand-kept copies
- (head vs each wrapped cause) until 2026-08-28, which is how a rule reaches one path
- and not the other."""
+ """The outage rules, applied to ONE exception. Typed first: the harness's own verdict
+ (``RoundFailed`` / ``GenerationResult`` ``.transient`` — a provider storm, a dropped socket
+ — and ``.quota`` — the vendor's usage limit, "not ours") decided from the CLI's own record,
+ which the bench must not re-guess from the message (N76: a gemini-cli 503 storm that killed a
+ baseline twice matched no marker and was scored as an error). Then the HTTP status, from the
+ harness's one table (``models.parts.retryable_status``: 408/409/429/5xx), and a spent key
+ pool. The string markers are the fallback, for exceptions nobody typed and for the error
+ TEXT of old records. ONE copy: this ran as two hand-kept copies (head vs each wrapped
+ cause) until 2026-08-28, which is how a rule reaches one path and not the other."""
+ if getattr(exc, "transient", False) is True or getattr(exc, "quota", False) is True:
+ return True
if type(exc).__name__ == "KeyPoolExhausted" or isinstance(exc, TimeoutError):
return True
status = getattr(exc, "status", None)
- if isinstance(status, int) and status in INFRA_STATUSES:
+ if isinstance(status, int) and retryable_status(status):
return True
return _has_marker(f"{type(exc).__name__}: {exc}")
+def judge_outage(status: RunStatus, picked_score: float | None) -> bool:
+ """A harness run that stopped because its in-loop judge never answered, with no round
+ judged and picked before it: the judge's outage, not the arm's result — dropped as
+ ``infra_failed`` and re-run fresh by ``--redo-status``. A run that had picked a round
+ before the outage is scored on that pick (owner, 2026-09-23). THE rule for run_bench
+ and compare_backends (N76)."""
+ return status is RunStatus.JUDGE_UNAVAILABLE and picked_score is None
+
+
def _chain(err: BaseException, limit: int = 32) -> Iterator[BaseException]:
"""``err`` and every exception it was raised from, each yielded once.
diff --git a/eval/bench/ab_plan.py b/eval/bench/ab_plan.py
index a895bf83..124fc654 100644
--- a/eval/bench/ab_plan.py
+++ b/eval/bench/ab_plan.py
@@ -499,7 +499,7 @@ def run_ab(
# pair is recorded the way a dead cell is (docs/EVAL.md §7 — an outage is
# not a score) so `--redo-status infra_failed` picks it up, and the loop
# goes on to the next prompt.
- status = "infra_failed" if is_infra_failure(str(e)) else "error"
+ status = "infra_failed" if is_infra_failure(e) else "error" # the exception: its typed flags and status
log.warning(
"%s: pinned plan failed (%s) — pair recorded %s, continuing",
item.id,
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.py
index 0ebf58c2..d94ce3c4 100644
--- a/eval/bench/compare_backends.py
+++ b/eval/bench/compare_backends.py
@@ -58,7 +58,7 @@ from datetime import UTC # noqa: E402
from bench._compare_report import CellResult, PairRow, build_compare_report # noqa: E402
from bench._fixed_eval import EvalOutcome, FixedEvaluator, rubric_for # noqa: E402
-from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402
+from bench._infra import is_budget_exhaustion, is_infra_failure, judge_outage # noqa: E402
from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402
from bench._oneshot import ( # noqa: E402
OneShotBackend,
@@ -295,7 +295,7 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOption
res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), summary.picked_score
res.harness_stop_reason = summary.stop_reason
res.harness_aborted_rounds = len(rec.extra.get("aborted_rounds") or [])
- if rec.status is RunStatus.JUDGE_UNAVAILABLE and summary.picked_score is None:
+ if judge_outage(rec.status, summary.picked_score):
# The run stopped because its judge never answered (p3_graphics_v2 mushroom_forest, 2026-09-23:
# two 900 s pro read timeouts after round 0). Scoring what it had then would grade a one-round
# run for someone else's outage — the asymmetry §7 ends for the one-shot arms — so the cell is
diff --git a/eval/bench/plan_stage_report.py b/eval/bench/plan_stage_report.py
index cc859fb5..0bf3ab9b 100644
--- a/eval/bench/plan_stage_report.py
+++ b/eval/bench/plan_stage_report.py
@@ -18,6 +18,7 @@ from pathlib import Path
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from bench._infra import is_infra_failure # noqa: E402
from bench._jsonl import read_jsonl # noqa: E402
#: what a validation failure was ABOUT, read off the error text. The mechanism under test
@@ -33,19 +34,16 @@ FAILURE_CLASSES: dict[str, str] = {
def outcome(row: dict) -> str:
"""``valid`` | ``planning_error`` (the code under test lost the call) | ``provider``.
- ``provider`` is dropped from the denominator, so it is deliberately narrow: only the
- model-side failures the harness cannot help. A harness-side death — a budget ceiling,
- an unexpected exception — is a loss of the run and must NOT be hidden here, so anything
- that is neither a ``PlanningError`` nor a known provider failure counts as a loss.
+ ``provider`` is dropped from the denominator by the bench's one outage rule
+ (``_infra.is_infra_failure``; N76 — this kept its own word list, which dropped a model's
+ MAX_TOKENS or HTTP 400 as weather and counted a spent key pool as a loss). A harness-side
+ death — a budget ceiling, an unexpected exception — is a loss of the run and must NOT be
+ hidden here, so anything that is neither a ``PlanningError`` nor an outage counts as a loss.
"""
if row["ok"]:
return "valid"
err = row.get("error", "")
- if "PlanningError" in err:
- return "planning_error"
- if any(w in err for w in ("ModelError", "BlockedReason", "429", "503", "Deadline", "RESOURCE_EXHAUSTED")):
- return "provider"
- return "planning_error"
+ return "provider" if "PlanningError" not in err and is_infra_failure(err) else "planning_error"
def failure_class(row: dict) -> str:
diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.py
index 4bec2a44..850d476e 100644
--- a/eval/bench/run_bench.py
+++ b/eval/bench/run_bench.py
@@ -45,7 +45,7 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resol
if str(_p) not in sys.path: # this tree's codeverse3d (harness/) + the `bench` package (eval/)
sys.path.insert(0, str(_p))
-from bench._infra import is_infra_failure # noqa: E402
+from bench._infra import is_infra_failure, judge_outage # noqa: E402
from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402
from codeverse3d.addons import select # noqa: E402
from codeverse3d.config import get_settings # noqa: E402
@@ -211,11 +211,13 @@ def result_from_record(item: BenchPrompt, rec: RunRecord, ws: Workspace) -> Benc
error = rec.error
if unjudged and not error:
error = f"no verdict in any of {len(rec.rounds)} round(s) — the judge was skipped every time"
+ # the judge's outage is not the arm's result: dropped and --redo-status-able, as compare_backends does
+ status = "infra_failed" if judge_outage(rec.status, s.picked_score) else s.stop_reason
return BenchItemResult(
id=item.id, tier=item.tier, category=item.category, score_baseline=s.baseline_score,
score_picked=s.picked_score, picked_round=s.picked_round,
rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd, minutes=round(rec.minutes or 0.0, 2),
- status=s.stop_reason, errors=error, workspace=str(ws.root),
+ status=status, errors=error, workspace=str(ws.root),
generator=rec.spec.backends.generator, judge=rec.spec.backends.judge,
)
diff --git a/eval/tests/test_bench.py b/eval/tests/test_bench.py
index 260cd903..c827eeab 100644
--- a/eval/tests/test_bench.py
+++ b/eval/tests/test_bench.py
@@ -166,16 +166,30 @@ def test_a_provider_outage_is_not_model_latency_and_not_an_error():
assert BenchItemResult.model_validate({"id": "old", "tier": "easy", "score_final": 0.7, "passed": True}).score_picked == 0.7
-def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):
[email protected]("outage", ["api_503", "cli_storm", "cli_quota", "judge_down"])
+def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path, outage):
"""The status has to be recorded in the first place — the reporter can only honour
- what run_battery wrote."""
+ what run_battery wrote. The harness's typed verdict (``RoundFailed.transient`` / ``.quota``)
+ decides before any wording (N76: a gemini-cli storm's message matches no marker), and a run
+ whose judge never answered is dropped the way compare_backends drops it."""
from codeverse3d.models.base import ModelError
+ from codeverse3d.tracks.steps import RoundFailed
battery = REPO / "bench" / "prompts" / "static_objects_v1.yaml"
out = tmp_path / "bench_out"
def storm(spec, ws, resume):
- raise ModelError("Gemini API error 503: The model is overloaded.", retryable=True, status=503)
+ if outage == "judge_down":
+ rec = _fake_run_fn({})(spec, ws, resume)
+ rec.status = RunStatus.JUDGE_UNAVAILABLE
+ for r in rec.rounds:
+ r.judgment = None
+ ws.write_json(ws.record_path, rec)
+ return rec
+ raise {"api_503": ModelError("Gemini API error 503: The model is overloaded.", retryable=True, status=503),
+ "cli_storm": RoundFailed("baseline: gemini-cli exit 1: [API Error: got status: 503 UNAVAILABLE]",
+ transient=True),
+ "cli_quota": RoundFailed("baseline: codex: You've hit your usage limit", quota=True)}[outage]
res = run_battery(battery, out, BenchOptions(parallel=1, limit=1), run_fn=storm)
assert [r.status for r in res] == ["infra_failed"]
diff --git a/eval/tests/test_plan_stage.py b/eval/tests/test_plan_stage.py
index d62d72de..1648fec7 100644
--- a/eval/tests/test_plan_stage.py
+++ b/eval/tests/test_plan_stage.py
@@ -61,6 +61,11 @@ def test_a_harness_side_death_is_a_loss_not_provider_weather() -> None:
assert outcome({"ok": False, "error": "ModelError: Gemini prompt blocked: BlockedReason.OTHER"}) == "provider"
assert outcome({"ok": False, "error": "KeyError: 'parts'"}) == "planning_error"
+ # one outage rule with the bench (N76): a model's own bad answer is a loss, a spent key pool is weather
+ for err, want in (("ModelError: Gemini returned no JSON (finish_reason=MAX_TOKENS)", "planning_error"),
+ ("ModelError: HTTP 400 invalid argument", "planning_error"),
+ ("KeyPoolExhausted: all 15 keys cooling", "provider")):
+ assert outcome({"ok": False, "error": err}) == want, err
def test_a_killed_arm_file_still_reports(tmp_path: Path, capsys) -> None: