Skip to content

Official Codebase of 3DCodeVerse

Python

bench: flag harness cells that waited on the provider instead of iterating

ziyao934f88ee73b6
diff --git a/harness/bench/_compare_report.py b/harness/bench/_compare_report.py
index bb470ef9..b6539fac 100644
--- a/harness/bench/_compare_report.py
+++ b/harness/bench/_compare_report.py
@@ -46,8 +46,15 @@ class CellResult(BaseModel):
tool_calls: int = 0
criteria: dict[str, float] = Field(default_factory=dict)
harness_status: str = ""
- harness_rounds: int = 0
+ harness_rounds: int = Field(default=0, description="rounds the harness run COMPLETED (baseline included)")
harness_loop_score: float | None = None
+ harness_stop_reason: str = Field(default="", description="record.extra['stop_reason'] of the harness run")
+ harness_aborted_rounds: int = Field(default=0, description="rounds the budget / wall-clock ceiling cut mid-way (record.extra['aborted_rounds'])")
+ degraded: bool = Field(default=False, description=(
+ "harness run stopped on the wall-clock ceiling with <=1 completed round and money left: it waited "
+ "on the provider, it did not iterate. Scored (the artifact is real) but flagged, so a storm cannot "
+ "pass as harness performance (compare_backends.flag_degraded, docs/EVAL.md)"))
+ degraded_reason: str = ""
sheet: str = ""
glb: str = ""
workspace: str = ""
@@ -118,6 +125,8 @@ class ArmStats(BaseModel):
mean_minutes: float
tool_calls: int
errors: int
+ degraded: int = Field(default=0, description="harness cells flagged storm-degraded (scored, but the run never iterated)")
+ ceiling_cut: int = Field(default=0, description="harness cells whose run had a round cut by the ceiling")
class PairStats(BaseModel):
@@ -157,6 +166,7 @@ def arm_stats(rows: list[CellResult]) -> list[ArmStats]:
mean_judge_usd=round(statistics.fmean(r.judge_cost_usd for r in ev), 4) if ev else 0.0,
mean_minutes=round(statistics.fmean(r.wall_s for r in ev) / 60, 2) if ev else 0.0,
tool_calls=sum(r.tool_calls for r in ev), errors=sum(1 for r in rs if r.status in ("error", "judge_error")),
+ degraded=sum(1 for r in ev if r.degraded), ceiling_cut=sum(1 for r in ev if r.harness_aborted_rounds > 0),
))
return sorted(out, key=lambda s: (s.kind != "harness", -(s.mean_score or -1)))
@@ -199,7 +209,7 @@ def _cell_text(r: CellResult | None) -> str:
if r.score is None:
return f"? ({r.status})"
flag = "" if r.build_ok else " ✗build"
- return f"{r.score:.2f}{'✓' if r.passed else ''}{flag}"
+ return f"{r.score:.2f}{'✓' if r.passed else ''}{flag}{'†' if r.degraded else ''}"
def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], meta: dict) -> str:
@@ -217,13 +227,17 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], me
"`n` counts cells that actually ran; `dropped` counts cells lost to a provider outage "
"(503 storm, timeout, exhausted pool) — those test nothing about the model and are excluded "
"from every rate on this page rather than scored 0.", "",
- "## arms", "", "| arm | kind | n | dropped | over budget | mean | median | pass | build ok | $gen | $judge | min | tool calls | errors |",
- "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
+ "`degraded` counts harness cells that stopped on the wall-clock ceiling with at most one completed round "
+ "and money left — the run waited on the provider instead of iterating; they are scored (the artifact is "
+ "real) but marked † so a storm cannot pass as harness performance. `cut` counts harness runs the "
+ "ceiling interrupted mid-round.", "",
+ "## arms", "", "| arm | kind | n | dropped | over budget | mean | median | pass | build ok | $gen | $judge | min | tool calls | errors | degraded | cut |",
+ "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
for s in arm_stats(rows):
md.append(f"| {s.arm} | {s.kind} | {s.n_evaluated} | {s.infra_failed or ''} | {s.budget_exhausted or ''} | {_f(s.mean_score)} | {_f(s.median_score)} | {_f(s.pass_rate, '.0%')} | "
f"{s.build_ok_rate:.0%} | {s.mean_gen_usd:.2f} | {s.mean_judge_usd:.3f} | {s.mean_minutes:.1f} | "
- f"{s.tool_calls} | {s.errors} |")
- md += ["", "## per prompt (score, ✓ = passed, ✗build = build failed)", "",
+ f"{s.tool_calls} | {s.errors} | {s.degraded or ''} | {s.ceiling_cut or ''} |")
+ md += ["", "## per prompt (score, ✓ = passed, ✗build = build failed, † = storm-degraded harness run)", "",
"| prompt | tier | " + " | ".join(arms) + " |", "|---|---|" + "---|" * len(arms)]
for pid in prompts:
tier = next((r.tier for r in rows if r.prompt_id == pid), "")
@@ -240,9 +254,10 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], me
hr = [r for r in rows if r.kind == "harness"]
if hr:
md += ["", "## harness runs (loop judge score vs fixed judge score)", "",
- "| prompt | arm | run status | rounds | loop score | fixed score | $run | min |", "|---|---|---|---|---|---|---|---|"]
- md += [f"| {r.prompt_id} | {r.arm} | {r.harness_status} | {r.harness_rounds} | {_f(r.harness_loop_score)} | {_f(r.score)} | "
- f"{r.gen_cost_usd:.2f} | {r.wall_s / 60:.1f} |" for r in sorted(hr, key=lambda r: (r.prompt_id, r.arm))]
+ "| prompt | arm | run status | stop | rounds | cut | loop score | fixed score | $run | min | degraded |", "|---|---|---|---|---|---|---|---|---|---|---|"]
+ md += [f"| {r.prompt_id} | {r.arm} | {r.harness_status} | {r.harness_stop_reason} | {r.harness_rounds} | {r.harness_aborted_rounds or ''} | "
+ f"{_f(r.harness_loop_score)} | {_f(r.score)} | {r.gen_cost_usd:.2f} | {r.wall_s / 60:.1f} | {'† ' + r.degraded_reason if r.degraded else ''} |"
+ for r in sorted(hr, key=lambda r: (r.prompt_id, r.arm))]
total_gen, total_judge, total_pw = sum(r.gen_cost_usd for r in rows), sum(r.judge_cost_usd for r in rows), sum(p.cost_usd for p in pairs)
md += ["", f"**Total cost**: generation ${total_gen:.2f} · fixed judge ${total_judge:.2f} · pairwise ${total_pw:.2f} "
f"(subscription CLIs report their own cost figure or 0)."]
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index db467fce..b398c4e3 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -132,6 +132,8 @@ class CompareOptions(BaseModel):
repair_attempts: int = 2
n_samples: int = 2
pairwise: bool = True
+ degraded_min_wall_s: float = Field(default=40 * 60, description="flag_degraded: a harness run this long that never iterated waited, it did not work")
+ degraded_max_rounds: int = Field(default=1, description="flag_degraded: completed rounds at or below this count")
redo_status: list[str] = Field(default_factory=list, description="re-run cells whose status is one of these")
@@ -222,6 +224,8 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, deps: CompareDeps,
res.gen_cost_usd = rec.total_usage.cost_usd
res.tool_calls = rec.total_usage.tool_calls
res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), rec.final_score
+ res.harness_stop_reason = str(rec.extra.get("stop_reason") or "")
+ res.harness_aborted_rounds = len(rec.extra.get("aborted_rounds") or [])
if not (run_ws.root / MODEL_FILE).is_file():
res.error = f"harness run produced no {MODEL_FILE} (status {rec.status.value}: {rec.error})"
return
@@ -272,10 +276,33 @@ def run_cell(battery: Battery, item: BenchPrompt, arm: Arm, out: Path, opts: Com
res.status = "infra_failed" if is_infra_failure(e) else "error" # same rule as the no-code path
res.error = (res.error + "; " if res.error else "") + f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1200:]}"
res.wall_s = round(time.time() - t0, 1)
+ flag_degraded(res, opts)
eval_ws.write_json(cell / "cell.json", res)
return res
+def flag_degraded(res: CellResult, opts: CompareOptions) -> None:
+ """Mark a harness cell whose run WAITED instead of iterating (compare_v4, 2026-08-25).
+
+ Under a day-long gemini-3.7-flash 503 storm 37 of 40 harness runs stopped on the
+ 45-minute wall-clock ceiling with 0–2 completed rounds while the 3 runs that met a
+ calm window finished 2–4 rounds in 27–40 min and scored 0.92–0.95. Those cells are
+ real artifacts and stay scored, but a paired mean over them measures the weather, not
+ the loop — so the cell says so. Rule: the harness stopped for *budget* while money
+ was left (i.e. the clock, not the dollars), completed at most ``degraded_max_rounds``
+ rounds, and the cell ran at least ``degraded_min_wall_s``. ``harness_aborted_rounds``
+ separately counts runs the ceiling interrupted mid-round.
+ """
+ if res.kind != "harness" or res.status not in ("scored", "build_failed"):
+ return
+ money_stop = res.gen_cost_usd >= 0.9 * opts.max_usd
+ if (res.harness_stop_reason == "budget" and not money_stop and res.harness_rounds <= opts.degraded_max_rounds
+ and res.wall_s >= opts.degraded_min_wall_s):
+ res.degraded = True
+ res.degraded_reason = (f"ceiling stop after {res.harness_rounds} completed round(s) in {res.wall_s / 60:.0f} min "
+ f"with ${opts.max_usd - res.gen_cost_usd:.2f} of the budget unspent")
+
+
def _fill_from_outcome(res: CellResult, o: EvalOutcome) -> None:
res.build_ok = o.build.ok
res.gate_errors = o.gate_errors
diff --git a/harness/bench/paired_compare.py b/harness/bench/paired_compare.py
index e57d6f36..2b0f47ca 100644
--- a/harness/bench/paired_compare.py
+++ b/harness/bench/paired_compare.py
@@ -24,6 +24,7 @@ import json
import math
import statistics
import sys
+from collections import defaultdict
from pathlib import Path
from pydantic import BaseModel, Field
@@ -64,6 +65,8 @@ class PairedStats(BaseModel):
n: int = Field(description="prompts with a score on BOTH arms")
dropped_infra: int = Field(default=0, description="pairs lost to a provider outage on either arm")
dropped_unscored: int = Field(default=0, description="pairs lost to an unscored, non-outage cell")
+ dropped_degraded: int = Field(default=0, description="pairs excluded because the harness cell is storm-degraded (exclude_degraded=True)")
+ degraded_kept: int = Field(default=0, description="pairs kept whose harness cell is storm-degraded (exclude_degraded=False)")
mean_harness: float | None = None
mean_oneshot: float | None = None
mean_delta: float | None = None
@@ -102,7 +105,7 @@ def _usable(r: CellResult | None) -> tuple[bool, str]:
def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_arm: str, *,
- tier: str = "all") -> PairedStats:
+ tier: str = "all", exclude_degraded: bool = False) -> PairedStats:
prompts = sorted({p for (p, a) in cells if a in (harness_arm, oneshot_arm)})
st = PairedStats(harness_arm=harness_arm, oneshot_arm=oneshot_arm, tier=tier, n=0)
hs: list[float] = []
@@ -124,6 +127,11 @@ def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_a
st.dropped_unscored += 1
continue
assert h is not None and o is not None and h.score is not None and o.score is not None
+ if h.degraded or o.degraded:
+ if exclude_degraded:
+ st.dropped_degraded += 1
+ continue
+ st.degraded_kept += 1
hs.append(h.score)
os_.append(o.score)
st.deltas[p] = round(h.score - o.score, 4)
@@ -166,35 +174,87 @@ def analyse(rows: list[CellResult]) -> list[PairedStats]:
oneshot = [a for a in arms if a.startswith("oneshot")]
tiers = sorted({r.tier for r in cells.values() if r.tier}, key=lambda t: ("easy", "medium", "hard").index(t) if t in ("easy", "medium", "hard") else 9)
out: list[PairedStats] = []
+ any_degraded = any(r.degraded for r in cells.values())
for ha in harness:
for oa in oneshot:
out.append(paired(cells, ha, oa))
+ if any_degraded: # the same comparison without the cells that measured the weather
+ st = paired(cells, ha, oa, exclude_degraded=True)
+ st.tier = "all −degraded"
+ out.append(st)
for t in tiers:
out.append(paired(cells, ha, oa, tier=t))
return out
+class ArmGateStats(BaseModel):
+ """Judge-free numbers per arm: what the deterministic gates and the build say."""
+
+ arm: str
+ n: int
+ build_ok_rate: float
+ mean_gate_errors: float | None
+ zero_gate_error_rate: float | None
+ mean_tris: float | None
+ degraded: int = 0
+
+
+def gate_stats(cells: dict[tuple[str, str], CellResult]) -> list[ArmGateStats]:
+ by: dict[str, list[CellResult]] = defaultdict(list)
+ for (_, arm), r in cells.items():
+ if r.status != "infra_failed":
+ by[arm].append(r)
+ out = []
+ for arm, rs in sorted(by.items(), key=lambda kv: (not kv[0].startswith("harness:"), kv[0])):
+ built = [r for r in rs if r.build_ok]
+ out.append(ArmGateStats(
+ arm=arm, n=len(rs), build_ok_rate=round(sum(1 for r in rs if r.build_ok) / len(rs), 4) if rs else 0.0,
+ mean_gate_errors=round(statistics.fmean(len(r.gate_errors) for r in built), 3) if built else None,
+ zero_gate_error_rate=round(sum(1 for r in built if not r.gate_errors) / len(built), 4) if built else None,
+ mean_tris=round(statistics.fmean(r.tris for r in built if r.tris is not None), 0)
+ if any(r.tris is not None for r in built) else None,
+ degraded=sum(1 for r in rs if r.degraded),
+ ))
+ return out
+
+
def _f(x: float | None, signed: bool = False) -> str:
if x is None:
return "—"
return f"{x:+.3f}" if signed else f"{x:.3f}"
-def render_markdown(stats: list[PairedStats], title: str) -> str:
+def render_gate_markdown(gs: list[ArmGateStats]) -> str:
+ lines = ["", "## judge-free (build + deterministic gates, cells that ran)", "",
+ "| arm | n | build ok | mean gate errors (built) | zero-gate-error (built) | mean tris | degraded |",
+ "|---|---|---|---|---|---|---|"]
+ for g in gs:
+ lines.append(f"| {g.arm} | {g.n} | {g.build_ok_rate:.0%} | {_f(g.mean_gate_errors)} | "
+ f"{'—' if g.zero_gate_error_rate is None else format(g.zero_gate_error_rate, '.0%')} | "
+ f"{'—' if g.mean_tris is None else int(g.mean_tris)} | {g.degraded or ''} |")
+ return "\n".join(lines) + "\n"
+
+
+def render_markdown(stats: list[PairedStats], title: str, gates: list[ArmGateStats] | None = None) -> str:
lines = [f"# paired harness − one-shot — {title}", "",
"Mean Δ with its paired 95 % t-interval and the exact two-sided sign test over prompts scored on BOTH arms; "
"`infra` = pairs dropped to a provider outage, `unscored` = pairs dropped to a cell with no score for any other reason. "
"A comparison is **supported** only when the interval excludes zero.", "",
- "| harness arm | one-shot arm | tier | n | infra | unscored | harness | one-shot | Δ | 95 % CI | W/L/T | sign p | pass h/o | build h/o | verdict |",
- "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
+ "`degraded` = pairs whose harness cell is storm-degraded (kept in every row except `all −degraded`, where they are excluded).", "",
+ "| harness arm | one-shot arm | tier | n | infra | unscored | degraded | harness | one-shot | Δ | 95 % CI | W/L/T | sign p | pass h/o | build h/o | verdict |",
+ "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
for s in stats:
ci = f"[{_f(s.ci95_low, True)}, {_f(s.ci95_high, True)}]" if s.ci95_low is not None else "—"
+ deg = f"{s.dropped_degraded} excl." if s.dropped_degraded else (str(s.degraded_kept) if s.degraded_kept else "")
lines.append(
- f"| {s.harness_arm} | {s.oneshot_arm} | {s.tier} | {s.n} | {s.dropped_infra} | {s.dropped_unscored} | "
+ f"| {s.harness_arm} | {s.oneshot_arm} | {s.tier} | {s.n} | {s.dropped_infra} | {s.dropped_unscored} | {deg} | "
f"{_f(s.mean_harness)} | {_f(s.mean_oneshot)} | {_f(s.mean_delta, True)} | {ci} | {s.wins}/{s.losses}/{s.ties} | "
f"{_f(s.sign_p)} | {_f(s.pass_rate_harness)}/{_f(s.pass_rate_oneshot)} | {_f(s.build_ok_harness)}/{_f(s.build_ok_oneshot)} | "
f"**{s.verdict}** |")
- return "\n".join(lines) + "\n"
+ out = "\n".join(lines) + "\n"
+ if gates:
+ out += render_gate_markdown(gates)
+ return out
def main(argv: list[str] | None = None) -> int:
@@ -204,9 +264,11 @@ def main(argv: list[str] | None = None) -> int:
out = Path(ns.out_dir)
rows = read_jsonl(out / "results.jsonl", CellResult)
stats = analyse(rows)
- md = render_markdown(stats, out.name)
+ gates = gate_stats(latest_cells(rows))
+ md = render_markdown(stats, out.name, gates)
(out / "paired.md").write_text(md)
- (out / "paired.json").write_text(json.dumps([s.model_dump(mode="json") for s in stats], indent=2))
+ (out / "paired.json").write_text(json.dumps({"paired": [s.model_dump(mode="json") for s in stats],
+ "gates": [g.model_dump(mode="json") for g in gates]}, indent=2))
print(md)
return 0
diff --git a/harness/tests/compare_bench/test_degraded.py b/harness/tests/compare_bench/test_degraded.py
new file mode 100644
index 00000000..9dc42227
--- /dev/null
+++ b/harness/tests/compare_bench/test_degraded.py
@@ -0,0 +1,91 @@
+"""compare_backends.flag_degraded: a harness run that waited on the provider is flagged, not trusted.
+
+compare_v4 (2026-08-25): 37 of 40 harness runs stopped on the 45-minute wall-clock ceiling with
+0–2 completed rounds during a flash 503 storm; the 3 runs that met a calm window did 2–4 rounds
+in 27–40 min and scored 0.92–0.95. A paired mean over the stormed cells measures the weather.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[2]
+if str(REPO) not in sys.path:
+ sys.path.insert(0, str(REPO))
+
+from bench._compare_report import CellResult, arm_stats, compare_markdown # noqa: E402
+from bench.compare_backends import ( # noqa: E402
+ CompareDeps,
+ CompareOptions,
+ flag_degraded,
+ parse_arm,
+ run_cell,
+)
+from bench.run_bench import Battery # noqa: E402
+from codeverse.contracts.common import Usage # noqa: E402
+from codeverse.contracts.run import RoundRecord, RunRecord, RunStatus # noqa: E402
+from tests.compare_bench.conftest import BATTERY, GOOD, FakeEvaluator # noqa: E402
+
+
+def _track(status: RunStatus, stop_reason: str, rounds: int, cost: float, aborted: int = 0):
+ def run(spec, ws, resume):
+ ws.src.mkdir(parents=True, exist_ok=True)
+ (ws.root / "src" / "model.py").write_text(GOOD.format(score=0.6))
+ rec = RunRecord(spec=spec, workspace=str(ws.root), status=status, final_score=0.6,
+ rounds=[RoundRecord(index=i, kind="baseline" if i == 0 else "refine") for i in range(rounds)],
+ total_usage=Usage(cost_usd=cost),
+ extra={"stop_reason": stop_reason, "aborted_rounds": [{"index": rounds}] * aborted})
+ ws.write_json(ws.record_path, rec)
+ return rec
+ return run
+
+
[email protected]("status,stop,rounds,cost,aborted,expect", [
+ (RunStatus.BUDGET, "budget", 1, 1.2, 1, True), # the storm case: clock stop, 1 round, money left
+ (RunStatus.BUDGET, "budget", 0, 0.9, 1, True), # never completed a round
+ (RunStatus.BUDGET, "budget", 2, 1.5, 1, False), # iterated twice: cut, but not degraded
+ (RunStatus.BUDGET, "budget", 1, 2.4, 0, False), # a MONEY stop (>= 90% of max_usd) is the protocol, not weather
+ (RunStatus.PASSED, "pass", 1, 0.8, 0, False), # passed after one round: fine
+ (RunStatus.PLATEAU, "plateau", 1, 0.7, 0, False),
+])
+def test_flag_degraded_rule(tmp_path, status, stop, rounds, cost, aborted, expect):
+ battery = Battery.load(BATTERY)
+ opts = CompareOptions(judge="gemini:x", loop_judge="gemini:x", max_usd=2.5, degraded_min_wall_s=0)
+ deps = CompareDeps(FakeEvaluator(), run_track=_track(status, stop, rounds, cost, aborted))
+ r = run_cell(battery, battery.prompts[0], parse_arm("harness:api-agent:gemini:gemini-3.7-flash"), tmp_path, opts, deps)
+ assert r.status == "scored" and r.score is not None
+ assert (r.harness_stop_reason, r.harness_rounds, r.harness_aborted_rounds) == (stop, rounds, aborted)
+ assert r.degraded is expect, r
+ if expect:
+ assert "ceiling stop" in r.degraded_reason and "unspent" in r.degraded_reason
+
+
+def test_degraded_needs_the_wall_clock_floor():
+ """A 2-minute harness run that stopped on `budget` with one round is not a storm victim."""
+ r = CellResult(prompt_id="p", arm="harness:x", kind="harness", status="scored", score=0.5, harness_rounds=1,
+ harness_stop_reason="budget", gen_cost_usd=0.5, wall_s=120)
+ flag_degraded(r, CompareOptions(judge="g:x"))
+ assert not r.degraded
+ r.wall_s = 45 * 60
+ flag_degraded(r, CompareOptions(judge="g:x"))
+ assert r.degraded
+
+
+def test_report_counts_and_marks_degraded_cells(tmp_path):
+ rows = [
+ CellResult(prompt_id="p0", tier="hard", arm="harness:x", kind="harness", status="scored", score=0.5,
+ build_ok=True, degraded=True, degraded_reason="ceiling stop after 0 completed round(s) in 45 min",
+ harness_stop_reason="budget", harness_aborted_rounds=1),
+ CellResult(prompt_id="p1", tier="hard", arm="harness:x", kind="harness", status="scored", score=0.9,
+ build_ok=True, harness_stop_reason="pass"),
+ CellResult(prompt_id="p0", tier="hard", arm="oneshot:x", kind="oneshot", status="scored", score=0.4, build_ok=True),
+ CellResult(prompt_id="p1", tier="hard", arm="oneshot:x", kind="oneshot", status="scored", score=0.3, build_ok=True),
+ ]
+ st = {s.arm: s for s in arm_stats(rows)}
+ assert st["harness:x"].degraded == 1 and st["harness:x"].ceiling_cut == 1 and st["oneshot:x"].degraded == 0
+ md = compare_markdown(tmp_path, rows, [], {"options": {}, "battery": {"name": "t"}})
+ assert "| degraded | cut |" in md and "0.50†" in md and "0.90 |" in md
+ assert "ceiling stop after 0 completed round(s)" in md
diff --git a/harness/tests/compare_bench/test_paired_compare.py b/harness/tests/compare_bench/test_paired_compare.py
index f47b7031..b7da21ba 100644
--- a/harness/tests/compare_bench/test_paired_compare.py
+++ b/harness/tests/compare_bench/test_paired_compare.py
@@ -65,4 +65,25 @@ def test_cli_writes_paired_md_and_json(tmp_path):
md = (tmp_path / "paired.md").read_text()
assert "| harness arm |" in md and HA in md and "**" in md
data = json.loads((tmp_path / "paired.json").read_text())
- assert data[0]["n"] == 2 and data[0]["wins"] == 2
+ assert data["paired"][0]["n"] == 2 and data["paired"][0]["wins"] == 2
+ assert md.count("## judge-free") == 1 and {g["arm"] for g in data["gates"]} == {HA, OA}
+
+
+def test_degraded_harness_cells_are_reported_and_can_be_excluded():
+ """A storm-degraded harness cell (flagged by compare_backends.flag_degraded) stays in the
+ headline row but is counted, and an `all −degraded` row repeats the comparison without it."""
+ rows = []
+ for i, (h, o, deg) in enumerate([(0.9, 0.5, False), (0.8, 0.6, False), (0.3, 0.6, True), (0.2, 0.5, True)]):
+ hc = _cell(f"p{i}", HA, h)
+ hc.degraded, hc.degraded_reason = deg, "ceiling stop after 0 completed round(s) in 45 min" if deg else ""
+ rows += [hc, _cell(f"p{i}", OA, o)]
+ stats = {s.tier: s for s in analyse(rows)}
+ assert stats["all"].n == 4 and stats["all"].degraded_kept == 2 and stats["all"].dropped_degraded == 0
+ assert stats["all"].mean_delta == round((0.4 + 0.2 - 0.3 - 0.3) / 4, 4)
+ excl = stats["all −degraded"]
+ assert excl.n == 2 and excl.dropped_degraded == 2 and excl.degraded_kept == 0
+ assert excl.mean_delta == round((0.4 + 0.2) / 2, 4) and excl.wins == 2
+ # no degraded cells → no extra row
+ plain = analyse([_cell("q", HA, 0.7), _cell("q", OA, 0.5)])
+ assert "all −degraded" not in {s.tier for s in plain}
+