The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(run_tests_parallel): name pytest ERRORs on the Summary line
Byrd-ITcd424473d15f
diff --git a/scripts/run_tests_parallel.py b/scripts/run_tests_parallel.pyindex 521a95eb647..3ccb2cfc228 100644--- a/scripts/run_tests_parallel.py+++ b/scripts/run_tests_parallel.py@@ -1126,10 +1126,17 @@ def _print_failure_report(crashed_files = [(f, o, s) for f, o, s in failures if s.get("crashed")]rest = [(f, s) for f, _o, s in failures if not s.get("crashed")]test_fail_files = [(f, s) for f, s in rest if s.get("failed", 0) > 0]+ # Setup/teardown ERRORs with no test failures get their own bucket;+ # they used to land under "all tests passed but pytest exited+ # non-zero", which reads like a warnings-as-errors quirk.+ test_error_files = [(f, s) for f, s in rest+ if s.get("failed", 0) == 0 and s.get("errors", 0) > 0]all_passed_but_nonzero = [(f, s) for f, s in rest- if s.get("failed", 0) == 0 and s.get("passed", 0) > 0]+ if s.get("failed", 0) == 0 and s.get("errors", 0) == 0+ and s.get("passed", 0) > 0]no_tests_ran = [(f, s) for f, s in rest- if s.get("failed", 0) == 0 and s.get("passed", 0) == 0]+ if s.get("failed", 0) == 0 and s.get("errors", 0) == 0+ and s.get("passed", 0) == 0]if crashed_files:print(f"=== {len(crashed_files)} file{'s' if len(crashed_files) != 1 else ''} where the interpreter CRASHED mid-run (native fault — a real bug, not a collection error; the tests that did run are not counted) ===")for file, output, _s in crashed_files:@@ -1140,6 +1147,12 @@ def _print_failure_report(for file, s in test_fail_files:nf = s.get("failed", 0)print(f" {_format_file(file, repo_root)} ({nf} test{'s' if nf != 1 else ''} failed)")+ if test_error_files:+ total_te = sum(s.get("errors", 0) for _, s in test_error_files)+ print(f"=== {len(test_error_files)} file{'s' if len(test_error_files) != 1 else ''} with pytest ERRORs (fixture setup/teardown or collection) ({total_te} error{'s' if total_te != 1 else ''}; grep '^ERROR ' in the failure output) ===")+ for file, s in test_error_files:+ ne = s.get("errors", 0)+ print(f" {_format_file(file, repo_root)} ({s.get('passed', 0)} passed, {ne} error{'s' if ne != 1 else ''})")if all_passed_but_nonzero:print(f"=== {len(all_passed_but_nonzero)} file{'s' if len(all_passed_but_nonzero) != 1 else ''} where all tests passed but pytest exited non-zero (warnings-as-errors, hook failures, etc.) ===")for file, s in all_passed_but_nonzero:@@ -1470,6 +1483,10 @@ def main() -> int:tests_failed = 0tests_skipped = 0skipped_files: list[tuple[Path, int]] = []+ # pytest setup/teardown ERRORs ("41 passed, 7 errors"). Not failures, but+ # they make pytest exit non-zero, so the Summary line must name them or it+ # reads "0 failed" under an exit code of 1.+ tests_errored = 0# Every collected outcome, not just pass/fail: a legitimately all-skipped# (platform-gated) file reports "2 skipped" and must NOT trip the# nothing-ran guard, whereas a file that died before collection reports@@ -1480,7 +1497,7 @@ def main() -> int:def _on_done(file: Path, started_at: float, fut: Future[tuple[Path, int, str, dict[str, int], float]]) -> None:nonlocal files_done, tests_done, pass_count, fail_count, tests_passed, tests_failed, tests_skipped- nonlocal tests_collected, files_crashed+ nonlocal tests_collected, files_crashed, tests_erroredn_tests = test_counts.get(file, 0)try:fpath, rc, output, summary, subproc_wall = fut.result()@@ -1507,6 +1524,7 @@ def main() -> int:tests_skipped += summary.get("skipped", 0)if summary.get("skipped", 0):skipped_files.append((fpath, summary["skipped"]))+ tests_errored += summary.get("errors", 0)files_crashed += summary.get("crashed", 0)tests_collected += sum(summary.get(k, 0)@@ -1566,7 +1584,13 @@ def main() -> int:f", {files_crashed} file{'s' if files_crashed != 1 else ''} CRASHED"if files_crashed else "")- print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{crashed_note}{skipped_note} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")+ # Setup/teardown ERRORs likewise have no failed-test count; without this+ # note a run with "41 passed, 7 errors" summarised as "0 failed" + exit 1.+ errors_note = (+ f", {tests_errored} error{'s' if tests_errored != 1 else ''}"+ if tests_errored else ""+ )+ print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{errors_note}{crashed_note}{skipped_note} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")# Host-OS gating note: tests marked for another OS were skipped by the# conftest hook, not run. Say so explicitly — a green local run on Linuxdiff --git a/tests/scripts/test_run_tests_parallel.py b/tests/scripts/test_run_tests_parallel.pyindex 262dcb047ca..865d4a900ca 100644--- a/tests/scripts/test_run_tests_parallel.py+++ b/tests/scripts/test_run_tests_parallel.py@@ -577,6 +577,74 @@ def test_interpreter_crash_is_reported_as_a_crash_not_as_no_tests_ran(tmp_path:assert "NO TESTS RAN" not in proc.stdout+def test_parse_summary_counts_setup_errors() -> None:+ """"41 passed, 7 errors" parses both counts (the input the Summary relies on)."""+ import importlib.util++ runner = Path(__file__).resolve().parents[2] / "scripts" / "run_tests_parallel.py"+ spec = importlib.util.spec_from_file_location("_rtp_errors_probe", runner)+ assert spec is not None and spec.loader is not None+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ fake = (+ "tests/tools/test_plugin_guard.py ....EEEEEEE\n"+ "ERROR tests/tools/test_plugin_guard.py::test_x - RuntimeError: boom\n"+ "======= 41 passed, 7 errors in 3.21s =======\n"+ )+ assert mod._parse_pytest_summary(fake) == {"passed": 41, "errors": 7}+++def test_setup_errors_are_named_on_the_summary_line(tmp_path: Path) -> None:+ """Setup ERRORs must appear on the Summary line, not hide behind "0 failed".++ A file whose tests all pass except for fixture-setup ERRORs exits 1, but+ the Summary used to read "N tests passed, 0 failed" and the file was+ filed under "all tests passed but pytest exited non-zero". The error+ count must be on the Summary line and in its own failure bucket.+ """+ probe_dir = tmp_path / "probe"+ probe_dir.mkdir()+ (probe_dir / "test_probe_setup_errors.py").write_text(+ textwrap.dedent(+ """+ import pytest++ @pytest.fixture+ def broken():+ raise RuntimeError("setup boom")++ def test_ok_one():+ assert True++ def test_ok_two():+ assert True++ @pytest.mark.parametrize("n", range(3))+ def test_needs_broken(broken, n):+ assert True+ """+ )+ )++ proc = _run_runner(probe_dir, "--file-retries", "0")++ assert proc.returncode == 1, proc.stdout+ summary = next(l for l in proc.stdout.splitlines() if "=== Summary:" in l)+ assert "2 tests passed, 0 failed, 3 errors" in summary, summary+ assert "with pytest ERRORs" in proc.stdout+ assert "(2 passed, 3 errors)" in proc.stdout+ assert "all tests passed but pytest exited non-zero" not in proc.stdout+ assert "where no tests ran" not in proc.stdout+++def test_clean_run_summary_has_no_errors_note(tmp_path: Path) -> None:+ probe_dir = _make_probe_dir(tmp_path)+ proc = _run_runner(probe_dir)+ assert proc.returncode == 0, proc.stdout+ summary = next(l for l in proc.stdout.splitlines() if "=== Summary:" in l)+ assert "error" not in summary, summary++# ── --files-from: file-backed explicit file lists ───────────────────────────## --files carries the whole list as ONE argv element, and Linux caps a