Skip to content

The agent that grows with you

fix(run_tests_parallel): name pytest ERRORs on the Summary line

Byrd-ITcd424473d15f
diff --git a/scripts/run_tests_parallel.py b/scripts/run_tests_parallel.py
index 521a95eb647..3ccb2cfc228 100644
--- a/scripts/run_tests_parallel.py
+++ b/scripts/run_tests_parallel.py
@@ -1126,10 +1126,17 @@ def _print_failure_report(
crashed_files = [(f, o, s) for f, o, s in failures if s.get("crashed")]
rest = [(f, s) for f, _o, s in failures if not s.get("crashed")]
test_fail_files = [(f, s) for f, s in rest if s.get("failed", 0) > 0]
+ # Setup/teardown ERRORs with no test failures get their own bucket;
+ # they used to land under "all tests passed but pytest exited
+ # non-zero", which reads like a warnings-as-errors quirk.
+ test_error_files = [(f, s) for f, s in rest
+ if s.get("failed", 0) == 0 and s.get("errors", 0) > 0]
all_passed_but_nonzero = [(f, s) for f, s in rest
- if s.get("failed", 0) == 0 and s.get("passed", 0) > 0]
+ if s.get("failed", 0) == 0 and s.get("errors", 0) == 0
+ and s.get("passed", 0) > 0]
no_tests_ran = [(f, s) for f, s in rest
- if s.get("failed", 0) == 0 and s.get("passed", 0) == 0]
+ if s.get("failed", 0) == 0 and s.get("errors", 0) == 0
+ and s.get("passed", 0) == 0]
if crashed_files:
print(f"=== {len(crashed_files)} file{'s' if len(crashed_files) != 1 else ''} where the interpreter CRASHED mid-run (native fault — a real bug, not a collection error; the tests that did run are not counted) ===")
for file, output, _s in crashed_files:
@@ -1140,6 +1147,12 @@ def _print_failure_report(
for file, s in test_fail_files:
nf = s.get("failed", 0)
print(f" {_format_file(file, repo_root)} ({nf} test{'s' if nf != 1 else ''} failed)")
+ if test_error_files:
+ total_te = sum(s.get("errors", 0) for _, s in test_error_files)
+ print(f"=== {len(test_error_files)} file{'s' if len(test_error_files) != 1 else ''} with pytest ERRORs (fixture setup/teardown or collection) ({total_te} error{'s' if total_te != 1 else ''}; grep '^ERROR ' in the failure output) ===")
+ for file, s in test_error_files:
+ ne = s.get("errors", 0)
+ print(f" {_format_file(file, repo_root)} ({s.get('passed', 0)} passed, {ne} error{'s' if ne != 1 else ''})")
if all_passed_but_nonzero:
print(f"=== {len(all_passed_but_nonzero)} file{'s' if len(all_passed_but_nonzero) != 1 else ''} where all tests passed but pytest exited non-zero (warnings-as-errors, hook failures, etc.) ===")
for file, s in all_passed_but_nonzero:
@@ -1470,6 +1483,10 @@ def main() -> int:
tests_failed = 0
tests_skipped = 0
skipped_files: list[tuple[Path, int]] = []
+ # pytest setup/teardown ERRORs ("41 passed, 7 errors"). Not failures, but
+ # they make pytest exit non-zero, so the Summary line must name them or it
+ # reads "0 failed" under an exit code of 1.
+ tests_errored = 0
# Every collected outcome, not just pass/fail: a legitimately all-skipped
# (platform-gated) file reports "2 skipped" and must NOT trip the
# nothing-ran guard, whereas a file that died before collection reports
@@ -1480,7 +1497,7 @@ def main() -> int:
def _on_done(file: Path, started_at: float, fut: Future[tuple[Path, int, str, dict[str, int], float]]) -> None:
nonlocal files_done, tests_done, pass_count, fail_count, tests_passed, tests_failed, tests_skipped
- nonlocal tests_collected, files_crashed
+ nonlocal tests_collected, files_crashed, tests_errored
n_tests = test_counts.get(file, 0)
try:
fpath, rc, output, summary, subproc_wall = fut.result()
@@ -1507,6 +1524,7 @@ def main() -> int:
tests_skipped += summary.get("skipped", 0)
if summary.get("skipped", 0):
skipped_files.append((fpath, summary["skipped"]))
+ tests_errored += summary.get("errors", 0)
files_crashed += summary.get("crashed", 0)
tests_collected += sum(
summary.get(k, 0)
@@ -1566,7 +1584,13 @@ def main() -> int:
f", {files_crashed} file{'s' if files_crashed != 1 else ''} CRASHED"
if files_crashed else ""
)
- print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{crashed_note}{skipped_note} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")
+ # Setup/teardown ERRORs likewise have no failed-test count; without this
+ # note a run with "41 passed, 7 errors" summarised as "0 failed" + exit 1.
+ errors_note = (
+ f", {tests_errored} error{'s' if tests_errored != 1 else ''}"
+ if tests_errored else ""
+ )
+ print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{errors_note}{crashed_note}{skipped_note} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")
# Host-OS gating note: tests marked for another OS were skipped by the
# conftest hook, not run. Say so explicitly — a green local run on Linux
diff --git a/tests/scripts/test_run_tests_parallel.py b/tests/scripts/test_run_tests_parallel.py
index 262dcb047ca..865d4a900ca 100644
--- a/tests/scripts/test_run_tests_parallel.py
+++ b/tests/scripts/test_run_tests_parallel.py
@@ -577,6 +577,74 @@ def test_interpreter_crash_is_reported_as_a_crash_not_as_no_tests_ran(tmp_path:
assert "NO TESTS RAN" not in proc.stdout
+def test_parse_summary_counts_setup_errors() -> None:
+ """"41 passed, 7 errors" parses both counts (the input the Summary relies on)."""
+ import importlib.util
+
+ runner = Path(__file__).resolve().parents[2] / "scripts" / "run_tests_parallel.py"
+ spec = importlib.util.spec_from_file_location("_rtp_errors_probe", runner)
+ assert spec is not None and spec.loader is not None
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ fake = (
+ "tests/tools/test_plugin_guard.py ....EEEEEEE\n"
+ "ERROR tests/tools/test_plugin_guard.py::test_x - RuntimeError: boom\n"
+ "======= 41 passed, 7 errors in 3.21s =======\n"
+ )
+ assert mod._parse_pytest_summary(fake) == {"passed": 41, "errors": 7}
+
+
+def test_setup_errors_are_named_on_the_summary_line(tmp_path: Path) -> None:
+ """Setup ERRORs must appear on the Summary line, not hide behind "0 failed".
+
+ A file whose tests all pass except for fixture-setup ERRORs exits 1, but
+ the Summary used to read "N tests passed, 0 failed" and the file was
+ filed under "all tests passed but pytest exited non-zero". The error
+ count must be on the Summary line and in its own failure bucket.
+ """
+ probe_dir = tmp_path / "probe"
+ probe_dir.mkdir()
+ (probe_dir / "test_probe_setup_errors.py").write_text(
+ textwrap.dedent(
+ """
+ import pytest
+
+ @pytest.fixture
+ def broken():
+ raise RuntimeError("setup boom")
+
+ def test_ok_one():
+ assert True
+
+ def test_ok_two():
+ assert True
+
+ @pytest.mark.parametrize("n", range(3))
+ def test_needs_broken(broken, n):
+ assert True
+ """
+ )
+ )
+
+ proc = _run_runner(probe_dir, "--file-retries", "0")
+
+ assert proc.returncode == 1, proc.stdout
+ summary = next(l for l in proc.stdout.splitlines() if "=== Summary:" in l)
+ assert "2 tests passed, 0 failed, 3 errors" in summary, summary
+ assert "with pytest ERRORs" in proc.stdout
+ assert "(2 passed, 3 errors)" in proc.stdout
+ assert "all tests passed but pytest exited non-zero" not in proc.stdout
+ assert "where no tests ran" not in proc.stdout
+
+
+def test_clean_run_summary_has_no_errors_note(tmp_path: Path) -> None:
+ probe_dir = _make_probe_dir(tmp_path)
+ proc = _run_runner(probe_dir)
+ assert proc.returncode == 0, proc.stdout
+ summary = next(l for l in proc.stdout.splitlines() if "=== Summary:" in l)
+ assert "error" not in summary, summary
+
+
# ── --files-from: file-backed explicit file lists ───────────────────────────
#
# --files carries the whole list as ONE argv element, and Linux caps a