The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(run_tests): count import/collection failures and missing named paths
Teknium97253cc182fd
diff --git a/scripts/run_tests_parallel.py b/scripts/run_tests_parallel.pyindex 3ccb2cfc228..2d1d968aab5 100644--- a/scripts/run_tests_parallel.py+++ b/scripts/run_tests_parallel.py@@ -57,7 +57,7 @@ import threadingimport timefrom concurrent.futures import ThreadPoolExecutor, Futurefrom pathlib import Path-from typing import Dict, List, Optional, Tuple+from typing import Callable, Dict, List, Optional, Tuple# The CI lane selector owns the platforms() spec resolver; share it so the# "skipped on this host" note and the lanes can never disagree.@@ -268,6 +268,13 @@ def _select_files(# were supplied, otherwise --paths (which itself defaults to 'tests').if args.paths_positional:roots = [repo_root / p for p in args.paths_positional]+ # Discovery skips a root that does not exist, which is right for+ # the --paths defaults but turns a mistyped or deleted file the+ # caller NAMED into a green run that never tested it.+ missing = [p for p, root in zip(args.paths_positional, roots) if not root.exists()]+ if missing:+ print(f"error: no such test path: {' '.join(missing)}", file=sys.stderr)+ sys.exit(2)else:roots = [repo_root / p for p in _split_pathspec(args.paths)]@@ -719,6 +726,8 @@ def _run_one_file_once(# (venv without pytest, -k that matches nothing) can't report green.rc = 0summary = _parse_pytest_summary(output)+ if rc != 0 and _COLLECTION_ERROR_RE.search(output):+ summary["collect_error"] = 1crash = _describe_interpreter_crash(rc, output) if rc != 0 else Noneif crash:# Same convention as the timeout path: the diagnosis leads the@@ -1111,6 +1120,78 @@ def _report_skips(return True+def _plural(n: int, word: str) -> str:+ return f"{n} {word}{'s' if n != 1 else ''}"+++# A file that dies while importing/collecting is interrupted by pytest with+# exit 2 and ``collected 0 items / 1 error``: it has no failed-test count, so+# it must be named as its own cause or a run reads "N passed, 0 failed".+_COLLECTION_ERROR_RE = re.compile(+ r"\d+ errors? during collection|^_+ ERROR collecting ", re.MULTILINE,+)+++def _collection_error_reason(output: str) -> str:+ """The one-line cause pytest gave for a file that failed to import/collect."""+ # The traceback's last ``E`` line is whole; the short-summary ``ERROR``+ # line is cut to the terminal width and only stands in under --tb=no.+ lines = output.splitlines()+ for prefix in ("E ", "ERROR "):+ for line in reversed(lines):+ if line.startswith(prefix):+ return line[len(prefix):].strip()+ return "(see the failure output above)"+++def _failure_cause(summary: dict[str, int]) -> str:+ """Bucket key for one failing file, most specific diagnosis first."""+ if summary.get("crashed"):+ return "crashed"+ if summary.get("collect_error"):+ return "collect"+ if summary.get("failed", 0):+ return "failed"+ if summary.get("errors", 0):+ return "errors"+ if summary.get("passed", 0):+ return "nonzero"+ return "none_ran"+++# cause -> (header after "=== N file(s) ", per-file detail(output, summary)),+# printed in this order.+_FAILURE_BUCKETS: dict[str, tuple[str, Callable[[str, dict[str, int]], str]]] = {+ "crashed": (+ "where the interpreter CRASHED mid-run (native fault — a real bug, not a "+ "collection error; the tests that did run are not counted)",+ lambda out, _s: out.splitlines()[0],+ ),+ "collect": (+ "that FAILED AT IMPORT/COLLECTION (0 tests ran)",+ lambda out, _s: _collection_error_reason(out),+ ),+ "failed": (+ "with test failures",+ lambda _o, s: f"({_plural(s.get('failed', 0), 'test')} failed)",+ ),+ # Setup/teardown ERRORs with no test failures: under "all tests passed but+ # pytest exited non-zero" they read like a warnings-as-errors quirk.+ "errors": (+ "with pytest ERRORs (fixture setup/teardown; grep '^ERROR ' in the failure output)",+ lambda _o, s: f"({s.get('passed', 0)} passed, {_plural(s.get('errors', 0), 'error')})",+ ),+ "nonzero": (+ "where all tests passed but pytest exited non-zero (warnings-as-errors, hook failures, etc.)",+ lambda _o, s: f"({s.get('passed', 0)} passed)",+ ),+ "none_ran": (+ "where no tests ran (timeout before collection, pytest usage error, etc.)",+ lambda _o, _s: "",+ ),+}++def _print_failure_report(failures: list[tuple[Path, str, dict[str, int]]], repo_root: Path,) -> None:@@ -1122,45 +1203,44 @@ def _print_failure_report(print(f"--- {_format_file(file, repo_root)} ---")print(output.rstrip())print()- # Split: files with actual test failures vs non-zero exit for other reasons- crashed_files = [(f, o, s) for f, o, s in failures if s.get("crashed")]- rest = [(f, s) for f, _o, s in failures if not s.get("crashed")]- test_fail_files = [(f, s) for f, s in rest if s.get("failed", 0) > 0]- # Setup/teardown ERRORs with no test failures get their own bucket;- # they used to land under "all tests passed but pytest exited- # non-zero", which reads like a warnings-as-errors quirk.- test_error_files = [(f, s) for f, s in rest- if s.get("failed", 0) == 0 and s.get("errors", 0) > 0]- all_passed_but_nonzero = [(f, s) for f, s in rest- if s.get("failed", 0) == 0 and s.get("errors", 0) == 0- and s.get("passed", 0) > 0]- no_tests_ran = [(f, s) for f, s in rest- if s.get("failed", 0) == 0 and s.get("errors", 0) == 0- and s.get("passed", 0) == 0]- if crashed_files:- print(f"=== {len(crashed_files)} file{'s' if len(crashed_files) != 1 else ''} where the interpreter CRASHED mid-run (native fault — a real bug, not a collection error; the tests that did run are not counted) ===")- for file, output, _s in crashed_files:- print(f" {_format_file(file, repo_root)} {output.splitlines()[0]}")- if test_fail_files:- total_tf = sum(s.get("failed", 0) for _, s in test_fail_files)- print(f"=== {len(test_fail_files)} file{'s' if len(test_fail_files) != 1 else ''} with test failures ({total_tf} test{'s' if total_tf != 1 else ''} failed) ===")- for file, s in test_fail_files:- nf = s.get("failed", 0)- print(f" {_format_file(file, repo_root)} ({nf} test{'s' if nf != 1 else ''} failed)")- if test_error_files:- total_te = sum(s.get("errors", 0) for _, s in test_error_files)- print(f"=== {len(test_error_files)} file{'s' if len(test_error_files) != 1 else ''} with pytest ERRORs (fixture setup/teardown or collection) ({total_te} error{'s' if total_te != 1 else ''}; grep '^ERROR ' in the failure output) ===")- for file, s in test_error_files:- ne = s.get("errors", 0)- print(f" {_format_file(file, repo_root)} ({s.get('passed', 0)} passed, {ne} error{'s' if ne != 1 else ''})")- if all_passed_but_nonzero:- print(f"=== {len(all_passed_but_nonzero)} file{'s' if len(all_passed_but_nonzero) != 1 else ''} where all tests passed but pytest exited non-zero (warnings-as-errors, hook failures, etc.) ===")- for file, s in all_passed_but_nonzero:- print(f" {_format_file(file, repo_root)} ({s.get('passed', 0)} passed)")- if no_tests_ran:- print(f"=== {len(no_tests_ran)} file{'s' if len(no_tests_ran) != 1 else ''} where no tests ran (collection/import error, timeout before collection, etc.) ===")- for file, s in no_tests_ran:- print(f" {_format_file(file, repo_root)}")+ by_cause: dict[str, list[tuple[Path, str, dict[str, int]]]] = {}+ for failure in failures:+ by_cause.setdefault(_failure_cause(failure[2]), []).append(failure)+ for cause, (header, detail) in _FAILURE_BUCKETS.items():+ group = by_cause.get(cause, [])+ if not group:+ continue+ print(f"=== {_plural(len(group), 'file')} {header} ===")+ for file, output, summary in group:+ print(f" {_format_file(file, repo_root)} {detail(output, summary)}".rstrip())+++def _summary_notes(+ failures: list[tuple[Path, str, dict[str, int]]],+ total_files: int,+ tests_errored: int,+ files_crashed: int,+ tests_skipped: int,+) -> str:+ """Everything the Summary line must say beyond passed/failed counts.++ ERRORs, crashes and import/collection failures have no failed-test count;+ without these notes a red run reads "N passed, 0 failed" on the one line+ everyone reads, while the exit code says 1.+ """+ notes = ""+ if tests_errored:+ notes += f", {_plural(tests_errored, 'error')}"+ if files_crashed:+ notes += f", {_plural(files_crashed, 'file')} CRASHED"+ if tests_skipped:+ notes += f", {tests_skipped} skipped"+ if failures:+ collect = sum(1 for _f, _o, s in failures if s.get("collect_error"))+ notes += f"; FAILED: {len(failures)} of {_plural(total_files, 'file')}"+ if collect:+ notes += f" ({collect} at import/collection)"+ return notesdef main() -> int:@@ -1577,20 +1657,8 @@ def main() -> int:elapsed = time.monotonic() - startedprint()pct = min(100, (tests_done / approx_total_tests * 100)) if approx_total_tests else 0- skipped_note = f", {tests_skipped} skipped" if tests_skipped else ""- # A crashed interpreter has no failed-test count; say so on the one line- # everyone reads, or "0 failed" + exit 1 looks like a runner bug.- crashed_note = (- f", {files_crashed} file{'s' if files_crashed != 1 else ''} CRASHED"- if files_crashed else ""- )- # Setup/teardown ERRORs likewise have no failed-test count; without this- # note a run with "41 passed, 7 errors" summarised as "0 failed" + exit 1.- errors_note = (- f", {tests_errored} error{'s' if tests_errored != 1 else ''}"- if tests_errored else ""- )- print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{errors_note}{crashed_note}{skipped_note} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")+ notes = _summary_notes(failures, len(files), tests_errored, files_crashed, tests_skipped)+ print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{notes} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")# Host-OS gating note: tests marked for another OS were skipped by the# conftest hook, not run. Say so explicitly — a green local run on Linuxdiff --git a/tests/scripts/test_run_tests_parallel.py b/tests/scripts/test_run_tests_parallel.pyindex 865d4a900ca..41e61da4663 100644--- a/tests/scripts/test_run_tests_parallel.py+++ b/tests/scripts/test_run_tests_parallel.py@@ -645,6 +645,52 @@ def test_clean_run_summary_has_no_errors_note(tmp_path: Path) -> None:assert "error" not in summary, summary+def test_import_error_file_is_a_failed_file_on_the_summary_line(tmp_path: Path) -> None:+ """A file that dies at import is a FAILED file on the Summary line, with its cause.++ pytest interrupts it with exit 2 and ``collected 0 items / 1 error``, so+ it has no failed-test count: the Summary used to read "2 tests passed,+ 0 failed" beside it. A file that collects nothing (pytest exit 5, e.g.+ platform-gated) stays a pass.+ """+ probe_dir = _make_probe_dir(tmp_path)+ (probe_dir / "test_probe_import_error.py").write_text(+ "@pytest.mark.parametrize('n', [1])\ndef test_n(n):\n assert n\n"+ )+ (probe_dir / "test_probe_collects_nothing.py").write_text("VALUE = 1\n")++ proc = _run_runner(probe_dir, "--file-retries", "0")++ assert proc.returncode == 1, proc.stdout+ summary = next(l for l in proc.stdout.splitlines() if "=== Summary:" in l)+ assert "2 tests passed, 0 failed, 1 error" in summary, summary+ assert "FAILED: 1 of 3 files (1 at import/collection)" in summary, summary+ lines = proc.stdout.splitlines()+ header = lines.index("=== 1 file that FAILED AT IMPORT/COLLECTION (0 tests ran) ===")+ cause = lines[header + 1]+ assert "test_probe_import_error.py" in cause, proc.stdout+ assert "NameError: name 'pytest' is not defined" in cause, proc.stdout+ assert "where no tests ran" not in proc.stdout+++def test_named_test_path_that_does_not_exist_fails_the_run(tmp_path: Path) -> None:+ """A mistyped or deleted file named on the command line must not run green without it."""+ probe_dir = _make_probe_dir(tmp_path)+ runner = _probe_root(tmp_path) / "scripts" / "run_tests_parallel.py"+ missing = probe_dir / "test_probe_not_there.py"++ proc = subprocess.run(+ [sys.executable, str(runner), str(probe_dir / "test_flagprobe.py"), str(missing),+ "-j", "1", "--file-timeout", "30"],+ cwd=probe_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,+ encoding="utf-8", errors="replace", timeout=60, check=False,+ )++ assert proc.returncode == 2, proc.stdout+ assert f"no such test path: {missing}" in proc.stdout+ assert "=== Summary:" not in proc.stdout++# ── --files-from: file-backed explicit file lists ───────────────────────────## --files carries the whole list as ONE argv element, and Linux caps a