Skip to content

The agent that grows with you

fix(run_tests): count import/collection failures and missing named paths

Teknium97253cc182fd
diff --git a/scripts/run_tests_parallel.py b/scripts/run_tests_parallel.py
index 3ccb2cfc228..2d1d968aab5 100644
--- a/scripts/run_tests_parallel.py
+++ b/scripts/run_tests_parallel.py
@@ -57,7 +57,7 @@ import threading
import time
from concurrent.futures import ThreadPoolExecutor, Future
from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Callable, Dict, List, Optional, Tuple
# The CI lane selector owns the platforms() spec resolver; share it so the
# "skipped on this host" note and the lanes can never disagree.
@@ -268,6 +268,13 @@ def _select_files(
# were supplied, otherwise --paths (which itself defaults to 'tests').
if args.paths_positional:
roots = [repo_root / p for p in args.paths_positional]
+ # Discovery skips a root that does not exist, which is right for
+ # the --paths defaults but turns a mistyped or deleted file the
+ # caller NAMED into a green run that never tested it.
+ missing = [p for p, root in zip(args.paths_positional, roots) if not root.exists()]
+ if missing:
+ print(f"error: no such test path: {' '.join(missing)}", file=sys.stderr)
+ sys.exit(2)
else:
roots = [repo_root / p for p in _split_pathspec(args.paths)]
@@ -719,6 +726,8 @@ def _run_one_file_once(
# (venv without pytest, -k that matches nothing) can't report green.
rc = 0
summary = _parse_pytest_summary(output)
+ if rc != 0 and _COLLECTION_ERROR_RE.search(output):
+ summary["collect_error"] = 1
crash = _describe_interpreter_crash(rc, output) if rc != 0 else None
if crash:
# Same convention as the timeout path: the diagnosis leads the
@@ -1111,6 +1120,78 @@ def _report_skips(
return True
+def _plural(n: int, word: str) -> str:
+ return f"{n} {word}{'s' if n != 1 else ''}"
+
+
+# A file that dies while importing/collecting is interrupted by pytest with
+# exit 2 and ``collected 0 items / 1 error``: it has no failed-test count, so
+# it must be named as its own cause or a run reads "N passed, 0 failed".
+_COLLECTION_ERROR_RE = re.compile(
+ r"\d+ errors? during collection|^_+ ERROR collecting ", re.MULTILINE,
+)
+
+
+def _collection_error_reason(output: str) -> str:
+ """The one-line cause pytest gave for a file that failed to import/collect."""
+ # The traceback's last ``E`` line is whole; the short-summary ``ERROR``
+ # line is cut to the terminal width and only stands in under --tb=no.
+ lines = output.splitlines()
+ for prefix in ("E ", "ERROR "):
+ for line in reversed(lines):
+ if line.startswith(prefix):
+ return line[len(prefix):].strip()
+ return "(see the failure output above)"
+
+
+def _failure_cause(summary: dict[str, int]) -> str:
+ """Bucket key for one failing file, most specific diagnosis first."""
+ if summary.get("crashed"):
+ return "crashed"
+ if summary.get("collect_error"):
+ return "collect"
+ if summary.get("failed", 0):
+ return "failed"
+ if summary.get("errors", 0):
+ return "errors"
+ if summary.get("passed", 0):
+ return "nonzero"
+ return "none_ran"
+
+
+# cause -> (header after "=== N file(s) ", per-file detail(output, summary)),
+# printed in this order.
+_FAILURE_BUCKETS: dict[str, tuple[str, Callable[[str, dict[str, int]], str]]] = {
+ "crashed": (
+ "where the interpreter CRASHED mid-run (native fault — a real bug, not a "
+ "collection error; the tests that did run are not counted)",
+ lambda out, _s: out.splitlines()[0],
+ ),
+ "collect": (
+ "that FAILED AT IMPORT/COLLECTION (0 tests ran)",
+ lambda out, _s: _collection_error_reason(out),
+ ),
+ "failed": (
+ "with test failures",
+ lambda _o, s: f"({_plural(s.get('failed', 0), 'test')} failed)",
+ ),
+ # Setup/teardown ERRORs with no test failures: under "all tests passed but
+ # pytest exited non-zero" they read like a warnings-as-errors quirk.
+ "errors": (
+ "with pytest ERRORs (fixture setup/teardown; grep '^ERROR ' in the failure output)",
+ lambda _o, s: f"({s.get('passed', 0)} passed, {_plural(s.get('errors', 0), 'error')})",
+ ),
+ "nonzero": (
+ "where all tests passed but pytest exited non-zero (warnings-as-errors, hook failures, etc.)",
+ lambda _o, s: f"({s.get('passed', 0)} passed)",
+ ),
+ "none_ran": (
+ "where no tests ran (timeout before collection, pytest usage error, etc.)",
+ lambda _o, _s: "",
+ ),
+}
+
+
def _print_failure_report(
failures: list[tuple[Path, str, dict[str, int]]], repo_root: Path,
) -> None:
@@ -1122,45 +1203,44 @@ def _print_failure_report(
print(f"--- {_format_file(file, repo_root)} ---")
print(output.rstrip())
print()
- # Split: files with actual test failures vs non-zero exit for other reasons
- crashed_files = [(f, o, s) for f, o, s in failures if s.get("crashed")]
- rest = [(f, s) for f, _o, s in failures if not s.get("crashed")]
- test_fail_files = [(f, s) for f, s in rest if s.get("failed", 0) > 0]
- # Setup/teardown ERRORs with no test failures get their own bucket;
- # they used to land under "all tests passed but pytest exited
- # non-zero", which reads like a warnings-as-errors quirk.
- test_error_files = [(f, s) for f, s in rest
- if s.get("failed", 0) == 0 and s.get("errors", 0) > 0]
- all_passed_but_nonzero = [(f, s) for f, s in rest
- if s.get("failed", 0) == 0 and s.get("errors", 0) == 0
- and s.get("passed", 0) > 0]
- no_tests_ran = [(f, s) for f, s in rest
- if s.get("failed", 0) == 0 and s.get("errors", 0) == 0
- and s.get("passed", 0) == 0]
- if crashed_files:
- print(f"=== {len(crashed_files)} file{'s' if len(crashed_files) != 1 else ''} where the interpreter CRASHED mid-run (native fault — a real bug, not a collection error; the tests that did run are not counted) ===")
- for file, output, _s in crashed_files:
- print(f" {_format_file(file, repo_root)} {output.splitlines()[0]}")
- if test_fail_files:
- total_tf = sum(s.get("failed", 0) for _, s in test_fail_files)
- print(f"=== {len(test_fail_files)} file{'s' if len(test_fail_files) != 1 else ''} with test failures ({total_tf} test{'s' if total_tf != 1 else ''} failed) ===")
- for file, s in test_fail_files:
- nf = s.get("failed", 0)
- print(f" {_format_file(file, repo_root)} ({nf} test{'s' if nf != 1 else ''} failed)")
- if test_error_files:
- total_te = sum(s.get("errors", 0) for _, s in test_error_files)
- print(f"=== {len(test_error_files)} file{'s' if len(test_error_files) != 1 else ''} with pytest ERRORs (fixture setup/teardown or collection) ({total_te} error{'s' if total_te != 1 else ''}; grep '^ERROR ' in the failure output) ===")
- for file, s in test_error_files:
- ne = s.get("errors", 0)
- print(f" {_format_file(file, repo_root)} ({s.get('passed', 0)} passed, {ne} error{'s' if ne != 1 else ''})")
- if all_passed_but_nonzero:
- print(f"=== {len(all_passed_but_nonzero)} file{'s' if len(all_passed_but_nonzero) != 1 else ''} where all tests passed but pytest exited non-zero (warnings-as-errors, hook failures, etc.) ===")
- for file, s in all_passed_but_nonzero:
- print(f" {_format_file(file, repo_root)} ({s.get('passed', 0)} passed)")
- if no_tests_ran:
- print(f"=== {len(no_tests_ran)} file{'s' if len(no_tests_ran) != 1 else ''} where no tests ran (collection/import error, timeout before collection, etc.) ===")
- for file, s in no_tests_ran:
- print(f" {_format_file(file, repo_root)}")
+ by_cause: dict[str, list[tuple[Path, str, dict[str, int]]]] = {}
+ for failure in failures:
+ by_cause.setdefault(_failure_cause(failure[2]), []).append(failure)
+ for cause, (header, detail) in _FAILURE_BUCKETS.items():
+ group = by_cause.get(cause, [])
+ if not group:
+ continue
+ print(f"=== {_plural(len(group), 'file')} {header} ===")
+ for file, output, summary in group:
+ print(f" {_format_file(file, repo_root)} {detail(output, summary)}".rstrip())
+
+
+def _summary_notes(
+ failures: list[tuple[Path, str, dict[str, int]]],
+ total_files: int,
+ tests_errored: int,
+ files_crashed: int,
+ tests_skipped: int,
+) -> str:
+ """Everything the Summary line must say beyond passed/failed counts.
+
+ ERRORs, crashes and import/collection failures have no failed-test count;
+ without these notes a red run reads "N passed, 0 failed" on the one line
+ everyone reads, while the exit code says 1.
+ """
+ notes = ""
+ if tests_errored:
+ notes += f", {_plural(tests_errored, 'error')}"
+ if files_crashed:
+ notes += f", {_plural(files_crashed, 'file')} CRASHED"
+ if tests_skipped:
+ notes += f", {tests_skipped} skipped"
+ if failures:
+ collect = sum(1 for _f, _o, s in failures if s.get("collect_error"))
+ notes += f"; FAILED: {len(failures)} of {_plural(total_files, 'file')}"
+ if collect:
+ notes += f" ({collect} at import/collection)"
+ return notes
def main() -> int:
@@ -1577,20 +1657,8 @@ def main() -> int:
elapsed = time.monotonic() - started
print()
pct = min(100, (tests_done / approx_total_tests * 100)) if approx_total_tests else 0
- skipped_note = f", {tests_skipped} skipped" if tests_skipped else ""
- # A crashed interpreter has no failed-test count; say so on the one line
- # everyone reads, or "0 failed" + exit 1 looks like a runner bug.
- crashed_note = (
- f", {files_crashed} file{'s' if files_crashed != 1 else ''} CRASHED"
- if files_crashed else ""
- )
- # Setup/teardown ERRORs likewise have no failed-test count; without this
- # note a run with "41 passed, 7 errors" summarised as "0 failed" + exit 1.
- errors_note = (
- f", {tests_errored} error{'s' if tests_errored != 1 else ''}"
- if tests_errored else ""
- )
- print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{errors_note}{crashed_note}{skipped_note} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")
+ notes = _summary_notes(failures, len(files), tests_errored, files_crashed, tests_skipped)
+ print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed{notes} ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===")
# Host-OS gating note: tests marked for another OS were skipped by the
# conftest hook, not run. Say so explicitly — a green local run on Linux
diff --git a/tests/scripts/test_run_tests_parallel.py b/tests/scripts/test_run_tests_parallel.py
index 865d4a900ca..41e61da4663 100644
--- a/tests/scripts/test_run_tests_parallel.py
+++ b/tests/scripts/test_run_tests_parallel.py
@@ -645,6 +645,52 @@ def test_clean_run_summary_has_no_errors_note(tmp_path: Path) -> None:
assert "error" not in summary, summary
+def test_import_error_file_is_a_failed_file_on_the_summary_line(tmp_path: Path) -> None:
+ """A file that dies at import is a FAILED file on the Summary line, with its cause.
+
+ pytest interrupts it with exit 2 and ``collected 0 items / 1 error``, so
+ it has no failed-test count: the Summary used to read "2 tests passed,
+ 0 failed" beside it. A file that collects nothing (pytest exit 5, e.g.
+ platform-gated) stays a pass.
+ """
+ probe_dir = _make_probe_dir(tmp_path)
+ (probe_dir / "test_probe_import_error.py").write_text(
+ "@pytest.mark.parametrize('n', [1])\ndef test_n(n):\n assert n\n"
+ )
+ (probe_dir / "test_probe_collects_nothing.py").write_text("VALUE = 1\n")
+
+ proc = _run_runner(probe_dir, "--file-retries", "0")
+
+ assert proc.returncode == 1, proc.stdout
+ summary = next(l for l in proc.stdout.splitlines() if "=== Summary:" in l)
+ assert "2 tests passed, 0 failed, 1 error" in summary, summary
+ assert "FAILED: 1 of 3 files (1 at import/collection)" in summary, summary
+ lines = proc.stdout.splitlines()
+ header = lines.index("=== 1 file that FAILED AT IMPORT/COLLECTION (0 tests ran) ===")
+ cause = lines[header + 1]
+ assert "test_probe_import_error.py" in cause, proc.stdout
+ assert "NameError: name 'pytest' is not defined" in cause, proc.stdout
+ assert "where no tests ran" not in proc.stdout
+
+
+def test_named_test_path_that_does_not_exist_fails_the_run(tmp_path: Path) -> None:
+ """A mistyped or deleted file named on the command line must not run green without it."""
+ probe_dir = _make_probe_dir(tmp_path)
+ runner = _probe_root(tmp_path) / "scripts" / "run_tests_parallel.py"
+ missing = probe_dir / "test_probe_not_there.py"
+
+ proc = subprocess.run(
+ [sys.executable, str(runner), str(probe_dir / "test_flagprobe.py"), str(missing),
+ "-j", "1", "--file-timeout", "30"],
+ cwd=probe_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ encoding="utf-8", errors="replace", timeout=60, check=False,
+ )
+
+ assert proc.returncode == 2, proc.stdout
+ assert f"no such test path: {missing}" in proc.stdout
+ assert "=== Summary:" not in proc.stdout
+
+
# ── --files-from: file-backed explicit file lists ───────────────────────────
#
# --files carries the whole list as ONE argv element, and Linux caps a