Skip to content
Python
5.1 KB
Raw Blame
#!/usr/bin/env python3
"""Report review signals in the Core test suite without judging test value."""

from __future__ import annotations

import argparse
import ast
import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any

TIMING_HOTSPOT_SECONDS = 5.0


def _candidate(kind: str, path: Path, **details: Any) -> dict[str, Any]:
    return {"kind": kind, "file": path.as_posix(), **details}


def source_candidates(tests_root: Path) -> tuple[list[dict[str, Any]], dict[str, int]]:
    candidates: list[dict[str, Any]] = []
    names: dict[str, list[Path]] = defaultdict(list)
    for path in sorted(tests_root.glob("test_*.py")):
        source = path.read_text(encoding="utf-8")
        tree = ast.parse(source, path.name)
        for node in ast.walk(tree):
            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith(
                "test_"
            ):
                names[node.name].append(path)
                if "legacy" in node.name.lower():
                    candidates.append(_candidate("legacy_name", path, test=node.name))
            if (
                isinstance(node, ast.Call)
                and isinstance(node.func, ast.Attribute)
                and node.func.attr == "sleep"
                and node.args
                and isinstance(node.args[0], ast.Constant)
                and isinstance(node.args[0].value, (int, float))
                and node.args[0].value > 0
            ):
                candidates.append(
                    _candidate(
                        "fixed_sleep",
                        path,
                        line=node.lineno,
                        seconds=float(node.args[0].value),
                    )
                )
        if "pytest.mark.xfail" in source:
            candidates.append(_candidate("xfail", path))
    for name, paths in sorted(names.items()):
        unique = sorted({path.as_posix() for path in paths})
        if len(unique) > 1:
            candidates.append(
                {
                    "kind": "duplicate_test_name",
                    "test": name,
                    "files": unique,
                }
            )
    candidates.sort(
        key=lambda item: (
            str(item["kind"]),
            str(item.get("file", "")),
            str(item.get("test", "")),
        )
    )
    return candidates, dict(sorted(Counter(item["kind"] for item in candidates).items()))


def timing_candidates(report_dir: Path) -> tuple[list[dict[str, Any]], dict[str, int]]:
    path = report_dir / "tests.jsonl"
    if not path.is_file():
        return [], {}
    rows = [
        json.loads(line)
        for line in path.read_text(encoding="utf-8").splitlines()
        if line.strip()
    ]
    candidates: list[dict[str, Any]] = []
    by_file: dict[str, list[dict[str, Any]]] = defaultdict(list)
    for row in rows:
        file = str(row.get("file", ""))
        if file:
            by_file[file].append(row)
        outcome = row.get("outcome")
        if outcome == "xfailed":
            candidates.append(
                {
                    "kind": "xfailed",
                    "file": file,
                    "nodeid": row.get("nodeid"),
                }
            )
        total = row.get("total_seconds")
        if isinstance(total, (int, float)) and total >= TIMING_HOTSPOT_SECONDS:
            candidates.append(
                {
                    "kind": "timing_hotspot",
                    "file": file,
                    "nodeid": row.get("nodeid"),
                    "total_seconds": total,
                }
            )
    for file, file_rows in sorted(by_file.items()):
        executed = [row for row in file_rows if row.get("outcome") != "not_run"]
        if executed and all(row.get("outcome") == "skipped" for row in executed):
            candidates.append({"kind": "all_skipped_file", "file": file})
    candidates.sort(
        key=lambda item: (
            str(item["kind"]),
            str(item.get("file", "")),
            str(item.get("nodeid", "")),
        )
    )
    return candidates, dict(sorted(Counter(item["kind"] for item in candidates).items()))


def audit(report_dir: Path, tests_root: Path) -> dict[str, Any]:
    source, source_counts = source_candidates(tests_root)
    timing, timing_counts = timing_candidates(report_dir)
    result = {
        "schema_version": 1,
        "source_signals": source_counts,
        "timing_signals": timing_counts,
        "candidates": [*source, *timing],
    }
    report_dir.mkdir(parents=True, exist_ok=True)
    (report_dir / "design-audit.json").write_text(
        json.dumps(result, indent=2, sort_keys=True) + "\n",
        encoding="utf-8",
    )
    return result


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("report_dir", type=Path)
    parser.add_argument("tests_root", type=Path)
    args = parser.parse_args()
    result = audit(args.report_dir.resolve(), args.tests_root.resolve())
    print(
        "Core test design signals: "
        f"source={result['source_signals']} timing={result['timing_signals']}"
    )


if __name__ == "__main__":
    main()