Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(selection): move the impacted-tests collection walk into core (#3292)

Raghav Chamadiya5cd12cc3df05
diff --git a/packages/cli/src/repowise/cli/commands/impacted_tests_cmd.py b/packages/cli/src/repowise/cli/commands/impacted_tests_cmd.py
index 7275b808b..3e7711172 100644
--- a/packages/cli/src/repowise/cli/commands/impacted_tests_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/impacted_tests_cmd.py
@@ -30,12 +30,9 @@ Examples:
from __future__ import annotations
-import sys
from pathlib import Path
-from typing import TYPE_CHECKING, NamedTuple
import click
-import structlog
from rich.table import Table
from repowise.cli.ci import (
@@ -55,18 +52,9 @@ from repowise.cli.helpers import (
silence_logs_for_machine_output_until_close,
)
from repowise.cli.output import emit_json
+from repowise.core.analysis.test_collection import plan_scopes, read_checkout, select
from repowise.core.analysis.test_selection import RUNNERS
-if TYPE_CHECKING:
- from repowise.core.pytest_roots import PytestRoots
-
-log = structlog.get_logger(__name__)
-
-# The whole reverse-import closure: a test importing a module that imports the
-# changed file runs it too. No depth limit: the walk ends when no node gains a
-# new seed, so a cut-off can never drop a test.
-_IMPORT_CLOSURE_DEPTH = sys.maxsize
-
# Candidates the table prints per changed file; json and list carry them all.
_TABLE_CANDIDATES = 10
@@ -142,23 +130,12 @@ def impacted_tests_command(
cannot_evaluate(fmt, exc.code, str(exc))
explain = explain.replace("\\", "/").removeprefix("./") if explain else None
- checkout = _read_checkout(repo_path)
- plan = _plan_scopes(repo_path, change, config, checkout) if config is not None else None
- routes = plan.routes if plan else []
- result = run_async(
- _collect(
- repo_path,
- change,
- checkout.roots,
- config,
- explain,
- routes,
- dict(checkout.pytest_texts),
- )
- )
+ checkout = read_checkout(repo_path)
+ plan = plan_scopes(change, config, checkout) if config is not None else None
+ result = run_async(_collect(repo_path, change, checkout, config, explain, plan))
result["diff"] = change.label
if plan is not None:
- result["selection"] = _select(repo_path, change, result, config, checkout, plan)
+ result["selection"] = select(change, result, config, checkout, plan)
if explain and fmt != "json":
_render_explain(result, explain)
return
@@ -186,539 +163,32 @@ def _selection_config(repo_path):
raise CannotEvaluateError("config_invalid", str(exc)) from exc
-class _Checkout(NamedTuple):
- """Tracked paths and the pytest files among them, each read once per run."""
-
- tracked: list[str]
- pytest_texts: list[tuple[str, str]]
- roots: PytestRoots
-
-
-def _read_checkout(repo_path) -> _Checkout:
- """Tracked files, their conftests and pytest configs, and pytest's collection roots."""
- from repowise.core import git_refs
- from repowise.core.analysis.test_selection import is_scan_source
- from repowise.core.pytest_roots import PYTEST_CONFIG_NAMES, read_pytest_roots
-
- root = Path(repo_path)
- tracked = git_refs.tracked_paths(str(root))
- texts = list(_texts(root, [p for p in tracked if is_scan_source(p)], pytest_only=True))
- roots = read_pytest_roots((p, t) for p, t in texts if Path(p).name in PYTEST_CONFIG_NAMES)
- return _Checkout(tracked, texts, roots)
-
-
-class _Plan(NamedTuple):
- """Who names each changed doc or asset, the scope each scoped file runs, and its routes."""
-
- namers: dict[str, list[str]]
- scopes: dict
- routes: list[str]
-
-
-def _plan_scopes(repo_path, change, config, checkout: _Checkout) -> _Plan:
- """The change's scopes, reading the sources once and only when a file needs its namers."""
- from repowise.core.analysis.selection_scopes import (
- keeps_full_run,
- needs_namers,
- trigger_scopes,
- )
- from repowise.core.analysis.test_selection import file_namers, is_scan_source
-
- paths = [*change.files, *change.deleted]
- # A file that runs everything anyway makes every namer search moot.
- blocked = any(keeps_full_run(p, config) for p in paths)
- asked = [] if blocked else [p for p in paths if needs_namers(p, config)]
- namers: dict[str, list[str]] = {}
- if asked:
- known = dict(checkout.pytest_texts) # conftests and pytest configs, read already
- root = Path(repo_path)
- texts = (
- (p, known[p]) if p in known else (p, _text(root / p) or "")
- for p in checkout.tracked
- if is_scan_source(p)
- )
- namers = file_namers(asked, texts)
- scopes = trigger_scopes(paths, checkout.tracked, namers, config)
- routes = sorted({r for scope in scopes.values() for r in scope.routes} - set(paths))
- return _Plan(namers, scopes, routes)
-
-
-async def _collect(
- repo_path,
- change,
- roots: PytestRoots | None = None,
- config=None,
- explain: str | None = None,
- scope_routes: list[str] | tuple[str, ...] = (),
- pytest_texts: dict | None = None,
-) -> dict:
- """Resolve the change's files to impacted tests + labelled fallbacks.
-
- With a selection *config*, files changed since the index was built are
- walked too, so selection can add the tests reaching them; so are
- *scope_routes*, the files a scope runs the tests of (:func:`_plan_scopes`).
- With *explain*, the import route from that test to a changed file is
- looked up as well. *pytest_texts* are the checkout's conftests and pytest
- configs by path (``_read_checkout``); without them no conftest route is
- narrowed.
- """
- from repowise.core.persistence.crud import (
- get_health_metrics,
- get_repository,
- get_test_coverage_summary,
- )
- from repowise.core.persistence.crud.analysis.coverage_map import MAX_TEST_COVERAGE_ROWS
-
- out = _empty_result(len(change.files) + len(change.deleted))
- if not out["changed_files"]:
- return out
+async def _collect(repo_path, change, checkout, config, explain: str | None, plan) -> dict:
+ """Open the index and collect the change's tests (``test_collection.collect``)."""
+ from repowise.core.analysis.test_collection import collect, empty_result
+ from repowise.core.persistence.crud import get_repository
+ if not (change.files or change.deleted):
+ return empty_result(0)
async with repo_index_session(Path(repo_path)) as opened:
if opened is None:
+ out = empty_result(len(change.files) + len(change.deleted))
out["no_index"] = True
return out
session, repo_id = opened
repo_row = await get_repository(session, repo_id)
- _indexed_commit(repo_path, repo_row.head_commit if repo_row else None, out)
-
- summary = await get_test_coverage_summary(session, repo_id)
- out["map_empty"] = summary.get("pair_count", 0) == 0
- out["map_truncated"] = summary.get("pair_count", 0) >= MAX_TEST_COVERAGE_ROWS
- measured = summary.get("ingested_commit_sha")
- out["map_current"] = out["map_empty"] or measured in {change.base, change.head} - {None}
-
- # Repo file keys back the filename-pattern fallback (same source the
- # aggregate coverage ingest resolves against).
- repo_keys = {m.file_path for m in await get_health_metrics(session, repo_id)}
- routes = [] if config is None else _gap_routes(repo_path, change, config, repo_keys, out)
- routes = sorted({*routes, *scope_routes})
- await _resolve_impacted(
- session,
- repo_id,
- _query_lines(change, measured),
- repo_keys,
- out,
- roots,
- routes,
- pytest_texts,
- )
- await _place_tests(session, repo_id, out)
- if explain:
- await _explain_route(session, repo_id, explain, change, out)
-
- return out
-
-
-async def _explain_route(session, repo_id: str, test: str, change, out: dict) -> None:
- """The dependency route from *test* to a changed file, for ``--explain`` only."""
- from repowise.core.analysis.test_reachability import dependency_path
-
- changed = [*change.files, *change.deleted]
- try:
- out["explain_route"] = await dependency_path(
- session, repo_id, test.split("::", 1)[0], changed
- )
- except Exception as exc: # the explanation still stands without its route
- log.debug("explain_route_failed", test=test, error=str(exc))
- out["explain_route"] = []
-
-
-async def _place_tests(session, repo_id: str, out: dict) -> None:
- """Record the tests the graph can see into, and those the indexer found it cannot."""
- from repowise.core.analysis.test_reachability import (
- always_run_test_files,
- placed_test_files,
- unscanned_test_files,
- )
-
- if out["graph_error"] is not None:
- return
- try:
- out["placed_tests"] = await placed_test_files(session, repo_id)
- out["always_run_tests"] = await always_run_test_files(session, repo_id)
- unscanned = sorted(await unscanned_test_files(session, repo_id) & out["placed_tests"])
- except Exception as exc:
- out["graph_error"] = f"{type(exc).__name__}: {exc}"
- return
- # An index older than the check cannot say which tests walk the tree, so
- # it is out of date like any other; unplaced tests run anyway.
- if unscanned and out["index_problem"] is None:
- out["index_problem"] = (
- f"The index has not checked {len(unscanned)} test file(s) for walking the source "
- f"tree or running the project (e.g. {unscanned[0]}); run `repowise update`."
- )
-
-
-def _indexed_commit(repo_path, row_commit: str | None, out: dict) -> None:
- """The commit the index describes: ``state.json`` first, then the repository row.
-
- The same order the server's freshness check reads. When both are recorded
- and disagree, the index's own history is in doubt, so the commit is unknown.
- """
- from repowise.core.workspace.update import read_state_commit
-
- state = read_state_commit(Path(repo_path))
- if state and row_commit and state != row_commit:
- out["index_problem"] = (
- f"The index records two commits ({state[:7]} in state.json, {row_commit[:7]} "
- "in its database), so what it describes is unknown; run `repowise update`."
- )
- return
- out["indexed_commit"] = state or row_commit
-
-
-def _gap_routes(repo_path, change, config, repo_keys: set[str], out: dict) -> list[str]:
- """Record the files changed since the indexed commit, and return those to walk.
-
- Only files whose edges may have moved are walked: each candidate is read at
- the indexed commit and at the base in one batch and compared.
- """
- from repowise.core.analysis.change_health.sources import read_blobs
- from repowise.core.analysis.changed_lines import index_gap
- from repowise.core.analysis.import_drift import edges_may_differ
- from repowise.core.analysis.test_selection import plan_gap, with_rewired
-
- out["index_gap"] = gap = index_gap(str(repo_path), out["indexed_commit"], change)
- if gap is None:
- return []
- plan = plan_gap(gap, config, [*change.files, *change.deleted])
- if plan.candidates:
- old, new = out["indexed_commit"], change.base
- specs = [(rev, p) for p in plan.candidates for rev in (old, new)]
- blobs = read_blobs(str(repo_path), specs)
- # Unreadable means nothing was compared, so every candidate may have moved.
- rewired = [
- p
- for p in plan.candidates
- if blobs is None or edges_may_differ(p, blobs.get((old, p)), blobs.get((new, p)))
- ]
- plan = with_rewired(plan, rewired, repo_keys)
- out["gap"] = plan
- return list(plan.targets)
-
-
-def _query_lines(change, measured: str | None) -> dict[str, set[int] | None]:
- """The lines to look up per file, on the side of the diff the map was measured at.
-
- Coverage measured at the change's base is keyed in the old file's line
- numbers, at its head in the new file's. Measured anywhere else, or for a
- deleted or hunkless file, only the file can be matched (``None``).
- """
- side = None
- if measured and measured == change.head:
- side = "new"
- elif measured and measured == change.base:
- side = "old"
- lines: dict[str, set[int] | None] = dict.fromkeys(change.deleted)
- for path, diff in change.files.items():
- lines[path] = _touched_lines(diff.hunks, side) if side and diff.hunks else None
- return lines
-
-
-def _touched_lines(hunks, side: str) -> set[int]:
- """Lines the hunks touch on *side*; for a pure insertion or deletion, its two neighbours."""
- lines: set[int] = set()
- for old_start, old_count, new_start, new_count in hunks:
- start, count = (old_start, old_count) if side == "old" else (new_start, new_count)
- lines.update(range(start, start + count) if count else {start, start + 1} - {0})
- return lines
-
-
-def _empty_result(changed_files: int) -> dict:
- return {
- "no_index": False,
- "map_empty": False,
- "map_current": True,
- "map_truncated": False,
- "indexed_commit": None,
- "index_problem": None,
- "index_gap": None,
- "gap": None,
- "graph_error": None,
- "placed_tests": None,
- "always_run_tests": {},
- "explain_route": [],
- "helper_importers": {},
- "conftest_notes": [],
- "changed_files": changed_files,
- "covered": {}, # test_id -> {test_file, source_files: [...]}
- "inferred": [], # {source_file, test_file, via}
- "unknown": [], # source_file (nothing knows of a test for it)
- }
-
-
-async def _resolve_impacted(
- session,
- repo_id: str,
- changed: dict[str, set[int] | None],
- repo_keys: set[str],
- out: dict,
- roots: PytestRoots | None = None,
- routes: tuple[str, ...] | list[str] = (),
- pytest_texts: dict | None = None,
-) -> dict:
- """Classify each changed file: covered tests, inferred tests, or unknown.
-
- Mutates and returns *out*. Split from ``_collect`` (which owns the DB
- bootstrap) so the diff -> lines -> tests path is testable against a seeded
- ``test_coverage`` table. A file mapped to ``None`` is matched by file, not
- by line (a deleted file, or a map measured at another commit). Every file
- also asks the graph: coverage records what one run executed, so it adds to
- the graph's answer and never replaces it.
-
- *routes* (files changed since the index was built) are looked up the same
- way, by file, but only to report the tests reaching them: none is guessed
- from its name or listed as unknown.
-
- Three tiers, and the output says which one answered for every file, because
- they are not interchangeable:
-
- ``covered``
- Recorded per-test coverage intersecting the changed *lines*. The only
- tier that knows about lines, and the only one that proves execution.
- ``inferred``
- What the graph names, beside any coverage. A changed test is its own
- candidate (``via="changed-test"``). Then the dependency graph, which
- reports which tier answered: tests whose calls reach the file
- (``via="call-graph"``), and every test that imports it, directly or
- through other modules or other tests (``via="import-graph"``). Both are
- recorded edges, and they find suites whose tests are named for
- behaviour rather than for the file. The filename pattern answers only
- when the graph is silent (``via="filename-pattern"``). All are
- file-level and all over-claim; none may be read as coverage.
- ``unknown``
- Nothing said anything. Run the full suite.
- """
- from repowise.core.analysis.health.coverage import paired_test_file
- from repowise.core.persistence.crud import tests_covering, tests_covering_files
-
- covered: dict[str, dict] = out["covered"]
- has_rows: set[str] = set()
- route_only = [r for r in routes if r not in changed]
- graph_targets = sorted({*changed, *route_only})
- by_file = {f: await tests_covering(session, repo_id, f, lines=ls) for f, ls in changed.items()}
- if route_only and not out.get("map_empty"):
- by_file.update(await tests_covering_files(session, repo_id, set(route_only)))
- for source_file, rows in sorted(by_file.items()):
- for r in rows:
- has_rows.add(source_file)
- entry = covered.setdefault(
- r["test_id"],
- {"test_file": r["test_file"], "source_files": []},
- )
- if source_file not in entry["source_files"]:
- entry["source_files"].append(source_file)
-
- if not graph_targets:
- return out
-
- try:
- candidates, importers, notes = await _graph_candidates(
- session, repo_id, graph_targets, roots, pytest_texts=pytest_texts
- )
- except Exception as exc:
- # Nothing the graph said can be trusted; selection runs everything.
- out["graph_error"] = f"{type(exc).__name__}: {exc}"
- candidates, importers, notes = {}, {}, []
- out["helper_importers"] = importers
- out["conftest_notes"] = notes
- for source_file in graph_targets:
- found = candidates.get(source_file)
- if found:
- out["inferred"].extend(
- {"source_file": source_file, "test_file": t, "via": via} for t, via in found
- )
- continue
- if source_file in has_rows or source_file not in changed:
- continue # coverage answered, or a route; a name-shaped guess adds nothing
- guess = paired_test_file(source_file, repo_keys)
- if guess:
- out["inferred"].append(
- {
- "source_file": source_file,
- "test_file": guess,
- "via": "filename-pattern",
- }
- )
- else:
- out["unknown"].append(source_file)
- return out
-
-
-async def _graph_candidates(
- session,
- repo_id: str,
- targets: list[str],
- roots: PytestRoots | None = None,
- *,
- pytest_texts: dict | None = None,
-) -> tuple[dict[str, list], dict[str, list[str]], list[str]]:
- """``{target: [(test file, via), ...]}`` from the graph, each test file's importers, notes.
-
- Candidates come strongest tier first. The importers map says, for every
- test file the walk met, which test files import it. A conftest reached only
- through its imports stands for the tests it can break
- (:mod:`repowise.core.analysis.conftest_routes`), read from *pytest_texts*;
- the notes say what was decided.
-
- One walk per tier for every target rather than one per file: the seed set
- is what makes it cheap. Uncapped, since a trimmed list would drop tests a
- change needs. The import walk treats a test file as a leaf, so the tests
- importing a candidate (a shared base class, a helper module under
- ``tests/``) are added until nothing new appears. Only code counts as a test
- node: a JSON or golden file the index flags as test material is data, not a
- route. A read failure raises.
- """
- from repowise.core.analysis.conftest_routes import scoped_candidates
- from repowise.core.analysis.test_reachability import load_test_files
- from repowise.core.analysis.test_selection import is_code_file, plugin_loader
-
- texts = pytest_texts or {}
- test_files = {f for f in await load_test_files(session, repo_id) if is_code_file(f)}
- found, entries = await _tier_picks(session, repo_id, targets, test_files, roots)
- seeds = {t for picks in found.values() for t in picks}
- parents, parent_entries = await _test_importers(session, repo_id, seeds, test_files)
- out, notes = await scoped_candidates(
- session,
- repo_id,
- found,
- entries,
- parents,
- parent_entries,
- test_files,
- texts.get,
- plugin_loader=plugin_loader(texts.items()),
- )
- return out, {test: sorted(found_by) for test, found_by in parents.items()}, notes
-
-
-def _all_tests(reached) -> tuple[str, ...]:
- """Every test a walk found, not the capped list it reports."""
- return reached.all_tests or tuple(reached.tests)
-
-
-async def _tier_picks(
- session, repo_id: str, targets: list[str], test_files: set[str], roots: PytestRoots | None
-) -> tuple[dict[str, dict[str, str]], dict[str, dict]]:
- """``{target: {test file: via}}``: a changed test itself, then the call and import walks.
-
- Also ``{target: {test file: the files it was reached through}}`` from the
- import walk.
- """
- from repowise.core.analysis.test_reachability import tests_reaching_by_tier
- from repowise.core.analysis.test_selection import is_runnable_test
-
- reaching = await tests_reaching_by_tier(session, repo_id, targets, test_files=test_files)
- importers = await tests_reaching_by_tier(
- session,
- repo_id,
- targets,
- call_depth=0,
- import_depth=_IMPORT_CLOSURE_DEPTH,
- test_files=test_files,
- )
- found: dict[str, dict[str, str]] = {}
- entries = {t: dict(r.entries or {}) for t, r in importers.items()}
- for target in targets:
- picks = found.setdefault(target, {})
- # A test the index has not seen yet (new in this change) is still its own pick.
- if target in test_files or is_runnable_test(target, roots):
- picks[target] = "changed-test"
- for reached, via in ((reaching.get(target), None), (importers.get(target), "import-graph")):
- for t in _all_tests(reached) if reached else ():
- picks.setdefault(t, via or reached.via)
- return found, entries
-
-
-async def _test_importers(
- session, repo_id: str, seeds: set[str], test_files: set[str]
-) -> tuple[dict[str, set[str]], dict[str, dict]]:
- """``{test file: test files importing it}``, walked until nothing new appears.
-
- Also ``{test file: {importer: the files it imports on that route}}``.
- """
- from repowise.core.analysis.test_reachability import tests_reaching_by_tier
-
- parents: dict[str, set[str]] = {}
- routes: dict[str, dict] = {}
- frontier, seen = set(seeds), set(seeds)
- while frontier:
- walked = await tests_reaching_by_tier(
+ return await collect(
session,
repo_id,
- sorted(frontier),
- call_depth=0,
- import_depth=_IMPORT_CLOSURE_DEPTH,
- test_files=test_files,
- )
- frontier = set()
- for test, reached in walked.items():
- parents.setdefault(test, set()).update(_all_tests(reached))
- routes.setdefault(test, {}).update(reached.entries or {})
- frontier.update(set(_all_tests(reached)) - seen)
- seen |= frontier
- return parents, routes
-
-
-def _select(repo_path, change, result: dict, config, checkout: _Checkout, plan: _Plan):
- """The run-all-or-subset decision for ``--format args`` / ``json``."""
- from repowise.core.analysis.test_selection import (
- SelectionInput,
- is_documentation,
- is_runnable_test,
- plugin_loader,
- select_tests,
- )
-
- named = {g["test_file"] for g in result["inferred"]}
- named |= {i["test_file"] for i in result["covered"].values() if i["test_file"]}
- root = Path(repo_path)
- tracked = checkout.tracked
- go_test_dirs = {str(Path(p).parent.as_posix()) for p in tracked if p.endswith("_test.go")}
- placed = result["placed_tests"]
- known_tests = sorted(p for p in tracked if is_runnable_test(p, checkout.roots))
- return select_tests(
- SelectionInput(
- changed=change.files,
- deleted=change.deleted,
- tiers=result,
- config=config,
- index_available=not result["no_index"],
- label=change.label,
- map_current=result["map_current"],
- map_truncated=result["map_truncated"],
- index_gap=result["index_gap"],
- gap=result["gap"],
- index_problem=result["index_problem"],
- graph_error=result["graph_error"],
- missing={f for f in named if not (root / f).is_file()},
- go_test_dirs=go_test_dirs,
- known_tests=known_tests,
- doc_readers={p: n[0] for p, n in plan.namers.items() if is_documentation(p)},
- plugin_loader=plugin_loader(checkout.pytest_texts),
- unplaced_tests=[] if placed is None else [t for t in known_tests if t not in placed],
- always_run_tests=result["always_run_tests"],
- scopes=plan.scopes,
+ repo_path,
+ change,
+ checkout.roots,
+ config,
+ explain,
+ plan.routes if plan else [],
+ dict(checkout.pytest_texts),
+ indexed_commit=repo_row.head_commit if repo_row else None,
)
- )
-
-
-def _text(path: Path) -> str | None:
- try:
- return path.read_text(encoding="utf-8", errors="ignore")
- except OSError:
- return None
-
-
-def _texts(root: Path, paths: list[str], *, pytest_only: bool = False):
- """``(path, text)`` for each readable file; with *pytest_only*, conftests and configs."""
- from repowise.core.analysis.test_selection import is_code_file
-
- for path in paths:
- if pytest_only and is_code_file(path) and not path.endswith("conftest.py"):
- continue
- if (text := _text(root / path)) is not None:
- yield path, text
def _machine_test_ids(result: dict) -> list[str]:
diff --git a/packages/core/src/repowise/core/analysis/test_collection.py b/packages/core/src/repowise/core/analysis/test_collection.py
new file mode 100644
index 000000000..129b0664b
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/test_collection.py
@@ -0,0 +1,594 @@
+"""Collect the evidence :func:`~repowise.core.analysis.test_selection.select_tests` decides from.
+
+One pipeline for every surface that names the tests a change needs: map each
+changed file to the tests whose recorded coverage touches its changed lines and
+to the tests the dependency graph shows reaching it, note what the index cannot
+see, then hand it all to :func:`select_tests` (:func:`select`).
+
+The inputs are plain data a server can build from what it holds: a
+:class:`Checkout` (tracked paths, pytest's conftests and configs, a text
+reader) and a change set. Git is read only for the files changed since the
+index was built.
+
+Ceiling: two reads still go through the repository path, the commit
+``state.json`` records (:func:`_indexed_commit`) and the git diff between the
+indexed commit and the change's base (:func:`_gap_routes`). A server without a
+checkout passes *indexed_commit* and gets "cannot tell what changed since the
+index was built" for the gap, which runs everything; taking the state commit
+and the gap as inputs is the upgrade.
+"""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, NamedTuple
+
+import structlog
+
+from ..pytest_roots import PytestRoots
+from .test_selection import Selection, TestSelectionConfig
+
+log = structlog.get_logger(__name__)
+
+# The whole reverse-import closure: a test importing a module that imports the
+# changed file runs it too. No depth limit: the walk ends when no node gains a
+# new seed, so a cut-off can never drop a test.
+_IMPORT_CLOSURE_DEPTH = sys.maxsize
+
+
+@dataclass(frozen=True)
+class Checkout:
+ """Tracked paths and the pytest files among them, each read once per selection.
+
+ *read* returns a tracked file's text (``None`` when unreadable) and
+ *exists* whether a path is present; :func:`read_checkout` answers both from
+ disk, a server may answer them from what it stores.
+ """
+
+ tracked: list[str]
+ pytest_texts: list[tuple[str, str]]
+ roots: PytestRoots
+ read: Callable[[str], str | None]
+ exists: Callable[[str], bool]
+
+
+def read_checkout(repo_path) -> Checkout:
+ """Tracked files, their conftests and pytest configs, and pytest's collection roots."""
+ from .. import git_refs
+ from ..pytest_roots import PYTEST_CONFIG_NAMES, read_pytest_roots
+ from .test_selection import is_code_file, is_scan_source
+
+ root = Path(repo_path)
+
+ def read(path: str) -> str | None:
+ try:
+ return (root / path).read_text(encoding="utf-8", errors="ignore")
+ except OSError:
+ return None
+
+ tracked = git_refs.tracked_paths(str(root))
+ texts = [
+ (p, text)
+ for p in tracked
+ if is_scan_source(p) and (p.endswith("conftest.py") or not is_code_file(p))
+ if (text := read(p)) is not None
+ ]
+ roots = read_pytest_roots((p, t) for p, t in texts if Path(p).name in PYTEST_CONFIG_NAMES)
+ return Checkout(tracked, texts, roots, read, lambda p: (root / p).is_file())
+
+
+class Plan(NamedTuple):
+ """Who names each changed doc or asset, the scope each scoped file runs, and its routes."""
+
+ namers: dict[str, list[str]]
+ scopes: dict
+ routes: list[str]
+
+
+def plan_scopes(change, config, checkout: Checkout) -> Plan:
+ """The change's scopes, reading the sources once and only when a file needs its namers."""
+ from .selection_scopes import (
+ keeps_full_run,
+ needs_namers,
+ trigger_scopes,
+ )
+ from .test_selection import file_namers, is_scan_source
+
+ paths = [*change.files, *change.deleted]
+ # A file that runs everything anyway makes every namer search moot.
+ blocked = any(keeps_full_run(p, config) for p in paths)
+ asked = [] if blocked else [p for p in paths if needs_namers(p, config)]
+ namers: dict[str, list[str]] = {}
+ if asked:
+ known = dict(checkout.pytest_texts) # conftests and pytest configs, read already
+ texts = (
+ (p, known[p]) if p in known else (p, checkout.read(p) or "")
+ for p in checkout.tracked
+ if is_scan_source(p)
+ )
+ namers = file_namers(asked, texts)
+ scopes = trigger_scopes(paths, checkout.tracked, namers, config)
+ routes = sorted({r for scope in scopes.values() for r in scope.routes} - set(paths))
+ return Plan(namers, scopes, routes)
+
+
+async def collect(
+ session,
+ repo_id: str,
+ repo_path,
+ change,
+ roots: PytestRoots | None = None,
+ config: TestSelectionConfig | None = None,
+ explain: str | None = None,
+ scope_routes: list[str] | tuple[str, ...] = (),
+ pytest_texts: dict | None = None,
+ *,
+ indexed_commit: str | None = None,
+) -> dict:
+ """Resolve the change's files to impacted tests + labelled fallbacks.
+
+ *indexed_commit* is the index's repository row commit. With a selection
+ *config*, files changed since the index was built are walked too, so
+ selection can add the tests reaching them; so are *scope_routes*, the
+ files a scope runs the tests of (:func:`plan_scopes`). With *explain*, the
+ import route from that test to a changed file is looked up as well.
+ *pytest_texts* are the checkout's conftests and pytest configs by path;
+ without them no conftest route is narrowed.
+ """
+ from ..persistence.crud import get_health_metrics, get_test_coverage_summary
+ from ..persistence.crud.analysis.coverage_map import MAX_TEST_COVERAGE_ROWS
+
+ out = empty_result(len(change.files) + len(change.deleted))
+ if not out["changed_files"]:
+ return out
+ _indexed_commit(repo_path, indexed_commit, out)
+
+ summary = await get_test_coverage_summary(session, repo_id)
+ out["map_empty"] = summary.get("pair_count", 0) == 0
+ out["map_truncated"] = summary.get("pair_count", 0) >= MAX_TEST_COVERAGE_ROWS
+ measured = summary.get("ingested_commit_sha")
+ out["map_current"] = out["map_empty"] or measured in {change.base, change.head} - {None}
+
+ # Repo file keys back the filename-pattern fallback (same source the
+ # aggregate coverage ingest resolves against).
+ repo_keys = {m.file_path for m in await get_health_metrics(session, repo_id)}
+ routes = [] if config is None else _gap_routes(repo_path, change, config, repo_keys, out)
+ routes = sorted({*routes, *scope_routes})
+ await resolve_impacted(
+ session,
+ repo_id,
+ query_lines(change, measured),
+ repo_keys,
+ out,
+ roots,
+ routes,
+ pytest_texts,
+ )
+ await _place_tests(session, repo_id, out)
+ if explain:
+ await _explain_route(session, repo_id, explain, change, out)
+ return out
+
+
+async def _explain_route(session, repo_id: str, test: str, change, out: dict) -> None:
+ """The dependency route from *test* to a changed file, for ``--explain`` only."""
+ from .test_reachability import dependency_path
+
+ changed = [*change.files, *change.deleted]
+ try:
+ out["explain_route"] = await dependency_path(
+ session, repo_id, test.split("::", 1)[0], changed
+ )
+ except Exception as exc: # the explanation still stands without its route
+ log.debug("explain_route_failed", test=test, error=str(exc))
+ out["explain_route"] = []
+
+
+async def _place_tests(session, repo_id: str, out: dict) -> None:
+ """Record the tests the graph can see into, and those the indexer found it cannot."""
+ from .test_reachability import (
+ always_run_test_files,
+ placed_test_files,
+ unscanned_test_files,
+ )
+
+ if out["graph_error"] is not None:
+ return
+ try:
+ out["placed_tests"] = await placed_test_files(session, repo_id)
+ out["always_run_tests"] = await always_run_test_files(session, repo_id)
+ unscanned = sorted(await unscanned_test_files(session, repo_id) & out["placed_tests"])
+ except Exception as exc:
+ out["graph_error"] = f"{type(exc).__name__}: {exc}"
+ return
+ # An index older than the check cannot say which tests walk the tree, so
+ # it is out of date like any other; unplaced tests run anyway.
+ if unscanned and out["index_problem"] is None:
+ out["index_problem"] = (
+ f"The index has not checked {len(unscanned)} test file(s) for walking the source "
+ f"tree or running the project (e.g. {unscanned[0]}); run `repowise update`."
+ )
+
+
+def _indexed_commit(repo_path, row_commit: str | None, out: dict) -> None:
+ """The commit the index describes: ``state.json`` first, then the repository row.
+
+ The same order the server's freshness check reads. When both are recorded
+ and disagree, the index's own history is in doubt, so the commit is unknown.
+ """
+ from ..workspace.update import read_state_commit
+
+ state = read_state_commit(Path(repo_path))
+ if state and row_commit and state != row_commit:
+ out["index_problem"] = (
+ f"The index records two commits ({state[:7]} in state.json, {row_commit[:7]} "
+ "in its database), so what it describes is unknown; run `repowise update`."
+ )
+ return
+ out["indexed_commit"] = state or row_commit
+
+
+def _gap_routes(repo_path, change, config, repo_keys: set[str], out: dict) -> list[str]:
+ """Record the files changed since the indexed commit, and return those to walk.
+
+ Only files whose edges may have moved are walked: each candidate is read at
+ the indexed commit and at the base in one batch and compared.
+ """
+ from .change_health.sources import read_blobs
+ from .changed_lines import index_gap
+ from .import_drift import edges_may_differ
+ from .test_selection import plan_gap, with_rewired
+
+ out["index_gap"] = gap = index_gap(str(repo_path), out["indexed_commit"], change)
+ if gap is None:
+ return []
+ plan = plan_gap(gap, config, [*change.files, *change.deleted])
+ if plan.candidates:
+ old, new = out["indexed_commit"], change.base
+ specs = [(rev, p) for p in plan.candidates for rev in (old, new)]
+ blobs = read_blobs(str(repo_path), specs)
+ # Unreadable means nothing was compared, so every candidate may have moved.
+ rewired = [
+ p
+ for p in plan.candidates
+ if blobs is None or edges_may_differ(p, blobs.get((old, p)), blobs.get((new, p)))
+ ]
+ plan = with_rewired(plan, rewired, repo_keys)
+ out["gap"] = plan
+ return list(plan.targets)
+
+
+def query_lines(change, measured: str | None) -> dict[str, set[int] | None]:
+ """The lines to look up per file, on the side of the diff the map was measured at.
+
+ Coverage measured at the change's base is keyed in the old file's line
+ numbers, at its head in the new file's. Measured anywhere else, or for a
+ deleted or hunkless file, only the file can be matched (``None``).
+ """
+ side = None
+ if measured and measured == change.head:
+ side = "new"
+ elif measured and measured == change.base:
+ side = "old"
+ lines: dict[str, set[int] | None] = dict.fromkeys(change.deleted)
+ for path, diff in change.files.items():
+ lines[path] = _touched_lines(diff.hunks, side) if side and diff.hunks else None
+ return lines
+
+
+def _touched_lines(hunks, side: str) -> set[int]:
+ """Lines the hunks touch on *side*; for a pure insertion or deletion, its two neighbours."""
+ lines: set[int] = set()
+ for old_start, old_count, new_start, new_count in hunks:
+ start, count = (old_start, old_count) if side == "old" else (new_start, new_count)
+ lines.update(range(start, start + count) if count else {start, start + 1} - {0})
+ return lines
+
+
+def empty_result(changed_files: int) -> dict:
+ return {
+ "no_index": False,
+ "map_empty": False,
+ "map_current": True,
+ "map_truncated": False,
+ "indexed_commit": None,
+ "index_problem": None,
+ "index_gap": None,
+ "gap": None,
+ "graph_error": None,
+ "placed_tests": None,
+ "always_run_tests": {},
+ "explain_route": [],
+ "helper_importers": {},
+ "conftest_notes": [],
+ "changed_files": changed_files,
+ "covered": {}, # test_id -> {test_file, source_files: [...]}
+ "inferred": [], # {source_file, test_file, via}
+ "unknown": [], # source_file (nothing knows of a test for it)
+ }
+
+
+async def resolve_impacted(
+ session,
+ repo_id: str,
+ changed: dict[str, set[int] | None],
+ repo_keys: set[str],
+ out: dict,
+ roots: PytestRoots | None = None,
+ routes: tuple[str, ...] | list[str] = (),
+ pytest_texts: dict | None = None,
+) -> dict:
+ """Classify each changed file: covered tests, inferred tests, or unknown.
+
+ Mutates and returns *out*. Split from :func:`collect` (which reads the
+ map summary and the gap) so the diff -> lines -> tests path is testable against a seeded
+ ``test_coverage`` table. A file mapped to ``None`` is matched by file, not
+ by line (a deleted file, or a map measured at another commit). Every file
+ also asks the graph: coverage records what one run executed, so it adds to
+ the graph's answer and never replaces it.
+
+ *routes* (files changed since the index was built) are looked up the same
+ way, by file, but only to report the tests reaching them: none is guessed
+ from its name or listed as unknown.
+
+ Three tiers, and the output says which one answered for every file, because
+ they are not interchangeable:
+
+ ``covered``
+ Recorded per-test coverage intersecting the changed *lines*. The only
+ tier that knows about lines, and the only one that proves execution.
+ ``inferred``
+ What the graph names, beside any coverage. A changed test is its own
+ candidate (``via="changed-test"``). Then the dependency graph, which
+ reports which tier answered: tests whose calls reach the file
+ (``via="call-graph"``), and every test that imports it, directly or
+ through other modules or other tests (``via="import-graph"``). Both are
+ recorded edges, and they find suites whose tests are named for
+ behaviour, not for the file. The filename pattern answers only
+ when the graph is silent (``via="filename-pattern"``). All are
+ file-level and all over-claim; none may be read as coverage.
+ ``unknown``
+ Nothing said anything. Run the full suite.
+ """
+ from ..persistence.crud import tests_covering, tests_covering_files
+ from .health.coverage import paired_test_file
+
+ covered: dict[str, dict] = out["covered"]
+ has_rows: set[str] = set()
+ route_only = [r for r in routes if r not in changed]
+ graph_targets = sorted({*changed, *route_only})
+ by_file = {f: await tests_covering(session, repo_id, f, lines=ls) for f, ls in changed.items()}
+ if route_only and not out.get("map_empty"):
+ by_file.update(await tests_covering_files(session, repo_id, set(route_only)))
+ for source_file, rows in sorted(by_file.items()):
+ for r in rows:
+ has_rows.add(source_file)
+ entry = covered.setdefault(
+ r["test_id"],
+ {"test_file": r["test_file"], "source_files": []},
+ )
+ if source_file not in entry["source_files"]:
+ entry["source_files"].append(source_file)
+
+ if not graph_targets:
+ return out
+
+ try:
+ candidates, importers, notes = await _graph_candidates(
+ session, repo_id, graph_targets, roots, pytest_texts=pytest_texts
+ )
+ except Exception as exc:
+ # Nothing the graph said can be trusted; selection runs everything.
+ out["graph_error"] = f"{type(exc).__name__}: {exc}"
+ candidates, importers, notes = {}, {}, []
+ out["helper_importers"] = importers
+ out["conftest_notes"] = notes
+ for source_file in graph_targets:
+ found = candidates.get(source_file)
+ if found:
+ out["inferred"].extend(
+ {"source_file": source_file, "test_file": t, "via": via} for t, via in found
+ )
+ continue
+ if source_file in has_rows or source_file not in changed:
+ continue # coverage answered, or a route; a name-shaped guess adds nothing
+ guess = paired_test_file(source_file, repo_keys)
+ if guess:
+ out["inferred"].append(
+ {
+ "source_file": source_file,
+ "test_file": guess,
+ "via": "filename-pattern",
+ }
+ )
+ else:
+ out["unknown"].append(source_file)
+ return out
+
+
+async def _graph_candidates(
+ session,
+ repo_id: str,
+ targets: list[str],
+ roots: PytestRoots | None = None,
+ *,
+ pytest_texts: dict | None = None,
+) -> tuple[dict[str, list], dict[str, list[str]], list[str]]:
+ """``{target: [(test file, via), ...]}`` from the graph, each test file's importers, notes.
+
+ Candidates come strongest tier first. The importers map says, for every
+ test file the walk met, which test files import it. A conftest reached only
+ through its imports stands for the tests it can break
+ (:mod:`repowise.core.analysis.conftest_routes`), read from *pytest_texts*;
+ the notes say what was decided.
+
+ One walk per tier for every target, not one per file: the seed set
+ is what makes it cheap. Uncapped, since a trimmed list would drop tests a
+ change needs. The import walk treats a test file as a leaf, so the tests
+ importing a candidate (a shared base class, a helper module under
+ ``tests/``) are added until nothing new appears. Only code counts as a test
+ node: a JSON or golden file the index flags as test material is data, not a
+ route. A read failure raises.
+ """
+ from .conftest_routes import scoped_candidates
+ from .test_reachability import load_test_files
+ from .test_selection import is_code_file, plugin_loader
+
+ texts = pytest_texts or {}
+ test_files = {f for f in await load_test_files(session, repo_id) if is_code_file(f)}
+ found, entries = await _tier_picks(session, repo_id, targets, test_files, roots)
+ seeds = {t for picks in found.values() for t in picks}
+ parents, parent_entries = await _test_importers(session, repo_id, seeds, test_files)
+ out, notes = await scoped_candidates(
+ session,
+ repo_id,
+ found,
+ entries,
+ parents,
+ parent_entries,
+ test_files,
+ texts.get,
+ plugin_loader=plugin_loader(texts.items()),
+ )
+ return out, {test: sorted(found_by) for test, found_by in parents.items()}, notes
+
+
+def _all_tests(reached) -> tuple[str, ...]:
+ """Every test a walk found, not the capped list it reports."""
+ return reached.all_tests or tuple(reached.tests)
+
+
+async def _tier_picks(
+ session, repo_id: str, targets: list[str], test_files: set[str], roots: PytestRoots | None
+) -> tuple[dict[str, dict[str, str]], dict[str, dict]]:
+ """``{target: {test file: via}}``: a changed test itself, then the call and import walks.
+
+ Also ``{target: {test file: the files it was reached through}}`` from the
+ import walk.
+ """
+ from .test_reachability import tests_reaching_by_tier
+ from .test_selection import is_runnable_test
+
+ reaching = await tests_reaching_by_tier(session, repo_id, targets, test_files=test_files)
+ importers = await tests_reaching_by_tier(
+ session,
+ repo_id,
+ targets,
+ call_depth=0,
+ import_depth=_IMPORT_CLOSURE_DEPTH,
+ test_files=test_files,
+ )
+ found: dict[str, dict[str, str]] = {}
+ entries = {t: dict(r.entries or {}) for t, r in importers.items()}
+ for target in targets:
+ picks = found.setdefault(target, {})
+ # A test the index has not seen yet (new in this change) is still its own pick.
+ if target in test_files or is_runnable_test(target, roots):
+ picks[target] = "changed-test"
+ for reached, via in ((reaching.get(target), None), (importers.get(target), "import-graph")):
+ for t in _all_tests(reached) if reached else ():
+ picks.setdefault(t, via or reached.via)
+ return found, entries
+
+
+async def _test_importers(
+ session, repo_id: str, seeds: set[str], test_files: set[str]
+) -> tuple[dict[str, set[str]], dict[str, dict]]:
+ """``{test file: test files importing it}``, walked until nothing new appears.
+
+ Also ``{test file: {importer: the files it imports on that route}}``.
+ """
+ from .test_reachability import tests_reaching_by_tier
+
+ parents: dict[str, set[str]] = {}
+ routes: dict[str, dict] = {}
+ frontier, seen = set(seeds), set(seeds)
+ while frontier:
+ walked = await tests_reaching_by_tier(
+ session,
+ repo_id,
+ sorted(frontier),
+ call_depth=0,
+ import_depth=_IMPORT_CLOSURE_DEPTH,
+ test_files=test_files,
+ )
+ frontier = set()
+ for test, reached in walked.items():
+ parents.setdefault(test, set()).update(_all_tests(reached))
+ routes.setdefault(test, {}).update(reached.entries or {})
+ frontier.update(set(_all_tests(reached)) - seen)
+ seen |= frontier
+ return parents, routes
+
+
+def select(change, result: dict, config, checkout: Checkout, plan: Plan) -> Selection:
+ """The run-all-or-subset decision over *result* (from :func:`collect`)."""
+ from .test_selection import (
+ SelectionInput,
+ is_documentation,
+ is_runnable_test,
+ plugin_loader,
+ select_tests,
+ )
+
+ named = {g["test_file"] for g in result["inferred"]}
+ named |= {i["test_file"] for i in result["covered"].values() if i["test_file"]}
+ tracked = checkout.tracked
+ go_test_dirs = {str(Path(p).parent.as_posix()) for p in tracked if p.endswith("_test.go")}
+ placed = result["placed_tests"]
+ known_tests = sorted(p for p in tracked if is_runnable_test(p, checkout.roots))
+ return select_tests(
+ SelectionInput(
+ changed=change.files,
+ deleted=change.deleted,
+ tiers=result,
+ config=config,
+ index_available=not result["no_index"],
+ label=change.label,
+ map_current=result["map_current"],
+ map_truncated=result["map_truncated"],
+ index_gap=result["index_gap"],
+ gap=result["gap"],
+ index_problem=result["index_problem"],
+ graph_error=result["graph_error"],
+ missing={f for f in named if not checkout.exists(f)},
+ go_test_dirs=go_test_dirs,
+ known_tests=known_tests,
+ doc_readers={p: n[0] for p, n in plan.namers.items() if is_documentation(p)},
+ plugin_loader=plugin_loader(checkout.pytest_texts),
+ unplaced_tests=[] if placed is None else [t for t in known_tests if t not in placed],
+ always_run_tests=result["always_run_tests"],
+ scopes=plan.scopes,
+ )
+ )
+
+
+async def select_for_change(
+ session,
+ repo_id: str,
+ repo_path,
+ change,
+ config: TestSelectionConfig,
+ checkout: Checkout,
+ *,
+ indexed_commit: str | None = None,
+) -> tuple[dict[str, Any], Selection]:
+ """:func:`plan_scopes`, :func:`collect` then :func:`select`: what *change* needs, and why."""
+ plan = plan_scopes(change, config, checkout)
+ result = await collect(
+ session,
+ repo_id,
+ repo_path,
+ change,
+ checkout.roots,
+ config,
+ None,
+ plan.routes,
+ dict(checkout.pytest_texts),
+ indexed_commit=indexed_commit,
+ )
+ result["diff"] = change.label
+ return result, select(change, result, config, checkout, plan)
diff --git a/tests/unit/cli/test_impacted_tests_cmd.py b/tests/unit/cli/test_impacted_tests_cmd.py
index b5e582961..537f0c7e5 100644
--- a/tests/unit/cli/test_impacted_tests_cmd.py
+++ b/tests/unit/cli/test_impacted_tests_cmd.py
@@ -246,12 +246,12 @@ def test_an_index_that_disagrees_about_its_commit_runs_everything(repo) -> None:
def test_a_graph_that_cannot_be_read_runs_everything(repo, monkeypatch) -> None:
- from repowise.cli.commands import impacted_tests_cmd
+ from repowise.core.analysis import test_collection
async def _broken(*_a, **_k):
raise RuntimeError("edge table locked")
- monkeypatch.setattr(impacted_tests_cmd, "_graph_candidates", _broken)
+ monkeypatch.setattr(test_collection, "_graph_candidates", _broken)
result = _run(repo, "main...feat", "--format", "args")
assert result.stdout == ":all\n"
assert "The graph could not be read (RuntimeError: edge table locked)" in _err(result)
diff --git a/tests/unit/persistence/test_impacted_tests_resolution.py b/tests/unit/persistence/test_impacted_tests_resolution.py
index 66a757a4b..de53c8328 100644
--- a/tests/unit/persistence/test_impacted_tests_resolution.py
+++ b/tests/unit/persistence/test_impacted_tests_resolution.py
@@ -1,6 +1,6 @@
"""The ``impacted-tests`` resolution path: changed lines -> impacted tests.
-Exercises :func:`_resolve_impacted` against a seeded ``test_coverage`` table so
+Exercises :func:`resolve_impacted` against a seeded ``test_coverage`` table so
the honest outcomes are pinned and stay distinguishable: a coverage-backed hit,
an inferred candidate when a changed file has no coverage rows (the import graph
first, a filename-pattern guess second), and "unknown" when nothing knows of a
@@ -9,8 +9,9 @@ test. The diff parser and CRUD line-intersection are tested separately.
from __future__ import annotations
-from repowise.cli.commands.impacted_tests_cmd import _empty_result, _resolve_impacted
from repowise.core.analysis.health.coverage import TestCoverage
+from repowise.core.analysis.test_collection import empty_result as _empty_result
+from repowise.core.analysis.test_collection import resolve_impacted as _resolve_impacted
from repowise.core.persistence.crud import save_test_coverage
from tests.unit.persistence.helpers import insert_repo
@@ -276,12 +277,12 @@ async def test_coverage_adds_to_the_graph_and_never_replaces_it(async_session) -
async def test_a_graph_read_failure_is_reported_not_swallowed(async_session, monkeypatch) -> None:
- from repowise.cli.commands import impacted_tests_cmd
+ from repowise.core.analysis import test_collection
async def _broken(*_a, **_k):
raise RuntimeError("edge table locked")
- monkeypatch.setattr(impacted_tests_cmd, "_graph_candidates", _broken)
+ monkeypatch.setattr(test_collection, "_graph_candidates", _broken)
repo = await insert_repo(async_session)
out = _empty_result(1)
await _resolve_impacted(async_session, repo.id, {"src/a.py": {1}}, set(), out)