Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(selection): walk the test graph in memory and find namers with git grep (#3340)
Raghav Chamadiya4354d12aa995
diff --git a/packages/core/src/repowise/core/analysis/conftest_routes.py b/packages/core/src/repowise/core/analysis/conftest_routes.pyindex 8a22403be..53df65525 100644--- a/packages/core/src/repowise/core/analysis/conftest_routes.py+++ b/packages/core/src/repowise/core/analysis/conftest_routes.py@@ -527,52 +527,97 @@ async def scoped_candidates(a conftest's text. Every conftest on a route that thismodule cannot narrow keeps today's answer: every test under it."""+ routes = _conftest_routes(found, entries, parents, parent_entries)+ _judge_routes(routes, read, plugin_loader)+ scope = await _load_scope(session, repo_id, routes, parents, test_files)+ out: dict[str, list[tuple[str, str]]] = {}+ notes: dict[tuple[str, str], list[str]] = {}+ for target, picks in found.items():+ out[target] = _narrow(target, dict(picks), routes.get(target, {}), scope, notes)+ return out, [_note(key, targets) for key, targets in notes.items()]+++def _conftest_routes(+ found: Mapping[str, dict[str, str]],+ entries: Mapping[str, Mapping[str, Collection[str]]],+ parents: Mapping[str, Collection[str]],+ parent_entries: Mapping[str, Mapping[str, Collection[str]]],+) -> dict[str, dict[str, _Route]]:+ """``{target: {conftest: its route}}`` for every conftest a target's picks reach."""+ # Test files each conftest imports, asked once per conftest per target.+ imported: dict[str, list[str]] = {}+ for test, importers in parents.items():+ for importer in importers:+ imported.setdefault(importer, []).append(test)routes: dict[str, dict[str, _Route]] = {}for target, picks in found.items():full = dict(picks)with_importers(full, parents)for conf in sorted(t for t in full if scope_kind(t) == "conftest" and t != target):via = set(entries.get(target, {}).get(conf, ()))- for h in full:- if h != conf and conf in parents.get(h, ()):- via |= set(parent_entries.get(h, {}).get(conf, ()))+ via.update(+ v+ for h in imported.get(conf, ())+ if h != conf and h in full+ for v in parent_entries.get(h, {}).get(conf, ())+ )# A parent conftest is not an import: pytest links a nested# conftest to it, and that parent's own decision covers the route.own = frozenset(v for v in via if scope_kind(v) != "conftest")routes.setdefault(target, {})[conf] = _Route(own, conf in picks and not via)+ return routes+- # One parse per conftest; each distinct route through it is matched against it.+def _judge_routes(+ routes: Mapping[str, Mapping[str, _Route]],+ read: Callable[[str], str | None],+ plugin_loader: str | None,+) -> None:+ """Set each route's use and reason. One parse per conftest; each distinct route through+ it is matched against it once."""facts: dict[str, ConftestFacts | None] = {}uses: dict[tuple[str, frozenset[str]], ConftestUse] = {}- for by_conf in routes.values():- for conf, route in by_conf.items():- if conf not in facts:- source = read(conf)- facts[conf] = None if source is None else ConftestFacts.parse(source, conf)- parsed = facts[conf]- if plugin_loader:- route.reason = f"{plugin_loader} loads pytest plugins by name"- elif not route.entries:- route.reason = f"{conf} is reached by a call alone" if route.direct else None- elif parsed is None:- route.reason = f"{conf} could not be read"- else:- key = (conf, route.entries)- if key not in uses:- uses[key] = parsed.use(route.entries)- route.use = uses[key]- route.reason = route.use.run_all-- narrowable = {- conf- for by_conf in routes.values()- for conf, r in by_conf.items()- if r.use is not None and r.reason is None- }+ for conf, route in (pair for by_conf in routes.values() for pair in by_conf.items()):+ if conf not in facts:+ source = read(conf)+ facts[conf] = None if source is None else ConftestFacts.parse(source, conf)+ _judge(conf, route, facts[conf], plugin_loader, uses)+++def _judge(+ conf: str,+ route: _Route,+ parsed: ConftestFacts | None,+ plugin_loader: str | None,+ uses: dict[tuple[str, frozenset[str]], ConftestUse],+) -> None:+ if plugin_loader:+ route.reason = f"{plugin_loader} loads pytest plugins by name"+ elif not route.entries:+ route.reason = f"{conf} is reached by a call alone" if route.direct else None+ elif parsed is None:+ route.reason = f"{conf} could not be read"+ else:+ key = (conf, route.entries)+ if key not in uses:+ uses[key] = parsed.use(route.entries)+ route.use = uses[key]+ route.reason = route.use.run_all+++async def _load_scope(+ session: AsyncSession,+ repo_id: str,+ routes: Mapping[str, Mapping[str, _Route]],+ parents: Mapping[str, Collection[str]],+ test_files: Collection[str],+) -> _Scope:+ """The edges into every narrowable conftest and its fixtures, read once for all targets."""+ judged = [(conf, r) for by_conf in routes.values() for conf, r in by_conf.items()]+ narrowable = {conf for conf, r in judged if r.use is not None and r.reason is None}declared = {f"{conf}::{name}"- for by_conf in routes.values()- for conf, r in by_conf.items()+ for conf, r in judgedif conf in narrowable and r.use is not Nonefor name in r.use.declared}@@ -582,19 +627,12 @@ async def scoped_candidates(r.use is not Noneand r.use.fixturesand not any(f"{conf}::{n}" in stamped for n in r.use.declared)- for by_conf in routes.values()- for conf, r in by_conf.items()+ for conf, r in judgedif conf in narrowable)# Asked only when a conftest's fixtures show no recorded request at all.recorded = await _records_fixture_requests(session, repo_id) if unstamped else True- scope = _Scope(edges, parents, test_files, recorded)-- out: dict[str, list[tuple[str, str]]] = {}- notes: dict[tuple[str, str], list[str]] = {}- for target, picks in found.items():- out[target] = _narrow(target, dict(picks), routes.get(target, {}), scope, notes)- return out, [_note(key, targets) for key, targets in notes.items()]+ return _Scope(edges, parents, test_files, recorded)def _note(key: tuple[str, str], targets: list[str]) -> str:@@ -612,10 +650,20 @@ class _Scope:test_files: Collection[str]# The index stamps fixture requests (see :func:`_records_fixture_requests`).recorded: bool+ _first: dict[str, str | None] = field(default_factory=dict, compare=False)def unresolved_conftest_import(self) -> str | None:return next((e.source for e in self.edges if e.target.startswith("external:")), None)+ def first_test_under(self, prefix: str) -> str | None:+ """The first runnable test file by path under *prefix*, asked once per conftest."""+ if prefix not in self._first:+ self._first[prefix] = min(+ (t for t in self.test_files if t.startswith(prefix) and is_runnable_test(t)),+ default=None,+ )+ return self._first[prefix]+def _narrow(target: str,@@ -668,10 +716,7 @@ def _narrow(if any(t.startswith(prefix) and is_runnable_test(t) for t in result):check = Noneelse:- check = min(- (t for t in scope.test_files if t.startswith(prefix) and is_runnable_test(t)),- default=None,- )+ check = scope.first_test_under(prefix)if check:result.setdefault(check, "conftest-import-check")_add_note(notes, _narrow_note(conf, routes[conf], check), target)diff --git a/packages/core/src/repowise/core/analysis/test_collection.py b/packages/core/src/repowise/core/analysis/test_collection.pyindex 309fa1410..acd34d4ac 100644--- a/packages/core/src/repowise/core/analysis/test_collection.py+++ b/packages/core/src/repowise/core/analysis/test_collection.py@@ -22,7 +22,7 @@ from __future__ import annotationsimport asyncioimport sys-from collections.abc import Callable, Collection, Mapping, Sequence+from collections.abc import Callable, Collection, Iterator, Mapping, Sequencefrom dataclasses import dataclassfrom pathlib import Path, PurePosixPathfrom typing import Any, NamedTuple@@ -46,7 +46,9 @@ class Checkout:*read* returns a tracked file's text (``None`` when unreadable) and*exists* whether a path is present; :func:`read_checkout` answers both from- disk, a server may answer them from what it stores.+ disk, a server may answer them from what it stores. *holding*, when given,+ answers which tracked files hold any of some names without reading each+ file here (``None`` when it cannot); without it every source is read."""tracked: list[str]@@ -54,6 +56,7 @@ class Checkout:roots: PytestRootsread: Callable[[str], str | None]exists: Callable[[str], bool]+ holding: Callable[[Collection[str]], Collection[str] | None] | None = Nonedef read_checkout(repo_path) -> Checkout:@@ -78,7 +81,37 @@ def read_checkout(repo_path) -> Checkout:if (text := read(p)) is not None]roots = read_pytest_roots((p, t) for p, t in texts if Path(p).name in PYTEST_CONFIG_NAMES)- return Checkout(tracked, texts, roots, read, lambda p: (root / p).is_file())+ return Checkout(+ tracked, texts, roots, read, lambda p: (root / p).is_file(), lambda n: _git_holding(root, n)+ )+++# A cold grep of a large checkout takes seconds; past this the full read answers.+_GREP_TIMEOUT_SECONDS = 30+++def _git_holding(root: Path, needles: Collection[str]) -> set[str] | None:+ """Tracked files whose working-tree bytes hold any of *needles*: one ``git grep``.++ The names go in on stdin (``-f -``), so no argument list grows with them.+ ``None`` when git cannot answer in time, or for a needle that is not plain+ ASCII on one line, whose matches only the Python read reports. Ceiling: a+ name split by bytes that are not UTF-8 matches the lenient read only.+ """+ from .doc_drift.suggest import git_run++ needles = sorted(set(needles))+ if not needles or not all(n.isascii() and n.isprintable() for n in needles):+ return None+ patterns = "".join(f"{n}\n" for n in needles).encode("utf-8")+ ran = git_run(+ root, "grep", "-l", "-z", "-F", "--no-color", "-f", "-", "--",+ stdin=patterns, timeout=_GREP_TIMEOUT_SECONDS,+ ) # fmt: skip+ # Exit 1 is "no file holds any".+ if ran is None or ran[0] not in (0, 1):+ return None+ return {p for p in ran[1].decode("utf-8", errors="replace").split("\0") if p}class SelectionCancelledError(Exception):@@ -116,7 +149,7 @@ def plan_scopes(needs_namers,trigger_scopes,)- from .test_selection import file_namers, is_scan_source+ from .test_selection import file_namerspaths = [*change.files, *change.deleted]# A file that runs everything anyway makes every namer search moot.@@ -124,18 +157,36 @@ def plan_scopes(asked = [] if blocked else [p for p in paths if needs_namers(p, config)]namers: dict[str, list[str]] = {}if asked:- known = dict(checkout.pytest_texts) # conftests and pytest configs, read already- texts = (- (p, known[p]) if p in known else (p, checkout.read(p) or "")- for p in checkout.tracked- if is_scan_source(p) and not _stop(cancelled)- )- namers = file_namers(asked, texts)+ namers = file_namers(asked, _scan_texts(checkout, asked, cancelled))scopes = trigger_scopes(paths, checkout.tracked, namers, config)routes = sorted({r for scope in scopes.values() for r in scope.routes} - set(paths))return Plan(namers, scopes, routes)+def _scan_texts(+ checkout: Checkout, asked: Collection[str], cancelled: Callable[[], bool]+) -> Iterator[tuple[str, str]]:+ """``(path, text)`` of every scan source, in tracked order, for :func:`file_namers`.++ With ``checkout.holding`` only the code files holding one of the asked+ names are read: a file holding none names nothing, so :func:`file_namers`+ answers the same. Which name each holds is still decided on its text, so+ one name inside another is attributed exactly as a full read would.+ """+ from .test_selection import is_scan_source++ known = dict(checkout.pytest_texts) # conftests and pytest configs, read already+ names = {PurePosixPath(p).name for p in asked}+ held = checkout.holding(names) if checkout.holding else None+ for p in checkout.tracked:+ if not is_scan_source(p) or _stop(cancelled):+ continue+ if p in known:+ yield p, known[p]+ elif held is None or p in held:+ yield p, checkout.read(p) or ""++async def collect(session,repo_id: str,@@ -277,16 +328,43 @@ def _gap_routes(repo_path, change, config, repo_keys: set[str], out: dict) -> lispecs = [(rev, p) for p in plan.candidates for rev in (old, new)]blobs = read_blobs(str(repo_path), specs)# Unreadable means nothing was compared, so every candidate may have moved.+ unread = _unread(repo_path, specs, blobs)rewired = [pfor p in plan.candidates- if blobs is None or edges_may_differ(p, blobs.get((old, p)), blobs.get((new, p)))+ if blobs is None+ or p in unread+ or edges_may_differ(p, blobs.get((old, p)), blobs.get((new, p)))]plan = with_rewired(plan, rewired, repo_keys)out["gap"] = planreturn list(plan.targets)+def _unread(repo_path, specs: list[tuple[str, str]], blobs: Mapping | None) -> set[str]:+ """Paths a blob read left out although the file exists at that revision.++ A read that stops early (an unparseable header) drops the rest, which+ would read as "added" or "deleted", i.e. unmoved. One batch existence check+ over the missing specs tells those apart from files really absent there;+ when it cannot answer, every missing path counts as unread.+ """+ if blobs is None:+ return set()+ missing = [spec for spec in specs if spec not in blobs]+ if not missing:+ return set()+ from .doc_drift.suggest import git_run++ request = "".join(f"{rev}:{path}\n" for rev, path in missing).encode("utf-8")+ ran = git_run(Path(repo_path), "cat-file", "--batch-check=%(objecttype)", stdin=request)+ lines = ran[1].decode("utf-8", "replace").splitlines() if ran and ran[0] == 0 else []+ if len(lines) != len(missing):+ return {path for _, path in missing}+ # A missing object echoes the spec back with " missing"; anything else exists.+ return {path for (_, path), line in zip(missing, lines, strict=True) if line == "blob"}++def query_lines(change, measured: str | None) -> dict[str, set[int] | None]:"""The lines to look up per file, on the side of the diff the map was measured at.@@ -331,6 +409,7 @@ def empty_result(changed_files: int) -> dict:"explain_route": [],"helper_importers": {},"conftest_notes": [],+ "test_hops": {}, # test file -> fewest hops from the change, for ordering only"changed_files": changed_files,"covered": {}, # test_id -> {test_file, source_files: [...]}"inferred": [], # {source_file, test_file, via}@@ -380,13 +459,34 @@ async def resolve_impacted(``unknown``Nothing said anything. Run the full suite."""+ route_only = [r for r in routes if r not in changed]+ graph_targets = sorted({*changed, *route_only})+ has_rows = await _record_coverage(session, repo_id, changed, route_only, out)+ if not graph_targets:+ return out+ try:+ graph = await _graph_candidates(+ session, repo_id, graph_targets, roots, pytest_texts=pytest_texts, changed=changed+ )+ except Exception as exc:+ # Nothing the graph said can be trusted; selection runs everything.+ out["graph_error"] = f"{type(exc).__name__}: {exc}"+ graph = GraphCandidates({}, {}, [], {})+ out["helper_importers"] = graph.importers+ out["conftest_notes"] = graph.notes+ out["test_hops"] = graph.hops+ _record_inferred(graph_targets, graph.candidates, changed, has_rows, repo_keys, out)+ return out+++async def _record_coverage(+ session, repo_id: str, changed: dict[str, set[int] | None], route_only: list[str], out: dict+) -> set[str]:+ """Fill ``out["covered"]`` from the per-test map; return the files it has rows for."""from ..persistence.crud import tests_covering, tests_covering_files- from .test_reachability import tests_matching_by_namecovered: dict[str, dict] = out["covered"]has_rows: set[str] = set()- route_only = [r for r in routes if r not in changed]- graph_targets = sorted({*changed, *route_only})by_file = {f: await tests_covering(session, repo_id, f, lines=ls) for f, ls in changed.items()}if route_only and not out.get("map_empty"):by_file.update(await tests_covering_files(session, repo_id, set(route_only)))@@ -399,41 +499,56 @@ async def resolve_impacted()if source_file not in entry["source_files"]:entry["source_files"].append(source_file)+ return has_rows- if not graph_targets:- return out- try:- candidates, importers, notes = await _graph_candidates(- session, repo_id, graph_targets, roots, pytest_texts=pytest_texts- )- except Exception as exc:- # Nothing the graph said can be trusted; selection runs everything.- out["graph_error"] = f"{type(exc).__name__}: {exc}"- candidates, importers, notes = {}, {}, []- out["helper_importers"] = importers- out["conftest_notes"] = notes+def _record_inferred(+ graph_targets: list[str],+ candidates: dict[str, list],+ changed: Collection[str],+ has_rows: set[str],+ repo_keys: set[str],+ out: dict,+) -> None:+ """``out["inferred"]`` and ``out["unknown"]``, in target order.++ The filename pattern answers only a changed file nothing else did; a+ route is only looked up to link tests to the change.+ """+ from .test_reachability import tests_matching_by_name++ unanswered = [+ f for f in graph_targets if not candidates.get(f) and f not in has_rows and f in changed+ ]+ # One name index for every file that needs a guess, not one per file.+ guesses = tests_matching_by_name(unanswered, repo_keys) if unanswered else {}+ asked = set(unanswered)for source_file in graph_targets:found = candidates.get(source_file)if found:out["inferred"].extend({"source_file": source_file, "test_file": t, "via": via} for t, via in found)- continue- if source_file in has_rows or source_file not in changed:- continue # coverage answered, or a route; a name-shaped guess adds nothing- guess = tests_matching_by_name([source_file], repo_keys).get(source_file)- if guess:- out["inferred"].append(- {- "source_file": source_file,- "test_file": guess.tests[0],- "via": "filename-pattern",- }- )- else:- out["unknown"].append(source_file)- return out+ elif source_file in asked:+ if guess := guesses.get(source_file):+ out["inferred"].append(+ {+ "source_file": source_file,+ "test_file": guess.tests[0],+ "via": "filename-pattern",+ }+ )+ else:+ out["unknown"].append(source_file)+++class GraphCandidates(NamedTuple):+ """What :func:`_graph_candidates` found: per-target picks, importers, notes, hops."""++ candidates: dict[str, list]+ importers: dict[str, list[str]]+ notes: list[str]+ hops: dict[str, int]async def _graph_candidates(@@ -443,7 +558,8 @@ async def _graph_candidates(roots: PytestRoots | None = None,*,pytest_texts: dict | None = None,-) -> tuple[dict[str, list], dict[str, list[str]], list[str]]:+ changed: Collection[str] | None = None,+) -> GraphCandidates:"""``{target: [(test file, via), ...]}`` from the graph, each test file's importers, notes.Candidates come strongest tier first. The importers map says, for every@@ -459,16 +575,21 @@ async def _graph_candidates(``tests/``) are added until nothing new appears. Only code counts as a testnode: a JSON or golden file the index flags as test material is data, not aroute. A read failure raises.++ The import edges are read once (:func:`load_import_graph`) and every walk+ runs over them in memory; *hops* is each candidate's distance to the+ *changed* files (every target when not given; :func:`_test_hops`)."""from .conftest_routes import scoped_candidates- from .test_reachability import load_test_files+ from .test_reachability import load_import_graph, load_test_filesfrom .test_selection import is_code_file, plugin_loadertexts = pytest_texts or {}test_files = {f for f in await load_test_files(session, repo_id) if is_code_file(f)}- found, entries = await _tier_picks(session, repo_id, targets, test_files, roots)+ graph = await load_import_graph(session, repo_id)+ found, entries, calls = await _tier_picks(session, repo_id, targets, test_files, roots, graph)seeds = {t for picks in found.values() for t in picks}- parents, parent_entries = await _test_importers(session, repo_id, seeds, test_files)+ parents, parent_entries = await _test_importers(session, repo_id, seeds, test_files, graph)out, notes = await scoped_candidates(session,repo_id,@@ -480,7 +601,38 @@ async def _graph_candidates(texts.get,plugin_loader=plugin_loader(texts.items()),)- return out, {test: sorted(found_by) for test, found_by in parents.items()}, notes+ importers = {test: sorted(found_by) for test, found_by in parents.items()}+ sources = targets if changed is None else [t for t in targets if t in changed]+ hops = _test_hops(out, calls, graph.hops(sources), set(sources))+ return GraphCandidates(out, importers, notes, hops)+++def _test_hops(+ candidates: Mapping[str, list],+ calls: Mapping[str, Mapping[str, Any]],+ imports: Mapping[str, int],+ sources: Collection[str],+) -> dict[str, int]:+ """``{test file: fewest hops from any of *sources*}`` for every candidate the graph measures.++ 0 for a changed test, else the fewer of the call walk's hops (*calls*, per+ target) and the file-dependency hops (*imports*, :meth:`ImportGraph.hops`+ from *sources*), so 1 is a direct edge. A scope or gap route is not a+ source. Ordering only: no test is selected for it. A test neither+ measures (a filename guess, one reached only through a route) is absent.+ """+ best: dict[str, int] = {}+ for target, picks in candidates.items():+ by_call = calls.get(target, {}) if target in sources else {}+ for test, via in picks:+ options = [best.get(test), imports.get(test)]+ if via == "changed-test" and target in sources:+ options.append(0)+ if (d := by_call.get(test)) is not None:+ options.append(d.hops)+ if known := [h for h in options if h is not None]:+ best[test] = min(known)+ return bestdef _all_tests(reached) -> tuple[str, ...]:@@ -489,17 +641,24 @@ def _all_tests(reached) -> tuple[str, ...]:async def _tier_picks(- session, repo_id: str, targets: list[str], test_files: set[str], roots: PytestRoots | None-) -> tuple[dict[str, dict[str, str]], dict[str, dict]]:+ session,+ repo_id: str,+ targets: list[str],+ test_files: set[str],+ roots: PytestRoots | None,+ graph=None,+) -> tuple[dict[str, dict[str, str]], dict[str, dict], dict[str, Mapping[str, Any]]]:"""``{target: {test file: via}}``: a changed test itself, then the call and import walks.Also ``{target: {test file: the files it was reached through}}`` from the- import walk.+ import walk, and the call walk's distances per target."""from .test_reachability import tests_reaching_by_tierfrom .test_selection import is_runnable_test- reaching = await tests_reaching_by_tier(session, repo_id, targets, test_files=test_files)+ reaching = await tests_reaching_by_tier(+ session, repo_id, targets, test_files=test_files, import_graph=graph+ )importers = await tests_reaching_by_tier(session,repo_id,@@ -507,6 +666,7 @@ async def _tier_picks(call_depth=0,import_depth=_IMPORT_CLOSURE_DEPTH,test_files=test_files,+ import_graph=graph,)found: dict[str, dict[str, str]] = {}entries = {t: dict(r.entries or {}) for t, r in importers.items()}@@ -518,11 +678,12 @@ async def _tier_picks(for reached, via in ((reaching.get(target), None), (importers.get(target), "import-graph")):for t in _all_tests(reached) if reached else ():picks.setdefault(t, via or reached.via)- return found, entries+ calls = {t: r.reach for t, r in reaching.items() if r.reach}+ return found, entries, callsasync def _test_importers(- session, repo_id: str, seeds: set[str], test_files: set[str]+ session, repo_id: str, seeds: set[str], test_files: set[str], graph=None) -> tuple[dict[str, set[str]], dict[str, dict]]:"""``{test file: test files importing it}``, walked until nothing new appears.@@ -541,6 +702,7 @@ async def _test_importers(call_depth=0,import_depth=_IMPORT_CLOSURE_DEPTH,test_files=test_files,+ import_graph=graph,)frontier = set()for test, reached in walked.items():@@ -665,7 +827,9 @@ async def narrow_scopes(log.debug("narrow_scopes_checkout_failed", error=str(exc))files = sorted({t.split("::", 1)[0] for t in scoped})try:- candidates, _, _ = await _graph_candidates(session, repo_id, files, pytest_texts=texts)+ candidates = (+ await _graph_candidates(session, repo_id, files, pytest_texts=texts)+ ).candidatesexcept Exception as exc: # the walk's own answer stands: every test under each scopelog.debug("narrow_scopes_failed", error=str(exc))candidates = {}diff --git a/packages/core/src/repowise/core/analysis/test_reachability.py b/packages/core/src/repowise/core/analysis/test_reachability.pyindex eed3d1b8d..8d08deaf9 100644--- a/packages/core/src/repowise/core/analysis/test_reachability.py+++ b/packages/core/src/repowise/core/analysis/test_reachability.py@@ -195,6 +195,7 @@ __all__ = ["MAX_TESTS_PER_TARGET","UNRELIABLE_CALL_ORIGINS","CallGraphView",+ "ImportGraph","ReachDistance","ReachedBy","any_tests_reaching",@@ -207,6 +208,7 @@ __all__ = ["files_reached_by_tests","files_with_paired_tests","imported_names_by_test",+ "load_import_graph","load_test_files","rank_tests","reach_into_symbols",@@ -665,6 +667,7 @@ async def tests_reaching_by_tier(symbol_seeds: Mapping[str, Collection[str]] | None = None,test_files: set[str] | None = None,avoid: Collection[str] = frozenset(),+ import_graph: ImportGraph | None = None,) -> dict[str, ReachedBy]:""":func:`tests_reaching`, also saying which tier answered each target.@@ -682,6 +685,9 @@ async def tests_reaching_by_tier(into a target's own symbols. The import tier isone hop, so it never passes through anything.+ *import_graph* (:func:`load_import_graph`) answers the import tier's+ levels from memory, for a caller that walks deep or more than once.+The call walk runs first; the import walk is then seeded with only thetargets it left unanswered, so the weaker tier never speaks over thestronger one and costs nothing where the stronger one already spoke.@@ -717,7 +723,7 @@ async def tests_reaching_by_tier(unanswered = [seed for seed in seeds if seed not in out]if unanswered and import_depth >= 1:found, entries = await _import_reaching(- session, repo_id, unanswered, test_files, import_depth+ session, repo_id, unanswered, test_files, import_depth, import_graph)for seed, tests in found.items():ordered = tuple(rank_tests(seed.split("::", 1)[0], tests))@@ -929,20 +935,78 @@ async def _imported_names(return out+@dataclass(frozen=True)+class ImportGraph:+ """Every file-dependency edge, reversed (``{file: files depending on it}``), read once.++ The import walk is unbounded on the selection path, so on a hub it runs+ dozens of levels; one bulk read replaces a query per level. Either source+ gives the same answer: the walk reaches a fixpoint, so edge order does not+ matter. Ceiling: read per selection, not kept across calls; caching it per+ index stamp is the upgrade path if the read shows up.+ """++ importers: Mapping[str, tuple[str, ...]]++ def edges_into(self, targets: Collection[str]) -> list[tuple[str, str]]:+ """``(dependent, dependency)`` pairs, as :func:`_edges_into` returns them."""+ return [(src, t) for t in targets for src in self.importers.get(t, ())]++ def hops(self, sources: Collection[str]) -> dict[str, int]:+ """``{file: fewest dependency hops from any of *sources*}``, sources at 0.++ One multi-source breadth-first pass through every file, tests+ included, so a test importing a reached helper is one hop past it.+ """+ best = dict.fromkeys(sources, 0)+ frontier = list(best)+ depth = 0+ while frontier:+ depth += 1+ fresh = [+ d for node in frontier for d in self.importers.get(node, ()) if d not in best+ ]+ fresh = list(dict.fromkeys(fresh))+ best.update(dict.fromkeys(fresh, depth))+ frontier = fresh+ return best+++async def load_import_graph(session: AsyncSession, repo_id: str) -> ImportGraph:+ """The edges the import walk follows (``_edges_into`` over every file), in one read."""+ params: dict[str, Any] = {"repo_id": repo_id}+ ets = _in_clause("e", sorted(FILE_DEPENDENCY_EDGE_TYPES), params)+ rows = await session.execute(+ text(+ "SELECT DISTINCT source_node_id, target_node_id FROM graph_edges "+ f"WHERE repository_id = :repo_id AND edge_type IN ({ets}){_LEGACY_CONFTEST_FILTER}"+ ),+ params,+ )+ importers: dict[str, list[str]] = {}+ for source, target in rows:+ importers.setdefault(target, []).append(source)+ # Sorted, so a bounded walk does not depend on the order rows come back in.+ return ImportGraph({t: tuple(sorted(s)) for t, s in importers.items()})++async def _import_reaching(session: AsyncSession,repo_id: str,seeds: list[str],test_files: set[str],max_depth: int,+ graph: ImportGraph | None = None,) -> tuple[dict[str, set[str]], dict[str, dict[str, set[str]]]]:"""Tests that import each seed file, directly or within *max_depth* hops.Also ``{seed: {test: the files it imports on a route to the seed}}``.+ With *graph* the levels are read from it instead of the database."""origins: dict[str, set[str]] = {s: {s} for s in seeds}- found: dict[str, set[str]] = {}- entries: dict[str, dict[str, set[str]]] = {}+ # (test, file it imports) -> seeds carried in: one set union per edge, not+ # a dict write per seed; split per seed once the walk ends.+ reached: dict[tuple[str, str], set[str]] = {}seed_set = set(seeds)frontier = list(seeds)@@ -951,16 +1015,19 @@ async def _import_reaching(breaklevel, frontier = frontier, []queued: set[str] = set()- for dependent, dependency in await _edges_into(- session, repo_id, level, sorted(FILE_DEPENDENCY_EDGE_TYPES), frozenset()- ):+ edges = (+ graph.edges_into(level)+ if graph is not None+ else await _edges_into(+ session, repo_id, level, sorted(FILE_DEPENDENCY_EDGE_TYPES), frozenset()+ )+ )+ for dependent, dependency in edges:carried = origins.get(dependency)if not carried:continueif dependent in test_files:- for seed in carried:- found.setdefault(seed, set()).add(dependent)- entries.setdefault(seed, {}).setdefault(dependent, set()).add(dependency)+ reached.setdefault((dependent, dependency), set()).update(carried)continueif dependent in seed_set:continue@@ -975,6 +1042,12 @@ async def _import_reaching(if dependent not in queued:queued.add(dependent)frontier.append(dependent)+ found: dict[str, set[str]] = {}+ entries: dict[str, dict[str, set[str]]] = {}+ for (test, dependency), carried in reached.items():+ for seed in carried:+ found.setdefault(seed, set()).add(test)+ entries.setdefault(seed, {}).setdefault(test, set()).add(dependency)return found, entriesdiff --git a/packages/core/src/repowise/core/analysis/test_selection.py b/packages/core/src/repowise/core/analysis/test_selection.pyindex 063f12f83..ecf7d4e81 100644--- a/packages/core/src/repowise/core/analysis/test_selection.py+++ b/packages/core/src/repowise/core/analysis/test_selection.py@@ -382,6 +382,12 @@ def plugin_loader(sources: Iterable[tuple[str, str]]) -> str | None:PACKAGE_INIT_REASON = "every import of the package runs it, and those are not all tracked"+# Path predicates the selection asks hundreds of thousands of times on a large+# change (once per test per changed file); each is pure in the path.+_PATH_MEMO = 1 << 16++[email protected]_cache(maxsize=_PATH_MEMO)def scope_kind(path: str) -> str | None:"""``test-package`` or ``conftest`` for a file every test under its directory runs.@@ -418,12 +424,13 @@ def is_runnable_test(path: str, roots: PytestRoots | None = None) -> bool:With *roots*, a Python file pytest's config leaves out of collection(``core/test_paths.py``) is not one: the same rule that stamps ``is_test``."""+ return _runnable_name(path) and (roots is None or is_test_path(path, roots=roots))++[email protected]_cache(maxsize=_PATH_MEMO)+def _runnable_name(path: str) -> bool:p = PurePosixPath(path)- return (- p.suffix.lower() in _TEST_CODE_SUFFIXES- and is_test_path(p.name)- and (roots is None or is_test_path(path, roots=roots))- )+ return p.suffix.lower() in _TEST_CODE_SUFFIXES and is_test_path(p.name)@functools.lru_cache(maxsize=8)@@ -704,7 +711,7 @@ def _scope_tests(scope: Scope, ev: _Evidence) -> list[str]:return []return [t- for t in _under({scope.root}, ev.known_tests)+ for t in _known_under(ev, {scope.root})if (not scope.suffixes or t.lower().endswith(scope.suffixes)) and t not in ev.deleted]@@ -952,6 +959,8 @@ class _Evidence:every_subset: frozenset[str] = frozenset()# Explanations that do not force a full run, gathered while deciding.notes: list[str] = field(default_factory=list)+ # Known tests under each directory set (:func:`_known_under`), per selection.+ under_memo: dict[frozenset[str], list[str]] = field(default_factory=dict, compare=False)@classmethoddef of(cls, inp: SelectionInput, deleted: set[str]) -> _Evidence:@@ -1056,7 +1065,16 @@ def _scope_files(path: str, found: list[_TestRef], basis: str) -> tuple[set[str]def _expand_scopes(tests: list[_TestRef], scopes: set[str], ev: _Evidence) -> list[_TestRef]:kept = [(t, f) for t, f in tests if f not in scopes]- return kept + [(t, t) for t in _tests_under(scopes, ev.known_tests) if t not in ev.deleted]+ dirs = {str(PurePosixPath(i).parent) for i in scopes}+ return kept + [(t, t) for t in _known_under(ev, dirs) if t not in ev.deleted]+++def _known_under(ev: _Evidence, dirs: Collection[str]) -> list[str]:+ """Known tests below any of *dirs*, worked out once per directory set per selection."""+ key = frozenset(dirs)+ if key not in ev.under_memo:+ ev.under_memo[key] = _under(key, ev.known_tests)+ return ev.under_memo[key]def _own_helper_reasons(path: str, ev: _Evidence) -> list[str]:diff --git a/packages/server/src/repowise/server/mcp_server/_test_selection.py b/packages/server/src/repowise/server/mcp_server/_test_selection.pyindex fa0badd1b..5e97ce63c 100644--- a/packages/server/src/repowise/server/mcp_server/_test_selection.py+++ b/packages/server/src/repowise/server/mcp_server/_test_selection.py@@ -23,10 +23,8 @@ from repowise.server.mcp_server._budget import OmissionCollector_REASONS_LIMIT = 3#: Changed files whose deciding evidence is named._BASIS_LIMIT = 10-#: Past this the agent is told to run the CLI instead of waiting. Ceiling: a-#: change whose triggers scope a whole workspace walks every route to it (two-#: minutes on a 2k-test repository); loading the graph once per call is the-#: upgrade that lifts it.+#: Past this the agent is told to run the CLI instead of waiting. Ceiling: the+#: call walk is per-level SQL; loading call edges once is the upgrade.SELECTION_TIMEOUT_SECONDS = 30.0#: What a failed selection tells the reader: it vouches for no subset.diff --git a/tests/unit/analysis/test_selection_scopes.py b/tests/unit/analysis/test_selection_scopes.pyindex 2dca88b8e..fda58ac80 100644--- a/tests/unit/analysis/test_selection_scopes.py+++ b/tests/unit/analysis/test_selection_scopes.py@@ -292,3 +292,102 @@ def test_unplaced_tests_run_with_a_scoped_selection() -> None:def test_scope_paths_are_posix() -> None:assert all(PurePosixPath(p).as_posix() == p for p in _TRACKED)+++def _namer_checkout(tmp_path):+ """A git checkout where one changed name sits inside another (``a.md`` in ``data.md``)."""+ import subprocess++ from repowise.core.analysis.changed_lines import ChangeSet, FileDiff+ from repowise.core.analysis.test_collection import read_checkout++ files = {+ "web/package.json": "{}",+ "web/src/app.ts": "import pkg from '../package.json'\n",+ "web/src/app.test.ts": "readFileSync('package.json')\n",+ "tools/build.py": "print('web/package.json', 'README.md')\n",+ "tools/data.py": "open('docs/data.md')\n",+ "docs/a.md": "# a\n",+ "docs/data.md": "# data\n",+ "src/mod.py": "x = 1\n",+ "pytest.ini": "[pytest]\naddopts = --doctest-glob=*.md\n",+ "README.md": "readme\n",+ }+ for path, text in files.items():+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / path).write_text(text, encoding="utf-8")+ subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True)+ subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True)+ changed = ("web/package.json", "README.md", "docs/a.md", "docs/data.md")+ change = ChangeSet(+ files={p: FileDiff(path=p) for p in changed},+ deleted=set(),+ label="t",+ base=None,+ head=None,+ )+ return change, read_checkout(tmp_path)+++def test_plan_scopes_names_the_same_files_from_git_grep_as_from_reading_them(tmp_path) -> None:+ """The grep only skips reading files that hold no changed name; nested names still count."""+ from dataclasses import replace++ from repowise.core.analysis.test_collection import plan_scopes++ change, checkout = _namer_checkout(tmp_path)+ assert checkout.holding is not None+ grepped = plan_scopes(change, _NONE, checkout)+ assert grepped == plan_scopes(change, _NONE, replace(checkout, holding=None))+ assert grepped.namers["web/package.json"] == [+ "tools/build.py",+ "web/src/app.test.ts",+ "web/src/app.ts",+ ]+ # "data.md" holds "a.md", so the file naming docs/data.md names docs/a.md too.+ assert grepped.namers["docs/a.md"][-1] == "tools/data.py"+ assert grepped.namers["docs/data.md"][-1] == "tools/data.py"+++def test_a_grep_that_cannot_answer_falls_back_to_reading_every_file(+ tmp_path, monkeypatch+) -> None:+ from dataclasses import replace++ from repowise.core.analysis import test_collection+ from repowise.core.analysis.doc_drift import suggest++ change, checkout = _namer_checkout(tmp_path)+ read = test_collection.plan_scopes(change, _NONE, replace(checkout, holding=None))+ monkeypatch.setattr(suggest, "git_run", lambda *_a, **_k: None) # timed out+ assert test_collection.plan_scopes(change, _NONE, checkout) == read+++def test_a_blob_read_that_stops_early_does_not_read_as_unmoved(tmp_path) -> None:+ """A path left out of a blob read counts as unread only where the file exists."""+ import subprocess++ from repowise.core.analysis.test_collection import _unread++ def git(*args: str) -> str:+ return subprocess.run(+ ["git", *args], cwd=tmp_path, check=True, capture_output=True, text=True+ ).stdout.strip()++ git("init", "-q")+ git("config", "user.email", "[email protected]")+ git("config", "user.name", "t")+ (tmp_path / "kept.py").write_text("import os\n", encoding="utf-8")+ git("add", "-A")+ git("commit", "-q", "-m", "one")+ old = git("rev-parse", "HEAD")+ (tmp_path / "added.py").write_text("import sys\n", encoding="utf-8")+ git("add", "-A")+ git("commit", "-q", "-m", "two")+ new = git("rev-parse", "HEAD")++ specs = [(old, "kept.py"), (new, "kept.py"), (old, "added.py"), (new, "added.py")]+ whole = {(old, "kept.py"): b"x", (new, "kept.py"): b"x", (new, "added.py"): b"y"}+ assert _unread(tmp_path, specs, whole) == set() # added.py is absent at old+ assert _unread(tmp_path, specs, {(old, "kept.py"): b"x"}) == {"kept.py", "added.py"}+ assert _unread(tmp_path, specs, None) == set() # nothing read: the caller fails closeddiff --git a/tests/unit/cli/test_impacted_tests_cmd.py b/tests/unit/cli/test_impacted_tests_cmd.pyindex d183e3273..f6ec819d8 100644--- a/tests/unit/cli/test_impacted_tests_cmd.py+++ b/tests/unit/cli/test_impacted_tests_cmd.py@@ -432,7 +432,9 @@ def test_explain_names_the_changed_file_and_the_route(repo) -> None:assert result.stdout.splitlines() == ["Selected: src/a.py changed (import-graph).","Route: tests/test_a.py -> src/a.py",- "Order: 1 of 1 (reaches a changed file through other files).",+ # tests/test_a.py imports src/a.py: one hop, a direct edge.+ "Order: 1 of 1 (its calls, a fixture or a direct import reach a changed file; "+ "1 hop(s)).",]missed = _run(repo, "main...feat", "--explain", "./tests/test_b.py")assert missed.stdout.startswith("Not selected: no changed file reaches it")@@ -447,18 +449,19 @@ def test_explain_joins_the_json_report(repo) -> None:"lines": ["Selected: src/a.py changed (import-graph).","Route: tests/test_a.py -> src/a.py",- "Order: 1 of 1 (reaches a changed file through other files).",+ "Order: 1 of 1 (its calls, a fixture or a direct import reach a changed file; "+ "1 hop(s)).",],"route": ["tests/test_a.py", "src/a.py"],"order": {"position": 1,"of": 1,"test": "tests/test_a.py",- "tier": "transitive",- "hops": None,+ "tier": "direct",+ "hops": 1,"co_change": 0,"failed_last_run": False,- "reason": "reaches a changed file through other files",+ "reason": "its calls, a fixture or a direct import reach a changed file; 1 hop(s)",},}assert data["selected"]["why"] == {"tests/test_a.py": "src/a.py changed (import-graph)"}@@ -515,7 +518,7 @@ def test_prioritize_lists_the_whole_suite_selected_first(repo) -> None:data = json.loads(_run(repo, "main...feat", "--format", "json", "--prioritize").stdout)assert data["run_all"] is False and data["selected"]["tests"] == ["tests/test_a.py"]assert [(o["test"], o["tier"], o["co_change"]) for o in data["order"]] == [- ("tests/test_a.py", "transitive", 0),+ ("tests/test_a.py", "direct", 0),("tests/test_c.py", "rest", 4),("tests/test_b.py", "rest", 0),]@@ -542,7 +545,8 @@ def test_selected_tests_run_in_order_and_a_last_failure_breaks_a_tie(repo) -> Noassert result.stdout == "tests/test_z.py tests/test_a.py\n"explained = _run(repo, "main...feat", "--explain", "tests/test_z.py")assert explained.stdout.splitlines()[-1] == (- "Order: 1 of 2 (reaches a changed file through other files; failed in the last run)."+ "Order: 1 of 2 (its calls, a fixture or a direct import reach a changed file; "+ "1 hop(s); failed in the last run).")diff --git a/tests/unit/persistence/test_impacted_tests_resolution.py b/tests/unit/persistence/test_impacted_tests_resolution.pyindex 14363fdfe..b5d182796 100644--- a/tests/unit/persistence/test_impacted_tests_resolution.py+++ b/tests/unit/persistence/test_impacted_tests_resolution.py@@ -167,6 +167,7 @@ async def test_a_changed_test_file_is_its_own_candidate(async_session) -> None:}]assert out["unknown"] == []+ assert out["test_hops"] == {"tests/test_bar.py": 0}async def _graph(session, repo_id, tests: set[str], files: set[str], imports: list[tuple]):@@ -220,6 +221,12 @@ async def test_the_import_graph_reaches_tests_through_modules_and_other_tests("tests/test_child.py": "import-graph",}assert out["unknown"] == []+ # Fewest dependency hops from the change, for ordering; 1 is a direct import.+ assert out["test_hops"] == {+ "tests/base.py": 1,+ "tests/test_api.py": 2,+ "tests/test_child.py": 2,+ }async def test_graph_candidates_are_not_capped(async_session) -> None:@@ -587,3 +594,41 @@ async def test_narrow_scopes_keeps_the_whole_scope_for_a_target_the_walk_does_no)assert out == {"src/unindexed.py": ["tests/test_a.py", "tests/test_b.py", "tests/test_c.py"]}+++async def test_the_ranking_orders_by_the_hops_the_collection_measures(async_session) -> None:+ """A test importing the changed file directly runs before one importing it through another."""+ from repowise.core.analysis.changed_lines import ChangeSet, FileDiff+ from repowise.core.analysis.test_ranking import rank_change+ from repowise.core.analysis.test_selection import Selection++ repo = await insert_repo(async_session)+ await _graph(+ async_session,+ repo.id,+ tests={"tests/test_far.py", "tests/test_near.py"},+ files={"src/api.py", "src/core.py"},+ imports=[+ ("src/api.py", "src/core.py"),+ ("tests/test_far.py", "src/api.py"),+ ("tests/test_near.py", "src/core.py"),+ ],+ )+ out = _empty_result(1)+ await _resolve_impacted(async_session, repo.id, {"src/core.py": None}, set(), out)+ assert out["test_hops"] == {"tests/test_near.py": 1, "tests/test_far.py": 2}++ change = ChangeSet({"src/core.py": FileDiff(path="src/core.py")}, set(), "t", None, None)+ tests = ("tests/test_far.py", "tests/test_near.py")+ selection = Selection(+ run_all=False,+ reasons=(),+ tests=tests,+ test_files=tests,+ basis={"src/core.py": "import-graph"},+ )+ ranked = await rank_change(None, repo.id, change, out, selection, lambda _p: None)+ assert [(r.test, r.tier, r.hops) for r in ranked] == [+ ("tests/test_near.py", "direct", 1),+ ("tests/test_far.py", "transitive", 2),+ ]diff --git a/tests/unit/persistence/test_test_reachability_walk.py b/tests/unit/persistence/test_test_reachability_walk.pyindex ab61f8f02..e716c509f 100644--- a/tests/unit/persistence/test_test_reachability_walk.py+++ b/tests/unit/persistence/test_test_reachability_walk.py@@ -647,3 +647,53 @@ async def test_a_hub_import_the_graph_cannot_tie_to_a_symbol_still_counts(async_"tests/test_barrel.py",}assert lonely.prerequisite is None+++async def test_the_import_walk_from_memory_matches_the_walk_from_the_database(async_session):+ """One bulk read answers the levels the per-level queries do, at any depth."""+ import sys++ from repowise.core.analysis.test_reachability import load_import_graph++ repo = await insert_repo(async_session)+ files = ["src/a.py", "src/b.py", "src/c.py", "src/d.py", "src/e.py"]+ tests = ["tests/test_a.py", "tests/test_d.py", "tests/helper.py", "tests/test_h.py"]+ await _seed(+ async_session,+ repo.id,+ nodes={**dict.fromkeys(files, False), **dict.fromkeys(tests, True)},+ edges=[+ # A diamond (d reaches a through b and c), a cycle (b <-> e) and a+ # test importing a test helper.+ ("src/b.py", "src/a.py", "imports"),+ ("src/c.py", "src/a.py", "imports"),+ ("src/d.py", "src/b.py", "imports"),+ ("src/d.py", "src/c.py", "type_use"),+ ("src/e.py", "src/b.py", "imports"),+ ("src/b.py", "src/e.py", "imports"),+ ("tests/test_a.py", "src/a.py", "imports"),+ ("tests/test_d.py", "src/d.py", "imports"),+ ("tests/helper.py", "src/e.py", "imports"),+ ("tests/test_h.py", "tests/helper.py", "imports"),+ ],+ )+ graph = await load_import_graph(async_session, repo.id)+ test_files = set(tests)+ for depth in (1, 2, sys.maxsize):+ args = dict(call_depth=0, import_depth=depth, test_files=test_files)+ stored = await by_tier(async_session, repo.id, ["src/a.py", "src/c.py"], **args)+ held = await by_tier(+ async_session, repo.id, ["src/a.py", "src/c.py"], import_graph=graph, **args+ )+ assert held == stored+ assert graph.hops(["src/a.py"]) == {+ "src/a.py": 0,+ "src/b.py": 1,+ "src/c.py": 1,+ "tests/test_a.py": 1,+ "src/d.py": 2,+ "src/e.py": 2,+ "tests/test_d.py": 3,+ "tests/helper.py": 3,+ "tests/test_h.py": 4,+ }