Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(impacted-tests): a conftest calling a self-contained function reaches only its module (#3772)

Swati Ahuja5abca6ef5eca
diff --git a/docs/layers/TEST_INTELLIGENCE.md b/docs/layers/TEST_INTELLIGENCE.md
index 504f635f2..aa08bdbb3 100644
--- a/docs/layers/TEST_INTELLIGENCE.md
+++ b/docs/layers/TEST_INTELLIGENCE.md
@@ -283,7 +283,7 @@ over the change's diff, so a measured map at either end of it matches by line:
"covered": [...],
"no_coverage_data": [...]
},
- "summary": "23 test(s) to run for this change; showing first 10; the rest can be skipped."
+ "summary": "23 test(s) to run for this change (9 of 412 test files); showing first 10; the rest can be skipped."
}
```
diff --git a/docs/start/CI.md b/docs/start/CI.md
index 2e129731c..51129c1f5 100644
--- a/docs/start/CI.md
+++ b/docs/start/CI.md
@@ -265,6 +265,7 @@ a per-test coverage map (`repowise coverage add`) makes it more precise.
- A changed `conftest.py`, test package `__init__.py` or imported test helper selects the tests under or importing it, not everything.
- A change that reaches a `conftest.py` only through its imports selects the tests that ask for a fixture using that change (directly, through other fixtures, `usefixtures` or `getfixturevalue`), plus one test under the conftest as an import check.
- Every test under the conftest still runs when an autouse fixture, a `pytest_*` hook or module-level code uses the change, or when the conftest imports it only for side effects.
+ - Reading a literal constant of an imported module, or calling one of its functions that uses nothing the module imports beyond standard modules that cannot load code by name (emptying a `functools` cache or a standard container counts), reaches only a change in that module itself. Ceiling: such a function that reads or runs repo files by path (`open`, `Path.read_text`, `os.system`) is judged by its imports only.
- It also runs everything there when a fixture request cannot be read from the code (a computed name, a mark kept in a variable, a config `usefixtures`), when the conftest loads plugins or cannot be read, and on an index built before this was recorded. The reasons name the cause.
- Ceiling: a change that only alters what the changed module does at import time, seen by a test that does not import it, is caught by the import check alone.
- Wiring no import states counts as a use: a JS/TS setup file named under `setupFiles`, `setupFilesAfterEnv`, `globalSetup` or `globalTeardown` in a `vitest.*`, `jest.*`, `vite.*` or `playwright.*` config or a `package.json` stands for the tests in that config's package, and a helper a test names by a path relative to itself (`Path(__file__).parent / "server.py"`, `path.join(__dirname, "server.ts")`) stands for that test. A `conftest.py` links only the files pytest collects (its `python_files`), so a helper beside the tests is not on every route the conftest reaches.
diff --git a/packages/core/src/repowise/core/analysis/conftest_routes.py b/packages/core/src/repowise/core/analysis/conftest_routes.py
index 7d5be9fe5..6f4c8e26b 100644
--- a/packages/core/src/repowise/core/analysis/conftest_routes.py
+++ b/packages/core/src/repowise/core/analysis/conftest_routes.py
@@ -29,12 +29,13 @@ through its imports can break a test through either:
Granularity is the module, not the function: a fixture that names anything in
a module on the route to the change counts as reaching it, however deep the
-change is. One exception: reading a literal constant of a module on the route
-(a name it binds once, at top level, to a literal, a tuple or a ``frozenset``
-of literals) depends on that module's text alone, so it reaches a change only
-in that module (:func:`literal_constants`). A binding the conftest never
-reads (``import app.models # registers them``) is imported for its side
-effects, which every test sees, so it keeps every test.
+change is. One exception: reading a literal constant of a module on the route,
+or calling a function of it that uses nothing the module imports beyond standard
+modules that cannot load code by name (:func:`local_names`), depends on that
+module's text alone, so it reaches a change only in that module, not one in a
+module it merely imports. A binding the conftest never reads (``import
+app.models # registers them``) is imported for its side effects, which every
+test sees, so it keeps every test.
Fails closed, keeping every test under the conftest with the reason, when the
conftest does not parse, loads plugins, star-imports a module on the route, has
@@ -48,14 +49,19 @@ Ceiling: an import-time side effect of the changed module itself (it registers
itself in a shared registry) that a test observes without any import route of
its own to that module is seen only through the import check. So is an
import-time side effect (plugin registration, a patch) elsewhere in the closure
-of a module whose literal constant the conftest reads, since that read still
-imports it. Upgrade path: a per-test map built from a coverage run, which
-records what each test executed.
+of a module whose local name the conftest reads, since that read still
+imports it. A local function that reads or runs repo files by path (``open``,
+``Path.read_text``, ``os.system``) is judged by its imports only, as the module
+grain already was. Upgrade path: a per-test map built from a coverage run,
+which records what each test executed.
"""
from __future__ import annotations
import ast
+import builtins
+import symtable
+from collections import Counter
from collections.abc import Callable, Collection, Iterator, Mapping
from dataclasses import dataclass, field
from pathlib import PurePosixPath
@@ -71,6 +77,7 @@ from ..ingestion.framework_edges.pytest_edges import (
fixture_declaration,
)
from ..ingestion.languages.python_modules import module_parts
+from ..ingestion.languages.python_strings import PY_DYNAMIC_LOAD_MARKERS
from .test_reachability import _in_clause
from .test_selection import is_runnable_test, scope_kind
@@ -79,6 +86,39 @@ from .test_selection import is_runnable_test, scope_kind
# Probing (`hasattr`) or removing (`delattr`) depends on the module, so counts.
_TARGET_CALLS = frozenset({"setattr"})
_DEFS = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
+# Builtins that run code or reach any module by name (``breakpoint`` and ``help``
+# import one named by the environment or the argument).
+_RUNS_BY_NAME = frozenset(
+ {"eval", "exec", "compile", "globals", "vars", "breakpoint", "help", "__import__"}
+)
+# Standard modules a local function may use: none of them imports or runs a
+# module named by a string, unlike ``importlib``, ``runpy``, ``pickle``,
+# ``pydoc``, ``unittest.mock``, ``logging.config`` or ``sys.modules``. An
+# allowlist so anything not vetted keeps the module grain.
+_NO_LOAD_BY_NAME = frozenset(
+ {
+ "__future__", "abc", "dataclasses", "enum", "types", "typing", # declarations
+ "collections", "functools", "itertools", "operator", "heapq", "bisect",
+ "copy", "weakref", "contextlib", # values and callables passed in
+ "re", "string", "textwrap", "json", "base64", "struct", "hashlib", "uuid",
+ "math", "random", "statistics", "decimal", "fractions", "numbers", # data
+ "datetime", "time", "calendar", "threading", "warnings",
+ "os", "pathlib", "shutil", "tempfile", "glob", "fnmatch", "stat", "errno",
+ "io", # files and processes by path: see the module ceiling
+ }
+) # fmt: skip
+# Attributes through which an allowed module still runs code by name or hops to
+# a denied one (``os.system('python -m app')``, ``shutil.os``, ``re.sys``); any
+# dunder attribute and any ``exec*``/``spawn*`` one count as well.
+_RUNS_BY_ATTRIBUTE = frozenset(
+ {
+ "system", "popen", "startfile", "posix_spawn", "posix_spawnp",
+ "register_archive_format", "register_unpack_format",
+ "sys", "_sys", "os", "modules",
+ }
+) # fmt: skip
+# The ``functools`` decorators whose ``cache_clear`` empties a function's cache.
+_CACHE_DECORATORS = frozenset({"cache", "lru_cache"})
@dataclass(frozen=True)
@@ -253,16 +293,211 @@ def _assigned(stmt: ast.stmt) -> tuple[str, ast.AST] | None:
return (target.id, value) if isinstance(target, ast.Name) and value is not None else None
-def literal_constants(source: str) -> frozenset[str]:
- """Names a Python module binds once, at its top level, to an immutable literal.
+def _stdlib_import(stmt: ast.stmt) -> bool:
+ """Whether *stmt* imports only from :data:`_NO_LOAD_BY_NAME`, by absolute name.
- Reading one depends on that module's text alone, not on what it imports.
- A name bound anywhere else in the module (a local, a ``global``, an import)
- is left out, and so is everything when the module star-imports. Ceiling:
- another module rebinding it at runtime (``mod.NAME = ...``) is not seen.
+ Ceiling: a repo module shadowing one of them on the import path is not seen.
+ """
+ if isinstance(stmt, ast.Import):
+ tops = [a.name.split(".")[0] for a in stmt.names]
+ elif isinstance(stmt, ast.ImportFrom) and not stmt.level and stmt.module:
+ tops = [stmt.module.split(".")[0]]
+ else:
+ return False
+ return all(t in _NO_LOAD_BY_NAME for t in tops)
+
+
+def _is_reset(attr: ast.AST, method: str, parents: Mapping[int, ast.AST]) -> bool:
+ """Whether *attr* is ``<x>.method`` called with no arguments as a statement."""
+ call = parents.get(id(attr))
+ return (
+ isinstance(attr, ast.Attribute)
+ and attr.attr == method
+ and isinstance(call, ast.Call)
+ and call.func is attr
+ and not call.args
+ and not call.keywords
+ and isinstance(parents.get(id(call)), ast.Expr)
+ )
+
+
+def _functools_bindings(body: list[ast.stmt]) -> tuple[set[str], set[str]]:
+ """``(names bound to functools.cache or lru_cache, names bound to functools)``."""
+ decorators: set[str] = set()
+ modules: set[str] = set()
+ for stmt in body:
+ if isinstance(stmt, ast.ImportFrom) and not stmt.level and stmt.module == "functools":
+ decorators |= {a.asname or a.name for a in stmt.names if a.name in _CACHE_DECORATORS}
+ elif isinstance(stmt, ast.Import):
+ modules |= {a.asname or a.name for a in stmt.names if a.name == "functools"}
+ return decorators, modules
+
+
+def _cached(
+ node: ast.FunctionDef | ast.AsyncFunctionDef, outside: Collection[str], body: list[ast.stmt]
+) -> bool:
+ """Whether *node*'s one decorator is ``functools.cache`` or ``functools.lru_cache``."""
+ if len(node.decorator_list) != 1:
+ return False
+ deco = node.decorator_list[0]
+ target = deco.func if isinstance(deco, ast.Call) else deco
+ decorators, modules = _functools_bindings(body)
+ named = (
+ target.id in decorators
+ if isinstance(target, ast.Name)
+ else isinstance(target, ast.Attribute)
+ and target.attr in _CACHE_DECORATORS
+ and getattr(target.value, "id", None) in modules
+ )
+ return named and all(n.id in outside for n in ast.walk(deco) if isinstance(n, ast.Name))
+
+
+@dataclass(frozen=True)
+class _ModuleNames:
+ """How a module binds its top-level names, as :func:`local_names` reads them."""
+
+ parents: Mapping[int, ast.AST]
+ # Names bound once at top level to an immutable literal.
+ literals: frozenset[str]
+ # Names that resolve to the standard library or a builtin.
+ outside: frozenset[str]
+ # Names whose standard-library state one method empties, and that method.
+ resets: Mapping[str, str]
+ functions: Mapping[str, ast.FunctionDef | ast.AsyncFunctionDef]
+
+
+def _runs_by_attribute(attr: str) -> bool:
+ """Whether reading *attr* may run or reach code by name (:data:`_RUNS_BY_ATTRIBUTE`)."""
+ return (
+ attr in _RUNS_BY_ATTRIBUTE
+ or attr.startswith(("exec", "spawn"))
+ or (attr.startswith("__") and attr.endswith("__"))
+ )
+
+
+def _globals_used(
+ node: ast.FunctionDef | ast.AsyncFunctionDef, scope: symtable.SymbolTable
+) -> set[str] | None:
+ """The module names *node* reads, or None when it imports or assigns outside itself."""
+ if any(m in ast.unparse(node) for m in PY_DYNAMIC_LOAD_MARKERS) or any(
+ (isinstance(sub, (ast.Import, ast.ImportFrom)) and not _stdlib_import(sub))
+ or (isinstance(sub, ast.Attribute) and _runs_by_attribute(sub.attr))
+ for sub in ast.walk(node)
+ ):
+ return None
+ # Decorators and defaults run in the module's scope, the body in its own.
+ defaults = [*node.decorator_list, *node.args.defaults, *node.args.kw_defaults]
+ used = {sub.id for e in defaults if e for sub in ast.walk(e) if isinstance(sub, ast.Name)}
+ tables = [scope]
+ while tables:
+ table = tables.pop()
+ tables += table.get_children()
+ for sym in (s for s in table.get_symbols() if s.is_global()):
+ if sym.is_assigned():
+ return None
+ if sym.is_referenced():
+ used.add(sym.get_name())
+ return used
+
+
+def _only_resets(node: ast.AST, name: str, names: _ModuleNames) -> bool:
+ """Whether every use of *name* in *node* empties it with its reset method."""
+ method = names.resets.get(name)
+ return method is not None and all(
+ _is_reset(names.parents.get(id(sub)), method, names.parents)
+ for sub in ast.walk(node)
+ if isinstance(sub, ast.Name) and sub.id == name
+ )
+
+
+def _function_deps(
+ node: ast.FunctionDef | ast.AsyncFunctionDef,
+ scope: symtable.SymbolTable | None,
+ names: _ModuleNames,
+) -> set[str] | None:
+ """The module functions *node* uses, or None when it uses anything not local."""
+ used = _globals_used(node, scope) if scope is not None else None
+ if used is None:
+ return None
+ deps: set[str] = set()
+ for name in used:
+ if _only_resets(node, name, names):
+ continue
+ if name in names.functions:
+ deps.add(name)
+ elif name not in names.literals and name not in names.outside:
+ return None
+ return deps
+
+
+def _module_names(tree: ast.Module, counts: Mapping[str, int]) -> _ModuleNames:
+ """How *tree* binds its top-level names, given how often each is bound."""
+ once = {n for n, c in counts.items() if c == 1}
+ assigned = dict(a for stmt in tree.body if (a := _assigned(stmt)) and a[0] in once)
+ # Dunder names (``__loader__``, ``__spec__``, ``__builtins__``) reach module machinery.
+ plain = (n for n in dir(builtins) if not (n.startswith("__") and n.endswith("__")))
+ outside = {n for n in plain if n not in counts and n not in _RUNS_BY_NAME} | {
+ n
+ for stmt in tree.body
+ if _stdlib_import(stmt)
+ for alias in getattr(stmt, "names", ())
+ if (n := _bound_names(alias)[0]) in once
+ }
+ literals = {n for n, value in assigned.items() if _is_literal(value, counts)}
+ functions = {
+ s.name: s
+ for s in tree.body
+ if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef)) and s.name in once
+ }
+ # A container's import-time contents could come from anywhere, so the module
+ # may only name it where it builds it.
+ at_import = Counter(
+ sub.id for node in _module_level(tree.body)[2] for sub in ast.walk(node)
+ if isinstance(sub, ast.Name)
+ ) # fmt: skip
+ resets = {
+ n: "clear"
+ for n, value in assigned.items()
+ if n not in literals
+ and at_import[n] == 1
+ and isinstance(value, (ast.Call, ast.Dict, ast.List, ast.Set))
+ and all(s.id in outside for s in ast.walk(value) if isinstance(s, ast.Name))
+ } | {n: "cache_clear" for n, node in functions.items() if _cached(node, outside, tree.body)}
+ return _ModuleNames(_parents(tree), frozenset(literals), frozenset(outside), resets, functions)
+
+
+def _not_local(deps: Mapping[str, set[str] | None]) -> set[str]:
+ """The functions that use something not local, directly or through another function."""
+ bad = {n for n, d in deps.items() if d is None}
+ grew = True
+ while grew:
+ grew = False
+ for name, d in deps.items():
+ if name not in bad and d and d & bad:
+ bad.add(name)
+ grew = True
+ return bad
+
+
+def local_names(source: str) -> frozenset[str]:
+ """Names a Python module binds once, at its top level, whose use is its own text alone.
+
+ A literal constant (a literal, a tuple or a ``frozenset`` of literals), and
+ a function whose decorators, defaults and body name only its own locals,
+ builtins, modules in :data:`_NO_LOAD_BY_NAME` and other such names of the
+ module: reading or calling either runs nothing the module imports from
+ elsewhere. Emptying state the standard library holds leaves the same state
+ whatever filled it, so ``NAME.cache_clear`` for a ``functools`` cache and
+ ``NAME.clear`` for a container the module names only where it builds it
+ are listed too (dotted), and a local function may call them. A name bound
+ anywhere else in the module (a local, a ``global``, an import) is left out,
+ and so is everything when the module star-imports. Ceiling: another module
+ rebinding it at runtime (``mod.NAME = ...``) is not seen, nor is an
+ import-time side effect that patches the standard library.
"""
try:
tree = ast.parse(source)
+ table = symtable.symtable(source, "<module>", "exec")
except (SyntaxError, ValueError):
return frozenset()
counts: dict[str, int] = {}
@@ -271,9 +506,16 @@ def literal_constants(source: str) -> frozenset[str]:
counts[n] = counts.get(n, 0) + 1
if "*" in counts:
return frozenset()
- assigned = (a for stmt in tree.body if (a := _assigned(stmt)))
+ names = _module_names(tree, counts)
+ functions = (t for t in table.get_children() if t.get_type() == "function")
+ scopes = {(t.get_name(), t.get_lineno()): t for t in functions}
+ deps = {
+ name: _function_deps(node, scopes.get((name, node.lineno)), names)
+ for name, node in names.functions.items()
+ }
+ local = set(names.functions) - _not_local(deps)
return frozenset(
- name for name, value in assigned if counts[name] == 1 and _is_literal(value, counts)
+ names.literals | local | {f"{n}.{method}" for n, method in names.resets.items()}
)
@@ -439,18 +681,18 @@ class ConftestFacts:
}
def use(
- self, entries: Collection[str], constants: Mapping[str, Collection[str]] | None = None
+ self, entries: Collection[str], local: Mapping[str, Collection[str]] | None = None
) -> ConftestUse:
"""What the conftest does with *entries*, the files it imports on a route.
- *constants* are, per entry the change is not in, its literal constants
- (:func:`literal_constants`): reading one does not reach the change.
+ *local* are, per entry the change is not in, its local names
+ (:func:`local_names`): reading or calling one does not reach the change.
"""
if self.error:
return ConftestUse(run_all=self.error)
# Sorted: an import binds the first entry it names, so order must not vary.
targets = [module_parts(e) for e in sorted(entries)]
- stable = {module_parts(e): names for e, names in (constants or {}).items()}
+ stable = {module_parts(e): names for e, names in (local or {}).items()}
star_import = ConftestUse(run_all=f"{self.path} star-imports a module on the route")
hot, matched, star = self._bind(self.imports, targets)
if star:
@@ -478,7 +720,7 @@ class ConftestFacts:
if (
load.id in hot
and not _patch_target(load, self.parents)
- and not self._reads_constant(load, hot[load.id], stable)
+ and not self._reads_local(load, hot[load.id], stable)
):
return ConftestUse(run_all=f"module-level code in {self.path} uses {load.id}")
if load.id in self.units:
@@ -522,24 +764,29 @@ class ConftestFacts:
continue
if _patch_target(load, self.parents):
r.patch = r.patch or load.id
- elif not self._reads_constant(load, path, stable):
+ elif not self._reads_local(load, path, stable):
r.run = r.run or load.id
# A local import nothing reads is there for its side effects.
if r.run is None and (unread := sorted(set(local) - {n.id for n in unit.loads})):
r.run = unread[0]
return r
- def _reads_constant(
+ def _reads_local(
self,
load: ast.Name,
path: tuple[str, ...],
stable: Mapping[tuple[str, ...], Collection[str]],
) -> bool:
- """Whether *load* (bound to *path*) only reads a literal constant of a *stable* entry."""
+ """Whether *load* (bound to *path*) only reads or calls a local name of a *stable* entry."""
outer, attrs = _dotted_read(load, self.parents)
+ if any(_runs_by_attribute(a) for a in attrs):
+ return False
full = (*path, *attrs)
+ # ``NAME.cache_clear()`` and the like: listed dotted, and only as the call itself.
+ reset = len(full) > 2 and _is_reset(outer, full[-1], self.parents)
return isinstance(getattr(outer, "ctx", None), ast.Load) and any(
- full[k] in names and _names(full[:k], entry, package=False)
+ (full[k] in names or (reset and k == len(full) - 2 and ".".join(full[k:]) in names))
+ and _names(full[:k], entry, package=False)
for entry, names in stable.items()
for k in range(1, len(full))
)
@@ -593,10 +840,10 @@ def conftest_use(
source: str,
path: str,
entries: Collection[str],
- constants: Mapping[str, Collection[str]] | None = None,
+ local: Mapping[str, Collection[str]] | None = None,
) -> ConftestUse:
"""What the conftest at *path* (text *source*) does with *entries*. Pure."""
- return ConftestFacts.parse(source, path).use(entries, constants)
+ return ConftestFacts.parse(source, path).use(entries, local)
@dataclass(frozen=True)
@@ -695,7 +942,7 @@ async def scoped_candidates(
*found* are the walks' picks per target and *entries* the files each was
reached through; *parents* the test files importing each test file and
*parent_entries* the files each importer reached it through; *read* returns
- a conftest's text and *read_source* any file's, for the literal constants a
+ a conftest's text and *read_source* any file's, for the local names a
conftest reads (without it, every read counts as reaching). Every conftest
on a route that this module cannot narrow keeps today's answer: every test
under it.
@@ -756,10 +1003,10 @@ def _judge_routes(
if conf not in facts:
source = read(conf)
facts[conf] = None if source is None else ConftestFacts.parse(source, conf)
- # The changed file's own constants may be what changed.
+ # The changed file's own local names may be what changed.
others = route.entries - {target}
for e in others - set(consts) if read_source else ():
- consts[e] = literal_constants(read_source(e) or "")
+ consts[e] = local_names(read_source(e) or "")
stable = {e: consts[e] for e in others if consts.get(e)}
_judge(conf, route, facts[conf], plugin_loader, uses, stable)
diff --git a/packages/core/src/repowise/core/analysis/test_collection.py b/packages/core/src/repowise/core/analysis/test_collection.py
index acf816016..93e1b760a 100644
--- a/packages/core/src/repowise/core/analysis/test_collection.py
+++ b/packages/core/src/repowise/core/analysis/test_collection.py
@@ -232,7 +232,7 @@ async def collect(
import route from that test to a changed file is looked up as well.
*pytest_texts* are the checkout's conftests and pytest configs by path;
without them no conftest route is narrowed. *read_source* (``Checkout.read``)
- lets that narrowing read the modules a conftest takes constants from.
+ lets that narrowing read the modules a conftest takes names from.
"""
from ..persistence.crud import get_health_metrics, get_test_coverage_summary
from ..persistence.crud.analysis.coverage_map import MAX_TEST_COVERAGE_ROWS
diff --git a/packages/core/src/repowise/core/analysis/test_selection.py b/packages/core/src/repowise/core/analysis/test_selection.py
index 2348d1dfa..2962b59f5 100644
--- a/packages/core/src/repowise/core/analysis/test_selection.py
+++ b/packages/core/src/repowise/core/analysis/test_selection.py
@@ -275,7 +275,9 @@ class Selection:
nothing was asked (no index), or a scope's ``ecosystem``,
``package-importers``, ``named-by`` or ``owner-package``. ``why`` says, per
selected test file, what put it in: the changed file and evidence that
- reached it, or the reason it runs with every subset.
+ reached it, or the reason it runs with every subset. ``known_tests`` are
+ the checkout's runnable test files (empty when not read), so a selection
+ can say how much of the suite it is.
"""
run_all: bool
@@ -287,6 +289,7 @@ class Selection:
skipped_files: tuple[str, ...] = ()
basis: Mapping[str, str] = field(default_factory=dict)
why: Mapping[str, str] = field(default_factory=dict)
+ known_tests: tuple[str, ...] = ()
def to_dict(self) -> dict[str, Any]:
return {
@@ -605,6 +608,7 @@ def select_tests(inp: SelectionInput) -> Selection:
skipped_files=tuple(triage.skipped),
basis=basis,
why={**scope_why, **gap_why, **_why(triage.code, per_file, evidence), **always},
+ known_tests=tuple(inp.known_tests),
)
diff --git a/packages/server/src/repowise/server/mcp_server/_test_selection.py b/packages/server/src/repowise/server/mcp_server/_test_selection.py
index 20e568a4a..12b198122 100644
--- a/packages/server/src/repowise/server/mcp_server/_test_selection.py
+++ b/packages/server/src/repowise/server/mcp_server/_test_selection.py
@@ -271,5 +271,11 @@ def _summary(selection: Selection, total: int, always: int, limit: int) -> str:
return f"Run every test: {lead}{named}"
if not total:
return "No test needs to run for this change."
+ # The share of the suite, so a selection holding most of it never reads as targeted.
+ known = set(selection.known_tests)
+ held = len(known & set(selection.test_files))
+ share = f" ({held} of {len(known)} test files)" if known else ""
rule = f", {always} of them because they run with every subset" if always else ""
- return f"{total} test(s) to run for this change{rule}{shown}; the rest can be skipped."
+ if known and held == len(known):
+ return f"{total} test(s) to run for this change, every test file there is{rule}{shown}."
+ return f"{total} test(s) to run for this change{share}{rule}{shown}; the rest can be skipped."
diff --git a/tests/unit/analysis/test_conftest_routes.py b/tests/unit/analysis/test_conftest_routes.py
index 36969b3fb..b647f21da 100644
--- a/tests/unit/analysis/test_conftest_routes.py
+++ b/tests/unit/analysis/test_conftest_routes.py
@@ -9,7 +9,7 @@ import pytest
from repowise.core.analysis.conftest_routes import (
ConftestFacts,
conftest_use,
- literal_constants,
+ local_names,
)
ROUTE = ["src/app/store.py"]
@@ -274,7 +274,7 @@ def test_star_imports_plugins_and_unparsable_files_fail_closed() -> None:
assert _use("def broken(:\n").run_all
-_STORE_CONSTANTS = {"src/app/store.py": literal_constants('SIZE = 3\nNAMES = ("a", "b")\n')}
+_STORE_CONSTANTS = {"src/app/store.py": local_names('SIZE = 3\nNAMES = ("a", "b")\n')}
def _autouse(imports: str, line: str, constants=None):
@@ -320,7 +320,7 @@ def test_module_level_code_reading_a_literal_constant_does_not_keep_every_test()
assert conftest_use(source, "tests/conftest.py", ROUTE).run_all
-def test_literal_constants_are_names_bound_once_at_top_level_to_immutable_literals() -> None:
+def test_local_constants_are_names_bound_once_at_top_level_to_immutable_literals() -> None:
source = dedent(
"""
from x import IMPORTED
@@ -344,7 +344,276 @@ def test_literal_constants_are_names_bound_once_at_top_level_to_immutable_litera
global GLOBAL
"""
)
- assert literal_constants(source) == {"A", "B", "C", "D", "E"}
- assert literal_constants("from x import *\nA = 1\n") == frozenset()
- assert literal_constants("frozenset = list\nA = frozenset(())\n") == frozenset()
- assert literal_constants("def broken(:\n") == frozenset()
+ # ``f`` only declares a global, so it is a local function too; ``LIST`` may only be emptied.
+ assert local_names(source) == {"A", "B", "C", "D", "E", "f", "LIST.clear"}
+ assert local_names("from x import *\nA = 1\n") == frozenset()
+ assert local_names("frozenset = list\nA = frozenset(())\n") == frozenset()
+ assert local_names("def broken(:\n") == frozenset()
+
+
+def test_local_functions_use_nothing_the_module_imports_from_elsewhere() -> None:
+ source = dedent(
+ """
+ import os
+ from collections import OrderedDict
+ from app.db import engine
+ from app.deco import wrap
+ from . import sibling
+
+ _cache = OrderedDict()
+ _seen = {}
+ SIZE = 3
+
+ def reset():
+ _cache.clear()
+ _seen.clear()
+
+ def doubled():
+ return SIZE * 2
+
+ def env():
+ return os.environ.get("X")
+
+ def lazy_std():
+ import json
+ return json.dumps({})
+
+ def calls_local():
+ reset()
+ return doubled()
+
+ def shadowed():
+ def inner(engine):
+ return engine
+ return inner
+
+ def uses_db():
+ return engine.connect()
+
+ def calls_db():
+ return uses_db()
+
+ def reads_state():
+ _seen.clear()
+ return len(_seen)
+
+ def lazy_repo():
+ from app.db import engine as e
+ return e
+
+ def relative():
+ return sibling.x
+
+ @wrap
+ def decorated():
+ return 1
+
+ def default(x=engine):
+ return x
+
+ def nested_db():
+ def inner():
+ return engine
+ return inner
+
+ def rebinds():
+ global LIMIT
+ LIMIT = 4
+
+ def by_name():
+ return importlib.import_module("app.db")
+
+ def evaluates():
+ return eval("engine")
+
+ def modules():
+ import sys
+ return sys.modules["app.db"]
+ """
+ )
+ names = local_names(source)
+ assert {"reset", "doubled", "env", "lazy_std", "calls_local", "shadowed", "SIZE"} <= names
+ assert not names & {
+ "uses_db",
+ "calls_db",
+ "reads_state",
+ "lazy_repo",
+ "relative",
+ "decorated",
+ "default",
+ "nested_db",
+ "rebinds",
+ "by_name",
+ "evaluates",
+ "modules",
+ }
+
+
+def test_emptying_standard_library_state_is_local_whatever_filled_it() -> None:
+ source = dedent(
+ """
+ import functools
+ from functools import cache, lru_cache
+ from app import db, deco
+
+ _seen = {}
+
+ @cache
+ def engine():
+ return db.connect()
+
+ @functools.lru_cache(maxsize=None)
+ def settings():
+ return db.settings()
+
+ @deco.wrap
+ @lru_cache
+ def stacked():
+ return 1
+
+ def reset_all():
+ engine.cache_clear()
+ settings.cache_clear()
+ _seen.clear()
+
+ def calls_engine():
+ return engine()
+ """
+ )
+ names = local_names(source)
+ assert {"engine.cache_clear", "settings.cache_clear", "_seen.clear", "reset_all"} <= names
+ assert not names & {"engine", "settings", "stacked.cache_clear", "calls_engine"}
+
+
+ "source",
+ [
+ "import runpy\ndef f():\n runpy.run_module('app.models')\n",
+ "from unittest import mock\ndef f():\n mock.patch('app.x.y').start()\n",
+ "import logging.config\ndef f():\n"
+ " logging.config.dictConfig({'version': 1, 'handlers': {'h': {'()': 'app.x.H'}}})\n",
+ "import pydoc\ndef f():\n return pydoc.locate('app.x.y')\n",
+ "import pickle\ndef f(b):\n return pickle.loads(b)\n",
+ "import importlib.metadata as m\ndef f():\n return m.entry_points()\n",
+ "def f():\n breakpoint()\n",
+ "import sys\ndef f():\n return sys.modules['app.x']\n",
+ ],
+)
+def test_a_standard_module_that_loads_code_by_name_is_not_local(source: str) -> None:
+ """Only standard modules vetted as never importing a named module count as outside."""
+ assert "f" not in local_names(source)
+
+
+ "body",
+ [
+ "os.system('python -m app.x')",
+ "os.popen('python -m app.x')",
+ "os.execv('python', ['python', '-m', 'app.x'])",
+ "os.spawnl(os.P_WAIT, 'python', 'python', '-m', 'app.x')",
+ "os.posix_spawn('python', ['python', '-m', 'app.x'], {})",
+ "os.startfile('app.py')",
+ "shutil.register_archive_format('x', print)",
+ "os.sys.modules['app.x'].run()",
+ "shutil.os.system('x')",
+ "re.sys.modules['app.x'].run()",
+ "threading._sys.modules['app.x'].run()",
+ "__loader__.load_module('app.x')",
+ "return __spec__.loader",
+ "return f.__globals__",
+ ],
+)
+def test_an_allowed_module_used_to_run_or_reach_code_by_name_is_not_local(body: str) -> None:
+ source = f"import os, re, shutil, threading\n\ndef f():\n {body}\n"
+ assert "f" not in local_names(source)
+ # The same modules used for paths and the environment stay local.
+ assert "g" in local_names(
+ "import os, shutil\n\ndef g():\n"
+ " os.makedirs(os.getcwd(), exist_ok=True)\n return os.environ.get('X')\n"
+ )
+
+
+def test_a_conftest_reading_through_a_dunder_attribute_still_reaches() -> None:
+ use = _autouse("from app import store", "store.reset.__globals__", _STORE_FUNCTIONS)
+ assert use.run_all and "autouse fixture _f uses store" in use.run_all
+
+
+ ("source", "local"),
+ [
+ ("import collections\nD = collections.OrderedDict()\ndef f():\n D.clear()\n", True),
+ # Filled at import from something the module imports.
+ ("from app import x\nD = {}\nD.update(x.ITEMS)\ndef f():\n D.clear()\n", False),
+ ("from app import x\nD = {}\nD['k'] = x.V\ndef f():\n D.clear()\n", False),
+ ("from app import x\nD = dict(x.ITEMS)\ndef f():\n D.clear()\n", False),
+ ],
+)
+def test_a_container_is_reset_locally_only_when_the_module_names_it_where_it_builds_it(
+ source: str, local: bool
+) -> None:
+ assert ("D.clear" in local_names(source)) is local
+ assert ("f" in local_names(source)) is local
+
+
+ ("source", "cached"),
+ [
+ ("import functools as ft\[email protected]\ndef g():\n return 1\n", True),
+ (
+ "from functools import lru_cache as memo\n@memo(maxsize=2)\ndef g():\n return 1\n",
+ True,
+ ),
+ # A ``cache`` that is not functools' has no known ``cache_clear``.
+ ("from app import cache\n@cache\ndef g():\n return 1\n", False),
+ ("import collections as functools\[email protected]\ndef g():\n return 1\n", False),
+ ("import functools\nclass c:\n cache = 1\[email protected]\ndef g():\n return 1\n", False),
+ ],
+)
+def test_only_a_functools_cache_has_a_local_cache_clear(source: str, cached: bool) -> None:
+ assert ("g.cache_clear" in local_names(source)) is cached
+
+
+_STORE_FUNCTIONS = {
+ "src/app/store.py": local_names(
+ "from functools import cache\nfrom app import db\n_seen = {}\n\n"
+ "def reset():\n _seen.clear()\n\n"
+ "def connect():\n return db.connect()\n\n"
+ "@cache\ndef engine():\n return db.engine()\n"
+ )
+}
+
+
+ ("imports", "line"),
+ [
+ ("from app import store", "store.reset()"),
+ ("from app.store import reset", "reset()"),
+ ("", "from app.store import reset; reset()"),
+ ("from app import store", "store._seen.clear()"),
+ ("from app.store import engine", "engine.cache_clear()"),
+ ],
+)
+def test_an_autouse_fixture_calling_a_local_function_reaches_only_its_module(
+ imports: str, line: str
+) -> None:
+ """``reset`` runs nothing ``store`` imports, so a change behind ``store`` cannot reach it."""
+ assert _autouse(imports, line, _STORE_FUNCTIONS).run_all is None
+ # The change is in ``store`` itself: the function may be what changed.
+ assert _autouse(imports, line).run_all
+
+
+ "line",
+ [
+ "store.connect()",
+ "store.engine()",
+ "clear = store.engine.cache_clear",
+ "assert store._seen",
+ "store._seen.pop('x')",
+ ],
+)
+def test_an_autouse_fixture_using_what_the_module_imports_or_holds_still_reaches(
+ line: str,
+) -> None:
+ use = _autouse("from app import store", line, _STORE_FUNCTIONS)
+ assert use.run_all and "autouse fixture _f uses store" in use.run_all
diff --git a/tests/unit/analysis/test_test_selection.py b/tests/unit/analysis/test_test_selection.py
index 774ba3c80..87c5f98e7 100644
--- a/tests/unit/analysis/test_test_selection.py
+++ b/tests/unit/analysis/test_test_selection.py
@@ -271,6 +271,7 @@ def test_a_route_through_a_test_package_init_selects_the_tests_under_it() -> Non
sel = _select(["src/a.py"], tiers, known_tests=_KNOWN)
assert not sel.run_all, sel.reasons
assert sel.tests == ("other/test_c.py", "tests/unit/test_a.py", "tests/unit/sub/test_b.py")
+ assert sel.known_tests == _KNOWN
def test_a_route_through_another_helper_still_runs_everything() -> None:
diff --git a/tests/unit/persistence/test_impacted_tests_resolution.py b/tests/unit/persistence/test_impacted_tests_resolution.py
index c6b77ea5b..2eeffe7e3 100644
--- a/tests/unit/persistence/test_impacted_tests_resolution.py
+++ b/tests/unit/persistence/test_impacted_tests_resolution.py
@@ -9,6 +9,8 @@ test. The diff parser and CRUD line-intersection are tested separately.
from __future__ import annotations
+import pytest
+
from repowise.core.analysis.health.coverage import TestCoverage
from repowise.core.analysis.test_collection import empty_result as _empty_result
from repowise.core.analysis.test_collection import resolve_impacted as _resolve_impacted
@@ -463,6 +465,41 @@ async def test_an_autouse_fixture_reading_a_constant_reaches_only_the_defining_m
assert "tests/conftest.py" in picks["src/app/store.py"]
+ ("body", "narrowed"),
+ [("_seen.clear()", True), ("util.flush()", False)],
+)
+async def test_an_autouse_fixture_calling_a_function_reaches_only_what_the_function_uses(
+ async_session, body: str, narrowed: bool
+) -> None:
+ """``store.reset()`` reaches a change behind store only when reset uses what store imports."""
+ repo = await insert_repo(async_session)
+ await _conftest_graph(async_session, repo.id, stamped=True)
+ source = (
+ "import pytest\nfrom app import store\n\n\[email protected](autouse=True)\n"
+ "def _fresh():\n store.reset()\n"
+ )
+ store = f"from app import util\n\n_seen = {{}}\n\ndef reset():\n {body}\n"
+ files = {"src/app/store.py": store}
+
+ out = _empty_result(1)
+ await _resolve_impacted(
+ async_session,
+ repo.id,
+ {"src/app/util.py": None},
+ set(),
+ out,
+ pytest_texts={"tests/conftest.py": source},
+ read_source=files.get,
+ )
+
+ picks = _pairs(out, "src/app/util.py")
+ if narrowed:
+ assert picks == {"tests/test_a.py": "conftest-import-check"}
+ else:
+ assert "tests/conftest.py" in picks
+
+
async def _add_edge(session, repo_id, source, target, edge_type, hint=None, **flags) -> None:
from repowise.core.persistence.models import GraphEdge
diff --git a/tests/unit/server/mcp/test_change_risk.py b/tests/unit/server/mcp/test_change_risk.py
index 145c83d57..d3b25b7fc 100644
--- a/tests/unit/server/mcp/test_change_risk.py
+++ b/tests/unit/server/mcp/test_change_risk.py
@@ -446,7 +446,40 @@ async def test_impacted_tests_falls_back_to_the_graph_without_a_map(tmp_path, mo
# when the weaker tier is allowed to speak.
assert it["why"] == {"tests/test_round_trips.py": "src/app.py changed (import-graph)"}
assert "line_coverage" not in it
- assert it["summary"].startswith("1 test(s) to run for this change")
+ # The only test file there is: a whole-suite selection never reads as targeted.
+ assert it["summary"] == "1 test(s) to run for this change, every test file there is."
+
+
+ ("files", "summary"),
+ [
+ (
+ ("tests/test_a.py",),
+ "2 test(s) to run for this change (1 of 4 test files); the rest can be skipped.",
+ ),
+ (
+ ("tests/test_a.py", "tests/test_b.py", "tests/test_c.py", "tests/test_d.py"),
+ "2 test(s) to run for this change, every test file there is.",
+ ),
+ # As many files as the suite holds is not the suite: one is not a known test.
+ (
+ ("tests/test_a.py", "tests/test_b.py", "tests/test_c.py", "other/test_x.py"),
+ "2 test(s) to run for this change (3 of 4 test files); the rest can be skipped.",
+ ),
+ ],
+)
+def test_a_selection_summary_says_how_much_of_the_suite_it_is(files, summary) -> None:
+ from repowise.core.analysis.test_selection import Selection
+ from repowise.server.mcp_server._test_selection import _summary
+
+ selection = Selection(
+ run_all=False,
+ reasons=(),
+ tests=("t1", "t2"),
+ test_files=files,
+ known_tests=("tests/test_a.py", "tests/test_b.py", "tests/test_c.py", "tests/test_d.py"),
+ )
+ assert _summary(selection, 2, 0, 10) == summary
@pytest.mark.asyncio