Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(dead-code): recognize lazy module attribute tuples (#3958)
Ruben Garcia1a0d742f5f79
diff --git a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py b/packages/core/src/repowise/core/analysis/dead_code/module_strings.pyindex 2370dfb45..a8c3d350f 100644--- a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py+++ b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py@@ -22,6 +22,10 @@ parent does, and the last segment is a top-level name that module defines, thestring uses both. Each match drops the finding: a string that resolves to themodule is how the module is loaded.+A lazy loader may instead pair two string literals as ``("module", "attr")``.+Only exact two-element tuples in a Python file defining ``__getattr__`` or+calling ``import_module`` count, and only for attributes that module defines.+Only strings that run count: a comment or a Python docstring that mentions adotted name loads nothing. A test file counts only whole module paths (afixture app it loads by name), never a member path (a ``mock.patch`` target).@@ -29,7 +33,10 @@ fixture app it loads by name), never a member path (a ``mock.patch`` target).from __future__ import annotations+import io+import keywordimport re+import tokenizefrom collections.abc import Iterable, Iteratorfrom dataclasses import dataclassfrom pathlib import PurePosixPath@@ -45,6 +52,77 @@ _MODULE_STRING_RE = re.compile()+def _has_module_getattr(tokens: list[tokenize.TokenInfo]) -> bool:+ """A module hook may be conditional, but never inside a class/function."""+ scopes: list[bool] = []+ start: list[str] = []+ header_scope = False+ for i, token in enumerate(tokens):+ if token.type == tokenize.ENCODING:+ continue+ if token.type == tokenize.INDENT:+ scopes.append(header_scope)+ elif token.type == tokenize.DEDENT:+ scopes.pop()+ elif token.type == tokenize.NEWLINE:+ header_scope = start[:1] in (["def"], ["class"]) or start == ["async", "def"]+ start = []+ else:+ if len(start) < 2:+ start.append(token.string)+ if (token.type == tokenize.NAME and token.string == "__getattr__"+ and tokens[i - 1].string == "def" and not any(scopes)):+ return True+ return False+++def _loader_tokens(blob: bytes) -> list[tokenize.TokenInfo]:+ """Tokens of a Python loader, without comments or continuation newlines."""+ try:+ tokens = [t for t in tokenize.tokenize(io.BytesIO(blob).readline)+ if t.type not in (tokenize.NL, tokenize.COMMENT)]+ except (tokenize.TokenError, SyntaxError, ValueError):+ return []+ if _has_module_getattr(tokens):+ return tokens+ for i, token in enumerate(tokens[:-1]):+ if token.type != tokenize.NAME:+ continue+ if (token.string == "import_module" and tokens[i + 1].string == "("+ and tokens[i - 1].string not in ("def", "class")):+ return tokens+ return []+++def _is_tuple_start(tokens: list[tokenize.TokenInfo], i: int) -> bool:+ previous = tokens[i - 1]+ call = previous.string in (")", "]", "}") or (+ previous.type == tokenize.NAME and not keyword.iskeyword(previous.string)+ )+ return (+ tokens[i].string == "(" and not call and tokens[i + 2].string == ","+ and tokens[i + 1].type == tokenize.STRING+ and tokens[i + 3].type == tokenize.STRING+ )+++def _module_pairs(tokens: list[tokenize.TokenInfo]) -> Iterator[tuple[str, str]]:+ """Exact two-string tuples, not calls, lists or concatenated literals."""+ for i in range(len(tokens) - 5):+ if not _is_tuple_start(tokens, i):+ continue+ end = i + 5 if tokens[i + 4].string == "," else i + 4+ if tokens[end].string != ")":+ continue+ matches = [_MODULE_STRING_RE.fullmatch(tokens[j].string.encode("utf-8"))+ for j in (i + 1, i + 3)]+ if any(m is None or m.group(2) for m in matches):+ continue+ module, attr = (m.group(1).decode("ascii") for m in matches)+ if "." not in attr:+ yield module, attr++class _ModuleIndex:"""The candidate modules, keyed by every dotted tail and by bare name."""@@ -118,6 +196,9 @@ def _named_by_strings(# uses it; a member path in a test is a ``mock.patch`` target, not a load.members = None if is_test_related_path(path) else source_mapreader = _Reader(path, index, loaders.get(path), members)+ for target, attr in reader.tuple_uses(blob):+ named.add(target)+ attrs.add((target, attr))# Tokenize only a file whose raw text names a candidate at all.if next(reader.uses(blob), None) is None:continue@@ -154,6 +235,22 @@ class _Reader:def _resolve(self, module: str) -> set[str]:return self.index.resolve(module, self.loader_dir) - {self.path}+ def tuple_uses(self, blob: bytes) -> Iterator[tuple[str, str]]:+ """Read exact ``(module, attr)`` string pairs in Python lazy loaders.++ Tokens keep tuple boundaries without treating comments, docstrings or+ concatenated literals as entries. Test files never use member names.+ """+ if self.members is None or not is_python(self.path):+ return+ if b"__getattr__" not in blob and b"import_module" not in blob:+ return+ for module, attr in _module_pairs(_loader_tokens(blob)):+ targets = self.index.resolve(module, _directory(self.path)) - {self.path}+ for target in targets:+ if defines_top_level(self.members.get(target, b""), attr):+ yield target, attr+def _member_owners(self, module: str, members: dict[str, bytes]) -> tuple[set[str], str]:"""For ``"pkg.module.Name"``: the modules ``pkg.module`` names that define ``Name``."""parent, _, name = module.rpartition(".")diff --git a/tests/unit/dead_code/test_module_strings.py b/tests/unit/dead_code/test_module_strings.pyindex 9350f6b83..bbafe7b45 100644--- a/tests/unit/dead_code/test_module_strings.py+++ b/tests/unit/dead_code/test_module_strings.py@@ -10,6 +10,8 @@ from __future__ import annotationsfrom types import SimpleNamespace+import pytest+from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKindfrom repowise.core.analysis.dead_code.models import DeadCodeFindingDatafrom repowise.core.analysis.dead_code.module_strings import drop_named_modules@@ -182,3 +184,115 @@ def test_python_that_does_not_decode_falls_back_to_the_whole_file():settings = b'# -*- coding: utf-8 -*-\nB = "hc.accounts.backends.EmailBackend"\nX = "\xff"\n'source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}assert _kept(_backend_findings(), source) == {(_BACKENDS, "helper")}+++_PAIR_MODULE = "pkg/_graph_utils.py"+_PAIR_SOURCE = b"def build_ctx(x):\n return {'ctx': x}\n\ndef helper(): ...\n"+_PAIR_LOADER = b"""import importlib+_LAZY_ATTRS = {"_build_ctx": ("_graph_utils", "build_ctx")}+def __getattr__(name):+ module_name, attr = _LAZY_ATTRS[name]+ return getattr(importlib.import_module(f"{__name__}.{module_name}"), attr)+"""++[email protected]("marker", [+ b"def __getattr__(name): ...\n",+ b"async def __getattr__(name): ...\n",+ b"if enabled:\n def __getattr__(name): ...\n",+ b"import_module(name)\n",+])[email protected]("pair", [+ b'("_graph_utils", "build_ctx")',+ b'(\n "_graph_utils", # module\n "build_ctx",\n)',+])+def test_loader_pair_names_only_the_defined_member(marker, pair):+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, name)+ for name in ("build_ctx", "helper")]+ source = {"pkg/__init__.py": marker + b"TABLE = {\"alias\": " + pair + b"}\n",+ _PAIR_MODULE: _PAIR_SOURCE}+ assert _kept(findings, source) == {(_PAIR_MODULE, "helper")}++[email protected]("pair", [+ b'("_graph_utils", "missing")',+ b'("_graph_utils", "build_ctx", "extra")',+ b'("_graph_utils", "build_" "ctx")',+ b'["_graph_utils", "build_ctx"]',+ b'call("_graph_utils", "build_ctx")',+ b'callbacks[0]("_graph_utils", "build_ctx")',+ b'"""("_graph_utils", "build_ctx")"""',+ b'# ("_graph_utils", "build_ctx")',+])+def test_invalid_or_nonexecuting_pairs_do_not_name_an_export(pair):+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, "build_ctx")]+ source = {"pkg/__init__.py": b"def __getattr__(name): ...\n" + pair + b"\n",+ _PAIR_MODULE: _PAIR_SOURCE}+ assert _kept(findings, source) == {(_PAIR_MODULE, "build_ctx")}++[email protected]("path", ["pkg/table.py", "pkg/tests/test_loader.py"])+def test_nonloader_or_test_pairs_do_not_name_an_export(path):+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, "build_ctx")]+ blob = _PAIR_LOADER if "tests/" in path else b'TABLE = ("pkg._graph_utils", "build_ctx")\n'+ assert _kept(findings, {path: blob, _PAIR_MODULE: _PAIR_SOURCE}) == {+ (_PAIR_MODULE, "build_ctx")+ }++[email protected]("table", [+ b'{"_build_ctx": ("_graph_utils", "build_ctx")}',+ b'dict([("_build_ctx", ("_graph_utils", "build_ctx"))])',+])+def test_renamed_lazy_reexport_with_the_real_graph_builder(tmp_path, table):+ from repowise.core.pipeline.incremental import build_repo_graph++ source = {+ "pkg/__init__.py": _PAIR_LOADER.replace(+ b'{"_build_ctx": ("_graph_utils", "build_ctx")}', table+ ),+ _PAIR_MODULE: _PAIR_SOURCE,+ "app/main.py": b"from pkg import _build_ctx\n\ndef run():\n return _build_ctx(1)\n\n"+ b"if __name__ == '__main__':\n run()\n",+ }+ for path, blob in source.items():+ target = tmp_path / path+ target.parent.mkdir(parents=True, exist_ok=True)+ target.write_bytes(blob)+ _, sources, builder, _, _ = build_repo_graph(+ tmp_path, [], collect_sources=True, read_only=True+ )+ report = DeadCodeAnalyzer(+ builder.graph(), {}, parsed_files=builder._parsed_files,+ source_map=sources, repo_root=tmp_path+ ).analyze(None)+ unused = {(f.file_path, f.symbol_name) for f in report.findings+ if f.kind is DeadCodeKind.UNUSED_EXPORT}+ assert (_PAIR_MODULE, "helper") in unused+ assert (_PAIR_MODULE, "build_ctx") not in unused+++def test_pair_cannot_reach_a_bare_module_outside_the_loader_directory():+ other = "other/_graph_utils.py"+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, other, "build_ctx")]+ assert _kept(findings, {"pkg/__init__.py": _PAIR_LOADER, other: _PAIR_SOURCE}) == {+ (other, "build_ctx")+ }++[email protected]("marker", [+ b'LABEL = "__getattr__"\n',+ b'def import_module(name): ...\n',+ b'class import_module(object): ...\n',+ b'class Proxy:\n def __getattr__(self, name): ...\n',+ b'def factory():\n def __getattr__(name): ...\n',+ b'class Proxy:\n async def __getattr__(self, name): ...\n',+])+def test_a_loader_marker_that_is_not_a_dynamic_load_does_not_enable_pairing(marker):+ finding = _finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, "build_ctx")+ source = {+ "pkg/__init__.py": marker + b'TABLE = ("_graph_utils", "build_ctx")\n',+ _PAIR_MODULE: _PAIR_SOURCE,+ }+ assert drop_named_modules([finding], source, ()) == [finding]+ assert not finding.safe_to_delete