Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(dead-code): recognize lazy module attribute tuples (#3958)

Ruben Garcia1a0d742f5f79
diff --git a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
index 2370dfb45..a8c3d350f 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
@@ -22,6 +22,10 @@ parent does, and the last segment is a top-level name that module defines, the
string uses both. Each match drops the finding: a string that resolves to the
module is how the module is loaded.
+A lazy loader may instead pair two string literals as ``("module", "attr")``.
+Only exact two-element tuples in a Python file defining ``__getattr__`` or
+calling ``import_module`` count, and only for attributes that module defines.
+
Only strings that run count: a comment or a Python docstring that mentions a
dotted name loads nothing. A test file counts only whole module paths (a
fixture app it loads by name), never a member path (a ``mock.patch`` target).
@@ -29,7 +33,10 @@ fixture app it loads by name), never a member path (a ``mock.patch`` target).
from __future__ import annotations
+import io
+import keyword
import re
+import tokenize
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from pathlib import PurePosixPath
@@ -45,6 +52,77 @@ _MODULE_STRING_RE = re.compile(
)
+def _has_module_getattr(tokens: list[tokenize.TokenInfo]) -> bool:
+ """A module hook may be conditional, but never inside a class/function."""
+ scopes: list[bool] = []
+ start: list[str] = []
+ header_scope = False
+ for i, token in enumerate(tokens):
+ if token.type == tokenize.ENCODING:
+ continue
+ if token.type == tokenize.INDENT:
+ scopes.append(header_scope)
+ elif token.type == tokenize.DEDENT:
+ scopes.pop()
+ elif token.type == tokenize.NEWLINE:
+ header_scope = start[:1] in (["def"], ["class"]) or start == ["async", "def"]
+ start = []
+ else:
+ if len(start) < 2:
+ start.append(token.string)
+ if (token.type == tokenize.NAME and token.string == "__getattr__"
+ and tokens[i - 1].string == "def" and not any(scopes)):
+ return True
+ return False
+
+
+def _loader_tokens(blob: bytes) -> list[tokenize.TokenInfo]:
+ """Tokens of a Python loader, without comments or continuation newlines."""
+ try:
+ tokens = [t for t in tokenize.tokenize(io.BytesIO(blob).readline)
+ if t.type not in (tokenize.NL, tokenize.COMMENT)]
+ except (tokenize.TokenError, SyntaxError, ValueError):
+ return []
+ if _has_module_getattr(tokens):
+ return tokens
+ for i, token in enumerate(tokens[:-1]):
+ if token.type != tokenize.NAME:
+ continue
+ if (token.string == "import_module" and tokens[i + 1].string == "("
+ and tokens[i - 1].string not in ("def", "class")):
+ return tokens
+ return []
+
+
+def _is_tuple_start(tokens: list[tokenize.TokenInfo], i: int) -> bool:
+ previous = tokens[i - 1]
+ call = previous.string in (")", "]", "}") or (
+ previous.type == tokenize.NAME and not keyword.iskeyword(previous.string)
+ )
+ return (
+ tokens[i].string == "(" and not call and tokens[i + 2].string == ","
+ and tokens[i + 1].type == tokenize.STRING
+ and tokens[i + 3].type == tokenize.STRING
+ )
+
+
+def _module_pairs(tokens: list[tokenize.TokenInfo]) -> Iterator[tuple[str, str]]:
+ """Exact two-string tuples, not calls, lists or concatenated literals."""
+ for i in range(len(tokens) - 5):
+ if not _is_tuple_start(tokens, i):
+ continue
+ end = i + 5 if tokens[i + 4].string == "," else i + 4
+ if tokens[end].string != ")":
+ continue
+ matches = [_MODULE_STRING_RE.fullmatch(tokens[j].string.encode("utf-8"))
+ for j in (i + 1, i + 3)]
+ if any(m is None or m.group(2) for m in matches):
+ continue
+ module, attr = (m.group(1).decode("ascii") for m in matches)
+ if "." not in attr:
+ yield module, attr
+
+
class _ModuleIndex:
"""The candidate modules, keyed by every dotted tail and by bare name."""
@@ -118,6 +196,9 @@ def _named_by_strings(
# uses it; a member path in a test is a ``mock.patch`` target, not a load.
members = None if is_test_related_path(path) else source_map
reader = _Reader(path, index, loaders.get(path), members)
+ for target, attr in reader.tuple_uses(blob):
+ named.add(target)
+ attrs.add((target, attr))
# Tokenize only a file whose raw text names a candidate at all.
if next(reader.uses(blob), None) is None:
continue
@@ -154,6 +235,22 @@ class _Reader:
def _resolve(self, module: str) -> set[str]:
return self.index.resolve(module, self.loader_dir) - {self.path}
+ def tuple_uses(self, blob: bytes) -> Iterator[tuple[str, str]]:
+ """Read exact ``(module, attr)`` string pairs in Python lazy loaders.
+
+ Tokens keep tuple boundaries without treating comments, docstrings or
+ concatenated literals as entries. Test files never use member names.
+ """
+ if self.members is None or not is_python(self.path):
+ return
+ if b"__getattr__" not in blob and b"import_module" not in blob:
+ return
+ for module, attr in _module_pairs(_loader_tokens(blob)):
+ targets = self.index.resolve(module, _directory(self.path)) - {self.path}
+ for target in targets:
+ if defines_top_level(self.members.get(target, b""), attr):
+ yield target, attr
+
def _member_owners(self, module: str, members: dict[str, bytes]) -> tuple[set[str], str]:
"""For ``"pkg.module.Name"``: the modules ``pkg.module`` names that define ``Name``."""
parent, _, name = module.rpartition(".")
diff --git a/tests/unit/dead_code/test_module_strings.py b/tests/unit/dead_code/test_module_strings.py
index 9350f6b83..bbafe7b45 100644
--- a/tests/unit/dead_code/test_module_strings.py
+++ b/tests/unit/dead_code/test_module_strings.py
@@ -10,6 +10,8 @@ from __future__ import annotations
from types import SimpleNamespace
+import pytest
+
from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKind
from repowise.core.analysis.dead_code.models import DeadCodeFindingData
from repowise.core.analysis.dead_code.module_strings import drop_named_modules
@@ -182,3 +184,115 @@ def test_python_that_does_not_decode_falls_back_to_the_whole_file():
settings = b'# -*- coding: utf-8 -*-\nB = "hc.accounts.backends.EmailBackend"\nX = "\xff"\n'
source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}
assert _kept(_backend_findings(), source) == {(_BACKENDS, "helper")}
+
+
+_PAIR_MODULE = "pkg/_graph_utils.py"
+_PAIR_SOURCE = b"def build_ctx(x):\n return {'ctx': x}\n\ndef helper(): ...\n"
+_PAIR_LOADER = b"""import importlib
+_LAZY_ATTRS = {"_build_ctx": ("_graph_utils", "build_ctx")}
+def __getattr__(name):
+ module_name, attr = _LAZY_ATTRS[name]
+ return getattr(importlib.import_module(f"{__name__}.{module_name}"), attr)
+"""
+
+
[email protected]("marker", [
+ b"def __getattr__(name): ...\n",
+ b"async def __getattr__(name): ...\n",
+ b"if enabled:\n def __getattr__(name): ...\n",
+ b"import_module(name)\n",
+])
+ b'("_graph_utils", "build_ctx")',
+ b'(\n "_graph_utils", # module\n "build_ctx",\n)',
+])
+def test_loader_pair_names_only_the_defined_member(marker, pair):
+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, name)
+ for name in ("build_ctx", "helper")]
+ source = {"pkg/__init__.py": marker + b"TABLE = {\"alias\": " + pair + b"}\n",
+ _PAIR_MODULE: _PAIR_SOURCE}
+ assert _kept(findings, source) == {(_PAIR_MODULE, "helper")}
+
+
+ b'("_graph_utils", "missing")',
+ b'("_graph_utils", "build_ctx", "extra")',
+ b'("_graph_utils", "build_" "ctx")',
+ b'["_graph_utils", "build_ctx"]',
+ b'call("_graph_utils", "build_ctx")',
+ b'callbacks[0]("_graph_utils", "build_ctx")',
+ b'"""("_graph_utils", "build_ctx")"""',
+ b'# ("_graph_utils", "build_ctx")',
+])
+def test_invalid_or_nonexecuting_pairs_do_not_name_an_export(pair):
+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, "build_ctx")]
+ source = {"pkg/__init__.py": b"def __getattr__(name): ...\n" + pair + b"\n",
+ _PAIR_MODULE: _PAIR_SOURCE}
+ assert _kept(findings, source) == {(_PAIR_MODULE, "build_ctx")}
+
+
[email protected]("path", ["pkg/table.py", "pkg/tests/test_loader.py"])
+def test_nonloader_or_test_pairs_do_not_name_an_export(path):
+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, "build_ctx")]
+ blob = _PAIR_LOADER if "tests/" in path else b'TABLE = ("pkg._graph_utils", "build_ctx")\n'
+ assert _kept(findings, {path: blob, _PAIR_MODULE: _PAIR_SOURCE}) == {
+ (_PAIR_MODULE, "build_ctx")
+ }
+
+
+ b'{"_build_ctx": ("_graph_utils", "build_ctx")}',
+ b'dict([("_build_ctx", ("_graph_utils", "build_ctx"))])',
+])
+def test_renamed_lazy_reexport_with_the_real_graph_builder(tmp_path, table):
+ from repowise.core.pipeline.incremental import build_repo_graph
+
+ source = {
+ "pkg/__init__.py": _PAIR_LOADER.replace(
+ b'{"_build_ctx": ("_graph_utils", "build_ctx")}', table
+ ),
+ _PAIR_MODULE: _PAIR_SOURCE,
+ "app/main.py": b"from pkg import _build_ctx\n\ndef run():\n return _build_ctx(1)\n\n"
+ b"if __name__ == '__main__':\n run()\n",
+ }
+ for path, blob in source.items():
+ target = tmp_path / path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_bytes(blob)
+ _, sources, builder, _, _ = build_repo_graph(
+ tmp_path, [], collect_sources=True, read_only=True
+ )
+ report = DeadCodeAnalyzer(
+ builder.graph(), {}, parsed_files=builder._parsed_files,
+ source_map=sources, repo_root=tmp_path
+ ).analyze(None)
+ unused = {(f.file_path, f.symbol_name) for f in report.findings
+ if f.kind is DeadCodeKind.UNUSED_EXPORT}
+ assert (_PAIR_MODULE, "helper") in unused
+ assert (_PAIR_MODULE, "build_ctx") not in unused
+
+
+def test_pair_cannot_reach_a_bare_module_outside_the_loader_directory():
+ other = "other/_graph_utils.py"
+ findings = [_finding(DeadCodeKind.UNUSED_EXPORT, other, "build_ctx")]
+ assert _kept(findings, {"pkg/__init__.py": _PAIR_LOADER, other: _PAIR_SOURCE}) == {
+ (other, "build_ctx")
+ }
+
+
[email protected]("marker", [
+ b'LABEL = "__getattr__"\n',
+ b'def import_module(name): ...\n',
+ b'class import_module(object): ...\n',
+ b'class Proxy:\n def __getattr__(self, name): ...\n',
+ b'def factory():\n def __getattr__(name): ...\n',
+ b'class Proxy:\n async def __getattr__(self, name): ...\n',
+])
+def test_a_loader_marker_that_is_not_a_dynamic_load_does_not_enable_pairing(marker):
+ finding = _finding(DeadCodeKind.UNUSED_EXPORT, _PAIR_MODULE, "build_ctx")
+ source = {
+ "pkg/__init__.py": marker + b'TABLE = ("_graph_utils", "build_ctx")\n',
+ _PAIR_MODULE: _PAIR_SOURCE,
+ }
+ assert drop_named_modules([finding], source, ()) == [finding]
+ assert not finding.safe_to_delete