Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(dead-code): count a module-path string naming a Python module as a use (#2924)

Raghav Chamadiya3e9e3a66cf3a
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
index e31b2d618..045c9136c 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
@@ -62,6 +62,7 @@ from .file_reachability import (
is_file_reachable,
)
from .models import DeadCodeFindingData, DeadCodeKind, DeadCodeReport
+from .module_strings import drop_named_modules
from .name_occurrences import (
IDENTIFIER_RE,
clamp_named_types,
@@ -970,6 +971,7 @@ class DeadCodeAnalyzer:
type_names = self._public_top_level_names(findings, kinds=_TYPE_DECLARATION_KINDS)
findings = drop_reference_assembly_api(findings, self._source_map, type_names)
findings = clamp_named_types(findings, self._source_map, type_names)
+ findings = drop_named_modules(findings, self._source_map, self._dynamic_import_files)
findings = clamp_entry_shaped(
findings, self._source_map, self._public_top_level_names(findings)
)
diff --git a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
index 9d9895a7f..d26c27d01 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
@@ -54,6 +54,8 @@ def read_source_text(
_DYNAMIC_IMPORT_MARKERS: dict[str, tuple[str, ...]] = {
".py": (
"importlib.import_module",
+ # ``from importlib import import_module``, then a bare call.
+ "import_module(",
"__import__(",
"importlib.reload",
"pkgutil.iter_modules",
diff --git a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
new file mode 100644
index 000000000..f2f12e3ac
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
@@ -0,0 +1,224 @@
+"""A Python module named by a module-path string is used.
+
+Python loads code by dotted name as often as by ``import``: an entry-point
+table (``"repowise.cli.main:cli"`` in ``pyproject.toml``), a Celery or Django
+setting, a lazy command table that imports ``"uninstall_cmd:uninstall_command"``
+only when the command runs, a package ``__getattr__`` mapping names to
+``"result_panels"``. None of these leaves an import edge, so the module reads
+as unreachable and the attribute after the colon as an unused export.
+
+Two spellings count:
+
+* a dotted name of two or more segments that is the tail of the module's own
+ dotted path (``cli.commands.uninstall_cmd``), written in any indexed file;
+* a bare module name, only inside a file that imports by name at runtime
+ (an ``import_module`` caller), and only for a module in that file's
+ directory or below it, which is where such a loader looks.
+
+``module:attr`` additionally uses ``attr`` of that module. A Django or plugin
+table names a member with a dot instead (``"hc.accounts.backends.EmailBackend"``
+in ``AUTHENTICATION_BACKENDS``): when the whole string names no module, its
+parent does, and the last segment is a top-level name that module defines, the
+string uses both. Each match drops the finding: a string that resolves to the
+module is how the module is loaded.
+
+Only strings that run count: a comment or a Python docstring that mentions a
+dotted name loads nothing. A test file counts only whole module paths (a
+fixture app it loads by name), never a member path (a ``mock.patch`` target).
+"""
+
+from __future__ import annotations
+
+import io
+import re
+import tokenize
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+
+from ...ingestion.languages.registry import REGISTRY
+from ...test_paths import is_test_related_path
+from .models import DeadCodeFindingData, DeadCodeKind
+
+#: A quoted module path, optionally followed by ``:attr``.
+_MODULE_STRING_RE = re.compile(
+ rb"""['"]([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)(?::([A-Za-z_]\w*))?['"]"""
+)
+
+
+def _module_parts(path: str) -> tuple[str, ...]:
+ """The dotted-path segments of a Python file (a package for ``__init__``)."""
+ parts = PurePosixPath(path).with_suffix("").parts
+ return parts[:-1] if parts and parts[-1] == "__init__" else parts
+
+
+class _ModuleIndex:
+ """The candidate modules, keyed by every dotted tail and by bare name."""
+
+ def __init__(self, paths: Iterable[str]) -> None:
+ self.by_tail: dict[str, set[str]] = {}
+ self.by_name: dict[str, set[str]] = {}
+ for path in paths:
+ parts = _module_parts(path)
+ if not parts:
+ continue
+ for start in range(len(parts) - 1):
+ self.by_tail.setdefault(".".join(parts[start:]), set()).add(path)
+ self.by_name.setdefault(parts[-1], set()).add(path)
+
+ def resolve(self, module: str, loader_dir: str | None) -> set[str]:
+ """The candidate files *module* names, as written in a file in *loader_dir*."""
+ found = set(self.by_tail.get(module, ()))
+ if loader_dir is not None and "." not in module:
+ prefix = f"{loader_dir}/" if loader_dir else ""
+ found.update(p for p in self.by_name.get(module, ()) if p.startswith(prefix))
+ return found
+
+
+def _directory(path: str) -> str:
+ parent = PurePosixPath(path).parent.as_posix()
+ return "" if parent == "." else parent
+
+
+def _is_python(path: str) -> bool:
+ return REGISTRY.from_extension(PurePosixPath(path).suffix) == "python"
+
+
+def drop_named_modules(
+ findings: list[DeadCodeFindingData],
+ source_map: dict[str, bytes],
+ runtime_importers: Iterable[str],
+) -> list[DeadCodeFindingData]:
+ """Drop Python file and export findings a module-path string names.
+
+ *runtime_importers* are the files that import by name at runtime, the
+ only place a bare module name is read as one. Returns a new list.
+ """
+ candidates = {f.file_path for f in findings if _is_candidate(f)}
+ if not candidates or not source_map:
+ return findings
+ loaders = {path: _directory(path) for path in runtime_importers}
+ named, attrs = _named_by_strings(source_map, _ModuleIndex(candidates), loaders)
+ return [f for f in findings if not _is_named(f, named, attrs)]
+
+
+def _is_candidate(finding: DeadCodeFindingData) -> bool:
+ kinds = (DeadCodeKind.UNREACHABLE_FILE, DeadCodeKind.UNUSED_EXPORT)
+ return finding.kind in kinds and _is_python(finding.file_path)
+
+
+def _is_named(
+ finding: DeadCodeFindingData, named: set[str], attrs: set[tuple[str, str]]
+) -> bool:
+ if finding.kind is DeadCodeKind.UNREACHABLE_FILE:
+ return finding.file_path in named
+ return (
+ finding.kind is DeadCodeKind.UNUSED_EXPORT
+ and (finding.file_path, finding.symbol_name) in attrs
+ )
+
+
+def _named_by_strings(
+ source_map: dict[str, bytes], index: _ModuleIndex, loaders: dict[str, str]
+) -> tuple[set[str], set[tuple[str, str]]]:
+ """Modules a string names, and the ``(module, attr)`` pairs ``module:attr`` names."""
+ named: set[str] = set()
+ attrs: set[tuple[str, str]] = set()
+ for path, blob in source_map.items():
+ # A test that loads a fixture module by name (``"t.unit.proj.app"``)
+ # uses it; a member path in a test is a ``mock.patch`` target, not a load.
+ members = None if is_test_related_path(path) else source_map
+ reader = _Reader(path, index, loaders.get(path), members)
+ # Tokenize only a file whose raw text names a candidate at all.
+ if next(reader.uses(blob), None) is None:
+ continue
+ for target, attr in reader.uses(_live_text(path, blob)):
+ named.add(target)
+ if attr:
+ attrs.add((target, attr))
+ return named, attrs
+
+
+@dataclass(frozen=True)
+class _Reader:
+ """Resolves the module-path strings written in one file.
+
+ *members* is the source to look a ``"pkg.module.Name"`` member up in, or
+ None to read only whole module paths.
+ """
+
+ path: str
+ index: _ModuleIndex
+ loader_dir: str | None
+ members: dict[str, bytes] | None
+
+ def uses(self, text: bytes) -> Iterator[tuple[str, str | None]]:
+ """``(module, attr)`` for each module-path string in *text*."""
+ for match in _MODULE_STRING_RE.finditer(text):
+ module = match.group(1).decode("ascii")
+ attr = match.group(2).decode("ascii") if match.group(2) else None
+ targets = self._resolve(module)
+ if not (targets or attr) and self.members is not None:
+ targets, attr = self._member_owners(module, self.members)
+ yield from ((target, attr) for target in targets)
+
+ def _resolve(self, module: str) -> set[str]:
+ return self.index.resolve(module, self.loader_dir) - {self.path}
+
+ def _member_owners(self, module: str, members: dict[str, bytes]) -> tuple[set[str], str]:
+ """For ``"pkg.module.Name"``: the modules ``pkg.module`` names that define ``Name``."""
+ parent, _, name = module.rpartition(".")
+ owners = self._resolve(parent) if parent else set()
+ return {t for t in owners if _defines_top_level(members.get(t, b""), name)}, name
+
+
+def _defines_top_level(blob: bytes, name: str) -> bool:
+ """Whether a Python module defines *name* at top level (def, class, assignment).
+
+ A re-export (``from x import Name``) or a tuple target is not seen, so such
+ a member stays reported.
+ """
+ word = re.escape(name.encode("ascii"))
+ pattern = (
+ rb"^(?:(?:async[ \t]+)?def|class)[ \t]+" + word + rb"\b"
+ rb"|^" + word + rb"[ \t]*(?::[^=\n]*)?=(?!=)"
+ )
+ return re.search(pattern, blob, re.MULTILINE) is not None
+
+
+#: Tokens after which a string starts an expression statement (a docstring).
+_STATEMENT_START = frozenset(
+ {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT, tokenize.ENCODING}
+)
+_TRIVIA = frozenset({tokenize.NL, tokenize.COMMENT})
+_COMMENT_LINE_RE = re.compile(rb"^[ \t]*#.*$", re.MULTILINE)
+
+
+def _live_text(path: str, blob: bytes) -> bytes:
+ """*blob* without comments and, for Python, without docstrings.
+
+ Python keeps only its string literals that are not a statement on their
+ own; other files drop only ``#`` comment lines. Python that does not tokenize
+ is kept whole, as before this filter.
+ """
+ if not _is_python(path):
+ return _COMMENT_LINE_RE.sub(b"", blob)
+ try:
+ tokens = [
+ t for t in tokenize.tokenize(io.BytesIO(blob).readline) if t.type not in _TRIVIA
+ ]
+ except (tokenize.TokenError, SyntaxError, ValueError):
+ return blob
+ return b"\n".join(
+ tok.string.encode("utf-8")
+ for i, tok in enumerate(tokens)
+ if tok.type == tokenize.STRING and not _is_statement(tokens, i)
+ )
+
+
+def _is_statement(tokens: list[tokenize.TokenInfo], i: int) -> bool:
+ """Whether ``tokens[i]`` is a statement on its own (a docstring)."""
+ # ENDMARKER always closes the stream, so ``i + 1`` exists.
+ if tokens[i - 1].type not in _STATEMENT_START:
+ return False
+ return tokens[i + 1].type in (tokenize.NEWLINE, tokenize.ENDMARKER)
diff --git a/tests/unit/dead_code/test_module_strings.py b/tests/unit/dead_code/test_module_strings.py
new file mode 100644
index 000000000..9350f6b83
--- /dev/null
+++ b/tests/unit/dead_code/test_module_strings.py
@@ -0,0 +1,184 @@
+"""A Python module named by a module-path string is used.
+
+From repowise itself: ``cli/main.py`` loads ``"uninstall_cmd:uninstall_command"``
+from ``cli/commands`` through ``import_module`` only when the command runs, and
+``cli/ui/__init__.py`` maps names to ``"result_panels"`` for a lazy
+``__getattr__``. Both modules were reported unreachable.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKind
+from repowise.core.analysis.dead_code.models import DeadCodeFindingData
+from repowise.core.analysis.dead_code.module_strings import drop_named_modules
+from tests.unit.dead_code._helpers import _build_graph
+
+_MAIN = b"""from importlib import import_module
+
+_LAZY = (
+ ("uninstall", "uninstall_cmd:uninstall_command"),
+)
+
+def load(name):
+ module, attr = dict(_LAZY)[name].split(":")
+ return getattr(import_module(f"repowise.cli.commands.{module}"), attr)
+"""
+
+
+def _finding(kind: DeadCodeKind, path: str, name: str | None = None) -> DeadCodeFindingData:
+ return DeadCodeFindingData(
+ kind=kind,
+ file_path=path,
+ symbol_name=name,
+ symbol_kind="function" if name else None,
+ confidence=0.4,
+ reason="test",
+ last_commit_at=None,
+ commit_count_90d=0,
+ lines=1,
+ evidence=[],
+ safe_to_delete=False,
+ primary_owner=None,
+ age_days=None,
+ )
+
+
+def _kept(findings, source, importers=()):
+ return {(f.file_path, f.symbol_name) for f in drop_named_modules(findings, source, importers)}
+
+
+_CMD = "pkg/repowise/cli/commands/uninstall_cmd.py"
+
+
+def test_lazy_table_entry_in_a_runtime_importer_names_module_and_attribute():
+ findings = [
+ _finding(DeadCodeKind.UNREACHABLE_FILE, _CMD),
+ _finding(DeadCodeKind.UNUSED_EXPORT, _CMD, "uninstall_command"),
+ _finding(DeadCodeKind.UNUSED_EXPORT, _CMD, "helper"),
+ ]
+ source = {"pkg/repowise/cli/main.py": _MAIN, _CMD: b"def uninstall_command(): ...\n"}
+ kept = _kept(findings, source, {"pkg/repowise/cli/main.py"})
+ assert kept == {(_CMD, "helper")}
+
+
+def test_a_bare_name_outside_a_runtime_importer_is_not_read_as_a_module():
+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, _CMD)]
+ source = {"pkg/repowise/cli/main.py": b'LABEL = "uninstall_cmd"\n'}
+ assert _kept(findings, source) == {(_CMD, None)}
+
+
+def test_a_bare_name_only_reaches_modules_under_the_importer():
+ other = "pkg/repowise/server/uninstall_cmd.py"
+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, other)]
+ assert _kept(findings, {"pkg/repowise/cli/main.py": _MAIN}, {"pkg/repowise/cli/main.py"}) == {
+ (other, None)
+ }
+
+
+def test_a_dotted_entry_point_in_pyproject_names_the_module():
+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, "src/tool/cli.py")]
+ source = {"pyproject.toml": b'[project.scripts]\ntool = "tool.cli:main"\n'}
+ assert _kept(findings, source) == set()
+
+
+def test_a_single_word_string_elsewhere_names_nothing():
+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, "src/tool/cli.py")]
+ assert _kept(findings, {"README.md": b'run "cli" to start\n'}) == {("src/tool/cli.py", None)}
+
+
+def test_the_analyzer_reads_a_string_map_in_a_lazy_package():
+ init = b"""from importlib import import_module
+
+_SUBMODULES = {"build_completion_panel": "result_panels"}
+
+def __getattr__(name):
+ return getattr(import_module(f"{__name__}.{_SUBMODULES[name]}"), name)
+"""
+ graph = _build_graph(
+ {
+ "cli/ui/__init__.py": {"symbols": []},
+ "cli/ui/result_panels.py": {"symbols": []},
+ "cli/ui/orphan.py": {"symbols": []},
+ }
+ )
+ source = {
+ "cli/ui/__init__.py": init,
+ "cli/ui/result_panels.py": b"def build_completion_panel(): ...\n",
+ "cli/ui/orphan.py": b"X = 1\n",
+ }
+ parsed = {path: SimpleNamespace(file_info=SimpleNamespace(abs_path=path)) for path in source}
+ report = DeadCodeAnalyzer(graph, parsed_files=parsed, source_map=source).analyze(
+ {"detect_unused_exports": False, "detect_zombie_packages": False, "min_confidence": 0.0}
+ )
+ files = {f.file_path for f in report.findings if f.kind is DeadCodeKind.UNREACHABLE_FILE}
+ assert files == {"cli/ui/orphan.py"}
+
+
+_BACKENDS = "hc/accounts/backends.py"
+_BACKENDS_SRC = b"class EmailBackend:\n pass\n\n\ndef helper():\n pass\n"
+
+
+def _backend_findings():
+ return [
+ _finding(DeadCodeKind.UNREACHABLE_FILE, _BACKENDS),
+ _finding(DeadCodeKind.UNUSED_EXPORT, _BACKENDS, "EmailBackend"),
+ _finding(DeadCodeKind.UNUSED_EXPORT, _BACKENDS, "helper"),
+ ]
+
+
+def test_a_django_setting_naming_a_class_uses_its_module_and_the_class():
+ settings = b'AUTHENTICATION_BACKENDS = [\n "hc.accounts.backends.EmailBackend",\n]\n'
+ source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}
+ assert _kept(_backend_findings(), source) == {(_BACKENDS, "helper")}
+
+
+def test_a_colon_entry_names_the_module_and_the_attribute():
+ redis = "celery/backends/redis.py"
+ findings = [
+ _finding(DeadCodeKind.UNREACHABLE_FILE, redis),
+ _finding(DeadCodeKind.UNUSED_EXPORT, redis, "RedisBackend"),
+ ]
+ source = {
+ "celery/app/backends.py": b"BACKEND_ALIASES = {'redis': 'celery.backends.redis:RedisBackend'}\n",
+ redis: b"class RedisBackend: ...\n",
+ }
+ assert _kept(findings, source) == set()
+
+
+def test_a_dotted_name_the_module_does_not_define_names_nothing():
+ settings = b'MIDDLEWARE = ["hc.accounts.backends.Missing"]\n'
+ source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}
+ kept = _kept(_backend_findings(), source)
+ assert kept == {(_BACKENDS, None), (_BACKENDS, "EmailBackend"), (_BACKENDS, "helper")}
+
+
+def test_a_docstring_comment_or_test_patch_target_names_nothing():
+ doc = b'"""Mirrors "hc.accounts.backends.EmailBackend"."""\n# see "hc.accounts.backends"\nX = 1\n'
+ test = b'@patch("hc.accounts.backends.EmailBackend")\ndef test_login(): ...\n'
+ toml = b'# tool = "hc.accounts.backends:EmailBackend"\n'
+ source = {
+ "hc/other.py": doc,
+ "hc/tests/test_backends.py": test,
+ "pyproject.toml": toml,
+ _BACKENDS: _BACKENDS_SRC,
+ }
+ assert _kept(_backend_findings(), source) == {
+ (_BACKENDS, None),
+ (_BACKENDS, "EmailBackend"),
+ (_BACKENDS, "helper"),
+ }
+
+
+def test_a_test_loading_a_fixture_app_by_module_path_uses_it():
+ app = "t/unit/bin/proj/app.py"
+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, app)]
+ source = {"t/unit/bin/test_control.py": b"OPTS = ['-A', 't.unit.bin.proj.app']\n", app: b"app = 1\n"}
+ assert _kept(findings, source) == set()
+
+
+def test_python_that_does_not_decode_falls_back_to_the_whole_file():
+ settings = b'# -*- coding: utf-8 -*-\nB = "hc.accounts.backends.EmailBackend"\nX = "\xff"\n'
+ source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}
+ assert _kept(_backend_findings(), source) == {(_BACKENDS, "helper")}