Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(dead-code): count a module-path string naming a Python module as a use (#2924)
Raghav Chamadiya3e9e3a66cf3a
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.pyindex e31b2d618..045c9136c 100644--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py@@ -62,6 +62,7 @@ from .file_reachability import (is_file_reachable,)from .models import DeadCodeFindingData, DeadCodeKind, DeadCodeReport+from .module_strings import drop_named_modulesfrom .name_occurrences import (IDENTIFIER_RE,clamp_named_types,@@ -970,6 +971,7 @@ class DeadCodeAnalyzer:type_names = self._public_top_level_names(findings, kinds=_TYPE_DECLARATION_KINDS)findings = drop_reference_assembly_api(findings, self._source_map, type_names)findings = clamp_named_types(findings, self._source_map, type_names)+ findings = drop_named_modules(findings, self._source_map, self._dynamic_import_files)findings = clamp_entry_shaped(findings, self._source_map, self._public_top_level_names(findings))diff --git a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.pyindex 9d9895a7f..d26c27d01 100644--- a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py+++ b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py@@ -54,6 +54,8 @@ def read_source_text(_DYNAMIC_IMPORT_MARKERS: dict[str, tuple[str, ...]] = {".py": ("importlib.import_module",+ # ``from importlib import import_module``, then a bare call.+ "import_module(","__import__(","importlib.reload","pkgutil.iter_modules",diff --git a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py b/packages/core/src/repowise/core/analysis/dead_code/module_strings.pynew file mode 100644index 000000000..f2f12e3ac--- /dev/null+++ b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py@@ -0,0 +1,224 @@+"""A Python module named by a module-path string is used.++Python loads code by dotted name as often as by ``import``: an entry-point+table (``"repowise.cli.main:cli"`` in ``pyproject.toml``), a Celery or Django+setting, a lazy command table that imports ``"uninstall_cmd:uninstall_command"``+only when the command runs, a package ``__getattr__`` mapping names to+``"result_panels"``. None of these leaves an import edge, so the module reads+as unreachable and the attribute after the colon as an unused export.++Two spellings count:++* a dotted name of two or more segments that is the tail of the module's own+ dotted path (``cli.commands.uninstall_cmd``), written in any indexed file;+* a bare module name, only inside a file that imports by name at runtime+ (an ``import_module`` caller), and only for a module in that file's+ directory or below it, which is where such a loader looks.++``module:attr`` additionally uses ``attr`` of that module. A Django or plugin+table names a member with a dot instead (``"hc.accounts.backends.EmailBackend"``+in ``AUTHENTICATION_BACKENDS``): when the whole string names no module, its+parent does, and the last segment is a top-level name that module defines, the+string uses both. Each match drops the finding: a string that resolves to the+module is how the module is loaded.++Only strings that run count: a comment or a Python docstring that mentions a+dotted name loads nothing. A test file counts only whole module paths (a+fixture app it loads by name), never a member path (a ``mock.patch`` target).+"""++from __future__ import annotations++import io+import re+import tokenize+from collections.abc import Iterable, Iterator+from dataclasses import dataclass+from pathlib import PurePosixPath++from ...ingestion.languages.registry import REGISTRY+from ...test_paths import is_test_related_path+from .models import DeadCodeFindingData, DeadCodeKind++#: A quoted module path, optionally followed by ``:attr``.+_MODULE_STRING_RE = re.compile(+ rb"""['"]([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)(?::([A-Za-z_]\w*))?['"]"""+)+++def _module_parts(path: str) -> tuple[str, ...]:+ """The dotted-path segments of a Python file (a package for ``__init__``)."""+ parts = PurePosixPath(path).with_suffix("").parts+ return parts[:-1] if parts and parts[-1] == "__init__" else parts+++class _ModuleIndex:+ """The candidate modules, keyed by every dotted tail and by bare name."""++ def __init__(self, paths: Iterable[str]) -> None:+ self.by_tail: dict[str, set[str]] = {}+ self.by_name: dict[str, set[str]] = {}+ for path in paths:+ parts = _module_parts(path)+ if not parts:+ continue+ for start in range(len(parts) - 1):+ self.by_tail.setdefault(".".join(parts[start:]), set()).add(path)+ self.by_name.setdefault(parts[-1], set()).add(path)++ def resolve(self, module: str, loader_dir: str | None) -> set[str]:+ """The candidate files *module* names, as written in a file in *loader_dir*."""+ found = set(self.by_tail.get(module, ()))+ if loader_dir is not None and "." not in module:+ prefix = f"{loader_dir}/" if loader_dir else ""+ found.update(p for p in self.by_name.get(module, ()) if p.startswith(prefix))+ return found+++def _directory(path: str) -> str:+ parent = PurePosixPath(path).parent.as_posix()+ return "" if parent == "." else parent+++def _is_python(path: str) -> bool:+ return REGISTRY.from_extension(PurePosixPath(path).suffix) == "python"+++def drop_named_modules(+ findings: list[DeadCodeFindingData],+ source_map: dict[str, bytes],+ runtime_importers: Iterable[str],+) -> list[DeadCodeFindingData]:+ """Drop Python file and export findings a module-path string names.++ *runtime_importers* are the files that import by name at runtime, the+ only place a bare module name is read as one. Returns a new list.+ """+ candidates = {f.file_path for f in findings if _is_candidate(f)}+ if not candidates or not source_map:+ return findings+ loaders = {path: _directory(path) for path in runtime_importers}+ named, attrs = _named_by_strings(source_map, _ModuleIndex(candidates), loaders)+ return [f for f in findings if not _is_named(f, named, attrs)]+++def _is_candidate(finding: DeadCodeFindingData) -> bool:+ kinds = (DeadCodeKind.UNREACHABLE_FILE, DeadCodeKind.UNUSED_EXPORT)+ return finding.kind in kinds and _is_python(finding.file_path)+++def _is_named(+ finding: DeadCodeFindingData, named: set[str], attrs: set[tuple[str, str]]+) -> bool:+ if finding.kind is DeadCodeKind.UNREACHABLE_FILE:+ return finding.file_path in named+ return (+ finding.kind is DeadCodeKind.UNUSED_EXPORT+ and (finding.file_path, finding.symbol_name) in attrs+ )+++def _named_by_strings(+ source_map: dict[str, bytes], index: _ModuleIndex, loaders: dict[str, str]+) -> tuple[set[str], set[tuple[str, str]]]:+ """Modules a string names, and the ``(module, attr)`` pairs ``module:attr`` names."""+ named: set[str] = set()+ attrs: set[tuple[str, str]] = set()+ for path, blob in source_map.items():+ # A test that loads a fixture module by name (``"t.unit.proj.app"``)+ # uses it; a member path in a test is a ``mock.patch`` target, not a load.+ members = None if is_test_related_path(path) else source_map+ reader = _Reader(path, index, loaders.get(path), members)+ # Tokenize only a file whose raw text names a candidate at all.+ if next(reader.uses(blob), None) is None:+ continue+ for target, attr in reader.uses(_live_text(path, blob)):+ named.add(target)+ if attr:+ attrs.add((target, attr))+ return named, attrs+++@dataclass(frozen=True)+class _Reader:+ """Resolves the module-path strings written in one file.++ *members* is the source to look a ``"pkg.module.Name"`` member up in, or+ None to read only whole module paths.+ """++ path: str+ index: _ModuleIndex+ loader_dir: str | None+ members: dict[str, bytes] | None++ def uses(self, text: bytes) -> Iterator[tuple[str, str | None]]:+ """``(module, attr)`` for each module-path string in *text*."""+ for match in _MODULE_STRING_RE.finditer(text):+ module = match.group(1).decode("ascii")+ attr = match.group(2).decode("ascii") if match.group(2) else None+ targets = self._resolve(module)+ if not (targets or attr) and self.members is not None:+ targets, attr = self._member_owners(module, self.members)+ yield from ((target, attr) for target in targets)++ def _resolve(self, module: str) -> set[str]:+ return self.index.resolve(module, self.loader_dir) - {self.path}++ def _member_owners(self, module: str, members: dict[str, bytes]) -> tuple[set[str], str]:+ """For ``"pkg.module.Name"``: the modules ``pkg.module`` names that define ``Name``."""+ parent, _, name = module.rpartition(".")+ owners = self._resolve(parent) if parent else set()+ return {t for t in owners if _defines_top_level(members.get(t, b""), name)}, name+++def _defines_top_level(blob: bytes, name: str) -> bool:+ """Whether a Python module defines *name* at top level (def, class, assignment).++ A re-export (``from x import Name``) or a tuple target is not seen, so such+ a member stays reported.+ """+ word = re.escape(name.encode("ascii"))+ pattern = (+ rb"^(?:(?:async[ \t]+)?def|class)[ \t]+" + word + rb"\b"+ rb"|^" + word + rb"[ \t]*(?::[^=\n]*)?=(?!=)"+ )+ return re.search(pattern, blob, re.MULTILINE) is not None+++#: Tokens after which a string starts an expression statement (a docstring).+_STATEMENT_START = frozenset(+ {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT, tokenize.ENCODING}+)+_TRIVIA = frozenset({tokenize.NL, tokenize.COMMENT})+_COMMENT_LINE_RE = re.compile(rb"^[ \t]*#.*$", re.MULTILINE)+++def _live_text(path: str, blob: bytes) -> bytes:+ """*blob* without comments and, for Python, without docstrings.++ Python keeps only its string literals that are not a statement on their+ own; other files drop only ``#`` comment lines. Python that does not tokenize+ is kept whole, as before this filter.+ """+ if not _is_python(path):+ return _COMMENT_LINE_RE.sub(b"", blob)+ try:+ tokens = [+ t for t in tokenize.tokenize(io.BytesIO(blob).readline) if t.type not in _TRIVIA+ ]+ except (tokenize.TokenError, SyntaxError, ValueError):+ return blob+ return b"\n".join(+ tok.string.encode("utf-8")+ for i, tok in enumerate(tokens)+ if tok.type == tokenize.STRING and not _is_statement(tokens, i)+ )+++def _is_statement(tokens: list[tokenize.TokenInfo], i: int) -> bool:+ """Whether ``tokens[i]`` is a statement on its own (a docstring)."""+ # ENDMARKER always closes the stream, so ``i + 1`` exists.+ if tokens[i - 1].type not in _STATEMENT_START:+ return False+ return tokens[i + 1].type in (tokenize.NEWLINE, tokenize.ENDMARKER)diff --git a/tests/unit/dead_code/test_module_strings.py b/tests/unit/dead_code/test_module_strings.pynew file mode 100644index 000000000..9350f6b83--- /dev/null+++ b/tests/unit/dead_code/test_module_strings.py@@ -0,0 +1,184 @@+"""A Python module named by a module-path string is used.++From repowise itself: ``cli/main.py`` loads ``"uninstall_cmd:uninstall_command"``+from ``cli/commands`` through ``import_module`` only when the command runs, and+``cli/ui/__init__.py`` maps names to ``"result_panels"`` for a lazy+``__getattr__``. Both modules were reported unreachable.+"""++from __future__ import annotations++from types import SimpleNamespace++from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKind+from repowise.core.analysis.dead_code.models import DeadCodeFindingData+from repowise.core.analysis.dead_code.module_strings import drop_named_modules+from tests.unit.dead_code._helpers import _build_graph++_MAIN = b"""from importlib import import_module++_LAZY = (+ ("uninstall", "uninstall_cmd:uninstall_command"),+)++def load(name):+ module, attr = dict(_LAZY)[name].split(":")+ return getattr(import_module(f"repowise.cli.commands.{module}"), attr)+"""+++def _finding(kind: DeadCodeKind, path: str, name: str | None = None) -> DeadCodeFindingData:+ return DeadCodeFindingData(+ kind=kind,+ file_path=path,+ symbol_name=name,+ symbol_kind="function" if name else None,+ confidence=0.4,+ reason="test",+ last_commit_at=None,+ commit_count_90d=0,+ lines=1,+ evidence=[],+ safe_to_delete=False,+ primary_owner=None,+ age_days=None,+ )+++def _kept(findings, source, importers=()):+ return {(f.file_path, f.symbol_name) for f in drop_named_modules(findings, source, importers)}+++_CMD = "pkg/repowise/cli/commands/uninstall_cmd.py"+++def test_lazy_table_entry_in_a_runtime_importer_names_module_and_attribute():+ findings = [+ _finding(DeadCodeKind.UNREACHABLE_FILE, _CMD),+ _finding(DeadCodeKind.UNUSED_EXPORT, _CMD, "uninstall_command"),+ _finding(DeadCodeKind.UNUSED_EXPORT, _CMD, "helper"),+ ]+ source = {"pkg/repowise/cli/main.py": _MAIN, _CMD: b"def uninstall_command(): ...\n"}+ kept = _kept(findings, source, {"pkg/repowise/cli/main.py"})+ assert kept == {(_CMD, "helper")}+++def test_a_bare_name_outside_a_runtime_importer_is_not_read_as_a_module():+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, _CMD)]+ source = {"pkg/repowise/cli/main.py": b'LABEL = "uninstall_cmd"\n'}+ assert _kept(findings, source) == {(_CMD, None)}+++def test_a_bare_name_only_reaches_modules_under_the_importer():+ other = "pkg/repowise/server/uninstall_cmd.py"+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, other)]+ assert _kept(findings, {"pkg/repowise/cli/main.py": _MAIN}, {"pkg/repowise/cli/main.py"}) == {+ (other, None)+ }+++def test_a_dotted_entry_point_in_pyproject_names_the_module():+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, "src/tool/cli.py")]+ source = {"pyproject.toml": b'[project.scripts]\ntool = "tool.cli:main"\n'}+ assert _kept(findings, source) == set()+++def test_a_single_word_string_elsewhere_names_nothing():+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, "src/tool/cli.py")]+ assert _kept(findings, {"README.md": b'run "cli" to start\n'}) == {("src/tool/cli.py", None)}+++def test_the_analyzer_reads_a_string_map_in_a_lazy_package():+ init = b"""from importlib import import_module++_SUBMODULES = {"build_completion_panel": "result_panels"}++def __getattr__(name):+ return getattr(import_module(f"{__name__}.{_SUBMODULES[name]}"), name)+"""+ graph = _build_graph(+ {+ "cli/ui/__init__.py": {"symbols": []},+ "cli/ui/result_panels.py": {"symbols": []},+ "cli/ui/orphan.py": {"symbols": []},+ }+ )+ source = {+ "cli/ui/__init__.py": init,+ "cli/ui/result_panels.py": b"def build_completion_panel(): ...\n",+ "cli/ui/orphan.py": b"X = 1\n",+ }+ parsed = {path: SimpleNamespace(file_info=SimpleNamespace(abs_path=path)) for path in source}+ report = DeadCodeAnalyzer(graph, parsed_files=parsed, source_map=source).analyze(+ {"detect_unused_exports": False, "detect_zombie_packages": False, "min_confidence": 0.0}+ )+ files = {f.file_path for f in report.findings if f.kind is DeadCodeKind.UNREACHABLE_FILE}+ assert files == {"cli/ui/orphan.py"}+++_BACKENDS = "hc/accounts/backends.py"+_BACKENDS_SRC = b"class EmailBackend:\n pass\n\n\ndef helper():\n pass\n"+++def _backend_findings():+ return [+ _finding(DeadCodeKind.UNREACHABLE_FILE, _BACKENDS),+ _finding(DeadCodeKind.UNUSED_EXPORT, _BACKENDS, "EmailBackend"),+ _finding(DeadCodeKind.UNUSED_EXPORT, _BACKENDS, "helper"),+ ]+++def test_a_django_setting_naming_a_class_uses_its_module_and_the_class():+ settings = b'AUTHENTICATION_BACKENDS = [\n "hc.accounts.backends.EmailBackend",\n]\n'+ source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}+ assert _kept(_backend_findings(), source) == {(_BACKENDS, "helper")}+++def test_a_colon_entry_names_the_module_and_the_attribute():+ redis = "celery/backends/redis.py"+ findings = [+ _finding(DeadCodeKind.UNREACHABLE_FILE, redis),+ _finding(DeadCodeKind.UNUSED_EXPORT, redis, "RedisBackend"),+ ]+ source = {+ "celery/app/backends.py": b"BACKEND_ALIASES = {'redis': 'celery.backends.redis:RedisBackend'}\n",+ redis: b"class RedisBackend: ...\n",+ }+ assert _kept(findings, source) == set()+++def test_a_dotted_name_the_module_does_not_define_names_nothing():+ settings = b'MIDDLEWARE = ["hc.accounts.backends.Missing"]\n'+ source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}+ kept = _kept(_backend_findings(), source)+ assert kept == {(_BACKENDS, None), (_BACKENDS, "EmailBackend"), (_BACKENDS, "helper")}+++def test_a_docstring_comment_or_test_patch_target_names_nothing():+ doc = b'"""Mirrors "hc.accounts.backends.EmailBackend"."""\n# see "hc.accounts.backends"\nX = 1\n'+ test = b'@patch("hc.accounts.backends.EmailBackend")\ndef test_login(): ...\n'+ toml = b'# tool = "hc.accounts.backends:EmailBackend"\n'+ source = {+ "hc/other.py": doc,+ "hc/tests/test_backends.py": test,+ "pyproject.toml": toml,+ _BACKENDS: _BACKENDS_SRC,+ }+ assert _kept(_backend_findings(), source) == {+ (_BACKENDS, None),+ (_BACKENDS, "EmailBackend"),+ (_BACKENDS, "helper"),+ }+++def test_a_test_loading_a_fixture_app_by_module_path_uses_it():+ app = "t/unit/bin/proj/app.py"+ findings = [_finding(DeadCodeKind.UNREACHABLE_FILE, app)]+ source = {"t/unit/bin/test_control.py": b"OPTS = ['-A', 't.unit.bin.proj.app']\n", app: b"app = 1\n"}+ assert _kept(findings, source) == set()+++def test_python_that_does_not_decode_falls_back_to_the_whole_file():+ settings = b'# -*- coding: utf-8 -*-\nB = "hc.accounts.backends.EmailBackend"\nX = "\xff"\n'+ source = {"hc/settings.py": settings, _BACKENDS: _BACKENDS_SRC}+ assert _kept(_backend_findings(), source) == {(_BACKENDS, "helper")}