Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(dead-code): treat the modules Django loads from an installed app as used (#2975)
Raghav Chamadiya60bacdc28299
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.pyindex 8f0615e60..df3535d05 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py@@ -19,6 +19,9 @@ class DynamicEdge:edge_type: DynamicKindhint_source: str # extractor nameweight: float = 1.0+ # The target's symbols the runtime reaches. Empty means the whole module:+ # a ``dynamic_uses`` edge without names counts every public symbol as used.+ imported_names: tuple[str, ...] = ()class DynamicHintExtractor(ABC):diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.pyindex 779b26214..8ad74d093 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py@@ -2,10 +2,99 @@ from __future__ import annotationsimport astimport re-from pathlib import Path+from collections.abc import Callable+from pathlib import Path, PurePosixPathfrom .base import DynamicEdge, DynamicHintExtractor+#: Reads a module's syntax tree and returns the class names Django uses there.+UsedNames = Callable[[ast.Module], set[str]]+++def _last_segment(node: ast.expr) -> str:+ """``AppConfig`` for ``AppConfig``, ``apps.AppConfig`` and ``admin.register(...)``."""+ if isinstance(node, ast.Call):+ node = node.func+ if isinstance(node, ast.Subscript): # ``ModelAdmin[Profile]``+ node = node.value+ if isinstance(node, ast.Attribute):+ return node.attr+ return node.id if isinstance(node, ast.Name) else ""+++def _app_config_classes(tree: ast.Module) -> set[str]:+ """The ``AppConfig`` subclasses, which the app registry instantiates."""+ return {+ node.name+ for node in tree.body+ if isinstance(node, ast.ClassDef)+ and any(_last_segment(base) == "AppConfig" for base in node.bases)+ }+++def _names_passed_to_register(tree: ast.Module) -> set[str]:+ """Names given as an admin class to ``site.register(Model, Admin)``."""+ return {+ arg.id+ for node in ast.walk(tree)+ if isinstance(node, ast.Call) and _last_segment(node.func) == "register"+ for arg in node.args[1:]+ if isinstance(arg, ast.Name)+ }+++def _registered_admin_classes(tree: ast.Module) -> set[str]:+ """Admin classes registered by ``@admin.register`` or ``site.register``,+ and the classes nested in them (``Media``), which the admin reads."""+ passed = _names_passed_to_register(tree)+ registered = [+ node+ for node in tree.body+ if isinstance(node, ast.ClassDef)+ and (node.name in passed or any(_last_segment(d) == "register" for d in node.decorator_list))+ ]+ return {node.name for node in registered} | {+ inner.name for node in registered for inner in node.body if isinstance(inner, ast.ClassDef)+ }+++#: Modules Django loads from every installed app by convention, as paths+#: inside the app package (``*`` is one module name). No source file imports+#: them: the app registry imports ``apps`` and ``models``, the admin+#: autodiscovers ``admin``, the template engine loads every ``templatetags``+#: library and ``manage.py`` finds each ``management/commands`` module whose+#: name does not start with ``_``. Where Django also uses classes of the+#: module, the second item names them, so only those count as used and a dead+#: helper beside them is still reported. ``signals``, ``urls`` and ``views``+#: are not here: Django never loads those by name, an app imports them itself.+_APP_CONVENTION_MODULES: tuple[tuple[str, UsedNames | None], ...] = (+ ("apps.py", _app_config_classes),+ ("admin.py", _registered_admin_classes),+ ("models.py", None),+ ("templatetags/*.py", None),+ ("management/commands/[!_]*.py", None),+)+++def _convention_match(rel_in_app: PurePosixPath) -> tuple[bool, UsedNames | None]:+ """Whether Django loads the app module at *rel_in_app*, and what names its used classes."""+ for pattern, used_names in _APP_CONVENTION_MODULES:+ # ``match`` anchors on the right only; equal depth anchors both ends.+ if len(rel_in_app.parts) == pattern.count("/") + 1 and rel_in_app.match(pattern):+ return True, used_names+ return False, None+++def _used_names(path: Path, used_names: UsedNames | None) -> tuple[str, ...]:+ """The names *used_names* finds in the module at *path*, sorted; none if unreadable."""+ if used_names is None:+ return ()+ try:+ tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"), filename=str(path))+ except (OSError, SyntaxError, ValueError):+ return ()+ return tuple(sorted(used_names(tree)))+def _app_to_path(app: str, repo_root: Path) -> str | None:"""Attempt to resolve a dotted app name to an __init__.py under repo_root."""@@ -35,10 +124,38 @@ def _module_to_path(module: str, repo_root: Path) -> str | None:return None+def _installed_app_init(entry: str, repo_root: Path) -> str | None:+ """The ``__init__.py`` of the app an ``INSTALLED_APPS`` entry names.++ An entry is an app package (``polls``) or an ``AppConfig`` path+ (``polls.apps.PollsConfig``), whose app is the package holding the module.+ """+ init = _app_to_path(entry, repo_root)+ if init is not None or "." not in entry:+ return init+ module_name = entry[: entry.rindex(".")]+ module = _module_to_path(module_name, repo_root)+ if module is None:+ return None+ module_path = PurePosixPath(module)+ if module_path.name != "__init__.py":+ app_dir = module_path.parent+ elif module_path.parent.name == "apps":+ # An ``apps`` package inside the app (``polls/apps/__init__.py``).+ app_dir = module_path.parent.parent+ else:+ # The config sits in the app's own ``__init__`` (``polls.PollsConfig``).+ return module+ package_init = (app_dir / "__init__.py").as_posix()+ return package_init if (repo_root / package_init).exists() else None++def _extract_string_list(node: ast.expr) -> list[str]:- """Extract string literals from an ast.List node."""+ """String literals of a list or tuple, or of a sum of them (``BASE + [...]``)."""+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):+ return _extract_string_list(node.left) + _extract_string_list(node.right)results: list[str] = []- if not isinstance(node, ast.List):+ if not isinstance(node, (ast.List, ast.Tuple)):return resultsfor elt in node.elts:if isinstance(elt, ast.Constant) and isinstance(elt.value, str):@@ -64,6 +181,9 @@ class DjangoDynamicHints(DynamicHintExtractor):def _scan_settings(self, repo_root: Path) -> list[DynamicEdge]:edges: list[DynamicEdge] = []+ # Convention modules per app ``__init__``, shared by every settings+ # file that installs the app (``base.py``, ``dev.py``, ``prod.py``).+ app_modules: dict[str, list[tuple[str, tuple[str, ...]]]] = {}# Collect all settings filessettings_files: list[Path] = list(self._rglob(repo_root, "settings.py"))@@ -90,25 +210,20 @@ class DjangoDynamicHints(DynamicHintExtractor):continuefor node in ast.walk(tree):- if not isinstance(node, ast.Assign):+ # ``INSTALLED_APPS += [...]`` extends the setting in place.+ if not isinstance(node, (ast.Assign, ast.AugAssign)):continue- for target in node.targets:+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]+ for target in targets:if not (isinstance(target, ast.Name)):continuename = target.idif name == "INSTALLED_APPS":for app in _extract_string_list(node.value):- resolved = _app_to_path(app, repo_root)- if resolved:- edges.append(- DynamicEdge(- source=rel_settings,- target=resolved,- edge_type="dynamic_imports",- hint_source=self.name,- )- )+ edges.extend(+ self._installed_app_edges(rel_settings, app, repo_root, app_modules)+ )elif name == "ROOT_URLCONF":module = _extract_string_value(node.value)@@ -139,6 +254,53 @@ class DjangoDynamicHints(DynamicHintExtractor):return edges+ def _installed_app_edges(+ self,+ rel_settings: str,+ app: str,+ repo_root: Path,+ app_modules: dict[str, list[tuple[str, tuple[str, ...]]]],+ ) -> list[DynamicEdge]:+ """Edges from a settings file to an installed app and its convention modules.++ A module whose classes Django uses gets a ``dynamic_uses`` edge naming+ them; any other gets ``dynamic_imports``, which marks the module+ loaded and none of its symbols used.+ """+ init = _installed_app_init(app, repo_root)+ if init is None:+ return []+ if init not in app_modules:+ app_modules[init] = self._convention_modules(repo_root, init)+ return [+ DynamicEdge(+ source=rel_settings,+ target=target,+ edge_type="dynamic_uses" if names else "dynamic_imports",+ hint_source=self.name,+ imported_names=names,+ )+ for target, names in [(init, ()), *app_modules[init]]+ ]++ def _convention_modules(+ self, repo_root: Path, app_init: str+ ) -> list[tuple[str, tuple[str, ...]]]:+ """The modules under an app that Django loads by convention, with the names it uses."""+ app_dir = repo_root / PurePosixPath(app_init).parent+ found: list[tuple[str, tuple[str, ...]]] = []+ # Through ``_rglob`` so a file the index excludes gets no edge.+ for path in self._rglob(app_dir, "*.py"):+ # A package ``__init__`` is reached through its package already.+ if path.name == "__init__.py":+ continue+ rel_in_app = PurePosixPath(path.relative_to(app_dir).as_posix())+ loaded, used_names = _convention_match(rel_in_app)+ if loaded:+ rel = path.relative_to(repo_root).as_posix()+ found.append((rel, _used_names(path, used_names)))+ return found+def _scan_urls(self, repo_root: Path) -> list[DynamicEdge]:edges: list[DynamicEdge] = []include_re = re.compile(r"""include\(\s*['\"]([\w\.]+)['\"]""")diff --git a/packages/core/src/repowise/core/ingestion/graph/_edges.py b/packages/core/src/repowise/core/ingestion/graph/_edges.pyindex 9a1bef8ed..aa40b0aa7 100644--- a/packages/core/src/repowise/core/ingestion/graph/_edges.py+++ b/packages/core/src/repowise/core/ingestion/graph/_edges.py@@ -150,12 +150,14 @@ class EdgesMixin:# a real `dynamic_*` edge.sub_type = e.edge_typegraph_edge_type = sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"+ names = {"imported_names": list(e.imported_names)} if e.imported_names else {}self._graph.add_edge(e.source,e.target,edge_type=graph_edge_type,hint_source=e.hint_source,weight=e.weight,+ **names,)# A ``:test`` hint used to set ``is_test`` on the source node. Only# the Rust hinter emits one, for `#[test]` / `#[cfg(test)]` markers,diff --git a/tests/unit/ingestion/test_django_dynamic_hints.py b/tests/unit/ingestion/test_django_dynamic_hints.pynew file mode 100644index 000000000..02257ba0c--- /dev/null+++ b/tests/unit/ingestion/test_django_dynamic_hints.py@@ -0,0 +1,193 @@+"""Django loads an installed app's convention modules by name.++From healthchecks: ``hc/*/admin.py``, ``hc/front/models.py``, ``apps.py``,+``management/commands/*.py`` and ``templatetags/*.py`` have no importer, so+each was reported as an unreachable file, and the ``AppConfig`` classes and+the admin ``Media`` inner classes as unused exports.+"""++from __future__ import annotations++from pathlib import Path++from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKind+from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder+from repowise.core.ingestion.dynamic_hints import HintRegistry+from repowise.core.ingestion.dynamic_hints.django import DjangoDynamicHints++_ADMIN = """from django.contrib import admin+from .models import Item, Order++[email protected](Item)+class ItemAdmin(admin.ModelAdmin):+ class Media:+ css = {"all": ("item.css",)}+++class OrderAdmin(admin.ModelAdmin):+ pass+++admin.site.register(Order, OrderAdmin)+++class UnregisteredAdmin(admin.ModelAdmin):+ pass+++def dead_helper():+ return 1+"""++_APPS = """from django.apps import AppConfig+++class ShopConfig(AppConfig):+ name = "proj.shop"+++def unused_in_apps():+ return 2+"""++_APP_FILES = {+ "__init__.py": "",+ "admin.py": _ADMIN,+ "apps.py": _APPS,+ "models.py": "class Item:\n pass\n\n\nclass Order:\n pass\n",+ "signals.py": "def on_save():\n return 3\n",+ "views.py": "def index():\n return 4\n",+ "templatetags/__init__.py": "",+ "templatetags/extras.py": "X = 1\n",+ "templatetags/nested/deep.py": "Y = 1\n",+ "management/__init__.py": "",+ "management/commands/__init__.py": "",+ "management/commands/prune.py": "class Command:\n pass\n",+ "management/commands/_shared.py": "Z = 1\n",+}+++def _write(root: Path, rel: str, text: str = "") -> None:+ path = root / rel+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(text, encoding="utf-8")+++def _project(root: Path, settings: str, settings_path: str = "proj/settings.py") -> None:+ _write(root, "proj/__init__.py")+ for rel, text in _APP_FILES.items():+ _write(root, f"proj/shop/{rel}", text)+ _write(root, settings_path, settings)+++def _edges(root: Path, source: str = "proj/settings.py") -> dict[str, tuple[str, tuple[str, ...]]]:+ return {+ e.target: (e.edge_type, e.imported_names)+ for e in DjangoDynamicHints().extract(root)+ if e.source == source+ }+++_SHOP_EDGES = {+ "proj/shop/__init__.py": ("dynamic_imports", ()),+ "proj/shop/admin.py": ("dynamic_uses", ("ItemAdmin", "Media", "OrderAdmin")),+ "proj/shop/apps.py": ("dynamic_uses", ("ShopConfig",)),+ "proj/shop/models.py": ("dynamic_imports", ()),+ "proj/shop/templatetags/extras.py": ("dynamic_imports", ()),+ "proj/shop/management/commands/prune.py": ("dynamic_imports", ()),+}+++def test_installed_app_convention_modules_get_edges(tmp_path: Path) -> None:+ _project(tmp_path, 'INSTALLED_APPS = ("django.contrib.admin", "proj.shop")\n')+ # No edge to signals, views, a nested templatetags module, a ``_`` command.+ assert _edges(tmp_path) == _SHOP_EDGES+++def test_extended_and_summed_settings_are_read(tmp_path: Path) -> None:+ _project(tmp_path, 'BASE = ["django.contrib.admin"]\nINSTALLED_APPS = BASE + ["proj.shop"]\n')+ assert _edges(tmp_path) == _SHOP_EDGES+ _write(tmp_path, "proj/settings.py", 'INSTALLED_APPS = []\nINSTALLED_APPS += ["proj.shop"]\n')+ assert _edges(tmp_path) == _SHOP_EDGES+++def test_split_settings_link_from_each_file_that_installs_the_app(tmp_path: Path) -> None:+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop"]\n', "proj/settings/base.py")+ _write(tmp_path, "proj/settings/__init__.py")+ _write(tmp_path, "proj/settings/dev.py", "DEBUG = True\n")+ assert _edges(tmp_path, "proj/settings/base.py") == _SHOP_EDGES+ assert _edges(tmp_path, "proj/settings/dev.py") == {}+++def test_an_app_config_path_names_the_app_package(tmp_path: Path) -> None:+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop.apps.ShopConfig"]\n')+ assert "proj/shop/admin.py" in _edges(tmp_path)+++def test_an_apps_package_config_path_names_the_app_package(tmp_path: Path) -> None:+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop.apps.ShopConfig"]\n')+ (tmp_path / "proj/shop/apps.py").unlink()+ _write(tmp_path, "proj/shop/apps/__init__.py", _APPS)+ _write(tmp_path, "proj/shop/apps/admin.py", "")+ edges = _edges(tmp_path)+ assert "proj/shop/admin.py" in edges+ assert "proj/shop/apps/admin.py" not in edges+++def test_an_app_not_installed_gets_no_edges(tmp_path: Path) -> None:+ _project(tmp_path, 'INSTALLED_APPS = ["django.contrib.admin"]\n')+ assert _edges(tmp_path) == {}+++def test_a_repo_without_django_settings_gets_no_edges(tmp_path: Path) -> None:+ for rel, text in _APP_FILES.items():+ _write(tmp_path, f"shop/{rel}", text)+ assert DjangoDynamicHints().extract(tmp_path) == []+++def _dead_code(repo: Path) -> set[tuple[DeadCodeKind, str, str | None]]:+ builder = GraphBuilder(repo_path=repo)+ parser = ASTParser()+ paths = []+ for fi in FileTraverser(repo).traverse():+ builder.add_file(parser.parse_file(fi, Path(fi.abs_path).read_bytes()))+ paths.append(fi.path)+ builder.build()+ builder.add_dynamic_edges(HintRegistry().extract_all(repo, file_paths=paths))+ report = DeadCodeAnalyzer(builder.graph(), git_meta_map={}).analyze(+ {"detect_zombie_packages": False, "min_confidence": 0.0}+ )+ return {(f.kind, f.file_path, f.symbol_name) for f in report.findings}+++def test_only_what_django_loads_counts_as_used(tmp_path: Path) -> None:+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop"]\n')+ found = _dead_code(tmp_path)+ unreachable = {path for kind, path, _ in found if kind is DeadCodeKind.UNREACHABLE_FILE}+ assert unreachable >= {+ "proj/shop/signals.py",+ "proj/shop/views.py",+ "proj/shop/templatetags/nested/deep.py",+ "proj/shop/management/commands/_shared.py",+ }+ assert not unreachable & {+ "proj/shop/admin.py",+ "proj/shop/apps.py",+ "proj/shop/models.py",+ "proj/shop/templatetags/extras.py",+ "proj/shop/management/commands/prune.py",+ }+ exports = {(path, name) for kind, path, name in found if kind is DeadCodeKind.UNUSED_EXPORT}+ # A dead helper beside the classes Django uses is still reported.+ assert {+ ("proj/shop/admin.py", "dead_helper"),+ ("proj/shop/admin.py", "UnregisteredAdmin"),+ ("proj/shop/apps.py", "unused_in_apps"),+ } <= exports+ assert not exports & {+ ("proj/shop/admin.py", "ItemAdmin"),+ ("proj/shop/admin.py", "Media"),+ ("proj/shop/admin.py", "OrderAdmin"),+ ("proj/shop/apps.py", "ShopConfig"),+ }