Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(dead-code): treat the modules Django loads from an installed app as used (#2975)

Raghav Chamadiya60bacdc28299
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
index 8f0615e60..df3535d05 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
@@ -19,6 +19,9 @@ class DynamicEdge:
edge_type: DynamicKind
hint_source: str # extractor name
weight: float = 1.0
+ # The target's symbols the runtime reaches. Empty means the whole module:
+ # a ``dynamic_uses`` edge without names counts every public symbol as used.
+ imported_names: tuple[str, ...] = ()
class DynamicHintExtractor(ABC):
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
index 779b26214..8ad74d093 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
@@ -2,10 +2,99 @@ from __future__ import annotations
import ast
import re
-from pathlib import Path
+from collections.abc import Callable
+from pathlib import Path, PurePosixPath
from .base import DynamicEdge, DynamicHintExtractor
+#: Reads a module's syntax tree and returns the class names Django uses there.
+UsedNames = Callable[[ast.Module], set[str]]
+
+
+def _last_segment(node: ast.expr) -> str:
+ """``AppConfig`` for ``AppConfig``, ``apps.AppConfig`` and ``admin.register(...)``."""
+ if isinstance(node, ast.Call):
+ node = node.func
+ if isinstance(node, ast.Subscript): # ``ModelAdmin[Profile]``
+ node = node.value
+ if isinstance(node, ast.Attribute):
+ return node.attr
+ return node.id if isinstance(node, ast.Name) else ""
+
+
+def _app_config_classes(tree: ast.Module) -> set[str]:
+ """The ``AppConfig`` subclasses, which the app registry instantiates."""
+ return {
+ node.name
+ for node in tree.body
+ if isinstance(node, ast.ClassDef)
+ and any(_last_segment(base) == "AppConfig" for base in node.bases)
+ }
+
+
+def _names_passed_to_register(tree: ast.Module) -> set[str]:
+ """Names given as an admin class to ``site.register(Model, Admin)``."""
+ return {
+ arg.id
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call) and _last_segment(node.func) == "register"
+ for arg in node.args[1:]
+ if isinstance(arg, ast.Name)
+ }
+
+
+def _registered_admin_classes(tree: ast.Module) -> set[str]:
+ """Admin classes registered by ``@admin.register`` or ``site.register``,
+ and the classes nested in them (``Media``), which the admin reads."""
+ passed = _names_passed_to_register(tree)
+ registered = [
+ node
+ for node in tree.body
+ if isinstance(node, ast.ClassDef)
+ and (node.name in passed or any(_last_segment(d) == "register" for d in node.decorator_list))
+ ]
+ return {node.name for node in registered} | {
+ inner.name for node in registered for inner in node.body if isinstance(inner, ast.ClassDef)
+ }
+
+
+#: Modules Django loads from every installed app by convention, as paths
+#: inside the app package (``*`` is one module name). No source file imports
+#: them: the app registry imports ``apps`` and ``models``, the admin
+#: autodiscovers ``admin``, the template engine loads every ``templatetags``
+#: library and ``manage.py`` finds each ``management/commands`` module whose
+#: name does not start with ``_``. Where Django also uses classes of the
+#: module, the second item names them, so only those count as used and a dead
+#: helper beside them is still reported. ``signals``, ``urls`` and ``views``
+#: are not here: Django never loads those by name, an app imports them itself.
+_APP_CONVENTION_MODULES: tuple[tuple[str, UsedNames | None], ...] = (
+ ("apps.py", _app_config_classes),
+ ("admin.py", _registered_admin_classes),
+ ("models.py", None),
+ ("templatetags/*.py", None),
+ ("management/commands/[!_]*.py", None),
+)
+
+
+def _convention_match(rel_in_app: PurePosixPath) -> tuple[bool, UsedNames | None]:
+ """Whether Django loads the app module at *rel_in_app*, and what names its used classes."""
+ for pattern, used_names in _APP_CONVENTION_MODULES:
+ # ``match`` anchors on the right only; equal depth anchors both ends.
+ if len(rel_in_app.parts) == pattern.count("/") + 1 and rel_in_app.match(pattern):
+ return True, used_names
+ return False, None
+
+
+def _used_names(path: Path, used_names: UsedNames | None) -> tuple[str, ...]:
+ """The names *used_names* finds in the module at *path*, sorted; none if unreadable."""
+ if used_names is None:
+ return ()
+ try:
+ tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"), filename=str(path))
+ except (OSError, SyntaxError, ValueError):
+ return ()
+ return tuple(sorted(used_names(tree)))
+
def _app_to_path(app: str, repo_root: Path) -> str | None:
"""Attempt to resolve a dotted app name to an __init__.py under repo_root."""
@@ -35,10 +124,38 @@ def _module_to_path(module: str, repo_root: Path) -> str | None:
return None
+def _installed_app_init(entry: str, repo_root: Path) -> str | None:
+ """The ``__init__.py`` of the app an ``INSTALLED_APPS`` entry names.
+
+ An entry is an app package (``polls``) or an ``AppConfig`` path
+ (``polls.apps.PollsConfig``), whose app is the package holding the module.
+ """
+ init = _app_to_path(entry, repo_root)
+ if init is not None or "." not in entry:
+ return init
+ module_name = entry[: entry.rindex(".")]
+ module = _module_to_path(module_name, repo_root)
+ if module is None:
+ return None
+ module_path = PurePosixPath(module)
+ if module_path.name != "__init__.py":
+ app_dir = module_path.parent
+ elif module_path.parent.name == "apps":
+ # An ``apps`` package inside the app (``polls/apps/__init__.py``).
+ app_dir = module_path.parent.parent
+ else:
+ # The config sits in the app's own ``__init__`` (``polls.PollsConfig``).
+ return module
+ package_init = (app_dir / "__init__.py").as_posix()
+ return package_init if (repo_root / package_init).exists() else None
+
+
def _extract_string_list(node: ast.expr) -> list[str]:
- """Extract string literals from an ast.List node."""
+ """String literals of a list or tuple, or of a sum of them (``BASE + [...]``)."""
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+ return _extract_string_list(node.left) + _extract_string_list(node.right)
results: list[str] = []
- if not isinstance(node, ast.List):
+ if not isinstance(node, (ast.List, ast.Tuple)):
return results
for elt in node.elts:
if isinstance(elt, ast.Constant) and isinstance(elt.value, str):
@@ -64,6 +181,9 @@ class DjangoDynamicHints(DynamicHintExtractor):
def _scan_settings(self, repo_root: Path) -> list[DynamicEdge]:
edges: list[DynamicEdge] = []
+ # Convention modules per app ``__init__``, shared by every settings
+ # file that installs the app (``base.py``, ``dev.py``, ``prod.py``).
+ app_modules: dict[str, list[tuple[str, tuple[str, ...]]]] = {}
# Collect all settings files
settings_files: list[Path] = list(self._rglob(repo_root, "settings.py"))
@@ -90,25 +210,20 @@ class DjangoDynamicHints(DynamicHintExtractor):
continue
for node in ast.walk(tree):
- if not isinstance(node, ast.Assign):
+ # ``INSTALLED_APPS += [...]`` extends the setting in place.
+ if not isinstance(node, (ast.Assign, ast.AugAssign)):
continue
- for target in node.targets:
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]
+ for target in targets:
if not (isinstance(target, ast.Name)):
continue
name = target.id
if name == "INSTALLED_APPS":
for app in _extract_string_list(node.value):
- resolved = _app_to_path(app, repo_root)
- if resolved:
- edges.append(
- DynamicEdge(
- source=rel_settings,
- target=resolved,
- edge_type="dynamic_imports",
- hint_source=self.name,
- )
- )
+ edges.extend(
+ self._installed_app_edges(rel_settings, app, repo_root, app_modules)
+ )
elif name == "ROOT_URLCONF":
module = _extract_string_value(node.value)
@@ -139,6 +254,53 @@ class DjangoDynamicHints(DynamicHintExtractor):
return edges
+ def _installed_app_edges(
+ self,
+ rel_settings: str,
+ app: str,
+ repo_root: Path,
+ app_modules: dict[str, list[tuple[str, tuple[str, ...]]]],
+ ) -> list[DynamicEdge]:
+ """Edges from a settings file to an installed app and its convention modules.
+
+ A module whose classes Django uses gets a ``dynamic_uses`` edge naming
+ them; any other gets ``dynamic_imports``, which marks the module
+ loaded and none of its symbols used.
+ """
+ init = _installed_app_init(app, repo_root)
+ if init is None:
+ return []
+ if init not in app_modules:
+ app_modules[init] = self._convention_modules(repo_root, init)
+ return [
+ DynamicEdge(
+ source=rel_settings,
+ target=target,
+ edge_type="dynamic_uses" if names else "dynamic_imports",
+ hint_source=self.name,
+ imported_names=names,
+ )
+ for target, names in [(init, ()), *app_modules[init]]
+ ]
+
+ def _convention_modules(
+ self, repo_root: Path, app_init: str
+ ) -> list[tuple[str, tuple[str, ...]]]:
+ """The modules under an app that Django loads by convention, with the names it uses."""
+ app_dir = repo_root / PurePosixPath(app_init).parent
+ found: list[tuple[str, tuple[str, ...]]] = []
+ # Through ``_rglob`` so a file the index excludes gets no edge.
+ for path in self._rglob(app_dir, "*.py"):
+ # A package ``__init__`` is reached through its package already.
+ if path.name == "__init__.py":
+ continue
+ rel_in_app = PurePosixPath(path.relative_to(app_dir).as_posix())
+ loaded, used_names = _convention_match(rel_in_app)
+ if loaded:
+ rel = path.relative_to(repo_root).as_posix()
+ found.append((rel, _used_names(path, used_names)))
+ return found
+
def _scan_urls(self, repo_root: Path) -> list[DynamicEdge]:
edges: list[DynamicEdge] = []
include_re = re.compile(r"""include\(\s*['\"]([\w\.]+)['\"]""")
diff --git a/packages/core/src/repowise/core/ingestion/graph/_edges.py b/packages/core/src/repowise/core/ingestion/graph/_edges.py
index 9a1bef8ed..aa40b0aa7 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_edges.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_edges.py
@@ -150,12 +150,14 @@ class EdgesMixin:
# a real `dynamic_*` edge.
sub_type = e.edge_type
graph_edge_type = sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"
+ names = {"imported_names": list(e.imported_names)} if e.imported_names else {}
self._graph.add_edge(
e.source,
e.target,
edge_type=graph_edge_type,
hint_source=e.hint_source,
weight=e.weight,
+ **names,
)
# A ``:test`` hint used to set ``is_test`` on the source node. Only
# the Rust hinter emits one, for `#[test]` / `#[cfg(test)]` markers,
diff --git a/tests/unit/ingestion/test_django_dynamic_hints.py b/tests/unit/ingestion/test_django_dynamic_hints.py
new file mode 100644
index 000000000..02257ba0c
--- /dev/null
+++ b/tests/unit/ingestion/test_django_dynamic_hints.py
@@ -0,0 +1,193 @@
+"""Django loads an installed app's convention modules by name.
+
+From healthchecks: ``hc/*/admin.py``, ``hc/front/models.py``, ``apps.py``,
+``management/commands/*.py`` and ``templatetags/*.py`` have no importer, so
+each was reported as an unreachable file, and the ``AppConfig`` classes and
+the admin ``Media`` inner classes as unused exports.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKind
+from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
+from repowise.core.ingestion.dynamic_hints import HintRegistry
+from repowise.core.ingestion.dynamic_hints.django import DjangoDynamicHints
+
+_ADMIN = """from django.contrib import admin
+from .models import Item, Order
+
+
+class ItemAdmin(admin.ModelAdmin):
+ class Media:
+ css = {"all": ("item.css",)}
+
+
+class OrderAdmin(admin.ModelAdmin):
+ pass
+
+
+admin.site.register(Order, OrderAdmin)
+
+
+class UnregisteredAdmin(admin.ModelAdmin):
+ pass
+
+
+def dead_helper():
+ return 1
+"""
+
+_APPS = """from django.apps import AppConfig
+
+
+class ShopConfig(AppConfig):
+ name = "proj.shop"
+
+
+def unused_in_apps():
+ return 2
+"""
+
+_APP_FILES = {
+ "__init__.py": "",
+ "admin.py": _ADMIN,
+ "apps.py": _APPS,
+ "models.py": "class Item:\n pass\n\n\nclass Order:\n pass\n",
+ "signals.py": "def on_save():\n return 3\n",
+ "views.py": "def index():\n return 4\n",
+ "templatetags/__init__.py": "",
+ "templatetags/extras.py": "X = 1\n",
+ "templatetags/nested/deep.py": "Y = 1\n",
+ "management/__init__.py": "",
+ "management/commands/__init__.py": "",
+ "management/commands/prune.py": "class Command:\n pass\n",
+ "management/commands/_shared.py": "Z = 1\n",
+}
+
+
+def _write(root: Path, rel: str, text: str = "") -> None:
+ path = root / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text, encoding="utf-8")
+
+
+def _project(root: Path, settings: str, settings_path: str = "proj/settings.py") -> None:
+ _write(root, "proj/__init__.py")
+ for rel, text in _APP_FILES.items():
+ _write(root, f"proj/shop/{rel}", text)
+ _write(root, settings_path, settings)
+
+
+def _edges(root: Path, source: str = "proj/settings.py") -> dict[str, tuple[str, tuple[str, ...]]]:
+ return {
+ e.target: (e.edge_type, e.imported_names)
+ for e in DjangoDynamicHints().extract(root)
+ if e.source == source
+ }
+
+
+_SHOP_EDGES = {
+ "proj/shop/__init__.py": ("dynamic_imports", ()),
+ "proj/shop/admin.py": ("dynamic_uses", ("ItemAdmin", "Media", "OrderAdmin")),
+ "proj/shop/apps.py": ("dynamic_uses", ("ShopConfig",)),
+ "proj/shop/models.py": ("dynamic_imports", ()),
+ "proj/shop/templatetags/extras.py": ("dynamic_imports", ()),
+ "proj/shop/management/commands/prune.py": ("dynamic_imports", ()),
+}
+
+
+def test_installed_app_convention_modules_get_edges(tmp_path: Path) -> None:
+ _project(tmp_path, 'INSTALLED_APPS = ("django.contrib.admin", "proj.shop")\n')
+ # No edge to signals, views, a nested templatetags module, a ``_`` command.
+ assert _edges(tmp_path) == _SHOP_EDGES
+
+
+def test_extended_and_summed_settings_are_read(tmp_path: Path) -> None:
+ _project(tmp_path, 'BASE = ["django.contrib.admin"]\nINSTALLED_APPS = BASE + ["proj.shop"]\n')
+ assert _edges(tmp_path) == _SHOP_EDGES
+ _write(tmp_path, "proj/settings.py", 'INSTALLED_APPS = []\nINSTALLED_APPS += ["proj.shop"]\n')
+ assert _edges(tmp_path) == _SHOP_EDGES
+
+
+def test_split_settings_link_from_each_file_that_installs_the_app(tmp_path: Path) -> None:
+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop"]\n', "proj/settings/base.py")
+ _write(tmp_path, "proj/settings/__init__.py")
+ _write(tmp_path, "proj/settings/dev.py", "DEBUG = True\n")
+ assert _edges(tmp_path, "proj/settings/base.py") == _SHOP_EDGES
+ assert _edges(tmp_path, "proj/settings/dev.py") == {}
+
+
+def test_an_app_config_path_names_the_app_package(tmp_path: Path) -> None:
+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop.apps.ShopConfig"]\n')
+ assert "proj/shop/admin.py" in _edges(tmp_path)
+
+
+def test_an_apps_package_config_path_names_the_app_package(tmp_path: Path) -> None:
+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop.apps.ShopConfig"]\n')
+ (tmp_path / "proj/shop/apps.py").unlink()
+ _write(tmp_path, "proj/shop/apps/__init__.py", _APPS)
+ _write(tmp_path, "proj/shop/apps/admin.py", "")
+ edges = _edges(tmp_path)
+ assert "proj/shop/admin.py" in edges
+ assert "proj/shop/apps/admin.py" not in edges
+
+
+def test_an_app_not_installed_gets_no_edges(tmp_path: Path) -> None:
+ _project(tmp_path, 'INSTALLED_APPS = ["django.contrib.admin"]\n')
+ assert _edges(tmp_path) == {}
+
+
+def test_a_repo_without_django_settings_gets_no_edges(tmp_path: Path) -> None:
+ for rel, text in _APP_FILES.items():
+ _write(tmp_path, f"shop/{rel}", text)
+ assert DjangoDynamicHints().extract(tmp_path) == []
+
+
+def _dead_code(repo: Path) -> set[tuple[DeadCodeKind, str, str | None]]:
+ builder = GraphBuilder(repo_path=repo)
+ parser = ASTParser()
+ paths = []
+ for fi in FileTraverser(repo).traverse():
+ builder.add_file(parser.parse_file(fi, Path(fi.abs_path).read_bytes()))
+ paths.append(fi.path)
+ builder.build()
+ builder.add_dynamic_edges(HintRegistry().extract_all(repo, file_paths=paths))
+ report = DeadCodeAnalyzer(builder.graph(), git_meta_map={}).analyze(
+ {"detect_zombie_packages": False, "min_confidence": 0.0}
+ )
+ return {(f.kind, f.file_path, f.symbol_name) for f in report.findings}
+
+
+def test_only_what_django_loads_counts_as_used(tmp_path: Path) -> None:
+ _project(tmp_path, 'INSTALLED_APPS = ["proj.shop"]\n')
+ found = _dead_code(tmp_path)
+ unreachable = {path for kind, path, _ in found if kind is DeadCodeKind.UNREACHABLE_FILE}
+ assert unreachable >= {
+ "proj/shop/signals.py",
+ "proj/shop/views.py",
+ "proj/shop/templatetags/nested/deep.py",
+ "proj/shop/management/commands/_shared.py",
+ }
+ assert not unreachable & {
+ "proj/shop/admin.py",
+ "proj/shop/apps.py",
+ "proj/shop/models.py",
+ "proj/shop/templatetags/extras.py",
+ "proj/shop/management/commands/prune.py",
+ }
+ exports = {(path, name) for kind, path, name in found if kind is DeadCodeKind.UNUSED_EXPORT}
+ # A dead helper beside the classes Django uses is still reported.
+ assert {
+ ("proj/shop/admin.py", "dead_helper"),
+ ("proj/shop/admin.py", "UnregisteredAdmin"),
+ ("proj/shop/apps.py", "unused_in_apps"),
+ } <= exports
+ assert not exports & {
+ ("proj/shop/admin.py", "ItemAdmin"),
+ ("proj/shop/admin.py", "Media"),
+ ("proj/shop/admin.py", "OrderAdmin"),
+ ("proj/shop/apps.py", "ShopConfig"),
+ }