Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(ingestion): link modules named by string in lazy registries (#3220)

Raghav Chamadiya0c0a0f613b7c
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
index c4ac684ff..9f66d0609 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
@@ -1463,10 +1463,11 @@ class DeadCodeAnalyzer:
# member of the file. Only these two: ``dynamic_imports`` means the
# module is loaded, which a plain import edge does not rescue either.
# A framework edge that names its exports (a Lambda handler) rescues
- # only those, through the importer check below.
+ # only those, through the importer check below. ``dynamic_hint`` is a
+ # whole-module dynamic use folded into a static edge on the same pair.
if any(
- edge.get("edge_type") in ("dynamic_uses", "framework")
- and not edge.get("imported_names")
+ (edge.get("edge_type") in ("dynamic_uses", "framework") and not edge.get("imported_names"))
+ or edge.get("dynamic_hint")
for edge in (
self.graph.get_edge_data(pred, node, {}) for pred in self.graph.predecessors(node)
)
diff --git a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
index d26c27d01..bebd7c81e 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py
@@ -19,6 +19,7 @@ from __future__ import annotations
from pathlib import Path
from repowise.core.ids import is_external
+from repowise.core.ingestion.languages.python_strings import PY_DYNAMIC_LOAD_MARKERS
from repowise.core.ingestion.models import is_dynamic_edge
@@ -52,14 +53,7 @@ def read_source_text(
# Patterns in source that indicate dynamic/runtime imports, keyed by suffix.
_DYNAMIC_IMPORT_MARKERS: dict[str, tuple[str, ...]] = {
- ".py": (
- "importlib.import_module",
- # ``from importlib import import_module``, then a bare call.
- "import_module(",
- "__import__(",
- "importlib.reload",
- "pkgutil.iter_modules",
- ),
+ ".py": PY_DYNAMIC_LOAD_MARKERS,
# JS/TS dynamic-dispatch markers. Each implies the surrounding file
# (and, by the package-co-location heuristic, its siblings) reaches
# code through a runtime mechanism the static import graph cannot
@@ -517,7 +511,8 @@ def find_dynamic_edge_files(graph) -> set[str]:
result: set[str] = set()
try:
for u, v, data in graph.edges(data=True):
- if not is_dynamic_edge(data.get("edge_type", "")):
+ # ``dynamic_hint``: a dynamic hint folded into a static edge.
+ if not (is_dynamic_edge(data.get("edge_type", "")) or data.get("dynamic_hint")):
continue
for endpoint in (u, v):
if endpoint is None:
diff --git a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
index f2f12e3ac..e1d387273 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py
@@ -29,14 +29,12 @@ fixture app it loads by name), never a member path (a ``mock.patch`` target).
from __future__ import annotations
-import io
import re
-import tokenize
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from pathlib import PurePosixPath
-from ...ingestion.languages.registry import REGISTRY
+from ...ingestion.languages.python_strings import defines_top_level, is_python, live_text
from ...test_paths import is_test_related_path
from .models import DeadCodeFindingData, DeadCodeKind
@@ -80,10 +78,6 @@ def _directory(path: str) -> str:
return "" if parent == "." else parent
-def _is_python(path: str) -> bool:
- return REGISTRY.from_extension(PurePosixPath(path).suffix) == "python"
-
-
def drop_named_modules(
findings: list[DeadCodeFindingData],
source_map: dict[str, bytes],
@@ -104,7 +98,7 @@ def drop_named_modules(
def _is_candidate(finding: DeadCodeFindingData) -> bool:
kinds = (DeadCodeKind.UNREACHABLE_FILE, DeadCodeKind.UNUSED_EXPORT)
- return finding.kind in kinds and _is_python(finding.file_path)
+ return finding.kind in kinds and is_python(finding.file_path)
def _is_named(
@@ -132,7 +126,7 @@ def _named_by_strings(
# Tokenize only a file whose raw text names a candidate at all.
if next(reader.uses(blob), None) is None:
continue
- for target, attr in reader.uses(_live_text(path, blob)):
+ for target, attr in reader.uses(live_text(path, blob)):
named.add(target)
if attr:
attrs.add((target, attr))
@@ -169,56 +163,4 @@ class _Reader:
"""For ``"pkg.module.Name"``: the modules ``pkg.module`` names that define ``Name``."""
parent, _, name = module.rpartition(".")
owners = self._resolve(parent) if parent else set()
- return {t for t in owners if _defines_top_level(members.get(t, b""), name)}, name
-
-
-def _defines_top_level(blob: bytes, name: str) -> bool:
- """Whether a Python module defines *name* at top level (def, class, assignment).
-
- A re-export (``from x import Name``) or a tuple target is not seen, so such
- a member stays reported.
- """
- word = re.escape(name.encode("ascii"))
- pattern = (
- rb"^(?:(?:async[ \t]+)?def|class)[ \t]+" + word + rb"\b"
- rb"|^" + word + rb"[ \t]*(?::[^=\n]*)?=(?!=)"
- )
- return re.search(pattern, blob, re.MULTILINE) is not None
-
-
-#: Tokens after which a string starts an expression statement (a docstring).
-_STATEMENT_START = frozenset(
- {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT, tokenize.ENCODING}
-)
-_TRIVIA = frozenset({tokenize.NL, tokenize.COMMENT})
-_COMMENT_LINE_RE = re.compile(rb"^[ \t]*#.*$", re.MULTILINE)
-
-
-def _live_text(path: str, blob: bytes) -> bytes:
- """*blob* without comments and, for Python, without docstrings.
-
- Python keeps only its string literals that are not a statement on their
- own; other files drop only ``#`` comment lines. Python that does not tokenize
- is kept whole, as before this filter.
- """
- if not _is_python(path):
- return _COMMENT_LINE_RE.sub(b"", blob)
- try:
- tokens = [
- t for t in tokenize.tokenize(io.BytesIO(blob).readline) if t.type not in _TRIVIA
- ]
- except (tokenize.TokenError, SyntaxError, ValueError):
- return blob
- return b"\n".join(
- tok.string.encode("utf-8")
- for i, tok in enumerate(tokens)
- if tok.type == tokenize.STRING and not _is_statement(tokens, i)
- )
-
-
-def _is_statement(tokens: list[tokenize.TokenInfo], i: int) -> bool:
- """Whether ``tokens[i]`` is a statement on its own (a docstring)."""
- # ENDMARKER always closes the stream, so ``i + 1`` exists.
- if tokens[i - 1].type not in _STATEMENT_START:
- return False
- return tokens[i + 1].type in (tokenize.NEWLINE, tokenize.ENDMARKER)
+ return {t for t in owners if defines_top_level(members.get(t, b""), name)}, name
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
index df3535d05..a0d01ebea 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
@@ -38,9 +38,25 @@ class DynamicHintExtractor(ABC):
# None -> extractors that need it build their own, as before.
_dotnet_index = None
+ # Ingestion's ``{repo-relative path: raw bytes}`` for the parsed files,
+ # attached by HintRegistry.extract_all when the caller holds one, so an
+ # extractor reading every file of a language reuses the parse pass's
+ # bytes. None (or a missing path) -> read from disk.
+ _source_map = None
+
@abstractmethod
def extract(self, repo_root: Path) -> list[DynamicEdge]: ...
+ def _source_bytes(self, abs_path: Path, rel: str) -> bytes:
+ """*rel*'s bytes from the attached source map, else from disk; ``b""`` if neither."""
+ # Same contract as ``ingestion.source_text.source_bytes``; collapse into it once both exist.
+ if self._source_map is not None and (data := self._source_map.get(rel)) is not None:
+ return data
+ try:
+ return abs_path.read_bytes()
+ except OSError:
+ return b""
+
def _rglob(self, root: Path, pattern: str) -> Iterator[Path]:
"""Pruned replacement for :py:meth:`pathlib.Path.rglob`.
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.py
index 38f542215..de43f334c 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.py
@@ -12,47 +12,228 @@ module is named by a *string* and loaded with ``importlib.import_module``
cls = getattr(module, _PROVIDERS[name][1])
Nothing statically imports ``my_pkg.providers.ollama`` or ``OllamaProvider``,
-so dead-code analysis would flag both the module and the class as unused.
-
-This extractor recovers those edges generically: in any file that uses
-dynamic-import machinery, every quoted string that resolves to a real
-in-repo module (via :func:`python_modules.build_python_module_index`) yields
-a ``dynamic_uses`` edge to that module's file. The analyzer then treats the
-target file as runtime-loaded and stops flagging its public members.
-
-Gating on the presence of dynamic-import machinery (rather than emitting an
-edge for any dotted string that happens to match a module) keeps the signal
-precise: a dotted string in a log message or docstring of an ordinary module
-is ignored. The mechanism is repo-agnostic — it works for any Python plugin
-registry, ``importlib`` loader, or entry-point dispatch table.
+so dead-code analysis would flag both the module and the class as unused,
+and test selection would never connect the registry to the module.
+
+This extractor recovers those edges generically. Every string it considers
+must resolve to a real in-repo module (via
+:func:`python_modules.build_python_module_index`); a string that names
+nothing in the repo adds nothing, and only strings that run count (a comment
+or docstring loads nothing). Three shapes are read:
+
+* **Dotted module paths** (``"pkg.sub.mod"``), only in files that use
+ dynamic-import machinery (:data:`_DYNAMIC_IMPORT_MARKERS`). Gating keeps a
+ dotted string in a log message of an ordinary module inert.
+* **Attribute references**: ``"pkg.mod:attr"`` in any file (the spelling of
+ console-script entry points, ASGI/WSGI app specs and lazy command
+ registries), and ``"pkg.mod.Name"`` in gated files outside tests (the
+ ``import_string`` / lazy command group spelling; in a test it is a patch
+ target). A relative ``"status_cmd:status_command"`` resolves under any
+ package the same file names by a dotted string, which is how lazy command
+ tables join a base package to short names. The module must define or
+ import the attribute (or define ``__getattr__``): ``"agent:start"`` as an
+ event name, next to a package called ``agent`` without a ``start``, adds
+ nothing.
+* **Templated loads** (``import_module(f"pkg.plugins.{name}")`` or
+ ``import_module("pkg.plugins." + name)``) in gated non-test code: every
+ module matching the template, where each hole is one name segment. A
+ template that starts with a hole adds nothing; one with a single-segment
+ prefix or more than :data:`_TEMPLATE_FANOUT_CAP` matches links the package
+ only. A call in a comment or docstring is skipped.
+
+The mechanism is repo-agnostic: plugin registries, ``importlib`` loaders,
+entry-point dispatch tables and lazy CLI groups all reduce to these shapes.
"""
from __future__ import annotations
import re
+from bisect import bisect
+from collections.abc import Callable
from pathlib import Path
+from ...test_paths import is_test_related_path
from ..languages.python_modules import build_python_module_index
+from ..languages.python_strings import PY_DYNAMIC_LOAD_MARKERS, defines_top_level, live_text
from .base import DynamicEdge, DynamicHintExtractor
-# Tokens that signal a file performs runtime module loading. An edge is only
-# emitted from files that contain at least one of these, so plain dotted
-# strings elsewhere never create spurious reachability.
-_DYNAMIC_IMPORT_MARKERS: tuple[str, ...] = (
- "importlib",
- "import_module",
- "__import__",
- "import_string", # Werkzeug / Flask / Django utilities
- "load_entry_point",
- "entry_points(", # importlib.metadata plugin discovery
- "pkgutil",
- "pkg_resources",
+# Dotted strings only produce edges from files that load modules at run time,
+# so plain dotted strings elsewhere never create spurious reachability.
+_DYNAMIC_IMPORT_MARKERS = tuple(m.encode("ascii") for m in PY_DYNAMIC_LOAD_MARKERS)
+
+# A quoted dotted path of two or more segments (``"pkg.sub.mod"``).
+_DOTTED_STRING_RE = re.compile(rb"""['"]([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)+)['"]""")
+# A quoted ``module:attr``: all an ungated file is scanned for.
+_ATTR_STRING_RE = re.compile(rb"""['"]([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*):([A-Za-z_]\w*)['"]""")
+# Its colon-first tail: a literal-led scan about ten times faster, so files
+# without one skip the full pattern.
+_ATTR_TAIL_RE = re.compile(rb""":[A-Za-z_]\w*['"]""")
+
+# ``import_module(f"...")`` / ``__import__(f"...")`` and the ``"prefix." + x``
+# concatenation. Group 2 is the f-string body, group 4 the literal prefix.
+_TEMPLATE_LOAD_RE = re.compile(
+ rb"""(?:import_module|__import__)\(\s*"""
+ rb"""(?:f(['"])([\w.{}]+)\1|(['"])((?:[a-zA-Z_]\w*\.)+)\3\s*\+)"""
)
+_HOLE_RE = re.compile(r"\{[^{}]*\}")
+_SEGMENT = r"[a-zA-Z_]\w*"
+_DOTTED_PREFIX_RE = re.compile(rf"{_SEGMENT}(?:\.{_SEGMENT})*")
+
+# Most modules one templated load may link before it links the package only.
+_TEMPLATE_FANOUT_CAP = 50
+
+# Top-level import statements, parenthesised continuation included.
+_TOP_IMPORT_RE = re.compile(rb"^(?:from[ \t]+[\w.]+[ \t]+)?import[ \t]+(\([^)]*\)|[^\n]*)", re.M)
+
+_TRIPLE_QUOTE_RE = re.compile(rb"\"\"\"|'''")
+
+_INIT_NAMES = ("__init__.py", "__init__.pyi")
+
+# Target file -> names reached there; an empty tuple means the whole module.
+_Refs = dict[str, tuple[str, ...]]
+# (target, attribute or None for the whole module, offset of the string)
+_Ref = tuple[str, str | None, int]
+
+
+def _add_ref(refs: _Refs, target: str, name: str | None) -> None:
+ """Record *target*; a whole-module reference absorbs any named one."""
+ if name is None or refs.get(target) == ():
+ refs[target] = ()
+ return
+ refs[target] = tuple(sorted({*refs.get(target, ()), name}))
+
+
+class ModuleStringResolver:
+ """Resolves module-path strings against one repo's modules.
+
+ *source_of* returns a module file's bytes, read only to check that an
+ attribute reference names something the module has. Attribute checks and
+ template expansions are memoised, so one instance serves a whole repo.
+ """
+
+ def __init__(self, module_index: dict[str, str], source_of: Callable[[str], bytes]) -> None:
+ self.module_index = module_index
+ self._source_of = source_of
+ self._attrs: dict[tuple[str, str], bool] = {}
+ self._templates: dict[str, list[str]] = {}
-# A quoted dotted path of two or more identifier segments — i.e. something
-# shaped like ``a.b`` / ``pkg.sub.mod``. Single-segment names are too
-# ambiguous (and resolve via ordinary stem matching anyway).
-_DOTTED_STRING_RE = re.compile(r"""['"]([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)+)['"]""")
+ def has_attr(self, target: str, name: str) -> bool:
+ """Whether a module defines, imports or lazily serves *name* at top level."""
+ key = (target, name)
+ if key not in self._attrs:
+ blob = self._source_of(target)
+ word = re.compile(rb"\b" + re.escape(name.encode("ascii")) + rb"\b")
+ self._attrs[key] = (
+ defines_top_level(blob, name)
+ or defines_top_level(blob, "__getattr__")
+ or any(word.search(m.group(1)) for m in _TOP_IMPORT_RE.finditer(blob))
+ )
+ return self._attrs[key]
+
+ def template_targets(self, template: str) -> list[str]:
+ """Files of every module matching a load template; each hole is one segment."""
+ if template not in self._templates:
+ self._templates[template] = self._expand(template)
+ return self._templates[template]
+
+ def _expand(self, template: str) -> list[str]:
+ prefix = _HOLE_RE.split(template, maxsplit=1)[0].rstrip(".")
+ if not _DOTTED_PREFIX_RE.fullmatch(prefix):
+ return [] # starts with a hole: would match any module
+ pattern = re.compile(_SEGMENT.join(re.escape(part) for part in _HOLE_RE.split(template)))
+ matched = [path for dotted, path in self.module_index.items() if pattern.fullmatch(dotted)]
+ # Ceiling: a one-segment prefix or a wide match reads as a plugin
+ # directory, not a short table, so only the package is linked. Upgrade
+ # path: resolve the hole's values when the source spells them out.
+ if "." not in prefix or len(matched) > _TEMPLATE_FANOUT_CAP:
+ package = self.module_index.get(prefix, "")
+ return [package] if package.endswith(_INIT_NAMES) else []
+ return matched
+
+
+def _string_refs(
+ text: bytes, module_index: dict[str, str], *, gated: bool, runtime: bool
+) -> list[_Ref]:
+ """Unverified ``(target, attr)`` references the quoted strings in *text* make."""
+ found: list[_Ref] = []
+ if gated:
+ for m in _DOTTED_STRING_RE.finditer(text):
+ module = m.group(1).decode("ascii")
+ if module in module_index:
+ found.append((module_index[module], None, m.start()))
+ continue
+ # A module path and the attribute it reads, not a type's qualifier.
+ parent, _, attr = module.rpartition(".")
+ if runtime and parent in module_index:
+ found.append((module_index[parent], attr, m.start()))
+ relative: list[tuple[str, str, int]] = []
+ for m in _ATTR_STRING_RE.finditer(text) if _ATTR_TAIL_RE.search(text) else ():
+ module, attr = m.group(1).decode("ascii"), m.group(2).decode("ascii")
+ target = module_index.get(module)
+ if target is not None:
+ found.append((target, attr, m.start()))
+ else:
+ relative.append((module, attr, m.start()))
+ if relative:
+ # Packages this file names: the roots a short reference joins.
+ bases = sorted(
+ {
+ dotted
+ for m in _DOTTED_STRING_RE.finditer(text)
+ if module_index.get(dotted := m.group(1).decode("ascii"), "").endswith(_INIT_NAMES)
+ }
+ )
+ for module, attr, start in relative:
+ found.extend(
+ (module_index[f"{b}.{module}"], attr, start)
+ for b in bases
+ if f"{b}.{module}" in module_index
+ )
+ return found
+
+
+def _maybe_not_code(blob: bytes, triples: list[int], start: int) -> bool:
+ """Whether the text at *start* may sit in a comment or a triple-quoted string.
+
+ Cheap and conservative: a ``#`` earlier on the line, or an odd count of
+ triple quotes before it (*triples* holds their offsets), says yes.
+ """
+ line_start = blob.rfind(b"\n", 0, start) + 1
+ return bool(bisect(triples, start) % 2) or b"#" in blob[line_start:start]
+
+
+def python_dynamic_refs(rel: str, blob: bytes, resolver: ModuleStringResolver) -> _Refs:
+ """Modules the source *blob* of *rel* names by string, as ``{target: names}``.
+
+ Pure over its inputs, so a caller holding source bytes needs no filesystem.
+ """
+ gated = any(marker in blob for marker in _DYNAMIC_IMPORT_MARKERS)
+ if not (gated or _ATTR_TAIL_RE.search(blob)):
+ return {}
+ # A loader outside tests; in a test a member path is a patch target.
+ runtime = not is_test_related_path(rel)
+ index = resolver.module_index
+ triples = [m.start() for m in _TRIPLE_QUOTE_RE.finditer(blob)]
+ refs: _Refs = {}
+ found = _string_refs(blob, index, gated=gated, runtime=runtime)
+ # Tokenize, which costs far more than the scan, only when a match may sit
+ # in a comment or docstring.
+ if any(_maybe_not_code(blob, triples, start) for _, _, start in found):
+ found = _string_refs(live_text(rel, blob), index, gated=gated, runtime=runtime)
+ for target, attr, _ in found:
+ if attr is None or resolver.has_attr(target, attr):
+ _add_ref(refs, target, attr)
+ if gated and runtime:
+ # Raw text, since f-strings are not plain string tokens on newer
+ # Pythons; a call in a comment or docstring is skipped by position.
+ for m in _TEMPLATE_LOAD_RE.finditer(blob):
+ if _maybe_not_code(blob, triples, m.start()):
+ continue
+ body = m.group(2) or m.group(4) + b"{}"
+ for target in resolver.template_targets(body.decode("ascii")):
+ _add_ref(refs, target, None)
+ refs.pop(rel, None)
+ return refs
class PythonDynamicHints(DynamicHintExtractor):
@@ -74,29 +255,23 @@ class PythonDynamicHints(DynamicHintExtractor):
if not module_index:
return []
+ resolver = ModuleStringResolver(
+ module_index, lambda target: self._source_bytes(repo_root / target, target)
+ )
edges: list[DynamicEdge] = []
for abs_path, rel in rel_by_abs.items():
- try:
- text = abs_path.read_text(encoding="utf-8", errors="ignore")
- except OSError:
- continue
- if not any(marker in text for marker in _DYNAMIC_IMPORT_MARKERS):
+ blob = self._source_bytes(abs_path, rel)
+ if not blob:
continue
-
- seen: set[str] = set()
- for match in _DOTTED_STRING_RE.finditer(text):
- dotted = match.group(1)
- if dotted in seen:
- continue
- seen.add(dotted)
- target = module_index.get(dotted)
- if target and target != rel:
- edges.append(
- DynamicEdge(
- source=rel,
- target=target,
- edge_type="dynamic_uses",
- hint_source=self.name,
- )
+ refs = python_dynamic_refs(rel, blob, resolver)
+ for target, names in sorted(refs.items()):
+ edges.append(
+ DynamicEdge(
+ source=rel,
+ target=target,
+ edge_type="dynamic_uses",
+ hint_source=self.name,
+ imported_names=names,
)
+ )
return edges
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
index 9fb8c3e77..d5d9a2151 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
@@ -84,6 +84,7 @@ class HintRegistry:
*,
dotnet_index: object | None = None,
file_paths: Iterable[str] | None = None,
+ source_map: dict[str, bytes] | None = None,
) -> list[DynamicEdge]:
"""Run every registered extractor and merge their edges.
@@ -106,6 +107,9 @@ class HintRegistry:
(``*.Designer.cs``) can no longer produce edges to files the index
does not contain. Omitted (tests, standalone use), the registry
falls back to its own pruned walk.
+
+ *source_map* is ingestion's ``{repo-relative path: raw bytes}``; an
+ extractor that reads whole files takes them from it instead of disk.
"""
edges: list[DynamicEdge] = []
if not self._extractors:
@@ -129,6 +133,7 @@ class HintRegistry:
for ex in self._extractors:
ex._walk_snapshot = snapshot
ex._dotnet_index = dotnet_index
+ ex._source_map = source_map
try:
with ThreadPoolExecutor(max_workers=self._max_workers) as pool:
@@ -148,6 +153,7 @@ class HintRegistry:
for ex in self._extractors:
ex._walk_snapshot = None
ex._dotnet_index = None
+ ex._source_map = None
# Thread-completion order varies run-to-run; sort so downstream graph
# construction (and therefore the exported KG) stays deterministic.
edges.sort(key=lambda e: (e.source, e.target, e.edge_type, e.hint_source, e.weight))
diff --git a/packages/core/src/repowise/core/ingestion/graph/_edges.py b/packages/core/src/repowise/core/ingestion/graph/_edges.py
index b702da9f5..4a39b664e 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_edges.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_edges.py
@@ -17,6 +17,7 @@ from ...co_change import (
canonical_pair,
parse_partners,
)
+from ..models import FILE_DEPENDENCY_EDGE_TYPES, is_dynamic_edge
from ..resolvers import ResolverContext
from ..resolvers.go import read_go_module_path, read_go_modules
from ._stem import build_stem_map
@@ -24,6 +25,29 @@ from ._stem import build_stem_map
log = structlog.get_logger(__name__)
+def _merge_dynamic_hint(existing: dict, hint) -> None:
+ """Fold a dynamic hint into the file edge already joining the same two files.
+
+ The graph holds one edge per pair, and hints run after static resolution,
+ so replacing the edge would drop what the import said (its type, the names
+ it binds). Names are unioned instead. A whole-module hint (no names) on a
+ static edge is kept as ``dynamic_hint`` so dead code still treats every
+ public member as reached. ``dynamic_hint`` lives on the live graph only:
+ persisted rows keep the static type and the unioned names (upgrade path: a
+ column, if a rehydrated graph ever runs the export rescue).
+ """
+ if is_dynamic_edge(existing.get("edge_type")):
+ # Two hints on one pair: a whole-module reach absorbs a named one.
+ if not hint.imported_names or not existing.get("imported_names"):
+ existing.pop("imported_names", None)
+ return
+ elif not hint.imported_names:
+ existing["dynamic_hint"] = hint.hint_source
+ return
+ names = existing.get("imported_names") or []
+ existing["imported_names"] = names + [n for n in hint.imported_names if n not in names]
+
+
class EdgesMixin:
"""Non-import edge construction for :class:`GraphBuilder`."""
@@ -150,6 +174,10 @@ class EdgesMixin:
# a real `dynamic_*` edge.
sub_type = e.edge_type
graph_edge_type = sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"
+ existing = self._graph.get_edge_data(e.source, e.target)
+ if existing is not None and existing.get("edge_type") in FILE_DEPENDENCY_EDGE_TYPES:
+ _merge_dynamic_hint(existing, e)
+ continue
names = {"imported_names": list(e.imported_names)} if e.imported_names else {}
self._graph.add_edge(
e.source,
diff --git a/packages/core/src/repowise/core/ingestion/languages/python_strings.py b/packages/core/src/repowise/core/ingestion/languages/python_strings.py
new file mode 100644
index 000000000..20feab71f
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/languages/python_strings.py
@@ -0,0 +1,87 @@
+"""Module-path strings in source text, and the part of a file that runs.
+
+Python names code by dotted string as often as by ``import`` (entry-point
+tables, plugin registries, lazy command tables). Both the dynamic-import hints
+(graph edges) and dead-code analysis (finding filters) read those strings, so
+the "only strings that run" filter, the "module defines this name" check and
+the dynamic-load markers live here once.
+"""
+
+from __future__ import annotations
+
+import io
+import re
+import tokenize
+from pathlib import PurePosixPath
+
+from .registry import REGISTRY
+
+#: Source tokens that mark a Python file as loading modules by name at run time.
+#: Substrings, so ``importlib`` also covers ``importlib.import_module``.
+PY_DYNAMIC_LOAD_MARKERS: tuple[str, ...] = (
+ "importlib",
+ "import_module",
+ "__import__",
+ "import_string", # Werkzeug / Flask / Django utilities
+ "load_entry_point",
+ "entry_points(", # importlib.metadata plugin discovery
+ "pkgutil",
+ "pkg_resources",
+ "lazy_subcommands", # lazy click group keyword: {"name": "pkg.mod.attr"}
+)
+
+
+def is_python(path: str) -> bool:
+ return REGISTRY.from_extension(PurePosixPath(path).suffix) == "python"
+
+
+def defines_top_level(blob: bytes, name: str) -> bool:
+ """Whether a Python module defines *name* at top level (def, class, assignment).
+
+ A re-export (``from x import Name``) or a tuple target is not seen, so such
+ a member stays reported.
+ """
+ word = re.escape(name.encode("ascii"))
+ pattern = (
+ rb"^(?:(?:async[ \t]+)?def|class)[ \t]+" + word + rb"\b"
+ rb"|^" + word + rb"[ \t]*(?::[^=\n]*)?=(?!=)"
+ )
+ return re.search(pattern, blob, re.MULTILINE) is not None
+
+
+#: Tokens after which a string starts an expression statement (a docstring).
+_STATEMENT_START = frozenset(
+ {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT, tokenize.ENCODING}
+)
+_TRIVIA = frozenset({tokenize.NL, tokenize.COMMENT})
+_COMMENT_LINE_RE = re.compile(rb"^[ \t]*#.*$", re.MULTILINE)
+
+
+def live_text(path: str, blob: bytes) -> bytes:
+ """*blob* without comments and, for Python, without docstrings.
+
+ Python keeps only its string literals that are not a statement on their
+ own; other files drop only ``#`` comment lines. Python that does not tokenize
+ is kept whole, as before this filter.
+ """
+ if not is_python(path):
+ return _COMMENT_LINE_RE.sub(b"", blob)
+ try:
+ tokens = [
+ t for t in tokenize.tokenize(io.BytesIO(blob).readline) if t.type not in _TRIVIA
+ ]
+ except (tokenize.TokenError, SyntaxError, ValueError):
+ return blob
+ return b"\n".join(
+ tok.string.encode("utf-8")
+ for i, tok in enumerate(tokens)
+ if tok.type == tokenize.STRING and not _is_statement(tokens, i)
+ )
+
+
+def _is_statement(tokens: list[tokenize.TokenInfo], i: int) -> bool:
+ """Whether ``tokens[i]`` is a statement on its own (a docstring)."""
+ # ENDMARKER always closes the stream, so ``i + 1`` exists.
+ if tokens[i - 1].type not in _STATEMENT_START:
+ return False
+ return tokens[i + 1].type in (tokenize.NEWLINE, tokenize.ENDMARKER)
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index 16d8481e6..b769da00d 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -234,6 +234,7 @@ def build_repo_graph(
Path(repo_path),
dotnet_index=graph_builder.dotnet_index,
file_paths=[fi.path for fi in file_infos],
+ source_map=source_map or None,
)
graph_builder.add_dynamic_edges(dynamic_edges)
if dynamic_edges:
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index f734dd587..850cef5c7 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -687,7 +687,10 @@ async def _run_ingestion(
dynamic_edges = await loop.run_in_executor(
None,
lambda: registry.extract_all(
- repo_path, dotnet_index=graph_builder.dotnet_index, file_paths=hint_paths
+ repo_path,
+ dotnet_index=graph_builder.dotnet_index,
+ file_paths=hint_paths,
+ source_map=source_map,
),
)
graph_builder.add_dynamic_edges(dynamic_edges)
diff --git a/tests/unit/ingestion/test_python_dead_code_fixes.py b/tests/unit/ingestion/test_python_dead_code_fixes.py
index 12a13575d..9bc27abf5 100644
--- a/tests/unit/ingestion/test_python_dead_code_fixes.py
+++ b/tests/unit/ingestion/test_python_dead_code_fixes.py
@@ -414,3 +414,35 @@ def test_dynamic_use_edge_marks_target_file_live() -> None:
g.add_edge("pkg/registry.py", "pkg/ollama.py", edge_type="dynamic_uses")
assert "OllamaProvider" not in _unused_export_names(g)
+
+
+def test_whole_module_hint_folded_into_an_import_marks_target_file_live() -> None:
+ """A whole-module hint on a pair that also has a static import keeps the
+ import edge and its names, and still keeps every public export live."""
+ g = nx.DiGraph()
+ for path in ("pkg/registry.py", "pkg/ollama.py"):
+ g.add_node(path, node_type="file", language="python", is_test=False, is_entry_point=False)
+ for name in ("OllamaProvider", "app"):
+ sym = f"pkg/ollama.py::{name}"
+ g.add_node(
+ sym,
+ node_type="symbol",
+ kind="class",
+ name=name,
+ visibility="public",
+ decorators=[],
+ start_line=1,
+ end_line=20,
+ file_path="pkg/ollama.py",
+ language="python",
+ )
+ g.add_edge("pkg/ollama.py", sym, edge_type="defines")
+ g.add_edge(
+ "pkg/registry.py",
+ "pkg/ollama.py",
+ edge_type="imports",
+ imported_names=["OllamaProvider"],
+ dynamic_hint="python_dynamic_import",
+ )
+
+ assert not {"OllamaProvider", "app"} & _unused_export_names(g)
diff --git a/tests/unit/ingestion/test_python_string_registry_hints.py b/tests/unit/ingestion/test_python_string_registry_hints.py
new file mode 100644
index 000000000..d2388668f
--- /dev/null
+++ b/tests/unit/ingestion/test_python_string_registry_hints.py
@@ -0,0 +1,257 @@
+"""Modules named by string in lazy registries become ``dynamic_uses`` edges.
+
+Lazy command tables, entry-point specs and templated ``import_module`` calls
+load a module by name, so no import edge exists. Each test pins one shape and
+one false-edge guard: a string must resolve to a module in the repo, and an
+attribute reference must name something that module has.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.core.ingestion.dynamic_hints.base import DynamicEdge
+from repowise.core.ingestion.dynamic_hints.python_imports import (
+ ModuleStringResolver,
+ PythonDynamicHints,
+ python_dynamic_refs,
+)
+from repowise.core.ingestion.graph import GraphBuilder
+from repowise.core.ingestion.languages.python_modules import build_python_module_index
+
+
+def _refs(files: dict[str, str], rel: str) -> dict[str, tuple[str, ...]]:
+ blobs = {path: src.encode() for path, src in files.items()}
+ resolver = ModuleStringResolver(
+ build_python_module_index(blobs), lambda path: blobs.get(path, b"")
+ )
+ return python_dynamic_refs(rel, blobs[rel], resolver)
+
+
+_CLI = {
+ "src/app/__init__.py": "",
+ "src/app/commands/__init__.py": "",
+ "src/app/commands/status_cmd.py": "def status_command():\n pass\n",
+ "src/app/commands/init_cmd/__init__.py": "from .command import init_command\n",
+ "src/app/commands/init_cmd/command.py": "def init_command():\n pass\n",
+}
+
+
+def test_relative_lazy_command_table_resolves_under_the_named_package() -> None:
+ files = {
+ **_CLI,
+ "src/app/main.py": (
+ '_PKG = "app.commands"\n'
+ "_COMMANDS = (\n"
+ ' ("status", "status_cmd:status_command"),\n'
+ ' ("init", "init_cmd:init_command"),\n'
+ ' ("gone", "missing_cmd:gone_command"),\n'
+ ")\n"
+ "for name, target in _COMMANDS:\n"
+ ' register(name, f"{_PKG}.{target}")\n'
+ ),
+ }
+ assert _refs(files, "src/app/main.py") == {
+ "src/app/commands/status_cmd.py": ("status_command",),
+ # A package re-exporting the command counts as having it.
+ "src/app/commands/init_cmd/__init__.py": ("init_command",),
+ }
+
+
+def test_absolute_entry_point_string_needs_no_loader_marker() -> None:
+ files = {**_CLI, "src/app/serve.py": 'run("app.commands.status_cmd:status_command")\n'}
+ assert _refs(files, "src/app/serve.py") == {
+ "src/app/commands/status_cmd.py": ("status_command",)
+ }
+
+
+def test_colon_string_naming_an_attribute_the_module_lacks_adds_nothing() -> None:
+ # An event key that happens to share its prefix with a package name.
+ files = {
+ "agent/__init__.py": "def run():\n pass\n",
+ "gateway.py": 'emit("agent:start")\nemit("agent:run")\n',
+ }
+ assert _refs(files, "gateway.py") == {"agent/__init__.py": ("run",)}
+
+
+def test_module_with_lazy_getattr_serves_any_attribute() -> None:
+ files = {
+ "agent/__init__.py": "def __getattr__(name):\n return name\n",
+ "gateway.py": 'load("agent:start")\n',
+ }
+ assert _refs(files, "gateway.py") == {"agent/__init__.py": ("start",)}
+
+
+def test_string_naming_no_repo_module_adds_nothing() -> None:
+ files = {**_CLI, "src/app/serve.py": 'run("uvicorn.main:run")\nopen("w:gz")\n'}
+ assert _refs(files, "src/app/serve.py") == {}
+
+
+def test_strings_in_comments_and_docstrings_add_nothing() -> None:
+ files = {
+ **_CLI,
+ "src/app/doc.py": (
+ '"""Run ``"app.commands.status_cmd:status_command"`` to see it."""\n'
+ "import importlib\n"
+ '# load("app.commands.status_cmd:status_command")\n'
+ "def f():\n"
+ ' """See "app.commands.status_cmd" for details."""\n'
+ ),
+ }
+ assert _refs(files, "src/app/doc.py") == {}
+
+
+def test_dotted_member_path_in_a_loader_file_names_the_member() -> None:
+ files = {
+ **_CLI,
+ "src/app/loader.py": (
+ "from importlib import import_module\n"
+ 'BACKENDS = ["app.commands.status_cmd.status_command", "app.commands.init_cmd"]\n'
+ ),
+ }
+ assert _refs(files, "src/app/loader.py") == {
+ "src/app/commands/status_cmd.py": ("status_command",),
+ "src/app/commands/init_cmd/__init__.py": (),
+ }
+
+
+def test_dotted_member_path_in_a_test_is_a_patch_target_not_a_load() -> None:
+ files = {
+ **_CLI,
+ "tests/test_status.py": (
+ "import importlib\n"
+ 'patch("app.commands.status_cmd.status_command")\n'
+ ),
+ }
+ assert _refs(files, "tests/test_status.py") == {}
+
+
+def test_lazy_group_keyword_gates_dotted_module_attr_strings() -> None:
+ files = {
+ **_CLI,
+ "src/app/cli.py": (
+ "@click.group(cls=LazyGroup, lazy_subcommands="
+ '{"status": "app.commands.status_cmd.status_command"})\n'
+ "def cli():\n pass\n"
+ ),
+ }
+ assert _refs(files, "src/app/cli.py") == {
+ "src/app/commands/status_cmd.py": ("status_command",)
+ }
+
+
+def test_templated_import_reaches_every_matching_module() -> None:
+ files = {
+ "pkg/__init__.py": "",
+ "pkg/plugins/__init__.py": "",
+ "pkg/plugins/a.py": "",
+ "pkg/plugins/b/__init__.py": "",
+ "pkg/plugins/b/oauth.py": "",
+ "pkg/other.py": "",
+ "pkg/loader.py": (
+ "import importlib\n"
+ "def load(name):\n"
+ ' importlib.import_module(f"pkg.plugins.{name}")\n'
+ ' importlib.import_module(f"pkg.plugins.{name}.oauth")\n'
+ ' importlib.import_module("pkg.plugins." + name)\n'
+ ' importlib.import_module(f"{name}.{name}")\n'
+ ),
+ }
+ assert _refs(files, "pkg/loader.py") == {
+ "pkg/plugins/a.py": (),
+ "pkg/plugins/b/__init__.py": (),
+ "pkg/plugins/b/oauth.py": (),
+ }
+
+
+def test_extractor_reads_attached_source_map_before_disk(tmp_path: Path) -> None:
+ for path, src in _CLI.items():
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text(src)
+ main = tmp_path / "src/app/main.py"
+ main.write_text("") # on disk: nothing; in the source map: the table
+ hints = PythonDynamicHints()
+ hints._source_map = {"src/app/main.py": b'P = "app.commands"\nT = "status_cmd:status_command"\n'}
+ edges = hints.extract(tmp_path)
+ assert [(e.source, e.target, e.edge_type, e.imported_names) for e in edges] == [
+ (
+ "src/app/main.py",
+ "src/app/commands/status_cmd.py",
+ "dynamic_uses",
+ ("status_command",),
+ )
+ ]
+
+
+def test_templated_import_in_a_comment_or_docstring_adds_nothing() -> None:
+ files = {
+ "pkg/__init__.py": "",
+ "pkg/plugins/__init__.py": "",
+ "pkg/plugins/a.py": "",
+ "pkg/loader.py": (
+ '"""Loads plugins with import_module(f"pkg.plugins.{name}")."""\n'
+ "import importlib\n"
+ "def load(name):\n"
+ ' # importlib.import_module(f"pkg.plugins.{name}")\n'
+ " return name\n"
+ ),
+ }
+ assert _refs(files, "pkg/loader.py") == {}
+
+
+def test_wide_or_one_segment_template_links_the_package_only() -> None:
+ plugins = {f"pkg/plugins/p{i}.py": "" for i in range(51)}
+ files = {
+ "pkg/__init__.py": "",
+ "pkg/plugins/__init__.py": "",
+ **plugins,
+ "pkg/loader.py": (
+ "import importlib\n"
+ "def load(name):\n"
+ ' importlib.import_module(f"pkg.plugins.{name}")\n'
+ ' importlib.import_module(f"pkg.{name}")\n'
+ ),
+ }
+ assert _refs(files, "pkg/loader.py") == {
+ "pkg/plugins/__init__.py": (),
+ "pkg/__init__.py": (),
+ }
+
+
+def _hint(source: str, target: str, names: tuple[str, ...] = ()) -> DynamicEdge:
+ return DynamicEdge(source, target, "dynamic_uses", "python_dynamic_import", imported_names=names)
+
+
+def _builder_with_import(names: list[str]) -> GraphBuilder:
+ builder = GraphBuilder()
+ graph = builder._graph
+ graph.add_node("a.py")
+ graph.add_node("b.py")
+ graph.add_edge("a.py", "b.py", edge_type="imports", imported_names=names)
+ return builder
+
+
+def test_hint_on_an_imported_pair_keeps_the_import_and_unions_names() -> None:
+ builder = _builder_with_import(["Foo"])
+ builder.add_dynamic_edges([_hint("a.py", "b.py", ("app",))])
+ data = builder._graph.get_edge_data("a.py", "b.py")
+ assert data["edge_type"] == "imports"
+ assert data["imported_names"] == ["Foo", "app"]
+ assert "dynamic_hint" not in data
+
+
+def test_whole_module_hint_on_an_imported_pair_is_kept_as_a_flag() -> None:
+ builder = _builder_with_import(["Foo"])
+ builder.add_dynamic_edges([_hint("a.py", "b.py")])
+ data = builder._graph.get_edge_data("a.py", "b.py")
+ assert (data["edge_type"], data["imported_names"]) == ("imports", ["Foo"])
+ assert data["dynamic_hint"] == "python_dynamic_import"
+
+
+def test_second_hint_on_a_pair_merges_and_whole_module_wins() -> None:
+ builder = GraphBuilder()
+ builder._graph.add_node("a.py")
+ builder.add_dynamic_edges([_hint("a.py", "b.py", ("x",)), _hint("a.py", "b.py", ("y",))])
+ assert builder._graph.get_edge_data("a.py", "b.py")["imported_names"] == ["x", "y"]
+ builder.add_dynamic_edges([_hint("a.py", "b.py")])
+ assert "imported_names" not in builder._graph.get_edge_data("a.py", "b.py")
diff --git a/tests/unit/test_no_bare_type_name_copies.py b/tests/unit/test_no_bare_type_name_copies.py
index 738d7c3d9..2241cd03c 100644
--- a/tests/unit/test_no_bare_type_name_copies.py
+++ b/tests/unit/test_no_bare_type_name_copies.py
@@ -95,6 +95,9 @@ _KNOWN: dict[str, int] = {
_PREFIX + "dynamic_hints/swift.py": 1,
# Splits an import statement's package path, not a type reference.
_PREFIX + "languages/jvm_same_package.py": 1,
+ # Splits a module-path string (`"pkg.mod.Name"`) into the module it loads
+ # and the attribute it reads; the head is a module path, not a qualifier.
+ _PREFIX + "dynamic_hints/python_imports.py": 1,
# Qualifies a PHP name against its `use` alias (the HEAD segment) and
# splits an FQN into the namespace it indexes by and the class: it asks
# which namespace, the opposite end from the shared helper.