Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(ingestion): link modules named by string in lazy registries (#3220)
Raghav Chamadiya0c0a0f613b7c
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.pyindex c4ac684ff..9f66d0609 100644--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py@@ -1463,10 +1463,11 @@ class DeadCodeAnalyzer:# member of the file. Only these two: ``dynamic_imports`` means the# module is loaded, which a plain import edge does not rescue either.# A framework edge that names its exports (a Lambda handler) rescues- # only those, through the importer check below.+ # only those, through the importer check below. ``dynamic_hint`` is a+ # whole-module dynamic use folded into a static edge on the same pair.if any(- edge.get("edge_type") in ("dynamic_uses", "framework")- and not edge.get("imported_names")+ (edge.get("edge_type") in ("dynamic_uses", "framework") and not edge.get("imported_names"))+ or edge.get("dynamic_hint")for edge in (self.graph.get_edge_data(pred, node, {}) for pred in self.graph.predecessors(node))diff --git a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.pyindex d26c27d01..bebd7c81e 100644--- a/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py+++ b/packages/core/src/repowise/core/analysis/dead_code/dynamic_markers.py@@ -19,6 +19,7 @@ from __future__ import annotationsfrom pathlib import Pathfrom repowise.core.ids import is_external+from repowise.core.ingestion.languages.python_strings import PY_DYNAMIC_LOAD_MARKERSfrom repowise.core.ingestion.models import is_dynamic_edge@@ -52,14 +53,7 @@ def read_source_text(# Patterns in source that indicate dynamic/runtime imports, keyed by suffix._DYNAMIC_IMPORT_MARKERS: dict[str, tuple[str, ...]] = {- ".py": (- "importlib.import_module",- # ``from importlib import import_module``, then a bare call.- "import_module(",- "__import__(",- "importlib.reload",- "pkgutil.iter_modules",- ),+ ".py": PY_DYNAMIC_LOAD_MARKERS,# JS/TS dynamic-dispatch markers. Each implies the surrounding file# (and, by the package-co-location heuristic, its siblings) reaches# code through a runtime mechanism the static import graph cannot@@ -517,7 +511,8 @@ def find_dynamic_edge_files(graph) -> set[str]:result: set[str] = set()try:for u, v, data in graph.edges(data=True):- if not is_dynamic_edge(data.get("edge_type", "")):+ # ``dynamic_hint``: a dynamic hint folded into a static edge.+ if not (is_dynamic_edge(data.get("edge_type", "")) or data.get("dynamic_hint")):continuefor endpoint in (u, v):if endpoint is None:diff --git a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py b/packages/core/src/repowise/core/analysis/dead_code/module_strings.pyindex f2f12e3ac..e1d387273 100644--- a/packages/core/src/repowise/core/analysis/dead_code/module_strings.py+++ b/packages/core/src/repowise/core/analysis/dead_code/module_strings.py@@ -29,14 +29,12 @@ fixture app it loads by name), never a member path (a ``mock.patch`` target).from __future__ import annotations-import ioimport re-import tokenizefrom collections.abc import Iterable, Iteratorfrom dataclasses import dataclassfrom pathlib import PurePosixPath-from ...ingestion.languages.registry import REGISTRY+from ...ingestion.languages.python_strings import defines_top_level, is_python, live_textfrom ...test_paths import is_test_related_pathfrom .models import DeadCodeFindingData, DeadCodeKind@@ -80,10 +78,6 @@ def _directory(path: str) -> str:return "" if parent == "." else parent-def _is_python(path: str) -> bool:- return REGISTRY.from_extension(PurePosixPath(path).suffix) == "python"--def drop_named_modules(findings: list[DeadCodeFindingData],source_map: dict[str, bytes],@@ -104,7 +98,7 @@ def drop_named_modules(def _is_candidate(finding: DeadCodeFindingData) -> bool:kinds = (DeadCodeKind.UNREACHABLE_FILE, DeadCodeKind.UNUSED_EXPORT)- return finding.kind in kinds and _is_python(finding.file_path)+ return finding.kind in kinds and is_python(finding.file_path)def _is_named(@@ -132,7 +126,7 @@ def _named_by_strings(# Tokenize only a file whose raw text names a candidate at all.if next(reader.uses(blob), None) is None:continue- for target, attr in reader.uses(_live_text(path, blob)):+ for target, attr in reader.uses(live_text(path, blob)):named.add(target)if attr:attrs.add((target, attr))@@ -169,56 +163,4 @@ class _Reader:"""For ``"pkg.module.Name"``: the modules ``pkg.module`` names that define ``Name``."""parent, _, name = module.rpartition(".")owners = self._resolve(parent) if parent else set()- return {t for t in owners if _defines_top_level(members.get(t, b""), name)}, name---def _defines_top_level(blob: bytes, name: str) -> bool:- """Whether a Python module defines *name* at top level (def, class, assignment).-- A re-export (``from x import Name``) or a tuple target is not seen, so such- a member stays reported.- """- word = re.escape(name.encode("ascii"))- pattern = (- rb"^(?:(?:async[ \t]+)?def|class)[ \t]+" + word + rb"\b"- rb"|^" + word + rb"[ \t]*(?::[^=\n]*)?=(?!=)"- )- return re.search(pattern, blob, re.MULTILINE) is not None---#: Tokens after which a string starts an expression statement (a docstring).-_STATEMENT_START = frozenset(- {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT, tokenize.ENCODING}-)-_TRIVIA = frozenset({tokenize.NL, tokenize.COMMENT})-_COMMENT_LINE_RE = re.compile(rb"^[ \t]*#.*$", re.MULTILINE)---def _live_text(path: str, blob: bytes) -> bytes:- """*blob* without comments and, for Python, without docstrings.-- Python keeps only its string literals that are not a statement on their- own; other files drop only ``#`` comment lines. Python that does not tokenize- is kept whole, as before this filter.- """- if not _is_python(path):- return _COMMENT_LINE_RE.sub(b"", blob)- try:- tokens = [- t for t in tokenize.tokenize(io.BytesIO(blob).readline) if t.type not in _TRIVIA- ]- except (tokenize.TokenError, SyntaxError, ValueError):- return blob- return b"\n".join(- tok.string.encode("utf-8")- for i, tok in enumerate(tokens)- if tok.type == tokenize.STRING and not _is_statement(tokens, i)- )---def _is_statement(tokens: list[tokenize.TokenInfo], i: int) -> bool:- """Whether ``tokens[i]`` is a statement on its own (a docstring)."""- # ENDMARKER always closes the stream, so ``i + 1`` exists.- if tokens[i - 1].type not in _STATEMENT_START:- return False- return tokens[i + 1].type in (tokenize.NEWLINE, tokenize.ENDMARKER)+ return {t for t in owners if defines_top_level(members.get(t, b""), name)}, namediff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.pyindex df3535d05..a0d01ebea 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py@@ -38,9 +38,25 @@ class DynamicHintExtractor(ABC):# None -> extractors that need it build their own, as before._dotnet_index = None+ # Ingestion's ``{repo-relative path: raw bytes}`` for the parsed files,+ # attached by HintRegistry.extract_all when the caller holds one, so an+ # extractor reading every file of a language reuses the parse pass's+ # bytes. None (or a missing path) -> read from disk.+ _source_map = None+@abstractmethoddef extract(self, repo_root: Path) -> list[DynamicEdge]: ...+ def _source_bytes(self, abs_path: Path, rel: str) -> bytes:+ """*rel*'s bytes from the attached source map, else from disk; ``b""`` if neither."""+ # Same contract as ``ingestion.source_text.source_bytes``; collapse into it once both exist.+ if self._source_map is not None and (data := self._source_map.get(rel)) is not None:+ return data+ try:+ return abs_path.read_bytes()+ except OSError:+ return b""+def _rglob(self, root: Path, pattern: str) -> Iterator[Path]:"""Pruned replacement for :py:meth:`pathlib.Path.rglob`.diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.pyindex 38f542215..de43f334c 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/python_imports.py@@ -12,47 +12,228 @@ module is named by a *string* and loaded with ``importlib.import_module``cls = getattr(module, _PROVIDERS[name][1])Nothing statically imports ``my_pkg.providers.ollama`` or ``OllamaProvider``,-so dead-code analysis would flag both the module and the class as unused.--This extractor recovers those edges generically: in any file that uses-dynamic-import machinery, every quoted string that resolves to a real-in-repo module (via :func:`python_modules.build_python_module_index`) yields-a ``dynamic_uses`` edge to that module's file. The analyzer then treats the-target file as runtime-loaded and stops flagging its public members.--Gating on the presence of dynamic-import machinery (rather than emitting an-edge for any dotted string that happens to match a module) keeps the signal-precise: a dotted string in a log message or docstring of an ordinary module-is ignored. The mechanism is repo-agnostic — it works for any Python plugin-registry, ``importlib`` loader, or entry-point dispatch table.+so dead-code analysis would flag both the module and the class as unused,+and test selection would never connect the registry to the module.++This extractor recovers those edges generically. Every string it considers+must resolve to a real in-repo module (via+:func:`python_modules.build_python_module_index`); a string that names+nothing in the repo adds nothing, and only strings that run count (a comment+or docstring loads nothing). Three shapes are read:++* **Dotted module paths** (``"pkg.sub.mod"``), only in files that use+ dynamic-import machinery (:data:`_DYNAMIC_IMPORT_MARKERS`). Gating keeps a+ dotted string in a log message of an ordinary module inert.+* **Attribute references**: ``"pkg.mod:attr"`` in any file (the spelling of+ console-script entry points, ASGI/WSGI app specs and lazy command+ registries), and ``"pkg.mod.Name"`` in gated files outside tests (the+ ``import_string`` / lazy command group spelling; in a test it is a patch+ target). A relative ``"status_cmd:status_command"`` resolves under any+ package the same file names by a dotted string, which is how lazy command+ tables join a base package to short names. The module must define or+ import the attribute (or define ``__getattr__``): ``"agent:start"`` as an+ event name, next to a package called ``agent`` without a ``start``, adds+ nothing.+* **Templated loads** (``import_module(f"pkg.plugins.{name}")`` or+ ``import_module("pkg.plugins." + name)``) in gated non-test code: every+ module matching the template, where each hole is one name segment. A+ template that starts with a hole adds nothing; one with a single-segment+ prefix or more than :data:`_TEMPLATE_FANOUT_CAP` matches links the package+ only. A call in a comment or docstring is skipped.++The mechanism is repo-agnostic: plugin registries, ``importlib`` loaders,+entry-point dispatch tables and lazy CLI groups all reduce to these shapes."""from __future__ import annotationsimport re+from bisect import bisect+from collections.abc import Callablefrom pathlib import Path+from ...test_paths import is_test_related_pathfrom ..languages.python_modules import build_python_module_index+from ..languages.python_strings import PY_DYNAMIC_LOAD_MARKERS, defines_top_level, live_textfrom .base import DynamicEdge, DynamicHintExtractor-# Tokens that signal a file performs runtime module loading. An edge is only-# emitted from files that contain at least one of these, so plain dotted-# strings elsewhere never create spurious reachability.-_DYNAMIC_IMPORT_MARKERS: tuple[str, ...] = (- "importlib",- "import_module",- "__import__",- "import_string", # Werkzeug / Flask / Django utilities- "load_entry_point",- "entry_points(", # importlib.metadata plugin discovery- "pkgutil",- "pkg_resources",+# Dotted strings only produce edges from files that load modules at run time,+# so plain dotted strings elsewhere never create spurious reachability.+_DYNAMIC_IMPORT_MARKERS = tuple(m.encode("ascii") for m in PY_DYNAMIC_LOAD_MARKERS)++# A quoted dotted path of two or more segments (``"pkg.sub.mod"``).+_DOTTED_STRING_RE = re.compile(rb"""['"]([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)+)['"]""")+# A quoted ``module:attr``: all an ungated file is scanned for.+_ATTR_STRING_RE = re.compile(rb"""['"]([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*):([A-Za-z_]\w*)['"]""")+# Its colon-first tail: a literal-led scan about ten times faster, so files+# without one skip the full pattern.+_ATTR_TAIL_RE = re.compile(rb""":[A-Za-z_]\w*['"]""")++# ``import_module(f"...")`` / ``__import__(f"...")`` and the ``"prefix." + x``+# concatenation. Group 2 is the f-string body, group 4 the literal prefix.+_TEMPLATE_LOAD_RE = re.compile(+ rb"""(?:import_module|__import__)\(\s*"""+ rb"""(?:f(['"])([\w.{}]+)\1|(['"])((?:[a-zA-Z_]\w*\.)+)\3\s*\+)""")+_HOLE_RE = re.compile(r"\{[^{}]*\}")+_SEGMENT = r"[a-zA-Z_]\w*"+_DOTTED_PREFIX_RE = re.compile(rf"{_SEGMENT}(?:\.{_SEGMENT})*")++# Most modules one templated load may link before it links the package only.+_TEMPLATE_FANOUT_CAP = 50++# Top-level import statements, parenthesised continuation included.+_TOP_IMPORT_RE = re.compile(rb"^(?:from[ \t]+[\w.]+[ \t]+)?import[ \t]+(\([^)]*\)|[^\n]*)", re.M)++_TRIPLE_QUOTE_RE = re.compile(rb"\"\"\"|'''")++_INIT_NAMES = ("__init__.py", "__init__.pyi")++# Target file -> names reached there; an empty tuple means the whole module.+_Refs = dict[str, tuple[str, ...]]+# (target, attribute or None for the whole module, offset of the string)+_Ref = tuple[str, str | None, int]+++def _add_ref(refs: _Refs, target: str, name: str | None) -> None:+ """Record *target*; a whole-module reference absorbs any named one."""+ if name is None or refs.get(target) == ():+ refs[target] = ()+ return+ refs[target] = tuple(sorted({*refs.get(target, ()), name}))+++class ModuleStringResolver:+ """Resolves module-path strings against one repo's modules.++ *source_of* returns a module file's bytes, read only to check that an+ attribute reference names something the module has. Attribute checks and+ template expansions are memoised, so one instance serves a whole repo.+ """++ def __init__(self, module_index: dict[str, str], source_of: Callable[[str], bytes]) -> None:+ self.module_index = module_index+ self._source_of = source_of+ self._attrs: dict[tuple[str, str], bool] = {}+ self._templates: dict[str, list[str]] = {}-# A quoted dotted path of two or more identifier segments — i.e. something-# shaped like ``a.b`` / ``pkg.sub.mod``. Single-segment names are too-# ambiguous (and resolve via ordinary stem matching anyway).-_DOTTED_STRING_RE = re.compile(r"""['"]([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)+)['"]""")+ def has_attr(self, target: str, name: str) -> bool:+ """Whether a module defines, imports or lazily serves *name* at top level."""+ key = (target, name)+ if key not in self._attrs:+ blob = self._source_of(target)+ word = re.compile(rb"\b" + re.escape(name.encode("ascii")) + rb"\b")+ self._attrs[key] = (+ defines_top_level(blob, name)+ or defines_top_level(blob, "__getattr__")+ or any(word.search(m.group(1)) for m in _TOP_IMPORT_RE.finditer(blob))+ )+ return self._attrs[key]++ def template_targets(self, template: str) -> list[str]:+ """Files of every module matching a load template; each hole is one segment."""+ if template not in self._templates:+ self._templates[template] = self._expand(template)+ return self._templates[template]++ def _expand(self, template: str) -> list[str]:+ prefix = _HOLE_RE.split(template, maxsplit=1)[0].rstrip(".")+ if not _DOTTED_PREFIX_RE.fullmatch(prefix):+ return [] # starts with a hole: would match any module+ pattern = re.compile(_SEGMENT.join(re.escape(part) for part in _HOLE_RE.split(template)))+ matched = [path for dotted, path in self.module_index.items() if pattern.fullmatch(dotted)]+ # Ceiling: a one-segment prefix or a wide match reads as a plugin+ # directory, not a short table, so only the package is linked. Upgrade+ # path: resolve the hole's values when the source spells them out.+ if "." not in prefix or len(matched) > _TEMPLATE_FANOUT_CAP:+ package = self.module_index.get(prefix, "")+ return [package] if package.endswith(_INIT_NAMES) else []+ return matched+++def _string_refs(+ text: bytes, module_index: dict[str, str], *, gated: bool, runtime: bool+) -> list[_Ref]:+ """Unverified ``(target, attr)`` references the quoted strings in *text* make."""+ found: list[_Ref] = []+ if gated:+ for m in _DOTTED_STRING_RE.finditer(text):+ module = m.group(1).decode("ascii")+ if module in module_index:+ found.append((module_index[module], None, m.start()))+ continue+ # A module path and the attribute it reads, not a type's qualifier.+ parent, _, attr = module.rpartition(".")+ if runtime and parent in module_index:+ found.append((module_index[parent], attr, m.start()))+ relative: list[tuple[str, str, int]] = []+ for m in _ATTR_STRING_RE.finditer(text) if _ATTR_TAIL_RE.search(text) else ():+ module, attr = m.group(1).decode("ascii"), m.group(2).decode("ascii")+ target = module_index.get(module)+ if target is not None:+ found.append((target, attr, m.start()))+ else:+ relative.append((module, attr, m.start()))+ if relative:+ # Packages this file names: the roots a short reference joins.+ bases = sorted(+ {+ dotted+ for m in _DOTTED_STRING_RE.finditer(text)+ if module_index.get(dotted := m.group(1).decode("ascii"), "").endswith(_INIT_NAMES)+ }+ )+ for module, attr, start in relative:+ found.extend(+ (module_index[f"{b}.{module}"], attr, start)+ for b in bases+ if f"{b}.{module}" in module_index+ )+ return found+++def _maybe_not_code(blob: bytes, triples: list[int], start: int) -> bool:+ """Whether the text at *start* may sit in a comment or a triple-quoted string.++ Cheap and conservative: a ``#`` earlier on the line, or an odd count of+ triple quotes before it (*triples* holds their offsets), says yes.+ """+ line_start = blob.rfind(b"\n", 0, start) + 1+ return bool(bisect(triples, start) % 2) or b"#" in blob[line_start:start]+++def python_dynamic_refs(rel: str, blob: bytes, resolver: ModuleStringResolver) -> _Refs:+ """Modules the source *blob* of *rel* names by string, as ``{target: names}``.++ Pure over its inputs, so a caller holding source bytes needs no filesystem.+ """+ gated = any(marker in blob for marker in _DYNAMIC_IMPORT_MARKERS)+ if not (gated or _ATTR_TAIL_RE.search(blob)):+ return {}+ # A loader outside tests; in a test a member path is a patch target.+ runtime = not is_test_related_path(rel)+ index = resolver.module_index+ triples = [m.start() for m in _TRIPLE_QUOTE_RE.finditer(blob)]+ refs: _Refs = {}+ found = _string_refs(blob, index, gated=gated, runtime=runtime)+ # Tokenize, which costs far more than the scan, only when a match may sit+ # in a comment or docstring.+ if any(_maybe_not_code(blob, triples, start) for _, _, start in found):+ found = _string_refs(live_text(rel, blob), index, gated=gated, runtime=runtime)+ for target, attr, _ in found:+ if attr is None or resolver.has_attr(target, attr):+ _add_ref(refs, target, attr)+ if gated and runtime:+ # Raw text, since f-strings are not plain string tokens on newer+ # Pythons; a call in a comment or docstring is skipped by position.+ for m in _TEMPLATE_LOAD_RE.finditer(blob):+ if _maybe_not_code(blob, triples, m.start()):+ continue+ body = m.group(2) or m.group(4) + b"{}"+ for target in resolver.template_targets(body.decode("ascii")):+ _add_ref(refs, target, None)+ refs.pop(rel, None)+ return refsclass PythonDynamicHints(DynamicHintExtractor):@@ -74,29 +255,23 @@ class PythonDynamicHints(DynamicHintExtractor):if not module_index:return []+ resolver = ModuleStringResolver(+ module_index, lambda target: self._source_bytes(repo_root / target, target)+ )edges: list[DynamicEdge] = []for abs_path, rel in rel_by_abs.items():- try:- text = abs_path.read_text(encoding="utf-8", errors="ignore")- except OSError:- continue- if not any(marker in text for marker in _DYNAMIC_IMPORT_MARKERS):+ blob = self._source_bytes(abs_path, rel)+ if not blob:continue-- seen: set[str] = set()- for match in _DOTTED_STRING_RE.finditer(text):- dotted = match.group(1)- if dotted in seen:- continue- seen.add(dotted)- target = module_index.get(dotted)- if target and target != rel:- edges.append(- DynamicEdge(- source=rel,- target=target,- edge_type="dynamic_uses",- hint_source=self.name,- )+ refs = python_dynamic_refs(rel, blob, resolver)+ for target, names in sorted(refs.items()):+ edges.append(+ DynamicEdge(+ source=rel,+ target=target,+ edge_type="dynamic_uses",+ hint_source=self.name,+ imported_names=names,)+ )return edgesdiff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.pyindex 9fb8c3e77..d5d9a2151 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py@@ -84,6 +84,7 @@ class HintRegistry:*,dotnet_index: object | None = None,file_paths: Iterable[str] | None = None,+ source_map: dict[str, bytes] | None = None,) -> list[DynamicEdge]:"""Run every registered extractor and merge their edges.@@ -106,6 +107,9 @@ class HintRegistry:(``*.Designer.cs``) can no longer produce edges to files the indexdoes not contain. Omitted (tests, standalone use), the registryfalls back to its own pruned walk.++ *source_map* is ingestion's ``{repo-relative path: raw bytes}``; an+ extractor that reads whole files takes them from it instead of disk."""edges: list[DynamicEdge] = []if not self._extractors:@@ -129,6 +133,7 @@ class HintRegistry:for ex in self._extractors:ex._walk_snapshot = snapshotex._dotnet_index = dotnet_index+ ex._source_map = source_maptry:with ThreadPoolExecutor(max_workers=self._max_workers) as pool:@@ -148,6 +153,7 @@ class HintRegistry:for ex in self._extractors:ex._walk_snapshot = Noneex._dotnet_index = None+ ex._source_map = None# Thread-completion order varies run-to-run; sort so downstream graph# construction (and therefore the exported KG) stays deterministic.edges.sort(key=lambda e: (e.source, e.target, e.edge_type, e.hint_source, e.weight))diff --git a/packages/core/src/repowise/core/ingestion/graph/_edges.py b/packages/core/src/repowise/core/ingestion/graph/_edges.pyindex b702da9f5..4a39b664e 100644--- a/packages/core/src/repowise/core/ingestion/graph/_edges.py+++ b/packages/core/src/repowise/core/ingestion/graph/_edges.py@@ -17,6 +17,7 @@ from ...co_change import (canonical_pair,parse_partners,)+from ..models import FILE_DEPENDENCY_EDGE_TYPES, is_dynamic_edgefrom ..resolvers import ResolverContextfrom ..resolvers.go import read_go_module_path, read_go_modulesfrom ._stem import build_stem_map@@ -24,6 +25,29 @@ from ._stem import build_stem_maplog = structlog.get_logger(__name__)+def _merge_dynamic_hint(existing: dict, hint) -> None:+ """Fold a dynamic hint into the file edge already joining the same two files.++ The graph holds one edge per pair, and hints run after static resolution,+ so replacing the edge would drop what the import said (its type, the names+ it binds). Names are unioned instead. A whole-module hint (no names) on a+ static edge is kept as ``dynamic_hint`` so dead code still treats every+ public member as reached. ``dynamic_hint`` lives on the live graph only:+ persisted rows keep the static type and the unioned names (upgrade path: a+ column, if a rehydrated graph ever runs the export rescue).+ """+ if is_dynamic_edge(existing.get("edge_type")):+ # Two hints on one pair: a whole-module reach absorbs a named one.+ if not hint.imported_names or not existing.get("imported_names"):+ existing.pop("imported_names", None)+ return+ elif not hint.imported_names:+ existing["dynamic_hint"] = hint.hint_source+ return+ names = existing.get("imported_names") or []+ existing["imported_names"] = names + [n for n in hint.imported_names if n not in names]++class EdgesMixin:"""Non-import edge construction for :class:`GraphBuilder`."""@@ -150,6 +174,10 @@ class EdgesMixin:# a real `dynamic_*` edge.sub_type = e.edge_typegraph_edge_type = sub_type if sub_type.startswith("dynamic") else f"dynamic_{sub_type}"+ existing = self._graph.get_edge_data(e.source, e.target)+ if existing is not None and existing.get("edge_type") in FILE_DEPENDENCY_EDGE_TYPES:+ _merge_dynamic_hint(existing, e)+ continuenames = {"imported_names": list(e.imported_names)} if e.imported_names else {}self._graph.add_edge(e.source,diff --git a/packages/core/src/repowise/core/ingestion/languages/python_strings.py b/packages/core/src/repowise/core/ingestion/languages/python_strings.pynew file mode 100644index 000000000..20feab71f--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/languages/python_strings.py@@ -0,0 +1,87 @@+"""Module-path strings in source text, and the part of a file that runs.++Python names code by dotted string as often as by ``import`` (entry-point+tables, plugin registries, lazy command tables). Both the dynamic-import hints+(graph edges) and dead-code analysis (finding filters) read those strings, so+the "only strings that run" filter, the "module defines this name" check and+the dynamic-load markers live here once.+"""++from __future__ import annotations++import io+import re+import tokenize+from pathlib import PurePosixPath++from .registry import REGISTRY++#: Source tokens that mark a Python file as loading modules by name at run time.+#: Substrings, so ``importlib`` also covers ``importlib.import_module``.+PY_DYNAMIC_LOAD_MARKERS: tuple[str, ...] = (+ "importlib",+ "import_module",+ "__import__",+ "import_string", # Werkzeug / Flask / Django utilities+ "load_entry_point",+ "entry_points(", # importlib.metadata plugin discovery+ "pkgutil",+ "pkg_resources",+ "lazy_subcommands", # lazy click group keyword: {"name": "pkg.mod.attr"}+)+++def is_python(path: str) -> bool:+ return REGISTRY.from_extension(PurePosixPath(path).suffix) == "python"+++def defines_top_level(blob: bytes, name: str) -> bool:+ """Whether a Python module defines *name* at top level (def, class, assignment).++ A re-export (``from x import Name``) or a tuple target is not seen, so such+ a member stays reported.+ """+ word = re.escape(name.encode("ascii"))+ pattern = (+ rb"^(?:(?:async[ \t]+)?def|class)[ \t]+" + word + rb"\b"+ rb"|^" + word + rb"[ \t]*(?::[^=\n]*)?=(?!=)"+ )+ return re.search(pattern, blob, re.MULTILINE) is not None+++#: Tokens after which a string starts an expression statement (a docstring).+_STATEMENT_START = frozenset(+ {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT, tokenize.ENCODING}+)+_TRIVIA = frozenset({tokenize.NL, tokenize.COMMENT})+_COMMENT_LINE_RE = re.compile(rb"^[ \t]*#.*$", re.MULTILINE)+++def live_text(path: str, blob: bytes) -> bytes:+ """*blob* without comments and, for Python, without docstrings.++ Python keeps only its string literals that are not a statement on their+ own; other files drop only ``#`` comment lines. Python that does not tokenize+ is kept whole, as before this filter.+ """+ if not is_python(path):+ return _COMMENT_LINE_RE.sub(b"", blob)+ try:+ tokens = [+ t for t in tokenize.tokenize(io.BytesIO(blob).readline) if t.type not in _TRIVIA+ ]+ except (tokenize.TokenError, SyntaxError, ValueError):+ return blob+ return b"\n".join(+ tok.string.encode("utf-8")+ for i, tok in enumerate(tokens)+ if tok.type == tokenize.STRING and not _is_statement(tokens, i)+ )+++def _is_statement(tokens: list[tokenize.TokenInfo], i: int) -> bool:+ """Whether ``tokens[i]`` is a statement on its own (a docstring)."""+ # ENDMARKER always closes the stream, so ``i + 1`` exists.+ if tokens[i - 1].type not in _STATEMENT_START:+ return False+ return tokens[i + 1].type in (tokenize.NEWLINE, tokenize.ENDMARKER)diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex 16d8481e6..b769da00d 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -234,6 +234,7 @@ def build_repo_graph(Path(repo_path),dotnet_index=graph_builder.dotnet_index,file_paths=[fi.path for fi in file_infos],+ source_map=source_map or None,)graph_builder.add_dynamic_edges(dynamic_edges)if dynamic_edges:diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.pyindex f734dd587..850cef5c7 100644--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py@@ -687,7 +687,10 @@ async def _run_ingestion(dynamic_edges = await loop.run_in_executor(None,lambda: registry.extract_all(- repo_path, dotnet_index=graph_builder.dotnet_index, file_paths=hint_paths+ repo_path,+ dotnet_index=graph_builder.dotnet_index,+ file_paths=hint_paths,+ source_map=source_map,),)graph_builder.add_dynamic_edges(dynamic_edges)diff --git a/tests/unit/ingestion/test_python_dead_code_fixes.py b/tests/unit/ingestion/test_python_dead_code_fixes.pyindex 12a13575d..9bc27abf5 100644--- a/tests/unit/ingestion/test_python_dead_code_fixes.py+++ b/tests/unit/ingestion/test_python_dead_code_fixes.py@@ -414,3 +414,35 @@ def test_dynamic_use_edge_marks_target_file_live() -> None:g.add_edge("pkg/registry.py", "pkg/ollama.py", edge_type="dynamic_uses")assert "OllamaProvider" not in _unused_export_names(g)+++def test_whole_module_hint_folded_into_an_import_marks_target_file_live() -> None:+ """A whole-module hint on a pair that also has a static import keeps the+ import edge and its names, and still keeps every public export live."""+ g = nx.DiGraph()+ for path in ("pkg/registry.py", "pkg/ollama.py"):+ g.add_node(path, node_type="file", language="python", is_test=False, is_entry_point=False)+ for name in ("OllamaProvider", "app"):+ sym = f"pkg/ollama.py::{name}"+ g.add_node(+ sym,+ node_type="symbol",+ kind="class",+ name=name,+ visibility="public",+ decorators=[],+ start_line=1,+ end_line=20,+ file_path="pkg/ollama.py",+ language="python",+ )+ g.add_edge("pkg/ollama.py", sym, edge_type="defines")+ g.add_edge(+ "pkg/registry.py",+ "pkg/ollama.py",+ edge_type="imports",+ imported_names=["OllamaProvider"],+ dynamic_hint="python_dynamic_import",+ )++ assert not {"OllamaProvider", "app"} & _unused_export_names(g)diff --git a/tests/unit/ingestion/test_python_string_registry_hints.py b/tests/unit/ingestion/test_python_string_registry_hints.pynew file mode 100644index 000000000..d2388668f--- /dev/null+++ b/tests/unit/ingestion/test_python_string_registry_hints.py@@ -0,0 +1,257 @@+"""Modules named by string in lazy registries become ``dynamic_uses`` edges.++Lazy command tables, entry-point specs and templated ``import_module`` calls+load a module by name, so no import edge exists. Each test pins one shape and+one false-edge guard: a string must resolve to a module in the repo, and an+attribute reference must name something that module has.+"""++from __future__ import annotations++from pathlib import Path++from repowise.core.ingestion.dynamic_hints.base import DynamicEdge+from repowise.core.ingestion.dynamic_hints.python_imports import (+ ModuleStringResolver,+ PythonDynamicHints,+ python_dynamic_refs,+)+from repowise.core.ingestion.graph import GraphBuilder+from repowise.core.ingestion.languages.python_modules import build_python_module_index+++def _refs(files: dict[str, str], rel: str) -> dict[str, tuple[str, ...]]:+ blobs = {path: src.encode() for path, src in files.items()}+ resolver = ModuleStringResolver(+ build_python_module_index(blobs), lambda path: blobs.get(path, b"")+ )+ return python_dynamic_refs(rel, blobs[rel], resolver)+++_CLI = {+ "src/app/__init__.py": "",+ "src/app/commands/__init__.py": "",+ "src/app/commands/status_cmd.py": "def status_command():\n pass\n",+ "src/app/commands/init_cmd/__init__.py": "from .command import init_command\n",+ "src/app/commands/init_cmd/command.py": "def init_command():\n pass\n",+}+++def test_relative_lazy_command_table_resolves_under_the_named_package() -> None:+ files = {+ **_CLI,+ "src/app/main.py": (+ '_PKG = "app.commands"\n'+ "_COMMANDS = (\n"+ ' ("status", "status_cmd:status_command"),\n'+ ' ("init", "init_cmd:init_command"),\n'+ ' ("gone", "missing_cmd:gone_command"),\n'+ ")\n"+ "for name, target in _COMMANDS:\n"+ ' register(name, f"{_PKG}.{target}")\n'+ ),+ }+ assert _refs(files, "src/app/main.py") == {+ "src/app/commands/status_cmd.py": ("status_command",),+ # A package re-exporting the command counts as having it.+ "src/app/commands/init_cmd/__init__.py": ("init_command",),+ }+++def test_absolute_entry_point_string_needs_no_loader_marker() -> None:+ files = {**_CLI, "src/app/serve.py": 'run("app.commands.status_cmd:status_command")\n'}+ assert _refs(files, "src/app/serve.py") == {+ "src/app/commands/status_cmd.py": ("status_command",)+ }+++def test_colon_string_naming_an_attribute_the_module_lacks_adds_nothing() -> None:+ # An event key that happens to share its prefix with a package name.+ files = {+ "agent/__init__.py": "def run():\n pass\n",+ "gateway.py": 'emit("agent:start")\nemit("agent:run")\n',+ }+ assert _refs(files, "gateway.py") == {"agent/__init__.py": ("run",)}+++def test_module_with_lazy_getattr_serves_any_attribute() -> None:+ files = {+ "agent/__init__.py": "def __getattr__(name):\n return name\n",+ "gateway.py": 'load("agent:start")\n',+ }+ assert _refs(files, "gateway.py") == {"agent/__init__.py": ("start",)}+++def test_string_naming_no_repo_module_adds_nothing() -> None:+ files = {**_CLI, "src/app/serve.py": 'run("uvicorn.main:run")\nopen("w:gz")\n'}+ assert _refs(files, "src/app/serve.py") == {}+++def test_strings_in_comments_and_docstrings_add_nothing() -> None:+ files = {+ **_CLI,+ "src/app/doc.py": (+ '"""Run ``"app.commands.status_cmd:status_command"`` to see it."""\n'+ "import importlib\n"+ '# load("app.commands.status_cmd:status_command")\n'+ "def f():\n"+ ' """See "app.commands.status_cmd" for details."""\n'+ ),+ }+ assert _refs(files, "src/app/doc.py") == {}+++def test_dotted_member_path_in_a_loader_file_names_the_member() -> None:+ files = {+ **_CLI,+ "src/app/loader.py": (+ "from importlib import import_module\n"+ 'BACKENDS = ["app.commands.status_cmd.status_command", "app.commands.init_cmd"]\n'+ ),+ }+ assert _refs(files, "src/app/loader.py") == {+ "src/app/commands/status_cmd.py": ("status_command",),+ "src/app/commands/init_cmd/__init__.py": (),+ }+++def test_dotted_member_path_in_a_test_is_a_patch_target_not_a_load() -> None:+ files = {+ **_CLI,+ "tests/test_status.py": (+ "import importlib\n"+ 'patch("app.commands.status_cmd.status_command")\n'+ ),+ }+ assert _refs(files, "tests/test_status.py") == {}+++def test_lazy_group_keyword_gates_dotted_module_attr_strings() -> None:+ files = {+ **_CLI,+ "src/app/cli.py": (+ "@click.group(cls=LazyGroup, lazy_subcommands="+ '{"status": "app.commands.status_cmd.status_command"})\n'+ "def cli():\n pass\n"+ ),+ }+ assert _refs(files, "src/app/cli.py") == {+ "src/app/commands/status_cmd.py": ("status_command",)+ }+++def test_templated_import_reaches_every_matching_module() -> None:+ files = {+ "pkg/__init__.py": "",+ "pkg/plugins/__init__.py": "",+ "pkg/plugins/a.py": "",+ "pkg/plugins/b/__init__.py": "",+ "pkg/plugins/b/oauth.py": "",+ "pkg/other.py": "",+ "pkg/loader.py": (+ "import importlib\n"+ "def load(name):\n"+ ' importlib.import_module(f"pkg.plugins.{name}")\n'+ ' importlib.import_module(f"pkg.plugins.{name}.oauth")\n'+ ' importlib.import_module("pkg.plugins." + name)\n'+ ' importlib.import_module(f"{name}.{name}")\n'+ ),+ }+ assert _refs(files, "pkg/loader.py") == {+ "pkg/plugins/a.py": (),+ "pkg/plugins/b/__init__.py": (),+ "pkg/plugins/b/oauth.py": (),+ }+++def test_extractor_reads_attached_source_map_before_disk(tmp_path: Path) -> None:+ for path, src in _CLI.items():+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / path).write_text(src)+ main = tmp_path / "src/app/main.py"+ main.write_text("") # on disk: nothing; in the source map: the table+ hints = PythonDynamicHints()+ hints._source_map = {"src/app/main.py": b'P = "app.commands"\nT = "status_cmd:status_command"\n'}+ edges = hints.extract(tmp_path)+ assert [(e.source, e.target, e.edge_type, e.imported_names) for e in edges] == [+ (+ "src/app/main.py",+ "src/app/commands/status_cmd.py",+ "dynamic_uses",+ ("status_command",),+ )+ ]+++def test_templated_import_in_a_comment_or_docstring_adds_nothing() -> None:+ files = {+ "pkg/__init__.py": "",+ "pkg/plugins/__init__.py": "",+ "pkg/plugins/a.py": "",+ "pkg/loader.py": (+ '"""Loads plugins with import_module(f"pkg.plugins.{name}")."""\n'+ "import importlib\n"+ "def load(name):\n"+ ' # importlib.import_module(f"pkg.plugins.{name}")\n'+ " return name\n"+ ),+ }+ assert _refs(files, "pkg/loader.py") == {}+++def test_wide_or_one_segment_template_links_the_package_only() -> None:+ plugins = {f"pkg/plugins/p{i}.py": "" for i in range(51)}+ files = {+ "pkg/__init__.py": "",+ "pkg/plugins/__init__.py": "",+ **plugins,+ "pkg/loader.py": (+ "import importlib\n"+ "def load(name):\n"+ ' importlib.import_module(f"pkg.plugins.{name}")\n'+ ' importlib.import_module(f"pkg.{name}")\n'+ ),+ }+ assert _refs(files, "pkg/loader.py") == {+ "pkg/plugins/__init__.py": (),+ "pkg/__init__.py": (),+ }+++def _hint(source: str, target: str, names: tuple[str, ...] = ()) -> DynamicEdge:+ return DynamicEdge(source, target, "dynamic_uses", "python_dynamic_import", imported_names=names)+++def _builder_with_import(names: list[str]) -> GraphBuilder:+ builder = GraphBuilder()+ graph = builder._graph+ graph.add_node("a.py")+ graph.add_node("b.py")+ graph.add_edge("a.py", "b.py", edge_type="imports", imported_names=names)+ return builder+++def test_hint_on_an_imported_pair_keeps_the_import_and_unions_names() -> None:+ builder = _builder_with_import(["Foo"])+ builder.add_dynamic_edges([_hint("a.py", "b.py", ("app",))])+ data = builder._graph.get_edge_data("a.py", "b.py")+ assert data["edge_type"] == "imports"+ assert data["imported_names"] == ["Foo", "app"]+ assert "dynamic_hint" not in data+++def test_whole_module_hint_on_an_imported_pair_is_kept_as_a_flag() -> None:+ builder = _builder_with_import(["Foo"])+ builder.add_dynamic_edges([_hint("a.py", "b.py")])+ data = builder._graph.get_edge_data("a.py", "b.py")+ assert (data["edge_type"], data["imported_names"]) == ("imports", ["Foo"])+ assert data["dynamic_hint"] == "python_dynamic_import"+++def test_second_hint_on_a_pair_merges_and_whole_module_wins() -> None:+ builder = GraphBuilder()+ builder._graph.add_node("a.py")+ builder.add_dynamic_edges([_hint("a.py", "b.py", ("x",)), _hint("a.py", "b.py", ("y",))])+ assert builder._graph.get_edge_data("a.py", "b.py")["imported_names"] == ["x", "y"]+ builder.add_dynamic_edges([_hint("a.py", "b.py")])+ assert "imported_names" not in builder._graph.get_edge_data("a.py", "b.py")diff --git a/tests/unit/test_no_bare_type_name_copies.py b/tests/unit/test_no_bare_type_name_copies.pyindex 738d7c3d9..2241cd03c 100644--- a/tests/unit/test_no_bare_type_name_copies.py+++ b/tests/unit/test_no_bare_type_name_copies.py@@ -95,6 +95,9 @@ _KNOWN: dict[str, int] = {_PREFIX + "dynamic_hints/swift.py": 1,# Splits an import statement's package path, not a type reference._PREFIX + "languages/jvm_same_package.py": 1,+ # Splits a module-path string (`"pkg.mod.Name"`) into the module it loads+ # and the attribute it reads; the head is a module path, not a qualifier.+ _PREFIX + "dynamic_hints/python_imports.py": 1,# Qualifies a PHP name against its `use` alias (the HEAD segment) and# splits an FQN into the namespace it indexes by and the class: it asks# which namespace, the opposite end from the shared helper.