Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(dead-code): make the dynamic-import package clamp a set lookup (#1222)

Raghav Chamadiya5a09ff3cf9fc
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
index 069cf1d80..009c1c940 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
@@ -493,6 +493,23 @@ class DeadCodeAnalyzer:
self._dynamic_import_files = find_dynamic_import_files(
parsed_files or {}
) | find_dynamic_edge_files(graph)
+ # Parent directories of the above, precomputed once.
+ #
+ # ``_make_unreachable_finding`` asks "does any dynamic-import file sit
+ # in the same package as this node". That used to be a scan over
+ # ``_dynamic_import_files`` constructing a ``Path`` per candidate, run
+ # once per unreachable candidate: O(candidates x dynamic_files) with a
+ # pathlib constant. A set of parent strings makes it one hash lookup,
+ # so the pass is linear in candidates again. On a 30k-file JS monorepo
+ # (8.8k files carrying dynamic-import markers) that took the dead-code
+ # phase from 374.9s to 10.1s with byte-identical findings.
+ #
+ # Derived state: this is only valid for the current
+ # ``_dynamic_import_files``. Treat both as immutable after __init__,
+ # or update them together.
+ self._dynamic_import_dirs: set[str] = {
+ str(Path(dif).parent) for dif in self._dynamic_import_files
+ }
self._jsx_namespace_files: set[str] = _find_jsx_namespace_files(parsed_files or {})
# Files substituted in via bundler ``resolve.alias`` config — named
# by the config, never imported by path.
@@ -711,12 +728,8 @@ class DeadCodeAnalyzer:
confidence = 0.4
# Reduce confidence when dynamic imports exist in the same package.
- if self._dynamic_import_files:
- node_pkg = str(Path(node).parent)
- for dif in self._dynamic_import_files:
- if str(Path(dif).parent) == node_pkg:
- confidence = min(confidence, 0.4)
- break
+ if self._dynamic_import_dirs and str(Path(node).parent) in self._dynamic_import_dirs:
+ confidence = min(confidence, 0.4)
# Runtime-load risk factors (config / bootstrap / database /
# environment / script). These are files the never-flag allowlist
diff --git a/packages/core/src/repowise/core/analysis/dead_code/constants.py b/packages/core/src/repowise/core/analysis/dead_code/constants.py
index 063a01d17..65f022237 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/constants.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/constants.py
@@ -12,11 +12,7 @@ from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTR
# Non-code languages that should never be flagged as dead code.
# Derived from the centralised LanguageRegistry — passthrough config/infra
# languages plus "unknown".
-_NON_CODE_LANGUAGES: frozenset[str] = frozenset(
- spec.tag
- for spec in _LANG_REGISTRY.all_specs()
- if spec.is_passthrough and (not spec.is_code or spec.is_infra) and spec.tag != "openapi"
-) | {"unknown"}
+_NON_CODE_LANGUAGES: frozenset[str] = _LANG_REGISTRY.unparseable_or_unknown_languages()
# Patterns that should never be flagged as dead.
_NEVER_FLAG_PATTERNS: tuple[str, ...] = (
diff --git a/packages/core/src/repowise/core/ingestion/languages/registry.py b/packages/core/src/repowise/core/ingestion/languages/registry.py
index 9ae64f974..0a6380164 100644
--- a/packages/core/src/repowise/core/ingestion/languages/registry.py
+++ b/packages/core/src/repowise/core/ingestion/languages/registry.py
@@ -126,6 +126,47 @@ class LanguageRegistry:
"""Return tags for languages with no AST parser."""
return frozenset(s.tag for s in self._specs.values() if s.is_passthrough)
+ def unparseable_data_languages(self) -> frozenset[str]:
+ """Config/markup/data plus infra passthrough tags, without ``openapi``
+ and ``unknown``.
+
+ The question is "we cannot extract structure from it, and there is none
+ to extract". That is narrower than :meth:`passthrough_languages`, which
+ also catches real code languages still awaiting a grammar (clojure,
+ elixir, haskell). For those, zero symbols is a gap in us rather than a
+ property of the file, so they must stay eligible for the analyses this
+ set exempts.
+
+ ``openapi`` is excluded because ``special_handlers`` extracts from it
+ without a tree-sitter grammar. ``unknown`` is excluded because it is not
+ a data format at all: it is the registry's "no idea what this is"
+ fallback, carrying ``is_passthrough=True`` only because there is no
+ grammar to point at. Callers that do want it say so, via
+ :meth:`unparseable_or_unknown_languages`.
+
+ Three call sites inlined this comprehension before it landed here (the
+ parser's passthrough skip, the traverser's generated-marker skip, and
+ dead code's non-code exemption) and they had already drifted on whether
+ ``unknown`` belonged. Same reasoning as ``core.test_paths``: one
+ spelling, so they cannot disagree.
+ """
+ return frozenset(
+ s.tag
+ for s in self._specs.values()
+ if s.is_passthrough
+ and (not s.is_code or s.is_infra)
+ and s.tag not in ("openapi", "unknown")
+ )
+
+ def unparseable_or_unknown_languages(self) -> frozenset[str]:
+ """:meth:`unparseable_data_languages` plus ``unknown``.
+
+ For callers that must not act on a file whose language was never
+ identified. Dead code is the one today: there, "unknown" means "no
+ evidence of reachability either way", so flagging it would be a guess.
+ """
+ return self.unparseable_data_languages() | {"unknown"}
+
def infra_languages(self) -> frozenset[str]:
"""Return tags for infrastructure languages."""
return frozenset(s.tag for s in self._specs.values() if s.is_infra)
diff --git a/packages/core/src/repowise/core/ingestion/parser.py b/packages/core/src/repowise/core/ingestion/parser.py
index 9df1b9262..77058744f 100644
--- a/packages/core/src/repowise/core/ingestion/parser.py
+++ b/packages/core/src/repowise/core/ingestion/parser.py
@@ -143,13 +143,7 @@ def _load_compiled_query(lang: str, grammar_tag: str | None = None) -> object |
# included (not the extra git-blame-only languages).
# Excludes "openapi" (handled by special_handlers) and "unknown".
-_PASSTHROUGH_LANGUAGES: frozenset[str] = frozenset(
- spec.tag
- for spec in _LANG_REGISTRY.all_specs()
- if spec.is_passthrough
- and (not spec.is_code or spec.is_infra)
- and spec.tag not in ("openapi", "unknown")
-)
+_PASSTHROUGH_LANGUAGES: frozenset[str] = _LANG_REGISTRY.unparseable_data_languages()
# ---------------------------------------------------------------------------
# Language registry — maps language tag → tree-sitter Language object
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index c74996372..db18edd93 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -211,13 +211,7 @@ _DEFAULT_MAX_FILE_SIZE_BYTES: int = 500 * 1024 # 500 KB
# adds no value.
# Languages for which generated-file detection is skipped — same as parser's
# passthrough set (no AST parsing, so reading 512 bytes for markers is pointless).
-_SKIP_GENERATED_CHECK: frozenset[str] = frozenset(
- spec.tag
- for spec in _LANG_REGISTRY.all_specs()
- if spec.is_passthrough
- and (not spec.is_code or spec.is_infra)
- and spec.tag not in ("openapi", "unknown")
-)
+_SKIP_GENERATED_CHECK: frozenset[str] = _LANG_REGISTRY.unparseable_data_languages()
class FileTraverser:
diff --git a/tests/unit/dead_code/test_dynamic_import_dirs.py b/tests/unit/dead_code/test_dynamic_import_dirs.py
new file mode 100644
index 000000000..d6fd752cb
--- /dev/null
+++ b/tests/unit/dead_code/test_dynamic_import_dirs.py
@@ -0,0 +1,145 @@
+"""Dynamic-import package clamp: semantics, and the derived set that drives it.
+
+``_make_unreachable_finding`` caps confidence at 0.4 when a dynamic-import file
+lives in the same package as the candidate. That used to be a scan over
+``_dynamic_import_files`` run once per candidate, constructing a ``Path`` per
+inner iteration: O(candidates x dynamic_files). It is now a precomputed set of
+parent directories plus one hash lookup.
+
+These tests pin the clamp's semantics (which package matches, which does not,
+and the repo-root case) so the rewrite cannot have moved them.
+"""
+
+from __future__ import annotations
+
+import networkx as nx
+import pytest
+
+from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKind
+
+from ._helpers import _build_graph, _old_date
+
+_DETECT_UNREACHABLE_ONLY = {
+ "detect_unused_exports": False,
+ "detect_unused_internals": False,
+ "detect_zombie_packages": False,
+ "min_confidence": 0.0,
+}
+
+
+def _orphan_graph() -> nx.DiGraph:
+ """Two packages, one carrying a dynamic edge and one not."""
+ return _build_graph(
+ nodes={
+ # pkg_a: orphan sits in the SAME package as a dynamic edge target
+ "pkg_a/orphan.py": {"symbol_count": 5, "symbols": []},
+ "pkg_a/dispatcher.py": {"symbol_count": 3, "symbols": []},
+ "pkg_a/handler.py": {"is_entry_point": True, "symbol_count": 3, "symbols": []},
+ # pkg_b: orphan sits in a package with NO dynamic edge at all
+ "pkg_b/orphan.py": {"symbol_count": 5, "symbols": []},
+ },
+ edges=[("pkg_a/dispatcher.py", "pkg_a/handler.py", {"edge_type": "dynamic"})],
+ )
+
+
+def _stale_meta(*paths: str) -> dict[str, dict]:
+ """Git metadata that scores an unreferenced file at full confidence."""
+ return {
+ p: {"commit_count_90d": 0, "last_commit_at": _old_date(days=400), "age_days": 500}
+ for p in paths
+ }
+
+
+def _unreachable(report, path: str):
+ hits = [
+ f for f in report.findings if f.file_path == path and f.kind == DeadCodeKind.UNREACHABLE_FILE
+ ]
+ assert len(hits) == 1, f"expected exactly one unreachable finding for {path}, got {hits}"
+ return hits[0]
+
+
+def test_same_package_dynamic_import_clamps_confidence():
+ """An orphan beside a dynamically-reached file stays a review candidate."""
+ analyzer = DeadCodeAnalyzer(
+ _orphan_graph(),
+ git_meta_map=_stale_meta("pkg_a/orphan.py", "pkg_b/orphan.py"),
+ )
+ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))
+
+ finding = _unreachable(report, "pkg_a/orphan.py")
+ assert finding.confidence == pytest.approx(0.4)
+ assert not finding.safe_to_delete
+
+
+def test_other_package_is_not_clamped():
+ """The discriminating case: the clamp is per-package, not repo-wide.
+
+ A membership test that dropped the directory would clamp every orphan in
+ the repo as soon as one dynamic import existed anywhere.
+ """
+ analyzer = DeadCodeAnalyzer(
+ _orphan_graph(),
+ git_meta_map=_stale_meta("pkg_a/orphan.py", "pkg_b/orphan.py"),
+ )
+ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))
+
+ finding = _unreachable(report, "pkg_b/orphan.py")
+ # Untouched for 400 days, in a package with no dynamic edge: unclamped.
+ assert finding.confidence == pytest.approx(1.0)
+
+
+def test_dynamic_import_dirs_holds_packages_not_files():
+ """The lookup key is the parent directory, and only for packages that have one."""
+ analyzer = DeadCodeAnalyzer(_orphan_graph(), git_meta_map={})
+
+ assert analyzer._dynamic_import_dirs == {"pkg_a"}
+
+
+def test_repo_root_dynamic_import_clamps_other_root_files():
+ """A dynamic import at the repo root clamps every other root-level orphan.
+
+ ``Path("main.py").parent`` is ``.``, so the root behaves as one package and
+ a single root-level dynamic import caps confidence for all its neighbours.
+ Surprising enough to pin: it is the case a future rewrite is most likely to
+ get wrong, and bare filenames are real node ids in this codebase.
+ """
+ graph = _build_graph(
+ nodes={
+ "loader.py": {"symbol_count": 3, "symbols": []},
+ "main.py": {"is_entry_point": True, "symbol_count": 3, "symbols": []},
+ "orphan.py": {"symbol_count": 5, "symbols": []},
+ "pkg_c/orphan.py": {"symbol_count": 5, "symbols": []},
+ },
+ edges=[("loader.py", "main.py", {"edge_type": "dynamic"})],
+ )
+ analyzer = DeadCodeAnalyzer(
+ graph, git_meta_map=_stale_meta("orphan.py", "pkg_c/orphan.py")
+ )
+ assert analyzer._dynamic_import_dirs == {"."}
+
+ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))
+
+ # Root-level neighbour of the dynamic import: clamped.
+ assert _unreachable(report, "orphan.py").confidence == pytest.approx(0.4)
+ # A real subpackage is unaffected by the root's dynamic import.
+ assert _unreachable(report, "pkg_c/orphan.py").confidence == pytest.approx(1.0)
+
+
+def test_clamp_reads_the_derived_set_not_the_file_set():
+ """The precomputed directory set is what the clamp consults.
+
+ Pins the derived-state contract documented on ``_dynamic_import_dirs``:
+ widening ``_dynamic_import_files`` alone must not move a verdict, because
+ the lookup no longer walks that collection. This is what fails if the
+ per-candidate scan is ever reintroduced.
+ """
+ graph = _orphan_graph()
+ meta = _stale_meta("pkg_a/orphan.py", "pkg_b/orphan.py")
+
+ analyzer = DeadCodeAnalyzer(graph, git_meta_map=meta)
+ # A dynamic-import file in pkg_b, registered ONLY in the file set.
+ analyzer._dynamic_import_files = analyzer._dynamic_import_files | {"pkg_b/late.py"}
+
+ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))
+
+ assert _unreachable(report, "pkg_b/orphan.py").confidence == pytest.approx(1.0)
diff --git a/tests/unit/ingestion/test_unparseable_language_sets.py b/tests/unit/ingestion/test_unparseable_language_sets.py
new file mode 100644
index 000000000..6cd4ece02
--- /dev/null
+++ b/tests/unit/ingestion/test_unparseable_language_sets.py
@@ -0,0 +1,77 @@
+"""The "no AST parser and no structure to extract" language set has one home.
+
+Three modules computed this comprehension verbatim -- the parser's passthrough
+skip, the traverser's generated-marker skip, and dead code's non-code
+exemption -- and they had already drifted on whether ``unknown`` belonged.
+Same failure mode ``core.test_paths`` was created to end, so the same fix: one
+spelling on the registry, and these tests keep the call sites pinned to it.
+"""
+
+from __future__ import annotations
+
+from repowise.core.analysis.dead_code.constants import _NON_CODE_LANGUAGES
+from repowise.core.ingestion.languages.registry import REGISTRY
+from repowise.core.ingestion.parser import _PASSTHROUGH_LANGUAGES
+from repowise.core.ingestion.traverser import _SKIP_GENERATED_CHECK
+
+
+def _reference_set() -> frozenset[str]:
+ """The predicate as it read at all three call sites before extraction."""
+ return frozenset(
+ spec.tag
+ for spec in REGISTRY.all_specs()
+ if spec.is_passthrough
+ and (not spec.is_code or spec.is_infra)
+ and spec.tag not in ("openapi", "unknown")
+ )
+
+
+def test_registry_set_matches_the_original_predicate():
+ """Extraction was behaviour-preserving, not a redefinition."""
+ assert REGISTRY.unparseable_data_languages() == _reference_set()
+
+
+def test_parser_and_traverser_use_the_shared_set():
+ assert REGISTRY.unparseable_data_languages() == _PASSTHROUGH_LANGUAGES
+ assert REGISTRY.unparseable_data_languages() == _SKIP_GENERATED_CHECK
+
+
+def test_dead_code_adds_unknown_and_says_so():
+ """Dead code alone treats ``unknown`` as non-code.
+
+ An unidentified language is no evidence of reachability either way, so
+ flagging it would be a guess. That is a real difference from the parser's
+ question, which is why it is a second named method rather than a third
+ copy of the comprehension.
+ """
+ assert REGISTRY.unparseable_or_unknown_languages() == _NON_CODE_LANGUAGES
+ assert REGISTRY.unparseable_data_languages() | {"unknown"} == _NON_CODE_LANGUAGES
+
+
+def test_unknown_is_excluded_from_the_base_set():
+ """Regression pin: ``unknown`` is itself a passthrough spec.
+
+ Deriving the base set by excluding only ``openapi`` silently pulls
+ ``unknown`` in, which would make the parser treat unidentified files as
+ passthrough and stop the traverser reading their generated-file markers.
+ """
+ assert "unknown" not in REGISTRY.unparseable_data_languages()
+ assert "unknown" in REGISTRY.unparseable_or_unknown_languages()
+
+
+def test_real_data_formats_are_in_the_set():
+ """Anchor the set to concrete members so an upstream spec flip is caught."""
+ for tag in ("markdown", "json", "yaml", "toml"):
+ assert tag in REGISTRY.unparseable_data_languages(), tag
+
+
+def test_passthrough_code_languages_are_not_in_the_set():
+ """Languages awaiting a grammar are NOT "data".
+
+ clojure/elixir/haskell are ``is_passthrough`` but real code: zero symbols
+ there is a gap in us, not a property of the file, so they must stay
+ eligible for the analyses this set exempts.
+ """
+ for tag in ("clojure", "elixir", "haskell"):
+ assert tag in REGISTRY.passthrough_languages(), tag
+ assert tag not in REGISTRY.unparseable_data_languages(), tag