Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(dead-code): make the dynamic-import package clamp a set lookup (#1222)
Raghav Chamadiya5a09ff3cf9fc
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.pyindex 069cf1d80..009c1c940 100644--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py@@ -493,6 +493,23 @@ class DeadCodeAnalyzer:self._dynamic_import_files = find_dynamic_import_files(parsed_files or {}) | find_dynamic_edge_files(graph)+ # Parent directories of the above, precomputed once.+ #+ # ``_make_unreachable_finding`` asks "does any dynamic-import file sit+ # in the same package as this node". That used to be a scan over+ # ``_dynamic_import_files`` constructing a ``Path`` per candidate, run+ # once per unreachable candidate: O(candidates x dynamic_files) with a+ # pathlib constant. A set of parent strings makes it one hash lookup,+ # so the pass is linear in candidates again. On a 30k-file JS monorepo+ # (8.8k files carrying dynamic-import markers) that took the dead-code+ # phase from 374.9s to 10.1s with byte-identical findings.+ #+ # Derived state: this is only valid for the current+ # ``_dynamic_import_files``. Treat both as immutable after __init__,+ # or update them together.+ self._dynamic_import_dirs: set[str] = {+ str(Path(dif).parent) for dif in self._dynamic_import_files+ }self._jsx_namespace_files: set[str] = _find_jsx_namespace_files(parsed_files or {})# Files substituted in via bundler ``resolve.alias`` config — named# by the config, never imported by path.@@ -711,12 +728,8 @@ class DeadCodeAnalyzer:confidence = 0.4# Reduce confidence when dynamic imports exist in the same package.- if self._dynamic_import_files:- node_pkg = str(Path(node).parent)- for dif in self._dynamic_import_files:- if str(Path(dif).parent) == node_pkg:- confidence = min(confidence, 0.4)- break+ if self._dynamic_import_dirs and str(Path(node).parent) in self._dynamic_import_dirs:+ confidence = min(confidence, 0.4)# Runtime-load risk factors (config / bootstrap / database /# environment / script). These are files the never-flag allowlistdiff --git a/packages/core/src/repowise/core/analysis/dead_code/constants.py b/packages/core/src/repowise/core/analysis/dead_code/constants.pyindex 063a01d17..65f022237 100644--- a/packages/core/src/repowise/core/analysis/dead_code/constants.py+++ b/packages/core/src/repowise/core/analysis/dead_code/constants.py@@ -12,11 +12,7 @@ from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTR# Non-code languages that should never be flagged as dead code.# Derived from the centralised LanguageRegistry — passthrough config/infra# languages plus "unknown".-_NON_CODE_LANGUAGES: frozenset[str] = frozenset(- spec.tag- for spec in _LANG_REGISTRY.all_specs()- if spec.is_passthrough and (not spec.is_code or spec.is_infra) and spec.tag != "openapi"-) | {"unknown"}+_NON_CODE_LANGUAGES: frozenset[str] = _LANG_REGISTRY.unparseable_or_unknown_languages()# Patterns that should never be flagged as dead._NEVER_FLAG_PATTERNS: tuple[str, ...] = (diff --git a/packages/core/src/repowise/core/ingestion/languages/registry.py b/packages/core/src/repowise/core/ingestion/languages/registry.pyindex 9ae64f974..0a6380164 100644--- a/packages/core/src/repowise/core/ingestion/languages/registry.py+++ b/packages/core/src/repowise/core/ingestion/languages/registry.py@@ -126,6 +126,47 @@ class LanguageRegistry:"""Return tags for languages with no AST parser."""return frozenset(s.tag for s in self._specs.values() if s.is_passthrough)+ def unparseable_data_languages(self) -> frozenset[str]:+ """Config/markup/data plus infra passthrough tags, without ``openapi``+ and ``unknown``.++ The question is "we cannot extract structure from it, and there is none+ to extract". That is narrower than :meth:`passthrough_languages`, which+ also catches real code languages still awaiting a grammar (clojure,+ elixir, haskell). For those, zero symbols is a gap in us rather than a+ property of the file, so they must stay eligible for the analyses this+ set exempts.++ ``openapi`` is excluded because ``special_handlers`` extracts from it+ without a tree-sitter grammar. ``unknown`` is excluded because it is not+ a data format at all: it is the registry's "no idea what this is"+ fallback, carrying ``is_passthrough=True`` only because there is no+ grammar to point at. Callers that do want it say so, via+ :meth:`unparseable_or_unknown_languages`.++ Three call sites inlined this comprehension before it landed here (the+ parser's passthrough skip, the traverser's generated-marker skip, and+ dead code's non-code exemption) and they had already drifted on whether+ ``unknown`` belonged. Same reasoning as ``core.test_paths``: one+ spelling, so they cannot disagree.+ """+ return frozenset(+ s.tag+ for s in self._specs.values()+ if s.is_passthrough+ and (not s.is_code or s.is_infra)+ and s.tag not in ("openapi", "unknown")+ )++ def unparseable_or_unknown_languages(self) -> frozenset[str]:+ """:meth:`unparseable_data_languages` plus ``unknown``.++ For callers that must not act on a file whose language was never+ identified. Dead code is the one today: there, "unknown" means "no+ evidence of reachability either way", so flagging it would be a guess.+ """+ return self.unparseable_data_languages() | {"unknown"}+def infra_languages(self) -> frozenset[str]:"""Return tags for infrastructure languages."""return frozenset(s.tag for s in self._specs.values() if s.is_infra)diff --git a/packages/core/src/repowise/core/ingestion/parser.py b/packages/core/src/repowise/core/ingestion/parser.pyindex 9df1b9262..77058744f 100644--- a/packages/core/src/repowise/core/ingestion/parser.py+++ b/packages/core/src/repowise/core/ingestion/parser.py@@ -143,13 +143,7 @@ def _load_compiled_query(lang: str, grammar_tag: str | None = None) -> object |# included (not the extra git-blame-only languages).# Excludes "openapi" (handled by special_handlers) and "unknown".-_PASSTHROUGH_LANGUAGES: frozenset[str] = frozenset(- spec.tag- for spec in _LANG_REGISTRY.all_specs()- if spec.is_passthrough- and (not spec.is_code or spec.is_infra)- and spec.tag not in ("openapi", "unknown")-)+_PASSTHROUGH_LANGUAGES: frozenset[str] = _LANG_REGISTRY.unparseable_data_languages()# ---------------------------------------------------------------------------# Language registry — maps language tag → tree-sitter Language objectdiff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex c74996372..db18edd93 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -211,13 +211,7 @@ _DEFAULT_MAX_FILE_SIZE_BYTES: int = 500 * 1024 # 500 KB# adds no value.# Languages for which generated-file detection is skipped — same as parser's# passthrough set (no AST parsing, so reading 512 bytes for markers is pointless).-_SKIP_GENERATED_CHECK: frozenset[str] = frozenset(- spec.tag- for spec in _LANG_REGISTRY.all_specs()- if spec.is_passthrough- and (not spec.is_code or spec.is_infra)- and spec.tag not in ("openapi", "unknown")-)+_SKIP_GENERATED_CHECK: frozenset[str] = _LANG_REGISTRY.unparseable_data_languages()class FileTraverser:diff --git a/tests/unit/dead_code/test_dynamic_import_dirs.py b/tests/unit/dead_code/test_dynamic_import_dirs.pynew file mode 100644index 000000000..d6fd752cb--- /dev/null+++ b/tests/unit/dead_code/test_dynamic_import_dirs.py@@ -0,0 +1,145 @@+"""Dynamic-import package clamp: semantics, and the derived set that drives it.++``_make_unreachable_finding`` caps confidence at 0.4 when a dynamic-import file+lives in the same package as the candidate. That used to be a scan over+``_dynamic_import_files`` run once per candidate, constructing a ``Path`` per+inner iteration: O(candidates x dynamic_files). It is now a precomputed set of+parent directories plus one hash lookup.++These tests pin the clamp's semantics (which package matches, which does not,+and the repo-root case) so the rewrite cannot have moved them.+"""++from __future__ import annotations++import networkx as nx+import pytest++from repowise.core.analysis.dead_code import DeadCodeAnalyzer, DeadCodeKind++from ._helpers import _build_graph, _old_date++_DETECT_UNREACHABLE_ONLY = {+ "detect_unused_exports": False,+ "detect_unused_internals": False,+ "detect_zombie_packages": False,+ "min_confidence": 0.0,+}+++def _orphan_graph() -> nx.DiGraph:+ """Two packages, one carrying a dynamic edge and one not."""+ return _build_graph(+ nodes={+ # pkg_a: orphan sits in the SAME package as a dynamic edge target+ "pkg_a/orphan.py": {"symbol_count": 5, "symbols": []},+ "pkg_a/dispatcher.py": {"symbol_count": 3, "symbols": []},+ "pkg_a/handler.py": {"is_entry_point": True, "symbol_count": 3, "symbols": []},+ # pkg_b: orphan sits in a package with NO dynamic edge at all+ "pkg_b/orphan.py": {"symbol_count": 5, "symbols": []},+ },+ edges=[("pkg_a/dispatcher.py", "pkg_a/handler.py", {"edge_type": "dynamic"})],+ )+++def _stale_meta(*paths: str) -> dict[str, dict]:+ """Git metadata that scores an unreferenced file at full confidence."""+ return {+ p: {"commit_count_90d": 0, "last_commit_at": _old_date(days=400), "age_days": 500}+ for p in paths+ }+++def _unreachable(report, path: str):+ hits = [+ f for f in report.findings if f.file_path == path and f.kind == DeadCodeKind.UNREACHABLE_FILE+ ]+ assert len(hits) == 1, f"expected exactly one unreachable finding for {path}, got {hits}"+ return hits[0]+++def test_same_package_dynamic_import_clamps_confidence():+ """An orphan beside a dynamically-reached file stays a review candidate."""+ analyzer = DeadCodeAnalyzer(+ _orphan_graph(),+ git_meta_map=_stale_meta("pkg_a/orphan.py", "pkg_b/orphan.py"),+ )+ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))++ finding = _unreachable(report, "pkg_a/orphan.py")+ assert finding.confidence == pytest.approx(0.4)+ assert not finding.safe_to_delete+++def test_other_package_is_not_clamped():+ """The discriminating case: the clamp is per-package, not repo-wide.++ A membership test that dropped the directory would clamp every orphan in+ the repo as soon as one dynamic import existed anywhere.+ """+ analyzer = DeadCodeAnalyzer(+ _orphan_graph(),+ git_meta_map=_stale_meta("pkg_a/orphan.py", "pkg_b/orphan.py"),+ )+ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))++ finding = _unreachable(report, "pkg_b/orphan.py")+ # Untouched for 400 days, in a package with no dynamic edge: unclamped.+ assert finding.confidence == pytest.approx(1.0)+++def test_dynamic_import_dirs_holds_packages_not_files():+ """The lookup key is the parent directory, and only for packages that have one."""+ analyzer = DeadCodeAnalyzer(_orphan_graph(), git_meta_map={})++ assert analyzer._dynamic_import_dirs == {"pkg_a"}+++def test_repo_root_dynamic_import_clamps_other_root_files():+ """A dynamic import at the repo root clamps every other root-level orphan.++ ``Path("main.py").parent`` is ``.``, so the root behaves as one package and+ a single root-level dynamic import caps confidence for all its neighbours.+ Surprising enough to pin: it is the case a future rewrite is most likely to+ get wrong, and bare filenames are real node ids in this codebase.+ """+ graph = _build_graph(+ nodes={+ "loader.py": {"symbol_count": 3, "symbols": []},+ "main.py": {"is_entry_point": True, "symbol_count": 3, "symbols": []},+ "orphan.py": {"symbol_count": 5, "symbols": []},+ "pkg_c/orphan.py": {"symbol_count": 5, "symbols": []},+ },+ edges=[("loader.py", "main.py", {"edge_type": "dynamic"})],+ )+ analyzer = DeadCodeAnalyzer(+ graph, git_meta_map=_stale_meta("orphan.py", "pkg_c/orphan.py")+ )+ assert analyzer._dynamic_import_dirs == {"."}++ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))++ # Root-level neighbour of the dynamic import: clamped.+ assert _unreachable(report, "orphan.py").confidence == pytest.approx(0.4)+ # A real subpackage is unaffected by the root's dynamic import.+ assert _unreachable(report, "pkg_c/orphan.py").confidence == pytest.approx(1.0)+++def test_clamp_reads_the_derived_set_not_the_file_set():+ """The precomputed directory set is what the clamp consults.++ Pins the derived-state contract documented on ``_dynamic_import_dirs``:+ widening ``_dynamic_import_files`` alone must not move a verdict, because+ the lookup no longer walks that collection. This is what fails if the+ per-candidate scan is ever reintroduced.+ """+ graph = _orphan_graph()+ meta = _stale_meta("pkg_a/orphan.py", "pkg_b/orphan.py")++ analyzer = DeadCodeAnalyzer(graph, git_meta_map=meta)+ # A dynamic-import file in pkg_b, registered ONLY in the file set.+ analyzer._dynamic_import_files = analyzer._dynamic_import_files | {"pkg_b/late.py"}++ report = analyzer.analyze(dict(_DETECT_UNREACHABLE_ONLY))++ assert _unreachable(report, "pkg_b/orphan.py").confidence == pytest.approx(1.0)diff --git a/tests/unit/ingestion/test_unparseable_language_sets.py b/tests/unit/ingestion/test_unparseable_language_sets.pynew file mode 100644index 000000000..6cd4ece02--- /dev/null+++ b/tests/unit/ingestion/test_unparseable_language_sets.py@@ -0,0 +1,77 @@+"""The "no AST parser and no structure to extract" language set has one home.++Three modules computed this comprehension verbatim -- the parser's passthrough+skip, the traverser's generated-marker skip, and dead code's non-code+exemption -- and they had already drifted on whether ``unknown`` belonged.+Same failure mode ``core.test_paths`` was created to end, so the same fix: one+spelling on the registry, and these tests keep the call sites pinned to it.+"""++from __future__ import annotations++from repowise.core.analysis.dead_code.constants import _NON_CODE_LANGUAGES+from repowise.core.ingestion.languages.registry import REGISTRY+from repowise.core.ingestion.parser import _PASSTHROUGH_LANGUAGES+from repowise.core.ingestion.traverser import _SKIP_GENERATED_CHECK+++def _reference_set() -> frozenset[str]:+ """The predicate as it read at all three call sites before extraction."""+ return frozenset(+ spec.tag+ for spec in REGISTRY.all_specs()+ if spec.is_passthrough+ and (not spec.is_code or spec.is_infra)+ and spec.tag not in ("openapi", "unknown")+ )+++def test_registry_set_matches_the_original_predicate():+ """Extraction was behaviour-preserving, not a redefinition."""+ assert REGISTRY.unparseable_data_languages() == _reference_set()+++def test_parser_and_traverser_use_the_shared_set():+ assert REGISTRY.unparseable_data_languages() == _PASSTHROUGH_LANGUAGES+ assert REGISTRY.unparseable_data_languages() == _SKIP_GENERATED_CHECK+++def test_dead_code_adds_unknown_and_says_so():+ """Dead code alone treats ``unknown`` as non-code.++ An unidentified language is no evidence of reachability either way, so+ flagging it would be a guess. That is a real difference from the parser's+ question, which is why it is a second named method rather than a third+ copy of the comprehension.+ """+ assert REGISTRY.unparseable_or_unknown_languages() == _NON_CODE_LANGUAGES+ assert REGISTRY.unparseable_data_languages() | {"unknown"} == _NON_CODE_LANGUAGES+++def test_unknown_is_excluded_from_the_base_set():+ """Regression pin: ``unknown`` is itself a passthrough spec.++ Deriving the base set by excluding only ``openapi`` silently pulls+ ``unknown`` in, which would make the parser treat unidentified files as+ passthrough and stop the traverser reading their generated-file markers.+ """+ assert "unknown" not in REGISTRY.unparseable_data_languages()+ assert "unknown" in REGISTRY.unparseable_or_unknown_languages()+++def test_real_data_formats_are_in_the_set():+ """Anchor the set to concrete members so an upstream spec flip is caught."""+ for tag in ("markdown", "json", "yaml", "toml"):+ assert tag in REGISTRY.unparseable_data_languages(), tag+++def test_passthrough_code_languages_are_not_in_the_set():+ """Languages awaiting a grammar are NOT "data".++ clojure/elixir/haskell are ``is_passthrough`` but real code: zero symbols+ there is a gap in us, not a property of the file, so they must stay+ eligible for the analyses this set exempts.+ """+ for tag in ("clojure", "elixir", "haskell"):+ assert tag in REGISTRY.passthrough_languages(), tag+ assert tag not in REGISTRY.unparseable_data_languages(), tag