Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(dead-code): one predicate answers "can anything reach this file" (#1559)

Swati Ahuja5962156dc6df
diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md
index 63ccacdf3..7ddbb40ea 100644
--- a/docs/architecture/ARCHITECTURE.md
+++ b/docs/architecture/ARCHITECTURE.md
@@ -938,7 +938,8 @@ The analyzer runs after `GitIndexer` during init (Step 3.6) and optionally durin
- Files matching `*migrations*`, `*schema*`, `*seed*`
- TypeScript `.d.ts` files
- Files where `is_api_contract == True`
-- Files in `.repowise/dead_code_whitelist.txt`
+- Files named in the `whitelist` key of the analyzer's config argument (an API
+ parameter; no CLI flag or config file populates it today)
- Symbols matching `config.dead_code.dynamic_patterns`
### 8.2 Dead Code in Generation Prompts
@@ -1679,7 +1680,6 @@ dead_code:
- "*Middleware"
- "register_*"
- "on_*"
- whitelist_file: .repowise/dead_code_whitelist.txt
analyze_on_update: true # re-analyze dead code on incremental updates
maintenance:
diff --git a/docs/architecture/deep-dives.md b/docs/architecture/deep-dives.md
index 4d5802813..5d2cdf2d2 100644
--- a/docs/architecture/deep-dives.md
+++ b/docs/architecture/deep-dives.md
@@ -32,16 +32,21 @@ Repowise's dead code analyzer finds these automatically using **pure graph trave
```
For each node in the dependency graph:
- Skip if: external package, non-code language, entry point, test file,
- fixture directory, __init__.py, config file, migration, etc.
+ Skip if: external package, non-code language, test file, fixture directory
- if in_degree(node) == 0:
- This file has zero importers → candidate for dead code
+ if not is_file_reachable(node):
+ Nothing can get to this file → candidate for dead code
```
-`in_degree` is just the count of incoming edges — files that import this one. If nobody imports it and it's not an entry point or test, it's suspicious.
+`is_file_reachable` is the single predicate for "can anything get to this file",
+shared with the repo-overview assembler so the two cannot disagree. It rescues
+entry points, API contracts, never-flag paths (`__init__.py`, config files,
+migrations, shell scripts), bundler-alias shims and the package-granular
+languages, and otherwise asks whether any *dependency* edge points at the file.
+That last part is not a raw `in_degree`: a co-change edge ("these two files were
+committed together"), a file's own symbols and a self-import are all excluded.
-**Why in_degree alone isn't enough:**
+**Why an import edge alone isn't enough:**
Consider `plugin_auth.py`. Nothing imports it directly because the plugin framework loads it dynamically at runtime via `importlib.import_module()`. The graph doesn't capture dynamic imports as edges because they don't appear in the AST as static import statements.
diff --git a/packages/core/src/repowise/core/analysis/dead_code/__init__.py b/packages/core/src/repowise/core/analysis/dead_code/__init__.py
index 6b3028988..a41ced2ef 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/__init__.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/__init__.py
@@ -15,8 +15,12 @@ Internal layout (Phase 1 refactor):
- ``analyzer.py`` — ``DeadCodeAnalyzer`` class + four detection methods.
- ``models.py`` — public dataclasses + ``DeadCodeKind`` enum.
-- ``constants.py`` — never-flag globs, framework decorators, fixture
- path segments, non-code languages.
+- ``constants.py`` — never-flag globs and ``never_flag_match`` over
+ them, framework decorators, fixture path segments,
+ non-code languages.
+- ``file_reachability.py`` — ``is_file_reachable``, the one answer to "can
+ anything reach this file", shared with the
+ repo-overview assembler.
- ``dynamic_markers.py`` — ``_DYNAMIC_IMPORT_MARKERS`` source-text dict
+ ``find_dynamic_import_files()`` scanner.
Phase 2 (A1/A2) work lands here.
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
index 585255b64..40fc8c97d 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
@@ -12,10 +12,9 @@ this package.
from __future__ import annotations
import fnmatch
-import os
import re
+from collections.abc import Set as AbstractSet
from datetime import UTC, datetime
-from functools import lru_cache
from pathlib import Path
from typing import Any
@@ -26,10 +25,10 @@ from .constants import (
_DEFAULT_DYNAMIC_PATTERNS,
_FRAMEWORK_DECORATOR_SUFFIXES,
_FRAMEWORK_DECORATORS,
- _NEVER_FLAG_PATTERNS,
_NEVER_PACKAGE_DIRS,
_NON_CODE_LANGUAGES,
_is_fixture_path,
+ never_flag_match,
)
from .contract_methods import is_contract_method
from .dynamic_markers import (
@@ -38,6 +37,7 @@ from .dynamic_markers import (
read_source_text,
)
from .file_reachability import (
+ PackageFileMap,
ReachabilityRescues,
build_package_file_map,
is_file_reachable,
@@ -593,94 +593,6 @@ def _is_synthetic_node(node: str) -> bool:
return node.startswith("external:") or node.startswith("framework:")
-@lru_cache(maxsize=8)
-def _never_flag_regex(patterns: tuple[str, ...]) -> re.Pattern[str]:
- """Compile *patterns* into one alternation regex equivalent to fnmatch.
-
- ``fnmatch.fnmatch(path, p)`` normcases both sides and matches the
- translated glob; doing that per (node x pattern) costs ~540 fnmatch
- calls per node and dominated the whole dead-code pass (measured: 50s of
- a 51s analyze() on a 13k-node graph, mostly Windows ``normcase``).
- One pre-normcased alternation keeps the exact same match semantics at
- one regex match per node.
- """
- return re.compile("|".join(fnmatch.translate(os.path.normcase(p)) for p in patterns))
-
-
-@lru_cache(maxsize=8)
-def _never_flag_suffix_index(
- patterns: tuple[str, ...],
-) -> tuple[re.Pattern[str] | None, dict[str, re.Pattern[str]], tuple[int, ...]]:
- """Split *patterns* into suffix-keyed buckets that can be skipped wholesale.
-
- ``_never_flag_regex`` puts all 579 patterns in one alternation, and
- ``.match()`` tries every branch at position 0 — most of them beginning
- ``.*``, so each branch scans the path. Measured cold on a 63k-node repo
- that is 206 microseconds per unique path and 12.8s of an 18.1s dead-code
- analysis, the single largest cost in the pass.
-
- The filter is sound because ``fnmatch.translate`` ends *each* alternative
- with ``\\Z`` and the join keeps that per-branch (``(?s:A)\\Z|(?s:B)\\Z``),
- so every branch has to match the whole string. A pattern whose text after
- its last ``*`` is the literal ``S`` can therefore only match a path ending
- in ``S``: testing it against a path that does not end in ``S`` is wasted
- work, never a dropped match. Patterns ending in ``*`` constrain nothing at
- the tail and stay in one always-tried group.
-
- Within a bucket the branches keep their original translated form, so the
- atomic groups ``fnmatch`` emits for interior ``*literal`` runs — which
- commit to the first occurrence and never retry a later one — behave
- exactly as they did in the single alternation. Alternation order does not
- matter to a boolean "did anything match".
-
- Returns ``(always_tried_regex_or_None, {suffix: regex}, suffix_lengths)``.
- """
- always: list[str] = []
- by_suffix: dict[str, list[str]] = {}
- for pattern in patterns:
- norm = os.path.normcase(pattern)
- # No pattern in the set uses ``?`` or a character class (checked by
- # test_never_flag_suffix_index_covers_pattern_shapes), so the text
- # after the last ``*`` is a plain literal tail.
- suffix = norm.rsplit("*", 1)[-1]
- translated = fnmatch.translate(norm)
- if suffix:
- by_suffix.setdefault(suffix, []).append(translated)
- else:
- always.append(translated)
- compiled = {s: re.compile("|".join(v)) for s, v in by_suffix.items()}
- return (
- re.compile("|".join(always)) if always else None,
- compiled,
- tuple(sorted({len(s) for s in compiled})),
- )
-
-
-@lru_cache(maxsize=131072)
-def _never_flag_regex_match(path: str) -> bool:
- """Memoized never-flag match for the default pattern set.
-
- Equivalent to ``_never_flag_regex(_NEVER_FLAG_PATTERNS).match(...)``, and
- pinned to it path-for-path by ``test_never_flag_regex.py``. Pure function
- of *path*: the pattern set is a module constant, so process-wide
- memoization is sound. The detector passes ask about the same node ids
- repeatedly (every graph node is checked by the unreachable-files and the
- unused-exports passes), which is what the cache is for; this function is
- what the *first* ask of each id costs.
- """
- norm = os.path.normcase(path)
- always, by_suffix, suffix_lengths = _never_flag_suffix_index(_NEVER_FLAG_PATTERNS)
- if always is not None and always.match(norm):
- return True
- for length in suffix_lengths:
- if length > len(norm):
- break
- bucket = by_suffix.get(norm[-length:])
- if bucket is not None and bucket.match(norm):
- return True
- return False
-
-
class DeadCodeAnalyzer:
"""Detects unreachable files, unused exports, unused internals, and
zombie packages using the dependency graph and git metadata.
@@ -745,20 +657,23 @@ class DeadCodeAnalyzer:
self._ts_export_aliases: dict[str, dict[str, str]] = _find_ts_export_aliases(
parsed_files or {}, source_map
)
- # Lazily-built rescue state for the shared reachability predicate: the
- # package-directory maps for Go / JVM / C-C++ plus the bundler-alias
- # targets found above. Built on first use so a graph that never
- # reaches the unreachable-files pass never pays for it.
- self._rescues: ReachabilityRescues | None = None
-
- def _reachability_rescues(self) -> ReachabilityRescues:
- """Return the cached rescue state, building it on first use."""
- if self._rescues is None:
- self._rescues = ReachabilityRescues(
- bundler_alias_targets=frozenset(self._bundler_alias_targets),
- package_files=build_package_file_map(self.graph),
- )
- return self._rescues
+ # Lazily-built package-directory maps for Go / JVM / C-C++, the one
+ # piece of the rescue state that costs a graph scan. Built on first use
+ # so a graph that never reaches the unreachable-files pass never pays
+ # for it. Cached here rather than on the ``ReachabilityRescues`` object
+ # because the whitelist arrives per ``analyze()`` call and the map does
+ # not.
+ self._package_files: PackageFileMap | None = None
+
+ def _reachability_rescues(self, whitelist: AbstractSet[str]) -> ReachabilityRescues:
+ """Assemble the rescue state the shared predicate reads."""
+ if self._package_files is None:
+ self._package_files = build_package_file_map(self.graph)
+ return ReachabilityRescues(
+ bundler_alias_targets=frozenset(self._bundler_alias_targets),
+ whitelist=frozenset(whitelist),
+ package_files=self._package_files,
+ )
def analyze(
self,
@@ -835,31 +750,34 @@ class DeadCodeAnalyzer:
dynamic_patterns: tuple[str, ...],
whitelist: set[str],
) -> list[DeadCodeFindingData]:
- """Detect files with in_degree == 0 that are not entry points, tests, or config."""
+ """Detect files nothing can reach that are not tests, fixtures, or config."""
findings = []
+ # One object for the whole pass rather than one per node: the rescues
+ # do not vary by candidate.
+ rescues = self._reachability_rescues(whitelist)
+
for node in self.graph.nodes():
if _is_synthetic_node(str(node)):
continue
node_data = self.graph.nodes[node]
+ # The three skips left here are *scoping*, not reachability: a test
+ # file is perfectly reachable, it is just not something this pass
+ # reports. Everything that answers "can anything get to this file"
+ # — entry points, API contracts, never-flag globs, the whitelist,
+ # barrels, bundler-alias shims and the package-granular languages
+ # (Go / JVM / C-C++) — lives in the predicate, which the overview
+ # assembler calls with the same state so the two cannot disagree.
+ # See :mod:`file_reachability`.
if node_data.get("language", "unknown") in _NON_CODE_LANGUAGES:
continue
- if node_data.get("is_entry_point", False):
- continue
if node_data.get("is_test", False):
continue
if _is_fixture_path(str(node)):
continue
- if self._should_never_flag(str(node), whitelist):
- continue
- # Barrels, API contracts, bundler-alias shims and the
- # package-granular languages (Go / JVM / C-C++) are all rescued
- # inside the shared predicate, which the overview assembler calls
- # with the same state so the two passes cannot disagree about what
- # "reachable" means. See :mod:`file_reachability`.
- if is_file_reachable(str(node), self.graph, self._reachability_rescues()):
+ if is_file_reachable(str(node), self.graph, rescues):
continue
finding = self._make_unreachable_finding(str(node), node_data, dynamic_patterns)
@@ -1036,9 +954,14 @@ class DeadCodeAnalyzer:
# Framework-instantiated files (Spring stereotypes, JAX-RS
# resources, Quarkus components, Spring Data repos, …) have
# no source-level caller; the runtime constructs them via
- # classpath scanning. Mirror the entry-point skip the
- # ``_detect_unreachable_files`` pass already does so an
- # ``@RestController`` class isn't reported as unused.
+ # classpath scanning, so an ``@RestController`` class must not be
+ # reported as an unused export.
+ #
+ # Deliberately not ``is_file_reachable``, which the sibling
+ # unreachable-files pass uses: this pass asks about *symbols*, and
+ # the predicate's barrel rescue is scoped to files on purpose — a
+ # genuine symbol defined in a barrel nobody imports should still be
+ # flagged. See ``BARREL_FILENAMES``'s scope note.
if node_data.get("is_entry_point", False):
continue
if node_data.get("is_test", False):
@@ -1638,10 +1561,16 @@ class DeadCodeAnalyzer:
# ------------------------------------------------------------------
def _should_never_flag(self, path: str, whitelist: set[str]) -> bool:
- """Return True if path should never be flagged as dead."""
+ """Return True if path should never be flagged as dead.
+
+ Used by the unused-export, unused-internal and zombie-package passes.
+ The unreachable-files pass does not call it: every limb below is also
+ asked by :func:`is_file_reachable`, which that pass calls anyway, and
+ two spellings of one rule is what this phase exists to remove.
+ """
if path in whitelist:
return True
- if _never_flag_regex_match(path):
+ if never_flag_match(path):
return True
# Workspace-driven never-flag — set by language warmups that read
# the build manifest (Gradle non-``main`` source sets, Cargo
diff --git a/packages/core/src/repowise/core/analysis/dead_code/constants.py b/packages/core/src/repowise/core/analysis/dead_code/constants.py
index 65f022237..2e0b9e859 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/constants.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/constants.py
@@ -3,10 +3,21 @@
These tuples / frozensets shape what the analyzer treats as "always
alive" (framework decorators, never-flag path globs) and where to skip
entirely (test fixture directories, non-code languages).
+
+``never_flag_match`` lives here rather than on the analyzer because the
+pattern list it reads is here and because it is a pure function of a path:
+:func:`~.file_reachability.is_file_reachable` needs it, and a matcher that
+only the analyzer could reach was the reason the two callers of that
+predicate answered "is this file reachable?" differently.
"""
from __future__ import annotations
+import fnmatch
+import os
+import re
+from functools import lru_cache
+
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
# Non-code languages that should never be flagged as dead code.
@@ -1074,6 +1085,94 @@ _TS_JSX_NAMESPACE_TYPES: frozenset[str] = frozenset(
)
+@lru_cache(maxsize=8)
+def _never_flag_regex(patterns: tuple[str, ...]) -> re.Pattern[str]:
+ """Compile *patterns* into one alternation regex equivalent to fnmatch.
+
+ ``fnmatch.fnmatch(path, p)`` normcases both sides and matches the
+ translated glob; doing that per (node x pattern) costs ~540 fnmatch
+ calls per node and dominated the whole dead-code pass (measured: 50s of
+ a 51s analyze() on a 13k-node graph, mostly Windows ``normcase``).
+ One pre-normcased alternation keeps the exact same match semantics at
+ one regex match per node.
+ """
+ return re.compile("|".join(fnmatch.translate(os.path.normcase(p)) for p in patterns))
+
+
+@lru_cache(maxsize=8)
+def _never_flag_suffix_index(
+ patterns: tuple[str, ...],
+) -> tuple[re.Pattern[str] | None, dict[str, re.Pattern[str]], tuple[int, ...]]:
+ """Split *patterns* into suffix-keyed buckets that can be skipped wholesale.
+
+ ``_never_flag_regex`` puts all 579 patterns in one alternation, and
+ ``.match()`` tries every branch at position 0 — most of them beginning
+ ``.*``, so each branch scans the path. Measured cold on a 63k-node repo
+ that is 206 microseconds per unique path and 12.8s of an 18.1s dead-code
+ analysis, the single largest cost in the pass.
+
+ The filter is sound because ``fnmatch.translate`` ends *each* alternative
+ with ``\\Z`` and the join keeps that per-branch (``(?s:A)\\Z|(?s:B)\\Z``),
+ so every branch has to match the whole string. A pattern whose text after
+ its last ``*`` is the literal ``S`` can therefore only match a path ending
+ in ``S``: testing it against a path that does not end in ``S`` is wasted
+ work, never a dropped match. Patterns ending in ``*`` constrain nothing at
+ the tail and stay in one always-tried group.
+
+ Within a bucket the branches keep their original translated form, so the
+ atomic groups ``fnmatch`` emits for interior ``*literal`` runs — which
+ commit to the first occurrence and never retry a later one — behave
+ exactly as they did in the single alternation. Alternation order does not
+ matter to a boolean "did anything match".
+
+ Returns ``(always_tried_regex_or_None, {suffix: regex}, suffix_lengths)``.
+ """
+ always: list[str] = []
+ by_suffix: dict[str, list[str]] = {}
+ for pattern in patterns:
+ norm = os.path.normcase(pattern)
+ # No pattern in the set uses ``?`` or a character class (checked by
+ # test_never_flag_suffix_index_covers_pattern_shapes), so the text
+ # after the last ``*`` is a plain literal tail.
+ suffix = norm.rsplit("*", 1)[-1]
+ translated = fnmatch.translate(norm)
+ if suffix:
+ by_suffix.setdefault(suffix, []).append(translated)
+ else:
+ always.append(translated)
+ compiled = {s: re.compile("|".join(v)) for s, v in by_suffix.items()}
+ return (
+ re.compile("|".join(always)) if always else None,
+ compiled,
+ tuple(sorted({len(s) for s in compiled})),
+ )
+
+
+@lru_cache(maxsize=131072)
+def never_flag_match(path: str) -> bool:
+ """Memoized never-flag match for the default pattern set.
+
+ Equivalent to ``_never_flag_regex(_NEVER_FLAG_PATTERNS).match(...)``, and
+ pinned to it path-for-path by ``test_never_flag_regex.py``. Pure function
+ of *path*: the pattern set is a module constant, so process-wide
+ memoization is sound. The detector passes ask about the same node ids
+ repeatedly (every graph node is checked by the unreachable-files and the
+ unused-exports passes), which is what the cache is for; this function is
+ what the *first* ask of each id costs.
+ """
+ norm = os.path.normcase(path)
+ always, by_suffix, suffix_lengths = _never_flag_suffix_index(_NEVER_FLAG_PATTERNS)
+ if always is not None and always.match(norm):
+ return True
+ for length in suffix_lengths:
+ if length > len(norm):
+ break
+ bucket = by_suffix.get(norm[-length:])
+ if bucket is not None and bucket.match(norm):
+ return True
+ return False
+
+
def _is_fixture_path(path: str) -> bool:
"""Return True if path is under a test fixture / sample data directory."""
path_lower = path.lower().replace("\\", "/")
diff --git a/packages/core/src/repowise/core/analysis/dead_code/file_reachability.py b/packages/core/src/repowise/core/analysis/dead_code/file_reachability.py
index 7e062b1b6..137dbfece 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/file_reachability.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/file_reachability.py
@@ -14,11 +14,13 @@ set and carried a list of the languages where the analyzer was known to be
kinder, standing in for per-language helpers it could not reach. This module
is that predicate with its state passed in, so both callers get one answer.
-Leaf in its own imports: the three per-language reachability helpers and
-:mod:`repowise.core.ids`, and nothing else from either subsystem. It does not
-reach the analyzer, and in particular it holds none of the analysis-time state
-that made the analyzer's version uncallable, which is the coupling that
-mattered.
+Leaf in its own imports: the three per-language reachability helpers,
+:mod:`constants` for the never-flag globs, and :mod:`repowise.core.ids`.
+Nothing else from either subsystem. It does not reach the analyzer, and in
+particular it holds none of the analysis-time state that made the analyzer's
+version uncallable, which is the coupling that mattered — ``constants`` is a
+sibling of the analyzer, not the analyzer, and its only non-stdlib import is
+the language registry.
It is not a free import, though, and the difference is worth stating rather
than implying. ``dead_code/__init__`` eagerly re-exports ``DeadCodeAnalyzer``,
@@ -42,6 +44,7 @@ from typing import Any
from repowise.core.ids import SYMBOL_SEP, file_path_of, is_external
+from .constants import never_flag_match
from .cpp_reachability import build_cpp_package_files, is_cpp_file_reachable, is_cpp_path
from .go_reachability import build_go_package_files, is_go_file_reachable
from .jvm_reachability import build_jvm_package_files, is_jvm_file_reachable
@@ -115,26 +118,33 @@ class ReachabilityRescues:
files in a live package carry no file-level in-edge at all; a caller
without the map would call an entire language unreachable. ``None`` means
"not checked", and the predicate resolves an unchecked package-granular
- file to reachable. That is the deliberate asymmetry, and it runs one way:
- a missed drop leaves one wrong flow on a page, an over-eager drop silently
- empties the section.
+ file to reachable — the forgiving direction, since a missed drop leaves
+ one wrong flow on a page while an over-eager drop silently empties the
+ section. Both production callers now supply the map, so that default is a
+ safety net for a caller that cannot, not a stance either of them takes.
+
+ ``whitelist`` is the path-exact exemption list from ``analyze()``'s config
+ argument. It is here so the analyzer's unreachable-files pass has *no*
+ reachability limb left of its own; no CLI path populates it today, so it is
+ empty in every shipped run. The assembler has no config to read and passes
+ none, which makes it narrower by exactly the paths an API caller listed.
``bundler_alias_targets`` is a narrow allowlist — a handful of shim paths
per repo — and defaults to empty. A caller that omits it is stricter than
the analyzer for exactly those paths, and no wider.
- Ceiling, deliberate: the analyzer applies two further rescues this
- predicate does not, the never-flag glob set and the user whitelist, both
- of which live outside the graph (``analyzer._never_flag_regex_match``, and
- dead-code config). Same for ``bundler_alias_targets``, which needs a
- source scan of the bundler configs. The upgrade path is to thread that
- state to the second caller — for the glob set, by moving the matcher down
- to :mod:`constants` where its pattern list already lives — not to widen
- the defaults into a blanket rescue. A blanket rescue would answer
- "reachable" for every unimported TS/JS file and stop being a predicate.
+ Ceiling, deliberate: ``bundler_alias_targets`` is still caller-supplied
+ because finding it needs a source scan of the bundler configs, which this
+ module has no business doing. The never-flag glob set used to be listed
+ here too; it is a pure function of a path, so it moved into
+ :mod:`constants` and the predicate now applies it to every caller. What
+ must *not* happen is widening the defaults into a blanket rescue: that
+ would answer "reachable" for every unimported TS/JS file and stop this
+ being a predicate.
"""
bundler_alias_targets: AbstractSet[str] = frozenset()
+ whitelist: AbstractSet[str] = frozenset()
package_files: PackageFileMap | None = None
@@ -270,6 +280,14 @@ def is_file_reachable(
# manifest (Gradle non-``main`` source sets, Cargo ``[[example]]`` targets).
if node_data.get("is_never_flag", False):
return True
+ # Convention globs: shell scripts invoked by name, framework file-system
+ # routes, migration scripts. Reached from outside the import graph the same
+ # way an entry point is, so this belongs to the question, not to the
+ # analyzer that used to own the matcher.
+ if never_flag_match(path):
+ return True
+ if path in rescues.whitelist:
+ return True
if PurePosixPath(path).name in BARREL_FILENAMES:
return True
# Bundler ``resolve.alias`` targets are reached through the aliased package
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.py
index 6be3ee90c..c75d835bd 100644
--- a/packages/core/src/repowise/core/generation/context/assembler.py
+++ b/packages/core/src/repowise/core/generation/context/assembler.py
@@ -11,6 +11,7 @@ import structlog
from repowise.core.analysis.dead_code.file_reachability import (
ReachabilityRescues,
+ build_package_file_map,
file_dependency_neighbors,
is_file_reachable,
)
@@ -213,9 +214,10 @@ def _flow_entry_is_reachable(flow: Any, graph: Any, rescues: ReachabilityRescues
the two cannot drift. This function only resolves a flow to a file path.
An unresolvable entry point stays. That is the one rescue local to this
- caller, and it runs in the forgiving direction for the reason the caller's
- own comment gives: a missed drop leaves one wrong flow on the page, while
- an over-eager drop silently empties the section.
+ caller, and it is the only forgiving-by-default limb left now that the
+ caller supplies the package map: a missed drop leaves one wrong flow on the
+ page, while an over-eager drop empties the section. Same asymmetry
+ ``ReachabilityRescues`` records for ``package_files``.
"""
node = graph.nodes.get(flow.entry_point_id, {})
path = node.get("file_path") or file_path_of(flow.entry_point_id) or ""
@@ -775,20 +777,31 @@ class ContextAssembler:
key=lambda f: (-f.entry_point_score, f.entry_point_id),
)
flow_graph = graph_builder.graph()
- # No package map, deliberately: an unchecked Go / JVM /
- # C-C++ file answers "reachable", so this section keeps
- # being more forgiving than the dead-code pass for exactly
- # those languages. Measured on test-repos/, supplying
- # ``build_package_file_map(flow_graph)`` here would flip
- # 0-43% of their files to unreachable (43% on
- # nlohmann-json, 29% on openclaw, 27% on Crow), and an
- # over-eager drop empties the section. Ceiling, not
- # oversight: closing it is one argument, and wants its own
- # measurement of what the overview actually loses.
+ # The package map is supplied, so a Go / JVM / C-C++ file
+ # is answered for rather than assumed reachable. It used to
+ # be withheld because the file-level cost looked large, but
+ # the file-level cost is the wrong quantity: a flow's entry
+ # point is a function in a file something imports, so the
+ # flipped files are mostly not flow entries. Measured over
+ # the indexed repositories by tracing flows for real, the
+ # map costs **1 flow of 1,416 kept**, on one repository,
+ # changes no rendered top five and empties no section.
+ #
+ # Residual risk this does not remove, and no test can: on a
+ # repository where every traced flow starts in a package
+ # with no live sibling, this empties the section rather
+ # than showing wrong flows. The corpus holds no such
+ # repository, but "measured on 42" is not "cannot happen",
+ # which is what the log below is for.
+ #
+ # Built once, and only when there is something to filter:
+ # three passes over the node ids of a graph that can reach
+ # 175k nodes.
+ flow_rescues = ReachabilityRescues(
+ package_files=build_package_file_map(flow_graph) if ranked else None
+ )
kept = [
- f
- for f in ranked
- if _flow_entry_is_reachable(f, flow_graph, ReachabilityRescues())
+ f for f in ranked if _flow_entry_is_reachable(f, flow_graph, flow_rescues)
]
if len(kept) != len(ranked):
# The section disappears when this empties it, and a
diff --git a/tests/unit/analysis/test_file_reachability.py b/tests/unit/analysis/test_file_reachability.py
index 3d9d7e372..7426af087 100644
--- a/tests/unit/analysis/test_file_reachability.py
+++ b/tests/unit/analysis/test_file_reachability.py
@@ -257,17 +257,17 @@ def test_the_predicate_is_the_only_thing_deciding_reachability():
What that set means, precisely: the analyzer's remaining skips are
candidacy rules, "is this a file we would ever report", and the assembler
- deliberately wants none of them applied to an execution flow. Note the
- split is not clean the other way — ``is_entry_point``, ``is_never_flag``
- and the ``__init__.py`` barrel are checked in *both* places, harmlessly,
- since every one of them is idempotent. And ``_should_never_flag`` is a
- reachability rule rather than a candidacy one (``*.sh`` is exempt because
- CI invokes it by name), which is exactly why ``ReachabilityRescues``
- records its glob set as a rescue the second caller is missing.
-
- Coverage ceiling: the fixture exercises the non-code-language,
- ``is_entry_point`` and ``is_test`` skips. ``_is_synthetic_node``,
- ``_is_fixture_path`` and the never-flag globs are not represented here.
+ deliberately wants none of them applied to an execution flow. The split is
+ now clean both ways: nothing reachability-shaped is checked twice.
+ ``is_entry_point``, ``is_never_flag``, the never-flag globs, the whitelist
+ and the ``__init__.py`` barrel are asked once, by the predicate. The globs
+ in particular are a reachability rule rather than a candidacy one — ``*.sh``
+ is exempt because CI invokes it by name — which is why they moved.
+
+ Coverage ceiling: the fixture exercises the two analyzer skips that remain
+ reachable here, non-code-language and ``is_test``, plus the entry-point,
+ never-flag and whitelist rescues now inside the predicate.
+ ``_is_synthetic_node`` and ``_is_fixture_path`` are not represented.
"""
from repowise.core.analysis.dead_code import DeadCodeAnalyzer
from repowise.core.analysis.dead_code.constants import _DEFAULT_DYNAMIC_PATTERNS
@@ -279,6 +279,7 @@ def test_the_predicate_is_the_only_thing_deciding_reachability():
"src/orphan.py": {},
"src/main.py": {"is_entry_point": True},
"src/pkg/__init__.py": {},
+ "scripts/deploy.sh": {"language": "shell"},
"tests/test_orphan.py": {"is_test": True},
"docs/notes.md": {"language": "markdown"},
},
@@ -289,16 +290,22 @@ def test_the_predicate_is_the_only_thing_deciding_reachability():
)
analyzer = DeadCodeAnalyzer(g)
- rescues = analyzer._reachability_rescues()
+ whitelist = {"src/orphan.py"}
+ rescues = analyzer._reachability_rescues(whitelist)
flagged = {
- f.file_path for f in analyzer._detect_unreachable_files(_DEFAULT_DYNAMIC_PATTERNS, set())
+ f.file_path
+ for f in analyzer._detect_unreachable_files(_DEFAULT_DYNAMIC_PATTERNS, whitelist)
}
unreachable = {n for n in g.nodes() if not is_file_reachable(str(n), g, rescues)}
- assert flagged == {"src/caller.py", "src/orphan.py"}
+ assert flagged == {"src/caller.py"}
assert "src/service.py" not in flagged
assert "src/main.py" not in flagged
assert "src/pkg/__init__.py" not in flagged
+ # The whitelist and the never-flag globs now reach the predicate, so they
+ # rescue the file rather than being applied beside it.
+ assert "src/orphan.py" not in unreachable
+ assert "scripts/deploy.sh" not in unreachable
# The gap between the two is candidacy and nothing else. A markdown file
# and a test file are both unreachable and neither is dead code, and the
diff --git a/tests/unit/dead_code/test_never_flag_regex.py b/tests/unit/dead_code/test_never_flag_regex.py
index 3bf555eff..07e53d068 100644
--- a/tests/unit/dead_code/test_never_flag_regex.py
+++ b/tests/unit/dead_code/test_never_flag_regex.py
@@ -8,11 +8,12 @@ import os
import networkx as nx
import pytest
-from repowise.core.analysis.dead_code.analyzer import (
- DeadCodeAnalyzer,
+from repowise.core.analysis.dead_code.analyzer import DeadCodeAnalyzer
+from repowise.core.analysis.dead_code.constants import (
+ _NEVER_FLAG_PATTERNS,
_never_flag_regex,
+ never_flag_match,
)
-from repowise.core.analysis.dead_code.constants import _NEVER_FLAG_PATTERNS
# Positives derived from the glob list plus negatives that brush close to it.
_PROBE_PATHS = [
@@ -83,12 +84,10 @@ class TestMemoizedMatch:
[*_PROBE_PATHS, "pkg/a.go::Handler", "cmd/main_test.go::TestX", "app/page.tsx::Page"],
)
def test_memoized_equals_direct(self, path):
- from repowise.core.analysis.dead_code.analyzer import _never_flag_regex_match
-
direct = bool(_never_flag_regex(_NEVER_FLAG_PATTERNS).match(os.path.normcase(path)))
- assert _never_flag_regex_match(path) == direct, path
+ assert never_flag_match(path) == direct, path
# Second call exercises the cached branch.
- assert _never_flag_regex_match(path) == direct, path
+ assert never_flag_match(path) == direct, path
class TestShouldNeverFlag:
@@ -115,7 +114,7 @@ class TestShouldNeverFlag:
class TestSuffixIndexEquivalence:
- r"""``_never_flag_regex_match`` buckets the patterns by the literal text
+ r"""``never_flag_match`` buckets the patterns by the literal text
after their last ``*`` and only tests the buckets a path's tail can reach.
That is sound only because ``fnmatch.translate`` end-anchors every
alternative, so these pin both the equivalence and the assumption.
@@ -163,19 +162,15 @@ class TestSuffixIndexEquivalence:
],
)
def test_matches_the_single_alternation(self, path):
- from repowise.core.analysis.dead_code.analyzer import _never_flag_regex_match
-
expected = bool(_never_flag_regex(_NEVER_FLAG_PATTERNS).match(os.path.normcase(path)))
- assert _never_flag_regex_match(path) == expected, path
+ assert never_flag_match(path) == expected, path
def test_synthesized_probe_per_pattern_agrees(self):
"""One concrete path per glob, plus near-miss variants around it, run
through the bucketed matcher rather than the raw regex."""
- from repowise.core.analysis.dead_code.analyzer import _never_flag_regex_match
-
regex = _never_flag_regex(_NEVER_FLAG_PATTERNS)
for pat in _NEVER_FLAG_PATTERNS:
base = pat.replace("**", "x").replace("*", "x")
for probe in (base, base.upper(), f"prefix/{base}", f"{base}/trailing.py"):
expected = bool(regex.match(os.path.normcase(probe)))
- assert _never_flag_regex_match(probe) == expected, (pat, probe)
+ assert never_flag_match(probe) == expected, (pat, probe)
diff --git a/tests/unit/generation/test_overview_execution_flows.py b/tests/unit/generation/test_overview_execution_flows.py
index 8dcc9f560..7a0e52e10 100644
--- a/tests/unit/generation/test_overview_execution_flows.py
+++ b/tests/unit/generation/test_overview_execution_flows.py
@@ -151,6 +151,33 @@ def test_conventional_entry_point_survives_having_no_importers(
assert [f["entry_point"] for f in ctx.execution_flows] == ["src/main.py::f"]
+ "path",
+ [
+ "src/features/api/index.ts", # barrel: reached by the names it forwards
+ "pkg/sub/__init__.py",
+ "scripts/release.sh", # never-flag glob: CI invokes it by name
+ "app/dashboard/page.tsx", # never-flag glob: a file-system route
+ ],
+)
+def test_filter_bails_out_where_dead_code_is_kinder(sample_config, sample_repo_structure, path):
+ """The analyzer rescues these from unreachable_file, so the flow stays.
+
+ The barrels the shared predicate always rescued. The never-flag globs it
+ did not: the matcher used to live on the analyzer, so this caller could
+ not ask and dropped a flow the dead-code pass would never have reported.
+ """
+ graph = _graph_with({path: {"is_entry_point": False}}, imports=[])
+ flows = [_flow(f"{path}::f", 0.95, ["a", "b"])]
+
+ assembler = ContextAssembler(sample_config)
+ ctx = assembler.assemble_repo_overview(
+ sample_repo_structure, {}, [], {}, graph_builder=_Builder(flows, graph)
+ )
+
+ assert [f["entry_point"] for f in ctx.execution_flows] == [f"{path}::f"]
+
+
@pytest.mark.parametrize(
"path",
[
@@ -159,19 +186,40 @@ def test_conventional_entry_point_survives_having_no_importers(
"src/Queue.kt",
"src/engine/queue.cpp",
"include/engine/queue.h",
- "src/features/api/index.ts", # barrel: reached by the names it forwards
- "pkg/sub/__init__.py",
],
)
-def test_filter_bails_out_where_dead_code_is_kinder(sample_config, sample_repo_structure, path):
- """The analyzer rescues these from unreachable_file, so the flow stays.
+def test_package_granular_languages_are_answered_not_assumed(
+ sample_config, sample_repo_structure, path
+):
+ """This caller supplies the package map, so it gets a real answer.
- The barrels are rescued by the shared predicate outright. The
- package-granular languages are rescued because this caller passes no
- package map, which the predicate reads as "not checked" and answers
- reachable — the forgiving direction, chosen at the call site rather than
- falling out of what state happened to be available. Over-dropping empties
- the section silently, which is the failure being avoided.
+ It used to pass ``package_files=None``, which the predicate reads as "not
+ checked" and resolves to reachable, so a Go / JVM / C-C++ file with no
+ package sibling and no importer kept its flow while the dead-code pass
+ reported the same file. The sibling case below is the other half: the map
+ rescues a real package member rather than blanket-dropping the language.
+ """
+ graph = _graph_with({path: {"is_entry_point": False}}, imports=[])
+ flows = [_flow(f"{path}::f", 0.95, ["a", "b"])]
+
+ assembler = ContextAssembler(sample_config)
+ ctx = assembler.assemble_repo_overview(
+ sample_repo_structure, {}, [], {}, graph_builder=_Builder(flows, graph)
+ )
+
+ assert ctx.execution_flows == []
+
+
[email protected]("path", ["cmd/app/main.go", "internal/api/service.pb.go"])
+def test_never_flag_wins_over_the_package_map(sample_config, sample_repo_structure, path):
+ """Precedence, and it is the one thing these two changes could get wrong.
+
+ Both arrived in the same change: the never-flag globs moved into the
+ predicate, and this caller started supplying the package map. A generated
+ ``.pb.go`` in a package with no live sibling is answered "unreachable" by
+ the map and "exempt" by the globs, so the order of those two checks decides
+ the flow. The globs run first, matching the analyzer, which has always
+ treated a never-flag path as reached from outside the graph.
"""
graph = _graph_with({path: {"is_entry_point": False}}, imports=[])
flows = [_flow(f"{path}::f", 0.95, ["a", "b"])]
@@ -184,6 +232,31 @@ def test_filter_bails_out_where_dead_code_is_kinder(sample_config, sample_repo_s
assert [f["entry_point"] for f in ctx.execution_flows] == [f"{path}::f"]
+def test_a_live_package_sibling_still_rescues_the_flow(sample_config, sample_repo_structure):
+ """The package map is granularity, not a blanket drop.
+
+ ``queue.go`` has no importer of its own, but ``server.go`` beside it in the
+ same package does, and a Go import names the package. So the flow stays —
+ which is the whole reason the map exists rather than a file-level rule.
+ """
+ graph = _graph_with(
+ {
+ "internal/scheduler/queue.go": {"is_entry_point": False},
+ "internal/scheduler/server.go": {"is_entry_point": False},
+ "cmd/app/main.go": {"is_entry_point": True},
+ },
+ imports=[("cmd/app/main.go", "internal/scheduler/server.go")],
+ )
+ flows = [_flow("internal/scheduler/queue.go::f", 0.95, ["a", "b"])]
+
+ assembler = ContextAssembler(sample_config)
+ ctx = assembler.assemble_repo_overview(
+ sample_repo_structure, {}, [], {}, graph_builder=_Builder(flows, graph)
+ )
+
+ assert [f["entry_point"] for f in ctx.execution_flows] == ["internal/scheduler/queue.go::f"]
+
+
@pytest.mark.parametrize("attr", ["is_api_contract", "is_never_flag"])
def test_flow_survives_the_rescues_the_analyzer_already_applied(
sample_config, sample_repo_structure, attr