Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(dead-code): stop reporting symbols a docs build or an API dump names (#1677)
Raghav Chamadiyad22dc3c2b8d4
diff --git a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py b/packages/cli/src/repowise/cli/commands/dead_code_cmd.pyindex 3cb25824e..b5470a316 100644--- a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py+++ b/packages/cli/src/repowise/cli/commands/dead_code_cmd.py@@ -228,7 +228,11 @@ def dead_code_command(source_map=source_map,repo_root=repo_path,unindexed_source_files=[- (skipped.path, skipped.reason) for skipped in traverser.stats.skipped_source_files+ (skipped.path, skipped.reason)+ for skipped in (+ *traverser.stats.skipped_source_files,+ *traverser.stats.unknown_language_files,+ )],)report = analyzer.analyze(config)diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.pyindex 0b1c8ae48..f451f233d 100644--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py@@ -840,10 +840,12 @@ class DeadCodeAnalyzer:once and asking "is this symbol even mentioned there" keeps thesuppression to the findings an unread importer could actually explain.- Bounded on purpose: these files are skipped *because* they are large,- and one corpus repo ships a 104 MB generated parser. A partial read can- only under-suppress (a missed mention leaves the finding as it was),- which is the safe direction to be wrong in.+ Bounded on purpose. Files skipped on size are skipped *because* they+ are large, and one corpus repo ships a 104 MB generated parser; files+ skipped for having no language spec are individually small but arrive+ in the hundreds. A partial read can only under-suppress (a missed+ mention leaves the finding as it was), which is the safe direction to+ be wrong in."""if self._unindexed_tokens is not None:return self._unindexed_tokens@@ -899,14 +901,22 @@ class DeadCodeAnalyzer:# "main", "utils", "app") that ``risk_factors`` deliberately keeps# out of its own token map because they cap ordinary modules. An# unread importer imports symbols, so match on symbols.+ #+ # Exported symbols only, for the same reason stated the other way:+ # an importer can only reach what a file exports, so a match on an+ # internal one is a name collision rather than evidence. That is+ # not hypothetical — a published API dump lists a public `add`, and+ # every unrelated private `add` in the repo would clamp with it.+ if finding.kind is DeadCodeKind.UNUSED_INTERNAL:+ continuename = finding.symbol_nameif not name or name not in tokens:continuefinding.confidence = min(finding.confidence, RISK_CAP_CONFIDENCE)finding.safe_to_delete = Falsefinding.evidence.append(- f"'{name}' appears in a source file that was not indexed "- f"({skipped_names}), so its importers could not be checked"+ f"'{name}' appears in a file that was not indexed "+ f"({skipped_names}), so its references could not be checked")return findingsdiff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex e29348ff6..cb02ab88c 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -112,6 +112,20 @@ class TraversalStats:"""skipped_source_files_truncated: bool = False"""True once the cap above dropped a name, so a reader can say "at least"."""+ unknown_language_files: list[SkippedSourceFile] = field(default_factory=list)+ """Reference-bearing files dropped for having no language spec.++ Kept apart from :attr:`skipped_source_files` rather than folded into it,+ for two reasons that both cut the same way. That list is capped at 50 and+ these run to hundreds per repo, so sharing it would truncate the size-skip+ records instantly and destroy the signal they exist for; and it is rendered+ to the user file by file during ingestion, where naming every unparsed+ `.rst` would bury the one dropped entry point it was built to surface.+ So this list is fed to the dead-code analyzer and deliberately not to that+ report.+ """+ unknown_language_files_truncated: bool = False+ """True once the cap above dropped a name, so a reader can say "at least"."""log = structlog.get_logger(__name__)@@ -122,6 +136,38 @@ _MAX_NESTED_REPO_PATHS = 50#: Cap on the skipped-source records retained in :class:`TraversalStats`._MAX_SKIPPED_SOURCE_PATHS = 50+#: Cap on the unknown-language records retained in :class:`TraversalStats`.+#: Ten times the one above because this list is machine-read rather than+#: printed, and the widest repo measured contributes 275. The cost of reading+#: them is bounded separately, by the analyzer's own scan budget.+_MAX_UNKNOWN_LANGUAGE_PATHS = 500++#: Extensions worth remembering when language detection fails: formats whose+#: job is to name code, so a symbol appearing in one is evidence the symbol is+#: reached from somewhere the index cannot see. Doc-include formats carry+#: ``literalinclude``/``<code-block src=…>`` directives; ``.api`` files are+#: binary-compatibility dumps listing every symbol a downstream consumer may+#: bind to; the markup formats bind handlers and types by name.+#:+#: An allowlist rather than a blocklist because the unknown-language tail is+#: dominated by formats that mention no code at all — translation catalogues,+#: licence text, vector art and dotfiles are 3,335 of the 4,277 files across+#: the measurement corpus, and feeding those to a name-matching clamp would+#: suppress findings on coincidence.+#: Only extensions with no language spec belong here — one that has a spec is+#: parsed and indexed already, so listing it would be dead weight.+_REFERENCE_BEARING_EXTENSIONS: frozenset[str] = frozenset(+ {+ ".api", # also matches Kotlin's .klib.api+ ".cshtml",+ ".properties",+ ".razor",+ ".rst",+ ".topic",+ ".xml",+ }+)+# ---------------------------------------------------------------------------# Blocklists# ---------------------------------------------------------------------------@@ -576,14 +622,23 @@ class FileTraverser:"""This traverser's size verdict for one file. See :func:`size_verdict`."""return size_verdict(abs_path, size_bytes, max_file_size_bytes=self.max_file_size_bytes)- def _record_skipped_source(self, rel_str: str, size_bytes: int, reason: str) -> None:- """Note a skipped source file by name. Caller holds ``_count_lock``."""- if len(self.stats.skipped_source_files) < _MAX_SKIPPED_SOURCE_PATHS:- self.stats.skipped_source_files.append(- SkippedSourceFile(rel_str, size_bytes // 1024, reason)- )- else:- self.stats.skipped_source_files_truncated = True+ def _record_skipped_source(+ self,+ records: list[SkippedSourceFile],+ cap: int,+ rel_str: str,+ size_bytes: int,+ reason: str,+ ) -> bool:+ """Note a skipped file by name. Caller holds ``_count_lock``.++ Returns False once ``cap`` is reached, so each caller can raise its own+ truncation flag.+ """+ if len(records) >= cap:+ return False+ records.append(SkippedSourceFile(rel_str, size_bytes // 1024, reason))+ return Truedef _build_file_info(self, abs_path: Path) -> FileInfo | None:try:@@ -604,8 +659,14 @@ class FileTraverser:if (reason := self._oversize_skip_reason(abs_path, size_bytes)) is not None:with self._count_lock:self.stats.skipped_oversized += 1- if reason.is_source:- self._record_skipped_source(rel_str, size_bytes, reason.reason)+ if reason.is_source and not self._record_skipped_source(+ self.stats.skipped_source_files,+ _MAX_SKIPPED_SOURCE_PATHS,+ rel_str,+ size_bytes,+ reason.reason,+ ):+ self.stats.skipped_source_files_truncated = Truelog.debug("Skipping oversized file",path=rel_str,@@ -659,6 +720,23 @@ class FileTraverser:if language == "unknown":with self._count_lock:self.stats.skipped_unknown_language += 1+ # Keep the path even though nothing here can parse it. A+ # later pass reads these off disk to ask whether a symbol+ # it is about to call dead is named in one; discarding the+ # path made that question unanswerable for every format+ # without a parser, which is where a docs tree that embeds+ # its samples by path lives. Records no language and builds+ # no FileInfo, so the graph is untouched either way.+ if abs_path.suffix.lower() in _REFERENCE_BEARING_EXTENSIONS and (+ not self._record_skipped_source(+ self.stats.unknown_language_files,+ _MAX_UNKNOWN_LANGUAGE_PATHS,+ rel_str,+ size_bytes,+ "unknown_language",+ )+ ):+ self.stats.unknown_language_files_truncated = Truereturn None# Generated file detection: only meaningful for code files. Skippingdiff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex a84e2e8ad..62606f8c6 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -522,10 +522,12 @@ def run_partial_analysis(# files they cover; the stored rows carry every other file, which is# what init's analyzer had and this path did not._dead_code_git_meta = {**(stored_git_meta or {}), **git_meta_map}- # Source files the walk dropped on size. This report is persisted- # repo-wide (see below), so without them an update would re-derive the- # verdicts init had clamped and write them back unclamped — the #1237- # cascade would return, looking like the fix had regressed.+ # Files the walk dropped on size or for having no language spec. This+ # report is persisted repo-wide (see below), so without them an update+ # would re-derive the verdicts init had clamped and write them back+ # unclamped — the #1237 cascade would return, looking like the fix had+ # regressed.+ _traversal_stats = getattr(graph_builder, "traversal_stats", None)_dead_code_analyzer = DeadCodeAnalyzer(graph_builder.graph(),_dead_code_git_meta,@@ -534,8 +536,9 @@ def run_partial_analysis(repo_root=repo_path,unindexed_source_files=[(skipped.path, skipped.reason)- for skipped in getattr(- getattr(graph_builder, "traversal_stats", None), "skipped_source_files", []+ for skipped in (+ *getattr(_traversal_stats, "skipped_source_files", []),+ *getattr(_traversal_stats, "unknown_language_files", []),)],)diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.pyindex b8e5f94dc..e883693a3 100644--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py@@ -54,12 +54,16 @@ async def _run_dead_code_analysis(if progress:progress.on_phase_start("dead_code", dead_code_steps)- # Source files the traverser dropped on size. Without them the- # analyzer cannot tell "nothing imports this" from "the importer was- # never read", which is the cascade in #1237.+ # Files the traverser dropped on size, plus those it dropped for having+ # no language spec. Without them the analyzer cannot tell "nothing+ # imports this" from "the importer was never read", which is the+ # cascade in #1237.unindexed_source_files = [(skipped.path, skipped.reason)- for skipped in getattr(traversal_stats, "skipped_source_files", [])+ for skipped in (+ *getattr(traversal_stats, "skipped_source_files", []),+ *getattr(traversal_stats, "unknown_language_files", []),+ )]analyzer = DeadCodeAnalyzer(diff --git a/tests/unit/analysis/test_dead_code_unindexed_importers.py b/tests/unit/analysis/test_dead_code_unindexed_importers.pyindex 6fa1f6281..fbd9955cc 100644--- a/tests/unit/analysis/test_dead_code_unindexed_importers.py+++ b/tests/unit/analysis/test_dead_code_unindexed_importers.py@@ -144,3 +144,33 @@ class TestUnindexedImporterClamp:)._unindexed_identifier_tokens()assert "FirstIdentifier" in tokensassert "SecondIdentifier" not in tokens, "total budget did not stop the scan"++ def test_a_doc_include_reaches_the_symbol_it_embeds(self, repo: Path) -> None:+ # The reference a docs build writes names a *path*, but the identifier+ # split treats `/`, `-` and `.` as boundaries, so the sample's type+ # name falls out of the attribute on its own and the existing+ # name-keyed clamp reaches it with no path matching anywhere.+ (repo / "Array-types.topic").write_text(+ '<code-block src="exposed-data-types/src/main/kotlin/ArrayExamples.kt"/>\n',+ encoding="utf-8",+ )+ finding = _finding(symbol_name="ArrayExamples", file_path="ArrayExamples.kt")+ _analyzer(repo, [("Array-types.topic", "unknown_language")])._clamp_for_unindexed_importers(+ [finding]+ )+ assert finding.safe_to_delete is False+ assert finding.confidence == RISK_CAP_CONFIDENCE++ def test_internal_findings_are_never_clamped_on_a_name_collision(self, repo: Path) -> None:+ # An importer can only reach what a file exports, so a match on an+ # internal symbol is a collision rather than evidence. Measured, not+ # hypothetical: a published API dump names a public `add`, and without+ # this guard every unrelated private `add` in the repo clamped with it.+ (repo / "surface.api").write_text("public final fun add (I)V\n", encoding="utf-8")+ finding = _finding(symbol_name="add", kind=DeadCodeKind.UNUSED_INTERNAL)+ _analyzer(repo, [("surface.api", "unknown_language")])._clamp_for_unindexed_importers(+ [finding]+ )+ assert finding.safe_to_delete is True+ assert finding.confidence == 0.9+ assert finding.evidence == []diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.pyindex cd4b1b8bc..64cac3f31 100644--- a/tests/unit/ingestion/test_traverser.py+++ b/tests/unit/ingestion/test_traverser.py@@ -1007,3 +1007,65 @@ def test_production_paths_that_merely_contain_the_word_are_not_tests(rel_path):from repowise.core.test_paths import is_test_related_pathassert not is_test_related_path(rel_path)+++class TestUnknownLanguageFileRecording:+ """Paths kept when language detection fails, for the dead-code clamp.++ The clamp reads files ingestion never parsed and refuses to call a symbol+ deletion-ready when an unread file names it. It could never be offered a+ file without a language spec, because the walk bumped a counter and threw+ the path away.+ """++ def test_records_a_reference_bearing_file(self, tmp_path: Path) -> None:+ (tmp_path / "guide.rst").write_text("literalinclude:: ../src/widget.py\n")+ traverser = FileTraverser(tmp_path)+ paths = [f.path for f in traverser.traverse()]++ # Still not indexed — recording a path registers no language.+ assert not any("guide.rst" in p for p in paths)+ assert traverser.stats.skipped_unknown_language == 1+ assert [s.path for s in traverser.stats.unknown_language_files] == ["guide.rst"]+ assert traverser.stats.unknown_language_files[0].reason == "unknown_language"++ def test_ignores_formats_that_name_no_code(self, tmp_path: Path) -> None:+ # The tail is dominated by these, and a name-matching clamp fed prose+ # suppresses findings on coincidence rather than on evidence.+ (tmp_path / "django.po").write_text('msgid "Widget"\nmsgstr ""\n')+ (tmp_path / "LICENSE.txt").write_text("Redistribution of Widget is permitted.\n")+ traverser = FileTraverser(tmp_path)+ list(traverser.traverse())++ assert traverser.stats.skipped_unknown_language == 2+ assert traverser.stats.unknown_language_files == []++ def test_does_not_consume_the_size_skip_budget(self, tmp_path: Path) -> None:+ # The two lists are separate so hundreds of unparsed docs cannot+ # truncate the oversized-source records, which are shown to the user.+ for i in range(traverser_mod._MAX_SKIPPED_SOURCE_PATHS + 5):+ (tmp_path / f"doc{i}.rst").write_text("text\n")+ big = tmp_path / "huge.py"+ big.write_bytes(b"x = 1\n" * 500_000) # over the source ceiling++ traverser = FileTraverser(tmp_path, max_file_size_kb=500)+ list(traverser.traverse())++ assert [s.path for s in traverser.stats.skipped_source_files] == ["huge.py"]+ assert traverser.stats.skipped_source_files_truncated is False+ assert len(traverser.stats.unknown_language_files) == 55++ def test_caps_the_list_and_flags_truncation(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr(traverser_mod, "_MAX_UNKNOWN_LANGUAGE_PATHS", 3)+ for i in range(6):+ (tmp_path / f"doc{i}.rst").write_text("text\n")++ traverser = FileTraverser(tmp_path)+ list(traverser.traverse())++ assert len(traverser.stats.unknown_language_files) == 3+ assert traverser.stats.unknown_language_files_truncated is True+ # The count stays exact whatever the cap does to the names.+ assert traverser.stats.skipped_unknown_language == 6