Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(dead-code): stop reporting symbols a docs build or an API dump names (#1677)

Raghav Chamadiyad22dc3c2b8d4
diff --git a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py b/packages/cli/src/repowise/cli/commands/dead_code_cmd.py
index 3cb25824e..b5470a316 100644
--- a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/dead_code_cmd.py
@@ -228,7 +228,11 @@ def dead_code_command(
source_map=source_map,
repo_root=repo_path,
unindexed_source_files=[
- (skipped.path, skipped.reason) for skipped in traverser.stats.skipped_source_files
+ (skipped.path, skipped.reason)
+ for skipped in (
+ *traverser.stats.skipped_source_files,
+ *traverser.stats.unknown_language_files,
+ )
],
)
report = analyzer.analyze(config)
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
index 0b1c8ae48..f451f233d 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
@@ -840,10 +840,12 @@ class DeadCodeAnalyzer:
once and asking "is this symbol even mentioned there" keeps the
suppression to the findings an unread importer could actually explain.
- Bounded on purpose: these files are skipped *because* they are large,
- and one corpus repo ships a 104 MB generated parser. A partial read can
- only under-suppress (a missed mention leaves the finding as it was),
- which is the safe direction to be wrong in.
+ Bounded on purpose. Files skipped on size are skipped *because* they
+ are large, and one corpus repo ships a 104 MB generated parser; files
+ skipped for having no language spec are individually small but arrive
+ in the hundreds. A partial read can only under-suppress (a missed
+ mention leaves the finding as it was), which is the safe direction to
+ be wrong in.
"""
if self._unindexed_tokens is not None:
return self._unindexed_tokens
@@ -899,14 +901,22 @@ class DeadCodeAnalyzer:
# "main", "utils", "app") that ``risk_factors`` deliberately keeps
# out of its own token map because they cap ordinary modules. An
# unread importer imports symbols, so match on symbols.
+ #
+ # Exported symbols only, for the same reason stated the other way:
+ # an importer can only reach what a file exports, so a match on an
+ # internal one is a name collision rather than evidence. That is
+ # not hypothetical — a published API dump lists a public `add`, and
+ # every unrelated private `add` in the repo would clamp with it.
+ if finding.kind is DeadCodeKind.UNUSED_INTERNAL:
+ continue
name = finding.symbol_name
if not name or name not in tokens:
continue
finding.confidence = min(finding.confidence, RISK_CAP_CONFIDENCE)
finding.safe_to_delete = False
finding.evidence.append(
- f"'{name}' appears in a source file that was not indexed "
- f"({skipped_names}), so its importers could not be checked"
+ f"'{name}' appears in a file that was not indexed "
+ f"({skipped_names}), so its references could not be checked"
)
return findings
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index e29348ff6..cb02ab88c 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -112,6 +112,20 @@ class TraversalStats:
"""
skipped_source_files_truncated: bool = False
"""True once the cap above dropped a name, so a reader can say "at least"."""
+ unknown_language_files: list[SkippedSourceFile] = field(default_factory=list)
+ """Reference-bearing files dropped for having no language spec.
+
+ Kept apart from :attr:`skipped_source_files` rather than folded into it,
+ for two reasons that both cut the same way. That list is capped at 50 and
+ these run to hundreds per repo, so sharing it would truncate the size-skip
+ records instantly and destroy the signal they exist for; and it is rendered
+ to the user file by file during ingestion, where naming every unparsed
+ `.rst` would bury the one dropped entry point it was built to surface.
+ So this list is fed to the dead-code analyzer and deliberately not to that
+ report.
+ """
+ unknown_language_files_truncated: bool = False
+ """True once the cap above dropped a name, so a reader can say "at least"."""
log = structlog.get_logger(__name__)
@@ -122,6 +136,38 @@ _MAX_NESTED_REPO_PATHS = 50
#: Cap on the skipped-source records retained in :class:`TraversalStats`.
_MAX_SKIPPED_SOURCE_PATHS = 50
+#: Cap on the unknown-language records retained in :class:`TraversalStats`.
+#: Ten times the one above because this list is machine-read rather than
+#: printed, and the widest repo measured contributes 275. The cost of reading
+#: them is bounded separately, by the analyzer's own scan budget.
+_MAX_UNKNOWN_LANGUAGE_PATHS = 500
+
+#: Extensions worth remembering when language detection fails: formats whose
+#: job is to name code, so a symbol appearing in one is evidence the symbol is
+#: reached from somewhere the index cannot see. Doc-include formats carry
+#: ``literalinclude``/``<code-block src=…>`` directives; ``.api`` files are
+#: binary-compatibility dumps listing every symbol a downstream consumer may
+#: bind to; the markup formats bind handlers and types by name.
+#:
+#: An allowlist rather than a blocklist because the unknown-language tail is
+#: dominated by formats that mention no code at all — translation catalogues,
+#: licence text, vector art and dotfiles are 3,335 of the 4,277 files across
+#: the measurement corpus, and feeding those to a name-matching clamp would
+#: suppress findings on coincidence.
+#: Only extensions with no language spec belong here — one that has a spec is
+#: parsed and indexed already, so listing it would be dead weight.
+_REFERENCE_BEARING_EXTENSIONS: frozenset[str] = frozenset(
+ {
+ ".api", # also matches Kotlin's .klib.api
+ ".cshtml",
+ ".properties",
+ ".razor",
+ ".rst",
+ ".topic",
+ ".xml",
+ }
+)
+
# ---------------------------------------------------------------------------
# Blocklists
# ---------------------------------------------------------------------------
@@ -576,14 +622,23 @@ class FileTraverser:
"""This traverser's size verdict for one file. See :func:`size_verdict`."""
return size_verdict(abs_path, size_bytes, max_file_size_bytes=self.max_file_size_bytes)
- def _record_skipped_source(self, rel_str: str, size_bytes: int, reason: str) -> None:
- """Note a skipped source file by name. Caller holds ``_count_lock``."""
- if len(self.stats.skipped_source_files) < _MAX_SKIPPED_SOURCE_PATHS:
- self.stats.skipped_source_files.append(
- SkippedSourceFile(rel_str, size_bytes // 1024, reason)
- )
- else:
- self.stats.skipped_source_files_truncated = True
+ def _record_skipped_source(
+ self,
+ records: list[SkippedSourceFile],
+ cap: int,
+ rel_str: str,
+ size_bytes: int,
+ reason: str,
+ ) -> bool:
+ """Note a skipped file by name. Caller holds ``_count_lock``.
+
+ Returns False once ``cap`` is reached, so each caller can raise its own
+ truncation flag.
+ """
+ if len(records) >= cap:
+ return False
+ records.append(SkippedSourceFile(rel_str, size_bytes // 1024, reason))
+ return True
def _build_file_info(self, abs_path: Path) -> FileInfo | None:
try:
@@ -604,8 +659,14 @@ class FileTraverser:
if (reason := self._oversize_skip_reason(abs_path, size_bytes)) is not None:
with self._count_lock:
self.stats.skipped_oversized += 1
- if reason.is_source:
- self._record_skipped_source(rel_str, size_bytes, reason.reason)
+ if reason.is_source and not self._record_skipped_source(
+ self.stats.skipped_source_files,
+ _MAX_SKIPPED_SOURCE_PATHS,
+ rel_str,
+ size_bytes,
+ reason.reason,
+ ):
+ self.stats.skipped_source_files_truncated = True
log.debug(
"Skipping oversized file",
path=rel_str,
@@ -659,6 +720,23 @@ class FileTraverser:
if language == "unknown":
with self._count_lock:
self.stats.skipped_unknown_language += 1
+ # Keep the path even though nothing here can parse it. A
+ # later pass reads these off disk to ask whether a symbol
+ # it is about to call dead is named in one; discarding the
+ # path made that question unanswerable for every format
+ # without a parser, which is where a docs tree that embeds
+ # its samples by path lives. Records no language and builds
+ # no FileInfo, so the graph is untouched either way.
+ if abs_path.suffix.lower() in _REFERENCE_BEARING_EXTENSIONS and (
+ not self._record_skipped_source(
+ self.stats.unknown_language_files,
+ _MAX_UNKNOWN_LANGUAGE_PATHS,
+ rel_str,
+ size_bytes,
+ "unknown_language",
+ )
+ ):
+ self.stats.unknown_language_files_truncated = True
return None
# Generated file detection: only meaningful for code files. Skipping
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index a84e2e8ad..62606f8c6 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -522,10 +522,12 @@ def run_partial_analysis(
# files they cover; the stored rows carry every other file, which is
# what init's analyzer had and this path did not.
_dead_code_git_meta = {**(stored_git_meta or {}), **git_meta_map}
- # Source files the walk dropped on size. This report is persisted
- # repo-wide (see below), so without them an update would re-derive the
- # verdicts init had clamped and write them back unclamped — the #1237
- # cascade would return, looking like the fix had regressed.
+ # Files the walk dropped on size or for having no language spec. This
+ # report is persisted repo-wide (see below), so without them an update
+ # would re-derive the verdicts init had clamped and write them back
+ # unclamped — the #1237 cascade would return, looking like the fix had
+ # regressed.
+ _traversal_stats = getattr(graph_builder, "traversal_stats", None)
_dead_code_analyzer = DeadCodeAnalyzer(
graph_builder.graph(),
_dead_code_git_meta,
@@ -534,8 +536,9 @@ def run_partial_analysis(
repo_root=repo_path,
unindexed_source_files=[
(skipped.path, skipped.reason)
- for skipped in getattr(
- getattr(graph_builder, "traversal_stats", None), "skipped_source_files", []
+ for skipped in (
+ *getattr(_traversal_stats, "skipped_source_files", []),
+ *getattr(_traversal_stats, "unknown_language_files", []),
)
],
)
diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.py
index b8e5f94dc..e883693a3 100644
--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py
+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py
@@ -54,12 +54,16 @@ async def _run_dead_code_analysis(
if progress:
progress.on_phase_start("dead_code", dead_code_steps)
- # Source files the traverser dropped on size. Without them the
- # analyzer cannot tell "nothing imports this" from "the importer was
- # never read", which is the cascade in #1237.
+ # Files the traverser dropped on size, plus those it dropped for having
+ # no language spec. Without them the analyzer cannot tell "nothing
+ # imports this" from "the importer was never read", which is the
+ # cascade in #1237.
unindexed_source_files = [
(skipped.path, skipped.reason)
- for skipped in getattr(traversal_stats, "skipped_source_files", [])
+ for skipped in (
+ *getattr(traversal_stats, "skipped_source_files", []),
+ *getattr(traversal_stats, "unknown_language_files", []),
+ )
]
analyzer = DeadCodeAnalyzer(
diff --git a/tests/unit/analysis/test_dead_code_unindexed_importers.py b/tests/unit/analysis/test_dead_code_unindexed_importers.py
index 6fa1f6281..fbd9955cc 100644
--- a/tests/unit/analysis/test_dead_code_unindexed_importers.py
+++ b/tests/unit/analysis/test_dead_code_unindexed_importers.py
@@ -144,3 +144,33 @@ class TestUnindexedImporterClamp:
)._unindexed_identifier_tokens()
assert "FirstIdentifier" in tokens
assert "SecondIdentifier" not in tokens, "total budget did not stop the scan"
+
+ def test_a_doc_include_reaches_the_symbol_it_embeds(self, repo: Path) -> None:
+ # The reference a docs build writes names a *path*, but the identifier
+ # split treats `/`, `-` and `.` as boundaries, so the sample's type
+ # name falls out of the attribute on its own and the existing
+ # name-keyed clamp reaches it with no path matching anywhere.
+ (repo / "Array-types.topic").write_text(
+ '<code-block src="exposed-data-types/src/main/kotlin/ArrayExamples.kt"/>\n',
+ encoding="utf-8",
+ )
+ finding = _finding(symbol_name="ArrayExamples", file_path="ArrayExamples.kt")
+ _analyzer(repo, [("Array-types.topic", "unknown_language")])._clamp_for_unindexed_importers(
+ [finding]
+ )
+ assert finding.safe_to_delete is False
+ assert finding.confidence == RISK_CAP_CONFIDENCE
+
+ def test_internal_findings_are_never_clamped_on_a_name_collision(self, repo: Path) -> None:
+ # An importer can only reach what a file exports, so a match on an
+ # internal symbol is a collision rather than evidence. Measured, not
+ # hypothetical: a published API dump names a public `add`, and without
+ # this guard every unrelated private `add` in the repo clamped with it.
+ (repo / "surface.api").write_text("public final fun add (I)V\n", encoding="utf-8")
+ finding = _finding(symbol_name="add", kind=DeadCodeKind.UNUSED_INTERNAL)
+ _analyzer(repo, [("surface.api", "unknown_language")])._clamp_for_unindexed_importers(
+ [finding]
+ )
+ assert finding.safe_to_delete is True
+ assert finding.confidence == 0.9
+ assert finding.evidence == []
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index cd4b1b8bc..64cac3f31 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -1007,3 +1007,65 @@ def test_production_paths_that_merely_contain_the_word_are_not_tests(rel_path):
from repowise.core.test_paths import is_test_related_path
assert not is_test_related_path(rel_path)
+
+
+class TestUnknownLanguageFileRecording:
+ """Paths kept when language detection fails, for the dead-code clamp.
+
+ The clamp reads files ingestion never parsed and refuses to call a symbol
+ deletion-ready when an unread file names it. It could never be offered a
+ file without a language spec, because the walk bumped a counter and threw
+ the path away.
+ """
+
+ def test_records_a_reference_bearing_file(self, tmp_path: Path) -> None:
+ (tmp_path / "guide.rst").write_text("literalinclude:: ../src/widget.py\n")
+ traverser = FileTraverser(tmp_path)
+ paths = [f.path for f in traverser.traverse()]
+
+ # Still not indexed — recording a path registers no language.
+ assert not any("guide.rst" in p for p in paths)
+ assert traverser.stats.skipped_unknown_language == 1
+ assert [s.path for s in traverser.stats.unknown_language_files] == ["guide.rst"]
+ assert traverser.stats.unknown_language_files[0].reason == "unknown_language"
+
+ def test_ignores_formats_that_name_no_code(self, tmp_path: Path) -> None:
+ # The tail is dominated by these, and a name-matching clamp fed prose
+ # suppresses findings on coincidence rather than on evidence.
+ (tmp_path / "django.po").write_text('msgid "Widget"\nmsgstr ""\n')
+ (tmp_path / "LICENSE.txt").write_text("Redistribution of Widget is permitted.\n")
+ traverser = FileTraverser(tmp_path)
+ list(traverser.traverse())
+
+ assert traverser.stats.skipped_unknown_language == 2
+ assert traverser.stats.unknown_language_files == []
+
+ def test_does_not_consume_the_size_skip_budget(self, tmp_path: Path) -> None:
+ # The two lists are separate so hundreds of unparsed docs cannot
+ # truncate the oversized-source records, which are shown to the user.
+ for i in range(traverser_mod._MAX_SKIPPED_SOURCE_PATHS + 5):
+ (tmp_path / f"doc{i}.rst").write_text("text\n")
+ big = tmp_path / "huge.py"
+ big.write_bytes(b"x = 1\n" * 500_000) # over the source ceiling
+
+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)
+ list(traverser.traverse())
+
+ assert [s.path for s in traverser.stats.skipped_source_files] == ["huge.py"]
+ assert traverser.stats.skipped_source_files_truncated is False
+ assert len(traverser.stats.unknown_language_files) == 55
+
+ def test_caps_the_list_and_flags_truncation(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(traverser_mod, "_MAX_UNKNOWN_LANGUAGE_PATHS", 3)
+ for i in range(6):
+ (tmp_path / f"doc{i}.rst").write_text("text\n")
+
+ traverser = FileTraverser(tmp_path)
+ list(traverser.traverse())
+
+ assert len(traverser.stats.unknown_language_files) == 3
+ assert traverser.stats.unknown_language_files_truncated is True
+ # The count stays exact whatever the cap does to the names.
+ assert traverser.stats.skipped_unknown_language == 6