Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): stop dropping large source files, and say so when we do (#1443)
Raghav Chamadiyaa7de4b9ebe98
diff --git a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py b/packages/cli/src/repowise/cli/commands/dead_code_cmd.pyindex 496220471..ab7f1ee38 100644--- a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py+++ b/packages/cli/src/repowise/cli/commands/dead_code_cmd.py@@ -218,6 +218,10 @@ def dead_code_command(git_meta_map,parsed_files=graph_builder._parsed_files,source_map=source_map,+ repo_root=repo_path,+ unindexed_source_files=[+ (skipped.path, skipped.reason) for skipped in traverser.stats.skipped_source_files+ ],)report = analyzer.analyze(config)diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.pyindex b8cab93e8..4aadb5be4 100644--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py@@ -46,6 +46,29 @@ from .jvm_reachability import build_jvm_package_files, is_jvm_file_reachablefrom .models import DeadCodeFindingData, DeadCodeKind, DeadCodeReportfrom .risk_factors import RISK_CAP_CONFIDENCE, path_risk_factors, risk_evidence+#: Identifier shape, matched over raw bytes so an unindexed file never has to+#: be decoded (these files are large by definition, and a decode would double+#: the peak). ASCII-only is deliberate: a non-ASCII identifier that fails to+#: match can only under-suppress.+_IDENTIFIER_RE = re.compile(rb"[A-Za-z_][A-Za-z0-9_]{2,}")++#: The confidence ceiling for a finding an unread file could explain is+#: ``RISK_CAP_CONFIDENCE``, imported above rather than redefined: it already+#: means "review candidate, never deletion-ready", it already sits exactly at+#: the default ``min_confidence`` so the finding still surfaces, and the CLI,+#: REST router and MCP tools already read it as the single source of truth. A+#: lower private constant would make these findings vanish from the report,+#: which is the wrong answer for a change whose whole premise is that silent+#: omission is the bug.++#: Read budgets for the unindexed scan. These files are skipped *because* they+#: are large — one corpus repo ships a 104 MB generated parser — so the scan is+#: bounded per file and in total. Tree-sitter costs ~95 MB of RSS per MB of+#: source; this pass is a byte scan and costs far less, but the bound is what+#: keeps it from ever becoming the expensive thing.+_UNINDEXED_SCAN_PER_FILE_BYTES = 4 * 1024 * 1024+_UNINDEXED_SCAN_TOTAL_BYTES = 32 * 1024 * 1024+# Symbol kinds that cannot be independently imported by name in any# supported language. Flagging them as "unused exports" is a guaranteed# false-positive — they're always accessed through an enclosing class /@@ -576,9 +599,20 @@ class DeadCodeAnalyzer:git_meta_map: dict | None = None,parsed_files: dict | None = None,source_map: dict[str, bytes] | None = None,+ repo_root: Path | None = None,+ unindexed_source_files: list[tuple[str, str]] | None = None,) -> None:self.graph = graphself.git_meta_map = git_meta_map or {}+ # Source files ingestion could not read (currently: dropped on size).+ # An unread importer is invisible to the graph, so "no importers" stops+ # meaning "unused" and starts meaning "we did not look" — which is how+ # one skipped entry point turned into 38 deletion-ready components at+ # high confidence (#1237). ``(path, reason)`` pairs; the reason rides+ # along so the evidence line can say why.+ self._unindexed_source_files = list(unindexed_source_files or [])+ self._repo_root = repo_root+ self._unindexed_tokens: frozenset[str] | None = None# Four prepasses below each scan every indexed file for text markers.# ``source_map`` is ingestion's ``{repo_relative_path: raw bytes}`` for# the same file set, so passing it turns four full-repo disk passes@@ -684,6 +718,12 @@ class DeadCodeAnalyzer:if on_step:on_step("zombie_packages")+ # Before the confidence filter, not after: a finding an unread importer+ # could explain must be able to fall *below* min_confidence and drop+ # out entirely, rather than being reported at a number it no longer+ # deserves.+ findings = self._clamp_for_unindexed_importers(findings)+min_conf = cfg.get("min_confidence", 0.4)findings = [f for f in findings if f.confidence >= min_conf]@@ -772,6 +812,84 @@ class DeadCodeAnalyzer:return findings+ def _unindexed_identifier_tokens(self) -> frozenset[str]:+ """Identifiers appearing in the source files ingestion never read.++ Suppressing every finding whenever any file was skipped would gut the+ feature on a repo with one vendored bundle. Reading the skipped files+ once and asking "is this symbol even mentioned there" keeps the+ suppression to the findings an unread importer could actually explain.++ Bounded on purpose: these files are skipped *because* they are large,+ and one corpus repo ships a 104 MB generated parser. A partial read can+ only under-suppress (a missed mention leaves the finding as it was),+ which is the safe direction to be wrong in.+ """+ if self._unindexed_tokens is not None:+ return self._unindexed_tokens+ if not self._unindexed_source_files or self._repo_root is None:+ self._unindexed_tokens = frozenset()+ return self._unindexed_tokens++ tokens: set[str] = set()+ budget = _UNINDEXED_SCAN_TOTAL_BYTES+ for rel_path, reason in self._unindexed_source_files:+ # A minified bundle is the one skipped file that must NOT feed this+ # set. It is machine-packed and carries its whole dependency tree+ # inlined, so a single 700 KB vendor bundle contributes tens of+ # thousands of identifiers and would clamp most of the report.+ if reason == "minified":+ continue+ if budget <= 0:+ break+ try:+ with open(self._repo_root / rel_path, "rb") as handle:+ blob = handle.read(min(_UNINDEXED_SCAN_PER_FILE_BYTES, budget))+ except OSError:+ continue+ budget -= len(blob)+ # ``finditer`` rather than ``findall``: the latter materialises+ # every match at once, roughly 500k bytes objects for a 4 MB blob.+ tokens.update(m.group().decode("ascii", "ignore") for m in _IDENTIFIER_RE.finditer(blob))+ self._unindexed_tokens = frozenset(tokens)+ return self._unindexed_tokens++ def _clamp_for_unindexed_importers(+ self, findings: list[DeadCodeFindingData]+ ) -> list[DeadCodeFindingData]:+ """Downgrade findings an unread file could explain.++ Mutates in place and returns the same list. Never raises a finding's+ confidence and never deletes one: a reader still sees the candidate,+ but it stops claiming to be deletion-ready on evidence we do not have.+ """+ if not self._unindexed_source_files:+ return findings+ tokens = self._unindexed_identifier_tokens()+ if not tokens:+ return findings++ skipped_names = ", ".join(path for path, _ in self._unindexed_source_files[:3])+ if len(self._unindexed_source_files) > 3:+ skipped_names += f" (+{len(self._unindexed_source_files) - 3} more)"++ for finding in findings:+ # Symbol findings only. A whole-file finding would have to match on+ # the path stem, and stems are exactly the broad words ("index",+ # "main", "utils", "app") that ``risk_factors`` deliberately keeps+ # out of its own token map because they cap ordinary modules. An+ # unread importer imports symbols, so match on symbols.+ name = finding.symbol_name+ if not name or name not in tokens:+ continue+ finding.confidence = min(finding.confidence, RISK_CAP_CONFIDENCE)+ finding.safe_to_delete = False+ finding.evidence.append(+ f"'{name}' appears in a source file that was not indexed "+ f"({skipped_names}), so its importers could not be checked"+ )+ return findings+def _make_unreachable_finding(self,node: str,diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.pyindex 817a0b678..1ae456801 100644--- a/packages/core/src/repowise/core/ingestion/graph/builder.py+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py@@ -61,6 +61,15 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd# Resolver-built DotNetProjectIndex, stashed by build() for the# dynamic-hints phase to reuse (see build()).self.dotnet_index: Any | None = None+ # ``TraversalStats`` for the walk that produced this graph, stashed by+ # the caller. Carried here rather than widened into the return tuples+ # of ``build_repo_graph``/``rebuild_graph_and_git`` (ten unpack sites+ # across the CLI, workspace and tests) because the only consumer is the+ # dead-code analyzer, which already receives this object. It needs to+ # know which source files the walk dropped on size: without that,+ # "nothing imports this" cannot be told apart from "the importer was+ # never read" (#1237).+ self.traversal_stats: Any | None = Noneself._repo_path: Path | None = Path(repo_path) if repo_path else None# Mirrors the traverser flags: when submodules or nested repos are# indexed, resolver filesystem scans must not prune them (both arediff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex 98fe04ed7..31da644ad 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -44,6 +44,34 @@ from .models import (# ---------------------------------------------------------------------------+class _OversizeSkip(NamedTuple):+ """Outcome of the second-tier size check.++ ``is_source`` is what decides whether the skip is worth telling the user+ about: a dropped 12 MB video is noise, a dropped 600 KB module is the bug+ this type exists to surface.+ """++ reason: str+ is_source: bool+++class SkippedSourceFile(NamedTuple):+ """A file with a real parser that traversal dropped on size.++ Carried as a record rather than folded into ``skipped_oversized`` because+ the count alone is what let this hide: "39 oversized" reads as lockfiles+ and images, so nobody looks, and a dropped entry point is indistinguishable+ from a dropped video. The path is the actionable part, and ``reason``+ separates "genuinely too big to parse" from "we think this is minified",+ which are different user actions.+ """++ path: str+ size_kb: int+ reason: str # "over_max_size" | "minified"++@dataclassclass TraversalStats:"""Counts collected during file traversal, broken down by skip reason."""@@ -73,6 +101,15 @@ class TraversalStats:"""nested_repo_paths_truncated: bool = False"""True once the cap above dropped a name, so a reader can say "at least"."""+ skipped_source_files: list[SkippedSourceFile] = field(default_factory=list)+ """Parseable source files dropped on size — the subset worth naming.++ Mirrors :attr:`nested_repo_paths`: capped at+ :data:`_MAX_SKIPPED_SOURCE_PATHS` so a pathological tree cannot grow this+ without bound, while ``skipped_oversized`` stays exact.+ """+ skipped_source_files_truncated: bool = False+ """True once the cap above dropped a name, so a reader can say "at least"."""log = structlog.get_logger(__name__)@@ -80,6 +117,9 @@ log = structlog.get_logger(__name__)#: Cap on the nested-repo names retained in :class:`TraversalStats`._MAX_NESTED_REPO_PATHS = 50+#: Cap on the skipped-source records retained in :class:`TraversalStats`.+_MAX_SKIPPED_SOURCE_PATHS = 50+# ---------------------------------------------------------------------------# Blocklists# ---------------------------------------------------------------------------@@ -226,9 +266,42 @@ _ENTRY_POINT_NAME_SUFFIXES: tuple[str, ...] = tuple(# question at query time, so the flag stored here and the fallback the MCP# tools use can never disagree (#1103).-# Default file-size limit+# Default file-size limit, applied to files we have no AST parser for:+# lockfiles, changelogs, JSON fixtures, images, videos. For those, size is the+# only signal available and 500 KB is generous._DEFAULT_MAX_FILE_SIZE_BYTES: int = 500 * 1024 # 500 KB+# Ceiling for files a parser *can* read. The 500 KB limit above was applied to+# every file before any language check, so a repo's largest hand-written+# modules were dropped with no page, no symbols and no wiki entry — on+# NousResearch/hermes-agent that silently removed the CLI, the gateway, the web+# server and the state layer: 6 files, 103,664 lines, 2,640 symbols (#1237).+#+# Why 2 MB and not "no limit": tree-sitter's peak cost is linear in source+# size at roughly **95 MB of peak RSS per MB of source** (measured 502 KB ->+# 95 MB, 1.36 MB -> 182 MB, 6.5 MB -> 669 MB, 46 MB -> 4,411 MB). The parse+# pool runs up to ``_MAX_PARSE_WORKERS`` (8) of these concurrently, so the cap+# is really a memory budget: 2 MB bounds one worker at ~230 MB and the pool at+# ~1.8 GB. Lifting it to 46 MB would be 4.4 GB *per worker*, and vendored+# tree-sitter grammars ship `parser.c` files of exactly that size — one repo in+# the corpus (codebase-memory-mcp) carries 268 of them, the largest 104 MB,+# which would want ~10 GB in a single worker.+_SOURCE_MAX_FILE_SIZE_BYTES: int = 2 * 1024 * 1024 # 2 MB++# A minified bundle has a real parser and a source extension, so neither the+# language check nor the size ceiling excludes it — but it is machine-written,+# yields junk symbols, and is exactly what the 500 KB limit used to catch by+# accident. Mean line length separates the two cleanly: hand-written source in+# the corpus runs 25-60 bytes/line (the hermes six: 41-50), while minified+# JavaScript runs 1,200+ and a single-line JSON blob runs into the millions.+# 200 leaves a wide margin over the widest real source measured.+_MINIFIED_MEAN_LINE_BYTES: int = 200++# Only the head of the file is read to decide: a minified bundle has no+# newlines to find, so a sample answers the question as well as the whole file+# and bounds the read on a file we are about to reject anyway.+_MINIFIED_SAMPLE_BYTES: int = 256 * 1024+# Languages for which generated-file detection is skipped. These files have# no AST parsing anyway, so reading 512 bytes to check for generated markers# adds no value.@@ -242,7 +315,11 @@ class FileTraverser:Args:repo_root: Absolute path to the repository root.- max_file_size_kb: Skip files larger than this. Default: 500 KB.+ max_file_size_kb: Skip files larger than this *when nothing can parse+ them* (lockfiles, changelogs, JSON fixtures, images, video).+ Default: 500 KB. Files whose language has an AST parser are bounded+ instead by :data:`_SOURCE_MAX_FILE_SIZE_BYTES`, a memory budget+ this argument cannot raise.extra_ignore_filename: Name of an additional gitignore-syntax file.Defaults to ``.repowiseIgnore``.extra_exclude_patterns: Additional gitignore-style patterns to exclude@@ -484,6 +561,19 @@ class FileTraverser:# Internal: FileInfo construction# ------------------------------------------------------------------+ def _oversize_skip_reason(self, abs_path: Path, size_bytes: int) -> _OversizeSkip | None:+ """This traverser's size verdict for one file. See :func:`size_verdict`."""+ return size_verdict(abs_path, size_bytes, max_file_size_bytes=self.max_file_size_bytes)++ def _record_skipped_source(self, rel_str: str, size_bytes: int, reason: str) -> None:+ """Note a skipped source file by name. Caller holds ``_count_lock``."""+ if len(self.stats.skipped_source_files) < _MAX_SKIPPED_SOURCE_PATHS:+ self.stats.skipped_source_files.append(+ SkippedSourceFile(rel_str, size_bytes // 1024, reason)+ )+ else:+ self.stats.skipped_source_files_truncated = True+def _build_file_info(self, abs_path: Path) -> FileInfo | None:try:stat = abs_path.stat()@@ -494,11 +584,23 @@ class FileTraverser:rel_path = abs_path.relative_to(self.repo_root)rel_str = rel_path.as_posix()- # Size limit- if size_bytes > self.max_file_size_bytes:+ # Size limit. Two ceilings, because one number cannot serve both jobs:+ # keeping multi-megabyte blobs out, and keeping a repo's biggest real+ # module in. The lookup that separates them is a dict hit with no I/O+ # (see :func:`_language_from_name_or_ext`), so this stays as cheap as+ # the single comparison it replaces and a 12 MB video is still+ # rejected on its stat alone.+ if (reason := self._oversize_skip_reason(abs_path, size_bytes)) is not None:with self._count_lock:self.stats.skipped_oversized += 1- log.debug("Skipping oversized file", path=rel_str, size_kb=size_bytes // 1024)+ if reason.is_source:+ self._record_skipped_source(rel_str, size_bytes, reason.reason)+ log.debug(+ "Skipping oversized file",+ path=rel_str,+ size_kb=size_bytes // 1024,+ reason=reason.reason,+ )return None# Blocked extension@@ -714,6 +816,68 @@ def _is_binary(abs_path: Path) -> bool:return True+def size_verdict(+ abs_path: Path,+ size_bytes: int,+ *,+ max_file_size_bytes: int = _DEFAULT_MAX_FILE_SIZE_BYTES,+) -> _OversizeSkip | None:+ """Whether this file is excluded on size, and why. None means "index it".++ Which ceiling applies depends on whether anything can actually read the+ file. A parserless file (lockfile, changelog, image, video) is bounded by+ *max_file_size_bytes*, the caller-facing knob. A file whose language has a+ parser is bounded by :data:`_SOURCE_MAX_FILE_SIZE_BYTES`, which is+ deliberately *not* configurable: it is a memory budget (~95 MB of peak RSS+ per MB of source, times an 8-worker parse pool), not a preference, so+ raising the knob must not be able to lift it.++ Shared with the MCP layer so that "why is this file missing" is answered by+ the same code that made it missing, rather than a second copy that can+ drift.+ """+ language = _language_from_name_or_ext(abs_path)+ # No parser, so there is nothing to gain by reading it and size is the only+ # signal available. Pre-existing behaviour, unchanged.+ if language is None or language in _SKIP_GENERATED_CHECK:+ if size_bytes > max_file_size_bytes:+ return _OversizeSkip("over_max_size", is_source=False)+ return None+ if size_bytes > _SOURCE_MAX_FILE_SIZE_BYTES:+ return _OversizeSkip("over_max_size", is_source=True)+ # Only files the old cap would have dropped are worth the sample read;+ # below it a minified bundle is small enough to be harmless.+ if size_bytes > max_file_size_bytes:+ minified = _looks_minified(abs_path)+ if minified is None:+ return _OversizeSkip("unreadable", is_source=True)+ if minified:+ return _OversizeSkip("minified", is_source=True)+ return None+++def _looks_minified(abs_path: Path) -> bool | None:+ """Return True if the file's mean line length says it is machine-packed.++ Reads only the head: a minified file's defining property is that it has+ almost no newlines, so the sample is representative by construction.++ Returns ``None`` when the file could not be read. The file still has to be+ skipped, but reporting an unreadable file as "looks minified" would send+ the user to check a bundler when the real problem is permissions.+ """+ try:+ with open(abs_path, "rb") as f:+ sample = f.read(_MINIFIED_SAMPLE_BYTES)+ except OSError:+ return None+ if not sample:+ return False+ # +1 so a file with no trailing newline is not divided by zero, and a+ # single-line file measures its own full length.+ return len(sample) / (sample.count(b"\n") + 1) > _MINIFIED_MEAN_LINE_BYTES++def _is_generated(abs_path: Path) -> bool:"""Return True if the file appears to be auto-generated."""name = abs_path.namediff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex 9409ae363..a9f8b105c 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -124,6 +124,10 @@ def build_repo_graph(include_submodules=include_submodules,include_nested_repos=include_nested_repos,)+ # Carry the walk's skip record to the dead-code analyzer. This report is+ # persisted repo-wide, so an update that could not see the skipped source+ # files would re-derive and write back the verdicts init had clamped.+ graph_builder.traversal_stats = traverser.stats# Parse the misses in process and serially: on the update path they are# change-sized. (Ceiling: a wiped/stale cache re-parses everything on one@@ -451,11 +455,22 @@ def run_partial_analysis(# files they cover; the stored rows carry every other file, which is# what init's analyzer had and this path did not._dead_code_git_meta = {**(stored_git_meta or {}), **git_meta_map}+ # Source files the walk dropped on size. This report is persisted+ # repo-wide (see below), so without them an update would re-derive the+ # verdicts init had clamped and write them back unclamped — the #1237+ # cascade would return, looking like the fix had regressed._dead_code_analyzer = DeadCodeAnalyzer(graph_builder.graph(),_dead_code_git_meta,parsed_files=graph_builder._parsed_files,source_map=source_map,+ repo_root=repo_path,+ unindexed_source_files=[+ (skipped.path, skipped.reason)+ for skipped in getattr(+ getattr(graph_builder, "traversal_stats", None), "skipped_source_files", []+ )+ ],)# Repo-wide, and persisted repo-wide. The detectors were always# repo-wide — the update path just discarded everything outside thediff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 104a30c30..2057c0610 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -496,7 +496,12 @@ async def run_pipeline(# entirely behind the CPU-bound dead-code + health work.dead_code_report, health_report, decision_report = await asyncio.gather(_run_dead_code_analysis(- graph_builder, git_meta_map, source_map=source_map, progress=progress+ graph_builder,+ git_meta_map,+ source_map=source_map,+ repo_path=repo_path,+ traversal_stats=traversal_stats,+ progress=progress,),_run_health_analysis(graph_builder,diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.pyindex b2609ac3c..3736b67e7 100644--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py@@ -40,6 +40,8 @@ async def _run_dead_code_analysis(git_meta_map: dict[str, dict],*,source_map: dict[str, bytes] | None = None,+ repo_path: Any | None = None,+ traversal_stats: Any | None = None,progress: ProgressCallback | None,) -> Any | None:"""Run dead code detection (pure graph traversal, no LLM)."""@@ -52,11 +54,21 @@ async def _run_dead_code_analysis(if progress:progress.on_phase_start("dead_code", dead_code_steps)+ # Source files the traverser dropped on size. Without them the+ # analyzer cannot tell "nothing imports this" from "the importer was+ # never read", which is the cascade in #1237.+ unindexed_source_files = [+ (skipped.path, skipped.reason)+ for skipped in getattr(traversal_stats, "skipped_source_files", [])+ ]+analyzer = DeadCodeAnalyzer(graph_builder.graph(),git_meta_map,parsed_files=graph_builder._parsed_files,source_map=source_map,+ repo_root=repo_path,+ unindexed_source_files=unindexed_source_files,)def _step(_stage: str) -> None:diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.pyindex 17fbde11f..6ff8d0834 100644--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py@@ -129,6 +129,32 @@ def _emit_traversal_summary(if skipped:progress.on_message("info", f" Excluded: {', '.join(skipped)}")+ # Name the skipped *source* files. The aggregate line above cannot carry+ # this: "39 oversized" reads as images and lockfiles, so a dropped entry+ # point looks like nothing at all. That silence is what let a repo index+ # with its CLI, gateway and web server missing and nothing saying so+ # (#1237). Blobs stay in the aggregate — only files with a real parser+ # earn a name here.+ skipped_sources = getattr(stats, "skipped_source_files", [])+ if skipped_sources:+ # Local import: this module defers its ingestion imports so the phase+ # can be loaded without building the language registry.+ from repowise.core.ingestion.traverser import _SOURCE_MAX_FILE_SIZE_BYTES++ ceiling_kb = _SOURCE_MAX_FILE_SIZE_BYTES // 1024+ for skipped_source in skipped_sources:+ detail = {+ "minified": "looks minified",+ "unreadable": "could not be read",+ }.get(skipped_source.reason, f"over the {ceiling_kb:,} KB limit")+ progress.on_message(+ "warning",+ f" Not indexed: {skipped_source.path} "+ f"({skipped_source.size_kb:,} KB, {detail})",+ )+ if getattr(stats, "skipped_source_files_truncated", False):+ progress.on_message("warning", " ...and more source files skipped on size")+if stats.lang_counts:ranked = sorted(stats.lang_counts.items(), key=lambda item: -item[1])lang_str = ", ".join(f"{lang} {count:,}" for lang, count in ranked[:_TOP_LANGUAGES_SHOWN])diff --git a/packages/server/src/repowise/server/mcp_server/tool_context/targets.py b/packages/server/src/repowise/server/mcp_server/tool_context/targets.pyindex a130cc96e..ebbda7737 100644--- a/packages/server/src/repowise/server/mcp_server/tool_context/targets.py+++ b/packages/server/src/repowise/server/mcp_server/tool_context/targets.py@@ -11,6 +11,7 @@ from __future__ import annotationsimport contextlibimport jsonimport re+from pathlib import Pathfrom typing import Anyfrom sqlalchemy import or_, select@@ -189,6 +190,40 @@ def _clean_signature(signature: str | None) -> str:return " ".join((signature or "").split())+def _size_exclusion_note(repo_root: Any, target: str) -> str | None:+ """Explain a missing file that ingestion dropped on size, else None.++ Computed live from the file on disk rather than read from the index,+ because a file excluded from ingestion leaves no row to read. Delegates the+ actual rule to ``size_verdict`` so this answer cannot drift from the code+ that produced the exclusion.+ """+ if repo_root is None:+ return None+ try:+ from repowise.core.ingestion.traverser import size_verdict++ root = Path(str(repo_root))+ abs_path = (root / target).resolve()+ # An index row is not a trust boundary; refuse anything outside the repo.+ abs_path.relative_to(root.resolve())+ size_bytes = abs_path.stat().st_size+ verdict = size_verdict(abs_path, size_bytes)+ except (OSError, ValueError):+ return None+ if verdict is None or not verdict.is_source:+ return None+ detail = {+ "minified": "it looks minified (machine-packed, not hand-written)",+ "unreadable": "it could not be read",+ }.get(verdict.reason, "it exceeds the maximum size for an indexed source file")+ return (+ f"'{target}' is not indexed: {detail} "+ f"({size_bytes // 1024:,} KB). It has no page and no symbols, and "+ "`repowise update` will not create them. Read the file directly."+ )++async def _resolve_one_target(session: AsyncSession,repository: Repository,@@ -372,9 +407,17 @@ async def _resolve_one_target()meta = res.scalar_one_or_none()if meta:+ # Say so when the file was excluded on size. The generic answer+ # below ("too few symbols or below the PageRank threshold")+ # sends the reader to regenerate docs, which cannot help: the+ # file was never read, so no amount of updating will produce a+ # page. Computed from the same function that made the decision,+ # so the two cannot drift (#1237).+ size_note = _size_exclusion_note(repo_root, target)return {"target": target,- "error": (+ "error": size_note+ or (f"'{target}' exists in the repository but has no wiki page. ""This usually means the file has too few symbols or is below ""the PageRank threshold. Run `repowise update` to regenerate docs."diff --git a/tests/unit/analysis/test_dead_code_unindexed_importers.py b/tests/unit/analysis/test_dead_code_unindexed_importers.pynew file mode 100644index 000000000..eab574fff--- /dev/null+++ b/tests/unit/analysis/test_dead_code_unindexed_importers.py@@ -0,0 +1,147 @@+"""A file ingestion never read must not turn into deletion-ready findings.++Regression cover for #1237: a large entry point was dropped by the traversal+size cap, so every component it imported looked like it had no importer and was+reported ``safe_to_delete`` at high confidence.++The clamp is exercised directly rather than through a detector: what needs+pinning is which findings it touches and which it must leave alone, and+building a graph that yields one finding per case would test the detectors+instead.+"""++from __future__ import annotations++from pathlib import Path++import networkx as nx+import pytest++from repowise.core.analysis.dead_code.analyzer import DeadCodeAnalyzer+from repowise.core.analysis.dead_code.models import DeadCodeFindingData, DeadCodeKind+from repowise.core.analysis.dead_code.risk_factors import RISK_CAP_CONFIDENCE+++def _finding(+ *,+ symbol_name: str | None,+ file_path: str = "widget.ts",+ kind: DeadCodeKind = DeadCodeKind.UNUSED_EXPORT,+) -> DeadCodeFindingData:+ return DeadCodeFindingData(+ kind=kind,+ file_path=file_path,+ symbol_name=symbol_name,+ symbol_kind="function",+ confidence=0.9,+ reason="test",+ last_commit_at=None,+ commit_count_90d=0,+ lines=10,+ package=None,+ evidence=[],+ safe_to_delete=True,+ primary_owner=None,+ age_days=None,+ )+++def _analyzer(repo: Path, skipped: list[tuple[str, str]]) -> DeadCodeAnalyzer:+ return DeadCodeAnalyzer(+ nx.DiGraph(), repo_root=repo, unindexed_source_files=skipped+ )+++def repo(tmp_path: Path) -> Path:+ # The importer ingestion could not read. It references Widget, not Orphan.+ (tmp_path / "App.tsx").write_text(+ "import { Widget } from './widget';\nexport function App() { return <Widget />; }\n",+ encoding="utf-8",+ )+ return tmp_path+++class TestUnindexedImporterClamp:+ def test_symbol_named_in_unindexed_file_is_not_deletion_ready(self, repo: Path) -> None:+ finding = _finding(symbol_name="Widget")+ _analyzer(repo, [("App.tsx", "over_max_size")])._clamp_for_unindexed_importers([finding])+ assert finding.safe_to_delete is False+ assert finding.confidence == RISK_CAP_CONFIDENCE+ assert any("not indexed" in e for e in finding.evidence)++ def test_clamped_finding_still_surfaces_at_the_default_floor(self, repo: Path) -> None:+ # The ceiling is RISK_CAP_CONFIDENCE, which sits exactly at the default+ # min_confidence so the finding stays visible as a review candidate.+ # Dropping it out of the report would be a second silent omission.+ finding = _finding(symbol_name="Widget")+ _analyzer(repo, [("App.tsx", "over_max_size")])._clamp_for_unindexed_importers([finding])+ assert finding.confidence >= 0.4++ def test_symbol_absent_from_unindexed_file_is_untouched(self, repo: Path) -> None:+ finding = _finding(symbol_name="Orphan")+ _analyzer(repo, [("App.tsx", "over_max_size")])._clamp_for_unindexed_importers([finding])+ assert finding.safe_to_delete is True+ assert finding.confidence == 0.9+ assert finding.evidence == []++ def test_file_findings_are_never_clamped_on_their_path_stem(self, repo: Path) -> None:+ # "index", "main", "utils", "app" are exactly the broad words that+ # risk_factors curates out of its own token map because they cap+ # ordinary modules. An unread importer imports symbols, not stems.+ (repo / "big.ts").write_text("const App = 1; const index = 2;\n", encoding="utf-8")+ finding = _finding(+ symbol_name=None, file_path="src/index.ts", kind=DeadCodeKind.UNREACHABLE_FILE+ )+ _analyzer(repo, [("big.ts", "over_max_size")])._clamp_for_unindexed_importers([finding])+ assert finding.safe_to_delete is True+ assert finding.confidence == 0.9++ def test_minified_files_do_not_feed_the_token_set(self, repo: Path) -> None:+ # A bundle carries its whole dependency tree inlined, so letting it+ # contribute identifiers would clamp most of the report.+ (repo / "vendor.js").write_text("var Widget=1,Orphan=2,render=3;\n", encoding="utf-8")+ analyzer = _analyzer(repo, [("vendor.js", "minified")])+ assert analyzer._unindexed_identifier_tokens() == frozenset()+ finding = _finding(symbol_name="Widget")+ analyzer._clamp_for_unindexed_importers([finding])+ assert finding.safe_to_delete is True++ def test_no_skipped_files_changes_nothing(self, repo: Path) -> None:+ finding = _finding(symbol_name="Widget")+ _analyzer(repo, [])._clamp_for_unindexed_importers([finding])+ assert finding.safe_to_delete is True+ assert finding.confidence == 0.9++ def test_missing_file_on_disk_does_not_raise(self, repo: Path) -> None:+ finding = _finding(symbol_name="Widget")+ analyzer = _analyzer(repo, [("does_not_exist.tsx", "over_max_size")])+ analyzer._clamp_for_unindexed_importers([finding])+ assert finding.safe_to_delete is True++ def test_per_file_read_is_bounded(self, repo: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ # These files are skipped because they are large; the scan must never+ # be the thing that loads one fully into memory.+ from repowise.core.analysis.dead_code import analyzer as analyzer_mod++ monkeypatch.setattr(analyzer_mod, "_UNINDEXED_SCAN_PER_FILE_BYTES", 64)+ (repo / "huge.ts").write_text(+ "EarlyIdentifier\n" + ("pad\n" * 200) + "LateIdentifier\n", encoding="utf-8"+ )+ tokens = _analyzer(repo, [("huge.ts", "over_max_size")])._unindexed_identifier_tokens()+ assert "EarlyIdentifier" in tokens+ assert "LateIdentifier" not in tokens, "read was not bounded by the per-file budget"++ def test_total_read_budget_stops_the_scan(+ self, repo: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ from repowise.core.analysis.dead_code import analyzer as analyzer_mod++ monkeypatch.setattr(analyzer_mod, "_UNINDEXED_SCAN_TOTAL_BYTES", 32)+ (repo / "one.ts").write_text("FirstIdentifier\n" + "pad\n" * 40, encoding="utf-8")+ (repo / "two.ts").write_text("SecondIdentifier\n", encoding="utf-8")+ tokens = _analyzer(+ repo, [("one.ts", "over_max_size"), ("two.ts", "over_max_size")]+ )._unindexed_identifier_tokens()+ assert "FirstIdentifier" in tokens+ assert "SecondIdentifier" not in tokens, "total budget did not stop the scan"diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.pyindex dfc9779f2..552c01575 100644--- a/tests/unit/ingestion/test_traverser.py+++ b/tests/unit/ingestion/test_traverser.py@@ -225,12 +225,81 @@ class TestFileTraverser:assert not any("app_ok.py" in p for p in paths)assert not any("debug.log" in p for p in paths)- def test_skips_oversized_files(self, tmp_path: Path) -> None:+ def test_skips_oversized_files_without_a_parser(self, tmp_path: Path) -> None:+ # No AST parser, so size is the only signal and the default cap holds.+ big = tmp_path / "fixture.json"+ big.write_bytes(b'{"k": 1}\n' * 200_000) # ~1.7 MB+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)+ paths = [f.path for f in traverser.traverse()]+ assert not any("fixture.json" in p for p in paths)+ assert traverser.stats.skipped_oversized == 1+ # Blobs stay in the aggregate — naming them would drown the signal.+ assert traverser.stats.skipped_source_files == []++ def test_indexes_source_past_the_default_cap(self, tmp_path: Path) -> None:+ # The #1237 regression: a repo's biggest hand-written module was+ # dropped before any language check ran.big = tmp_path / "big.py"big.write_bytes(b"x = 1\n" * 200_000) # ~1.2 MBtraverser = FileTraverser(tmp_path, max_file_size_kb=500)paths = [f.path for f in traverser.traverse()]- assert not any("big.py" in p for p in paths)+ assert any("big.py" in p for p in paths)+ assert traverser.stats.skipped_oversized == 0++ def test_skips_source_past_the_source_ceiling(self, tmp_path: Path) -> None:+ # The ceiling is a memory budget: tree-sitter peaks at ~95 MB of RSS+ # per MB of source and the parse pool runs 8 workers.+ big = tmp_path / "huge.py"+ big.write_bytes(b"x = 1\n" * 500_000) # ~2.9 MB, over the 2 MB ceiling+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)+ paths = [f.path for f in traverser.traverse()]+ assert not any("huge.py" in p for p in paths)+ assert traverser.stats.skipped_oversized == 1+ # ...and it is named, because a silently dropped module is the bug.+ assert [s.path for s in traverser.stats.skipped_source_files] == ["huge.py"]+ assert traverser.stats.skipped_source_files[0].reason == "over_max_size"++ def test_skips_minified_bundle_under_the_ceiling(self, tmp_path: Path) -> None:+ # Real parser, source extension, under the ceiling — only the line+ # shape distinguishes it from hand-written code.+ bundle = tmp_path / "vendor.js"+ bundle.write_bytes(b"!function(){" + b";".join(b"var a=1" for _ in range(90_000)) + b"}();")+ assert bundle.stat().st_size > 500 * 1024+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)+ paths = [f.path for f in traverser.traverse()]+ assert not any("vendor.js" in p for p in paths)+ assert traverser.stats.skipped_oversized == 1+ assert traverser.stats.skipped_source_files[0].reason == "minified"++ def test_source_ceiling_cannot_be_raised_by_max_file_size_kb(self, tmp_path: Path) -> None:+ # The source ceiling is a memory budget (~95 MB RSS per MB of source,+ # times an 8-worker pool), not a preference. Raising the caller-facing+ # knob must not be able to lift it, or the OOM guard is optional.+ big = tmp_path / "huge.py"+ big.write_bytes(b"x = 1\n" * 500_000) # ~2.9 MB, over the 2 MB ceiling+ traverser = FileTraverser(tmp_path, max_file_size_kb=100_000) # 100 MB+ paths = [f.path for f in traverser.traverse()]+ assert not any("huge.py" in p for p in paths)+ assert traverser.stats.skipped_oversized == 1++ def test_max_file_size_kb_still_governs_parserless_files(self, tmp_path: Path) -> None:+ # ...but it remains the knob for everything with no parser.+ blob = tmp_path / "fixture.json"+ blob.write_bytes(b'{"k": 1}\n' * 200_000) # ~1.7 MB+ traverser = FileTraverser(tmp_path, max_file_size_kb=100_000)+ paths = [f.path for f in traverser.traverse()]+ assert any("fixture.json" in p for p in paths)++ def test_normal_source_is_not_mistaken_for_minified(self, tmp_path: Path) -> None:+ # Long-ish but ordinary lines must survive the guard: the threshold is+ # 200 bytes/line and real source measured 25-60.+ wide = tmp_path / "wide.py"+ line = b"result = compute(" + b"argument_name, " * 8 + b"final)\n" # ~140 bytes+ wide.write_bytes(line * 5_000)+ assert wide.stat().st_size > 500 * 1024+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)+ paths = [f.path for f in traverser.traverse()]+ assert any("wide.py" in p for p in paths)def test_deterministic_ordering(self, simple_repo: Path) -> None:traverser = FileTraverser(simple_repo)@@ -508,8 +577,10 @@ class TestTraversalStats:assert traverser.stats.included >= 1def test_stats_counts_oversized_skips(self, tmp_path: Path) -> None:- big = tmp_path / "big.py"- big.write_bytes(b"x = 1\n" * 200_000)+ # A parserless blob past the cap: the counter must still fire, or the+ # size guard has been removed rather than narrowed.+ big = tmp_path / "big.json"+ big.write_bytes(b'{"k": 1}\n' * 200_000)(tmp_path / "small.py").write_text("pass")traverser = FileTraverser(tmp_path, max_file_size_kb=500)list(traverser.traverse())