Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): stop dropping large source files, and say so when we do (#1443)

Raghav Chamadiyaa7de4b9ebe98
diff --git a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py b/packages/cli/src/repowise/cli/commands/dead_code_cmd.py
index 496220471..ab7f1ee38 100644
--- a/packages/cli/src/repowise/cli/commands/dead_code_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/dead_code_cmd.py
@@ -218,6 +218,10 @@ def dead_code_command(
git_meta_map,
parsed_files=graph_builder._parsed_files,
source_map=source_map,
+ repo_root=repo_path,
+ unindexed_source_files=[
+ (skipped.path, skipped.reason) for skipped in traverser.stats.skipped_source_files
+ ],
)
report = analyzer.analyze(config)
diff --git a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
index b8cab93e8..4aadb5be4 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/analyzer.py
@@ -46,6 +46,29 @@ from .jvm_reachability import build_jvm_package_files, is_jvm_file_reachable
from .models import DeadCodeFindingData, DeadCodeKind, DeadCodeReport
from .risk_factors import RISK_CAP_CONFIDENCE, path_risk_factors, risk_evidence
+#: Identifier shape, matched over raw bytes so an unindexed file never has to
+#: be decoded (these files are large by definition, and a decode would double
+#: the peak). ASCII-only is deliberate: a non-ASCII identifier that fails to
+#: match can only under-suppress.
+_IDENTIFIER_RE = re.compile(rb"[A-Za-z_][A-Za-z0-9_]{2,}")
+
+#: The confidence ceiling for a finding an unread file could explain is
+#: ``RISK_CAP_CONFIDENCE``, imported above rather than redefined: it already
+#: means "review candidate, never deletion-ready", it already sits exactly at
+#: the default ``min_confidence`` so the finding still surfaces, and the CLI,
+#: REST router and MCP tools already read it as the single source of truth. A
+#: lower private constant would make these findings vanish from the report,
+#: which is the wrong answer for a change whose whole premise is that silent
+#: omission is the bug.
+
+#: Read budgets for the unindexed scan. These files are skipped *because* they
+#: are large — one corpus repo ships a 104 MB generated parser — so the scan is
+#: bounded per file and in total. Tree-sitter costs ~95 MB of RSS per MB of
+#: source; this pass is a byte scan and costs far less, but the bound is what
+#: keeps it from ever becoming the expensive thing.
+_UNINDEXED_SCAN_PER_FILE_BYTES = 4 * 1024 * 1024
+_UNINDEXED_SCAN_TOTAL_BYTES = 32 * 1024 * 1024
+
# Symbol kinds that cannot be independently imported by name in any
# supported language. Flagging them as "unused exports" is a guaranteed
# false-positive — they're always accessed through an enclosing class /
@@ -576,9 +599,20 @@ class DeadCodeAnalyzer:
git_meta_map: dict | None = None,
parsed_files: dict | None = None,
source_map: dict[str, bytes] | None = None,
+ repo_root: Path | None = None,
+ unindexed_source_files: list[tuple[str, str]] | None = None,
) -> None:
self.graph = graph
self.git_meta_map = git_meta_map or {}
+ # Source files ingestion could not read (currently: dropped on size).
+ # An unread importer is invisible to the graph, so "no importers" stops
+ # meaning "unused" and starts meaning "we did not look" — which is how
+ # one skipped entry point turned into 38 deletion-ready components at
+ # high confidence (#1237). ``(path, reason)`` pairs; the reason rides
+ # along so the evidence line can say why.
+ self._unindexed_source_files = list(unindexed_source_files or [])
+ self._repo_root = repo_root
+ self._unindexed_tokens: frozenset[str] | None = None
# Four prepasses below each scan every indexed file for text markers.
# ``source_map`` is ingestion's ``{repo_relative_path: raw bytes}`` for
# the same file set, so passing it turns four full-repo disk passes
@@ -684,6 +718,12 @@ class DeadCodeAnalyzer:
if on_step:
on_step("zombie_packages")
+ # Before the confidence filter, not after: a finding an unread importer
+ # could explain must be able to fall *below* min_confidence and drop
+ # out entirely, rather than being reported at a number it no longer
+ # deserves.
+ findings = self._clamp_for_unindexed_importers(findings)
+
min_conf = cfg.get("min_confidence", 0.4)
findings = [f for f in findings if f.confidence >= min_conf]
@@ -772,6 +812,84 @@ class DeadCodeAnalyzer:
return findings
+ def _unindexed_identifier_tokens(self) -> frozenset[str]:
+ """Identifiers appearing in the source files ingestion never read.
+
+ Suppressing every finding whenever any file was skipped would gut the
+ feature on a repo with one vendored bundle. Reading the skipped files
+ once and asking "is this symbol even mentioned there" keeps the
+ suppression to the findings an unread importer could actually explain.
+
+ Bounded on purpose: these files are skipped *because* they are large,
+ and one corpus repo ships a 104 MB generated parser. A partial read can
+ only under-suppress (a missed mention leaves the finding as it was),
+ which is the safe direction to be wrong in.
+ """
+ if self._unindexed_tokens is not None:
+ return self._unindexed_tokens
+ if not self._unindexed_source_files or self._repo_root is None:
+ self._unindexed_tokens = frozenset()
+ return self._unindexed_tokens
+
+ tokens: set[str] = set()
+ budget = _UNINDEXED_SCAN_TOTAL_BYTES
+ for rel_path, reason in self._unindexed_source_files:
+ # A minified bundle is the one skipped file that must NOT feed this
+ # set. It is machine-packed and carries its whole dependency tree
+ # inlined, so a single 700 KB vendor bundle contributes tens of
+ # thousands of identifiers and would clamp most of the report.
+ if reason == "minified":
+ continue
+ if budget <= 0:
+ break
+ try:
+ with open(self._repo_root / rel_path, "rb") as handle:
+ blob = handle.read(min(_UNINDEXED_SCAN_PER_FILE_BYTES, budget))
+ except OSError:
+ continue
+ budget -= len(blob)
+ # ``finditer`` rather than ``findall``: the latter materialises
+ # every match at once, roughly 500k bytes objects for a 4 MB blob.
+ tokens.update(m.group().decode("ascii", "ignore") for m in _IDENTIFIER_RE.finditer(blob))
+ self._unindexed_tokens = frozenset(tokens)
+ return self._unindexed_tokens
+
+ def _clamp_for_unindexed_importers(
+ self, findings: list[DeadCodeFindingData]
+ ) -> list[DeadCodeFindingData]:
+ """Downgrade findings an unread file could explain.
+
+ Mutates in place and returns the same list. Never raises a finding's
+ confidence and never deletes one: a reader still sees the candidate,
+ but it stops claiming to be deletion-ready on evidence we do not have.
+ """
+ if not self._unindexed_source_files:
+ return findings
+ tokens = self._unindexed_identifier_tokens()
+ if not tokens:
+ return findings
+
+ skipped_names = ", ".join(path for path, _ in self._unindexed_source_files[:3])
+ if len(self._unindexed_source_files) > 3:
+ skipped_names += f" (+{len(self._unindexed_source_files) - 3} more)"
+
+ for finding in findings:
+ # Symbol findings only. A whole-file finding would have to match on
+ # the path stem, and stems are exactly the broad words ("index",
+ # "main", "utils", "app") that ``risk_factors`` deliberately keeps
+ # out of its own token map because they cap ordinary modules. An
+ # unread importer imports symbols, so match on symbols.
+ name = finding.symbol_name
+ if not name or name not in tokens:
+ continue
+ finding.confidence = min(finding.confidence, RISK_CAP_CONFIDENCE)
+ finding.safe_to_delete = False
+ finding.evidence.append(
+ f"'{name}' appears in a source file that was not indexed "
+ f"({skipped_names}), so its importers could not be checked"
+ )
+ return findings
+
def _make_unreachable_finding(
self,
node: str,
diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.py
index 817a0b678..1ae456801 100644
--- a/packages/core/src/repowise/core/ingestion/graph/builder.py
+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py
@@ -61,6 +61,15 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
# Resolver-built DotNetProjectIndex, stashed by build() for the
# dynamic-hints phase to reuse (see build()).
self.dotnet_index: Any | None = None
+ # ``TraversalStats`` for the walk that produced this graph, stashed by
+ # the caller. Carried here rather than widened into the return tuples
+ # of ``build_repo_graph``/``rebuild_graph_and_git`` (ten unpack sites
+ # across the CLI, workspace and tests) because the only consumer is the
+ # dead-code analyzer, which already receives this object. It needs to
+ # know which source files the walk dropped on size: without that,
+ # "nothing imports this" cannot be told apart from "the importer was
+ # never read" (#1237).
+ self.traversal_stats: Any | None = None
self._repo_path: Path | None = Path(repo_path) if repo_path else None
# Mirrors the traverser flags: when submodules or nested repos are
# indexed, resolver filesystem scans must not prune them (both are
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index 98fe04ed7..31da644ad 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -44,6 +44,34 @@ from .models import (
# ---------------------------------------------------------------------------
+class _OversizeSkip(NamedTuple):
+ """Outcome of the second-tier size check.
+
+ ``is_source`` is what decides whether the skip is worth telling the user
+ about: a dropped 12 MB video is noise, a dropped 600 KB module is the bug
+ this type exists to surface.
+ """
+
+ reason: str
+ is_source: bool
+
+
+class SkippedSourceFile(NamedTuple):
+ """A file with a real parser that traversal dropped on size.
+
+ Carried as a record rather than folded into ``skipped_oversized`` because
+ the count alone is what let this hide: "39 oversized" reads as lockfiles
+ and images, so nobody looks, and a dropped entry point is indistinguishable
+ from a dropped video. The path is the actionable part, and ``reason``
+ separates "genuinely too big to parse" from "we think this is minified",
+ which are different user actions.
+ """
+
+ path: str
+ size_kb: int
+ reason: str # "over_max_size" | "minified"
+
+
@dataclass
class TraversalStats:
"""Counts collected during file traversal, broken down by skip reason."""
@@ -73,6 +101,15 @@ class TraversalStats:
"""
nested_repo_paths_truncated: bool = False
"""True once the cap above dropped a name, so a reader can say "at least"."""
+ skipped_source_files: list[SkippedSourceFile] = field(default_factory=list)
+ """Parseable source files dropped on size — the subset worth naming.
+
+ Mirrors :attr:`nested_repo_paths`: capped at
+ :data:`_MAX_SKIPPED_SOURCE_PATHS` so a pathological tree cannot grow this
+ without bound, while ``skipped_oversized`` stays exact.
+ """
+ skipped_source_files_truncated: bool = False
+ """True once the cap above dropped a name, so a reader can say "at least"."""
log = structlog.get_logger(__name__)
@@ -80,6 +117,9 @@ log = structlog.get_logger(__name__)
#: Cap on the nested-repo names retained in :class:`TraversalStats`.
_MAX_NESTED_REPO_PATHS = 50
+#: Cap on the skipped-source records retained in :class:`TraversalStats`.
+_MAX_SKIPPED_SOURCE_PATHS = 50
+
# ---------------------------------------------------------------------------
# Blocklists
# ---------------------------------------------------------------------------
@@ -226,9 +266,42 @@ _ENTRY_POINT_NAME_SUFFIXES: tuple[str, ...] = tuple(
# question at query time, so the flag stored here and the fallback the MCP
# tools use can never disagree (#1103).
-# Default file-size limit
+# Default file-size limit, applied to files we have no AST parser for:
+# lockfiles, changelogs, JSON fixtures, images, videos. For those, size is the
+# only signal available and 500 KB is generous.
_DEFAULT_MAX_FILE_SIZE_BYTES: int = 500 * 1024 # 500 KB
+# Ceiling for files a parser *can* read. The 500 KB limit above was applied to
+# every file before any language check, so a repo's largest hand-written
+# modules were dropped with no page, no symbols and no wiki entry — on
+# NousResearch/hermes-agent that silently removed the CLI, the gateway, the web
+# server and the state layer: 6 files, 103,664 lines, 2,640 symbols (#1237).
+#
+# Why 2 MB and not "no limit": tree-sitter's peak cost is linear in source
+# size at roughly **95 MB of peak RSS per MB of source** (measured 502 KB ->
+# 95 MB, 1.36 MB -> 182 MB, 6.5 MB -> 669 MB, 46 MB -> 4,411 MB). The parse
+# pool runs up to ``_MAX_PARSE_WORKERS`` (8) of these concurrently, so the cap
+# is really a memory budget: 2 MB bounds one worker at ~230 MB and the pool at
+# ~1.8 GB. Lifting it to 46 MB would be 4.4 GB *per worker*, and vendored
+# tree-sitter grammars ship `parser.c` files of exactly that size — one repo in
+# the corpus (codebase-memory-mcp) carries 268 of them, the largest 104 MB,
+# which would want ~10 GB in a single worker.
+_SOURCE_MAX_FILE_SIZE_BYTES: int = 2 * 1024 * 1024 # 2 MB
+
+# A minified bundle has a real parser and a source extension, so neither the
+# language check nor the size ceiling excludes it — but it is machine-written,
+# yields junk symbols, and is exactly what the 500 KB limit used to catch by
+# accident. Mean line length separates the two cleanly: hand-written source in
+# the corpus runs 25-60 bytes/line (the hermes six: 41-50), while minified
+# JavaScript runs 1,200+ and a single-line JSON blob runs into the millions.
+# 200 leaves a wide margin over the widest real source measured.
+_MINIFIED_MEAN_LINE_BYTES: int = 200
+
+# Only the head of the file is read to decide: a minified bundle has no
+# newlines to find, so a sample answers the question as well as the whole file
+# and bounds the read on a file we are about to reject anyway.
+_MINIFIED_SAMPLE_BYTES: int = 256 * 1024
+
# Languages for which generated-file detection is skipped. These files have
# no AST parsing anyway, so reading 512 bytes to check for generated markers
# adds no value.
@@ -242,7 +315,11 @@ class FileTraverser:
Args:
repo_root: Absolute path to the repository root.
- max_file_size_kb: Skip files larger than this. Default: 500 KB.
+ max_file_size_kb: Skip files larger than this *when nothing can parse
+ them* (lockfiles, changelogs, JSON fixtures, images, video).
+ Default: 500 KB. Files whose language has an AST parser are bounded
+ instead by :data:`_SOURCE_MAX_FILE_SIZE_BYTES`, a memory budget
+ this argument cannot raise.
extra_ignore_filename: Name of an additional gitignore-syntax file.
Defaults to ``.repowiseIgnore``.
extra_exclude_patterns: Additional gitignore-style patterns to exclude
@@ -484,6 +561,19 @@ class FileTraverser:
# Internal: FileInfo construction
# ------------------------------------------------------------------
+ def _oversize_skip_reason(self, abs_path: Path, size_bytes: int) -> _OversizeSkip | None:
+ """This traverser's size verdict for one file. See :func:`size_verdict`."""
+ return size_verdict(abs_path, size_bytes, max_file_size_bytes=self.max_file_size_bytes)
+
+ def _record_skipped_source(self, rel_str: str, size_bytes: int, reason: str) -> None:
+ """Note a skipped source file by name. Caller holds ``_count_lock``."""
+ if len(self.stats.skipped_source_files) < _MAX_SKIPPED_SOURCE_PATHS:
+ self.stats.skipped_source_files.append(
+ SkippedSourceFile(rel_str, size_bytes // 1024, reason)
+ )
+ else:
+ self.stats.skipped_source_files_truncated = True
+
def _build_file_info(self, abs_path: Path) -> FileInfo | None:
try:
stat = abs_path.stat()
@@ -494,11 +584,23 @@ class FileTraverser:
rel_path = abs_path.relative_to(self.repo_root)
rel_str = rel_path.as_posix()
- # Size limit
- if size_bytes > self.max_file_size_bytes:
+ # Size limit. Two ceilings, because one number cannot serve both jobs:
+ # keeping multi-megabyte blobs out, and keeping a repo's biggest real
+ # module in. The lookup that separates them is a dict hit with no I/O
+ # (see :func:`_language_from_name_or_ext`), so this stays as cheap as
+ # the single comparison it replaces and a 12 MB video is still
+ # rejected on its stat alone.
+ if (reason := self._oversize_skip_reason(abs_path, size_bytes)) is not None:
with self._count_lock:
self.stats.skipped_oversized += 1
- log.debug("Skipping oversized file", path=rel_str, size_kb=size_bytes // 1024)
+ if reason.is_source:
+ self._record_skipped_source(rel_str, size_bytes, reason.reason)
+ log.debug(
+ "Skipping oversized file",
+ path=rel_str,
+ size_kb=size_bytes // 1024,
+ reason=reason.reason,
+ )
return None
# Blocked extension
@@ -714,6 +816,68 @@ def _is_binary(abs_path: Path) -> bool:
return True
+def size_verdict(
+ abs_path: Path,
+ size_bytes: int,
+ *,
+ max_file_size_bytes: int = _DEFAULT_MAX_FILE_SIZE_BYTES,
+) -> _OversizeSkip | None:
+ """Whether this file is excluded on size, and why. None means "index it".
+
+ Which ceiling applies depends on whether anything can actually read the
+ file. A parserless file (lockfile, changelog, image, video) is bounded by
+ *max_file_size_bytes*, the caller-facing knob. A file whose language has a
+ parser is bounded by :data:`_SOURCE_MAX_FILE_SIZE_BYTES`, which is
+ deliberately *not* configurable: it is a memory budget (~95 MB of peak RSS
+ per MB of source, times an 8-worker parse pool), not a preference, so
+ raising the knob must not be able to lift it.
+
+ Shared with the MCP layer so that "why is this file missing" is answered by
+ the same code that made it missing, rather than a second copy that can
+ drift.
+ """
+ language = _language_from_name_or_ext(abs_path)
+ # No parser, so there is nothing to gain by reading it and size is the only
+ # signal available. Pre-existing behaviour, unchanged.
+ if language is None or language in _SKIP_GENERATED_CHECK:
+ if size_bytes > max_file_size_bytes:
+ return _OversizeSkip("over_max_size", is_source=False)
+ return None
+ if size_bytes > _SOURCE_MAX_FILE_SIZE_BYTES:
+ return _OversizeSkip("over_max_size", is_source=True)
+ # Only files the old cap would have dropped are worth the sample read;
+ # below it a minified bundle is small enough to be harmless.
+ if size_bytes > max_file_size_bytes:
+ minified = _looks_minified(abs_path)
+ if minified is None:
+ return _OversizeSkip("unreadable", is_source=True)
+ if minified:
+ return _OversizeSkip("minified", is_source=True)
+ return None
+
+
+def _looks_minified(abs_path: Path) -> bool | None:
+ """Return True if the file's mean line length says it is machine-packed.
+
+ Reads only the head: a minified file's defining property is that it has
+ almost no newlines, so the sample is representative by construction.
+
+ Returns ``None`` when the file could not be read. The file still has to be
+ skipped, but reporting an unreadable file as "looks minified" would send
+ the user to check a bundler when the real problem is permissions.
+ """
+ try:
+ with open(abs_path, "rb") as f:
+ sample = f.read(_MINIFIED_SAMPLE_BYTES)
+ except OSError:
+ return None
+ if not sample:
+ return False
+ # +1 so a file with no trailing newline is not divided by zero, and a
+ # single-line file measures its own full length.
+ return len(sample) / (sample.count(b"\n") + 1) > _MINIFIED_MEAN_LINE_BYTES
+
+
def _is_generated(abs_path: Path) -> bool:
"""Return True if the file appears to be auto-generated."""
name = abs_path.name
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index 9409ae363..a9f8b105c 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -124,6 +124,10 @@ def build_repo_graph(
include_submodules=include_submodules,
include_nested_repos=include_nested_repos,
)
+ # Carry the walk's skip record to the dead-code analyzer. This report is
+ # persisted repo-wide, so an update that could not see the skipped source
+ # files would re-derive and write back the verdicts init had clamped.
+ graph_builder.traversal_stats = traverser.stats
# Parse the misses in process and serially: on the update path they are
# change-sized. (Ceiling: a wiped/stale cache re-parses everything on one
@@ -451,11 +455,22 @@ def run_partial_analysis(
# files they cover; the stored rows carry every other file, which is
# what init's analyzer had and this path did not.
_dead_code_git_meta = {**(stored_git_meta or {}), **git_meta_map}
+ # Source files the walk dropped on size. This report is persisted
+ # repo-wide (see below), so without them an update would re-derive the
+ # verdicts init had clamped and write them back unclamped — the #1237
+ # cascade would return, looking like the fix had regressed.
_dead_code_analyzer = DeadCodeAnalyzer(
graph_builder.graph(),
_dead_code_git_meta,
parsed_files=graph_builder._parsed_files,
source_map=source_map,
+ repo_root=repo_path,
+ unindexed_source_files=[
+ (skipped.path, skipped.reason)
+ for skipped in getattr(
+ getattr(graph_builder, "traversal_stats", None), "skipped_source_files", []
+ )
+ ],
)
# Repo-wide, and persisted repo-wide. The detectors were always
# repo-wide — the update path just discarded everything outside the
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 104a30c30..2057c0610 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -496,7 +496,12 @@ async def run_pipeline(
# entirely behind the CPU-bound dead-code + health work.
dead_code_report, health_report, decision_report = await asyncio.gather(
_run_dead_code_analysis(
- graph_builder, git_meta_map, source_map=source_map, progress=progress
+ graph_builder,
+ git_meta_map,
+ source_map=source_map,
+ repo_path=repo_path,
+ traversal_stats=traversal_stats,
+ progress=progress,
),
_run_health_analysis(
graph_builder,
diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.py
index b2609ac3c..3736b67e7 100644
--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py
+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py
@@ -40,6 +40,8 @@ async def _run_dead_code_analysis(
git_meta_map: dict[str, dict],
*,
source_map: dict[str, bytes] | None = None,
+ repo_path: Any | None = None,
+ traversal_stats: Any | None = None,
progress: ProgressCallback | None,
) -> Any | None:
"""Run dead code detection (pure graph traversal, no LLM)."""
@@ -52,11 +54,21 @@ async def _run_dead_code_analysis(
if progress:
progress.on_phase_start("dead_code", dead_code_steps)
+ # Source files the traverser dropped on size. Without them the
+ # analyzer cannot tell "nothing imports this" from "the importer was
+ # never read", which is the cascade in #1237.
+ unindexed_source_files = [
+ (skipped.path, skipped.reason)
+ for skipped in getattr(traversal_stats, "skipped_source_files", [])
+ ]
+
analyzer = DeadCodeAnalyzer(
graph_builder.graph(),
git_meta_map,
parsed_files=graph_builder._parsed_files,
source_map=source_map,
+ repo_root=repo_path,
+ unindexed_source_files=unindexed_source_files,
)
def _step(_stage: str) -> None:
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index 17fbde11f..6ff8d0834 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -129,6 +129,32 @@ def _emit_traversal_summary(
if skipped:
progress.on_message("info", f" Excluded: {', '.join(skipped)}")
+ # Name the skipped *source* files. The aggregate line above cannot carry
+ # this: "39 oversized" reads as images and lockfiles, so a dropped entry
+ # point looks like nothing at all. That silence is what let a repo index
+ # with its CLI, gateway and web server missing and nothing saying so
+ # (#1237). Blobs stay in the aggregate — only files with a real parser
+ # earn a name here.
+ skipped_sources = getattr(stats, "skipped_source_files", [])
+ if skipped_sources:
+ # Local import: this module defers its ingestion imports so the phase
+ # can be loaded without building the language registry.
+ from repowise.core.ingestion.traverser import _SOURCE_MAX_FILE_SIZE_BYTES
+
+ ceiling_kb = _SOURCE_MAX_FILE_SIZE_BYTES // 1024
+ for skipped_source in skipped_sources:
+ detail = {
+ "minified": "looks minified",
+ "unreadable": "could not be read",
+ }.get(skipped_source.reason, f"over the {ceiling_kb:,} KB limit")
+ progress.on_message(
+ "warning",
+ f" Not indexed: {skipped_source.path} "
+ f"({skipped_source.size_kb:,} KB, {detail})",
+ )
+ if getattr(stats, "skipped_source_files_truncated", False):
+ progress.on_message("warning", " ...and more source files skipped on size")
+
if stats.lang_counts:
ranked = sorted(stats.lang_counts.items(), key=lambda item: -item[1])
lang_str = ", ".join(f"{lang} {count:,}" for lang, count in ranked[:_TOP_LANGUAGES_SHOWN])
diff --git a/packages/server/src/repowise/server/mcp_server/tool_context/targets.py b/packages/server/src/repowise/server/mcp_server/tool_context/targets.py
index a130cc96e..ebbda7737 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_context/targets.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_context/targets.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import contextlib
import json
import re
+from pathlib import Path
from typing import Any
from sqlalchemy import or_, select
@@ -189,6 +190,40 @@ def _clean_signature(signature: str | None) -> str:
return " ".join((signature or "").split())
+def _size_exclusion_note(repo_root: Any, target: str) -> str | None:
+ """Explain a missing file that ingestion dropped on size, else None.
+
+ Computed live from the file on disk rather than read from the index,
+ because a file excluded from ingestion leaves no row to read. Delegates the
+ actual rule to ``size_verdict`` so this answer cannot drift from the code
+ that produced the exclusion.
+ """
+ if repo_root is None:
+ return None
+ try:
+ from repowise.core.ingestion.traverser import size_verdict
+
+ root = Path(str(repo_root))
+ abs_path = (root / target).resolve()
+ # An index row is not a trust boundary; refuse anything outside the repo.
+ abs_path.relative_to(root.resolve())
+ size_bytes = abs_path.stat().st_size
+ verdict = size_verdict(abs_path, size_bytes)
+ except (OSError, ValueError):
+ return None
+ if verdict is None or not verdict.is_source:
+ return None
+ detail = {
+ "minified": "it looks minified (machine-packed, not hand-written)",
+ "unreadable": "it could not be read",
+ }.get(verdict.reason, "it exceeds the maximum size for an indexed source file")
+ return (
+ f"'{target}' is not indexed: {detail} "
+ f"({size_bytes // 1024:,} KB). It has no page and no symbols, and "
+ "`repowise update` will not create them. Read the file directly."
+ )
+
+
async def _resolve_one_target(
session: AsyncSession,
repository: Repository,
@@ -372,9 +407,17 @@ async def _resolve_one_target(
)
meta = res.scalar_one_or_none()
if meta:
+ # Say so when the file was excluded on size. The generic answer
+ # below ("too few symbols or below the PageRank threshold")
+ # sends the reader to regenerate docs, which cannot help: the
+ # file was never read, so no amount of updating will produce a
+ # page. Computed from the same function that made the decision,
+ # so the two cannot drift (#1237).
+ size_note = _size_exclusion_note(repo_root, target)
return {
"target": target,
- "error": (
+ "error": size_note
+ or (
f"'{target}' exists in the repository but has no wiki page. "
"This usually means the file has too few symbols or is below "
"the PageRank threshold. Run `repowise update` to regenerate docs."
diff --git a/tests/unit/analysis/test_dead_code_unindexed_importers.py b/tests/unit/analysis/test_dead_code_unindexed_importers.py
new file mode 100644
index 000000000..eab574fff
--- /dev/null
+++ b/tests/unit/analysis/test_dead_code_unindexed_importers.py
@@ -0,0 +1,147 @@
+"""A file ingestion never read must not turn into deletion-ready findings.
+
+Regression cover for #1237: a large entry point was dropped by the traversal
+size cap, so every component it imported looked like it had no importer and was
+reported ``safe_to_delete`` at high confidence.
+
+The clamp is exercised directly rather than through a detector: what needs
+pinning is which findings it touches and which it must leave alone, and
+building a graph that yields one finding per case would test the detectors
+instead.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import networkx as nx
+import pytest
+
+from repowise.core.analysis.dead_code.analyzer import DeadCodeAnalyzer
+from repowise.core.analysis.dead_code.models import DeadCodeFindingData, DeadCodeKind
+from repowise.core.analysis.dead_code.risk_factors import RISK_CAP_CONFIDENCE
+
+
+def _finding(
+ *,
+ symbol_name: str | None,
+ file_path: str = "widget.ts",
+ kind: DeadCodeKind = DeadCodeKind.UNUSED_EXPORT,
+) -> DeadCodeFindingData:
+ return DeadCodeFindingData(
+ kind=kind,
+ file_path=file_path,
+ symbol_name=symbol_name,
+ symbol_kind="function",
+ confidence=0.9,
+ reason="test",
+ last_commit_at=None,
+ commit_count_90d=0,
+ lines=10,
+ package=None,
+ evidence=[],
+ safe_to_delete=True,
+ primary_owner=None,
+ age_days=None,
+ )
+
+
+def _analyzer(repo: Path, skipped: list[tuple[str, str]]) -> DeadCodeAnalyzer:
+ return DeadCodeAnalyzer(
+ nx.DiGraph(), repo_root=repo, unindexed_source_files=skipped
+ )
+
+
+def repo(tmp_path: Path) -> Path:
+ # The importer ingestion could not read. It references Widget, not Orphan.
+ (tmp_path / "App.tsx").write_text(
+ "import { Widget } from './widget';\nexport function App() { return <Widget />; }\n",
+ encoding="utf-8",
+ )
+ return tmp_path
+
+
+class TestUnindexedImporterClamp:
+ def test_symbol_named_in_unindexed_file_is_not_deletion_ready(self, repo: Path) -> None:
+ finding = _finding(symbol_name="Widget")
+ _analyzer(repo, [("App.tsx", "over_max_size")])._clamp_for_unindexed_importers([finding])
+ assert finding.safe_to_delete is False
+ assert finding.confidence == RISK_CAP_CONFIDENCE
+ assert any("not indexed" in e for e in finding.evidence)
+
+ def test_clamped_finding_still_surfaces_at_the_default_floor(self, repo: Path) -> None:
+ # The ceiling is RISK_CAP_CONFIDENCE, which sits exactly at the default
+ # min_confidence so the finding stays visible as a review candidate.
+ # Dropping it out of the report would be a second silent omission.
+ finding = _finding(symbol_name="Widget")
+ _analyzer(repo, [("App.tsx", "over_max_size")])._clamp_for_unindexed_importers([finding])
+ assert finding.confidence >= 0.4
+
+ def test_symbol_absent_from_unindexed_file_is_untouched(self, repo: Path) -> None:
+ finding = _finding(symbol_name="Orphan")
+ _analyzer(repo, [("App.tsx", "over_max_size")])._clamp_for_unindexed_importers([finding])
+ assert finding.safe_to_delete is True
+ assert finding.confidence == 0.9
+ assert finding.evidence == []
+
+ def test_file_findings_are_never_clamped_on_their_path_stem(self, repo: Path) -> None:
+ # "index", "main", "utils", "app" are exactly the broad words that
+ # risk_factors curates out of its own token map because they cap
+ # ordinary modules. An unread importer imports symbols, not stems.
+ (repo / "big.ts").write_text("const App = 1; const index = 2;\n", encoding="utf-8")
+ finding = _finding(
+ symbol_name=None, file_path="src/index.ts", kind=DeadCodeKind.UNREACHABLE_FILE
+ )
+ _analyzer(repo, [("big.ts", "over_max_size")])._clamp_for_unindexed_importers([finding])
+ assert finding.safe_to_delete is True
+ assert finding.confidence == 0.9
+
+ def test_minified_files_do_not_feed_the_token_set(self, repo: Path) -> None:
+ # A bundle carries its whole dependency tree inlined, so letting it
+ # contribute identifiers would clamp most of the report.
+ (repo / "vendor.js").write_text("var Widget=1,Orphan=2,render=3;\n", encoding="utf-8")
+ analyzer = _analyzer(repo, [("vendor.js", "minified")])
+ assert analyzer._unindexed_identifier_tokens() == frozenset()
+ finding = _finding(symbol_name="Widget")
+ analyzer._clamp_for_unindexed_importers([finding])
+ assert finding.safe_to_delete is True
+
+ def test_no_skipped_files_changes_nothing(self, repo: Path) -> None:
+ finding = _finding(symbol_name="Widget")
+ _analyzer(repo, [])._clamp_for_unindexed_importers([finding])
+ assert finding.safe_to_delete is True
+ assert finding.confidence == 0.9
+
+ def test_missing_file_on_disk_does_not_raise(self, repo: Path) -> None:
+ finding = _finding(symbol_name="Widget")
+ analyzer = _analyzer(repo, [("does_not_exist.tsx", "over_max_size")])
+ analyzer._clamp_for_unindexed_importers([finding])
+ assert finding.safe_to_delete is True
+
+ def test_per_file_read_is_bounded(self, repo: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ # These files are skipped because they are large; the scan must never
+ # be the thing that loads one fully into memory.
+ from repowise.core.analysis.dead_code import analyzer as analyzer_mod
+
+ monkeypatch.setattr(analyzer_mod, "_UNINDEXED_SCAN_PER_FILE_BYTES", 64)
+ (repo / "huge.ts").write_text(
+ "EarlyIdentifier\n" + ("pad\n" * 200) + "LateIdentifier\n", encoding="utf-8"
+ )
+ tokens = _analyzer(repo, [("huge.ts", "over_max_size")])._unindexed_identifier_tokens()
+ assert "EarlyIdentifier" in tokens
+ assert "LateIdentifier" not in tokens, "read was not bounded by the per-file budget"
+
+ def test_total_read_budget_stops_the_scan(
+ self, repo: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from repowise.core.analysis.dead_code import analyzer as analyzer_mod
+
+ monkeypatch.setattr(analyzer_mod, "_UNINDEXED_SCAN_TOTAL_BYTES", 32)
+ (repo / "one.ts").write_text("FirstIdentifier\n" + "pad\n" * 40, encoding="utf-8")
+ (repo / "two.ts").write_text("SecondIdentifier\n", encoding="utf-8")
+ tokens = _analyzer(
+ repo, [("one.ts", "over_max_size"), ("two.ts", "over_max_size")]
+ )._unindexed_identifier_tokens()
+ assert "FirstIdentifier" in tokens
+ assert "SecondIdentifier" not in tokens, "total budget did not stop the scan"
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index dfc9779f2..552c01575 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -225,12 +225,81 @@ class TestFileTraverser:
assert not any("app_ok.py" in p for p in paths)
assert not any("debug.log" in p for p in paths)
- def test_skips_oversized_files(self, tmp_path: Path) -> None:
+ def test_skips_oversized_files_without_a_parser(self, tmp_path: Path) -> None:
+ # No AST parser, so size is the only signal and the default cap holds.
+ big = tmp_path / "fixture.json"
+ big.write_bytes(b'{"k": 1}\n' * 200_000) # ~1.7 MB
+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)
+ paths = [f.path for f in traverser.traverse()]
+ assert not any("fixture.json" in p for p in paths)
+ assert traverser.stats.skipped_oversized == 1
+ # Blobs stay in the aggregate — naming them would drown the signal.
+ assert traverser.stats.skipped_source_files == []
+
+ def test_indexes_source_past_the_default_cap(self, tmp_path: Path) -> None:
+ # The #1237 regression: a repo's biggest hand-written module was
+ # dropped before any language check ran.
big = tmp_path / "big.py"
big.write_bytes(b"x = 1\n" * 200_000) # ~1.2 MB
traverser = FileTraverser(tmp_path, max_file_size_kb=500)
paths = [f.path for f in traverser.traverse()]
- assert not any("big.py" in p for p in paths)
+ assert any("big.py" in p for p in paths)
+ assert traverser.stats.skipped_oversized == 0
+
+ def test_skips_source_past_the_source_ceiling(self, tmp_path: Path) -> None:
+ # The ceiling is a memory budget: tree-sitter peaks at ~95 MB of RSS
+ # per MB of source and the parse pool runs 8 workers.
+ big = tmp_path / "huge.py"
+ big.write_bytes(b"x = 1\n" * 500_000) # ~2.9 MB, over the 2 MB ceiling
+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)
+ paths = [f.path for f in traverser.traverse()]
+ assert not any("huge.py" in p for p in paths)
+ assert traverser.stats.skipped_oversized == 1
+ # ...and it is named, because a silently dropped module is the bug.
+ assert [s.path for s in traverser.stats.skipped_source_files] == ["huge.py"]
+ assert traverser.stats.skipped_source_files[0].reason == "over_max_size"
+
+ def test_skips_minified_bundle_under_the_ceiling(self, tmp_path: Path) -> None:
+ # Real parser, source extension, under the ceiling — only the line
+ # shape distinguishes it from hand-written code.
+ bundle = tmp_path / "vendor.js"
+ bundle.write_bytes(b"!function(){" + b";".join(b"var a=1" for _ in range(90_000)) + b"}();")
+ assert bundle.stat().st_size > 500 * 1024
+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)
+ paths = [f.path for f in traverser.traverse()]
+ assert not any("vendor.js" in p for p in paths)
+ assert traverser.stats.skipped_oversized == 1
+ assert traverser.stats.skipped_source_files[0].reason == "minified"
+
+ def test_source_ceiling_cannot_be_raised_by_max_file_size_kb(self, tmp_path: Path) -> None:
+ # The source ceiling is a memory budget (~95 MB RSS per MB of source,
+ # times an 8-worker pool), not a preference. Raising the caller-facing
+ # knob must not be able to lift it, or the OOM guard is optional.
+ big = tmp_path / "huge.py"
+ big.write_bytes(b"x = 1\n" * 500_000) # ~2.9 MB, over the 2 MB ceiling
+ traverser = FileTraverser(tmp_path, max_file_size_kb=100_000) # 100 MB
+ paths = [f.path for f in traverser.traverse()]
+ assert not any("huge.py" in p for p in paths)
+ assert traverser.stats.skipped_oversized == 1
+
+ def test_max_file_size_kb_still_governs_parserless_files(self, tmp_path: Path) -> None:
+ # ...but it remains the knob for everything with no parser.
+ blob = tmp_path / "fixture.json"
+ blob.write_bytes(b'{"k": 1}\n' * 200_000) # ~1.7 MB
+ traverser = FileTraverser(tmp_path, max_file_size_kb=100_000)
+ paths = [f.path for f in traverser.traverse()]
+ assert any("fixture.json" in p for p in paths)
+
+ def test_normal_source_is_not_mistaken_for_minified(self, tmp_path: Path) -> None:
+ # Long-ish but ordinary lines must survive the guard: the threshold is
+ # 200 bytes/line and real source measured 25-60.
+ wide = tmp_path / "wide.py"
+ line = b"result = compute(" + b"argument_name, " * 8 + b"final)\n" # ~140 bytes
+ wide.write_bytes(line * 5_000)
+ assert wide.stat().st_size > 500 * 1024
+ traverser = FileTraverser(tmp_path, max_file_size_kb=500)
+ paths = [f.path for f in traverser.traverse()]
+ assert any("wide.py" in p for p in paths)
def test_deterministic_ordering(self, simple_repo: Path) -> None:
traverser = FileTraverser(simple_repo)
@@ -508,8 +577,10 @@ class TestTraversalStats:
assert traverser.stats.included >= 1
def test_stats_counts_oversized_skips(self, tmp_path: Path) -> None:
- big = tmp_path / "big.py"
- big.write_bytes(b"x = 1\n" * 200_000)
+ # A parserless blob past the cap: the counter must still fire, or the
+ # size guard has been removed rather than narrowed.
+ big = tmp_path / "big.json"
+ big.write_bytes(b'{"k": 1}\n' * 200_000)
(tmp_path / "small.py").write_text("pass")
traverser = FileTraverser(tmp_path, max_file_size_kb=500)
list(traverser.traverse())