Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix: route coverage discovery and repo pre-scan through the shared pruned walk (#897)

Raghav Chamadiya77f011589a85
diff --git a/packages/cli/src/repowise/cli/ui/repo_scanner.py b/packages/cli/src/repowise/cli/ui/repo_scanner.py
index 2ec898e9c..0cf13655a 100644
--- a/packages/cli/src/repowise/cli/ui/repo_scanner.py
+++ b/packages/cli/src/repowise/cli/ui/repo_scanner.py
@@ -10,6 +10,7 @@ from pathlib import Path
from rich.console import Console
from rich.panel import Panel
+from repowise.core.fs_walk import PRUNED_DIRS, walk_repo
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
@@ -51,39 +52,26 @@ for _lang, _exts in _LANG_MAP.items():
for _ext in _exts:
_EXT_TO_LANG[_ext] = _lang
-_SKIP_DIRS = {
- "node_modules",
- ".venv",
- "venv",
- "__pycache__",
- "dist",
- "build",
- ".next",
- "target",
- "vendor",
- ".git",
- ".hg",
- "env",
- ".tox",
- ".mypy_cache",
- ".ruff_cache",
- ".pytest_cache",
- "site-packages",
-}
+# Shared junk set plus names too ambiguous for the global prune list; a
+# miscounted scan stat is harmless, a wrongly unindexed dir is not.
+_SKIP_DIRS = PRUNED_DIRS | frozenset({"dist", "build", "target", "vendor", "env", "site-packages"})
def quick_repo_scan(repo_path: Path) -> RepoScanInfo:
"""Fast pre-scan: count files, detect languages, count git commits.
- No AST parsing — just ``os.walk`` + extension histogram + ``git rev-list --count``.
- Typically completes in <2s even on large repos.
+ No AST parsing — just the shared pruned walk + extension histogram +
+ ``git rev-list --count``. The walk skips junk dirs, nested git repos
+ (vendored/sibling checkouts must not inflate the stats), and junction
+ cycles. Typically completes in <2s even on large repos.
"""
info = RepoScanInfo()
dir_counts: dict[str, int] = {}
- for dirpath, dirnames, filenames in os.walk(repo_path):
- # Prune heavy/irrelevant directories in-place
- dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS and not d.startswith(".")]
+ for dirpath, dirnames, filenames in walk_repo(repo_path, prune_dirs=_SKIP_DIRS):
+ # Additionally skip all remaining dotdirs (IDE/tool config), like
+ # the pre-fs_walk scan always did.
+ dirnames[:] = [d for d in dirnames if not d.startswith(".")]
rel_dir = os.path.relpath(dirpath, repo_path)
top_dir = rel_dir.split(os.sep)[0] if rel_dir != "." else "."
diff --git a/packages/core/src/repowise/core/analysis/health/coverage/discovery.py b/packages/core/src/repowise/core/analysis/health/coverage/discovery.py
index 5950cb674..c4ae5e4d2 100644
--- a/packages/core/src/repowise/core/analysis/health/coverage/discovery.py
+++ b/packages/core/src/repowise/core/analysis/health/coverage/discovery.py
@@ -24,9 +24,13 @@ Two jobs that make ingested coverage *actually line up* with the repo:
from __future__ import annotations
from collections import defaultdict
+from collections.abc import Iterable
from dataclasses import dataclass, field
+from itertools import chain
from pathlib import Path
+from repowise.core.fs_walk import PRUNED_DIRS, WalkSnapshot
+
from .detector import parse as parse_coverage
from .model import ContextCoverageReport, CoverageReport, FileCoverage, TestCoverage
@@ -51,22 +55,11 @@ DEFAULT_DISCOVERY_GLOBS: tuple[str, ...] = (
)
# Directories we never descend into when expanding ``**`` patterns — heavy,
-# vendored, or irrelevant. Keeps discovery bounded on large repos.
-_PRUNE_DIRS = frozenset(
- {
- ".git",
- "node_modules",
- ".venv",
- "venv",
- "dist",
- "build",
- "__pycache__",
- ".next",
- ".mypy_cache",
- ".pytest_cache",
- ".tox",
- }
-)
+# vendored, or irrelevant. The shared junk set plus derived-output names;
+# NOT ``coverage``/``target`` (that is where the reports live). Applied at
+# traversal time via the shared pruned walk, and post-hoc as a safety net
+# for the non-recursive glob paths.
+_PRUNE_DIRS = PRUNED_DIRS | frozenset({"dist", "build"})
# Hard cap on discovered artifacts — a sane upper bound that still covers
# polyglot monorepos with several per-package reports.
@@ -154,14 +147,45 @@ def discover_artifacts(
"""Return coverage report files under *repo_root*, de-duplicated.
Globs the filesystem directly (the report dirs are excluded from the
- indexed file set). Results are pruned of vendored dirs, de-duplicated
- by resolved path, and capped at :data:`_MAX_ARTIFACTS`.
+ indexed file set). Recursive ``**`` patterns are expanded through the
+ shared pruned walk — ONE filesystem pass answers all of them, and the
+ walk never descends into junk trees or nested git repos (recursive
+ ``Path.glob`` did, and was filtered only post-hoc). Literal patterns
+ stay direct file checks and single-level wildcards stay plain globs,
+ so each pattern keeps its original location scoping (``lcov.info``
+ matches at the root only, ``**/clover.xml`` anywhere). Results keep
+ pattern-priority order, are de-duplicated by resolved path, and are
+ capped at :data:`_MAX_ARTIFACTS`.
"""
patterns = tuple(globs) if globs else DEFAULT_DISCOVERY_GLOBS
+ snapshot: WalkSnapshot | None = None
+
+ def _expand(pattern: str) -> Iterable[Path]:
+ nonlocal snapshot
+ if not any(ch in pattern for ch in "*?["):
+ direct = repo_root / pattern
+ return (direct,) if direct.is_file() else ()
+ if "**" not in pattern:
+ # Single-level wildcards cannot recurse; a plain glob is bounded.
+ return repo_root.glob(pattern)
+ # Recursive pattern: split at the first ``**`` into a fixed (or
+ # shallow-globbed) root and a tail served from the shared snapshot.
+ prefix, _, tail = pattern.partition("**")
+ prefix = prefix.rstrip("/")
+ tail = tail.lstrip("/") or "*"
+ if snapshot is None:
+ snapshot = WalkSnapshot(repo_root, prune_dirs=_PRUNE_DIRS)
+ snap = snapshot
+ if any(ch in prefix for ch in "*?["):
+ roots = [d for d in repo_root.glob(prefix) if d.is_dir()]
+ else:
+ roots = [repo_root / prefix]
+ return chain.from_iterable(snap.iter_glob(r, tail) for r in roots)
+
seen: set[Path] = set()
out: list[Path] = []
for pattern in patterns:
- for match in repo_root.glob(pattern):
+ for match in _expand(pattern):
if not match.is_file():
continue
try:
diff --git a/tests/unit/cli/test_repo_scanner.py b/tests/unit/cli/test_repo_scanner.py
new file mode 100644
index 000000000..5b074fe77
--- /dev/null
+++ b/tests/unit/cli/test_repo_scanner.py
@@ -0,0 +1,70 @@
+"""Tests for the pre-scan repo walk (``repowise.cli.ui.repo_scanner``).
+
+The pre-scan feeds the mode-selection UI (file counts, language mix, exclude
+suggestions), so a walk that wanders into vendored checkouts or junk trees
+shows the user stats for a different codebase.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.cli.ui.repo_scanner import quick_repo_scan
+
+
+def _write(root: Path, rel: str, content: str = "x") -> None:
+ p = root / rel
+ p.parent.mkdir(parents=True, exist_ok=True)
+ p.write_text(content, encoding="utf-8")
+
+
+def test_counts_and_language_histogram(tmp_path: Path) -> None:
+ _write(tmp_path, "src/app.py")
+ _write(tmp_path, "src/util.py")
+ _write(tmp_path, "web/index.ts")
+ info = quick_repo_scan(tmp_path)
+ assert info.total_files == 3
+ assert info.language_counts["Python"] == 2
+ assert info.language_counts["TypeScript"] == 1
+
+
+def test_junk_dirs_not_counted(tmp_path: Path) -> None:
+ _write(tmp_path, "src/app.py")
+ _write(tmp_path, "node_modules/dep/index.js")
+ _write(tmp_path, ".venv/lib/site.py")
+ _write(tmp_path, "dist/bundle.js")
+ _write(tmp_path, "vendor/lib.go")
+ info = quick_repo_scan(tmp_path)
+ assert info.total_files == 1
+ assert "JavaScript" not in info.language_counts
+ assert "Go" not in info.language_counts
+
+
+def test_nested_git_repo_not_counted(tmp_path: Path) -> None:
+ """A sibling/vendored checkout must not inflate the pre-scan stats or
+ the large-dir exclude suggestions."""
+ _write(tmp_path, "src/app.py")
+ (tmp_path / "sibling-repo" / ".git").mkdir(parents=True)
+ for i in range(60):
+ _write(tmp_path, f"sibling-repo/src/file{i}.go")
+ info = quick_repo_scan(tmp_path)
+ assert info.total_files == 1
+ assert "Go" not in info.language_counts
+ assert all(d != "sibling-repo" for d, _ in info.large_dirs)
+
+
+def test_own_git_dir_not_counted_but_root_exempt(tmp_path: Path) -> None:
+ """The repo's own .git is skipped as a junk dir, not as a nested repo."""
+ (tmp_path / ".git").mkdir()
+ _write(tmp_path, ".git/config")
+ _write(tmp_path, "main.py")
+ info = quick_repo_scan(tmp_path)
+ assert info.total_files == 1
+
+
+def test_large_dirs_reported(tmp_path: Path) -> None:
+ for i in range(60):
+ _write(tmp_path, f"generated/file{i}.py")
+ _write(tmp_path, "src/app.py")
+ info = quick_repo_scan(tmp_path)
+ assert ("generated", 60) in info.large_dirs
diff --git a/tests/unit/health/test_coverage_discovery.py b/tests/unit/health/test_coverage_discovery.py
index a97f0068c..42433f38c 100644
--- a/tests/unit/health/test_coverage_discovery.py
+++ b/tests/unit/health/test_coverage_discovery.py
@@ -147,6 +147,80 @@ def test_discover_finds_common_locations(tmp_path: Path) -> None:
assert all("node_modules" not in p.parts for p in found)
+def test_discover_recursive_patterns_reach_deep_files(tmp_path: Path) -> None:
+ """The ``**`` patterns keep their reach through the pruned-walk rewrite."""
+ deep_cob = tmp_path / "pkg" / "reports" / "cobertura.xml"
+ deep_cob.parent.mkdir(parents=True)
+ deep_cob.write_text("<coverage/>")
+ deep_lcov = tmp_path / "coverage" / "unit" / "deep" / "lcov.info"
+ deep_lcov.parent.mkdir(parents=True)
+ deep_lcov.write_text("SF:a\nend_of_record\n")
+ rust = tmp_path / "target" / "llvm-cov" / "html" / "cov.lcov"
+ rust.parent.mkdir(parents=True)
+ rust.write_text("SF:a\nend_of_record\n")
+
+ found = set(discover_artifacts(tmp_path))
+ assert {deep_cob, deep_lcov, rust} <= found
+
+
+def test_discover_root_only_patterns_stay_root_only(tmp_path: Path) -> None:
+ """Literal patterns must NOT gain match-anywhere semantics: a checked-in
+ fixture named ``lcov.info``/``coverage.xml`` deep in the tree is not a
+ report for THIS repo."""
+ (tmp_path / "lcov.info").write_text("SF:a\nend_of_record\n")
+ fixture = tmp_path / "tests" / "fixtures" / "lcov.info"
+ fixture.parent.mkdir(parents=True)
+ fixture.write_text("SF:b\nend_of_record\n")
+ stray_xml = tmp_path / "some" / "dir" / "coverage.xml"
+ stray_xml.parent.mkdir(parents=True)
+ stray_xml.write_text("<coverage/>")
+
+ found = set(discover_artifacts(tmp_path))
+ assert tmp_path / "lcov.info" in found
+ assert fixture not in found
+ assert stray_xml not in found
+
+
+def test_discover_prunes_nested_git_repos(tmp_path: Path) -> None:
+ """A vendored/sibling checkout's reports belong to a different repo."""
+ (tmp_path / "coverage").mkdir()
+ (tmp_path / "coverage" / "lcov.info").write_text("SF:a\nend_of_record\n")
+ sibling = tmp_path / "vendored-repo"
+ (sibling / ".git").mkdir(parents=True)
+ (sibling / "reports").mkdir()
+ (sibling / "reports" / "cobertura.xml").write_text("<coverage/>")
+
+ found = discover_artifacts(tmp_path)
+ assert all("vendored-repo" not in p.parts for p in found)
+ assert tmp_path / "coverage" / "lcov.info" in set(found)
+
+
+def test_discover_pattern_priority_order_preserved(tmp_path: Path) -> None:
+ """Earlier (more canonical) patterns yield earlier results."""
+ canonical = tmp_path / "coverage" / "lcov.info"
+ canonical.parent.mkdir()
+ canonical.write_text("SF:a\nend_of_record\n")
+ late = tmp_path / "sub" / "clover.xml"
+ late.parent.mkdir()
+ late.write_text("<coverage/>")
+
+ found = discover_artifacts(tmp_path)
+ assert found.index(canonical) < found.index(late)
+
+
+def test_discover_user_glob_override_still_works(tmp_path: Path) -> None:
+ """CoverageConfig.artifacts globs route through the same expansion."""
+ deep = tmp_path / "out" / "cov" / "report.info"
+ deep.parent.mkdir(parents=True)
+ deep.write_text("SF:a\nend_of_record\n")
+ shallow = tmp_path / "reports" / "cov.xml"
+ shallow.parent.mkdir()
+ shallow.write_text("<coverage/>")
+
+ assert set(discover_artifacts(tmp_path, globs=["out/**/*.info"])) == {deep}
+ assert set(discover_artifacts(tmp_path, globs=["reports/*.xml"])) == {shallow}
+
+
def test_build_coverage_map_end_to_end(tmp_path: Path) -> None:
lcov = "SF:/ci/build/src/a.ts\nDA:1,1\nDA:2,0\nend_of_record\n"
report_path = tmp_path / "coverage" / "lcov.info"
diff --git a/tests/unit/ingestion/test_fs_walk.py b/tests/unit/ingestion/test_fs_walk.py
index 21b9d00e0..02b3f62a0 100644
--- a/tests/unit/ingestion/test_fs_walk.py
+++ b/tests/unit/ingestion/test_fs_walk.py
@@ -1,6 +1,6 @@
"""Unit tests for the shared pruned filesystem walk (``repowise.core.fs_walk``)
-plus the regression guard that keeps new unpruned ``rglob`` calls out of
-``packages/core``.
+plus the regression guard that keeps new unpruned walk call sites (``rglob``,
+``os.walk``, recursive ``glob``) out of packages/core, cli, and server.
Why this exists: unpruned ``Path.rglob`` over a repo that physically contains
sibling/vendored repos (or ``node_modules`` / ``.venv``) has caused two
@@ -276,43 +276,102 @@ class TestExternalSystemsDiscoverDepth:
# ---------------------------------------------------------------------------
-# Regression guard — no new direct rglob calls in packages/core
+# Regression guard — no new direct filesystem-walk call sites
# ---------------------------------------------------------------------------
-
-# Files allowed to mention ``.rglob(``. Additions require justification:
-# either the walk root is a tightly-bounded subdirectory that cannot contain
-# junk trees or nested repos, or the file IS the shared implementation.
-_RGLOB_ALLOWLIST = {
- # The shared implementation (docstring references rglob semantics).
- "fs_walk.py",
- # Decision mining: bounded to an explicit docs/ directory.
- "analysis/decisions/extractor.py",
+#
+# Three call shapes, three histories:
+# .rglob( two multi-minute init stalls plus foreign-manifest leaks
+# (see module docstring)
+# os.walk( every hand-rolled walk re-invented (and drifted) its own
+# prune list, and none carried the nested-git or
+# junction-cycle guards
+# .glob("...** recursive Path.glob descends into node_modules/.venv and
+# nested repos even when matches are filtered afterwards;
+# coverage discovery shipped exactly this bug
+#
+# Allowlist additions require a justification comment: the walk root is a
+# tightly-bounded directory that cannot contain junk trees or nested repos,
+# the file IS the shared implementation, or migration onto fs_walk is a
+# tracked follow-up. Matching is on comment-stripped lines, so prose
+# mentions of these names in ``#`` comments do not trip the guard
+# (docstring mentions still do — keep call syntax out of docstrings).
+
+_GUARD_KINDS: tuple[tuple[str, re.Pattern[str]], ...] = (
+ ("rglob", re.compile(r"\.rglob\(")),
+ ("os.walk", re.compile(r"\bos\.walk\(")),
+ ("recursive-glob", re.compile(r"""\.glob\(\s*[rbuf]*['"][^'"]*\*\*""")),
+)
+
+_WALK_ALLOWLIST: dict[tuple[str, str], frozenset[str]] = {
+ # The shared implementation (uses os.walk; docstrings reference rglob).
+ ("core", "fs_walk.py"): frozenset({"rglob", "os.walk"}),
+ # The gitignore-aware index walk; migration onto walk_repo is a tracked
+ # follow-up (it must keep its pathspec layer on top).
+ ("core", "ingestion/traverser.py"): frozenset({"os.walk"}),
+ # Repo DISCOVERY: exists to find nested .git dirs, prunes in-place;
+ # migration onto walk_repo(prune_nested_git=False) is a tracked follow-up.
+ ("core", "workspace/scanner.py"): frozenset({"os.walk"}),
+ # Decision mining: pruned in-place walks + an rglob bounded to docs/;
+ # migration is a tracked follow-up (sequenced after the source_map reuse).
+ ("core", "analysis/decisions/extractor.py"): frozenset({"rglob", "os.walk"}),
+ # Sizes .repowise/ only — repowise-owned, cannot contain junk trees.
+ ("cli", "commands/status_cmd.py"): frozenset({"rglob"}),
+ # Scans repowise's own packaged web/ui source dirs, not the user repo.
+ ("cli", "commands/serve_cmd.py"): frozenset({"rglob"}),
+ # Sizes .repowise/ only — repowise-owned, cannot contain junk trees.
+ ("server", "routers/overview.py"): frozenset({"rglob"}),
}
-_RGLOB_RE = re.compile(r"\.rglob\(")
-
-def _core_src_root() -> Path:
- # tests/unit/ingestion/ → repo root → packages/core/src/repowise/core
+def _guarded_src_roots() -> dict[str, Path]:
+ # tests/unit/ingestion/ → repo root → packages/<pkg>/src/repowise/<pkg>
repo_root = Path(__file__).resolve().parents[3]
- return repo_root / "packages" / "core" / "src" / "repowise" / "core"
+ return {
+ "core": repo_root / "packages" / "core" / "src" / "repowise" / "core",
+ "cli": repo_root / "packages" / "cli" / "src" / "repowise" / "cli",
+ "server": repo_root / "packages" / "server" / "src" / "repowise" / "server",
+ }
+
+
+def _strip_comments(text: str) -> str:
+ return "\n".join(line.split("#", 1)[0] for line in text.splitlines())
-class TestNoUnprunedRglobRegression:
- def test_no_new_rglob_call_sites_in_core(self) -> None:
- core = _core_src_root()
- assert core.is_dir(), f"layout changed? {core}"
+class TestNoUnprunedWalkRegression:
+ def test_no_new_walk_call_sites(self) -> None:
offenders: list[str] = []
- for py in core.rglob("*.py"): # tests may rglob; core may not.
- rel = py.relative_to(core).as_posix()
- if rel in _RGLOB_ALLOWLIST:
- continue
- text = py.read_text(encoding="utf-8", errors="ignore")
- if _RGLOB_RE.search(text):
- offenders.append(rel)
+ for tree, root in _guarded_src_roots().items():
+ assert root.is_dir(), f"layout changed? {root}"
+ for py in root.rglob("*.py"): # tests may rglob; src may not.
+ rel = py.relative_to(root).as_posix()
+ allowed = _WALK_ALLOWLIST.get((tree, rel), frozenset())
+ text = _strip_comments(py.read_text(encoding="utf-8", errors="ignore"))
+ for kind, pattern in _GUARD_KINDS:
+ if kind in allowed:
+ continue
+ if pattern.search(text):
+ offenders.append(f"{tree}:{rel} [{kind}]")
assert not offenders, (
- "Direct .rglob( calls found in packages/core — use "
- "repowise.core.fs_walk.iter_glob/walk_repo instead (prunes "
- "node_modules/.venv AND nested git repos). Unpruned rglob has "
- f"caused multi-minute init stalls twice. Offenders: {offenders}"
+ "Direct filesystem-walk calls found — use "
+ "repowise.core.fs_walk.iter_glob/walk_repo/WalkSnapshot instead "
+ "(prunes node_modules/.venv AND nested git repos, guards against "
+ "junction cycles). Unpruned walks have caused multi-minute init "
+ f"stalls and cross-repo manifest leaks. Offenders: {offenders}"
)
+
+ def test_allowlist_entries_still_needed(self) -> None:
+ """An allowlist entry whose file no longer matches its kinds is stale —
+ prune it so the exemption cannot silently cover future code."""
+ roots = _guarded_src_roots()
+ kind_res = dict(_GUARD_KINDS)
+ stale: list[str] = []
+ for (tree, rel), kinds in _WALK_ALLOWLIST.items():
+ py = roots[tree] / rel
+ if not py.is_file():
+ stale.append(f"{tree}:{rel} (file gone)")
+ continue
+ text = _strip_comments(py.read_text(encoding="utf-8", errors="ignore"))
+ for kind in kinds:
+ if not kind_res[kind].search(text):
+ stale.append(f"{tree}:{rel} [{kind}]")
+ assert not stale, f"Stale allowlist entries — remove them: {stale}"