Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix: route coverage discovery and repo pre-scan through the shared pruned walk (#897)
Raghav Chamadiya77f011589a85
diff --git a/packages/cli/src/repowise/cli/ui/repo_scanner.py b/packages/cli/src/repowise/cli/ui/repo_scanner.pyindex 2ec898e9c..0cf13655a 100644--- a/packages/cli/src/repowise/cli/ui/repo_scanner.py+++ b/packages/cli/src/repowise/cli/ui/repo_scanner.py@@ -10,6 +10,7 @@ from pathlib import Pathfrom rich.console import Consolefrom rich.panel import Panel+from repowise.core.fs_walk import PRUNED_DIRS, walk_repofrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY@@ -51,39 +52,26 @@ for _lang, _exts in _LANG_MAP.items():for _ext in _exts:_EXT_TO_LANG[_ext] = _lang-_SKIP_DIRS = {- "node_modules",- ".venv",- "venv",- "__pycache__",- "dist",- "build",- ".next",- "target",- "vendor",- ".git",- ".hg",- "env",- ".tox",- ".mypy_cache",- ".ruff_cache",- ".pytest_cache",- "site-packages",-}+# Shared junk set plus names too ambiguous for the global prune list; a+# miscounted scan stat is harmless, a wrongly unindexed dir is not.+_SKIP_DIRS = PRUNED_DIRS | frozenset({"dist", "build", "target", "vendor", "env", "site-packages"})def quick_repo_scan(repo_path: Path) -> RepoScanInfo:"""Fast pre-scan: count files, detect languages, count git commits.- No AST parsing — just ``os.walk`` + extension histogram + ``git rev-list --count``.- Typically completes in <2s even on large repos.+ No AST parsing — just the shared pruned walk + extension histogram ++ ``git rev-list --count``. The walk skips junk dirs, nested git repos+ (vendored/sibling checkouts must not inflate the stats), and junction+ cycles. Typically completes in <2s even on large repos."""info = RepoScanInfo()dir_counts: dict[str, int] = {}- for dirpath, dirnames, filenames in os.walk(repo_path):- # Prune heavy/irrelevant directories in-place- dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS and not d.startswith(".")]+ for dirpath, dirnames, filenames in walk_repo(repo_path, prune_dirs=_SKIP_DIRS):+ # Additionally skip all remaining dotdirs (IDE/tool config), like+ # the pre-fs_walk scan always did.+ dirnames[:] = [d for d in dirnames if not d.startswith(".")]rel_dir = os.path.relpath(dirpath, repo_path)top_dir = rel_dir.split(os.sep)[0] if rel_dir != "." else "."diff --git a/packages/core/src/repowise/core/analysis/health/coverage/discovery.py b/packages/core/src/repowise/core/analysis/health/coverage/discovery.pyindex 5950cb674..c4ae5e4d2 100644--- a/packages/core/src/repowise/core/analysis/health/coverage/discovery.py+++ b/packages/core/src/repowise/core/analysis/health/coverage/discovery.py@@ -24,9 +24,13 @@ Two jobs that make ingested coverage *actually line up* with the repo:from __future__ import annotationsfrom collections import defaultdict+from collections.abc import Iterablefrom dataclasses import dataclass, field+from itertools import chainfrom pathlib import Path+from repowise.core.fs_walk import PRUNED_DIRS, WalkSnapshot+from .detector import parse as parse_coveragefrom .model import ContextCoverageReport, CoverageReport, FileCoverage, TestCoverage@@ -51,22 +55,11 @@ DEFAULT_DISCOVERY_GLOBS: tuple[str, ...] = ()# Directories we never descend into when expanding ``**`` patterns — heavy,-# vendored, or irrelevant. Keeps discovery bounded on large repos.-_PRUNE_DIRS = frozenset(- {- ".git",- "node_modules",- ".venv",- "venv",- "dist",- "build",- "__pycache__",- ".next",- ".mypy_cache",- ".pytest_cache",- ".tox",- }-)+# vendored, or irrelevant. The shared junk set plus derived-output names;+# NOT ``coverage``/``target`` (that is where the reports live). Applied at+# traversal time via the shared pruned walk, and post-hoc as a safety net+# for the non-recursive glob paths.+_PRUNE_DIRS = PRUNED_DIRS | frozenset({"dist", "build"})# Hard cap on discovered artifacts — a sane upper bound that still covers# polyglot monorepos with several per-package reports.@@ -154,14 +147,45 @@ def discover_artifacts("""Return coverage report files under *repo_root*, de-duplicated.Globs the filesystem directly (the report dirs are excluded from the- indexed file set). Results are pruned of vendored dirs, de-duplicated- by resolved path, and capped at :data:`_MAX_ARTIFACTS`.+ indexed file set). Recursive ``**`` patterns are expanded through the+ shared pruned walk — ONE filesystem pass answers all of them, and the+ walk never descends into junk trees or nested git repos (recursive+ ``Path.glob`` did, and was filtered only post-hoc). Literal patterns+ stay direct file checks and single-level wildcards stay plain globs,+ so each pattern keeps its original location scoping (``lcov.info``+ matches at the root only, ``**/clover.xml`` anywhere). Results keep+ pattern-priority order, are de-duplicated by resolved path, and are+ capped at :data:`_MAX_ARTIFACTS`."""patterns = tuple(globs) if globs else DEFAULT_DISCOVERY_GLOBS+ snapshot: WalkSnapshot | None = None++ def _expand(pattern: str) -> Iterable[Path]:+ nonlocal snapshot+ if not any(ch in pattern for ch in "*?["):+ direct = repo_root / pattern+ return (direct,) if direct.is_file() else ()+ if "**" not in pattern:+ # Single-level wildcards cannot recurse; a plain glob is bounded.+ return repo_root.glob(pattern)+ # Recursive pattern: split at the first ``**`` into a fixed (or+ # shallow-globbed) root and a tail served from the shared snapshot.+ prefix, _, tail = pattern.partition("**")+ prefix = prefix.rstrip("/")+ tail = tail.lstrip("/") or "*"+ if snapshot is None:+ snapshot = WalkSnapshot(repo_root, prune_dirs=_PRUNE_DIRS)+ snap = snapshot+ if any(ch in prefix for ch in "*?["):+ roots = [d for d in repo_root.glob(prefix) if d.is_dir()]+ else:+ roots = [repo_root / prefix]+ return chain.from_iterable(snap.iter_glob(r, tail) for r in roots)+seen: set[Path] = set()out: list[Path] = []for pattern in patterns:- for match in repo_root.glob(pattern):+ for match in _expand(pattern):if not match.is_file():continuetry:diff --git a/tests/unit/cli/test_repo_scanner.py b/tests/unit/cli/test_repo_scanner.pynew file mode 100644index 000000000..5b074fe77--- /dev/null+++ b/tests/unit/cli/test_repo_scanner.py@@ -0,0 +1,70 @@+"""Tests for the pre-scan repo walk (``repowise.cli.ui.repo_scanner``).++The pre-scan feeds the mode-selection UI (file counts, language mix, exclude+suggestions), so a walk that wanders into vendored checkouts or junk trees+shows the user stats for a different codebase.+"""++from __future__ import annotations++from pathlib import Path++from repowise.cli.ui.repo_scanner import quick_repo_scan+++def _write(root: Path, rel: str, content: str = "x") -> None:+ p = root / rel+ p.parent.mkdir(parents=True, exist_ok=True)+ p.write_text(content, encoding="utf-8")+++def test_counts_and_language_histogram(tmp_path: Path) -> None:+ _write(tmp_path, "src/app.py")+ _write(tmp_path, "src/util.py")+ _write(tmp_path, "web/index.ts")+ info = quick_repo_scan(tmp_path)+ assert info.total_files == 3+ assert info.language_counts["Python"] == 2+ assert info.language_counts["TypeScript"] == 1+++def test_junk_dirs_not_counted(tmp_path: Path) -> None:+ _write(tmp_path, "src/app.py")+ _write(tmp_path, "node_modules/dep/index.js")+ _write(tmp_path, ".venv/lib/site.py")+ _write(tmp_path, "dist/bundle.js")+ _write(tmp_path, "vendor/lib.go")+ info = quick_repo_scan(tmp_path)+ assert info.total_files == 1+ assert "JavaScript" not in info.language_counts+ assert "Go" not in info.language_counts+++def test_nested_git_repo_not_counted(tmp_path: Path) -> None:+ """A sibling/vendored checkout must not inflate the pre-scan stats or+ the large-dir exclude suggestions."""+ _write(tmp_path, "src/app.py")+ (tmp_path / "sibling-repo" / ".git").mkdir(parents=True)+ for i in range(60):+ _write(tmp_path, f"sibling-repo/src/file{i}.go")+ info = quick_repo_scan(tmp_path)+ assert info.total_files == 1+ assert "Go" not in info.language_counts+ assert all(d != "sibling-repo" for d, _ in info.large_dirs)+++def test_own_git_dir_not_counted_but_root_exempt(tmp_path: Path) -> None:+ """The repo's own .git is skipped as a junk dir, not as a nested repo."""+ (tmp_path / ".git").mkdir()+ _write(tmp_path, ".git/config")+ _write(tmp_path, "main.py")+ info = quick_repo_scan(tmp_path)+ assert info.total_files == 1+++def test_large_dirs_reported(tmp_path: Path) -> None:+ for i in range(60):+ _write(tmp_path, f"generated/file{i}.py")+ _write(tmp_path, "src/app.py")+ info = quick_repo_scan(tmp_path)+ assert ("generated", 60) in info.large_dirsdiff --git a/tests/unit/health/test_coverage_discovery.py b/tests/unit/health/test_coverage_discovery.pyindex a97f0068c..42433f38c 100644--- a/tests/unit/health/test_coverage_discovery.py+++ b/tests/unit/health/test_coverage_discovery.py@@ -147,6 +147,80 @@ def test_discover_finds_common_locations(tmp_path: Path) -> None:assert all("node_modules" not in p.parts for p in found)+def test_discover_recursive_patterns_reach_deep_files(tmp_path: Path) -> None:+ """The ``**`` patterns keep their reach through the pruned-walk rewrite."""+ deep_cob = tmp_path / "pkg" / "reports" / "cobertura.xml"+ deep_cob.parent.mkdir(parents=True)+ deep_cob.write_text("<coverage/>")+ deep_lcov = tmp_path / "coverage" / "unit" / "deep" / "lcov.info"+ deep_lcov.parent.mkdir(parents=True)+ deep_lcov.write_text("SF:a\nend_of_record\n")+ rust = tmp_path / "target" / "llvm-cov" / "html" / "cov.lcov"+ rust.parent.mkdir(parents=True)+ rust.write_text("SF:a\nend_of_record\n")++ found = set(discover_artifacts(tmp_path))+ assert {deep_cob, deep_lcov, rust} <= found+++def test_discover_root_only_patterns_stay_root_only(tmp_path: Path) -> None:+ """Literal patterns must NOT gain match-anywhere semantics: a checked-in+ fixture named ``lcov.info``/``coverage.xml`` deep in the tree is not a+ report for THIS repo."""+ (tmp_path / "lcov.info").write_text("SF:a\nend_of_record\n")+ fixture = tmp_path / "tests" / "fixtures" / "lcov.info"+ fixture.parent.mkdir(parents=True)+ fixture.write_text("SF:b\nend_of_record\n")+ stray_xml = tmp_path / "some" / "dir" / "coverage.xml"+ stray_xml.parent.mkdir(parents=True)+ stray_xml.write_text("<coverage/>")++ found = set(discover_artifacts(tmp_path))+ assert tmp_path / "lcov.info" in found+ assert fixture not in found+ assert stray_xml not in found+++def test_discover_prunes_nested_git_repos(tmp_path: Path) -> None:+ """A vendored/sibling checkout's reports belong to a different repo."""+ (tmp_path / "coverage").mkdir()+ (tmp_path / "coverage" / "lcov.info").write_text("SF:a\nend_of_record\n")+ sibling = tmp_path / "vendored-repo"+ (sibling / ".git").mkdir(parents=True)+ (sibling / "reports").mkdir()+ (sibling / "reports" / "cobertura.xml").write_text("<coverage/>")++ found = discover_artifacts(tmp_path)+ assert all("vendored-repo" not in p.parts for p in found)+ assert tmp_path / "coverage" / "lcov.info" in set(found)+++def test_discover_pattern_priority_order_preserved(tmp_path: Path) -> None:+ """Earlier (more canonical) patterns yield earlier results."""+ canonical = tmp_path / "coverage" / "lcov.info"+ canonical.parent.mkdir()+ canonical.write_text("SF:a\nend_of_record\n")+ late = tmp_path / "sub" / "clover.xml"+ late.parent.mkdir()+ late.write_text("<coverage/>")++ found = discover_artifacts(tmp_path)+ assert found.index(canonical) < found.index(late)+++def test_discover_user_glob_override_still_works(tmp_path: Path) -> None:+ """CoverageConfig.artifacts globs route through the same expansion."""+ deep = tmp_path / "out" / "cov" / "report.info"+ deep.parent.mkdir(parents=True)+ deep.write_text("SF:a\nend_of_record\n")+ shallow = tmp_path / "reports" / "cov.xml"+ shallow.parent.mkdir()+ shallow.write_text("<coverage/>")++ assert set(discover_artifacts(tmp_path, globs=["out/**/*.info"])) == {deep}+ assert set(discover_artifacts(tmp_path, globs=["reports/*.xml"])) == {shallow}++def test_build_coverage_map_end_to_end(tmp_path: Path) -> None:lcov = "SF:/ci/build/src/a.ts\nDA:1,1\nDA:2,0\nend_of_record\n"report_path = tmp_path / "coverage" / "lcov.info"diff --git a/tests/unit/ingestion/test_fs_walk.py b/tests/unit/ingestion/test_fs_walk.pyindex 21b9d00e0..02b3f62a0 100644--- a/tests/unit/ingestion/test_fs_walk.py+++ b/tests/unit/ingestion/test_fs_walk.py@@ -1,6 +1,6 @@"""Unit tests for the shared pruned filesystem walk (``repowise.core.fs_walk``)-plus the regression guard that keeps new unpruned ``rglob`` calls out of-``packages/core``.+plus the regression guard that keeps new unpruned walk call sites (``rglob``,+``os.walk``, recursive ``glob``) out of packages/core, cli, and server.Why this exists: unpruned ``Path.rglob`` over a repo that physically containssibling/vendored repos (or ``node_modules`` / ``.venv``) has caused two@@ -276,43 +276,102 @@ class TestExternalSystemsDiscoverDepth:# ----------------------------------------------------------------------------# Regression guard — no new direct rglob calls in packages/core+# Regression guard — no new direct filesystem-walk call sites# -----------------------------------------------------------------------------# Files allowed to mention ``.rglob(``. Additions require justification:-# either the walk root is a tightly-bounded subdirectory that cannot contain-# junk trees or nested repos, or the file IS the shared implementation.-_RGLOB_ALLOWLIST = {- # The shared implementation (docstring references rglob semantics).- "fs_walk.py",- # Decision mining: bounded to an explicit docs/ directory.- "analysis/decisions/extractor.py",+#+# Three call shapes, three histories:+# .rglob( two multi-minute init stalls plus foreign-manifest leaks+# (see module docstring)+# os.walk( every hand-rolled walk re-invented (and drifted) its own+# prune list, and none carried the nested-git or+# junction-cycle guards+# .glob("...** recursive Path.glob descends into node_modules/.venv and+# nested repos even when matches are filtered afterwards;+# coverage discovery shipped exactly this bug+#+# Allowlist additions require a justification comment: the walk root is a+# tightly-bounded directory that cannot contain junk trees or nested repos,+# the file IS the shared implementation, or migration onto fs_walk is a+# tracked follow-up. Matching is on comment-stripped lines, so prose+# mentions of these names in ``#`` comments do not trip the guard+# (docstring mentions still do — keep call syntax out of docstrings).++_GUARD_KINDS: tuple[tuple[str, re.Pattern[str]], ...] = (+ ("rglob", re.compile(r"\.rglob\(")),+ ("os.walk", re.compile(r"\bos\.walk\(")),+ ("recursive-glob", re.compile(r"""\.glob\(\s*[rbuf]*['"][^'"]*\*\*""")),+)++_WALK_ALLOWLIST: dict[tuple[str, str], frozenset[str]] = {+ # The shared implementation (uses os.walk; docstrings reference rglob).+ ("core", "fs_walk.py"): frozenset({"rglob", "os.walk"}),+ # The gitignore-aware index walk; migration onto walk_repo is a tracked+ # follow-up (it must keep its pathspec layer on top).+ ("core", "ingestion/traverser.py"): frozenset({"os.walk"}),+ # Repo DISCOVERY: exists to find nested .git dirs, prunes in-place;+ # migration onto walk_repo(prune_nested_git=False) is a tracked follow-up.+ ("core", "workspace/scanner.py"): frozenset({"os.walk"}),+ # Decision mining: pruned in-place walks + an rglob bounded to docs/;+ # migration is a tracked follow-up (sequenced after the source_map reuse).+ ("core", "analysis/decisions/extractor.py"): frozenset({"rglob", "os.walk"}),+ # Sizes .repowise/ only — repowise-owned, cannot contain junk trees.+ ("cli", "commands/status_cmd.py"): frozenset({"rglob"}),+ # Scans repowise's own packaged web/ui source dirs, not the user repo.+ ("cli", "commands/serve_cmd.py"): frozenset({"rglob"}),+ # Sizes .repowise/ only — repowise-owned, cannot contain junk trees.+ ("server", "routers/overview.py"): frozenset({"rglob"}),}-_RGLOB_RE = re.compile(r"\.rglob\(")--def _core_src_root() -> Path:- # tests/unit/ingestion/ → repo root → packages/core/src/repowise/core+def _guarded_src_roots() -> dict[str, Path]:+ # tests/unit/ingestion/ → repo root → packages/<pkg>/src/repowise/<pkg>repo_root = Path(__file__).resolve().parents[3]- return repo_root / "packages" / "core" / "src" / "repowise" / "core"+ return {+ "core": repo_root / "packages" / "core" / "src" / "repowise" / "core",+ "cli": repo_root / "packages" / "cli" / "src" / "repowise" / "cli",+ "server": repo_root / "packages" / "server" / "src" / "repowise" / "server",+ }+++def _strip_comments(text: str) -> str:+ return "\n".join(line.split("#", 1)[0] for line in text.splitlines())-class TestNoUnprunedRglobRegression:- def test_no_new_rglob_call_sites_in_core(self) -> None:- core = _core_src_root()- assert core.is_dir(), f"layout changed? {core}"+class TestNoUnprunedWalkRegression:+ def test_no_new_walk_call_sites(self) -> None:offenders: list[str] = []- for py in core.rglob("*.py"): # tests may rglob; core may not.- rel = py.relative_to(core).as_posix()- if rel in _RGLOB_ALLOWLIST:- continue- text = py.read_text(encoding="utf-8", errors="ignore")- if _RGLOB_RE.search(text):- offenders.append(rel)+ for tree, root in _guarded_src_roots().items():+ assert root.is_dir(), f"layout changed? {root}"+ for py in root.rglob("*.py"): # tests may rglob; src may not.+ rel = py.relative_to(root).as_posix()+ allowed = _WALK_ALLOWLIST.get((tree, rel), frozenset())+ text = _strip_comments(py.read_text(encoding="utf-8", errors="ignore"))+ for kind, pattern in _GUARD_KINDS:+ if kind in allowed:+ continue+ if pattern.search(text):+ offenders.append(f"{tree}:{rel} [{kind}]")assert not offenders, (- "Direct .rglob( calls found in packages/core — use "- "repowise.core.fs_walk.iter_glob/walk_repo instead (prunes "- "node_modules/.venv AND nested git repos). Unpruned rglob has "- f"caused multi-minute init stalls twice. Offenders: {offenders}"+ "Direct filesystem-walk calls found — use "+ "repowise.core.fs_walk.iter_glob/walk_repo/WalkSnapshot instead "+ "(prunes node_modules/.venv AND nested git repos, guards against "+ "junction cycles). Unpruned walks have caused multi-minute init "+ f"stalls and cross-repo manifest leaks. Offenders: {offenders}")++ def test_allowlist_entries_still_needed(self) -> None:+ """An allowlist entry whose file no longer matches its kinds is stale —+ prune it so the exemption cannot silently cover future code."""+ roots = _guarded_src_roots()+ kind_res = dict(_GUARD_KINDS)+ stale: list[str] = []+ for (tree, rel), kinds in _WALK_ALLOWLIST.items():+ py = roots[tree] / rel+ if not py.is_file():+ stale.append(f"{tree}:{rel} (file gone)")+ continue+ text = _strip_comments(py.read_text(encoding="utf-8", errors="ignore"))+ for kind in kinds:+ if not kind_res[kind].search(text):+ stale.append(f"{tree}:{rel} [{kind}]")+ assert not stale, f"Stale allowlist entries — remove them: {stale}"