Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(core): pruned filesystem walks — stop scanning nested repos and junk trees (#380)
Raghav Chamadiya718c8efa9356
diff --git a/packages/core/src/repowise/core/fs_walk.py b/packages/core/src/repowise/core/fs_walk.pynew file mode 100644index 000000000..76da6687a--- /dev/null+++ b/packages/core/src/repowise/core/fs_walk.py@@ -0,0 +1,228 @@+"""Pruned filesystem walking — the ONE way to scan a repo tree.++Every repo-wide file-discovery scan in repowise MUST go through this module.+``Path.rglob`` descends into ``node_modules``, ``.venv``, ``__pycache__`` and+— critically — **nested git repositories** (a checkout that physically+contains sibling/vendored repos). That has two failure modes we have shipped+and fixed more than once:++1. **Performance** — an unpruned walk over a directory that holds 30+ sibling+ repos (each with its own ``node_modules`` / ``.venv``) turns a <2s scan+ into minutes, while holding the GIL and starving concurrent phases.+2. **Correctness** — manifests from a *different* repo (``hugo/go.mod``,+ ``eShop/*.sln``) leak into the current repo's resolver context.++History: this logic first shipped privately in ``dynamic_hints/_walk.py``+(perf incident: 5-10 min stalls per extractor), then again in+``resolvers/ts_workspace.py`` (PR #368: 365s → 0.02s), while ~20 other+``rglob`` call sites stayed unpruned. This module is the single shared+implementation; ``tests/unit/ingestion/test_fs_walk.py`` carries a guard+test that fails on any new direct ``rglob`` call in ``packages/core``.++Guarantees:+ - Junk directories are pruned **at traversal time** (``dirnames[:] =``),+ never post-hoc.+ - Nested git repos (a subdirectory with its own ``.git`` dir *or* file —+ worktrees and submodules use a ``.git`` file) are pruned by default.+ Detection is free: ``.git`` appears in ``os.walk``'s own listings.+ - Symlinks are never followed; junction/hard-link cycles are detected via+ realpath tracking (Windows ``os.walk`` can't see them via inodes).+ - Depth is capped as a final safety net.+"""++from __future__ import annotations++import fnmatch+import os+from collections.abc import Iterable, Iterator+from pathlib import Path++import structlog++log = structlog.get_logger(__name__)++__all__ = ["PRUNED_DIRS", "PRUNED_DIRS_DERIVED", "iter_glob", "walk_repo"]++# Directory basenames that can NEVER hold first-party source or manifests:+# VCS metadata, package/venv caches, and tool caches. Pruned at every level+# of the walk. Keep this list strictly to never-source names: anything a+# real project could plausibly use as a source directory (``build``,+# ``out``, ``dist``, ``coverage`` — coveragepy's main package literally+# lives in ``coverage/``) belongs in :data:`PRUNED_DIRS_DERIVED` instead,+# and ambiguous names (``bin``, ``obj``, ``lib``, ``vendor``, ``target``)+# stay out of both — callers post-filter (dotnet skips ``bin``/``obj``,+# go skips ``vendor``).+PRUNED_DIRS: frozenset[str] = frozenset(+ {+ # VCS / metadata+ ".git",+ ".hg",+ ".svn",+ # Python environments / caches+ ".venv",+ "venv",+ ".env",+ "__pycache__",+ ".mypy_cache",+ ".pytest_cache",+ ".ruff_cache",+ ".tox",+ ".nox",+ # JS / TS+ "node_modules",+ ".next",+ ".nuxt",+ ".turbo",+ ".parcel-cache",+ ".yarn",+ ".pnpm-store",+ # Tool caches+ ".gradle",+ ".cache",+ # Editor / IDE+ ".idea",+ ".vscode",+ # Coverage / test outputs (unambiguous names only)+ "htmlcov",+ ".nyc_output",+ # Repowise / data+ ".repowise",+ ".lancedb",+ }+)++# PRUNED_DIRS plus common *derived-output* names that occasionally double as+# real source dirs. Use for scans where derived trees would only add noise+# (dynamic hints' settings/router extraction); do NOT use for manifest+# discovery, where a module legitimately rooted at ``build/`` or+# ``coverage/`` must still be found.+PRUNED_DIRS_DERIVED: frozenset[str] = PRUNED_DIRS | frozenset({"dist", "build", "out", "coverage"})++# Safety net: bail if walk depth grows pathologically. Real-world repos+# rarely exceed depth 20; anything beyond strongly suggests a cycle.+_MAX_WALK_DEPTH = 64+++def walk_repo(+ root: Path | str,+ *,+ prune_dirs: frozenset[str] = PRUNED_DIRS,+ prune_nested_git: bool = True,+ max_depth: int = _MAX_WALK_DEPTH,+) -> Iterator[tuple[Path, list[str], list[str]]]:+ """``os.walk`` over *root* with pruning. Yields ``(dirpath, dirnames, filenames)``.++ Use this (instead of :func:`iter_glob`) when one pass should collect+ several kinds of files — match against ``filenames`` yourself.++ Args:+ root: Directory to walk.+ prune_dirs: Directory *basenames* skipped at every level.+ prune_nested_git: Skip any non-root directory that contains its own+ ``.git`` entry (dir or file) — i.e. another repo.+ max_depth: Hard depth cap (cycle safety net).++ ``dirnames`` is yielded *after* pruning; callers may prune further by+ mutating it in place, exactly like ``os.walk``.+ """+ root_path = Path(root)+ if not root_path.is_dir():+ return++ try:+ root_real = os.path.realpath(root_path)+ except OSError:+ return+ base_depth = root_real.rstrip(os.sep).count(os.sep)+ visited_real: set[str] = {root_real}++ is_root = True+ for dirpath, dirnames, filenames in os.walk(root_path, followlinks=False):+ # Nested git repo? ``.git`` shows up in this dir's own listing — no+ # extra stat calls needed. The walk root itself is exempt (it IS the+ # repo being scanned); top-down os.walk always yields it first.+ if is_root:+ is_root = False+ elif prune_nested_git and (".git" in dirnames or ".git" in filenames):+ dirnames[:] = []+ continue++ # Skip junk directories at every level.+ dirnames[:] = [d for d in dirnames if d not in prune_dirs]++ # Drop any dir whose realpath we've already entered — catches+ # Windows junctions and other cycles ``os.walk`` can't detect via+ # inode (Windows doesn't expose real inodes outside NTFS proper).+ cycle_pruned: list[str] = []+ for d in dirnames:+ child = os.path.join(dirpath, d)+ try:+ child_real = os.path.realpath(child)+ except OSError:+ cycle_pruned.append(d)+ continue+ if child_real in visited_real:+ cycle_pruned.append(d)+ continue+ visited_real.add(child_real)+ if cycle_pruned:+ dirnames[:] = [d for d in dirnames if d not in cycle_pruned]++ # Final safety net: refuse to descend beyond max_depth.+ cur_depth = dirpath.rstrip(os.sep).count(os.sep) - base_depth+ if cur_depth >= max_depth:+ log.warning(+ "fs_walk.depth_exceeded",+ root=str(root_path),+ depth=cur_depth,+ at=dirpath,+ )+ dirnames[:] = []+ continue++ yield Path(dirpath), dirnames, filenames+++def _matches(rel_posix: str, basename: str, pattern: str) -> bool:+ """``rglob`` matching semantics for one already-walked entry.++ Patterns without ``/`` match the basename (``"*.go"``, ``"go.mod"``).+ Patterns with ``/`` match the *tail* of the root-relative posix path,+ mirroring ``Path.rglob`` == ``glob("**/" + pattern)``:+ ``"META-INF/services"``, ``"META-INF/spring/*.imports"``.+ """+ if "/" not in pattern:+ return fnmatch.fnmatch(basename, pattern)+ return fnmatch.fnmatch(rel_posix, pattern) or fnmatch.fnmatch(rel_posix, "*/" + pattern)+++def iter_glob(+ root: Path | str,+ patterns: str | Iterable[str],+ *,+ prune_dirs: frozenset[str] = PRUNED_DIRS,+ prune_nested_git: bool = True,+) -> Iterator[Path]:+ """Pruned drop-in for ``root.rglob(pattern)``.++ Mirrors ``rglob`` semantics — files *and* directories whose name matches+ are yielded — but skips junk directories, nested git repos, symlinks,+ and cycles (see :func:`walk_repo`).++ Accepts one pattern or several (one walk, any-match). Patterns may+ contain ``/`` to match a relative-path tail (``"META-INF/services"``).+ """+ root_path = Path(root)+ pats = (patterns,) if isinstance(patterns, str) else tuple(patterns)++ for dirpath, dirnames, filenames in walk_repo(+ root_path, prune_dirs=prune_dirs, prune_nested_git=prune_nested_git+ ):+ rel_dir = dirpath.relative_to(root_path).as_posix()+ prefix = "" if rel_dir == "." else rel_dir + "/"+ for name in filenames:+ if any(_matches(prefix + name, name, p) for p in pats):+ yield dirpath / name+ for name in dirnames:+ if any(_matches(prefix + name, name, p) for p in pats):+ yield dirpath / namediff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.pyindex 0a4918739..b0e25b2ab 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py@@ -1,154 +1,31 @@"""Pruned filesystem walk for dynamic-hint extractors.-Every extractor in this package used to call ``repo_root.rglob(pattern)``-directly, which descended into ``node_modules``, ``.venv``, ``.next``,-``__pycache__`` and other multi-million-file junk trees. On a polyrepo-with sibling ``backend/`` and ``frontend/`` directories the dynamic-hints-phase could stall for 5–10+ minutes per extractor with the progress bar-stuck at zero.--``iter_glob`` uses :func:`os.walk` with in-place ``dirnames`` pruning so-known-junk subdirectories are skipped at the point of traversal, not-afterwards. It also guards against:-- - **Symlink loops** — ``os.walk`` is called with ``followlinks=False``.- - **Junction loops** (Windows) — caught via :func:`os.path.realpath`- cycle detection.- - **Pathological recursive copies** — same cycle detector handles real- directories that share an inode through hard-linking, junctions, or- repeated nested-copy mistakes.- - **Catastrophic depth** — a hard depth cap as a final safety net.--Semantics match :py:meth:`pathlib.Path.rglob`: both files and directories-whose basename matches the glob are yielded.+The implementation now lives in :mod:`repowise.core.fs_walk` — the single+shared pruned walk used by every repo-wide scan (resolvers, external+systems, tsconfig discovery, dynamic hints). This module re-exports it so+existing extractor imports keep working.++Behavior note vs the historical private copy: the shared walk also prunes+**nested git repositories** (a subdirectory with its own ``.git``), which+is what dynamic hints want too — settings/router files from a vendored or+sibling checkout are not this repo's hints.++Dynamic hints keep the *derived-output* prune set (``dist``/``build``/+``out``/``coverage``), matching the historical list here: hint extraction+reads source files, and derived copies of them only add noise."""from __future__ import annotations-import fnmatch-import osfrom collections.abc import Iteratorfrom pathlib import Path-import structlog+from repowise.core.fs_walk import PRUNED_DIRS_DERIVED as PRUNED_DIRS+from repowise.core.fs_walk import iter_glob as _shared_iter_glob-log = structlog.get_logger(__name__)--# Directory basenames that are never source-bearing. Pruned at every-# level of the walk. Keep this list conservative: anything ambiguously-# named (``bin``, ``obj``, ``lib``) belongs out of here so we don't-# accidentally drop real source.-PRUNED_DIRS: frozenset[str] = frozenset(- {- # VCS / metadata- ".git",- ".hg",- ".svn",- # Python environments / caches- ".venv",- "venv",- ".env",- "__pycache__",- ".mypy_cache",- ".pytest_cache",- ".ruff_cache",- ".tox",- ".nox",- # JS / TS- "node_modules",- ".next",- ".nuxt",- ".turbo",- ".parcel-cache",- # Build artifacts (multi-language)- "dist",- "build",- "out",- ".gradle",- ".cache",- # Editor / IDE- ".idea",- ".vscode",- # Coverage / test outputs- "coverage",- "htmlcov",- ".nyc_output",- # Repowise / data- ".repowise",- ".lancedb",- }-)--# Safety net: bail if walk depth grows pathologically. Real-world repos-# rarely exceed depth 20; anything beyond strongly suggests a cycle.-_MAX_WALK_DEPTH = 64+__all__ = ["PRUNED_DIRS", "iter_glob"]def iter_glob(root: Path, pattern: str) -> Iterator[Path]:- """Recursively yield paths under ``root`` whose basename matches ``pattern``.-- Mirrors :py:meth:`pathlib.Path.rglob` semantics — files *and*- directories with a matching basename are yielded — but:- - prunes well-known junk directories at every level,- - never follows symlinks (``os.walk(followlinks=False)``),- - detects junction / hard-link cycles via realpath tracking,- - caps total depth as a final safety net.-- Args:- root: Directory to walk. May be a string or :class:`Path`.- pattern: A :mod:`fnmatch`-style glob applied to the basename.- Examples: ``"*.go"``, ``"settings.py"``, ``"package.json"``,- ``"tsconfig*.json"``.- """- root_path = Path(root)- if not root_path.exists():- return-- try:- root_real = os.path.realpath(root_path)- except OSError:- return- base_depth = root_real.rstrip(os.sep).count(os.sep)- visited_real: set[str] = {root_real}-- for dirpath, dirnames, filenames in os.walk(root_path, followlinks=False):- # Skip junk directories at every level.- dirnames[:] = [d for d in dirnames if d not in PRUNED_DIRS]-- # Drop any dir whose realpath we've already entered — catches- # Windows junctions and any other cycle that os.walk can't detect- # via inode (Windows doesn't expose real inodes for filesystems- # outside of NTFS proper).- pruned: list[str] = []- for d in list(dirnames):- child = os.path.join(dirpath, d)- try:- child_real = os.path.realpath(child)- except OSError:- pruned.append(d)- continue- if child_real in visited_real:- pruned.append(d)- continue- visited_real.add(child_real)- if pruned:- dirnames[:] = [d for d in dirnames if d not in pruned]-- # Final safety net: refuse to descend beyond _MAX_WALK_DEPTH.- cur_depth = dirpath.rstrip(os.sep).count(os.sep) - base_depth- if cur_depth >= _MAX_WALK_DEPTH:- log.warning(- "dynamic_hints.walk_depth_exceeded",- root=str(root_path),- depth=cur_depth,- at=dirpath,- )- dirnames[:] = []- continue-- for name in filenames:- if fnmatch.fnmatch(name, pattern):- yield Path(dirpath) / name- for name in dirnames:- if fnmatch.fnmatch(name, pattern):- yield Path(dirpath) / name+ """Historical dynamic-hints walk: shared iter_glob + derived-dirs pruning."""+ return _shared_iter_glob(root, pattern, prune_dirs=PRUNED_DIRS)diff --git a/packages/core/src/repowise/core/ingestion/external_systems/__init__.py b/packages/core/src/repowise/core/ingestion/external_systems/__init__.pyindex bec436987..61f8ace33 100644--- a/packages/core/src/repowise/core/ingestion/external_systems/__init__.py+++ b/packages/core/src/repowise/core/ingestion/external_systems/__init__.py@@ -78,17 +78,25 @@ def extract_external_systems(def _discover(repo_root: Path) -> list[Path]:"""Walk the repo (depth ≤ 4) collecting recognised manifest files."""+ from repowise.core.fs_walk import PRUNED_DIRS, walk_repo+seen: list[Path] = []- skip_dirs = {".git", "node_modules", "__pycache__", ".venv", "venv", "target", "dist", "build"}root = Path(repo_root)- for path in root.rglob("*"):- rel_parts = path.relative_to(root).parts- if len(rel_parts) > 4 or any(part in skip_dirs for part in rel_parts):- continue- if not path.is_file():- continue- if _parser_for(path) is not None:- seen.append(path)+ # The historical depth cap was a *post-hoc* filter on an unbounded+ # ``rglob("*")`` — the walk itself still descended into everything.+ # walk_repo prunes junk dirs and nested git repos at traversal time;+ # clearing ``dirnames`` at depth 3 makes the bound real (files yielded+ # there have 4 relative parts, matching the old ``len(rel_parts) > 4``).+ # Historical skip list included target/dist/build — preserve it on top+ # of the shared never-source set.+ prune = PRUNED_DIRS | {"target", "dist", "build"}+ for dirpath, dirnames, filenames in walk_repo(root, prune_dirs=prune):+ if len(dirpath.relative_to(root).parts) >= 3:+ dirnames[:] = []+ for fname in filenames:+ path = dirpath / fname+ if _parser_for(path) is not None:+ seen.append(path)return seendiff --git a/packages/core/src/repowise/core/ingestion/external_systems/bazel.py b/packages/core/src/repowise/core/ingestion/external_systems/bazel.pyindex 1db86de34..d973fc786 100644--- a/packages/core/src/repowise/core/ingestion/external_systems/bazel.py+++ b/packages/core/src/repowise/core/ingestion/external_systems/bazel.py@@ -280,8 +280,10 @@ def discover_bazel_packages(repo_root: Path, *, max_files: int = 5000) -> list[Bout: list[BazelFile] = []skip_dirs = {".git", "node_modules", "bazel-bin", "bazel-out", "bazel-testlogs","bazel-" + repo_root.name, ".venv", "venv", "build"}+ from repowise.core.fs_walk import iter_glob+for candidate in ("BUILD", "BUILD.bazel"):- for p in repo_root.rglob(candidate):+ for p in iter_glob(repo_root, candidate):try:rel = p.resolve().relative_to(repo_root).as_posix()except ValueError:diff --git a/packages/core/src/repowise/core/ingestion/external_systems/cmake.py b/packages/core/src/repowise/core/ingestion/external_systems/cmake.pyindex fef1c20e8..fa6f90d20 100644--- a/packages/core/src/repowise/core/ingestion/external_systems/cmake.py+++ b/packages/core/src/repowise/core/ingestion/external_systems/cmake.py@@ -631,7 +631,9 @@ def discover_cmake_reactor(skip_dirs = {".git", "build", "_build", "out", "_deps", "cmake-build-debug","cmake-build-release", "node_modules", ".venv", "venv"}if len(out) < max_files:- for cml in repo_root.rglob("CMakeLists.txt"):+ from repowise.core.fs_walk import iter_glob++ for cml in iter_glob(repo_root, "CMakeLists.txt"):try:rel = cml.resolve().relative_to(repo_root).as_posix()except ValueError:diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.pyindex 862b5ef03..f50922783 100644--- a/packages/core/src/repowise/core/ingestion/graph/builder.py+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py@@ -46,11 +46,17 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd*,exclude_patterns: list[str] | None = None,centrality_cache_dir: Path | str | None = None,+ include_submodules: bool = False,+ include_nested_repos: bool = False,) -> None:self._graph: nx.DiGraph = nx.DiGraph()self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFileself._built = Falseself._repo_path: Path | None = Path(repo_path) if repo_path else None+ # Mirrors the traverser flags: when submodules or nested repos are+ # indexed, resolver filesystem scans must not prune them (both are+ # ``.git``-bearing subdirs to fs_walk's prune_nested_git).+ self._prune_nested_git: bool = not (include_submodules or include_nested_repos)self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)# Optional structure-keyed disk cache for betweenness (the most# expensive metric kernel). Opt-in via *centrality_cache_dir* — the@@ -239,14 +245,17 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehydstem_map = build_stem_map(path_set)# Construct resolver context- go_modules = read_go_modules(self._repo_path)+ go_modules = read_go_modules(self._repo_path, prune_nested_git=self._prune_nested_git)ctx = ResolverContext(path_set=path_set,stem_map=stem_map,graph=self._graph,repo_path=self._repo_path,+ prune_nested_git=self._prune_nested_git,tsconfig_resolver=self._tsconfig_resolver,- go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),+ go_module_path=(+ go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)+ ),go_modules=go_modules,has_sfc_files=any(p.endswith((".vue", ".svelte", ".astro")) for p in path_set),parsed_files=self._parsed_files,diff --git a/packages/core/src/repowise/core/ingestion/resolvers/context.py b/packages/core/src/repowise/core/ingestion/resolvers/context.pyindex cf7cb0abf..54df8cb20 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/context.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/context.py@@ -27,6 +27,13 @@ class ResolverContext:graph: nx.DiGraphrepo_path: Path | None = None+ # Whether repo-wide filesystem scans (fs_walk) skip nested git repos.+ # True by default — sibling/vendored checkouts must not leak manifests.+ # Set False when the pipeline indexes ``.git``-bearing subdirs (either+ # ``include_submodules`` or ``include_nested_repos``) so resolver scans+ # see the same files the traverser indexed.+ prune_nested_git: bool = True+# Language-specific statetsconfig_resolver: Any | None = Nonego_module_path: str | None = Nonediff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.pyindex 3346162c9..69a701b58 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.py@@ -11,6 +11,8 @@ from __future__ import annotationsimport refrom pathlib import Path+from repowise.core.fs_walk import iter_glob+# Default ImplicitUsings set for SDK ``Microsoft.NET.Sdk`` projects.# Source: dotnet/sdk repo, Sdks/Microsoft.NET.Sdk/targets/Microsoft.NET.Sdk.ImplicitNamespaceImports.props_DEFAULT_IMPLICIT_USINGS: frozenset[str] = frozenset(@@ -58,6 +60,7 @@ def collect_project_global_usings(sdk_is_web: bool = False,*,project_texts: dict[Path, str] | None = None,+ prune_nested_git: bool = True,) -> set[str]:"""Walk *project_dir* for global using sources and return the merged set.@@ -86,7 +89,7 @@ def collect_project_global_usings(return resultskip = {"bin", "obj", ".vs", "node_modules"}- for cs_path in project_dir.rglob("*.cs"):+ for cs_path in iter_glob(project_dir, "*.cs", prune_nested_git=prune_nested_git):if any(part in skip for part in cs_path.parts):continuetry:diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.pyindex 6c12a4178..a58a71ecc 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py@@ -8,6 +8,8 @@ from typing import TYPE_CHECKINGimport structlog+from repowise.core.fs_walk import iter_glob+from .global_usings import collect_project_global_usingsfrom .msbuild import MSBuildProject, find_csproj_files, find_directory_build_props, parse_csprojfrom .namespace_map import build_namespace_map@@ -147,7 +149,7 @@ class DotNetProjectIndex:_CS_WALK_SKIP_DIRS = frozenset({"bin", "obj", ".vs", "node_modules", ".git", "packages"})-def _walk_repo_cs_files(repo_path: Path) -> list[Path]:+def _walk_repo_cs_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:"""Single repo-wide rglob for ``*.cs`` files, dedup by resolved path.Lives at module scope (not nested inside ``build_index``) so it's@@ -156,7 +158,7 @@ def _walk_repo_cs_files(repo_path: Path) -> list[Path]:"""seen: set[Path] = set()out: list[Path] = []- for cs in repo_path.rglob("*.cs"):+ for cs in iter_glob(repo_path, "*.cs", prune_nested_git=prune_nested_git):if any(part in _CS_WALK_SKIP_DIRS for part in cs.parts):continuetry:@@ -205,7 +207,7 @@ def _bucket_files_by_project(return out-def build_index(repo_path: Path) -> DotNetProjectIndex:+def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProjectIndex:"""Walk *repo_path* and construct a fully-populated DotNetProjectIndex.Performance note: a previous version of this function walked the@@ -223,7 +225,7 @@ def build_index(repo_path: Path) -> DotNetProjectIndex:index = DotNetProjectIndex(repo_path=repo_path)# ---- 1. Parse every .csproj ----- for csproj_path in find_csproj_files(repo_path):+ for csproj_path in find_csproj_files(repo_path, prune_nested_git=prune_nested_git):proj = parse_csproj(csproj_path)if proj is None:continue@@ -232,7 +234,7 @@ def build_index(repo_path: Path) -> DotNetProjectIndex:index.package_refs[proj.path] = set(proj.package_references)# ---- 2. Walk .sln files (informational; surfaces orphaned .csprojs) ----- index.sln_paths = find_sln_files(repo_path)+ index.sln_paths = find_sln_files(repo_path, prune_nested_git=prune_nested_git)for sln in index.sln_paths:for entry in parse_sln(sln):if entry.csproj not in index.projects:@@ -246,7 +248,7 @@ def build_index(repo_path: Path) -> DotNetProjectIndex:index.package_refs.setdefault(proj.path, set()).update(proj.package_references)# ---- 3. Single master walk: enumerate .cs files & read each once ----- all_cs_files = _walk_repo_cs_files(repo_path)+ all_cs_files = _walk_repo_cs_files(repo_path, prune_nested_git=prune_nested_git)cs_texts: dict[Path, str] = {}for f in all_cs_files:try:@@ -317,6 +319,6 @@ def get_or_build_index(ctx: "ResolverContext") -> DotNetProjectIndex | None:cached = getattr(ctx, _INDEX_KEY, None)if cached is not None:return cached- index = build_index(ctx.repo_path)+ index = build_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)setattr(ctx, _INDEX_KEY, index)return indexdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.pyindex ed849e6e2..829279dfc 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py@@ -14,6 +14,8 @@ from xml.etree import ElementTree as ETimport structlog+from repowise.core.fs_walk import iter_glob+log = structlog.get_logger(__name__)@@ -87,11 +89,11 @@ def parse_csproj(csproj_path: Path) -> MSBuildProject | None:return project-def find_csproj_files(repo_path: Path) -> list[Path]:+def find_csproj_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:"""Return all .csproj files under *repo_path*, skipping bin/obj output."""skip = {"bin", "obj", ".vs", "node_modules", ".git", "packages", "TestResults"}out: list[Path] = []- for csproj in repo_path.rglob("*.csproj"):+ for csproj in iter_glob(repo_path, "*.csproj", prune_nested_git=prune_nested_git):if any(part in skip for part in csproj.parts):continueout.append(csproj)diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.pyindex b1005488b..3fe510857 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py@@ -16,6 +16,8 @@ from pathlib import Pathimport structlog+from repowise.core.fs_walk import iter_glob+log = structlog.get_logger(__name__)# Project("{TYPE}") = "Name", "rel\path.csproj", "{GUID}"@@ -56,10 +58,10 @@ def parse_sln(sln_path: Path) -> list[SolutionEntry]:return out-def find_sln_files(repo_path: Path) -> list[Path]:+def find_sln_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:skip = {".git", "node_modules", "bin", "obj"}out: list[Path] = []- for sln in repo_path.rglob("*.sln"):+ for sln in iter_glob(repo_path, "*.sln", prune_nested_git=prune_nested_git):if any(part in skip for part in sln.parts):continueout.append(sln)diff --git a/packages/core/src/repowise/core/ingestion/resolvers/go.py b/packages/core/src/repowise/core/ingestion/resolvers/go.pyindex 2120dde32..61bedb375 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/go.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/go.py@@ -4,6 +4,8 @@ from __future__ import annotationsfrom pathlib import Path+from repowise.core.fs_walk import iter_glob+from .context import ResolverContext_GO_MOD_SKIP_DIRS = frozenset({"vendor", "node_modules", ".git"})@@ -30,7 +32,9 @@ def read_go_module_path(repo_path: Path | None) -> str | None:return _read_module_directive(go_mod)-def read_go_modules(repo_path: Path | None) -> tuple[tuple[str, str], ...]:+def read_go_modules(+ repo_path: Path | None, *, prune_nested_git: bool = True+) -> tuple[tuple[str, str], ...]:"""Discover every ``go.mod`` under *repo_path* and return``((module_dir_posix, module_path), ...)`` sorted longest-module-path-first.@@ -40,7 +44,7 @@ def read_go_modules(repo_path: Path | None) -> tuple[tuple[str, str], ...]:if repo_path is None or not repo_path.is_dir():return ()found: list[tuple[str, str]] = []- for go_mod in repo_path.rglob("go.mod"):+ for go_mod in iter_glob(repo_path, "go.mod", prune_nested_git=prune_nested_git):# Skip vendored / nested-package directoriesrel_parts = go_mod.relative_to(repo_path).partsif any(part in _GO_MOD_SKIP_DIRS for part in rel_parts):diff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.pyindex cc09adbd1..f874e2c81 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py@@ -19,6 +19,8 @@ from typing import TYPE_CHECKINGimport structlog+from repowise.core.fs_walk import iter_glob+if TYPE_CHECKING:from .context import ResolverContext@@ -289,7 +291,9 @@ def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]# Index building# ----------------------------------------------------------------------------def build_jvm_gradle_index(repo_path: Path | None) -> JvmGradleIndex:+def build_jvm_gradle_index(+ repo_path: Path | None, *, prune_nested_git: bool = True+) -> JvmGradleIndex:"""Walk Gradle config + JVM sources to build the package-resolution index."""index = JvmGradleIndex()if repo_path is None or not repo_path.is_dir():@@ -349,7 +353,7 @@ def build_jvm_gradle_index(repo_path: Path | None) -> JvmGradleIndex:continuerel_roots.append(rel)- for jvm_file in root_path.rglob("*"):+ for jvm_file in iter_glob(root_path, "*", prune_nested_git=prune_nested_git):if jvm_file.suffix not in _JVM_EXTENSIONS or not jvm_file.is_file():continuetry:@@ -380,7 +384,7 @@ def get_or_build_jvm_gradle_index(ctx: "ResolverContext") -> JvmGradleIndex:cached = getattr(ctx, "_jvm_gradle_index", None)if cached is not None:return cached- index = build_jvm_gradle_index(ctx.repo_path)+ index = build_jvm_gradle_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)ctx._jvm_gradle_index = index # type: ignore[attr-defined]return indexdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.pyindex b4d91f2d5..08943ef0b 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py@@ -21,6 +21,8 @@ from typing import TYPE_CHECKINGimport structlog+from repowise.core.fs_walk import iter_glob+if TYPE_CHECKING:from .context import ResolverContext@@ -175,7 +177,9 @@ _JPMS_PROVIDES_RE = re.compile()-def _scan_jpms_provides(repo_path: Path) -> dict[str, tuple[str, ...]]:+def _scan_jpms_provides(+ repo_path: Path, *, prune_nested_git: bool = True+) -> dict[str, tuple[str, ...]]:"""Scan ``module-info.java`` files for ``provides X with Y, Z`` directives.Returns a mapping iface_fqn → impl_fqns identical in shape to@@ -185,7 +189,7 @@ def _scan_jpms_provides(repo_path: Path) -> dict[str, tuple[str, ...]]:the warmup fast."""out: dict[str, list[str]] = {}- for mi in repo_path.rglob("module-info.java"):+ for mi in iter_glob(repo_path, "module-info.java", prune_nested_git=prune_nested_git):if not mi.is_file():continuetry:@@ -200,10 +204,14 @@ def _scan_jpms_provides(repo_path: Path) -> dict[str, tuple[str, ...]]:return {k: tuple(v) for k, v in out.items()}-def _scan_meta_inf_services(repo_path: Path) -> dict[str, tuple[str, ...]]:+def _scan_meta_inf_services(+ repo_path: Path, *, prune_nested_git: bool = True+) -> dict[str, tuple[str, ...]]:"""Scan META-INF/services/ directories for SPI declarations."""services: dict[str, list[str]] = {}- for services_dir in repo_path.rglob("META-INF/services"):+ for services_dir in iter_glob(+ repo_path, "META-INF/services", prune_nested_git=prune_nested_git+ ):if not services_dir.is_dir():continuetry:@@ -230,12 +238,16 @@ def _scan_meta_inf_services(repo_path: Path) -> dict[str, tuple[str, ...]]:return {k: tuple(v) for k, v in services.items()}-def _scan_spring_autoconfig(repo_path: Path) -> dict[str, tuple[str, ...]]:+def _scan_spring_autoconfig(+ repo_path: Path, *, prune_nested_git: bool = True+) -> dict[str, tuple[str, ...]]:"""Scan spring.factories and Boot 3 AutoConfiguration.imports."""result: dict[str, list[str]] = {}# spring.factories (Boot 2 style)- for factories in repo_path.rglob("META-INF/spring.factories"):+ for factories in iter_glob(+ repo_path, "META-INF/spring.factories", prune_nested_git=prune_nested_git+ ):if not factories.is_file():continuetry:@@ -264,7 +276,9 @@ def _scan_spring_autoconfig(repo_path: Path) -> dict[str, tuple[str, ...]]:result.setdefault(rel, []).extend(current_values)# Boot 3 style: META-INF/spring/*.imports- for imports_file in repo_path.rglob("META-INF/spring/*.imports"):+ for imports_file in iter_glob(+ repo_path, "META-INF/spring/*.imports", prune_nested_git=prune_nested_git+ ):if not imports_file.is_file():continuetry:@@ -336,8 +350,8 @@ def build_jvm_workspace_index(ctx: "ResolverContext") -> JvmWorkspaceIndex:# (cheap glob, O(matching files)). Both populate ``services``; later# phases treat any FQN listed there as reachable.try:- services = _scan_meta_inf_services(repo_path)- jpms = _scan_jpms_provides(repo_path)+ services = _scan_meta_inf_services(repo_path, prune_nested_git=ctx.prune_nested_git)+ jpms = _scan_jpms_provides(repo_path, prune_nested_git=ctx.prune_nested_git)merged: dict[str, list[str]] = {k: list(v) for k, v in services.items()}for iface, impls in jpms.items():merged.setdefault(iface, []).extend(impls)@@ -345,7 +359,9 @@ def build_jvm_workspace_index(ctx: "ResolverContext") -> JvmWorkspaceIndex:except Exception:passtry:- index.autoconfig_imports = _scan_spring_autoconfig(repo_path)+ index.autoconfig_imports = _scan_spring_autoconfig(+ repo_path, prune_nested_git=ctx.prune_nested_git+ )except Exception:passdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.py b/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.pyindex 82e8c2687..e135458d5 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.py@@ -17,6 +17,8 @@ from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import TYPE_CHECKING+from repowise.core.fs_walk import iter_glob+if TYPE_CHECKING:from .context import ResolverContext@@ -76,7 +78,9 @@ def _is_rails_repo(repo_path: Path) -> bool:return (repo_path / "config" / "application.rb").is_file()-def build_rails_index(repo_path: Path | None) -> RailsIndex | None:+def build_rails_index(+ repo_path: Path | None, *, prune_nested_git: bool = True+) -> RailsIndex | None:if repo_path is None or not repo_path.is_dir():return Noneif not _is_rails_repo(repo_path):@@ -90,7 +94,7 @@ def build_rails_index(repo_path: Path | None) -> RailsIndex | None:if not root_path.is_dir():continueindex.autoload_roots.append(root)- for rb in root_path.rglob("*.rb"):+ for rb in iter_glob(root_path, "*.rb", prune_nested_git=prune_nested_git):try:rel = rb.relative_to(repo_resolved).as_posix()except ValueError:@@ -110,6 +114,6 @@ def get_or_build_rails_index(ctx: "ResolverContext") -> RailsIndex | None:cached = getattr(ctx, "_ruby_rails_index", "__sentinel__")if cached != "__sentinel__":return cached # type: ignore[return-value]- index = build_rails_index(ctx.repo_path)+ index = build_rails_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)ctx._ruby_rails_index = index # type: ignore[attr-defined]return indexdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/scala_build.py b/packages/core/src/repowise/core/ingestion/resolvers/scala_build.pyindex 51ed3e255..a5ae07c58 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/scala_build.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/scala_build.py@@ -16,6 +16,8 @@ from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import TYPE_CHECKING+from repowise.core.fs_walk import iter_glob+if TYPE_CHECKING:from .context import ResolverContext@@ -82,7 +84,9 @@ def _parse_build_sc(repo_path: Path) -> dict[str, str]:return result-def _scan_packages(repo_path: Path, project_dir: str, build_tool: str) -> dict[str, list[str]]:+def _scan_packages(+ repo_path: Path, project_dir: str, build_tool: str, *, prune_nested_git: bool = True+) -> dict[str, list[str]]:"""Walk a project's source tree, recording package declarations."""found: dict[str, list[str]] = {}proj_path = (repo_path / project_dir).resolve() if project_dir else repo_path.resolve()@@ -107,7 +111,7 @@ def _scan_packages(repo_path: Path, project_dir: str, build_tool: str) -> dict[sif not root.is_dir() or root in seen:continueseen.add(root)- for src in root.rglob("*.scala"):+ for src in iter_glob(root, "*.scala", prune_nested_git=prune_nested_git):try:rel = src.relative_to(repo_path.resolve()).as_posix()except ValueError:@@ -123,7 +127,9 @@ def _scan_packages(repo_path: Path, project_dir: str, build_tool: str) -> dict[sreturn found-def build_scala_index(repo_path: Path | None) -> ScalaProjectIndex:+def build_scala_index(+ repo_path: Path | None, *, prune_nested_git: bool = True+) -> ScalaProjectIndex:if repo_path is None or not repo_path.is_dir():return ScalaProjectIndex()sbt_projects = _parse_build_sbt(repo_path)@@ -143,7 +149,10 @@ def build_scala_index(repo_path: Path | None) -> ScalaProjectIndex:return ScalaProjectIndex()for _name, project_dir in index.projects.items():- for pkg, files in _scan_packages(repo_path, project_dir, index.build_tool).items():+ scanned = _scan_packages(+ repo_path, project_dir, index.build_tool, prune_nested_git=prune_nested_git+ )+ for pkg, files in scanned.items():index.package_to_files.setdefault(pkg, []).extend(files)return index@@ -152,7 +161,7 @@ def get_or_build_scala_index(ctx: "ResolverContext") -> ScalaProjectIndex:cached = getattr(ctx, "_scala_index", None)if cached is not None:return cached- index = build_scala_index(ctx.repo_path)+ index = build_scala_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)ctx._scala_index = index # type: ignore[attr-defined]return indexdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.py b/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.pyindex 9e2512ae6..debc87c53 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.py@@ -19,6 +19,8 @@ import refrom pathlib import Pathfrom typing import TYPE_CHECKING+from repowise.core.fs_walk import iter_glob+if TYPE_CHECKING:from .context import ResolverContext@@ -61,7 +63,9 @@ def parse_package_swift(path: Path) -> dict[str, str]:return result-def build_swift_targets(repo_path: Path | None) -> dict[str, str]:+def build_swift_targets(+ repo_path: Path | None, *, prune_nested_git: bool = True+) -> dict[str, str]:"""Walk the repo for every ``Package.swift``, merge their target maps.Each target's source dir is prefixed with the package's directory@@ -71,7 +75,7 @@ def build_swift_targets(repo_path: Path | None) -> dict[str, str]:if repo_path is None or not repo_path.is_dir():return {}merged: dict[str, str] = {}- for pkg_swift in repo_path.rglob("Package.swift"):+ for pkg_swift in iter_glob(repo_path, "Package.swift", prune_nested_git=prune_nested_git):try:pkg_dir = pkg_swift.parent.relative_to(repo_path).as_posix()except ValueError:@@ -86,7 +90,7 @@ def get_or_build_swift_targets(ctx: "ResolverContext") -> dict[str, str]:cached = getattr(ctx, "_swift_targets", None)if cached is not None:return cached- mapping = build_swift_targets(ctx.repo_path)+ mapping = build_swift_targets(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)ctx._swift_targets = mapping # type: ignore[attr-defined]return mappingdiff --git a/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.pyindex 657e1ed8e..0eae9e64b 100644--- a/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py+++ b/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py@@ -25,7 +25,6 @@ flattened to the first plausible source target. Packages without anfrom __future__ import annotationsimport json-import osfrom dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any@@ -84,11 +83,18 @@ class _RepoFileScan:package_jsons: list[Path] = field(default_factory=list)-def _scan_repo_files(repo_path: Path) -> _RepoFileScan:- """One pruned ``os.walk`` collecting every finder's target files."""+def _scan_repo_files(repo_path: Path, *, prune_nested_git: bool = True) -> _RepoFileScan:+ """One pruned walk collecting every finder's target files.++ Uses the shared :func:`repowise.core.fs_walk.walk_repo` (nested-git+ pruning, cycle guard) with this module's deliberately-narrow prune set.+ """+ from repowise.core.fs_walk import walk_repo+scan = _RepoFileScan()- for dirpath, dirnames, filenames in os.walk(repo_path):- dirnames[:] = [d for d in dirnames if d not in _SCAN_PRUNE_DIRS]+ for dirpath, _dirnames, filenames in walk_repo(+ repo_path, prune_dirs=_SCAN_PRUNE_DIRS, prune_nested_git=prune_nested_git+ ):for fname in filenames:if fname.endswith(".mdx"):scan.mdx_files.append(Path(dirpath) / fname)@@ -104,7 +110,11 @@ def _get_repo_scan(ctx: "ResolverContext") -> _RepoFileScan:cached = getattr(ctx, "_ts_repo_file_scan", None)if cached is not None:return cached- scan = _scan_repo_files(ctx.repo_path) if ctx.repo_path is not None else _RepoFileScan()+ scan = (+ _scan_repo_files(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)+ if ctx.repo_path is not None+ else _RepoFileScan()+ )ctx._ts_repo_file_scan = scan # type: ignore[attr-defined]return scandiff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex 05219a05f..b16de7a4a 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -468,9 +468,18 @@ class FileTraverser:# ------------------------------------------------------------------def _detect_monorepo(self) -> tuple[list[PackageInfo], bool]:- """Detect package sub-directories by looking for manifest files."""+ """Detect package sub-directories by looking for manifest files.++ Candidate dirs the main traversal would never enter (nested git+ repos, submodules, gitignored/blocked dirs) are rejected up front:+ a "package" the walk skips must not be reported — and, before this+ guard, each such candidate was expensively ``rglob``-scanned for+ language/entry-point detection (minutes per sibling repo on a+ directory that physically contains other checkouts).+ """packages: list[PackageInfo] = []seen_paths: set[str] = set()+ prune_nested = not self._include_nested_reposfor depth in (1, 2):pattern = "/".join(["*"] * depth) + "/*"@@ -478,12 +487,17 @@ class FileTraverser:if candidate.name not in _MANIFEST_FILES:continuepkg_dir = candidate.parent- rel_pkg = pkg_dir.relative_to(self.repo_root).as_posix()+ rel_pkg_path = pkg_dir.relative_to(self.repo_root)+ rel_pkg = rel_pkg_path.as_posix()if rel_pkg in seen_paths:continue+ if self._dir_chain_skipped(rel_pkg_path):+ continueseen_paths.add(rel_pkg)- lang = _primary_language_in(pkg_dir)- entry_pts = _find_entry_points_in(pkg_dir, self.repo_root)+ lang = _primary_language_in(pkg_dir, prune_nested_git=prune_nested)+ entry_pts = _find_entry_points_in(+ pkg_dir, self.repo_root, prune_nested_git=prune_nested+ )packages.append(PackageInfo(name=pkg_dir.name,@@ -497,6 +511,20 @@ class FileTraverser:packages.sort(key=lambda p: p.path)return packages, len(packages) > 1+ def _dir_chain_skipped(self, rel_dir: Path) -> bool:+ """True if *rel_dir* (or any ancestor) would be pruned by ``_walk``.++ Reuses :meth:`_should_skip_dir` level by level so monorepo package+ detection has exactly the same boundary semantics as file traversal+ (blocked dirs, submodules, nested git repos, gitignore/excludes).+ """+ cur = Path()+ for part in rel_dir.parts:+ cur = cur / part+ if self._should_skip_dir(part, cur, self.repo_root / cur):+ return True+ return False+# ---------------------------------------------------------------------------# Module-level helpers@@ -590,12 +618,16 @@ def _stem_is_entry_point(abs_path: Path) -> bool:return stem in _ENTRY_POINT_STEMS-def _primary_language_in(directory: Path) -> LanguageTag:+def _primary_language_in(directory: Path, *, prune_nested_git: bool = True) -> LanguageTag:+ from repowise.core.fs_walk import walk_repo+counts: dict[str, int] = {}try:- for item in directory.rglob("*"):- if item.is_file():- lang = _detect_language(item)+ for dirpath, _dirnames, filenames in walk_repo(+ directory, prune_nested_git=prune_nested_git+ ):+ for fname in filenames:+ lang = _detect_language(dirpath / fname)if lang not in ("unknown", "yaml", "json", "markdown", "toml"):counts[lang] = counts.get(lang, 0) + 1except OSError:@@ -605,12 +637,19 @@ def _primary_language_in(directory: Path) -> LanguageTag:return max(counts, key=lambda k: counts[k]) # type: ignore[return-value]-def _find_entry_points_in(directory: Path, repo_root: Path) -> list[str]:+def _find_entry_points_in(+ directory: Path, repo_root: Path, *, prune_nested_git: bool = True+) -> list[str]:+ from repowise.core.fs_walk import walk_repo+result: list[str] = []try:- for item in directory.rglob("*"):- if item.is_file() and item.name in _ENTRY_POINT_NAMES:- result.append(item.relative_to(repo_root).as_posix())+ for dirpath, _dirnames, filenames in walk_repo(+ directory, prune_nested_git=prune_nested_git+ ):+ for fname in filenames:+ if fname in _ENTRY_POINT_NAMES:+ result.append((dirpath / fname).relative_to(repo_root).as_posix())except OSError:passreturn sorted(result)diff --git a/packages/core/src/repowise/core/ingestion/tsconfig_resolver.py b/packages/core/src/repowise/core/ingestion/tsconfig_resolver.pyindex 851843f37..341cf0658 100644--- a/packages/core/src/repowise/core/ingestion/tsconfig_resolver.py+++ b/packages/core/src/repowise/core/ingestion/tsconfig_resolver.py@@ -80,11 +80,16 @@ class TsconfigResolver:Args:repo_path: Absolute path to the repository root.path_set: Set of POSIX-relative file paths known to the graph builder.+ prune_nested_git: Skip nested git repos during config discovery+ (set False when the pipeline runs with ``--include-submodules``)."""- def __init__(self, repo_path: Path, path_set: set[str]) -> None:+ def __init__(+ self, repo_path: Path, path_set: set[str], *, prune_nested_git: bool = True+ ) -> None:self._repo_path = repo_path.resolve()self._path_set = path_set+ self._prune_nested_git = prune_nested_git# directory (absolute) -> ResolvedConfigself._dir_to_config: dict[Path, ResolvedConfig] = {}@@ -150,9 +155,13 @@ class TsconfigResolver:# Group candidates by directory, assign priority (lower = higher).dir_candidates: dict[Path, list[tuple[int, Path]]] = {}- for config_file in self._repo_path.rglob("*.json"):- if "node_modules" in config_file.parts:- continue+ from repowise.core.fs_walk import iter_glob++ for config_file in iter_glob(+ self._repo_path,+ ("tsconfig*.json", "jsconfig.json"),+ prune_nested_git=self._prune_nested_git,+ ):name = config_file.nameif name == "tsconfig.json":priority = 0diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.pyindex 525a5a362..f0aa3f88c 100644--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py@@ -246,6 +246,8 @@ async def _run_ingestion(repo_path=repo_path,exclude_patterns=exclude_patterns,centrality_cache_dir=repo_path / ".repowise",+ include_submodules=include_submodules,+ include_nested_repos=include_nested_repos,)loop = asyncio.get_running_loop()@@ -335,7 +337,11 @@ async def _run_ingestion(if progress:progress.on_phase_start("tsconfig", None)_path_set = set(graph_builder._parsed_files.keys())- _resolver = TsconfigResolver(repo_path=repo_path, path_set=_path_set)+ _resolver = TsconfigResolver(+ repo_path=repo_path,+ path_set=_path_set,+ prune_nested_git=not (include_submodules or include_nested_repos),+ )graph_builder.set_tsconfig_resolver(_resolver)_phase_done(progress, "tsconfig")except Exception as _resolver_exc:diff --git a/packages/core/src/repowise/core/workspace/cross_repo.py b/packages/core/src/repowise/core/workspace/cross_repo.pyindex 39dad4a43..5ce1b2f88 100644--- a/packages/core/src/repowise/core/workspace/cross_repo.py+++ b/packages/core/src/repowise/core/workspace/cross_repo.py@@ -523,14 +523,18 @@ def _scan_csproj("""from xml.etree import ElementTree as ET+ from repowise.core.fs_walk import iter_glob+results: list[CrossRepoPackageDep] = []skip = {"bin", "obj", ".vs", "packages", "node_modules", ".git", "TestResults"}# Pre-compute the assembly-name → repo-alias map so we can resolve- # internal-NuGet references in a second pass.+ # internal-NuGet references in a second pass. Each *selected* workspace+ # repo is walked from its own root; iter_glob's nested-git pruning keeps+ # any physically-nested unselected repo out of the scan.assembly_to_repo: dict[str, str] = {}for sib_alias, sib_path in repo_paths.items():- for csproj in sib_path.rglob("*.csproj"):+ for csproj in iter_glob(sib_path, "*.csproj"):if any(part in skip for part in csproj.parts):continuetry:@@ -545,7 +549,7 @@ def _scan_csproj(breakassembly_to_repo[assembly_name] = sib_alias- for csproj in repo_path.rglob("*.csproj"):+ for csproj in iter_glob(repo_path, "*.csproj"):if any(part in skip for part in csproj.parts):continuetry:diff --git a/tests/unit/ingestion/test_fs_walk.py b/tests/unit/ingestion/test_fs_walk.pynew file mode 100644index 000000000..b2faf393f--- /dev/null+++ b/tests/unit/ingestion/test_fs_walk.py@@ -0,0 +1,312 @@+"""Unit tests for the shared pruned filesystem walk (``repowise.core.fs_walk``)+plus the regression guard that keeps new unpruned ``rglob`` calls out of+``packages/core``.++Why this exists: unpruned ``Path.rglob`` over a repo that physically contains+sibling/vendored repos (or ``node_modules`` / ``.venv``) has caused two+separate multi-minute perf incidents AND leaks foreign manifests+(``hugo/go.mod``, ``eShop/*.sln``) into the current repo's resolver context.+Every repo-wide scan must go through ``fs_walk``.+"""++from __future__ import annotations++import re+from pathlib import Path++from repowise.core.fs_walk import PRUNED_DIRS, PRUNED_DIRS_DERIVED, iter_glob, walk_repo+++def _write(root: Path, rel: str, content: str = "x") -> None:+ p = root / rel+ p.parent.mkdir(parents=True, exist_ok=True)+ p.write_text(content, encoding="utf-8")+++# ---------------------------------------------------------------------------+# iter_glob — basic rglob parity+# ---------------------------------------------------------------------------+++class TestIterGlobParity:+ def test_matches_rglob_on_clean_tree(self, tmp_path: Path) -> None:+ """On a tree with no junk dirs, iter_glob == rglob exactly."""+ _write(tmp_path, "go.mod")+ _write(tmp_path, "services/foo/go.mod")+ _write(tmp_path, "libs/bar/baz/go.mod")+ _write(tmp_path, "libs/bar/baz/main.go")+ expected = sorted(tmp_path.rglob("go.mod"))+ assert sorted(iter_glob(tmp_path, "go.mod")) == expected++ def test_yields_matching_directories_too(self, tmp_path: Path) -> None:+ """rglob semantics: directories with a matching basename are yielded."""+ (tmp_path / "src" / "META-INF" / "services").mkdir(parents=True)+ results = list(iter_glob(tmp_path, "services"))+ assert results == [tmp_path / "src" / "META-INF" / "services"]++ def test_slash_pattern_matches_path_tail(self, tmp_path: Path) -> None:+ """Patterns with '/' mirror rglob("META-INF/services") semantics."""+ (tmp_path / "a" / "META-INF" / "services").mkdir(parents=True)+ (tmp_path / "b" / "other" / "services").mkdir(parents=True)+ expected = sorted(tmp_path.rglob("META-INF/services"))+ assert sorted(iter_glob(tmp_path, "META-INF/services")) == expected+ assert (tmp_path / "b" / "other" / "services") not in set(+ iter_glob(tmp_path, "META-INF/services")+ )++ def test_slash_pattern_with_glob_component(self, tmp_path: Path) -> None:+ _write(tmp_path, "x/META-INF/spring/auto.imports")+ _write(tmp_path, "x/META-INF/spring/readme.txt")+ expected = sorted(tmp_path.rglob("META-INF/spring/*.imports"))+ assert sorted(iter_glob(tmp_path, "META-INF/spring/*.imports")) == expected++ def test_slash_pattern_matches_at_root(self, tmp_path: Path) -> None:+ """rglob("META-INF/services") also matches directly under the root."""+ (tmp_path / "META-INF" / "services").mkdir(parents=True)+ expected = sorted(tmp_path.rglob("META-INF/services"))+ assert sorted(iter_glob(tmp_path, "META-INF/services")) == expected++ def test_multiple_patterns_single_walk(self, tmp_path: Path) -> None:+ _write(tmp_path, "app/tsconfig.json")+ _write(tmp_path, "app/jsconfig.json")+ _write(tmp_path, "app/package.json")+ found = set(iter_glob(tmp_path, ("tsconfig*.json", "jsconfig.json")))+ assert found == {tmp_path / "app" / "tsconfig.json", tmp_path / "app" / "jsconfig.json"}++ def test_missing_root_yields_nothing(self, tmp_path: Path) -> None:+ assert list(iter_glob(tmp_path / "nope", "*.go")) == []+++# ---------------------------------------------------------------------------+# Pruning — junk dirs and nested git repos+# ---------------------------------------------------------------------------+++class TestPruning:+ def test_junk_dirs_pruned(self, tmp_path: Path) -> None:+ _write(tmp_path, "real/go.mod")+ _write(tmp_path, "node_modules/dep/go.mod")+ _write(tmp_path, ".venv/lib/go.mod")+ _write(tmp_path, "sub/__pycache__/go.mod")+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "real" / "go.mod"]++ def test_nested_git_dir_pruned(self, tmp_path: Path) -> None:+ """A subdirectory with its own .git directory is another repo — skip it."""+ _write(tmp_path, "go.mod")+ (tmp_path / "sibling-repo" / ".git").mkdir(parents=True)+ _write(tmp_path, "sibling-repo/go.mod")+ _write(tmp_path, "sibling-repo/deep/go.mod")+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "go.mod"]++ def test_nested_git_file_pruned(self, tmp_path: Path) -> None:+ """Worktrees and submodules mark themselves with a .git FILE."""+ _write(tmp_path, "main.go")+ _write(tmp_path, "submodule/.git", "gitdir: ../.git/modules/submodule")+ _write(tmp_path, "submodule/sub.go")+ assert list(iter_glob(tmp_path, "*.go")) == [tmp_path / "main.go"]++ def test_walk_root_with_git_not_pruned(self, tmp_path: Path) -> None:+ """The walk root IS the repo being scanned — its own .git is exempt."""+ (tmp_path / ".git").mkdir()+ _write(tmp_path, "src/go.mod")+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "src" / "go.mod"]++ def test_nested_git_pruning_can_be_disabled(self, tmp_path: Path) -> None:+ (tmp_path / "vendored" / ".git").mkdir(parents=True)+ _write(tmp_path, "vendored/go.mod")+ found = list(iter_glob(tmp_path, "go.mod", prune_nested_git=False))+ assert found == [tmp_path / "vendored" / "go.mod"]++ def test_custom_prune_dirs(self, tmp_path: Path) -> None:+ """Callers may pass their own prune set (narrower or broader)."""+ _write(tmp_path, "dist/package.json")+ _write(tmp_path, "node_modules/x/package.json")+ narrow = frozenset({"node_modules"})+ found = set(iter_glob(tmp_path, "package.json", prune_dirs=narrow))+ assert found == {tmp_path / "dist" / "package.json"}++ def test_default_keeps_possible_source_dirs(self, tmp_path: Path) -> None:+ """The DEFAULT set must not prune dirs real projects use for source —+ coveragepy's main package is literally ``coverage/``; a module rooted+ at ``build/go.mod`` must stay discoverable."""+ _write(tmp_path, "coverage/__init__.py")+ _write(tmp_path, "build/go.mod")+ assert list(iter_glob(tmp_path, "__init__.py")) == [tmp_path / "coverage" / "__init__.py"]+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "build" / "go.mod"]++ def test_derived_set_prunes_output_dirs(self, tmp_path: Path) -> None:+ """PRUNED_DIRS_DERIVED (dynamic hints) prunes derived-output names."""+ _write(tmp_path, "dist/settings.py")+ _write(tmp_path, "src/settings.py")+ found = list(iter_glob(tmp_path, "settings.py", prune_dirs=PRUNED_DIRS_DERIVED))+ assert found == [tmp_path / "src" / "settings.py"]++ def test_default_prune_set_contents(self) -> None:+ """Canary: the dirs behind past incidents stay pruned."""+ for d in ("node_modules", ".venv", "__pycache__", ".git", ".repowise", ".next"):+ assert d in PRUNED_DIRS+ for d in ("dist", "build", "out", "coverage"):+ assert d not in PRUNED_DIRS # possible source dirs — derived set only+ assert d in PRUNED_DIRS_DERIVED+++# ---------------------------------------------------------------------------+# walk_repo — low-level walk+# ---------------------------------------------------------------------------+++class TestWalkRepo:+ def test_caller_can_bound_depth_via_dirnames(self, tmp_path: Path) -> None:+ _write(tmp_path, "a/b/c/d/deep.txt")+ _write(tmp_path, "a/shallow.txt")+ seen: list[str] = []+ for dirpath, dirnames, filenames in walk_repo(tmp_path):+ if len(dirpath.relative_to(tmp_path).parts) >= 1:+ dirnames[:] = []+ seen.extend(filenames)+ assert "shallow.txt" in seen+ assert "deep.txt" not in seen++ def test_nested_git_subtree_never_yielded(self, tmp_path: Path) -> None:+ _write(tmp_path, "ours.txt")+ (tmp_path / "other" / ".git").mkdir(parents=True)+ _write(tmp_path, "other/theirs.txt")+ yielded_dirs = [d for d, _, _ in walk_repo(tmp_path)]+ assert tmp_path / "other" not in yielded_dirs+++# ---------------------------------------------------------------------------+# --include-submodules plumbing — scanners must honor prune_nested_git=False+# ---------------------------------------------------------------------------+++class TestSubmodulePlumbing:+ def test_read_go_modules_can_include_submodules(self, tmp_path: Path) -> None:+ from repowise.core.ingestion.resolvers.go import read_go_modules++ _write(tmp_path, "go.mod", "module example.com/root\n")+ _write(tmp_path, "vendored/.git", "gitdir: ../.git/modules/vendored")+ _write(tmp_path, "vendored/go.mod", "module example.com/vendored\n")++ default = dict(read_go_modules(tmp_path))+ assert default == {"": "example.com/root"}++ included = dict(read_go_modules(tmp_path, prune_nested_git=False))+ assert included == {"": "example.com/root", "vendored": "example.com/vendored"}++ def test_graph_builder_threads_include_submodules(self, tmp_path: Path) -> None:+ """Either traverser flag that indexes .git-bearing subdirs must also+ stop resolver scans from pruning them."""+ from repowise.core.ingestion import GraphBuilder++ assert GraphBuilder(tmp_path)._prune_nested_git is True+ assert GraphBuilder(tmp_path, include_submodules=True)._prune_nested_git is False+ assert GraphBuilder(tmp_path, include_nested_repos=True)._prune_nested_git is False++ def test_resolver_context_default(self) -> None:+ from repowise.core.ingestion.resolvers.context import ResolverContext++ ctx = ResolverContext(path_set=set(), stem_map={}, graph=None)+ assert ctx.prune_nested_git is True+++# ---------------------------------------------------------------------------+# Traverser monorepo detection — must share _walk's boundary semantics+# ---------------------------------------------------------------------------+++class TestMonorepoDetectionBoundaries:+ def test_nested_repo_not_reported_as_package(self, tmp_path: Path) -> None:+ """A sibling/vendored repo inside the tree is not a package of THIS+ repo — and must not be rglob-scanned for language/entry points+ (that scan ran for minutes per sibling on multi-repo directories)."""+ from repowise.core.ingestion.traverser import FileTraverser++ _write(tmp_path, "packages/app/package.json", "{}")+ _write(tmp_path, "packages/app/index.ts", "export {}")+ (tmp_path / "sibling" / ".git").mkdir(parents=True)+ _write(tmp_path, "sibling/package.json", "{}")+ _write(tmp_path, "node_modules/dep/package.json", "{}")++ traverser = FileTraverser(tmp_path)+ packages, _ = traverser._detect_monorepo()+ paths = {p.path for p in packages}+ assert "packages/app" in paths+ assert "sibling" not in paths+ assert all(not p.startswith("node_modules") for p in paths)++ def test_nested_repo_reported_when_opted_in(self, tmp_path: Path) -> None:+ from repowise.core.ingestion.traverser import FileTraverser++ (tmp_path / "sibling" / ".git").mkdir(parents=True)+ _write(tmp_path, "sibling/package.json", "{}")++ traverser = FileTraverser(tmp_path, include_nested_repos=True)+ packages, _ = traverser._detect_monorepo()+ assert "sibling" in {p.path for p in packages}+++# ---------------------------------------------------------------------------+# external_systems._discover — depth bound must match the historical rglob+# ---------------------------------------------------------------------------+++class TestExternalSystemsDiscoverDepth:+ def test_depth_bound_matches_old_rglob_semantics(self, tmp_path: Path) -> None:+ """Old code: rglob("*") + skip files with >4 relative parts. The+ walk_repo rewrite bounds descent by clearing dirnames at dir-depth 3;+ files at exactly 4 relative parts must stay IN, 5 must stay OUT."""+ from repowise.core.ingestion.external_systems import _discover++ _write(tmp_path, "package.json", "{}") # 1 part+ _write(tmp_path, "a/package.json", "{}") # 2 parts+ _write(tmp_path, "a/b/c/package.json", "{}") # 4 parts — boundary, kept+ _write(tmp_path, "a/b/c/d/package.json", "{}") # 5 parts — too deep+ found = {p.relative_to(tmp_path).as_posix() for p in _discover(tmp_path)}+ assert "package.json" in found+ assert "a/package.json" in found+ assert "a/b/c/package.json" in found+ assert "a/b/c/d/package.json" not in found+++# ---------------------------------------------------------------------------+# Regression guard — no new direct rglob calls in packages/core+# ---------------------------------------------------------------------------++# Files allowed to mention ``.rglob(``. Additions require justification:+# either the walk root is a tightly-bounded subdirectory that cannot contain+# junk trees or nested repos, or the file IS the shared implementation.+_RGLOB_ALLOWLIST = {+ # The shared implementation (docstring references rglob semantics).+ "fs_walk.py",+ # Decision mining: bounded to an explicit docs/ directory.+ "analysis/decisions/extractor.py",+}++_RGLOB_RE = re.compile(r"\.rglob\(")+++def _core_src_root() -> Path:+ # tests/unit/ingestion/ → repo root → packages/core/src/repowise/core+ repo_root = Path(__file__).resolve().parents[3]+ return repo_root / "packages" / "core" / "src" / "repowise" / "core"+++class TestNoUnprunedRglobRegression:+ def test_no_new_rglob_call_sites_in_core(self) -> None:+ core = _core_src_root()+ assert core.is_dir(), f"layout changed? {core}"+ offenders: list[str] = []+ for py in core.rglob("*.py"): # tests may rglob; core may not.+ rel = py.relative_to(core).as_posix()+ if rel in _RGLOB_ALLOWLIST:+ continue+ text = py.read_text(encoding="utf-8", errors="ignore")+ if _RGLOB_RE.search(text):+ offenders.append(rel)+ assert not offenders, (+ "Direct .rglob( calls found in packages/core — use "+ "repowise.core.fs_walk.iter_glob/walk_repo instead (prunes "+ "node_modules/.venv AND nested git repos). Unpruned rglob has "+ f"caused multi-minute init stalls twice. Offenders: {offenders}"+ )