Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(core): pruned filesystem walks — stop scanning nested repos and junk trees (#380)

Raghav Chamadiya718c8efa9356
diff --git a/packages/core/src/repowise/core/fs_walk.py b/packages/core/src/repowise/core/fs_walk.py
new file mode 100644
index 000000000..76da6687a
--- /dev/null
+++ b/packages/core/src/repowise/core/fs_walk.py
@@ -0,0 +1,228 @@
+"""Pruned filesystem walking — the ONE way to scan a repo tree.
+
+Every repo-wide file-discovery scan in repowise MUST go through this module.
+``Path.rglob`` descends into ``node_modules``, ``.venv``, ``__pycache__`` and
+— critically — **nested git repositories** (a checkout that physically
+contains sibling/vendored repos). That has two failure modes we have shipped
+and fixed more than once:
+
+1. **Performance** — an unpruned walk over a directory that holds 30+ sibling
+ repos (each with its own ``node_modules`` / ``.venv``) turns a <2s scan
+ into minutes, while holding the GIL and starving concurrent phases.
+2. **Correctness** — manifests from a *different* repo (``hugo/go.mod``,
+ ``eShop/*.sln``) leak into the current repo's resolver context.
+
+History: this logic first shipped privately in ``dynamic_hints/_walk.py``
+(perf incident: 5-10 min stalls per extractor), then again in
+``resolvers/ts_workspace.py`` (PR #368: 365s → 0.02s), while ~20 other
+``rglob`` call sites stayed unpruned. This module is the single shared
+implementation; ``tests/unit/ingestion/test_fs_walk.py`` carries a guard
+test that fails on any new direct ``rglob`` call in ``packages/core``.
+
+Guarantees:
+ - Junk directories are pruned **at traversal time** (``dirnames[:] =``),
+ never post-hoc.
+ - Nested git repos (a subdirectory with its own ``.git`` dir *or* file —
+ worktrees and submodules use a ``.git`` file) are pruned by default.
+ Detection is free: ``.git`` appears in ``os.walk``'s own listings.
+ - Symlinks are never followed; junction/hard-link cycles are detected via
+ realpath tracking (Windows ``os.walk`` can't see them via inodes).
+ - Depth is capped as a final safety net.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+__all__ = ["PRUNED_DIRS", "PRUNED_DIRS_DERIVED", "iter_glob", "walk_repo"]
+
+# Directory basenames that can NEVER hold first-party source or manifests:
+# VCS metadata, package/venv caches, and tool caches. Pruned at every level
+# of the walk. Keep this list strictly to never-source names: anything a
+# real project could plausibly use as a source directory (``build``,
+# ``out``, ``dist``, ``coverage`` — coveragepy's main package literally
+# lives in ``coverage/``) belongs in :data:`PRUNED_DIRS_DERIVED` instead,
+# and ambiguous names (``bin``, ``obj``, ``lib``, ``vendor``, ``target``)
+# stay out of both — callers post-filter (dotnet skips ``bin``/``obj``,
+# go skips ``vendor``).
+PRUNED_DIRS: frozenset[str] = frozenset(
+ {
+ # VCS / metadata
+ ".git",
+ ".hg",
+ ".svn",
+ # Python environments / caches
+ ".venv",
+ "venv",
+ ".env",
+ "__pycache__",
+ ".mypy_cache",
+ ".pytest_cache",
+ ".ruff_cache",
+ ".tox",
+ ".nox",
+ # JS / TS
+ "node_modules",
+ ".next",
+ ".nuxt",
+ ".turbo",
+ ".parcel-cache",
+ ".yarn",
+ ".pnpm-store",
+ # Tool caches
+ ".gradle",
+ ".cache",
+ # Editor / IDE
+ ".idea",
+ ".vscode",
+ # Coverage / test outputs (unambiguous names only)
+ "htmlcov",
+ ".nyc_output",
+ # Repowise / data
+ ".repowise",
+ ".lancedb",
+ }
+)
+
+# PRUNED_DIRS plus common *derived-output* names that occasionally double as
+# real source dirs. Use for scans where derived trees would only add noise
+# (dynamic hints' settings/router extraction); do NOT use for manifest
+# discovery, where a module legitimately rooted at ``build/`` or
+# ``coverage/`` must still be found.
+PRUNED_DIRS_DERIVED: frozenset[str] = PRUNED_DIRS | frozenset({"dist", "build", "out", "coverage"})
+
+# Safety net: bail if walk depth grows pathologically. Real-world repos
+# rarely exceed depth 20; anything beyond strongly suggests a cycle.
+_MAX_WALK_DEPTH = 64
+
+
+def walk_repo(
+ root: Path | str,
+ *,
+ prune_dirs: frozenset[str] = PRUNED_DIRS,
+ prune_nested_git: bool = True,
+ max_depth: int = _MAX_WALK_DEPTH,
+) -> Iterator[tuple[Path, list[str], list[str]]]:
+ """``os.walk`` over *root* with pruning. Yields ``(dirpath, dirnames, filenames)``.
+
+ Use this (instead of :func:`iter_glob`) when one pass should collect
+ several kinds of files — match against ``filenames`` yourself.
+
+ Args:
+ root: Directory to walk.
+ prune_dirs: Directory *basenames* skipped at every level.
+ prune_nested_git: Skip any non-root directory that contains its own
+ ``.git`` entry (dir or file) — i.e. another repo.
+ max_depth: Hard depth cap (cycle safety net).
+
+ ``dirnames`` is yielded *after* pruning; callers may prune further by
+ mutating it in place, exactly like ``os.walk``.
+ """
+ root_path = Path(root)
+ if not root_path.is_dir():
+ return
+
+ try:
+ root_real = os.path.realpath(root_path)
+ except OSError:
+ return
+ base_depth = root_real.rstrip(os.sep).count(os.sep)
+ visited_real: set[str] = {root_real}
+
+ is_root = True
+ for dirpath, dirnames, filenames in os.walk(root_path, followlinks=False):
+ # Nested git repo? ``.git`` shows up in this dir's own listing — no
+ # extra stat calls needed. The walk root itself is exempt (it IS the
+ # repo being scanned); top-down os.walk always yields it first.
+ if is_root:
+ is_root = False
+ elif prune_nested_git and (".git" in dirnames or ".git" in filenames):
+ dirnames[:] = []
+ continue
+
+ # Skip junk directories at every level.
+ dirnames[:] = [d for d in dirnames if d not in prune_dirs]
+
+ # Drop any dir whose realpath we've already entered — catches
+ # Windows junctions and other cycles ``os.walk`` can't detect via
+ # inode (Windows doesn't expose real inodes outside NTFS proper).
+ cycle_pruned: list[str] = []
+ for d in dirnames:
+ child = os.path.join(dirpath, d)
+ try:
+ child_real = os.path.realpath(child)
+ except OSError:
+ cycle_pruned.append(d)
+ continue
+ if child_real in visited_real:
+ cycle_pruned.append(d)
+ continue
+ visited_real.add(child_real)
+ if cycle_pruned:
+ dirnames[:] = [d for d in dirnames if d not in cycle_pruned]
+
+ # Final safety net: refuse to descend beyond max_depth.
+ cur_depth = dirpath.rstrip(os.sep).count(os.sep) - base_depth
+ if cur_depth >= max_depth:
+ log.warning(
+ "fs_walk.depth_exceeded",
+ root=str(root_path),
+ depth=cur_depth,
+ at=dirpath,
+ )
+ dirnames[:] = []
+ continue
+
+ yield Path(dirpath), dirnames, filenames
+
+
+def _matches(rel_posix: str, basename: str, pattern: str) -> bool:
+ """``rglob`` matching semantics for one already-walked entry.
+
+ Patterns without ``/`` match the basename (``"*.go"``, ``"go.mod"``).
+ Patterns with ``/`` match the *tail* of the root-relative posix path,
+ mirroring ``Path.rglob`` == ``glob("**/" + pattern)``:
+ ``"META-INF/services"``, ``"META-INF/spring/*.imports"``.
+ """
+ if "/" not in pattern:
+ return fnmatch.fnmatch(basename, pattern)
+ return fnmatch.fnmatch(rel_posix, pattern) or fnmatch.fnmatch(rel_posix, "*/" + pattern)
+
+
+def iter_glob(
+ root: Path | str,
+ patterns: str | Iterable[str],
+ *,
+ prune_dirs: frozenset[str] = PRUNED_DIRS,
+ prune_nested_git: bool = True,
+) -> Iterator[Path]:
+ """Pruned drop-in for ``root.rglob(pattern)``.
+
+ Mirrors ``rglob`` semantics — files *and* directories whose name matches
+ are yielded — but skips junk directories, nested git repos, symlinks,
+ and cycles (see :func:`walk_repo`).
+
+ Accepts one pattern or several (one walk, any-match). Patterns may
+ contain ``/`` to match a relative-path tail (``"META-INF/services"``).
+ """
+ root_path = Path(root)
+ pats = (patterns,) if isinstance(patterns, str) else tuple(patterns)
+
+ for dirpath, dirnames, filenames in walk_repo(
+ root_path, prune_dirs=prune_dirs, prune_nested_git=prune_nested_git
+ ):
+ rel_dir = dirpath.relative_to(root_path).as_posix()
+ prefix = "" if rel_dir == "." else rel_dir + "/"
+ for name in filenames:
+ if any(_matches(prefix + name, name, p) for p in pats):
+ yield dirpath / name
+ for name in dirnames:
+ if any(_matches(prefix + name, name, p) for p in pats):
+ yield dirpath / name
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py
index 0a4918739..b0e25b2ab 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py
@@ -1,154 +1,31 @@
"""Pruned filesystem walk for dynamic-hint extractors.
-Every extractor in this package used to call ``repo_root.rglob(pattern)``
-directly, which descended into ``node_modules``, ``.venv``, ``.next``,
-``__pycache__`` and other multi-million-file junk trees. On a polyrepo
-with sibling ``backend/`` and ``frontend/`` directories the dynamic-hints
-phase could stall for 5–10+ minutes per extractor with the progress bar
-stuck at zero.
-
-``iter_glob`` uses :func:`os.walk` with in-place ``dirnames`` pruning so
-known-junk subdirectories are skipped at the point of traversal, not
-afterwards. It also guards against:
-
- - **Symlink loops** — ``os.walk`` is called with ``followlinks=False``.
- - **Junction loops** (Windows) — caught via :func:`os.path.realpath`
- cycle detection.
- - **Pathological recursive copies** — same cycle detector handles real
- directories that share an inode through hard-linking, junctions, or
- repeated nested-copy mistakes.
- - **Catastrophic depth** — a hard depth cap as a final safety net.
-
-Semantics match :py:meth:`pathlib.Path.rglob`: both files and directories
-whose basename matches the glob are yielded.
+The implementation now lives in :mod:`repowise.core.fs_walk` — the single
+shared pruned walk used by every repo-wide scan (resolvers, external
+systems, tsconfig discovery, dynamic hints). This module re-exports it so
+existing extractor imports keep working.
+
+Behavior note vs the historical private copy: the shared walk also prunes
+**nested git repositories** (a subdirectory with its own ``.git``), which
+is what dynamic hints want too — settings/router files from a vendored or
+sibling checkout are not this repo's hints.
+
+Dynamic hints keep the *derived-output* prune set (``dist``/``build``/
+``out``/``coverage``), matching the historical list here: hint extraction
+reads source files, and derived copies of them only add noise.
"""
from __future__ import annotations
-import fnmatch
-import os
from collections.abc import Iterator
from pathlib import Path
-import structlog
+from repowise.core.fs_walk import PRUNED_DIRS_DERIVED as PRUNED_DIRS
+from repowise.core.fs_walk import iter_glob as _shared_iter_glob
-log = structlog.get_logger(__name__)
-
-# Directory basenames that are never source-bearing. Pruned at every
-# level of the walk. Keep this list conservative: anything ambiguously
-# named (``bin``, ``obj``, ``lib``) belongs out of here so we don't
-# accidentally drop real source.
-PRUNED_DIRS: frozenset[str] = frozenset(
- {
- # VCS / metadata
- ".git",
- ".hg",
- ".svn",
- # Python environments / caches
- ".venv",
- "venv",
- ".env",
- "__pycache__",
- ".mypy_cache",
- ".pytest_cache",
- ".ruff_cache",
- ".tox",
- ".nox",
- # JS / TS
- "node_modules",
- ".next",
- ".nuxt",
- ".turbo",
- ".parcel-cache",
- # Build artifacts (multi-language)
- "dist",
- "build",
- "out",
- ".gradle",
- ".cache",
- # Editor / IDE
- ".idea",
- ".vscode",
- # Coverage / test outputs
- "coverage",
- "htmlcov",
- ".nyc_output",
- # Repowise / data
- ".repowise",
- ".lancedb",
- }
-)
-
-# Safety net: bail if walk depth grows pathologically. Real-world repos
-# rarely exceed depth 20; anything beyond strongly suggests a cycle.
-_MAX_WALK_DEPTH = 64
+__all__ = ["PRUNED_DIRS", "iter_glob"]
def iter_glob(root: Path, pattern: str) -> Iterator[Path]:
- """Recursively yield paths under ``root`` whose basename matches ``pattern``.
-
- Mirrors :py:meth:`pathlib.Path.rglob` semantics — files *and*
- directories with a matching basename are yielded — but:
- - prunes well-known junk directories at every level,
- - never follows symlinks (``os.walk(followlinks=False)``),
- - detects junction / hard-link cycles via realpath tracking,
- - caps total depth as a final safety net.
-
- Args:
- root: Directory to walk. May be a string or :class:`Path`.
- pattern: A :mod:`fnmatch`-style glob applied to the basename.
- Examples: ``"*.go"``, ``"settings.py"``, ``"package.json"``,
- ``"tsconfig*.json"``.
- """
- root_path = Path(root)
- if not root_path.exists():
- return
-
- try:
- root_real = os.path.realpath(root_path)
- except OSError:
- return
- base_depth = root_real.rstrip(os.sep).count(os.sep)
- visited_real: set[str] = {root_real}
-
- for dirpath, dirnames, filenames in os.walk(root_path, followlinks=False):
- # Skip junk directories at every level.
- dirnames[:] = [d for d in dirnames if d not in PRUNED_DIRS]
-
- # Drop any dir whose realpath we've already entered — catches
- # Windows junctions and any other cycle that os.walk can't detect
- # via inode (Windows doesn't expose real inodes for filesystems
- # outside of NTFS proper).
- pruned: list[str] = []
- for d in list(dirnames):
- child = os.path.join(dirpath, d)
- try:
- child_real = os.path.realpath(child)
- except OSError:
- pruned.append(d)
- continue
- if child_real in visited_real:
- pruned.append(d)
- continue
- visited_real.add(child_real)
- if pruned:
- dirnames[:] = [d for d in dirnames if d not in pruned]
-
- # Final safety net: refuse to descend beyond _MAX_WALK_DEPTH.
- cur_depth = dirpath.rstrip(os.sep).count(os.sep) - base_depth
- if cur_depth >= _MAX_WALK_DEPTH:
- log.warning(
- "dynamic_hints.walk_depth_exceeded",
- root=str(root_path),
- depth=cur_depth,
- at=dirpath,
- )
- dirnames[:] = []
- continue
-
- for name in filenames:
- if fnmatch.fnmatch(name, pattern):
- yield Path(dirpath) / name
- for name in dirnames:
- if fnmatch.fnmatch(name, pattern):
- yield Path(dirpath) / name
+ """Historical dynamic-hints walk: shared iter_glob + derived-dirs pruning."""
+ return _shared_iter_glob(root, pattern, prune_dirs=PRUNED_DIRS)
diff --git a/packages/core/src/repowise/core/ingestion/external_systems/__init__.py b/packages/core/src/repowise/core/ingestion/external_systems/__init__.py
index bec436987..61f8ace33 100644
--- a/packages/core/src/repowise/core/ingestion/external_systems/__init__.py
+++ b/packages/core/src/repowise/core/ingestion/external_systems/__init__.py
@@ -78,17 +78,25 @@ def extract_external_systems(
def _discover(repo_root: Path) -> list[Path]:
"""Walk the repo (depth ≤ 4) collecting recognised manifest files."""
+ from repowise.core.fs_walk import PRUNED_DIRS, walk_repo
+
seen: list[Path] = []
- skip_dirs = {".git", "node_modules", "__pycache__", ".venv", "venv", "target", "dist", "build"}
root = Path(repo_root)
- for path in root.rglob("*"):
- rel_parts = path.relative_to(root).parts
- if len(rel_parts) > 4 or any(part in skip_dirs for part in rel_parts):
- continue
- if not path.is_file():
- continue
- if _parser_for(path) is not None:
- seen.append(path)
+ # The historical depth cap was a *post-hoc* filter on an unbounded
+ # ``rglob("*")`` — the walk itself still descended into everything.
+ # walk_repo prunes junk dirs and nested git repos at traversal time;
+ # clearing ``dirnames`` at depth 3 makes the bound real (files yielded
+ # there have 4 relative parts, matching the old ``len(rel_parts) > 4``).
+ # Historical skip list included target/dist/build — preserve it on top
+ # of the shared never-source set.
+ prune = PRUNED_DIRS | {"target", "dist", "build"}
+ for dirpath, dirnames, filenames in walk_repo(root, prune_dirs=prune):
+ if len(dirpath.relative_to(root).parts) >= 3:
+ dirnames[:] = []
+ for fname in filenames:
+ path = dirpath / fname
+ if _parser_for(path) is not None:
+ seen.append(path)
return seen
diff --git a/packages/core/src/repowise/core/ingestion/external_systems/bazel.py b/packages/core/src/repowise/core/ingestion/external_systems/bazel.py
index 1db86de34..d973fc786 100644
--- a/packages/core/src/repowise/core/ingestion/external_systems/bazel.py
+++ b/packages/core/src/repowise/core/ingestion/external_systems/bazel.py
@@ -280,8 +280,10 @@ def discover_bazel_packages(repo_root: Path, *, max_files: int = 5000) -> list[B
out: list[BazelFile] = []
skip_dirs = {".git", "node_modules", "bazel-bin", "bazel-out", "bazel-testlogs",
"bazel-" + repo_root.name, ".venv", "venv", "build"}
+ from repowise.core.fs_walk import iter_glob
+
for candidate in ("BUILD", "BUILD.bazel"):
- for p in repo_root.rglob(candidate):
+ for p in iter_glob(repo_root, candidate):
try:
rel = p.resolve().relative_to(repo_root).as_posix()
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/external_systems/cmake.py b/packages/core/src/repowise/core/ingestion/external_systems/cmake.py
index fef1c20e8..fa6f90d20 100644
--- a/packages/core/src/repowise/core/ingestion/external_systems/cmake.py
+++ b/packages/core/src/repowise/core/ingestion/external_systems/cmake.py
@@ -631,7 +631,9 @@ def discover_cmake_reactor(
skip_dirs = {".git", "build", "_build", "out", "_deps", "cmake-build-debug",
"cmake-build-release", "node_modules", ".venv", "venv"}
if len(out) < max_files:
- for cml in repo_root.rglob("CMakeLists.txt"):
+ from repowise.core.fs_walk import iter_glob
+
+ for cml in iter_glob(repo_root, "CMakeLists.txt"):
try:
rel = cml.resolve().relative_to(repo_root).as_posix()
except ValueError:
diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.py
index 862b5ef03..f50922783 100644
--- a/packages/core/src/repowise/core/ingestion/graph/builder.py
+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py
@@ -46,11 +46,17 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
*,
exclude_patterns: list[str] | None = None,
centrality_cache_dir: Path | str | None = None,
+ include_submodules: bool = False,
+ include_nested_repos: bool = False,
) -> None:
self._graph: nx.DiGraph = nx.DiGraph()
self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile
self._built = False
self._repo_path: Path | None = Path(repo_path) if repo_path else None
+ # Mirrors the traverser flags: when submodules or nested repos are
+ # indexed, resolver filesystem scans must not prune them (both are
+ # ``.git``-bearing subdirs to fs_walk's prune_nested_git).
+ self._prune_nested_git: bool = not (include_submodules or include_nested_repos)
self._tsconfig_resolver: Any | None = None # TsconfigResolver (lazy import)
# Optional structure-keyed disk cache for betweenness (the most
# expensive metric kernel). Opt-in via *centrality_cache_dir* — the
@@ -239,14 +245,17 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
stem_map = build_stem_map(path_set)
# Construct resolver context
- go_modules = read_go_modules(self._repo_path)
+ go_modules = read_go_modules(self._repo_path, prune_nested_git=self._prune_nested_git)
ctx = ResolverContext(
path_set=path_set,
stem_map=stem_map,
graph=self._graph,
repo_path=self._repo_path,
+ prune_nested_git=self._prune_nested_git,
tsconfig_resolver=self._tsconfig_resolver,
- go_module_path=(go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)),
+ go_module_path=(
+ go_modules[-1][1] if go_modules else read_go_module_path(self._repo_path)
+ ),
go_modules=go_modules,
has_sfc_files=any(p.endswith((".vue", ".svelte", ".astro")) for p in path_set),
parsed_files=self._parsed_files,
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/context.py b/packages/core/src/repowise/core/ingestion/resolvers/context.py
index cf7cb0abf..54df8cb20 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/context.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/context.py
@@ -27,6 +27,13 @@ class ResolverContext:
graph: nx.DiGraph
repo_path: Path | None = None
+ # Whether repo-wide filesystem scans (fs_walk) skip nested git repos.
+ # True by default — sibling/vendored checkouts must not leak manifests.
+ # Set False when the pipeline indexes ``.git``-bearing subdirs (either
+ # ``include_submodules`` or ``include_nested_repos``) so resolver scans
+ # see the same files the traverser indexed.
+ prune_nested_git: bool = True
+
# Language-specific state
tsconfig_resolver: Any | None = None
go_module_path: str | None = None
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.py
index 3346162c9..69a701b58 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/global_usings.py
@@ -11,6 +11,8 @@ from __future__ import annotations
import re
from pathlib import Path
+from repowise.core.fs_walk import iter_glob
+
# Default ImplicitUsings set for SDK ``Microsoft.NET.Sdk`` projects.
# Source: dotnet/sdk repo, Sdks/Microsoft.NET.Sdk/targets/Microsoft.NET.Sdk.ImplicitNamespaceImports.props
_DEFAULT_IMPLICIT_USINGS: frozenset[str] = frozenset(
@@ -58,6 +60,7 @@ def collect_project_global_usings(
sdk_is_web: bool = False,
*,
project_texts: dict[Path, str] | None = None,
+ prune_nested_git: bool = True,
) -> set[str]:
"""Walk *project_dir* for global using sources and return the merged set.
@@ -86,7 +89,7 @@ def collect_project_global_usings(
return result
skip = {"bin", "obj", ".vs", "node_modules"}
- for cs_path in project_dir.rglob("*.cs"):
+ for cs_path in iter_glob(project_dir, "*.cs", prune_nested_git=prune_nested_git):
if any(part in skip for part in cs_path.parts):
continue
try:
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py
index 6c12a4178..a58a71ecc 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/index.py
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
import structlog
+from repowise.core.fs_walk import iter_glob
+
from .global_usings import collect_project_global_usings
from .msbuild import MSBuildProject, find_csproj_files, find_directory_build_props, parse_csproj
from .namespace_map import build_namespace_map
@@ -147,7 +149,7 @@ class DotNetProjectIndex:
_CS_WALK_SKIP_DIRS = frozenset({"bin", "obj", ".vs", "node_modules", ".git", "packages"})
-def _walk_repo_cs_files(repo_path: Path) -> list[Path]:
+def _walk_repo_cs_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:
"""Single repo-wide rglob for ``*.cs`` files, dedup by resolved path.
Lives at module scope (not nested inside ``build_index``) so it's
@@ -156,7 +158,7 @@ def _walk_repo_cs_files(repo_path: Path) -> list[Path]:
"""
seen: set[Path] = set()
out: list[Path] = []
- for cs in repo_path.rglob("*.cs"):
+ for cs in iter_glob(repo_path, "*.cs", prune_nested_git=prune_nested_git):
if any(part in _CS_WALK_SKIP_DIRS for part in cs.parts):
continue
try:
@@ -205,7 +207,7 @@ def _bucket_files_by_project(
return out
-def build_index(repo_path: Path) -> DotNetProjectIndex:
+def build_index(repo_path: Path, *, prune_nested_git: bool = True) -> DotNetProjectIndex:
"""Walk *repo_path* and construct a fully-populated DotNetProjectIndex.
Performance note: a previous version of this function walked the
@@ -223,7 +225,7 @@ def build_index(repo_path: Path) -> DotNetProjectIndex:
index = DotNetProjectIndex(repo_path=repo_path)
# ---- 1. Parse every .csproj ----
- for csproj_path in find_csproj_files(repo_path):
+ for csproj_path in find_csproj_files(repo_path, prune_nested_git=prune_nested_git):
proj = parse_csproj(csproj_path)
if proj is None:
continue
@@ -232,7 +234,7 @@ def build_index(repo_path: Path) -> DotNetProjectIndex:
index.package_refs[proj.path] = set(proj.package_references)
# ---- 2. Walk .sln files (informational; surfaces orphaned .csprojs) ----
- index.sln_paths = find_sln_files(repo_path)
+ index.sln_paths = find_sln_files(repo_path, prune_nested_git=prune_nested_git)
for sln in index.sln_paths:
for entry in parse_sln(sln):
if entry.csproj not in index.projects:
@@ -246,7 +248,7 @@ def build_index(repo_path: Path) -> DotNetProjectIndex:
index.package_refs.setdefault(proj.path, set()).update(proj.package_references)
# ---- 3. Single master walk: enumerate .cs files & read each once ----
- all_cs_files = _walk_repo_cs_files(repo_path)
+ all_cs_files = _walk_repo_cs_files(repo_path, prune_nested_git=prune_nested_git)
cs_texts: dict[Path, str] = {}
for f in all_cs_files:
try:
@@ -317,6 +319,6 @@ def get_or_build_index(ctx: "ResolverContext") -> DotNetProjectIndex | None:
cached = getattr(ctx, _INDEX_KEY, None)
if cached is not None:
return cached
- index = build_index(ctx.repo_path)
+ index = build_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
setattr(ctx, _INDEX_KEY, index)
return index
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py
index ed849e6e2..829279dfc 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/msbuild.py
@@ -14,6 +14,8 @@ from xml.etree import ElementTree as ET
import structlog
+from repowise.core.fs_walk import iter_glob
+
log = structlog.get_logger(__name__)
@@ -87,11 +89,11 @@ def parse_csproj(csproj_path: Path) -> MSBuildProject | None:
return project
-def find_csproj_files(repo_path: Path) -> list[Path]:
+def find_csproj_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:
"""Return all .csproj files under *repo_path*, skipping bin/obj output."""
skip = {"bin", "obj", ".vs", "node_modules", ".git", "packages", "TestResults"}
out: list[Path] = []
- for csproj in repo_path.rglob("*.csproj"):
+ for csproj in iter_glob(repo_path, "*.csproj", prune_nested_git=prune_nested_git):
if any(part in skip for part in csproj.parts):
continue
out.append(csproj)
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py
index b1005488b..3fe510857 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/dotnet/solution.py
@@ -16,6 +16,8 @@ from pathlib import Path
import structlog
+from repowise.core.fs_walk import iter_glob
+
log = structlog.get_logger(__name__)
# Project("{TYPE}") = "Name", "rel\path.csproj", "{GUID}"
@@ -56,10 +58,10 @@ def parse_sln(sln_path: Path) -> list[SolutionEntry]:
return out
-def find_sln_files(repo_path: Path) -> list[Path]:
+def find_sln_files(repo_path: Path, *, prune_nested_git: bool = True) -> list[Path]:
skip = {".git", "node_modules", "bin", "obj"}
out: list[Path] = []
- for sln in repo_path.rglob("*.sln"):
+ for sln in iter_glob(repo_path, "*.sln", prune_nested_git=prune_nested_git):
if any(part in skip for part in sln.parts):
continue
out.append(sln)
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/go.py b/packages/core/src/repowise/core/ingestion/resolvers/go.py
index 2120dde32..61bedb375 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/go.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/go.py
@@ -4,6 +4,8 @@ from __future__ import annotations
from pathlib import Path
+from repowise.core.fs_walk import iter_glob
+
from .context import ResolverContext
_GO_MOD_SKIP_DIRS = frozenset({"vendor", "node_modules", ".git"})
@@ -30,7 +32,9 @@ def read_go_module_path(repo_path: Path | None) -> str | None:
return _read_module_directive(go_mod)
-def read_go_modules(repo_path: Path | None) -> tuple[tuple[str, str], ...]:
+def read_go_modules(
+ repo_path: Path | None, *, prune_nested_git: bool = True
+) -> tuple[tuple[str, str], ...]:
"""Discover every ``go.mod`` under *repo_path* and return
``((module_dir_posix, module_path), ...)`` sorted longest-module-path-first.
@@ -40,7 +44,7 @@ def read_go_modules(repo_path: Path | None) -> tuple[tuple[str, str], ...]:
if repo_path is None or not repo_path.is_dir():
return ()
found: list[tuple[str, str]] = []
- for go_mod in repo_path.rglob("go.mod"):
+ for go_mod in iter_glob(repo_path, "go.mod", prune_nested_git=prune_nested_git):
# Skip vendored / nested-package directories
rel_parts = go_mod.relative_to(repo_path).parts
if any(part in _GO_MOD_SKIP_DIRS for part in rel_parts):
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py
index cc09adbd1..f874e2c81 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_gradle.py
@@ -19,6 +19,8 @@ from typing import TYPE_CHECKING
import structlog
+from repowise.core.fs_walk import iter_glob
+
if TYPE_CHECKING:
from .context import ResolverContext
@@ -289,7 +291,9 @@ def _extract_file_info(abs_path: str) -> tuple[str, tuple[str, ...], bool, bool]
# Index building
# ---------------------------------------------------------------------------
-def build_jvm_gradle_index(repo_path: Path | None) -> JvmGradleIndex:
+def build_jvm_gradle_index(
+ repo_path: Path | None, *, prune_nested_git: bool = True
+) -> JvmGradleIndex:
"""Walk Gradle config + JVM sources to build the package-resolution index."""
index = JvmGradleIndex()
if repo_path is None or not repo_path.is_dir():
@@ -349,7 +353,7 @@ def build_jvm_gradle_index(repo_path: Path | None) -> JvmGradleIndex:
continue
rel_roots.append(rel)
- for jvm_file in root_path.rglob("*"):
+ for jvm_file in iter_glob(root_path, "*", prune_nested_git=prune_nested_git):
if jvm_file.suffix not in _JVM_EXTENSIONS or not jvm_file.is_file():
continue
try:
@@ -380,7 +384,7 @@ def get_or_build_jvm_gradle_index(ctx: "ResolverContext") -> JvmGradleIndex:
cached = getattr(ctx, "_jvm_gradle_index", None)
if cached is not None:
return cached
- index = build_jvm_gradle_index(ctx.repo_path)
+ index = build_jvm_gradle_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
ctx._jvm_gradle_index = index # type: ignore[attr-defined]
return index
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py
index b4d91f2d5..08943ef0b 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/jvm_workspace.py
@@ -21,6 +21,8 @@ from typing import TYPE_CHECKING
import structlog
+from repowise.core.fs_walk import iter_glob
+
if TYPE_CHECKING:
from .context import ResolverContext
@@ -175,7 +177,9 @@ _JPMS_PROVIDES_RE = re.compile(
)
-def _scan_jpms_provides(repo_path: Path) -> dict[str, tuple[str, ...]]:
+def _scan_jpms_provides(
+ repo_path: Path, *, prune_nested_git: bool = True
+) -> dict[str, tuple[str, ...]]:
"""Scan ``module-info.java`` files for ``provides X with Y, Z`` directives.
Returns a mapping iface_fqn → impl_fqns identical in shape to
@@ -185,7 +189,7 @@ def _scan_jpms_provides(repo_path: Path) -> dict[str, tuple[str, ...]]:
the warmup fast.
"""
out: dict[str, list[str]] = {}
- for mi in repo_path.rglob("module-info.java"):
+ for mi in iter_glob(repo_path, "module-info.java", prune_nested_git=prune_nested_git):
if not mi.is_file():
continue
try:
@@ -200,10 +204,14 @@ def _scan_jpms_provides(repo_path: Path) -> dict[str, tuple[str, ...]]:
return {k: tuple(v) for k, v in out.items()}
-def _scan_meta_inf_services(repo_path: Path) -> dict[str, tuple[str, ...]]:
+def _scan_meta_inf_services(
+ repo_path: Path, *, prune_nested_git: bool = True
+) -> dict[str, tuple[str, ...]]:
"""Scan META-INF/services/ directories for SPI declarations."""
services: dict[str, list[str]] = {}
- for services_dir in repo_path.rglob("META-INF/services"):
+ for services_dir in iter_glob(
+ repo_path, "META-INF/services", prune_nested_git=prune_nested_git
+ ):
if not services_dir.is_dir():
continue
try:
@@ -230,12 +238,16 @@ def _scan_meta_inf_services(repo_path: Path) -> dict[str, tuple[str, ...]]:
return {k: tuple(v) for k, v in services.items()}
-def _scan_spring_autoconfig(repo_path: Path) -> dict[str, tuple[str, ...]]:
+def _scan_spring_autoconfig(
+ repo_path: Path, *, prune_nested_git: bool = True
+) -> dict[str, tuple[str, ...]]:
"""Scan spring.factories and Boot 3 AutoConfiguration.imports."""
result: dict[str, list[str]] = {}
# spring.factories (Boot 2 style)
- for factories in repo_path.rglob("META-INF/spring.factories"):
+ for factories in iter_glob(
+ repo_path, "META-INF/spring.factories", prune_nested_git=prune_nested_git
+ ):
if not factories.is_file():
continue
try:
@@ -264,7 +276,9 @@ def _scan_spring_autoconfig(repo_path: Path) -> dict[str, tuple[str, ...]]:
result.setdefault(rel, []).extend(current_values)
# Boot 3 style: META-INF/spring/*.imports
- for imports_file in repo_path.rglob("META-INF/spring/*.imports"):
+ for imports_file in iter_glob(
+ repo_path, "META-INF/spring/*.imports", prune_nested_git=prune_nested_git
+ ):
if not imports_file.is_file():
continue
try:
@@ -336,8 +350,8 @@ def build_jvm_workspace_index(ctx: "ResolverContext") -> JvmWorkspaceIndex:
# (cheap glob, O(matching files)). Both populate ``services``; later
# phases treat any FQN listed there as reachable.
try:
- services = _scan_meta_inf_services(repo_path)
- jpms = _scan_jpms_provides(repo_path)
+ services = _scan_meta_inf_services(repo_path, prune_nested_git=ctx.prune_nested_git)
+ jpms = _scan_jpms_provides(repo_path, prune_nested_git=ctx.prune_nested_git)
merged: dict[str, list[str]] = {k: list(v) for k, v in services.items()}
for iface, impls in jpms.items():
merged.setdefault(iface, []).extend(impls)
@@ -345,7 +359,9 @@ def build_jvm_workspace_index(ctx: "ResolverContext") -> JvmWorkspaceIndex:
except Exception:
pass
try:
- index.autoconfig_imports = _scan_spring_autoconfig(repo_path)
+ index.autoconfig_imports = _scan_spring_autoconfig(
+ repo_path, prune_nested_git=ctx.prune_nested_git
+ )
except Exception:
pass
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.py b/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.py
index 82e8c2687..e135458d5 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/ruby_rails.py
@@ -17,6 +17,8 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING
+from repowise.core.fs_walk import iter_glob
+
if TYPE_CHECKING:
from .context import ResolverContext
@@ -76,7 +78,9 @@ def _is_rails_repo(repo_path: Path) -> bool:
return (repo_path / "config" / "application.rb").is_file()
-def build_rails_index(repo_path: Path | None) -> RailsIndex | None:
+def build_rails_index(
+ repo_path: Path | None, *, prune_nested_git: bool = True
+) -> RailsIndex | None:
if repo_path is None or not repo_path.is_dir():
return None
if not _is_rails_repo(repo_path):
@@ -90,7 +94,7 @@ def build_rails_index(repo_path: Path | None) -> RailsIndex | None:
if not root_path.is_dir():
continue
index.autoload_roots.append(root)
- for rb in root_path.rglob("*.rb"):
+ for rb in iter_glob(root_path, "*.rb", prune_nested_git=prune_nested_git):
try:
rel = rb.relative_to(repo_resolved).as_posix()
except ValueError:
@@ -110,6 +114,6 @@ def get_or_build_rails_index(ctx: "ResolverContext") -> RailsIndex | None:
cached = getattr(ctx, "_ruby_rails_index", "__sentinel__")
if cached != "__sentinel__":
return cached # type: ignore[return-value]
- index = build_rails_index(ctx.repo_path)
+ index = build_rails_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
ctx._ruby_rails_index = index # type: ignore[attr-defined]
return index
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/scala_build.py b/packages/core/src/repowise/core/ingestion/resolvers/scala_build.py
index 51ed3e255..a5ae07c58 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/scala_build.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/scala_build.py
@@ -16,6 +16,8 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING
+from repowise.core.fs_walk import iter_glob
+
if TYPE_CHECKING:
from .context import ResolverContext
@@ -82,7 +84,9 @@ def _parse_build_sc(repo_path: Path) -> dict[str, str]:
return result
-def _scan_packages(repo_path: Path, project_dir: str, build_tool: str) -> dict[str, list[str]]:
+def _scan_packages(
+ repo_path: Path, project_dir: str, build_tool: str, *, prune_nested_git: bool = True
+) -> dict[str, list[str]]:
"""Walk a project's source tree, recording package declarations."""
found: dict[str, list[str]] = {}
proj_path = (repo_path / project_dir).resolve() if project_dir else repo_path.resolve()
@@ -107,7 +111,7 @@ def _scan_packages(repo_path: Path, project_dir: str, build_tool: str) -> dict[s
if not root.is_dir() or root in seen:
continue
seen.add(root)
- for src in root.rglob("*.scala"):
+ for src in iter_glob(root, "*.scala", prune_nested_git=prune_nested_git):
try:
rel = src.relative_to(repo_path.resolve()).as_posix()
except ValueError:
@@ -123,7 +127,9 @@ def _scan_packages(repo_path: Path, project_dir: str, build_tool: str) -> dict[s
return found
-def build_scala_index(repo_path: Path | None) -> ScalaProjectIndex:
+def build_scala_index(
+ repo_path: Path | None, *, prune_nested_git: bool = True
+) -> ScalaProjectIndex:
if repo_path is None or not repo_path.is_dir():
return ScalaProjectIndex()
sbt_projects = _parse_build_sbt(repo_path)
@@ -143,7 +149,10 @@ def build_scala_index(repo_path: Path | None) -> ScalaProjectIndex:
return ScalaProjectIndex()
for _name, project_dir in index.projects.items():
- for pkg, files in _scan_packages(repo_path, project_dir, index.build_tool).items():
+ scanned = _scan_packages(
+ repo_path, project_dir, index.build_tool, prune_nested_git=prune_nested_git
+ )
+ for pkg, files in scanned.items():
index.package_to_files.setdefault(pkg, []).extend(files)
return index
@@ -152,7 +161,7 @@ def get_or_build_scala_index(ctx: "ResolverContext") -> ScalaProjectIndex:
cached = getattr(ctx, "_scala_index", None)
if cached is not None:
return cached
- index = build_scala_index(ctx.repo_path)
+ index = build_scala_index(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
ctx._scala_index = index # type: ignore[attr-defined]
return index
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.py b/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.py
index 9e2512ae6..debc87c53 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/swift_spm.py
@@ -19,6 +19,8 @@ import re
from pathlib import Path
from typing import TYPE_CHECKING
+from repowise.core.fs_walk import iter_glob
+
if TYPE_CHECKING:
from .context import ResolverContext
@@ -61,7 +63,9 @@ def parse_package_swift(path: Path) -> dict[str, str]:
return result
-def build_swift_targets(repo_path: Path | None) -> dict[str, str]:
+def build_swift_targets(
+ repo_path: Path | None, *, prune_nested_git: bool = True
+) -> dict[str, str]:
"""Walk the repo for every ``Package.swift``, merge their target maps.
Each target's source dir is prefixed with the package's directory
@@ -71,7 +75,7 @@ def build_swift_targets(repo_path: Path | None) -> dict[str, str]:
if repo_path is None or not repo_path.is_dir():
return {}
merged: dict[str, str] = {}
- for pkg_swift in repo_path.rglob("Package.swift"):
+ for pkg_swift in iter_glob(repo_path, "Package.swift", prune_nested_git=prune_nested_git):
try:
pkg_dir = pkg_swift.parent.relative_to(repo_path).as_posix()
except ValueError:
@@ -86,7 +90,7 @@ def get_or_build_swift_targets(ctx: "ResolverContext") -> dict[str, str]:
cached = getattr(ctx, "_swift_targets", None)
if cached is not None:
return cached
- mapping = build_swift_targets(ctx.repo_path)
+ mapping = build_swift_targets(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
ctx._swift_targets = mapping # type: ignore[attr-defined]
return mapping
diff --git a/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py b/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py
index 657e1ed8e..0eae9e64b 100644
--- a/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py
+++ b/packages/core/src/repowise/core/ingestion/resolvers/ts_workspace.py
@@ -25,7 +25,6 @@ flattened to the first plausible source target. Packages without an
from __future__ import annotations
import json
-import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -84,11 +83,18 @@ class _RepoFileScan:
package_jsons: list[Path] = field(default_factory=list)
-def _scan_repo_files(repo_path: Path) -> _RepoFileScan:
- """One pruned ``os.walk`` collecting every finder's target files."""
+def _scan_repo_files(repo_path: Path, *, prune_nested_git: bool = True) -> _RepoFileScan:
+ """One pruned walk collecting every finder's target files.
+
+ Uses the shared :func:`repowise.core.fs_walk.walk_repo` (nested-git
+ pruning, cycle guard) with this module's deliberately-narrow prune set.
+ """
+ from repowise.core.fs_walk import walk_repo
+
scan = _RepoFileScan()
- for dirpath, dirnames, filenames in os.walk(repo_path):
- dirnames[:] = [d for d in dirnames if d not in _SCAN_PRUNE_DIRS]
+ for dirpath, _dirnames, filenames in walk_repo(
+ repo_path, prune_dirs=_SCAN_PRUNE_DIRS, prune_nested_git=prune_nested_git
+ ):
for fname in filenames:
if fname.endswith(".mdx"):
scan.mdx_files.append(Path(dirpath) / fname)
@@ -104,7 +110,11 @@ def _get_repo_scan(ctx: "ResolverContext") -> _RepoFileScan:
cached = getattr(ctx, "_ts_repo_file_scan", None)
if cached is not None:
return cached
- scan = _scan_repo_files(ctx.repo_path) if ctx.repo_path is not None else _RepoFileScan()
+ scan = (
+ _scan_repo_files(ctx.repo_path, prune_nested_git=ctx.prune_nested_git)
+ if ctx.repo_path is not None
+ else _RepoFileScan()
+ )
ctx._ts_repo_file_scan = scan # type: ignore[attr-defined]
return scan
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index 05219a05f..b16de7a4a 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -468,9 +468,18 @@ class FileTraverser:
# ------------------------------------------------------------------
def _detect_monorepo(self) -> tuple[list[PackageInfo], bool]:
- """Detect package sub-directories by looking for manifest files."""
+ """Detect package sub-directories by looking for manifest files.
+
+ Candidate dirs the main traversal would never enter (nested git
+ repos, submodules, gitignored/blocked dirs) are rejected up front:
+ a "package" the walk skips must not be reported — and, before this
+ guard, each such candidate was expensively ``rglob``-scanned for
+ language/entry-point detection (minutes per sibling repo on a
+ directory that physically contains other checkouts).
+ """
packages: list[PackageInfo] = []
seen_paths: set[str] = set()
+ prune_nested = not self._include_nested_repos
for depth in (1, 2):
pattern = "/".join(["*"] * depth) + "/*"
@@ -478,12 +487,17 @@ class FileTraverser:
if candidate.name not in _MANIFEST_FILES:
continue
pkg_dir = candidate.parent
- rel_pkg = pkg_dir.relative_to(self.repo_root).as_posix()
+ rel_pkg_path = pkg_dir.relative_to(self.repo_root)
+ rel_pkg = rel_pkg_path.as_posix()
if rel_pkg in seen_paths:
continue
+ if self._dir_chain_skipped(rel_pkg_path):
+ continue
seen_paths.add(rel_pkg)
- lang = _primary_language_in(pkg_dir)
- entry_pts = _find_entry_points_in(pkg_dir, self.repo_root)
+ lang = _primary_language_in(pkg_dir, prune_nested_git=prune_nested)
+ entry_pts = _find_entry_points_in(
+ pkg_dir, self.repo_root, prune_nested_git=prune_nested
+ )
packages.append(
PackageInfo(
name=pkg_dir.name,
@@ -497,6 +511,20 @@ class FileTraverser:
packages.sort(key=lambda p: p.path)
return packages, len(packages) > 1
+ def _dir_chain_skipped(self, rel_dir: Path) -> bool:
+ """True if *rel_dir* (or any ancestor) would be pruned by ``_walk``.
+
+ Reuses :meth:`_should_skip_dir` level by level so monorepo package
+ detection has exactly the same boundary semantics as file traversal
+ (blocked dirs, submodules, nested git repos, gitignore/excludes).
+ """
+ cur = Path()
+ for part in rel_dir.parts:
+ cur = cur / part
+ if self._should_skip_dir(part, cur, self.repo_root / cur):
+ return True
+ return False
+
# ---------------------------------------------------------------------------
# Module-level helpers
@@ -590,12 +618,16 @@ def _stem_is_entry_point(abs_path: Path) -> bool:
return stem in _ENTRY_POINT_STEMS
-def _primary_language_in(directory: Path) -> LanguageTag:
+def _primary_language_in(directory: Path, *, prune_nested_git: bool = True) -> LanguageTag:
+ from repowise.core.fs_walk import walk_repo
+
counts: dict[str, int] = {}
try:
- for item in directory.rglob("*"):
- if item.is_file():
- lang = _detect_language(item)
+ for dirpath, _dirnames, filenames in walk_repo(
+ directory, prune_nested_git=prune_nested_git
+ ):
+ for fname in filenames:
+ lang = _detect_language(dirpath / fname)
if lang not in ("unknown", "yaml", "json", "markdown", "toml"):
counts[lang] = counts.get(lang, 0) + 1
except OSError:
@@ -605,12 +637,19 @@ def _primary_language_in(directory: Path) -> LanguageTag:
return max(counts, key=lambda k: counts[k]) # type: ignore[return-value]
-def _find_entry_points_in(directory: Path, repo_root: Path) -> list[str]:
+def _find_entry_points_in(
+ directory: Path, repo_root: Path, *, prune_nested_git: bool = True
+) -> list[str]:
+ from repowise.core.fs_walk import walk_repo
+
result: list[str] = []
try:
- for item in directory.rglob("*"):
- if item.is_file() and item.name in _ENTRY_POINT_NAMES:
- result.append(item.relative_to(repo_root).as_posix())
+ for dirpath, _dirnames, filenames in walk_repo(
+ directory, prune_nested_git=prune_nested_git
+ ):
+ for fname in filenames:
+ if fname in _ENTRY_POINT_NAMES:
+ result.append((dirpath / fname).relative_to(repo_root).as_posix())
except OSError:
pass
return sorted(result)
diff --git a/packages/core/src/repowise/core/ingestion/tsconfig_resolver.py b/packages/core/src/repowise/core/ingestion/tsconfig_resolver.py
index 851843f37..341cf0658 100644
--- a/packages/core/src/repowise/core/ingestion/tsconfig_resolver.py
+++ b/packages/core/src/repowise/core/ingestion/tsconfig_resolver.py
@@ -80,11 +80,16 @@ class TsconfigResolver:
Args:
repo_path: Absolute path to the repository root.
path_set: Set of POSIX-relative file paths known to the graph builder.
+ prune_nested_git: Skip nested git repos during config discovery
+ (set False when the pipeline runs with ``--include-submodules``).
"""
- def __init__(self, repo_path: Path, path_set: set[str]) -> None:
+ def __init__(
+ self, repo_path: Path, path_set: set[str], *, prune_nested_git: bool = True
+ ) -> None:
self._repo_path = repo_path.resolve()
self._path_set = path_set
+ self._prune_nested_git = prune_nested_git
# directory (absolute) -> ResolvedConfig
self._dir_to_config: dict[Path, ResolvedConfig] = {}
@@ -150,9 +155,13 @@ class TsconfigResolver:
# Group candidates by directory, assign priority (lower = higher).
dir_candidates: dict[Path, list[tuple[int, Path]]] = {}
- for config_file in self._repo_path.rglob("*.json"):
- if "node_modules" in config_file.parts:
- continue
+ from repowise.core.fs_walk import iter_glob
+
+ for config_file in iter_glob(
+ self._repo_path,
+ ("tsconfig*.json", "jsconfig.json"),
+ prune_nested_git=self._prune_nested_git,
+ ):
name = config_file.name
if name == "tsconfig.json":
priority = 0
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index 525a5a362..f0aa3f88c 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -246,6 +246,8 @@ async def _run_ingestion(
repo_path=repo_path,
exclude_patterns=exclude_patterns,
centrality_cache_dir=repo_path / ".repowise",
+ include_submodules=include_submodules,
+ include_nested_repos=include_nested_repos,
)
loop = asyncio.get_running_loop()
@@ -335,7 +337,11 @@ async def _run_ingestion(
if progress:
progress.on_phase_start("tsconfig", None)
_path_set = set(graph_builder._parsed_files.keys())
- _resolver = TsconfigResolver(repo_path=repo_path, path_set=_path_set)
+ _resolver = TsconfigResolver(
+ repo_path=repo_path,
+ path_set=_path_set,
+ prune_nested_git=not (include_submodules or include_nested_repos),
+ )
graph_builder.set_tsconfig_resolver(_resolver)
_phase_done(progress, "tsconfig")
except Exception as _resolver_exc:
diff --git a/packages/core/src/repowise/core/workspace/cross_repo.py b/packages/core/src/repowise/core/workspace/cross_repo.py
index 39dad4a43..5ce1b2f88 100644
--- a/packages/core/src/repowise/core/workspace/cross_repo.py
+++ b/packages/core/src/repowise/core/workspace/cross_repo.py
@@ -523,14 +523,18 @@ def _scan_csproj(
"""
from xml.etree import ElementTree as ET
+ from repowise.core.fs_walk import iter_glob
+
results: list[CrossRepoPackageDep] = []
skip = {"bin", "obj", ".vs", "packages", "node_modules", ".git", "TestResults"}
# Pre-compute the assembly-name → repo-alias map so we can resolve
- # internal-NuGet references in a second pass.
+ # internal-NuGet references in a second pass. Each *selected* workspace
+ # repo is walked from its own root; iter_glob's nested-git pruning keeps
+ # any physically-nested unselected repo out of the scan.
assembly_to_repo: dict[str, str] = {}
for sib_alias, sib_path in repo_paths.items():
- for csproj in sib_path.rglob("*.csproj"):
+ for csproj in iter_glob(sib_path, "*.csproj"):
if any(part in skip for part in csproj.parts):
continue
try:
@@ -545,7 +549,7 @@ def _scan_csproj(
break
assembly_to_repo[assembly_name] = sib_alias
- for csproj in repo_path.rglob("*.csproj"):
+ for csproj in iter_glob(repo_path, "*.csproj"):
if any(part in skip for part in csproj.parts):
continue
try:
diff --git a/tests/unit/ingestion/test_fs_walk.py b/tests/unit/ingestion/test_fs_walk.py
new file mode 100644
index 000000000..b2faf393f
--- /dev/null
+++ b/tests/unit/ingestion/test_fs_walk.py
@@ -0,0 +1,312 @@
+"""Unit tests for the shared pruned filesystem walk (``repowise.core.fs_walk``)
+plus the regression guard that keeps new unpruned ``rglob`` calls out of
+``packages/core``.
+
+Why this exists: unpruned ``Path.rglob`` over a repo that physically contains
+sibling/vendored repos (or ``node_modules`` / ``.venv``) has caused two
+separate multi-minute perf incidents AND leaks foreign manifests
+(``hugo/go.mod``, ``eShop/*.sln``) into the current repo's resolver context.
+Every repo-wide scan must go through ``fs_walk``.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from repowise.core.fs_walk import PRUNED_DIRS, PRUNED_DIRS_DERIVED, iter_glob, walk_repo
+
+
+def _write(root: Path, rel: str, content: str = "x") -> None:
+ p = root / rel
+ p.parent.mkdir(parents=True, exist_ok=True)
+ p.write_text(content, encoding="utf-8")
+
+
+# ---------------------------------------------------------------------------
+# iter_glob — basic rglob parity
+# ---------------------------------------------------------------------------
+
+
+class TestIterGlobParity:
+ def test_matches_rglob_on_clean_tree(self, tmp_path: Path) -> None:
+ """On a tree with no junk dirs, iter_glob == rglob exactly."""
+ _write(tmp_path, "go.mod")
+ _write(tmp_path, "services/foo/go.mod")
+ _write(tmp_path, "libs/bar/baz/go.mod")
+ _write(tmp_path, "libs/bar/baz/main.go")
+ expected = sorted(tmp_path.rglob("go.mod"))
+ assert sorted(iter_glob(tmp_path, "go.mod")) == expected
+
+ def test_yields_matching_directories_too(self, tmp_path: Path) -> None:
+ """rglob semantics: directories with a matching basename are yielded."""
+ (tmp_path / "src" / "META-INF" / "services").mkdir(parents=True)
+ results = list(iter_glob(tmp_path, "services"))
+ assert results == [tmp_path / "src" / "META-INF" / "services"]
+
+ def test_slash_pattern_matches_path_tail(self, tmp_path: Path) -> None:
+ """Patterns with '/' mirror rglob("META-INF/services") semantics."""
+ (tmp_path / "a" / "META-INF" / "services").mkdir(parents=True)
+ (tmp_path / "b" / "other" / "services").mkdir(parents=True)
+ expected = sorted(tmp_path.rglob("META-INF/services"))
+ assert sorted(iter_glob(tmp_path, "META-INF/services")) == expected
+ assert (tmp_path / "b" / "other" / "services") not in set(
+ iter_glob(tmp_path, "META-INF/services")
+ )
+
+ def test_slash_pattern_with_glob_component(self, tmp_path: Path) -> None:
+ _write(tmp_path, "x/META-INF/spring/auto.imports")
+ _write(tmp_path, "x/META-INF/spring/readme.txt")
+ expected = sorted(tmp_path.rglob("META-INF/spring/*.imports"))
+ assert sorted(iter_glob(tmp_path, "META-INF/spring/*.imports")) == expected
+
+ def test_slash_pattern_matches_at_root(self, tmp_path: Path) -> None:
+ """rglob("META-INF/services") also matches directly under the root."""
+ (tmp_path / "META-INF" / "services").mkdir(parents=True)
+ expected = sorted(tmp_path.rglob("META-INF/services"))
+ assert sorted(iter_glob(tmp_path, "META-INF/services")) == expected
+
+ def test_multiple_patterns_single_walk(self, tmp_path: Path) -> None:
+ _write(tmp_path, "app/tsconfig.json")
+ _write(tmp_path, "app/jsconfig.json")
+ _write(tmp_path, "app/package.json")
+ found = set(iter_glob(tmp_path, ("tsconfig*.json", "jsconfig.json")))
+ assert found == {tmp_path / "app" / "tsconfig.json", tmp_path / "app" / "jsconfig.json"}
+
+ def test_missing_root_yields_nothing(self, tmp_path: Path) -> None:
+ assert list(iter_glob(tmp_path / "nope", "*.go")) == []
+
+
+# ---------------------------------------------------------------------------
+# Pruning — junk dirs and nested git repos
+# ---------------------------------------------------------------------------
+
+
+class TestPruning:
+ def test_junk_dirs_pruned(self, tmp_path: Path) -> None:
+ _write(tmp_path, "real/go.mod")
+ _write(tmp_path, "node_modules/dep/go.mod")
+ _write(tmp_path, ".venv/lib/go.mod")
+ _write(tmp_path, "sub/__pycache__/go.mod")
+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "real" / "go.mod"]
+
+ def test_nested_git_dir_pruned(self, tmp_path: Path) -> None:
+ """A subdirectory with its own .git directory is another repo — skip it."""
+ _write(tmp_path, "go.mod")
+ (tmp_path / "sibling-repo" / ".git").mkdir(parents=True)
+ _write(tmp_path, "sibling-repo/go.mod")
+ _write(tmp_path, "sibling-repo/deep/go.mod")
+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "go.mod"]
+
+ def test_nested_git_file_pruned(self, tmp_path: Path) -> None:
+ """Worktrees and submodules mark themselves with a .git FILE."""
+ _write(tmp_path, "main.go")
+ _write(tmp_path, "submodule/.git", "gitdir: ../.git/modules/submodule")
+ _write(tmp_path, "submodule/sub.go")
+ assert list(iter_glob(tmp_path, "*.go")) == [tmp_path / "main.go"]
+
+ def test_walk_root_with_git_not_pruned(self, tmp_path: Path) -> None:
+ """The walk root IS the repo being scanned — its own .git is exempt."""
+ (tmp_path / ".git").mkdir()
+ _write(tmp_path, "src/go.mod")
+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "src" / "go.mod"]
+
+ def test_nested_git_pruning_can_be_disabled(self, tmp_path: Path) -> None:
+ (tmp_path / "vendored" / ".git").mkdir(parents=True)
+ _write(tmp_path, "vendored/go.mod")
+ found = list(iter_glob(tmp_path, "go.mod", prune_nested_git=False))
+ assert found == [tmp_path / "vendored" / "go.mod"]
+
+ def test_custom_prune_dirs(self, tmp_path: Path) -> None:
+ """Callers may pass their own prune set (narrower or broader)."""
+ _write(tmp_path, "dist/package.json")
+ _write(tmp_path, "node_modules/x/package.json")
+ narrow = frozenset({"node_modules"})
+ found = set(iter_glob(tmp_path, "package.json", prune_dirs=narrow))
+ assert found == {tmp_path / "dist" / "package.json"}
+
+ def test_default_keeps_possible_source_dirs(self, tmp_path: Path) -> None:
+ """The DEFAULT set must not prune dirs real projects use for source —
+ coveragepy's main package is literally ``coverage/``; a module rooted
+ at ``build/go.mod`` must stay discoverable."""
+ _write(tmp_path, "coverage/__init__.py")
+ _write(tmp_path, "build/go.mod")
+ assert list(iter_glob(tmp_path, "__init__.py")) == [tmp_path / "coverage" / "__init__.py"]
+ assert list(iter_glob(tmp_path, "go.mod")) == [tmp_path / "build" / "go.mod"]
+
+ def test_derived_set_prunes_output_dirs(self, tmp_path: Path) -> None:
+ """PRUNED_DIRS_DERIVED (dynamic hints) prunes derived-output names."""
+ _write(tmp_path, "dist/settings.py")
+ _write(tmp_path, "src/settings.py")
+ found = list(iter_glob(tmp_path, "settings.py", prune_dirs=PRUNED_DIRS_DERIVED))
+ assert found == [tmp_path / "src" / "settings.py"]
+
+ def test_default_prune_set_contents(self) -> None:
+ """Canary: the dirs behind past incidents stay pruned."""
+ for d in ("node_modules", ".venv", "__pycache__", ".git", ".repowise", ".next"):
+ assert d in PRUNED_DIRS
+ for d in ("dist", "build", "out", "coverage"):
+ assert d not in PRUNED_DIRS # possible source dirs — derived set only
+ assert d in PRUNED_DIRS_DERIVED
+
+
+# ---------------------------------------------------------------------------
+# walk_repo — low-level walk
+# ---------------------------------------------------------------------------
+
+
+class TestWalkRepo:
+ def test_caller_can_bound_depth_via_dirnames(self, tmp_path: Path) -> None:
+ _write(tmp_path, "a/b/c/d/deep.txt")
+ _write(tmp_path, "a/shallow.txt")
+ seen: list[str] = []
+ for dirpath, dirnames, filenames in walk_repo(tmp_path):
+ if len(dirpath.relative_to(tmp_path).parts) >= 1:
+ dirnames[:] = []
+ seen.extend(filenames)
+ assert "shallow.txt" in seen
+ assert "deep.txt" not in seen
+
+ def test_nested_git_subtree_never_yielded(self, tmp_path: Path) -> None:
+ _write(tmp_path, "ours.txt")
+ (tmp_path / "other" / ".git").mkdir(parents=True)
+ _write(tmp_path, "other/theirs.txt")
+ yielded_dirs = [d for d, _, _ in walk_repo(tmp_path)]
+ assert tmp_path / "other" not in yielded_dirs
+
+
+# ---------------------------------------------------------------------------
+# --include-submodules plumbing — scanners must honor prune_nested_git=False
+# ---------------------------------------------------------------------------
+
+
+class TestSubmodulePlumbing:
+ def test_read_go_modules_can_include_submodules(self, tmp_path: Path) -> None:
+ from repowise.core.ingestion.resolvers.go import read_go_modules
+
+ _write(tmp_path, "go.mod", "module example.com/root\n")
+ _write(tmp_path, "vendored/.git", "gitdir: ../.git/modules/vendored")
+ _write(tmp_path, "vendored/go.mod", "module example.com/vendored\n")
+
+ default = dict(read_go_modules(tmp_path))
+ assert default == {"": "example.com/root"}
+
+ included = dict(read_go_modules(tmp_path, prune_nested_git=False))
+ assert included == {"": "example.com/root", "vendored": "example.com/vendored"}
+
+ def test_graph_builder_threads_include_submodules(self, tmp_path: Path) -> None:
+ """Either traverser flag that indexes .git-bearing subdirs must also
+ stop resolver scans from pruning them."""
+ from repowise.core.ingestion import GraphBuilder
+
+ assert GraphBuilder(tmp_path)._prune_nested_git is True
+ assert GraphBuilder(tmp_path, include_submodules=True)._prune_nested_git is False
+ assert GraphBuilder(tmp_path, include_nested_repos=True)._prune_nested_git is False
+
+ def test_resolver_context_default(self) -> None:
+ from repowise.core.ingestion.resolvers.context import ResolverContext
+
+ ctx = ResolverContext(path_set=set(), stem_map={}, graph=None)
+ assert ctx.prune_nested_git is True
+
+
+# ---------------------------------------------------------------------------
+# Traverser monorepo detection — must share _walk's boundary semantics
+# ---------------------------------------------------------------------------
+
+
+class TestMonorepoDetectionBoundaries:
+ def test_nested_repo_not_reported_as_package(self, tmp_path: Path) -> None:
+ """A sibling/vendored repo inside the tree is not a package of THIS
+ repo — and must not be rglob-scanned for language/entry points
+ (that scan ran for minutes per sibling on multi-repo directories)."""
+ from repowise.core.ingestion.traverser import FileTraverser
+
+ _write(tmp_path, "packages/app/package.json", "{}")
+ _write(tmp_path, "packages/app/index.ts", "export {}")
+ (tmp_path / "sibling" / ".git").mkdir(parents=True)
+ _write(tmp_path, "sibling/package.json", "{}")
+ _write(tmp_path, "node_modules/dep/package.json", "{}")
+
+ traverser = FileTraverser(tmp_path)
+ packages, _ = traverser._detect_monorepo()
+ paths = {p.path for p in packages}
+ assert "packages/app" in paths
+ assert "sibling" not in paths
+ assert all(not p.startswith("node_modules") for p in paths)
+
+ def test_nested_repo_reported_when_opted_in(self, tmp_path: Path) -> None:
+ from repowise.core.ingestion.traverser import FileTraverser
+
+ (tmp_path / "sibling" / ".git").mkdir(parents=True)
+ _write(tmp_path, "sibling/package.json", "{}")
+
+ traverser = FileTraverser(tmp_path, include_nested_repos=True)
+ packages, _ = traverser._detect_monorepo()
+ assert "sibling" in {p.path for p in packages}
+
+
+# ---------------------------------------------------------------------------
+# external_systems._discover — depth bound must match the historical rglob
+# ---------------------------------------------------------------------------
+
+
+class TestExternalSystemsDiscoverDepth:
+ def test_depth_bound_matches_old_rglob_semantics(self, tmp_path: Path) -> None:
+ """Old code: rglob("*") + skip files with >4 relative parts. The
+ walk_repo rewrite bounds descent by clearing dirnames at dir-depth 3;
+ files at exactly 4 relative parts must stay IN, 5 must stay OUT."""
+ from repowise.core.ingestion.external_systems import _discover
+
+ _write(tmp_path, "package.json", "{}") # 1 part
+ _write(tmp_path, "a/package.json", "{}") # 2 parts
+ _write(tmp_path, "a/b/c/package.json", "{}") # 4 parts — boundary, kept
+ _write(tmp_path, "a/b/c/d/package.json", "{}") # 5 parts — too deep
+ found = {p.relative_to(tmp_path).as_posix() for p in _discover(tmp_path)}
+ assert "package.json" in found
+ assert "a/package.json" in found
+ assert "a/b/c/package.json" in found
+ assert "a/b/c/d/package.json" not in found
+
+
+# ---------------------------------------------------------------------------
+# Regression guard — no new direct rglob calls in packages/core
+# ---------------------------------------------------------------------------
+
+# Files allowed to mention ``.rglob(``. Additions require justification:
+# either the walk root is a tightly-bounded subdirectory that cannot contain
+# junk trees or nested repos, or the file IS the shared implementation.
+_RGLOB_ALLOWLIST = {
+ # The shared implementation (docstring references rglob semantics).
+ "fs_walk.py",
+ # Decision mining: bounded to an explicit docs/ directory.
+ "analysis/decisions/extractor.py",
+}
+
+_RGLOB_RE = re.compile(r"\.rglob\(")
+
+
+def _core_src_root() -> Path:
+ # tests/unit/ingestion/ → repo root → packages/core/src/repowise/core
+ repo_root = Path(__file__).resolve().parents[3]
+ return repo_root / "packages" / "core" / "src" / "repowise" / "core"
+
+
+class TestNoUnprunedRglobRegression:
+ def test_no_new_rglob_call_sites_in_core(self) -> None:
+ core = _core_src_root()
+ assert core.is_dir(), f"layout changed? {core}"
+ offenders: list[str] = []
+ for py in core.rglob("*.py"): # tests may rglob; core may not.
+ rel = py.relative_to(core).as_posix()
+ if rel in _RGLOB_ALLOWLIST:
+ continue
+ text = py.read_text(encoding="utf-8", errors="ignore")
+ if _RGLOB_RE.search(text):
+ offenders.append(rel)
+ assert not offenders, (
+ "Direct .rglob( calls found in packages/core — use "
+ "repowise.core.fs_walk.iter_glob/walk_repo instead (prunes "
+ "node_modules/.venv AND nested git repos). Unpruned rglob has "
+ f"caused multi-minute init stalls twice. Offenders: {offenders}"
+ )