Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(ingestion): resolve only links during the pruned repo walk (#4560)
Raghav Chamadiyad39d784a8ba4
diff --git a/packages/core/src/repowise/core/fs_walk.py b/packages/core/src/repowise/core/fs_walk.pyindex d114868c0..a48d93b9a 100644--- a/packages/core/src/repowise/core/fs_walk.py+++ b/packages/core/src/repowise/core/fs_walk.py@@ -26,7 +26,9 @@ Guarantees:worktrees and submodules use a ``.git`` file) are pruned by default.Detection is free: ``.git`` appears in ``os.walk``'s own listings.- Symlinks are never followed; junction/hard-link cycles are detected via- realpath tracking (Windows ``os.walk`` can't see them via inodes).+ realpath tracking (Windows ``os.walk`` can't see them via inodes). Only a+ link is resolved; a plain directory's real path is its parent's plus its+ name, read from the listing at no extra system call.- A link (symlink or junction) whose target is **inside** the walk root ispruned, so the target's subtree is walked exactly once and under its ownname. Recording the target as visited instead let whichever of the pair@@ -39,6 +41,7 @@ from __future__ import annotationsimport fnmatchimport os+import statfrom collections.abc import Callable, Iterable, Iteratorfrom pathlib import Path@@ -128,6 +131,68 @@ def _is_within(candidate: str, root_real: str) -> bool:return cand == root or cand.startswith(root + os.sep)+def _is_reparse_point(entry: os.DirEntry[str]) -> bool:+ """True for a Windows junction or other mount point (never off Windows).++ Free: on Windows ``stat(follow_symlinks=False)`` is served from the+ listing's own attributes. A failed read answers True so ``realpath`` decides.+ """+ if os.name != "nt":+ return False+ try:+ attrs = entry.stat(follow_symlinks=False).st_file_attributes+ except OSError:+ return True+ return bool(attrs & stat.FILE_ATTRIBUTE_REPARSE_POINT)+++def _scan(top: str) -> Iterator[tuple[str, list[str], list[str], set[str]]]:+ """``os.walk(top, followlinks=False)``, top-down, plus each listing's links.++ ``os.walk`` reads every entry's link state and drops it, then ``realpath``+ had to ask the filesystem again: two ``_getfinalpathname`` calls per+ directory on Windows, about 2/3 of a pruned walk. Keeping the state lets+ :func:`walk_repo` resolve only the links. Mutating the yielded dirnames+ prunes the descent, exactly as with ``os.walk``.+ """+ stack = [top]+ while stack:+ dirpath = stack.pop()+ dirnames: list[str] = []+ filenames: list[str] = []+ links: set[str] = set()+ symlinks: set[str] = set()+ try:+ with os.scandir(dirpath) as it:+ for entry in it:+ try:+ is_dir = entry.is_dir()+ except OSError:+ is_dir = False+ if not is_dir:+ filenames.append(entry.name)+ continue+ dirnames.append(entry.name)+ try:+ is_symlink = entry.is_symlink()+ except OSError:+ is_symlink = True+ if is_symlink:+ symlinks.add(entry.name)+ links.add(entry.name)+ elif _is_reparse_point(entry):+ links.add(entry.name)+ except OSError:+ continue+ yield dirpath, dirnames, filenames, links+ # Like ``os.walk(followlinks=False)``: a symlink is listed, never+ # entered; a junction is entered, so walk_repo's realpath checks+ # are what stop it.+ stack.extend(+ os.path.join(dirpath, d) for d in reversed(dirnames) if d not in symlinks+ )++def walk_repo(root: Path | str,*,@@ -136,7 +201,7 @@ def walk_repo(max_depth: int = _MAX_WALK_DEPTH,prune: Callable[[Path], bool] | None = None,) -> Iterator[tuple[Path, list[str], list[str]]]:- """``os.walk`` over *root* with pruning. Yields ``(dirpath, dirnames, filenames)``.+ """Top-down walk over *root* with pruning. Yields ``(dirpath, dirnames, filenames)``.Use this (instead of :func:`iter_glob`) when one pass should collectseveral kinds of files — match against ``filenames`` yourself.@@ -165,12 +230,16 @@ def walk_repo(returnbase_depth = root_real.rstrip(os.sep).count(os.sep)visited_real: set[str] = {root_real}+ # Resolved path of each directory queued for descent, keyed by the path+ # the walk yields it under, so a plain child resolves by a join.+ top = os.fspath(root_path)+ real_of: dict[str, str] = {top: root_real}is_root = True- for dirpath, dirnames, filenames in os.walk(root_path, followlinks=False):+ for dirpath, dirnames, filenames, links in _scan(top):# Nested git repo? ``.git`` shows up in this dir's own listing — no# extra stat calls needed. The walk root itself is exempt (it IS the- # repo being scanned); top-down os.walk always yields it first.+ # repo being scanned); the top-down walk always yields it first.if is_root:is_root = Falseelif prune_nested_git and (".git" in dirnames or ".git" in filenames):@@ -196,20 +265,27 @@ def walk_repo(# tree. That is not exotic — macOS ``/tmp`` is a symlink to# ``/private/tmp``, container bind mounts and mapped drives do the same,# and a repo checked out under one would have indexed as empty.- # Resolving the parent once per directory keeps both sides of the- # comparison resolved to the same depth, so only a genuine reparse point- # differs.- try:- parent_real = os.path.realpath(dirpath)- except OSError:- parent_real = str(dirpath)- for d in dirnames:- child = os.path.join(dirpath, d)+ # Carrying each directory's resolved path down from its parent keeps+ # both sides of the comparison resolved to the same depth, so only a+ # genuine reparse point differs.+ parent_real = real_of.pop(dirpath, None)+ if parent_real is None:try:- child_real = os.path.realpath(child)+ parent_real = os.path.realpath(dirpath)except OSError:- cycle_pruned.append(d)- continue+ parent_real = str(dirpath)+ for d in dirnames:+ child = os.path.join(dirpath, d)+ # Only a link can resolve anywhere but ``parent_real/d``, so only a+ # link pays for ``realpath``.+ if d in links:+ try:+ child_real = os.path.realpath(child)+ except OSError:+ cycle_pruned.append(d)+ continue+ else:+ child_real = os.path.join(parent_real, d)# A reparse point — symlink, junction, bind mount — resolving# somewhere else. Its target is NOT territory this entry owns, and# recording it as visited is what let a link *delete* its target's@@ -236,6 +312,7 @@ def walk_repo(cycle_pruned.append(d)continuevisited_real.add(child_real)+ real_of[child] = child_realif cycle_pruned:dirnames[:] = [d for d in dirnames if d not in cycle_pruned]diff --git a/tests/unit/ingestion/test_fs_walk.py b/tests/unit/ingestion/test_fs_walk.pyindex a01934c5b..c214cf83d 100644--- a/tests/unit/ingestion/test_fs_walk.py+++ b/tests/unit/ingestion/test_fs_walk.py@@ -424,6 +424,70 @@ class TestLinkCyclesStillTerminate:# what moves. Four files: the three seeded plus the external one, once.assert files == sorted([*_EXPECTED_FILES, "vendored/pkg/ext.py"]), files[:8]+ @pytest.mark.parametrize("link", [_symlink, _junction], ids=["symlink", "junction"])+ def test_siblings_linking_to_each_other_terminate(self, tmp_path: Path, link) -> None:+ _seed_tree(tmp_path)+ _write(tmp_path, "other/o.py")+ link(tmp_path / "svc" / "to_other", tmp_path / "other")+ link(tmp_path / "other" / "to_svc", tmp_path / "svc")++ dirs, files = _walked(tmp_path)+ assert dirs == sorted([*_EXPECTED_DIRS, "other"])+ assert files == sorted([*_EXPECTED_FILES, "other/o.py"])++ def test_an_out_of_tree_sibling_loop_lists_each_file_once(self, tmp_path: Path) -> None:+ """Two junctions into one external tree, the first at a subdirectory+ of the second's target. A plain directory under the second resolves by+ joining onto its parent, so it must still meet the visited set."""+ outside = tmp_path.parent / f"{tmp_path.name}_ext_siblings"+ _write(outside, "a/a.py")+ _write(outside, "b/b.py")+ _junction(outside / "a" / "to_b", outside / "b")+ _junction(outside / "b" / "to_a", outside / "a")+ root = tmp_path / "repo"+ root.mkdir()+ _seed_tree(root)+ _junction(root / "a_first", outside / "a")+ _junction(root / "b_all", outside)++ _dirs, files = _walked(root)+ ext = [f for f in files if f.endswith(("a.py", "b.py"))]+ assert sorted(Path(f).name for f in ext) == ["a.py", "b.py"], ext+++class TestRealpathOnlyForLinks:+ """``realpath`` costs two ``_getfinalpathname`` calls on Windows and was+ paid for every directory; a plain directory resolves by a join."""++ def test_an_unlinked_tree_resolves_only_the_root(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _seed_tree(tmp_path)+ _write(tmp_path, "deep/a/b/c/d.py")+ calls: list[str] = []+ real = os.path.realpath+ monkeypatch.setattr(+ os.path, "realpath", lambda p, *a, **k: calls.append(p) or real(p, *a, **k)+ )++ dirs, _files = _walked(tmp_path)+ assert len(dirs) == 8+ assert len(calls) == 1++ def test_a_link_is_still_resolved(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _seed_tree(tmp_path)+ _symlink(tmp_path / "linked", tmp_path / "svc")+ calls: list[str] = []+ real = os.path.realpath+ monkeypatch.setattr(+ os.path, "realpath", lambda p, *a, **k: calls.append(p) or real(p, *a, **k)+ )++ assert _walked(tmp_path) == (_EXPECTED_DIRS, _EXPECTED_FILES)+ assert len(calls) == 2+# ---------------------------------------------------------------------------# --include-submodules plumbing — scanners must honor prune_nested_git=False