Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(ingestion): resolve only links during the pruned repo walk (#4560)

Raghav Chamadiyad39d784a8ba4
diff --git a/packages/core/src/repowise/core/fs_walk.py b/packages/core/src/repowise/core/fs_walk.py
index d114868c0..a48d93b9a 100644
--- a/packages/core/src/repowise/core/fs_walk.py
+++ b/packages/core/src/repowise/core/fs_walk.py
@@ -26,7 +26,9 @@ Guarantees:
worktrees and submodules use a ``.git`` file) are pruned by default.
Detection is free: ``.git`` appears in ``os.walk``'s own listings.
- Symlinks are never followed; junction/hard-link cycles are detected via
- realpath tracking (Windows ``os.walk`` can't see them via inodes).
+ realpath tracking (Windows ``os.walk`` can't see them via inodes). Only a
+ link is resolved; a plain directory's real path is its parent's plus its
+ name, read from the listing at no extra system call.
- A link (symlink or junction) whose target is **inside** the walk root is
pruned, so the target's subtree is walked exactly once and under its own
name. Recording the target as visited instead let whichever of the pair
@@ -39,6 +41,7 @@ from __future__ import annotations
import fnmatch
import os
+import stat
from collections.abc import Callable, Iterable, Iterator
from pathlib import Path
@@ -128,6 +131,68 @@ def _is_within(candidate: str, root_real: str) -> bool:
return cand == root or cand.startswith(root + os.sep)
+def _is_reparse_point(entry: os.DirEntry[str]) -> bool:
+ """True for a Windows junction or other mount point (never off Windows).
+
+ Free: on Windows ``stat(follow_symlinks=False)`` is served from the
+ listing's own attributes. A failed read answers True so ``realpath`` decides.
+ """
+ if os.name != "nt":
+ return False
+ try:
+ attrs = entry.stat(follow_symlinks=False).st_file_attributes
+ except OSError:
+ return True
+ return bool(attrs & stat.FILE_ATTRIBUTE_REPARSE_POINT)
+
+
+def _scan(top: str) -> Iterator[tuple[str, list[str], list[str], set[str]]]:
+ """``os.walk(top, followlinks=False)``, top-down, plus each listing's links.
+
+ ``os.walk`` reads every entry's link state and drops it, then ``realpath``
+ had to ask the filesystem again: two ``_getfinalpathname`` calls per
+ directory on Windows, about 2/3 of a pruned walk. Keeping the state lets
+ :func:`walk_repo` resolve only the links. Mutating the yielded dirnames
+ prunes the descent, exactly as with ``os.walk``.
+ """
+ stack = [top]
+ while stack:
+ dirpath = stack.pop()
+ dirnames: list[str] = []
+ filenames: list[str] = []
+ links: set[str] = set()
+ symlinks: set[str] = set()
+ try:
+ with os.scandir(dirpath) as it:
+ for entry in it:
+ try:
+ is_dir = entry.is_dir()
+ except OSError:
+ is_dir = False
+ if not is_dir:
+ filenames.append(entry.name)
+ continue
+ dirnames.append(entry.name)
+ try:
+ is_symlink = entry.is_symlink()
+ except OSError:
+ is_symlink = True
+ if is_symlink:
+ symlinks.add(entry.name)
+ links.add(entry.name)
+ elif _is_reparse_point(entry):
+ links.add(entry.name)
+ except OSError:
+ continue
+ yield dirpath, dirnames, filenames, links
+ # Like ``os.walk(followlinks=False)``: a symlink is listed, never
+ # entered; a junction is entered, so walk_repo's realpath checks
+ # are what stop it.
+ stack.extend(
+ os.path.join(dirpath, d) for d in reversed(dirnames) if d not in symlinks
+ )
+
+
def walk_repo(
root: Path | str,
*,
@@ -136,7 +201,7 @@ def walk_repo(
max_depth: int = _MAX_WALK_DEPTH,
prune: Callable[[Path], bool] | None = None,
) -> Iterator[tuple[Path, list[str], list[str]]]:
- """``os.walk`` over *root* with pruning. Yields ``(dirpath, dirnames, filenames)``.
+ """Top-down walk over *root* with pruning. Yields ``(dirpath, dirnames, filenames)``.
Use this (instead of :func:`iter_glob`) when one pass should collect
several kinds of files — match against ``filenames`` yourself.
@@ -165,12 +230,16 @@ def walk_repo(
return
base_depth = root_real.rstrip(os.sep).count(os.sep)
visited_real: set[str] = {root_real}
+ # Resolved path of each directory queued for descent, keyed by the path
+ # the walk yields it under, so a plain child resolves by a join.
+ top = os.fspath(root_path)
+ real_of: dict[str, str] = {top: root_real}
is_root = True
- for dirpath, dirnames, filenames in os.walk(root_path, followlinks=False):
+ for dirpath, dirnames, filenames, links in _scan(top):
# Nested git repo? ``.git`` shows up in this dir's own listing — no
# extra stat calls needed. The walk root itself is exempt (it IS the
- # repo being scanned); top-down os.walk always yields it first.
+ # repo being scanned); the top-down walk always yields it first.
if is_root:
is_root = False
elif prune_nested_git and (".git" in dirnames or ".git" in filenames):
@@ -196,20 +265,27 @@ def walk_repo(
# tree. That is not exotic — macOS ``/tmp`` is a symlink to
# ``/private/tmp``, container bind mounts and mapped drives do the same,
# and a repo checked out under one would have indexed as empty.
- # Resolving the parent once per directory keeps both sides of the
- # comparison resolved to the same depth, so only a genuine reparse point
- # differs.
- try:
- parent_real = os.path.realpath(dirpath)
- except OSError:
- parent_real = str(dirpath)
- for d in dirnames:
- child = os.path.join(dirpath, d)
+ # Carrying each directory's resolved path down from its parent keeps
+ # both sides of the comparison resolved to the same depth, so only a
+ # genuine reparse point differs.
+ parent_real = real_of.pop(dirpath, None)
+ if parent_real is None:
try:
- child_real = os.path.realpath(child)
+ parent_real = os.path.realpath(dirpath)
except OSError:
- cycle_pruned.append(d)
- continue
+ parent_real = str(dirpath)
+ for d in dirnames:
+ child = os.path.join(dirpath, d)
+ # Only a link can resolve anywhere but ``parent_real/d``, so only a
+ # link pays for ``realpath``.
+ if d in links:
+ try:
+ child_real = os.path.realpath(child)
+ except OSError:
+ cycle_pruned.append(d)
+ continue
+ else:
+ child_real = os.path.join(parent_real, d)
# A reparse point — symlink, junction, bind mount — resolving
# somewhere else. Its target is NOT territory this entry owns, and
# recording it as visited is what let a link *delete* its target's
@@ -236,6 +312,7 @@ def walk_repo(
cycle_pruned.append(d)
continue
visited_real.add(child_real)
+ real_of[child] = child_real
if cycle_pruned:
dirnames[:] = [d for d in dirnames if d not in cycle_pruned]
diff --git a/tests/unit/ingestion/test_fs_walk.py b/tests/unit/ingestion/test_fs_walk.py
index a01934c5b..c214cf83d 100644
--- a/tests/unit/ingestion/test_fs_walk.py
+++ b/tests/unit/ingestion/test_fs_walk.py
@@ -424,6 +424,70 @@ class TestLinkCyclesStillTerminate:
# what moves. Four files: the three seeded plus the external one, once.
assert files == sorted([*_EXPECTED_FILES, "vendored/pkg/ext.py"]), files[:8]
+ @pytest.mark.parametrize("link", [_symlink, _junction], ids=["symlink", "junction"])
+ def test_siblings_linking_to_each_other_terminate(self, tmp_path: Path, link) -> None:
+ _seed_tree(tmp_path)
+ _write(tmp_path, "other/o.py")
+ link(tmp_path / "svc" / "to_other", tmp_path / "other")
+ link(tmp_path / "other" / "to_svc", tmp_path / "svc")
+
+ dirs, files = _walked(tmp_path)
+ assert dirs == sorted([*_EXPECTED_DIRS, "other"])
+ assert files == sorted([*_EXPECTED_FILES, "other/o.py"])
+
+ def test_an_out_of_tree_sibling_loop_lists_each_file_once(self, tmp_path: Path) -> None:
+ """Two junctions into one external tree, the first at a subdirectory
+ of the second's target. A plain directory under the second resolves by
+ joining onto its parent, so it must still meet the visited set."""
+ outside = tmp_path.parent / f"{tmp_path.name}_ext_siblings"
+ _write(outside, "a/a.py")
+ _write(outside, "b/b.py")
+ _junction(outside / "a" / "to_b", outside / "b")
+ _junction(outside / "b" / "to_a", outside / "a")
+ root = tmp_path / "repo"
+ root.mkdir()
+ _seed_tree(root)
+ _junction(root / "a_first", outside / "a")
+ _junction(root / "b_all", outside)
+
+ _dirs, files = _walked(root)
+ ext = [f for f in files if f.endswith(("a.py", "b.py"))]
+ assert sorted(Path(f).name for f in ext) == ["a.py", "b.py"], ext
+
+
+class TestRealpathOnlyForLinks:
+ """``realpath`` costs two ``_getfinalpathname`` calls on Windows and was
+ paid for every directory; a plain directory resolves by a join."""
+
+ def test_an_unlinked_tree_resolves_only_the_root(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _seed_tree(tmp_path)
+ _write(tmp_path, "deep/a/b/c/d.py")
+ calls: list[str] = []
+ real = os.path.realpath
+ monkeypatch.setattr(
+ os.path, "realpath", lambda p, *a, **k: calls.append(p) or real(p, *a, **k)
+ )
+
+ dirs, _files = _walked(tmp_path)
+ assert len(dirs) == 8
+ assert len(calls) == 1
+
+ def test_a_link_is_still_resolved(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _seed_tree(tmp_path)
+ _symlink(tmp_path / "linked", tmp_path / "svc")
+ calls: list[str] = []
+ real = os.path.realpath
+ monkeypatch.setattr(
+ os.path, "realpath", lambda p, *a, **k: calls.append(p) or real(p, *a, **k)
+ )
+
+ assert _walked(tmp_path) == (_EXPECTED_DIRS, _EXPECTED_FILES)
+ assert len(calls) == 2
+
# ---------------------------------------------------------------------------
# --include-submodules plumbing — scanners must honor prune_nested_git=False