Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix: use output-directory detection across source discovery walks (#3454)
Yunus Ozena9be4ce9c6ef
diff --git a/packages/core/src/repowise/core/analysis/decisions/adr.py b/packages/core/src/repowise/core/analysis/decisions/adr.pyindex 7a83f9481..44b1f76d4 100644--- a/packages/core/src/repowise/core/analysis/decisions/adr.py+++ b/packages/core/src/repowise/core/analysis/decisions/adr.py@@ -6,7 +6,7 @@ import refrom pathlib import Pathfrom repowise.core.fs_walk import walk_repo-from repowise.core.ingestion.traverser import load_gitignore_spec+from repowise.core.ingestion.traverser import is_output_dir, load_gitignore_specfrom .source_files import _SKIP_DIRS@@ -85,7 +85,9 @@ def find_adr_files(repo_path: Path) -> list[Path]:ignore = load_gitignore_spec(repo_path)conventional: list[Path] = []loose: list[Path] = []- for dirpath, dirnames, filenames in walk_repo(repo_path, prune_dirs=_SKIP_DIRS):+ for dirpath, dirnames, filenames in walk_repo(+ repo_path, prune_dirs=_SKIP_DIRS, prune=is_output_dir+ ):rel_dir = dirpath.relative_to(repo_path).as_posix()rel_dir = "" if rel_dir == "." else rel_dir# Prune ignored and packaging-metadata subtrees in place. Ignoreddiff --git a/packages/core/src/repowise/core/analysis/decisions/source_files.py b/packages/core/src/repowise/core/analysis/decisions/source_files.pyindex c12b8012c..1292b5344 100644--- a/packages/core/src/repowise/core/analysis/decisions/source_files.py+++ b/packages/core/src/repowise/core/analysis/decisions/source_files.py@@ -7,11 +7,13 @@ from collections.abc import Iteratorfrom pathlib import Pathfrom repowise.core.fs_walk import PRUNED_DIRS, walk_repo+from repowise.core.ingestion.traverser import is_output_dir-# fs_walk's never-source set plus the two derived-output names the decision-# walks have always skipped: a bundled ``dist``/``build`` copy of a source file-# would double every marker it contains.-_SKIP_DIRS = PRUNED_DIRS | {"dist", "build"}+# fs_walk's never-source set. The decision walks also pass ``is_output_dir``+# as ``prune``: a bundled ``dist``/``build`` copy of a source file would double+# every marker it contains, while a source package with one of those names is+# indexed and must be read.+_SKIP_DIRS = PRUNED_DIRS_BINARY_EXTENSIONS = frozenset({@@ -102,7 +104,9 @@ def iter_source_files(repo_path: Path) -> Iterator[Path]:"""tracked = tracked_files(repo_path)- for dirpath, dirnames, filenames in walk_repo(repo_path, prune_dirs=_SKIP_DIRS):+ for dirpath, dirnames, filenames in walk_repo(+ repo_path, prune_dirs=_SKIP_DIRS, prune=is_output_dir+ ):# Skip setuptools build metadata: PKG-INFO embeds the README# verbatim, so example marker lines in docs become spurious# decisions. Same risk for *.dist-info from wheels.diff --git a/packages/core/src/repowise/core/fs_walk.py b/packages/core/src/repowise/core/fs_walk.pyindex 33ee37d89..d114868c0 100644--- a/packages/core/src/repowise/core/fs_walk.py+++ b/packages/core/src/repowise/core/fs_walk.py@@ -39,7 +39,7 @@ from __future__ import annotationsimport fnmatchimport os-from collections.abc import Iterable, Iterator+from collections.abc import Callable, Iterable, Iteratorfrom pathlib import Pathimport structlog@@ -96,12 +96,14 @@ PRUNED_DIRS: frozenset[str] = frozenset(})-# PRUNED_DIRS plus common *derived-output* names that occasionally double as-# real source dirs. Use for scans where derived trees would only add noise+# PRUNED_DIRS plus derived-output names that occasionally double as real+# source dirs. Use for scans where derived trees would only add noise# (dynamic hints' settings/router extraction); do NOT use for manifest-# discovery, where a module legitimately rooted at ``build/`` or-# ``coverage/`` must still be found.-PRUNED_DIRS_DERIVED: frozenset[str] = PRUNED_DIRS | frozenset({"dist", "build", "out", "coverage"})+# discovery. ``build``, ``coverage``, ``dist`` and ``target`` are not listed:+# the traverser's ``is_output_dir`` decides them per directory, so pass it as+# ``prune`` to skip only real output and still read a source package that has+# one of those names, as the indexer does.+PRUNED_DIRS_DERIVED: frozenset[str] = PRUNED_DIRS | frozenset({"out"})# Safety net: bail if walk depth grows pathologically. Real-world repos# rarely exceed depth 20; anything beyond strongly suggests a cycle.@@ -132,6 +134,7 @@ def walk_repo(prune_dirs: frozenset[str] = PRUNED_DIRS,prune_nested_git: bool = True,max_depth: int = _MAX_WALK_DEPTH,+ prune: Callable[[Path], bool] | None = None,) -> Iterator[tuple[Path, list[str], list[str]]]:"""``os.walk`` over *root* with pruning. Yields ``(dirpath, dirnames, filenames)``.@@ -144,6 +147,10 @@ def walk_repo(prune_nested_git: Skip any non-root directory that contains its own``.git`` entry (dir or file) — i.e. another repo.max_depth: Hard depth cap (cycle safety net).+ prune: Optional per-directory check, called with each+ subdirectory's path after *prune_dirs*; a ``True``+ answer skips it. For decisions a name cannot make+ (``is_output_dir`` for ``build``/``dist``/...).``dirnames`` is yielded *after* pruning; callers may prune further bymutating it in place, exactly like ``os.walk``.@@ -172,6 +179,9 @@ def walk_repo(# Skip junk directories at every level.dirnames[:] = [d for d in dirnames if d not in prune_dirs]+ if prune is not None:+ parent = Path(dirpath)+ dirnames[:] = [d for d in dirnames if not prune(parent / d)]# Drop any dir whose realpath we've already entered — catches# Windows junctions and other cycles ``os.walk`` can't detect via@@ -273,7 +283,7 @@ class WalkSnapshot:root outside the snapshot tree falls back to a live walk."""- __slots__ = ("_entries", "_prune_dirs", "_prune_nested_git", "root")+ __slots__ = ("_entries", "_prune", "_prune_dirs", "_prune_nested_git", "root")def __init__(self,@@ -281,14 +291,16 @@ class WalkSnapshot:*,prune_dirs: frozenset[str] = PRUNED_DIRS,prune_nested_git: bool = True,+ prune: Callable[[Path], bool] | None = None,) -> None:self.root = Path(root)self._prune_dirs = prune_dirsself._prune_nested_git = prune_nested_git+ self._prune = prune# (dirpath, root-relative posix dir ('' for the root), dirnames, filenames)self._entries: list[tuple[Path, str, list[str], list[str]]] = []for dirpath, dirnames, filenames in walk_repo(- self.root, prune_dirs=prune_dirs, prune_nested_git=prune_nested_git+ self.root, prune_dirs=prune_dirs, prune_nested_git=prune_nested_git, prune=prune):rel = dirpath.relative_to(self.root).as_posix()self._entries.append(@@ -312,6 +324,7 @@ class WalkSnapshot:snap.root = Path(root)snap._prune_dirs = PRUNED_DIRSsnap._prune_nested_git = True+ snap._prune = Nonefiles_by_dir: dict[str, set[str]] = {"": set()}subdirs: dict[str, set[str]] = {"": set()}for rel in rel_paths:@@ -356,6 +369,7 @@ class WalkSnapshot:pats,prune_dirs=self._prune_dirs,prune_nested_git=self._prune_nested_git,+ prune=self._prune,)return@@ -402,6 +416,7 @@ def iter_glob(*,prune_dirs: frozenset[str] = PRUNED_DIRS,prune_nested_git: bool = True,+ prune: Callable[[Path], bool] | None = None,) -> Iterator[Path]:"""Pruned drop-in for ``root.rglob(pattern)``.@@ -416,7 +431,7 @@ def iter_glob(pats = (patterns,) if isinstance(patterns, str) else tuple(patterns)for dirpath, dirnames, filenames in walk_repo(- root_path, prune_dirs=prune_dirs, prune_nested_git=prune_nested_git+ root_path, prune_dirs=prune_dirs, prune_nested_git=prune_nested_git, prune=prune):rel_dir = dirpath.relative_to(root_path).as_posix()prefix = "" if rel_dir == "." else rel_dir + "/"diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 9fbff872f..436a496c3 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -31,6 +31,7 @@ from typing import Anyfrom repowise.core import fs_walkfrom repowise.core.exclusion import build_exclude_spec, is_excluded+from repowise.core.ingestion.traverser import is_output_dirfrom .grouping import ConceptGroup@@ -901,7 +902,9 @@ def _scan_tree(repo_root: Path) -> tuple[list[tuple[str, str]], frozenset[str]]:truncated = Falsespec = _exclude_spec(repo_root)- for dirpath, dirnames, filenames in fs_walk.walk_repo(repo_root, prune_dirs=_EXTRA_PRUNED_DIRS):+ for dirpath, dirnames, filenames in fs_walk.walk_repo(+ repo_root, prune_dirs=_EXTRA_PRUNED_DIRS, prune=is_output_dir+ ):# Hidden directories are tool and agent territory — CI definitions,# editor state, hook configuration — never the subsystem prose that# answers "was this term built". The walker already prunes the nameddiff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.pyindex b0e25b2ab..5c1c99a90 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/_walk.py@@ -10,9 +10,10 @@ Behavior note vs the historical private copy: the shared walk also prunesis what dynamic hints want too — settings/router files from a vendored orsibling checkout are not this repo's hints.-Dynamic hints keep the *derived-output* prune set (``dist``/``build``/-``out``/``coverage``), matching the historical list here: hint extraction-reads source files, and derived copies of them only add noise.+Dynamic hints keep the *derived-output* prune set and skip build and report+output through the traverser's ``is_output_dir``: hint extraction reads+source files, and derived copies of them only add noise, while a source+package named ``build`` or ``coverage`` is indexed and must be read."""from __future__ import annotations@@ -22,10 +23,11 @@ from pathlib import Pathfrom repowise.core.fs_walk import PRUNED_DIRS_DERIVED as PRUNED_DIRSfrom repowise.core.fs_walk import iter_glob as _shared_iter_glob+from repowise.core.ingestion.traverser import is_output_dir-__all__ = ["PRUNED_DIRS", "iter_glob"]+__all__ = ["PRUNED_DIRS", "is_output_dir", "iter_glob"]def iter_glob(root: Path, pattern: str) -> Iterator[Path]:"""Historical dynamic-hints walk: shared iter_glob + derived-dirs pruning."""- return _shared_iter_glob(root, pattern, prune_dirs=PRUNED_DIRS)+ return _shared_iter_glob(root, pattern, prune_dirs=PRUNED_DIRS, prune=is_output_dir)diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.pyindex d59c81393..c1fd8057e 100644--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py@@ -123,9 +123,9 @@ class HintRegistry:if file_paths is not None:snapshot = WalkSnapshot.from_files(repo_root, file_paths)else:- from ._walk import PRUNED_DIRS+ from ._walk import PRUNED_DIRS, is_output_dir- snapshot = WalkSnapshot(repo_root, prune_dirs=PRUNED_DIRS)+ snapshot = WalkSnapshot(repo_root, prune_dirs=PRUNED_DIRS, prune=is_output_dir)except Exception as e: # pragma: no cover - snapshot is best-effortlog.warning("dynamic_hints_snapshot_failed", error=str(e))for ex in self._extractors:diff --git a/packages/core/src/repowise/core/workspace/extractors/service_boundary.py b/packages/core/src/repowise/core/workspace/extractors/service_boundary.pyindex 38462e64a..ba0f60cc9 100644--- a/packages/core/src/repowise/core/workspace/extractors/service_boundary.py+++ b/packages/core/src/repowise/core/workspace/extractors/service_boundary.py@@ -21,6 +21,7 @@ from pathlib import Pathfrom repowise.core.fs_walk import walk_repofrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY+from repowise.core.ingestion.traverser import is_output_dirfrom repowise.core.workspace.manifests import SERVICE_MARKERS# ---------------------------------------------------------------------------@@ -34,9 +35,6 @@ _BLOCKED_DIRS = frozenset("__pycache__",".venv","venv",- "dist",- "build",- "target","vendor",".next",".nuxt",@@ -110,10 +108,12 @@ def detect_service_boundaries(repo_path: Path) -> list[ServiceBoundary]:repo_root = repo_path.resolve()boundaries: list[ServiceBoundary] = []- for dirpath, dirnames, filenames in walk_repo(repo_root):- # ``walk_repo`` already prunes junk dirs and nested git repos; layer on- # the derived-output names (dist/build/target/...) it intentionally- # leaves ambiguous, plus dotfile dirs, that are never service roots.+ for dirpath, dirnames, filenames in walk_repo(repo_root, prune=is_output_dir):+ # ``walk_repo`` already prunes junk dirs and nested git repos, and+ # ``is_output_dir`` prunes build and report output (``dist``, ``build``,+ # ``target``, ...) while keeping a source package with one of those+ # names. Layer on the other derived names it leaves ambiguous, plus+ # dotfile dirs, that are never service roots.dirnames[:] = [d for d in dirnames if d not in _BLOCKED_DIRS and not d.startswith(".")]current = Path(dirpath)diff --git a/tests/unit/analysis/test_decision_source_map_reuse.py b/tests/unit/analysis/test_decision_source_map_reuse.pyindex 3af3a7332..81c935fe3 100644--- a/tests/unit/analysis/test_decision_source_map_reuse.py+++ b/tests/unit/analysis/test_decision_source_map_reuse.py@@ -12,6 +12,7 @@ from __future__ import annotationsfrom pathlib import Pathfrom repowise.core.analysis.decision_extractor import DecisionExtractor+from repowise.core.analysis.decisions.source_files import iter_source_files# Fixtures use ``\n``-escaped single-line string concatenation on purpose: no# PHYSICAL line in this committed file begins with a comment marker, so a real@@ -115,3 +116,14 @@ async def test_restrict_skips_missing_files(tmp_path):restrict_to_files=["src/app.py", "src/gone.py"])assert {d.evidence_file for d in decisions} == {"src/app.py"}+++def test_fallback_walk_reads_a_source_package_named_build(tmp_path: Path) -> None:+ """The walk prunes a ``dist`` bundle but not a source package named ``build`` (#3237)."""+ for rel in ("src/build/render.ts", "dist/index.js", "src/app.py"):+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / rel).write_text("// DECISION: x\n", encoding="utf-8")++ found = {p.relative_to(tmp_path).as_posix() for p in iter_source_files(tmp_path)}++ assert found == {"src/build/render.ts", "src/app.py"}diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pyindex 731a27548..4dcc261c9 100644--- a/tests/unit/generation/test_vocabulary_house_terms.py+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -1248,3 +1248,21 @@ def test_heading_definition_after_fenced_block_is_extracted_correctly("Blast radius is the set of files a change can reach through the import graph.")++def test_scan_tree_reads_a_source_package_named_coverage(tmp_path: Path) -> None:+ """An output-shaped name prunes only output: a package named ``coverage``+ is read, a ``dist`` bundle is not (#3237)."""+ (tmp_path / "pkg" / "coverage").mkdir(parents=True)+ (tmp_path / "pkg" / "coverage" / "__init__.py").write_text(+ '"""Measures which lines the suite executes."""\n', encoding="utf-8"+ )+ (tmp_path / "web" / "dist").mkdir(parents=True)+ (tmp_path / "web" / "dist" / "bundle.js").write_text(+ "/** Bundled application entry. */\nexport default 1;\n", encoding="utf-8"+ )++ prose, dir_names = _scan_tree(tmp_path)++ assert [path for path, _ in prose] == ["pkg/coverage/__init__.py"]+ assert "coverage" in dir_names+ assert "dist" not in dir_namesdiff --git a/tests/unit/ingestion/test_dynamic_hints_file_feed.py b/tests/unit/ingestion/test_dynamic_hints_file_feed.pyindex eb5528482..8f584fe46 100644--- a/tests/unit/ingestion/test_dynamic_hints_file_feed.py+++ b/tests/unit/ingestion/test_dynamic_hints_file_feed.py@@ -21,6 +21,7 @@ import pytestfrom repowise.core.fs_walk import PRUNED_DIRS_DERIVED, WalkSnapshotfrom repowise.core.ingestion.dynamic_hints import HintRegistry+from repowise.core.ingestion.dynamic_hints.base import DynamicHintExtractordef _write(root: Path, rel: str, text: str) -> None:@@ -100,3 +101,40 @@ class TestFileFeedEquivalence:if "Bar.Designer" not in e.target + e.sourceand "local_settings" not in e.target + e.source]+++class _SettingsProbe(DynamicHintExtractor):+ """Records which ``settings.py`` files the hint walk reaches."""++ name = "settings_probe"++ def __init__(self) -> None:+ self.found: list[str] = []++ def extract(self, repo_root: Path) -> list:+ self.found = sorted(+ p.relative_to(repo_root).as_posix() for p in self._rglob(repo_root, "settings.py")+ )+ return []+++class TestOutputNamedDirectories:+ """A source package named ``build`` is walked; a ``dist`` bundle is not (#3237)."""++ @pytest.fixture+ def tree(self, tmp_path: Path) -> Path:+ _write(tmp_path, "src/build/__init__.py", "")+ _write(tmp_path, "src/build/settings.py", 'INSTALLED_APPS = ["myapp"]\n')+ _write(tmp_path, "dist/index.js", "module.exports = {}\n")+ _write(tmp_path, "dist/lib/settings.py", 'INSTALLED_APPS = ["myapp"]\n')+ return tmp_path++ def test_registry_snapshot(self, tree: Path) -> None:+ probe = _SettingsProbe()+ HintRegistry([probe]).extract_all(tree)+ assert probe.found == ["src/build/settings.py"]++ def test_live_walk(self, tree: Path) -> None:+ probe = _SettingsProbe()+ probe.extract(tree)+ assert probe.found == ["src/build/settings.py"]diff --git a/tests/unit/ingestion/test_fs_walk.py b/tests/unit/ingestion/test_fs_walk.pyindex 2405b5336..d1efe3fb5 100644--- a/tests/unit/ingestion/test_fs_walk.py+++ b/tests/unit/ingestion/test_fs_walk.py@@ -143,22 +143,34 @@ class TestPruning:assert list(iter_glob(tmp_path, "app.py")) == [tmp_path / "Library" / "app.py"]assert list(iter_glob(tmp_path, "worker.py")) == [tmp_path / "Logs" / "worker.py"]- def test_derived_set_prunes_output_dirs(self, tmp_path: Path) -> None:- """PRUNED_DIRS_DERIVED (dynamic hints) prunes derived-output names."""- _write(tmp_path, "dist/settings.py")+ def test_derived_set_prunes_out(self, tmp_path: Path) -> None:+ """PRUNED_DIRS_DERIVED (dynamic hints) prunes ``out`` by name."""+ _write(tmp_path, "out/settings.py")_write(tmp_path, "src/settings.py")found = list(iter_glob(tmp_path, "settings.py", prune_dirs=PRUNED_DIRS_DERIVED))assert found == [tmp_path / "src" / "settings.py"]+ def test_prune_predicate_decides_per_directory(self, tmp_path: Path) -> None:+ """``prune`` sees each directory's path, so two dirs with one name can differ."""+ _write(tmp_path, "a/gen/settings.py")+ _write(tmp_path, "b/gen/settings.py")+ found = list(+ iter_glob(tmp_path, "settings.py", prune=lambda d: d == tmp_path / "a" / "gen")+ )+ assert found == [tmp_path / "b" / "gen" / "settings.py"]+def test_default_prune_set_contents(self) -> None:"""Canary: the dirs behind past incidents stay pruned."""for d in ("node_modules", ".venv", "__pycache__", ".git", ".repowise", ".next"):assert d in PRUNED_DIRSfor d in ("Library", "Temp", "Logs", "UserSettings", "MemoryCaptures"):assert d not in PRUNED_DIRS- for d in ("dist", "build", "out", "coverage"):+ for d in ("dist", "build", "out", "coverage", "target"):assert d not in PRUNED_DIRS # possible source dirs — derived set only- assert d in PRUNED_DIRS_DERIVED+ # The traverser's is_output_dir decides these per directory.+ for d in ("dist", "build", "coverage", "target"):+ assert d not in PRUNED_DIRS_DERIVED+ assert "out" in PRUNED_DIRS_DERIVED# ---------------------------------------------------------------------------diff --git a/tests/unit/ingestion/test_walk_snapshot.py b/tests/unit/ingestion/test_walk_snapshot.pyindex b175b3abb..b02c1bb1f 100644--- a/tests/unit/ingestion/test_walk_snapshot.py+++ b/tests/unit/ingestion/test_walk_snapshot.py@@ -31,7 +31,7 @@ def tree(tmp_path: Path) -> Path:"tests/test_a.py","tests/sub/test_b.py","node_modules/dep/index.js", # pruned dir- "build/gen.go", # pruned (derived set)+ "out/gen.go", # pruned (derived set)"vendored/.git/HEAD", # nested git repo"vendored/inner.go","META-INF/services/com.example.Impl",@@ -80,7 +80,7 @@ class TestWalkSnapshotEquivalence:snap = WalkSnapshot(tree, prune_dirs=PRUNED_DIRS_DERIVED)hits = {p.name for p in snap.iter_glob(tree, ("*.go", "*.js"))}assert "index.js" not in hits # node_modules pruned- assert "gen.go" not in hits # build pruned (derived set)+ assert "gen.go" not in hits # out pruned (derived set)assert "inner.go" not in hits # nested git repo pruneddef test_out_of_tree_root_falls_back_to_live_walk(self, tree: Path, tmp_path_factory) -> None:diff --git a/tests/unit/workspace/test_extractor_traversal.py b/tests/unit/workspace/test_extractor_traversal.pyindex 9002079fa..e99828444 100644--- a/tests/unit/workspace/test_extractor_traversal.py+++ b/tests/unit/workspace/test_extractor_traversal.py@@ -94,3 +94,21 @@ def test_service_boundary_skips_nested_git_repo(tmp_path: Path) -> None:names = {b.service_name for b in boundaries}assert "sibling-service" not in names+++def test_service_boundary_under_a_source_package_named_build(tmp_path: Path) -> None:+ # A service rooted in a source package named ``build`` is a boundary; one+ # under a ``dist`` bundle is not (#3237).+ for rel, text in (+ ("services/build/pyproject.toml", "[project]\nname='b'\n"),+ ("services/build/main.py", "print('svc')\n"),+ ("services/dist/package.json", '{"name": "d"}\n'),+ ("services/dist/index.js", "module.exports = {}\n"),+ ):+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / rel).write_text(text, encoding="utf-8")++ paths = {b.service_path for b in detect_service_boundaries(tmp_path)}++ assert "services/build" in paths+ assert "services/dist" not in paths