Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): index source packages named build, coverage, dist or target (#3221)
Raghav Chamadiyabce07edfc2a4
diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.mdindex 91080536a..e9b135f8e 100644--- a/docs/architecture/ARCHITECTURE.md+++ b/docs/architecture/ARCHITECTURE.md@@ -394,8 +394,12 @@ priority order:patterns passed at runtime from `--exclude/-x` CLI flags or`repo.settings["exclude_patterns"]` (set via Web UI or REST API PATCH). Appliedto both directory pruning (entire subtree skipped) and individual file filtering.-5. Hardcoded blocklist (`node_modules`, `.git`, `__pycache__`, `dist`, `build`,- `*.lock`, `*.min.js`, generated protobuf files, etc.)+5. Hardcoded blocklist (`node_modules`, `.git`, `__pycache__`, `vendor`,+ `*.lock`, `*.min.js`, generated protobuf files, etc.). `build`, `coverage`,+ `dist` and `target` are pruned only when the directory itself holds no+ hand-written source file (JavaScript, C/C++ and data formats do not count,+ since builds emit them), so a source package with one of those names is+ indexed and a committed bundle is not6. Auto-detection of generated files (files with `// Code generated` headers)7. Binary files (detected by null bytes in first 8KB)8. Files over `max_file_size_kb` (default: 500KB)diff --git a/packages/cli/src/repowise/cli/commands/watch_cmd.py b/packages/cli/src/repowise/cli/commands/watch_cmd.pyindex 67a918812..12efb4137 100644--- a/packages/cli/src/repowise/cli/commands/watch_cmd.py+++ b/packages/cli/src/repowise/cli/commands/watch_cmd.py@@ -35,7 +35,9 @@ _SELF_WRITTEN_FILES: frozenset[str] = frozenset({"CLAUDE.md", "AGENTS.md", ".mcp_SELF_WRITTEN_DIRS: frozenset[str] = frozenset({".claude", ".codex", ".cursor", ".gemini"})-def _event_paths(event: object, repo_path: Path) -> set[str]:+def _event_paths(+ event: object, repo_path: Path, output_dirs: dict[Path, bool] | None = None+) -> set[str]:"""The watchable repo-relative paths a filesystem event touched.Both ends of a move are considered. Editors that save atomically (the@@ -47,6 +49,9 @@ def _event_paths(event: object, repo_path: Path) -> set[str]:Paths outside the watched root are dropped rather than raising: anunguarded ``relative_to`` here used to take watchdog's dispatcher threaddown with it, leaving ``watch`` apparently running but permanently deaf.++ *output_dirs* memoizes the output-directory listings for one debounce+ window, so a build writing N files lists its output root once."""found: set[str] = set()for attr in ("src_path", "dest_path"):@@ -57,7 +62,7 @@ def _event_paths(event: object, repo_path: Path) -> set[str]:rel = str(Path(raw).relative_to(repo_path))except ValueError:continue- if is_watchable_path(rel):+ if is_watchable_path(rel, repo_path, output_dirs):found.add(rel)return found@@ -82,7 +87,9 @@ def _release_own_update_lock(repo_path: Path) -> None:release_update_lock(repo_path)-def is_watchable_path(rel_path: str) -> bool:+def is_watchable_path(+ rel_path: str, repo_path: Path | None = None, output_dirs: dict[Path, bool] | None = None+) -> bool:"""Whether a change to *rel_path* (repo-relative) should trigger an update.Two filters. The first is the traversal blocklist, so a write inside@@ -97,7 +104,7 @@ def is_watchable_path(rel_path: str) -> bool:return Falseif posix.split("/", 1)[0] in _SELF_WRITTEN_DIRS:return False- return is_candidate_source_path(posix)+ return is_candidate_source_path(posix, repo_path, output_dirs)# ---------------------------------------------------------------------------@@ -122,6 +129,7 @@ def _watch_single_repo(ensure_repowise_dir(repo_path)changed_paths: set[str] = set()+ output_dirs: dict[Path, bool] = {} # per debounce window, see _event_pathslock = threading.Lock()# Serialises the updates themselves. The debounce timer only guarantees a# quiet gap before a run starts, not that the previous run has finished —@@ -136,6 +144,7 @@ def _watch_single_repo(with lock:paths = set(changed_paths)changed_paths.clear()+ output_dirs.clear()timer = Noneif not paths:@@ -170,7 +179,7 @@ def _watch_single_repo(nonlocal timerif event.is_directory:return- watched = _event_paths(event, repo_path)+ watched = _event_paths(event, repo_path, output_dirs)if not watched:return@@ -221,6 +230,7 @@ def _watch_workspace(# Per-repo state: each repo gets its own change set and debounce timerrepo_locks: dict[str, threading.Lock] = {}repo_changed: dict[str, set[str]] = {}+ repo_output_dirs: dict[str, dict[Path, bool]] = {} # see _event_pathsrepo_timers: dict[str, threading.Timer | None] = {}# Serialises a repo's own updates; see the single-repo watcher.repo_update_locks: dict[str, threading.Lock] = {}@@ -228,6 +238,7 @@ def _watch_workspace(for entry in ws_config.repos:repo_locks[entry.alias] = threading.Lock()repo_changed[entry.alias] = set()+ repo_output_dirs[entry.alias] = {}repo_timers[entry.alias] = Nonerepo_update_locks[entry.alias] = threading.Lock()@@ -239,6 +250,7 @@ def _watch_workspace(with repo_locks[alias]:paths = set(repo_changed[alias])repo_changed[alias].clear()+ repo_output_dirs[alias].clear()repo_timers[alias] = Noneif not paths:@@ -285,7 +297,7 @@ def _watch_workspace(def on_any_event(self, event):if event.is_directory:return- watched = _event_paths(event, self._repo_path)+ watched = _event_paths(event, self._repo_path, repo_output_dirs[self._alias])if not watched:returndiff --git a/packages/core/src/repowise/core/ingestion/change_detector.py b/packages/core/src/repowise/core/ingestion/change_detector.pyindex 268f33c05..b6732b587 100644--- a/packages/core/src/repowise/core/ingestion/change_detector.py+++ b/packages/core/src/repowise/core/ingestion/change_detector.py@@ -169,13 +169,14 @@ class ChangeDetector:return []results: list[FileDiff] = []+ output_dirs: dict[Path, bool] = {} # one listing per output-named dirtry:head_commit = repo.head.commit# ``None`` as the diff target means "the working tree", so this# covers staged and unstaged changes in one pass.for item in head_commit.diff(None):path = (item.b_path or item.a_path or "").replace("\\", "/")- if not is_candidate_source_path(path):+ if not is_candidate_source_path(path, self.repo_path, output_dirs):continueresults.append(self._file_diff_from_item(item))untracked = list(repo.untracked_files)@@ -185,7 +186,7 @@ class ChangeDetector:for rel_path in untracked:path = rel_path.replace("\\", "/")- if not is_candidate_source_path(path):+ if not is_candidate_source_path(path, self.repo_path, output_dirs):continueabs_path = self.repo_path / pathif not abs_path.is_file():@@ -607,8 +608,9 @@ def has_working_tree_changes(repo_path: Path) -> bool:changed = (item.b_path or item.a_path or "" for item in repo.head.commit.diff(None))+ output_dirs: dict[Path, bool] = {}return any(- is_candidate_source_path(p.replace("\\", "/"))+ is_candidate_source_path(p.replace("\\", "/"), repo_path, output_dirs)for p in (*changed, *repo.untracked_files))except Exception as exc:diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/records.py b/packages/core/src/repowise/core/ingestion/git_indexer/records.pyindex 4917244dd..7467b60c4 100644--- a/packages/core/src/repowise/core/ingestion/git_indexer/records.py+++ b/packages/core/src/repowise/core/ingestion/git_indexer/records.py@@ -638,19 +638,21 @@ def _history_tier_files(repo_path: Path, paths: Iterable[str]) -> set[str]:binaries: the traverser's own blocklists, plus a sniff for NUL bytes."""from ..traverser import (- _BLOCKED_DIRS,_BLOCKED_EXTENSIONS,_BLOCKED_FILENAME_SPEC,_is_binary,+ in_blocked_dir,)+ output_dirs: dict[Path, bool] = {} # one listing per output-named dir+def _keep(file_path: str) -> bool:- parts = file_path.split("/")- if any(part in _BLOCKED_DIRS for part in parts[:-1]):+ name = file_path.rsplit("/", 1)[-1]+ if Path(name).suffix.lower() in _BLOCKED_EXTENSIONS:return False- if Path(file_path).suffix.lower() in _BLOCKED_EXTENSIONS:+ if _BLOCKED_FILENAME_SPEC.match_file(name):return False- if _BLOCKED_FILENAME_SPEC.match_file(parts[-1]):+ if in_blocked_dir(file_path, repo_path, output_dirs):return Falsereturn not _is_binary(repo_path / file_path)diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex b65fb50cf..4da78fb8d 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -210,13 +210,9 @@ _BLOCKED_DIRS: frozenset[str] = frozenset(".mypy_cache",".ruff_cache",".tox",- "dist",- "build",".next",- "target", # Rust / Maven".gradle","vendor", # Go / PHP- "coverage","htmlcov",".eggs","site-packages",@@ -321,6 +317,24 @@ _MINIFIED_SAMPLE_BYTES: int = 256 * 1024# Languages with no AST parsing, so generated-marker sniffing buys nothing._SKIP_GENERATED_CHECK: frozenset[str] = _LANG_REGISTRY.unparseable_data_languages()+#: Names build and report tools write to that are also ordinary package names+#: (``analysis/coverage/``, coverage.py's own ``coverage/``, a Go ``build``+#: package, a ``build`` route). Pruned unless the directory itself holds a+#: hand-written source file: output roots hold bundles, objects and reports+#: directly and keep any copied source a level deeper (``build/lib/``,+#: ``target/generated-sources/``). Ceiling: a package holding only JavaScript+#: or C directly, or whose files all sit in subdirectories, stays pruned, and+#: no ignore file can re-include a pruned directory yet. A ``package.json`` is+#: no evidence: bundlers write one beside their output.+_OUTPUT_DIR_NAMES: frozenset[str] = frozenset({"build", "coverage", "dist", "target"})++#: Languages a build or report writes (bundles, generated C, HTML reports),+#: so their files are no evidence that an output-named directory is source.+_EMITTED_LANGUAGES: frozenset[str] = _SKIP_GENERATED_CHECK | {"c", "cpp", "javascript", "objectivec"}++#: Filename endings a compiler emits in otherwise hand-written languages.+_EMITTED_SUFFIXES: tuple[str, ...] = (".d.ts", ".d.mts", ".d.cts", ".qmltypes", *_GENERATED_SUFFIXES)+class FileTraverser:"""Traverse a repository and yield FileInfo for each documentable file.@@ -592,7 +606,8 @@ class FileTraverser:if dir_ignore is not None and dir_ignore.extra.match_file(dirname + "/"):return Truenested = dir_ignore is not None and dir_ignore.gitignore.match_file(dirname + "/")- return self._dir_gitignored(rel_str, nested)+ # Last, so a gitignored output tree is pruned before anything lists it.+ return self._dir_gitignored(rel_str, nested) or is_output_dir(self.repo_root / rel_path)def _dir_gitignored(self, rel_str: str, nested_match: bool) -> bool:"""Whether ``.gitignore`` prunes the directory, which it may not if git tracks a file in it."""@@ -1345,14 +1360,77 @@ def _parse_gitmodules(repo_root: Path) -> frozenset[str]:return frozenset()-def is_candidate_source_path(rel_path: str) -> bool:+def _is_handwritten_source_name(name: str) -> bool:+ """Whether a file named *name* is source a person writes, never a build writes."""+ if name.endswith(_EMITTED_SUFFIXES):+ return False+ language = EXTENSION_TO_LANGUAGE.get(os.path.splitext(name)[1].lower())+ return language is not None and language not in _EMITTED_LANGUAGES+++def is_output_dir(abs_dir: Path) -> bool:+ """Whether *abs_dir* is a build or report output root (see ``_OUTPUT_DIR_NAMES``).++ One listing of the directory itself, and only for an output-shaped name.+ An unlistable directory is not output: a deleted source package must still+ reach the change sources as deletions.+ """+ if abs_dir.name not in _OUTPUT_DIR_NAMES:+ return False+ try:+ with os.scandir(abs_dir) as entries:+ return not any(+ _is_handwritten_source_name(e.name) and e.is_file() for e in entries+ )+ except OSError:+ return False+++def in_blocked_dir(+ rel_path: str, repo_root: Path | None, cache: dict[Path, bool] | None = None+) -> bool:+ """Whether a directory on *rel_path* (POSIX, repo-relative) is blocklisted.++ With no *repo_root* there is nothing to list, so every output-shaped name+ counts as output, the conservative answer. A caller checking many paths+ passes one *cache* for the batch, so each output-named directory is listed+ once. A hand-written file directly in a directory cached as output lists+ it again, since that file may be what makes it source; build output never+ has such a name, so a burst of it stays cached.+ """+ *dirs, name = rel_path.split("/")+ for depth, part in enumerate(dirs, 1):+ if part in _BLOCKED_DIRS:+ return True+ if part not in _OUTPUT_DIR_NAMES:+ continue+ if repo_root is None:+ return True+ abs_dir = repo_root.joinpath(*dirs[:depth])+ cached = None if cache is None else cache.get(abs_dir)+ if cached is None or (+ cached and depth == len(dirs) and _is_handwritten_source_name(name)+ ):+ cached = is_output_dir(abs_dir)+ if cache is not None:+ cache[abs_dir] = cached+ if cached:+ return True+ return False+++def is_candidate_source_path(+ rel_path: str, repo_root: Path | None = None, cache: dict[Path, bool] | None = None+) -> bool:"""Whether *rel_path* is shaped like a file this repo would index.Path-shape only: the directory blocklist, the blocked extensions/filename- patterns, and the known-language extension map. No disk access, no- gitignore, no binary/size/generated checks, so a ``True`` answer means- "worth handing to the pipeline", never "will be indexed". :class:`FileTraverser`- still applies the full test on the files it walks.+ patterns, and the known-language extension map. No gitignore, no+ binary/size/generated checks, so a ``True`` answer means "worth handing to+ the pipeline", never "will be indexed". :class:`FileTraverser` still+ applies the full test on the files it walks. The only disk access is the+ output-directory listing of :func:`in_blocked_dir`, given *repo_root*+ (and memoized in *cache* when given).It serves the change sources that only see a path: the working-tree diffand the file watcher.@@ -1360,10 +1438,11 @@ def is_candidate_source_path(rel_path: str) -> bool:One known false negative: an extensionless script that :class:`FileTraverser`accepts by shebang is rejected here, since deciding that means reading it."""- parts = Path(rel_path.replace("\\", "/")).parts+ posix = rel_path.replace("\\", "/")+ parts = Path(posix).partsif not parts:return False- if any(part in _BLOCKED_DIRS for part in parts[:-1]):+ if in_blocked_dir(posix, repo_root, cache):return Falsename = parts[-1]diff --git a/tests/unit/ingestion/test_git_history_tier.py b/tests/unit/ingestion/test_git_history_tier.pyindex 50e2c8021..c3e6e772b 100644--- a/tests/unit/ingestion/test_git_history_tier.py+++ b/tests/unit/ingestion/test_git_history_tier.py@@ -61,6 +61,7 @@ def _build(root: Path) -> None:"logo.png": b"\x89PNG\r\n\x1a\n\x00\x00\x00binary","yarn.lock": "# lock\n","vendor/lib/notes.md": "vendored\n",+ "dist/report.md": "built\n",},"feat: initial",1,@@ -89,7 +90,7 @@ async def test_non_code_files_get_history_from_the_shared_walk(tmp_path, monkeyp_summary, rows = await GitIndexer(tmp_path, tier=GitIndexTier.FULL).index_repo("r")meta = {row["file_path"]: row for row in rows}- # Binary, lock and vendored files get no row at all.+ # Binary, lock, vendored and build-output files get no row at all.assert set(meta) == {"a.py", "b.py", "README.md", ".github/workflows/ci.yml", "config.json"}for path in ("README.md", ".github/workflows/ci.yml", "config.json"):@@ -157,3 +158,15 @@ async def test_follow_renames_mode_gives_non_code_files_no_rows(tmp_path, monkeyassert "a.py" in pathsassert not {"README.md", ".github/workflows/ci.yml", "config.json"} & paths+++def test_a_source_package_named_like_an_output_dir_keeps_its_history(tmp_path) -> None:+ from repowise.core.ingestion.git_indexer.records import _history_tier_files++ for rel in ("pkg/coverage/parsers.py", "pkg/coverage/README.md", "dist/notes.md"):+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / rel).write_text("x\n", encoding="utf-8")++ kept = _history_tier_files(tmp_path, ["pkg/coverage/README.md", "dist/notes.md"])++ assert kept == {"pkg/coverage/README.md"}diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.pyindex 657651ae7..8753c476b 100644--- a/tests/unit/ingestion/test_traverser.py+++ b/tests/unit/ingestion/test_traverser.py@@ -191,6 +191,46 @@ class TestFileTraverser:assert not any(p.startswith("Library/") for p in paths)assert not any(p.startswith("Temp/") for p in paths)+ def test_source_packages_named_like_output_dirs_are_indexed(self, tmp_path: Path) -> None:+ files = {+ "src/pkg/analysis/coverage/__init__.py": "",+ "src/pkg/analysis/coverage/parsers.py": "def parse(): ...\n",+ "internal/build/build.go": "package build\n",+ "app/(platform)/build/page.tsx": "export default function P() {}\n",+ "coverage/__init__.py": "", # a project's own root package+ }+ for rel, text in files.items():+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / rel).write_text(text, encoding="utf-8")++ paths = {f.path for f in FileTraverser(tmp_path).traverse()}++ assert set(files) <= paths++ def test_output_trees_stay_pruned_without_gitignore(self, tmp_path: Path) -> None:+ files = {+ # Bundles and declarations a compiler writes, committed.+ "packages/web/dist/index.js": "var a = 1;\n",+ "packages/web/dist/index.d.ts": "export {};\n",+ # Copied source sits a level below the output root.+ "build/lib/pkg/__init__.py": "",+ "build/lib/pkg/mod.py": "x = 1\n",+ "target/generated-sources/Gen.java": "class Gen {}\n",+ "coverage/lcov-report/prettify.js": "var p;\n",+ # Generated C from a configure step, and generated stubs.+ "build/config.h": "#define X 1\n",+ "build/api_pb2.py": "x = 1\n",+ "vendor/github.com/x/y/y.go": "package y\n",+ }+ for rel, text in files.items():+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / rel).write_text(text, encoding="utf-8")+ (tmp_path / "app.py").write_text("x = 1\n", encoding="utf-8")++ paths = {f.path for f in FileTraverser(tmp_path).traverse()}++ assert paths == {"app.py"}+def test_traverses_e2e_specs(self, tmp_path: Path) -> None:"""e2e spec files are indexed so they can get a file_page (#1497).@@ -1457,9 +1497,10 @@ class TestPackageScanPruning:assert "pkg/index.html" in pruneddef test_a_committed_build_dir_is_excluded_too(self, tmp_path: Path) -> None:- """Not just gitignored trees. ``dist`` is in ``_BLOCKED_DIRS``, so the- traverser never indexes it even when it is committed — and a directory- nothing indexes must not name the package's language."""+ """Not just gitignored trees. A ``dist`` holding only bundles is an+ output root, so the traverser never indexes it even when it is+ committed, and a directory nothing indexes must not name the package's+ language."""pkg = self._pkg(tmp_path, "node_modules/\n") # dist/ deliberately trackeddist = pkg / "dist"dist.mkdir()diff --git a/tests/unit/ingestion/test_working_tree_changes.py b/tests/unit/ingestion/test_working_tree_changes.pyindex 21bf3ae9e..2f3f64267 100644--- a/tests/unit/ingestion/test_working_tree_changes.py+++ b/tests/unit/ingestion/test_working_tree_changes.py@@ -265,3 +265,38 @@ class TestIsCandidateSourcePath:def test_empty_path(self) -> None:assert is_candidate_source_path("") is False++ def test_output_named_dirs_are_listed_given_the_root(self, tmp_path: Path) -> None:+ # A source package named like a report dir passes; a bundle dir does not.+ (tmp_path / "pkg" / "coverage").mkdir(parents=True)+ (tmp_path / "pkg" / "coverage" / "parsers.py").write_text("x = 1\n", encoding="utf-8")+ (tmp_path / "dist").mkdir()+ (tmp_path / "dist" / "bundle.js").write_text("var a;\n", encoding="utf-8")++ assert is_candidate_source_path("pkg/coverage/parsers.py", tmp_path) is True+ assert is_candidate_source_path("dist/bundle.js", tmp_path) is False+ # No root to list: the conservative answer.+ assert is_candidate_source_path("pkg/coverage/parsers.py") is False+ # A deleted package still surfaces its deletions.+ assert is_candidate_source_path("gone/coverage/parsers.py", tmp_path) is True++ def test_a_shared_cache_lists_each_output_dir_once(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ from repowise.core.ingestion import traverser++ (tmp_path / "dist").mkdir()+ for i in range(5):+ (tmp_path / "dist" / f"chunk{i}.js").write_text("var a;\n", encoding="utf-8")+ listed: list[Path] = []+ real = traverser.is_output_dir+ monkeypatch.setattr(traverser, "is_output_dir", lambda d: listed.append(d) or real(d))+ cache: dict[Path, bool] = {}++ for i in range(5):+ assert is_candidate_source_path(f"dist/chunk{i}.js", tmp_path, cache) is False+ assert listed == [tmp_path / "dist"]++ # A hand-written file landing in it later is seen despite the cache.+ (tmp_path / "dist" / "cli.py").write_text("x = 1\n", encoding="utf-8")+ assert is_candidate_source_path("dist/cli.py", tmp_path, cache) is True