Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): index source packages named build, coverage, dist or target (#3221)

Raghav Chamadiyabce07edfc2a4
diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md
index 91080536a..e9b135f8e 100644
--- a/docs/architecture/ARCHITECTURE.md
+++ b/docs/architecture/ARCHITECTURE.md
@@ -394,8 +394,12 @@ priority order:
patterns passed at runtime from `--exclude/-x` CLI flags or
`repo.settings["exclude_patterns"]` (set via Web UI or REST API PATCH). Applied
to both directory pruning (entire subtree skipped) and individual file filtering.
-5. Hardcoded blocklist (`node_modules`, `.git`, `__pycache__`, `dist`, `build`,
- `*.lock`, `*.min.js`, generated protobuf files, etc.)
+5. Hardcoded blocklist (`node_modules`, `.git`, `__pycache__`, `vendor`,
+ `*.lock`, `*.min.js`, generated protobuf files, etc.). `build`, `coverage`,
+ `dist` and `target` are pruned only when the directory itself holds no
+ hand-written source file (JavaScript, C/C++ and data formats do not count,
+ since builds emit them), so a source package with one of those names is
+ indexed and a committed bundle is not
6. Auto-detection of generated files (files with `// Code generated` headers)
7. Binary files (detected by null bytes in first 8KB)
8. Files over `max_file_size_kb` (default: 500KB)
diff --git a/packages/cli/src/repowise/cli/commands/watch_cmd.py b/packages/cli/src/repowise/cli/commands/watch_cmd.py
index 67a918812..12efb4137 100644
--- a/packages/cli/src/repowise/cli/commands/watch_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/watch_cmd.py
@@ -35,7 +35,9 @@ _SELF_WRITTEN_FILES: frozenset[str] = frozenset({"CLAUDE.md", "AGENTS.md", ".mcp
_SELF_WRITTEN_DIRS: frozenset[str] = frozenset({".claude", ".codex", ".cursor", ".gemini"})
-def _event_paths(event: object, repo_path: Path) -> set[str]:
+def _event_paths(
+ event: object, repo_path: Path, output_dirs: dict[Path, bool] | None = None
+) -> set[str]:
"""The watchable repo-relative paths a filesystem event touched.
Both ends of a move are considered. Editors that save atomically (the
@@ -47,6 +49,9 @@ def _event_paths(event: object, repo_path: Path) -> set[str]:
Paths outside the watched root are dropped rather than raising: an
unguarded ``relative_to`` here used to take watchdog's dispatcher thread
down with it, leaving ``watch`` apparently running but permanently deaf.
+
+ *output_dirs* memoizes the output-directory listings for one debounce
+ window, so a build writing N files lists its output root once.
"""
found: set[str] = set()
for attr in ("src_path", "dest_path"):
@@ -57,7 +62,7 @@ def _event_paths(event: object, repo_path: Path) -> set[str]:
rel = str(Path(raw).relative_to(repo_path))
except ValueError:
continue
- if is_watchable_path(rel):
+ if is_watchable_path(rel, repo_path, output_dirs):
found.add(rel)
return found
@@ -82,7 +87,9 @@ def _release_own_update_lock(repo_path: Path) -> None:
release_update_lock(repo_path)
-def is_watchable_path(rel_path: str) -> bool:
+def is_watchable_path(
+ rel_path: str, repo_path: Path | None = None, output_dirs: dict[Path, bool] | None = None
+) -> bool:
"""Whether a change to *rel_path* (repo-relative) should trigger an update.
Two filters. The first is the traversal blocklist, so a write inside
@@ -97,7 +104,7 @@ def is_watchable_path(rel_path: str) -> bool:
return False
if posix.split("/", 1)[0] in _SELF_WRITTEN_DIRS:
return False
- return is_candidate_source_path(posix)
+ return is_candidate_source_path(posix, repo_path, output_dirs)
# ---------------------------------------------------------------------------
@@ -122,6 +129,7 @@ def _watch_single_repo(
ensure_repowise_dir(repo_path)
changed_paths: set[str] = set()
+ output_dirs: dict[Path, bool] = {} # per debounce window, see _event_paths
lock = threading.Lock()
# Serialises the updates themselves. The debounce timer only guarantees a
# quiet gap before a run starts, not that the previous run has finished —
@@ -136,6 +144,7 @@ def _watch_single_repo(
with lock:
paths = set(changed_paths)
changed_paths.clear()
+ output_dirs.clear()
timer = None
if not paths:
@@ -170,7 +179,7 @@ def _watch_single_repo(
nonlocal timer
if event.is_directory:
return
- watched = _event_paths(event, repo_path)
+ watched = _event_paths(event, repo_path, output_dirs)
if not watched:
return
@@ -221,6 +230,7 @@ def _watch_workspace(
# Per-repo state: each repo gets its own change set and debounce timer
repo_locks: dict[str, threading.Lock] = {}
repo_changed: dict[str, set[str]] = {}
+ repo_output_dirs: dict[str, dict[Path, bool]] = {} # see _event_paths
repo_timers: dict[str, threading.Timer | None] = {}
# Serialises a repo's own updates; see the single-repo watcher.
repo_update_locks: dict[str, threading.Lock] = {}
@@ -228,6 +238,7 @@ def _watch_workspace(
for entry in ws_config.repos:
repo_locks[entry.alias] = threading.Lock()
repo_changed[entry.alias] = set()
+ repo_output_dirs[entry.alias] = {}
repo_timers[entry.alias] = None
repo_update_locks[entry.alias] = threading.Lock()
@@ -239,6 +250,7 @@ def _watch_workspace(
with repo_locks[alias]:
paths = set(repo_changed[alias])
repo_changed[alias].clear()
+ repo_output_dirs[alias].clear()
repo_timers[alias] = None
if not paths:
@@ -285,7 +297,7 @@ def _watch_workspace(
def on_any_event(self, event):
if event.is_directory:
return
- watched = _event_paths(event, self._repo_path)
+ watched = _event_paths(event, self._repo_path, repo_output_dirs[self._alias])
if not watched:
return
diff --git a/packages/core/src/repowise/core/ingestion/change_detector.py b/packages/core/src/repowise/core/ingestion/change_detector.py
index 268f33c05..b6732b587 100644
--- a/packages/core/src/repowise/core/ingestion/change_detector.py
+++ b/packages/core/src/repowise/core/ingestion/change_detector.py
@@ -169,13 +169,14 @@ class ChangeDetector:
return []
results: list[FileDiff] = []
+ output_dirs: dict[Path, bool] = {} # one listing per output-named dir
try:
head_commit = repo.head.commit
# ``None`` as the diff target means "the working tree", so this
# covers staged and unstaged changes in one pass.
for item in head_commit.diff(None):
path = (item.b_path or item.a_path or "").replace("\\", "/")
- if not is_candidate_source_path(path):
+ if not is_candidate_source_path(path, self.repo_path, output_dirs):
continue
results.append(self._file_diff_from_item(item))
untracked = list(repo.untracked_files)
@@ -185,7 +186,7 @@ class ChangeDetector:
for rel_path in untracked:
path = rel_path.replace("\\", "/")
- if not is_candidate_source_path(path):
+ if not is_candidate_source_path(path, self.repo_path, output_dirs):
continue
abs_path = self.repo_path / path
if not abs_path.is_file():
@@ -607,8 +608,9 @@ def has_working_tree_changes(repo_path: Path) -> bool:
changed = (
item.b_path or item.a_path or "" for item in repo.head.commit.diff(None)
)
+ output_dirs: dict[Path, bool] = {}
return any(
- is_candidate_source_path(p.replace("\\", "/"))
+ is_candidate_source_path(p.replace("\\", "/"), repo_path, output_dirs)
for p in (*changed, *repo.untracked_files)
)
except Exception as exc:
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/records.py b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
index 4917244dd..7467b60c4 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/records.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
@@ -638,19 +638,21 @@ def _history_tier_files(repo_path: Path, paths: Iterable[str]) -> set[str]:
binaries: the traverser's own blocklists, plus a sniff for NUL bytes.
"""
from ..traverser import (
- _BLOCKED_DIRS,
_BLOCKED_EXTENSIONS,
_BLOCKED_FILENAME_SPEC,
_is_binary,
+ in_blocked_dir,
)
+ output_dirs: dict[Path, bool] = {} # one listing per output-named dir
+
def _keep(file_path: str) -> bool:
- parts = file_path.split("/")
- if any(part in _BLOCKED_DIRS for part in parts[:-1]):
+ name = file_path.rsplit("/", 1)[-1]
+ if Path(name).suffix.lower() in _BLOCKED_EXTENSIONS:
return False
- if Path(file_path).suffix.lower() in _BLOCKED_EXTENSIONS:
+ if _BLOCKED_FILENAME_SPEC.match_file(name):
return False
- if _BLOCKED_FILENAME_SPEC.match_file(parts[-1]):
+ if in_blocked_dir(file_path, repo_path, output_dirs):
return False
return not _is_binary(repo_path / file_path)
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index b65fb50cf..4da78fb8d 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -210,13 +210,9 @@ _BLOCKED_DIRS: frozenset[str] = frozenset(
".mypy_cache",
".ruff_cache",
".tox",
- "dist",
- "build",
".next",
- "target", # Rust / Maven
".gradle",
"vendor", # Go / PHP
- "coverage",
"htmlcov",
".eggs",
"site-packages",
@@ -321,6 +317,24 @@ _MINIFIED_SAMPLE_BYTES: int = 256 * 1024
# Languages with no AST parsing, so generated-marker sniffing buys nothing.
_SKIP_GENERATED_CHECK: frozenset[str] = _LANG_REGISTRY.unparseable_data_languages()
+#: Names build and report tools write to that are also ordinary package names
+#: (``analysis/coverage/``, coverage.py's own ``coverage/``, a Go ``build``
+#: package, a ``build`` route). Pruned unless the directory itself holds a
+#: hand-written source file: output roots hold bundles, objects and reports
+#: directly and keep any copied source a level deeper (``build/lib/``,
+#: ``target/generated-sources/``). Ceiling: a package holding only JavaScript
+#: or C directly, or whose files all sit in subdirectories, stays pruned, and
+#: no ignore file can re-include a pruned directory yet. A ``package.json`` is
+#: no evidence: bundlers write one beside their output.
+_OUTPUT_DIR_NAMES: frozenset[str] = frozenset({"build", "coverage", "dist", "target"})
+
+#: Languages a build or report writes (bundles, generated C, HTML reports),
+#: so their files are no evidence that an output-named directory is source.
+_EMITTED_LANGUAGES: frozenset[str] = _SKIP_GENERATED_CHECK | {"c", "cpp", "javascript", "objectivec"}
+
+#: Filename endings a compiler emits in otherwise hand-written languages.
+_EMITTED_SUFFIXES: tuple[str, ...] = (".d.ts", ".d.mts", ".d.cts", ".qmltypes", *_GENERATED_SUFFIXES)
+
class FileTraverser:
"""Traverse a repository and yield FileInfo for each documentable file.
@@ -592,7 +606,8 @@ class FileTraverser:
if dir_ignore is not None and dir_ignore.extra.match_file(dirname + "/"):
return True
nested = dir_ignore is not None and dir_ignore.gitignore.match_file(dirname + "/")
- return self._dir_gitignored(rel_str, nested)
+ # Last, so a gitignored output tree is pruned before anything lists it.
+ return self._dir_gitignored(rel_str, nested) or is_output_dir(self.repo_root / rel_path)
def _dir_gitignored(self, rel_str: str, nested_match: bool) -> bool:
"""Whether ``.gitignore`` prunes the directory, which it may not if git tracks a file in it."""
@@ -1345,14 +1360,77 @@ def _parse_gitmodules(repo_root: Path) -> frozenset[str]:
return frozenset()
-def is_candidate_source_path(rel_path: str) -> bool:
+def _is_handwritten_source_name(name: str) -> bool:
+ """Whether a file named *name* is source a person writes, never a build writes."""
+ if name.endswith(_EMITTED_SUFFIXES):
+ return False
+ language = EXTENSION_TO_LANGUAGE.get(os.path.splitext(name)[1].lower())
+ return language is not None and language not in _EMITTED_LANGUAGES
+
+
+def is_output_dir(abs_dir: Path) -> bool:
+ """Whether *abs_dir* is a build or report output root (see ``_OUTPUT_DIR_NAMES``).
+
+ One listing of the directory itself, and only for an output-shaped name.
+ An unlistable directory is not output: a deleted source package must still
+ reach the change sources as deletions.
+ """
+ if abs_dir.name not in _OUTPUT_DIR_NAMES:
+ return False
+ try:
+ with os.scandir(abs_dir) as entries:
+ return not any(
+ _is_handwritten_source_name(e.name) and e.is_file() for e in entries
+ )
+ except OSError:
+ return False
+
+
+def in_blocked_dir(
+ rel_path: str, repo_root: Path | None, cache: dict[Path, bool] | None = None
+) -> bool:
+ """Whether a directory on *rel_path* (POSIX, repo-relative) is blocklisted.
+
+ With no *repo_root* there is nothing to list, so every output-shaped name
+ counts as output, the conservative answer. A caller checking many paths
+ passes one *cache* for the batch, so each output-named directory is listed
+ once. A hand-written file directly in a directory cached as output lists
+ it again, since that file may be what makes it source; build output never
+ has such a name, so a burst of it stays cached.
+ """
+ *dirs, name = rel_path.split("/")
+ for depth, part in enumerate(dirs, 1):
+ if part in _BLOCKED_DIRS:
+ return True
+ if part not in _OUTPUT_DIR_NAMES:
+ continue
+ if repo_root is None:
+ return True
+ abs_dir = repo_root.joinpath(*dirs[:depth])
+ cached = None if cache is None else cache.get(abs_dir)
+ if cached is None or (
+ cached and depth == len(dirs) and _is_handwritten_source_name(name)
+ ):
+ cached = is_output_dir(abs_dir)
+ if cache is not None:
+ cache[abs_dir] = cached
+ if cached:
+ return True
+ return False
+
+
+def is_candidate_source_path(
+ rel_path: str, repo_root: Path | None = None, cache: dict[Path, bool] | None = None
+) -> bool:
"""Whether *rel_path* is shaped like a file this repo would index.
Path-shape only: the directory blocklist, the blocked extensions/filename
- patterns, and the known-language extension map. No disk access, no
- gitignore, no binary/size/generated checks, so a ``True`` answer means
- "worth handing to the pipeline", never "will be indexed". :class:`FileTraverser`
- still applies the full test on the files it walks.
+ patterns, and the known-language extension map. No gitignore, no
+ binary/size/generated checks, so a ``True`` answer means "worth handing to
+ the pipeline", never "will be indexed". :class:`FileTraverser` still
+ applies the full test on the files it walks. The only disk access is the
+ output-directory listing of :func:`in_blocked_dir`, given *repo_root*
+ (and memoized in *cache* when given).
It serves the change sources that only see a path: the working-tree diff
and the file watcher.
@@ -1360,10 +1438,11 @@ def is_candidate_source_path(rel_path: str) -> bool:
One known false negative: an extensionless script that :class:`FileTraverser`
accepts by shebang is rejected here, since deciding that means reading it.
"""
- parts = Path(rel_path.replace("\\", "/")).parts
+ posix = rel_path.replace("\\", "/")
+ parts = Path(posix).parts
if not parts:
return False
- if any(part in _BLOCKED_DIRS for part in parts[:-1]):
+ if in_blocked_dir(posix, repo_root, cache):
return False
name = parts[-1]
diff --git a/tests/unit/ingestion/test_git_history_tier.py b/tests/unit/ingestion/test_git_history_tier.py
index 50e2c8021..c3e6e772b 100644
--- a/tests/unit/ingestion/test_git_history_tier.py
+++ b/tests/unit/ingestion/test_git_history_tier.py
@@ -61,6 +61,7 @@ def _build(root: Path) -> None:
"logo.png": b"\x89PNG\r\n\x1a\n\x00\x00\x00binary",
"yarn.lock": "# lock\n",
"vendor/lib/notes.md": "vendored\n",
+ "dist/report.md": "built\n",
},
"feat: initial",
1,
@@ -89,7 +90,7 @@ async def test_non_code_files_get_history_from_the_shared_walk(tmp_path, monkeyp
_summary, rows = await GitIndexer(tmp_path, tier=GitIndexTier.FULL).index_repo("r")
meta = {row["file_path"]: row for row in rows}
- # Binary, lock and vendored files get no row at all.
+ # Binary, lock, vendored and build-output files get no row at all.
assert set(meta) == {"a.py", "b.py", "README.md", ".github/workflows/ci.yml", "config.json"}
for path in ("README.md", ".github/workflows/ci.yml", "config.json"):
@@ -157,3 +158,15 @@ async def test_follow_renames_mode_gives_non_code_files_no_rows(tmp_path, monkey
assert "a.py" in paths
assert not {"README.md", ".github/workflows/ci.yml", "config.json"} & paths
+
+
+def test_a_source_package_named_like_an_output_dir_keeps_its_history(tmp_path) -> None:
+ from repowise.core.ingestion.git_indexer.records import _history_tier_files
+
+ for rel in ("pkg/coverage/parsers.py", "pkg/coverage/README.md", "dist/notes.md"):
+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / rel).write_text("x\n", encoding="utf-8")
+
+ kept = _history_tier_files(tmp_path, ["pkg/coverage/README.md", "dist/notes.md"])
+
+ assert kept == {"pkg/coverage/README.md"}
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index 657651ae7..8753c476b 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -191,6 +191,46 @@ class TestFileTraverser:
assert not any(p.startswith("Library/") for p in paths)
assert not any(p.startswith("Temp/") for p in paths)
+ def test_source_packages_named_like_output_dirs_are_indexed(self, tmp_path: Path) -> None:
+ files = {
+ "src/pkg/analysis/coverage/__init__.py": "",
+ "src/pkg/analysis/coverage/parsers.py": "def parse(): ...\n",
+ "internal/build/build.go": "package build\n",
+ "app/(platform)/build/page.tsx": "export default function P() {}\n",
+ "coverage/__init__.py": "", # a project's own root package
+ }
+ for rel, text in files.items():
+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / rel).write_text(text, encoding="utf-8")
+
+ paths = {f.path for f in FileTraverser(tmp_path).traverse()}
+
+ assert set(files) <= paths
+
+ def test_output_trees_stay_pruned_without_gitignore(self, tmp_path: Path) -> None:
+ files = {
+ # Bundles and declarations a compiler writes, committed.
+ "packages/web/dist/index.js": "var a = 1;\n",
+ "packages/web/dist/index.d.ts": "export {};\n",
+ # Copied source sits a level below the output root.
+ "build/lib/pkg/__init__.py": "",
+ "build/lib/pkg/mod.py": "x = 1\n",
+ "target/generated-sources/Gen.java": "class Gen {}\n",
+ "coverage/lcov-report/prettify.js": "var p;\n",
+ # Generated C from a configure step, and generated stubs.
+ "build/config.h": "#define X 1\n",
+ "build/api_pb2.py": "x = 1\n",
+ "vendor/github.com/x/y/y.go": "package y\n",
+ }
+ for rel, text in files.items():
+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / rel).write_text(text, encoding="utf-8")
+ (tmp_path / "app.py").write_text("x = 1\n", encoding="utf-8")
+
+ paths = {f.path for f in FileTraverser(tmp_path).traverse()}
+
+ assert paths == {"app.py"}
+
def test_traverses_e2e_specs(self, tmp_path: Path) -> None:
"""e2e spec files are indexed so they can get a file_page (#1497).
@@ -1457,9 +1497,10 @@ class TestPackageScanPruning:
assert "pkg/index.html" in pruned
def test_a_committed_build_dir_is_excluded_too(self, tmp_path: Path) -> None:
- """Not just gitignored trees. ``dist`` is in ``_BLOCKED_DIRS``, so the
- traverser never indexes it even when it is committed — and a directory
- nothing indexes must not name the package's language."""
+ """Not just gitignored trees. A ``dist`` holding only bundles is an
+ output root, so the traverser never indexes it even when it is
+ committed, and a directory nothing indexes must not name the package's
+ language."""
pkg = self._pkg(tmp_path, "node_modules/\n") # dist/ deliberately tracked
dist = pkg / "dist"
dist.mkdir()
diff --git a/tests/unit/ingestion/test_working_tree_changes.py b/tests/unit/ingestion/test_working_tree_changes.py
index 21bf3ae9e..2f3f64267 100644
--- a/tests/unit/ingestion/test_working_tree_changes.py
+++ b/tests/unit/ingestion/test_working_tree_changes.py
@@ -265,3 +265,38 @@ class TestIsCandidateSourcePath:
def test_empty_path(self) -> None:
assert is_candidate_source_path("") is False
+
+ def test_output_named_dirs_are_listed_given_the_root(self, tmp_path: Path) -> None:
+ # A source package named like a report dir passes; a bundle dir does not.
+ (tmp_path / "pkg" / "coverage").mkdir(parents=True)
+ (tmp_path / "pkg" / "coverage" / "parsers.py").write_text("x = 1\n", encoding="utf-8")
+ (tmp_path / "dist").mkdir()
+ (tmp_path / "dist" / "bundle.js").write_text("var a;\n", encoding="utf-8")
+
+ assert is_candidate_source_path("pkg/coverage/parsers.py", tmp_path) is True
+ assert is_candidate_source_path("dist/bundle.js", tmp_path) is False
+ # No root to list: the conservative answer.
+ assert is_candidate_source_path("pkg/coverage/parsers.py") is False
+ # A deleted package still surfaces its deletions.
+ assert is_candidate_source_path("gone/coverage/parsers.py", tmp_path) is True
+
+ def test_a_shared_cache_lists_each_output_dir_once(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from repowise.core.ingestion import traverser
+
+ (tmp_path / "dist").mkdir()
+ for i in range(5):
+ (tmp_path / "dist" / f"chunk{i}.js").write_text("var a;\n", encoding="utf-8")
+ listed: list[Path] = []
+ real = traverser.is_output_dir
+ monkeypatch.setattr(traverser, "is_output_dir", lambda d: listed.append(d) or real(d))
+ cache: dict[Path, bool] = {}
+
+ for i in range(5):
+ assert is_candidate_source_path(f"dist/chunk{i}.js", tmp_path, cache) is False
+ assert listed == [tmp_path / "dist"]
+
+ # A hand-written file landing in it later is seen despite the cache.
+ (tmp_path / "dist" / "cli.py").write_text("x = 1\n", encoding="utf-8")
+ assert is_candidate_source_path("dist/cli.py", tmp_path, cache) is True