Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(update): persist updates for a subdirectory index and scope git rows to it (#4195)

Taejeong Kwon67cfb6488e20
diff --git a/docs/start/TROUBLESHOOTING.md b/docs/start/TROUBLESHOOTING.md
index e3d1bd630..d7b1c288f 100644
--- a/docs/start/TROUBLESHOOTING.md
+++ b/docs/start/TROUBLESHOOTING.md
@@ -60,6 +60,21 @@ repowise init --max-file-pages 2000 # cap file pages, highest importanc
The parse pool defaults to at most eight worker processes; each one holds its
own parser, so lowering the count is the most direct memory lever.
+**Indexing one directory of a large repository.** When the path you index is a
+subdirectory of a larger git repository, git rows (per-file history, per-commit
+file detail) are recorded for that directory only, so the database stays
+proportional to the directory rather than to the whole repository. Co-change
+partners outside the directory are not recorded. Set `REPOWISE_GIT_SCOPE=repo`
+to record the whole repository instead. An index created before this change
+keeps its old rows until `repowise init --force` rebuilds it.
+
+**An update spends minutes after the files are already indexed.** After an
+update commits, repowise scans recent commits for what each did to code health.
+A commit that touches more than 300 files is skipped, so a bulk import cannot
+stall it; change the limit with `REPOWISE_COMMIT_HEALTH_MAX_FILES` (`0` removes
+it), or turn the scan off with `REPOWISE_COMMIT_HEALTH_COMMITS=0`. An update on
+a `--mode fast` index does not run the scan.
+
**Indexing cost more than expected.** `repowise init --dry-run` shows the
estimate without writing a wiki, `--test-run` generates for the top 10 files
only, and `--skip-tests --skip-infra` narrows scope. Lower `--concurrency` if
diff --git a/packages/core/src/repowise/core/analysis/change_health/commit_scan.py b/packages/core/src/repowise/core/analysis/change_health/commit_scan.py
index c3b7b84bf..d79bf45e6 100644
--- a/packages/core/src/repowise/core/analysis/change_health/commit_scan.py
+++ b/packages/core/src/repowise/core/analysis/change_health/commit_scan.py
@@ -11,6 +11,8 @@ by a wall clock instead, leaving older commits unscanned.
from __future__ import annotations
+import os
+import subprocess
import time
from dataclasses import dataclass, field
from typing import Any
@@ -18,7 +20,7 @@ from typing import Any
from repowise.core.repo_config import health_rules_fingerprint
from .models import ChangeFinding, ChangeHealthDelta
-from .service import ChangeHealthDeltaService, DeltaRequest
+from .service import MAX_CHANGED_FILES, ChangeHealthDeltaService, DeltaRequest
#: Findings kept per commit. The delta row carries the true totals, so a
#: capped commit still reports honestly how much it is not showing.
@@ -28,6 +30,55 @@ MAX_FINDINGS_PER_COMMIT = 50
#: commit, and storing it would make "no findings" ambiguous forever.
_STORED_STATUSES = frozenset({"available", "partial"})
+#: A commit touching more files than this is not analysed. The budget below is
+#: only checked *between* commits, so a single bulk commit (an initial import
+#: of tens of thousands of files) would otherwise run for many minutes and
+#: ignore it. The default is the comparison's own ceiling. It counts every
+#: changed file, while the comparison counts only eligible code files, so a
+#: commit of many non-code files is skipped here where it used to be analysed;
+#: that is intended, since reading its change shape is the cost being avoided.
+#: ``0`` removes the cap.
+def max_files_per_commit() -> int:
+ raw = os.environ.get("REPOWISE_COMMIT_HEALTH_MAX_FILES")
+ if raw is None:
+ return MAX_CHANGED_FILES
+ try:
+ value = int(raw)
+ except ValueError:
+ return MAX_CHANGED_FILES
+ return value if value >= 0 else MAX_CHANGED_FILES
+
+
+def changed_file_count(repo_path: str, sha: str) -> int | None:
+ """Files *sha* changes against its first parent (all of them for a root commit).
+
+ ``None`` when git cannot answer, so the caller analyses the commit as it
+ always did rather than skipping one it merely failed to measure.
+ """
+ try:
+ out = subprocess.run(
+ [
+ "git",
+ "-C",
+ repo_path,
+ "diff-tree",
+ "--root",
+ "-r",
+ "-m",
+ "--first-parent",
+ "--name-only",
+ "--no-commit-id",
+ "-z",
+ sha,
+ ],
+ capture_output=True,
+ timeout=30,
+ check=True,
+ ).stdout
+ except (OSError, subprocess.SubprocessError):
+ return None
+ return len({name for name in out.split(b"\0") if name})
+
@dataclass(slots=True)
class CommitHealthScan:
@@ -122,10 +173,16 @@ def scan_commits(
repo_path=repo_path, rules_fingerprint=health_rules_fingerprint(repo_path)
)
started = time.monotonic()
+ file_cap = max_files_per_commit()
for sha in shas:
if budget_seconds and time.monotonic() - started >= budget_seconds:
scan.exhausted_budget = True
break
+ if file_cap:
+ count = changed_file_count(repo_path, sha)
+ if count is not None and count > file_cap:
+ scan.skipped += 1
+ continue
try:
delta = service.compare(DeltaRequest(repo_path, sha))
except Exception:
diff --git a/packages/core/src/repowise/core/ingestion/change_detector.py b/packages/core/src/repowise/core/ingestion/change_detector.py
index 4f2e48fa3..51314ae66 100644
--- a/packages/core/src/repowise/core/ingestion/change_detector.py
+++ b/packages/core/src/repowise/core/ingestion/change_detector.py
@@ -114,6 +114,10 @@ class ChangeDetector:
def __init__(self, repo_path: Path) -> None:
self.repo_path = repo_path
self._repo: object = None # lazy git.Repo
+ # Where repo_path sits inside the git working tree ("" when it is the
+ # root). Git reports root-relative paths; the index is keyed relative
+ # to repo_path, so every path is translated through this prefix.
+ self._prefix: str = ""
# ------------------------------------------------------------------
# Public API
@@ -136,12 +140,17 @@ class ChangeDetector:
try:
base_commit = repo.commit(base_ref)
until_commit = repo.commit(until_ref)
- diff_items = base_commit.diff(until_commit)
+ if self._prefix:
+ diff_items = base_commit.diff(until_commit, paths=self._prefix)
+ else:
+ diff_items = base_commit.diff(until_commit)
except Exception as exc:
log.warning("git diff failed", base=base_ref, until=until_ref, error=str(exc))
return []
- return [self._file_diff_from_item(item) for item in diff_items]
+ return [
+ self._file_diff_from_item(item) for item in diff_items if self._item_in_scope(item)
+ ]
def get_working_tree_changes(self) -> list[FileDiff]:
"""Return FileDiffs for changes that are in the working tree, not in git.
@@ -174,12 +183,22 @@ class ChangeDetector:
head_commit = repo.head.commit
# ``None`` as the diff target means "the working tree", so this
# covers staged and unstaged changes in one pass.
- for item in head_commit.diff(None):
- path = (item.b_path or item.a_path or "").replace("\\", "/")
- if not _counts_as_change(path, self.repo_path, output_dirs, item):
+ items = (
+ head_commit.diff(None, paths=self._prefix)
+ if self._prefix
+ else head_commit.diff(None)
+ )
+ for item in items:
+ if not self._item_in_scope(item):
+ continue
+ # Git returns root-relative paths for a nested index. Prefer
+ # the in-scope side (a rename may cross the module boundary),
+ # then apply the same source/output filter as root indexing.
+ path = self._within(item.b_path) or self._within(item.a_path) or ""
+ if not path or not _counts_as_change(path, self.repo_path, output_dirs, item):
continue
results.append(self._file_diff_from_item(item))
- untracked = list(repo.untracked_files)
+ untracked = self._untracked_files(repo)
except Exception as exc:
log.warning("working tree diff failed", error=str(exc))
return []
@@ -256,6 +275,32 @@ class ChangeDetector:
symbol_diff=SymbolDiff(added=list(new_parsed.symbols)) if new_parsed else None,
)
+ def _within(self, path: str | None) -> str | None:
+ """*path* (git-root-relative) relative to repo_path, or None if outside it."""
+ if path is None:
+ return None
+ path = path.replace("\\", "/")
+ if not self._prefix:
+ return path
+ if path.startswith(self._prefix + "/"):
+ return path[len(self._prefix) + 1 :]
+ return None
+
+ def _item_in_scope(self, item: object) -> bool:
+ return self._within(item.a_path) is not None or self._within(item.b_path) is not None
+
+ def _untracked_files(self, repo: object) -> list[str]:
+ """Untracked, non-ignored files inside repo_path, relative to it.
+
+ Scoped to the module when it is nested: ``repo.untracked_files`` walks
+ the whole working tree, which for a module of a large repository is
+ most of the cost of detecting one changed file.
+ """
+ if not self._prefix:
+ return list(repo.untracked_files)
+ listing = repo.git.ls_files("--others", "--exclude-standard", "-z", "--", self._prefix)
+ return [p for p in (self._within(raw) for raw in listing.split("\0") if raw) if p]
+
def _file_diff_from_item(self, item: object) -> FileDiff:
"""Build a :class:`FileDiff` from one GitPython ``Diff``.
@@ -265,25 +310,10 @@ class ChangeDetector:
makes the same code correct for a diff whose right-hand side *is* the
working tree.
"""
- status: Literal["added", "deleted", "modified", "renamed"]
- old_path: str | None = None
- new_path: str | None = None
-
- change_type = item.change_type
- if change_type == "A":
- status = "added"
- new_path = item.b_path
- elif change_type == "D":
- status = "deleted"
- old_path = item.a_path
- elif change_type == "R":
- status = "renamed"
- old_path = item.a_path
- new_path = item.b_path
- else:
- status = "modified"
- old_path = item.a_path
- new_path = item.b_path
+ # Paths inside the module, however deep the module sits in the repo.
+ status, old_path, new_path = _classify_change(
+ item.change_type, self._within(item.a_path), self._within(item.b_path)
+ )
path = new_path or old_path or ""
@@ -505,6 +535,7 @@ class ChangeDetector:
from repowise.core.git_repo import Repo
self._repo = Repo(self.repo_path, search_parent_directories=True)
+ self._prefix = _prefix_in_repo(self._repo, self.repo_path)
return self._repo
except Exception as exc:
log.info(
@@ -583,6 +614,38 @@ class ChangeDetector:
)
+_ChangeStatus = Literal["added", "deleted", "modified", "renamed"]
+
+
+def _classify_change(
+ change_type: str, a_path: str | None, b_path: str | None
+) -> tuple[_ChangeStatus, str | None, str | None]:
+ """``(status, old_path, new_path)`` for one git change, paths already in scope."""
+ if change_type == "A":
+ return "added", None, b_path
+ if change_type == "D":
+ return "deleted", a_path, None
+ if change_type != "R":
+ return "modified", a_path, b_path
+ if a_path and b_path:
+ return "renamed", a_path, b_path
+ if b_path:
+ # Moved into the module from outside it.
+ return "added", None, b_path
+ # Moved out of the module.
+ return "deleted", a_path, None
+
+
+def _prefix_in_repo(repo: object, repo_path: Path) -> str:
+ """Where *repo_path* sits inside *repo*'s working tree, POSIX, "" at the root."""
+ try:
+ root = Path(repo.working_tree_dir).resolve()
+ parts = Path(repo_path).resolve().relative_to(root).parts
+ except Exception:
+ return ""
+ return "/".join(parts)
+
+
def has_working_tree_changes(repo_path: Path) -> bool:
"""Whether the working tree holds changes ``HEAD`` does not.
@@ -605,15 +668,24 @@ def has_working_tree_changes(repo_path: Path) -> bool:
except Exception:
return False
try:
+ detector = ChangeDetector(repo_path)
+ detector._repo = repo
+ detector._prefix = _prefix_in_repo(repo, repo_path)
+ items = (
+ repo.head.commit.diff(None, paths=detector._prefix)
+ if detector._prefix
+ else repo.head.commit.diff(None)
+ )
output_dirs: dict[Path, bool] = {}
+ for item in items:
+ if not detector._item_in_scope(item):
+ continue
+ path = detector._within(item.b_path) or detector._within(item.a_path)
+ if path and _counts_as_change(path, repo_path, output_dirs, item):
+ return True
return any(
- _counts_as_change(
- (item.b_path or item.a_path or "").replace("\\", "/"), repo_path, output_dirs, item
- )
- for item in repo.head.commit.diff(None)
- ) or any(
_counts_as_change(p.replace("\\", "/"), repo_path, output_dirs)
- for p in repo.untracked_files
+ for p in detector._untracked_files(repo)
)
except Exception as exc:
log.warning("working tree check failed", path=str(repo_path), error=str(exc))
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/_full_index.py b/packages/core/src/repowise/core/ingestion/git_indexer/_full_index.py
index a0dc7f2a4..04093124f 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/_full_index.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/_full_index.py
@@ -52,11 +52,14 @@ def merge_code_signals(
compute_percentiles(code_results)
-def build_commit_tables(commit_sink: list[dict]) -> tuple[list[dict], list[dict]]:
+def build_commit_tables(
+ commit_sink: list[dict], path_prefix: str = ""
+) -> tuple[list[dict], list[dict]]:
"""Per-commit rows and their file detail from the commit-index walk's parsed diffs.
Empty in rename-tracking mode (no batched commit index). Failure-isolated
- so a change_risk hiccup never breaks file-level git metadata.
+ so a change_risk hiccup never breaks file-level git metadata. *path_prefix*
+ narrows the per-file detail (not the commit rows) to one directory.
"""
if not commit_sink:
return [], []
@@ -64,7 +67,7 @@ def build_commit_tables(commit_sink: list[dict]) -> tuple[list[dict], list[dict]
from .commit_rows import build_commit_file_rows, build_commit_rows
built = build_commit_rows(commit_sink)
- built_files = build_commit_file_rows(commit_sink)
+ built_files = build_commit_file_rows(commit_sink, path_prefix)
except Exception as exc:
logger.debug("commit_rows_build_failed", error=str(exc))
return [], []
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py b/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py
index 225a1d398..a7fa77fd9 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/commit_rows.py
@@ -161,15 +161,25 @@ def _file_rows_for(sha: str, changes: list) -> list[dict]:
return rows
-def build_commit_file_rows(parsed_commits: list[dict]) -> list[dict]:
+def build_commit_file_rows(parsed_commits: list[dict], path_prefix: str = "") -> list[dict]:
"""Build ``git_commit_files`` rows from the same sunk records.
The per-file detail ``build_commit_rows`` aggregates away. Same input, same
walk, so the two must always be built from one batch.
+
+ *path_prefix* keeps only files under that git-root-relative directory. The
+ commit rows keep describing the whole commit; only the per-file detail is
+ narrowed, which is what stops a bulk commit of a large repository from
+ storing a row per file for a directory that indexes a handful of them.
"""
+ prefix = f"{path_prefix}/" if path_prefix else ""
+
+ def _scoped(changes: list) -> list:
+ return [c for c in changes if (c[0] or "").startswith(prefix)] if prefix else changes
+
return [
row
for c in parsed_commits
if c.get("sha")
- for row in _file_rows_for(c["sha"], c.get("changes") or [])
+ for row in _file_rows_for(c["sha"], _scoped(c.get("changes") or []))
]
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
index 46afc0fd8..31d20adb1 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
@@ -419,7 +419,7 @@ class GitIndexer:
# History-tier rows take none of the code signals.
code_results = [meta for meta in results if meta["file_path"] not in history_files]
merge_code_signals(code_results, walk, prior_defects, trace_index)
- commit_rows, commit_file_rows = build_commit_tables(commit_sink)
+ commit_rows, commit_file_rows = build_commit_tables(commit_sink, self._scope_prefix(repo))
duration = time.monotonic() - start
hotspots = sum(1 for m in results if m.get("is_hotspot", False))
@@ -745,6 +745,13 @@ class GitIndexer:
drawn from the same tracked-file set (the parsed files miss .rst,
LICENSE and the like), and the co-change walk reads that set too.
"""
+ # Callers hand over paths relative to the indexed directory; git and the
+ # rows this indexer writes are keyed relative to the repository root.
+ module_prefix = self._module_prefix(repo)
+ if module_prefix:
+ changed_file_paths = [f"{module_prefix}/{p}" for p in changed_file_paths]
+ if all_files:
+ all_files = {f"{module_prefix}/{p}" for p in all_files}
tier_paths = (self._get_tracked_files(repo) or all_files) if all_files else None
history_files = _history_tier_files(
self.repo_path,
@@ -934,7 +941,7 @@ class GitIndexer:
provenance_classifier=self._provenance_classifier(),
)
rows = build_commit_rows(sink)
- files = build_commit_file_rows(sink)
+ files = build_commit_file_rows(sink, self._scope_prefix(repo))
# Only after both succeed: the sink is the caller's list, and a
# partial extend would outlive the failure that caused it.
if file_rows_sink is not None:
@@ -1337,7 +1344,8 @@ class GitIndexer:
def _get_tracked_files(self, repo: Any) -> list[str]:
try:
- output = repo.git.ls_files()
+ prefix = self._scope_prefix(repo)
+ output = repo.git.ls_files("--", prefix) if prefix else repo.git.ls_files()
paths = [f for f in output.splitlines() if f.strip()]
if self._exclude.patterns:
paths = [p for p in paths if not self._exclude.match_file(p)]
@@ -1346,6 +1354,30 @@ class GitIndexer:
logger.warning("Failed to list tracked files", error=str(exc))
return []
+ def _module_prefix(self, repo: Any) -> str:
+ """Where the indexed path sits inside the git working tree ("" at the root).
+
+ Git, and so every key this indexer writes, is relative to the root; the
+ index above it is relative to the indexed directory.
+ """
+ from ..change_detector import _prefix_in_repo
+
+ return _prefix_in_repo(repo, self.repo_path)
+
+ def _scope_prefix(self, repo: Any) -> str:
+ """Directory whose files get git rows, "" for the whole repository.
+
+ An index of a subdirectory records history for that subdirectory only.
+ Without this, indexing one module of a large repository wrote a row per
+ file of the whole repository, and spent most of the git pass on files
+ nothing could ever look up. ``REPOWISE_GIT_SCOPE=repo`` restores the
+ whole-repository behaviour, which keeps co-change partners that live
+ outside the module.
+ """
+ if os.environ.get("REPOWISE_GIT_SCOPE", "module").strip().lower() == "repo":
+ return ""
+ return self._module_prefix(repo)
+
# ------------------------------------------------------------------
# Backward-compatible instance shims
#
diff --git a/packages/core/src/repowise/core/pipeline/commit_health.py b/packages/core/src/repowise/core/pipeline/commit_health.py
index 52a86a026..c428813dc 100644
--- a/packages/core/src/repowise/core/pipeline/commit_health.py
+++ b/packages/core/src/repowise/core/pipeline/commit_health.py
@@ -25,7 +25,9 @@ a later one instead of staying "not analysed" until a full index.
from __future__ import annotations
import asyncio
+import json
import os
+from pathlib import Path
from typing import Any
import structlog
@@ -65,6 +67,19 @@ def budget_seconds(default: float = DEFAULT_BUDGET_SECONDS) -> float:
return _env_number("REPOWISE_COMMIT_HEALTH_BUDGET_SECONDS", default)
+def _is_fast_index(repo_path: str) -> bool:
+ """Whether the index was built with ``--mode fast`` (essential git tier).
+
+ Fast mode exists to skip the expensive per-commit and per-file git signals,
+ so an update on such an index must not buy them back one commit at a time.
+ """
+ try:
+ state = json.loads((Path(repo_path) / ".repowise" / "state.json").read_text("utf-8"))
+ except (OSError, ValueError):
+ return False
+ return isinstance(state, dict) and state.get("git_tier") == "essential"
+
+
def recent_shas(commit_rows: list[dict] | None) -> list[str]:
"""Shas from persisted commit rows, newest first.
@@ -202,7 +217,7 @@ async def refresh_commit_health_after_update(
limit = commit_limit()
nothing = dict.fromkeys(("candidates", "scanned", "skipped", "stored", "findings"), 0)
- if limit <= 0:
+ if limit <= 0 or _is_fast_index(repo_path):
return nothing
try:
async with get_session(session_factory) as session:
diff --git a/packages/core/src/repowise/core/pipeline/persist/prune.py b/packages/core/src/repowise/core/pipeline/persist/prune.py
index ccffba53d..fa441496e 100644
--- a/packages/core/src/repowise/core/pipeline/persist/prune.py
+++ b/packages/core/src/repowise/core/pipeline/persist/prune.py
@@ -201,9 +201,15 @@ class _FileLiveness:
so an update with no deletions never pays for it.
"""
- def __init__(self, repo_path: Any) -> None:
+ def __init__(self, repo_path: Any, *, git_root_keyed: bool = False) -> None:
self._root = Path(repo_path)
self._tracked: frozenset[str] | None = None
+ # Rows written by the git indexer are keyed relative to the git root,
+ # which is not ``_root`` when the index covers a subdirectory. Every
+ # other table is keyed relative to ``_root``, so only a git-keyed
+ # instance resolves paths against the toplevel.
+ self._git_root_keyed = git_root_keyed
+ self._toplevel: Path | bool | None = False
# The same path is asked about once per table it has rows in, and a
# stat under an antivirus scanner is not free.
self._memo: dict[str, bool] = {}
@@ -215,11 +221,21 @@ class _FileLiveness:
self._memo[path] = answer
return answer
+ def _git_toplevel(self) -> Path | None:
+ if self._toplevel is False:
+ from repowise.core.git_refs import toplevel
+
+ top = toplevel(str(self._root))
+ resolved = Path(top).resolve() if top else None
+ self._toplevel = resolved if resolved and resolved != self._root.resolve() else None
+ return self._toplevel # type: ignore[return-value]
+
def _is_live(self, path: str) -> bool:
- if (self._root / path).exists():
+ root = (self._git_toplevel() or self._root) if self._git_root_keyed else self._root
+ if (root / path).exists():
return True
if self._tracked is None:
- self._tracked = _git_tracked_paths(self._root)
+ self._tracked = _git_tracked_paths(root)
return path in self._tracked
@@ -321,13 +337,14 @@ async def prune_deleted_file_rows(
hint = live_hint or set()
liveness = _FileLiveness(repo_path)
+ git_liveness = _FileLiveness(repo_path, git_root_keyed=True)
refusals: list[str] = []
deleted: set[str] = set()
- def _dead(persisted: set[str], label: str) -> list[str]:
+ def _dead(persisted: set[str], label: str, live: _FileLiveness = liveness) -> list[str]:
"""Paths in *persisted* that no longer exist, or [] if the floor fired."""
persisted = {p for p in persisted if not _is_synthetic_node(p)}
- dead = [p for p in persisted if p not in hint and not liveness.is_live(p)]
+ dead = [p for p in persisted if p not in hint and not live.is_live(p)]
if (
len(dead) > _PRUNE_FLOOR_MIN_ROWS
and persisted
@@ -342,14 +359,16 @@ async def prune_deleted_file_rows(
deleted.update(dead)
return dead
- async def _prune_table(model: Any, column: Any, label: str) -> None:
+ async def _prune_table(
+ model: Any, column: Any, label: str, live: _FileLiveness = liveness
+ ) -> None:
persisted = set(
(await session.execute(select(column).where(model.repository_id == repo_id).distinct()))
.scalars()
.all()
)
persisted.discard(None)
- dead = _dead(persisted, label)
+ dead = _dead(persisted, label, live)
for i in range(0, len(dead), _PRUNE_CHUNK):
await session.execute(
delete(model).where(
@@ -426,6 +445,6 @@ async def prune_deleted_file_rows(
# git_metadata is keyed off the git indexer on a full run, but an
# incremental run only indexes the changed files, so the same liveness
# question is the only authority available here too.
- await _prune_table(GitMetadata, GitMetadata.file_path, "git_metadata")
+ await _prune_table(GitMetadata, GitMetadata.file_path, "git_metadata", git_liveness)
return len(deleted), refusals
diff --git a/tests/unit/analysis/test_commit_scan_bulk_commits.py b/tests/unit/analysis/test_commit_scan_bulk_commits.py
new file mode 100644
index 000000000..dbc9eac1b
--- /dev/null
+++ b/tests/unit/analysis/test_commit_scan_bulk_commits.py
@@ -0,0 +1,137 @@
+"""A bulk commit must not be able to stall the commit-health scan.
+
+The scan's wall-clock budget is only checked between commits, so one commit
+that touches tens of thousands of files (an initial import) ran for minutes
+regardless of it. Such commits are skipped, and a fast index (essential git
+tier) does not run the scan on update at all.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+
+import git as gitpython
+import pytest
+
+from repowise.core.analysis.change_health.commit_scan import (
+ changed_file_count,
+ max_files_per_commit,
+ scan_commits,
+)
+from repowise.core.analysis.change_health.service import MAX_CHANGED_FILES
+from repowise.core.pipeline.commit_health import (
+ _is_fast_index,
+ refresh_commit_health_after_update,
+)
+
+
+def repo(tmp_path: Path) -> tuple[Path, list[str]]:
+ """Three commits: a 5-file root, a 1-file change, and a merge."""
+ r = gitpython.Repo.init(tmp_path, initial_branch="main")
+ with r.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+ names = [f"f{i}.py" for i in range(5)]
+ for n in names:
+ (tmp_path / n).write_text("x = 1\n", encoding="utf-8")
+ r.index.add(names)
+ root = r.index.commit("bulk root")
+ (tmp_path / "f0.py").write_text("x = 2\n", encoding="utf-8")
+ r.index.add(["f0.py"])
+ small = r.index.commit("small")
+ r.close()
+ return tmp_path, [root.hexsha, small.hexsha]
+
+
+def test_changed_file_count_covers_root_and_ordinary_commits(repo) -> None:
+ path, (root, small) = repo
+
+ assert changed_file_count(str(path), root) == 5
+ assert changed_file_count(str(path), small) == 1
+
+
+def test_changed_file_count_is_none_when_git_cannot_answer(tmp_path: Path) -> None:
+ assert changed_file_count(str(tmp_path / "missing"), "a" * 40) is None
+
+
+class _Service:
+ def __init__(self) -> None:
+ self.seen: list[str] = []
+
+ def compare(self, request):
+ self.seen.append(request.revspec)
+ raise RuntimeError("not under test")
+
+
+def test_a_commit_over_the_cap_is_skipped_without_analysis(repo, monkeypatch) -> None:
+ path, (root, small) = repo
+ monkeypatch.setenv("REPOWISE_COMMIT_HEALTH_MAX_FILES", "3")
+ service = _Service()
+
+ scan = scan_commits(str(path), [root, small], service=service)
+
+ assert service.seen == [small]
+ assert scan.skipped == 2 # the bulk root by the cap, the small one by the stub's error
+
+
+def test_the_cap_can_be_disabled(repo, monkeypatch) -> None:
+ path, (root, small) = repo
+ monkeypatch.setenv("REPOWISE_COMMIT_HEALTH_MAX_FILES", "0")
+ service = _Service()
+
+ scan_commits(str(path), [root, small], service=service)
+
+ assert service.seen == [root, small]
+
+
+ ("raw", "expected"),
+ [("nope", MAX_CHANGED_FILES), ("-4", MAX_CHANGED_FILES), ("12", 12)],
+)
+def test_cap_override_ignores_junk(monkeypatch, raw: str, expected: int) -> None:
+ monkeypatch.setenv("REPOWISE_COMMIT_HEALTH_MAX_FILES", raw)
+
+ assert max_files_per_commit() == expected
+
+
+ ("state", "fast"),
+ [
+ ({"git_tier": "essential"}, True),
+ ({"git_tier": "full"}, False),
+ ({}, False),
+ ("not-an-object", False),
+ ],
+)
+def test_fast_index_is_read_from_state(tmp_path: Path, state, fast: bool) -> None:
+ (tmp_path / ".repowise").mkdir()
+ (tmp_path / ".repowise" / "state.json").write_text(json.dumps(state), encoding="utf-8")
+
+ assert _is_fast_index(str(tmp_path)) is fast
+
+
+def test_a_missing_or_corrupt_state_file_is_not_a_fast_index(tmp_path: Path) -> None:
+ assert _is_fast_index(str(tmp_path)) is False
+ (tmp_path / ".repowise").mkdir()
+ (tmp_path / ".repowise" / "state.json").write_text("{broken", encoding="utf-8")
+ assert _is_fast_index(str(tmp_path)) is False
+
+
+def test_an_update_on_a_fast_index_never_opens_a_session(tmp_path: Path) -> None:
+ (tmp_path / ".repowise").mkdir()
+ (tmp_path / ".repowise" / "state.json").write_text(
+ json.dumps({"git_tier": "essential"}), encoding="utf-8"
+ )
+ opened: list[bool] = []
+
+ def factory():
+ opened.append(True)
+ raise AssertionError("a fast index must not be scanned")
+
+ stats = asyncio.run(refresh_commit_health_after_update(factory, "repo", str(tmp_path)))
+
+ assert opened == []
+ assert stats["scanned"] == 0
diff --git a/tests/unit/ingestion/test_change_detector_nested_module.py b/tests/unit/ingestion/test_change_detector_nested_module.py
new file mode 100644
index 000000000..daad6255d
--- /dev/null
+++ b/tests/unit/ingestion/test_change_detector_nested_module.py
@@ -0,0 +1,144 @@
+"""Change detection when the indexed path is a subdirectory of the git repo.
+
+Git reports paths relative to the repository root, while the index is keyed
+relative to the directory that was indexed. For a module inside a larger
+repository the two differ by a prefix, and an update that kept git's form
+matched nothing in the index: it reported success and persisted no change.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import git as gitpython
+import pytest
+
+from repowise.core.ingestion.change_detector import (
+ ChangeDetector,
+ has_working_tree_changes,
+)
+
+MODULE = "svc/module"
+
+
+def repo(tmp_path: Path) -> Path:
+ """A git repo whose indexed module lives two directories down."""
+ r = gitpython.Repo.init(tmp_path)
+ with r.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+ (tmp_path / MODULE).mkdir(parents=True)
+ (tmp_path / "other").mkdir()
+ (tmp_path / MODULE / "main.py").write_text("def alpha():\n return 1\n", encoding="utf-8")
+ (tmp_path / "other" / "x.py").write_text("def outside():\n return 0\n", encoding="utf-8")
+ r.index.add([f"{MODULE}/main.py", "other/x.py"])
+ r.index.commit("init")
+ r.close()
+ return tmp_path
+
+
+def _paths(diffs) -> set[str]:
+ return {d.path for d in diffs}
+
+
+def test_working_tree_paths_are_relative_to_the_module(repo: Path) -> None:
+ module = repo / MODULE
+ (module / "main.py").write_text("def alpha():\n return 2\n", encoding="utf-8")
+
+ diffs = ChangeDetector(module).get_working_tree_changes()
+
+ assert _paths(diffs) == {"main.py"}
+ assert diffs[0].status == "modified"
+ assert diffs[0].new_parsed is not None
+ assert diffs[0].new_parsed.file_info.path == "main.py"
+
+
+def test_changes_outside_the_module_are_ignored(repo: Path) -> None:
+ (repo / "other" / "x.py").write_text("def outside():\n return 9\n", encoding="utf-8")
+ (repo / "other" / "new.py").write_text("def fresh():\n return 1\n", encoding="utf-8")
+
+ detector = ChangeDetector(repo / MODULE)
+
+ assert detector.get_working_tree_changes() == []
+ assert has_working_tree_changes(repo / MODULE) is False
+
+
+def test_repowe_generated_editor_file_inside_module_is_ignored(repo: Path) -> None:
+ module = repo / MODULE
+ from repowise.core.generation.editor_files.base import BaseEditorFileGenerator
+ from repowise.core.generation.editor_files.claude_md import ClaudeMdGenerator
+
+ generated = module / ".claude" / "CLAUDE.md"
+ generated.parent.mkdir(parents=True)
+ tag = ClaudeMdGenerator.marker_tag
+ generated.write_text(
+ ClaudeMdGenerator.user_placeholder
+ + BaseEditorFileGenerator.MARKER_START_FMT.format(tag=tag)
+ + "\nLast indexed: today\n"
+ + BaseEditorFileGenerator.MARKER_END_FMT.format(tag=tag)
+ + "\n",
+ encoding="utf-8",
+ )
+
+ detector = ChangeDetector(module)
+
+ assert detector.get_working_tree_changes() == []
+ assert has_working_tree_changes(module) is False
+
+
+def test_untracked_files_inside_the_module_are_relative_to_it(repo: Path) -> None:
+ module = repo / MODULE
+ (module / "pkg").mkdir()
+ (module / "pkg" / "added.py").write_text("def added():\n return 1\n", encoding="utf-8")
+
+ diffs = ChangeDetector(module).get_working_tree_changes()
+
+ assert _paths(diffs) == {"pkg/added.py"}
+ assert diffs[0].status == "added"
+ assert has_working_tree_changes(module) is True
+
+
+def test_commit_range_diff_is_scoped_and_relative(repo: Path) -> None:
+ r = gitpython.Repo(repo)
+ (repo / MODULE / "main.py").write_text("def alpha():\n return 3\n", encoding="utf-8")
+ (repo / "other" / "x.py").write_text("def outside():\n return 4\n", encoding="utf-8")
+ r.index.add([f"{MODULE}/main.py", "other/x.py"])
+ r.index.commit("touch both")
+ r.close()
+
+ diffs = ChangeDetector(repo / MODULE).get_changed_files("HEAD~1", "HEAD")
+
+ assert _paths(diffs) == {"main.py"}
+
+
+def test_repo_root_behaviour_is_unchanged(repo: Path) -> None:
+ (repo / "other" / "x.py").write_text("def outside():\n return 5\n", encoding="utf-8")
+
+ assert _paths(ChangeDetector(repo).get_working_tree_changes()) == {"other/x.py"}
+
+
+def test_a_sibling_sharing_the_prefix_is_outside_the_module(tmp_path: Path) -> None:
+ # Indexing svc/mod must not pick up svc/module, whose path starts with it.
+ r = gitpython.Repo.init(tmp_path)
+ with r.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+ for d in ("svc/mod", "svc/module"):
+ (tmp_path / d).mkdir(parents=True)
+ (tmp_path / d / "a.py").write_text("def a():\n return 1\n", encoding="utf-8")
+ r.index.add(["svc/mod/a.py", "svc/module/a.py"])
+ r.index.commit("init")
+ (tmp_path / "svc/module/a.py").write_text("def a():\n return 2\n", encoding="utf-8")
+ r.index.add(["svc/module/a.py"])
+ r.index.commit("touch the sibling")
+ r.close()
+
+ (tmp_path / "svc/module/a.py").write_text("def a():\n return 3\n", encoding="utf-8")
+ (tmp_path / "svc/module/new.py").write_text("def n():\n return 1\n", encoding="utf-8")
+ module = tmp_path / "svc" / "mod"
+ detector = ChangeDetector(module)
+
+ assert detector.get_working_tree_changes() == []
+ assert detector.get_changed_files("HEAD~1", "HEAD") == []
+ assert has_working_tree_changes(module) is False
diff --git a/tests/unit/ingestion/test_git_indexer_module_scope.py b/tests/unit/ingestion/test_git_indexer_module_scope.py
new file mode 100644
index 000000000..8481a203e
--- /dev/null
+++ b/tests/unit/ingestion/test_git_indexer_module_scope.py
@@ -0,0 +1,94 @@
+"""Git rows for an index of a subdirectory cover that subdirectory only.
+
+Indexing one module of a large repository used to write a git row for every
+tracked file of the whole repository, plus a commit-file row per file of every
+bulk commit. For a 22-file module of a 40,000-file repository that was about
+35 MiB of a 64 MiB database, for files nothing could look up.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import git as gitpython
+import pytest
+
+from repowise.core.ingestion.git_indexer import GitIndexer
+from repowise.core.ingestion.git_indexer.commit_rows import build_commit_file_rows
+
+MODULE = "svc/module"
+
+
+def repo(tmp_path: Path) -> Path:
+ """One bulk commit over the module and a sibling tree, then a module-only edit."""
+ r = gitpython.Repo.init(tmp_path)
+ with r.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+ (tmp_path / MODULE).mkdir(parents=True)
+ (tmp_path / "other").mkdir()
+ names = [f"{MODULE}/a.py", f"{MODULE}/b.py"]
+ names += [f"other/o{i}.py" for i in range(6)]
+ for n in names:
+ (tmp_path / n).write_text("x = 1\n", encoding="utf-8")
+ r.index.add(names)
+ r.index.commit("bulk import")
+ (tmp_path / MODULE / "a.py").write_text("x = 2\n", encoding="utf-8")
+ r.index.add([f"{MODULE}/a.py"])
+ r.index.commit("edit module")
+ r.close()
+ return tmp_path
+
+
+def _paths(rows: list[dict]) -> set[str]:
+ return {r["file_path"] for r in rows}
+
+
+async def test_a_module_index_records_git_rows_for_the_module_only(repo: Path) -> None:
+ summary, meta = await GitIndexer(repo / MODULE).index_repo("module")
+
+ assert _paths(meta) == {f"{MODULE}/a.py", f"{MODULE}/b.py"}
+ assert _paths(summary.commit_file_rows) == {f"{MODULE}/a.py", f"{MODULE}/b.py"}
+
+
+async def test_commit_rows_still_describe_the_whole_commit(repo: Path) -> None:
+ summary, _ = await GitIndexer(repo / MODULE).index_repo("module")
+
+ assert len(summary.commit_rows) == 2
+
+
+async def test_the_repo_scope_switch_restores_whole_repository_rows(
+ repo: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setenv("REPOWISE_GIT_SCOPE", "repo")
+
+ _, meta = await GitIndexer(repo / MODULE).index_repo("module")
+
+ assert len(_paths(meta)) == 8
+
+
+async def test_indexing_the_repo_root_is_unchanged(repo: Path) -> None:
+ _, meta = await GitIndexer(repo).index_repo("repo")
+
+ assert len(_paths(meta)) == 8
+
+
+async def test_an_update_maps_module_paths_back_to_the_repo_key(repo: Path) -> None:
+ meta = await GitIndexer(repo / MODULE).index_changed_files(["a.py"])
+
+ assert [m["file_path"] for m in meta] == [f"{MODULE}/a.py"]
+ # History was found: the path handed to git is the root-relative one.
+ assert meta[0]["commit_count_total"] == 2
+
+
+def test_file_rows_can_be_narrowed_to_a_prefix() -> None:
+ commits = [
+ {
+ "sha": "a" * 40,
+ "changes": [(f"{MODULE}/a.py", 1, 0), ("other/x.py", 2, 0), (None, 0, 0)],
+ }
+ ]
+
+ assert _paths(build_commit_file_rows(commits, MODULE)) == {f"{MODULE}/a.py"}
+ assert len(build_commit_file_rows(commits)) == 2
diff --git a/tests/unit/pipeline/test_file_liveness_nested.py b/tests/unit/pipeline/test_file_liveness_nested.py
new file mode 100644
index 000000000..6b962a7df
--- /dev/null
+++ b/tests/unit/pipeline/test_file_liveness_nested.py
@@ -0,0 +1,70 @@
+"""The deleted-file prune must not read a nested index as a mass deletion.
+
+Git-metadata rows are keyed relative to the git root. When the index covers a
+subdirectory, asking only "is this path under the indexed directory?" made
+every one of them look deleted, so the prune refused the run and reported a
+degraded update on every update.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import git as gitpython
+import pytest
+
+from repowise.core.pipeline.persist import _FileLiveness
+
+
+def nested(tmp_path: Path) -> Path:
+ r = gitpython.Repo.init(tmp_path)
+ with r.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+ (tmp_path / "svc" / "module").mkdir(parents=True)
+ (tmp_path / "other").mkdir()
+ (tmp_path / "svc" / "module" / "a.py").write_text("a = 1\n", encoding="utf-8")
+ (tmp_path / "other" / "b.py").write_text("b = 1\n", encoding="utf-8")
+ r.index.add(["svc/module/a.py", "other/b.py"])
+ r.index.commit("init")
+ r.close()
+ return tmp_path
+
+
+def test_a_root_relative_path_that_exists_is_live(nested: Path) -> None:
+ liveness = _FileLiveness(nested / "svc" / "module", git_root_keyed=True)
+
+ assert liveness.is_live("svc/module/a.py")
+ assert liveness.is_live("other/b.py")
+
+
+def test_a_module_relative_path_is_still_live(nested: Path) -> None:
+ assert _FileLiveness(nested / "svc" / "module").is_live("a.py")
+
+
+def test_a_genuinely_missing_path_is_still_dead(nested: Path) -> None:
+ assert not _FileLiveness(nested / "svc" / "module").is_live("gone.py")
+ git_keyed = _FileLiveness(nested / "svc" / "module", git_root_keyed=True)
+ assert not git_keyed.is_live("svc/module/gone.py")
+
+
+def test_an_index_relative_path_is_not_resolved_against_the_git_root(nested: Path) -> None:
+ # A deleted module file must not look live because the repo root has a
+ # file of the same name.
+ (nested / "a.py").write_text("root = 1\n", encoding="utf-8")
+ (nested / "svc" / "module" / "a.py").unlink()
+ r = gitpython.Repo(nested)
+ r.git.rm("--cached", "-q", "svc/module/a.py")
+ r.close()
+
+ assert not _FileLiveness(nested / "svc" / "module").is_live("a.py")
+ git_keyed = _FileLiveness(nested / "svc" / "module", git_root_keyed=True)
+ assert not git_keyed.is_live("svc/module/a.py")
+ assert git_keyed.is_live("a.py")
+
+
+def test_the_repo_root_behaves_as_before(nested: Path) -> None:
+ for liveness in (_FileLiveness(nested), _FileLiveness(nested, git_root_keyed=True)):
+ assert liveness.is_live("other/b.py")
+ assert not liveness.is_live("nope.py")