Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf: XL-repo indexing pass (cpp hints regex, git deep walk, xaml index reuse) (#459)

Raghav Chamadiyaec6a97bb5d45
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
index 6930fd55e..5619bc46a 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/base.py
@@ -26,6 +26,11 @@ class DynamicHintExtractor(ABC):
# extractor use in tests keeps working unchanged).
_walk_snapshot = None
+ # Prebuilt DotNetProjectIndex, attached by HintRegistry.extract_all when
+ # the caller already built one (the graph resolvers do, on every C# repo).
+ # None -> extractors that need it build their own, as before.
+ _dotnet_index = None
+
@abstractmethod
def extract(self, repo_root: Path) -> list[DynamicEdge]: ...
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/cpp.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/cpp.py
index a52db9bd2..06a9695d6 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/cpp.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/cpp.py
@@ -55,6 +55,48 @@ _FUNC_DEF_RE = re.compile(
re.MULTILINE,
)
+# Statement / block delimiters. No component of _FUNC_DEF_RE can match
+# across a ``;``, ``{`` or ``}``: the prefix class, the ``[^;{}]*`` arg
+# span, the whitespace runs and the trailing-return class all exclude
+# them. The single exception is the ``noexcept\(([^)]*)\)`` argument
+# span, handled by the fallback below.
+_DELIM_RE = re.compile(r"[;{}]")
+
+# A _FUNC_DEF_RE match can only cross a delimiter through noexcept
+# arguments that themselves contain ``;``/``{``/``}``. Files containing
+# that (pathological, never seen in real code) take the full-text scan.
+_NOEXCEPT_HOLE_RE = re.compile(r"noexcept\s*\([^)]*[;{}]")
+
+
+def _iter_func_defs(text: str):
+ """Yield ``_FUNC_DEF_RE`` matches via delimiter-windowed search.
+
+ Equivalent to ``_FUNC_DEF_RE.finditer(text)`` but immune to the
+ pattern's catastrophic backtracking on long runs of prefix-class
+ characters (large initializer lists, expression-template headers),
+ which cost tens of seconds per file at WinUI-monorepo scale.
+
+ Every match ends at a ``{`` and, delimiter-crossing being impossible
+ (see ``_DELIM_RE``), lies entirely between the previous delimiter
+ and that brace. Searching the original pattern window-by-window with
+ ``pattern.search(text, pos, endpos)`` preserves ``^``/anchor
+ semantics against the full string, so each windowed hit is a
+ verbatim full-text match, at most one per window (one ``{``).
+ Windows without a ``(`` cannot satisfy the mandatory arg span and
+ are skipped outright.
+ """
+ if _NOEXCEPT_HOLE_RE.search(text):
+ yield from _FUNC_DEF_RE.finditer(text)
+ return
+ pos = 0
+ for dm in _DELIM_RE.finditer(text):
+ d = dm.start()
+ if text[d] == "{" and text.find("(", pos, d) != -1:
+ m = _FUNC_DEF_RE.search(text, pos, d + 1)
+ if m is not None:
+ yield m
+ pos = d + 1
+
class CppDynamicHints(DynamicHintExtractor):
"""Discover C++ function-pointer assignments, dlopen/dlsym, Qt connect."""
@@ -81,7 +123,7 @@ class CppDynamicHints(DynamicHintExtractor):
continue
rel = rel_path.as_posix()
sources.append((src, text, rel))
- for match in _FUNC_DEF_RE.finditer(text):
+ for match in _iter_func_defs(text):
func_to_files.setdefault(match.group(1), []).append(rel)
for _src, text, rel in sources:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
index 3722f6910..1a0fcf70a 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/registry.py
@@ -77,13 +77,21 @@ class HintRegistry:
]
self._max_workers = max_workers or min(_DEFAULT_MAX_WORKERS, len(self._extractors))
- def extract_all(self, repo_root: Path) -> list[DynamicEdge]:
+ def extract_all(
+ self, repo_root: Path, *, dotnet_index: object | None = None
+ ) -> list[DynamicEdge]:
"""Run every registered extractor and merge their edges.
Extractors run in a thread pool — their work is I/O-bound and
releases the GIL during filesystem reads, so threads are the
right tool (no per-process startup overhead, no pickling).
The returned list is deterministically sorted.
+
+ *dotnet_index* is an optional prebuilt
+ :class:`~repowise.core.ingestion.resolvers.dotnet.DotNetProjectIndex`.
+ The XAML extractor needs the repo's type map and otherwise rebuilds
+ the index from scratch (a full .cs walk + read + scan); passing the
+ one the graph resolvers already built skips that duplicate work.
"""
edges: list[DynamicEdge] = []
if not self._extractors:
@@ -103,6 +111,7 @@ class HintRegistry:
log.warning("dynamic_hints_snapshot_failed", error=str(e))
for ex in self._extractors:
ex._walk_snapshot = snapshot
+ ex._dotnet_index = dotnet_index
try:
with ThreadPoolExecutor(max_workers=self._max_workers) as pool:
@@ -121,6 +130,7 @@ class HintRegistry:
finally:
for ex in self._extractors:
ex._walk_snapshot = None
+ ex._dotnet_index = None
# Thread-completion order varies run-to-run; sort so downstream graph
# construction (and therefore the exported KG) stays deterministic.
edges.sort(key=lambda e: (e.source, e.target, e.edge_type, e.hint_source, e.weight))
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py
index 531824059..8602b6878 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/xaml.py
@@ -100,10 +100,14 @@ class XamlDynamicHints(DynamicHintExtractor):
return []
# Reuse the existing C# type index — it already walks every .cs
- # file under every .csproj and dedupes builtins. Building it
- # here keeps XAML resolution cross-project / cross-repo
- # consistent with how `using` directives resolve.
- type_map = _load_type_map(repo_root)
+ # file under every .csproj and dedupes builtins. Reusing it keeps
+ # XAML resolution cross-project / cross-repo consistent with how
+ # `using` directives resolve. The registry attaches the index the
+ # graph resolvers built; standalone use rebuilds it from disk.
+ if self._dotnet_index is not None:
+ type_map = self._dotnet_index.type_map
+ else:
+ type_map = _load_type_map(repo_root)
# Even without a .NET project we can still resolve xaml→xaml
# ResourceDictionary references, so don't early-exit on an empty
# type_map — only the C# binding pass is gated on it.
diff --git a/packages/core/src/repowise/core/ingestion/git_commit_index.py b/packages/core/src/repowise/core/ingestion/git_commit_index.py
index 5fb248abb..8671d8d0e 100644
--- a/packages/core/src/repowise/core/ingestion/git_commit_index.py
+++ b/packages/core/src/repowise/core/ingestion/git_commit_index.py
@@ -212,3 +212,146 @@ def load_commit_index(
indexable_files=len(indexable_files),
)
return bucket
+
+
+def load_deep_commit_index(
+ repo: object,
+ per_file_limit: int,
+ wanted_files: set[str],
+ *,
+ skip: int,
+ deep_limit: int,
+ provenance_classifier: object | None = None,
+) -> dict[str, list[_CommitRec]]:
+ """Bucket commits OLDER than the recent window for *wanted_files* only.
+
+ Walks ``git log --skip=<skip> -<deep_limit>`` — the region strictly
+ beyond what :func:`load_commit_index` parsed (``--skip`` counts after
+ ``--no-merges`` filtering, exactly like the window walk's ``-N`` cap,
+ so the two regions partition the history with no gap or overlap).
+
+ *wanted_files* is the set of indexable paths the window index missed;
+ only their commits are retained, newest first, capped at
+ *per_file_limit* per file — mirroring the per-file fallback's
+ ``git log -<limit> -- <file>`` cap. One subprocess replaces one
+ fallback spawn per missed file, which dominates the git phase on
+ repos whose history is much deeper than the window (a 9k-commit
+ monorepo left 3,295 of 4,857 files to the fallback).
+
+ The records differ from the per-file fallback in one documented way:
+ churn comes from the repo-wide diff (rename rows attribute edit churn
+ through the rename) rather than the pathspec-limited diff (which
+ shows a rename as a whole-file addition), and simplification-rare
+ merge commits never appear (``--no-merges``, like the window walk).
+ That makes deep-bucketed files CONSISTENT with window-indexed files,
+ which always had repo-walk semantics. Files absent from this bucket
+ (pre-rename names the marker parser cannot resolve, or history deeper
+ than *deep_limit*) keep the per-file fallback path.
+
+ Failures return an empty dict; every missed file then falls back to
+ the per-file path exactly as before.
+ """
+ from .git_indexer import (
+ _LOG_FORMAT,
+ _RECORD_SEP,
+ _CommitRec,
+ _extract_rename_paths,
+ _parse_commit_record,
+ )
+ from .git_indexer.agent_provenance import AgentProvenanceClassifier
+
+ if not wanted_files:
+ return {}
+ if provenance_classifier is None:
+ provenance_classifier = AgentProvenanceClassifier()
+
+ try:
+ raw = repo.git.log( # type: ignore[attr-defined]
+ f"--skip={skip}",
+ f"-{deep_limit}",
+ "--numstat",
+ "--no-merges",
+ f"--format={_LOG_FORMAT}",
+ )
+ except Exception as exc:
+ logger.warning("deep_commit_index_failed", error=str(exc))
+ return {}
+
+ if not raw:
+ return {}
+
+ bucket: dict[str, list[_CommitRec]] = {}
+ commits_parsed = 0
+
+ for record in raw.split(_RECORD_SEP):
+ if not record.strip():
+ continue
+ parsed = _parse_commit_record(record)
+ if parsed is None:
+ continue
+ header, numstat_lines = parsed
+ commits_parsed += 1
+
+ # Classified lazily — only commits that actually touch a wanted
+ # file pay the provenance regexes (the window walk classifies
+ # every commit because the commit sink needs the labels).
+ prov = None
+
+ for line in numstat_lines:
+ cols = line.split("\t")
+ if len(cols) < 3:
+ continue
+ stat_path = cols[2]
+ if "=>" in stat_path:
+ seen: set[str] = set()
+ _old_path, new_path = _extract_rename_paths(stat_path, seen)
+ target = new_path or stat_path
+ else:
+ target = stat_path
+
+ if target not in wanted_files:
+ continue
+ records = bucket.setdefault(target, [])
+ if len(records) >= per_file_limit:
+ continue
+
+ try:
+ added = int(cols[0]) if cols[0] != "-" else 0
+ deleted = int(cols[1]) if cols[1] != "-" else 0
+ except ValueError:
+ added = 0
+ deleted = 0
+
+ if prov is None:
+ prov = provenance_classifier.classify( # type: ignore[attr-defined]
+ header["author_name"],
+ header["author_email"],
+ header["committer_name"],
+ header["committer_email"],
+ f"{header['subject']}\n{header['body']}",
+ )
+
+ records.append(
+ _CommitRec(
+ sha=header["sha"],
+ author_name=header["author_name"],
+ author_email=header["author_email"],
+ ts=header["ts"],
+ is_merge=header["is_merge"],
+ subject=header["subject"],
+ body=header["body"],
+ added=added,
+ deleted=deleted,
+ agent=prov.agent,
+ agent_tier=prov.autonomy_tier,
+ )
+ )
+
+ logger.debug(
+ "deep_commit_index_built",
+ commits_parsed=commits_parsed,
+ files_bucketed=len(bucket),
+ wanted_files=len(wanted_files),
+ skip=skip,
+ )
+ return bucket
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py b/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py
index 0c4ed2929..7ba1516eb 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/_constants.py
@@ -63,6 +63,21 @@ _MIN_MESSAGE_LEN = 12
# Default per-file commit history depth.
_DEFAULT_COMMIT_LIMIT: int = 500
+# Depth of the second, deeper repo-wide log walk that serves files absent
+# from the recent-window commit index. On repos whose history is much
+# deeper than _DEFAULT_COMMIT_LIMIT, most tracked files miss the window
+# and previously each took a per-file ``git log`` subprocess (3,295
+# spawns on a 9k-commit WinUI monorepo). One --skip walk this deep
+# replaces them. Files older than window+deep still take the per-file
+# path, so behaviour degrades gracefully on very deep histories.
+_DEEP_WALK_COMMIT_LIMIT: int = 20_000
+
+# Minimum number of window-index misses before the deep walk pays for
+# itself. Below this, the handful of per-file ``git log`` fallbacks is
+# cheaper than walking the full history again (small repos typically
+# have zero misses and skip the deep walk entirely).
+_DEEP_WALK_MIN_FALLBACK: int = 100
+
# Per-file persisted contributor / commit fan-out. Previously hard-coded
# to 5 / 10 inline, which silently hid co-owners and meaningful history on
# multi-team modules. 50 is generous enough that any realistic UI surface
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
index 473e0abaa..748440c45 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
@@ -21,6 +21,8 @@ from typing import Any
import structlog
from ._constants import (
+ _DEEP_WALK_COMMIT_LIMIT,
+ _DEEP_WALK_MIN_FALLBACK,
_DEFAULT_CO_CHANGE_COMMIT_LIMIT,
_DEFAULT_CO_CHANGE_MIN_COUNT,
_DEFAULT_COMMIT_LIMIT,
@@ -122,8 +124,9 @@ class GitIndexer:
commit_index: dict[str, list[_CommitRec]] = {}
commit_sink: list[dict] = []
prov_clf = self._provenance_classifier()
+ deep_index: dict[str, list[_CommitRec]] = {}
if not self.follow_renames:
- from ..git_commit_index import load_commit_index
+ from ..git_commit_index import load_commit_index, load_deep_commit_index
commit_index = load_commit_index(
repo,
@@ -133,6 +136,21 @@ class GitIndexer:
provenance_classifier=prov_clf,
)
+ # Files the recent window never saw would each spawn a per-file
+ # ``git log`` fallback below. When there are many (deep-history
+ # repos), one --skip walk over the older region replaces them;
+ # files it still misses keep the per-file path.
+ missed = {fp for fp in indexable_files if fp not in commit_index}
+ if len(missed) >= _DEEP_WALK_MIN_FALLBACK:
+ deep_index = load_deep_commit_index(
+ repo,
+ self.commit_limit,
+ missed,
+ skip=self.commit_limit,
+ deep_limit=_DEEP_WALK_COMMIT_LIMIT,
+ provenance_classifier=prov_clf,
+ )
+
include_blame = self.tier.includes_blame
as_of_ts = self._resolve_as_of_ts(repo, commit_index)
get_thread_repo, close_thread_repos = self._thread_repo_pool()
@@ -148,6 +166,8 @@ class GitIndexer:
try:
thread_repo = get_thread_repo()
precomputed = commit_index.get(file_path) if commit_index else None
+ if precomputed is None:
+ precomputed = deep_index.get(file_path)
return index_file(
thread_repo,
file_path,
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/records.py b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
index b6050f70c..aaa7f4198 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/records.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/records.py
@@ -131,7 +131,7 @@ class GitIndexSummary:
commit_rows: list[dict] = field(default_factory=list)
-_RENAME_RE = re.compile(r"\{(.+?) => (.+?)\}")
+_RENAME_RE = re.compile(r"\{(.*?) => (.*?)\}")
def _extract_rename_paths(stat_path: str, known_paths: set[str]) -> tuple[str | None, str | None]:
@@ -141,17 +141,22 @@ def _extract_rename_paths(stat_path: str, known_paths: set[str]) -> tuple[str |
10\t5\t{old => new}/shared_suffix
10\t5\told_dir/{old_name => new_name}.py
-
- This helper parses both forms, adds both expanded paths to *known_paths*,
- and returns ``(old_path, new_path)`` so the caller can attribute churn to
- the correct file. Returns ``(None, None)`` if the pattern is not found.
+ 10\t5\tsrc/{ => newdir}/shared_suffix
+
+ The third form has an EMPTY side: a directory inserted into (or removed
+ from) the middle of a path. Expanding the empty side leaves a doubled
+ slash at the splice point, which is collapsed so the result matches the
+ tracked path. This helper parses all forms, adds both expanded paths to
+ *known_paths*, and returns ``(old_path, new_path)`` so the caller can
+ attribute churn to the correct file. Returns ``(None, None)`` if the
+ pattern is not found.
"""
m = _RENAME_RE.search(stat_path)
if m:
prefix = stat_path[: m.start()]
suffix = stat_path[m.end() :]
- old_path = prefix + m.group(1) + suffix
- new_path = prefix + m.group(2) + suffix
+ old_path = (prefix + m.group(1) + suffix).replace("//", "/")
+ new_path = (prefix + m.group(2) + suffix).replace("//", "/")
known_paths.add(old_path)
known_paths.add(new_path)
return old_path, new_path
diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.py
index 1515e12bc..4d2922bd5 100644
--- a/packages/core/src/repowise/core/ingestion/graph/builder.py
+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py
@@ -52,6 +52,9 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
self._graph: nx.DiGraph = nx.DiGraph()
self._parsed_files: dict[str, ParsedFile] = {} # path → ParsedFile
self._built = False
+ # Resolver-built DotNetProjectIndex, stashed by build() for the
+ # dynamic-hints phase to reuse (see build()).
+ self.dotnet_index: Any | None = None
self._repo_path: Path | None = Path(repo_path) if repo_path else None
# Mirrors the traverser flags: when submodules or nested repos are
# indexed, resolver filesystem scans must not prune them (both are
@@ -431,6 +434,15 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
self._built = True
+ # Keep the resolver-built DotNetProjectIndex reachable after the
+ # context goes out of scope: the dynamic-hints phase (XAML) needs
+ # the same type map and otherwise rebuilds the index from disk.
+ # Only stashed when this build pruned nested git repos, because a
+ # standalone rebuild always prunes — the maps must be identical.
+ self.dotnet_index = (
+ getattr(ctx, "_dotnet_index", None) if self._prune_nested_git else None
+ )
+
# Count edge types for logging
edge_counts: dict[str, int] = {}
for _, _, d in self._graph.edges(data=True):
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index 3cbd4e3aa..344965535 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -144,7 +144,9 @@ def build_repo_graph(
try:
from repowise.core.ingestion.dynamic_hints import HintRegistry
- dynamic_edges = HintRegistry().extract_all(Path(repo_path))
+ dynamic_edges = HintRegistry().extract_all(
+ Path(repo_path), dotnet_index=graph_builder.dotnet_index
+ )
graph_builder.add_dynamic_edges(dynamic_edges)
if dynamic_edges:
log(f"Dynamic hint edges added: [cyan]{len(dynamic_edges)}[/cyan]")
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index f0aa3f88c..2f28e6f45 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -377,7 +377,12 @@ async def _run_ingestion(
from repowise.core.ingestion.dynamic_hints import HintRegistry
registry = HintRegistry()
- dynamic_edges = await loop.run_in_executor(None, registry.extract_all, repo_path)
+ dynamic_edges = await loop.run_in_executor(
+ None,
+ lambda: registry.extract_all(
+ repo_path, dotnet_index=graph_builder.dotnet_index
+ ),
+ )
graph_builder.add_dynamic_edges(dynamic_edges)
logger.info("dynamic_hints_added", count=len(dynamic_edges))
except Exception as hints_exc:
diff --git a/tests/unit/ingestion/test_cpp_func_def_window.py b/tests/unit/ingestion/test_cpp_func_def_window.py
new file mode 100644
index 000000000..4c6a40a3a
--- /dev/null
+++ b/tests/unit/ingestion/test_cpp_func_def_window.py
@@ -0,0 +1,137 @@
+"""Equivalence tests for the delimiter-windowed _FUNC_DEF_RE scan.
+
+``_iter_func_defs`` must return exactly what a full-text
+``_FUNC_DEF_RE.finditer`` returns — same spans, same captured names, in
+the same order — on every C++ shape we can think of, including the
+catastrophic-backtracking inputs that motivated the windowing and the
+``noexcept`` argument hole that forces the full-text fallback.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.ingestion.dynamic_hints.cpp import (
+ _FUNC_DEF_RE,
+ _NOEXCEPT_HOLE_RE,
+ _iter_func_defs,
+)
+
+
+def _reference(text: str) -> list[tuple[int, int, str]]:
+ return [(m.start(), m.end(), m.group(1)) for m in _FUNC_DEF_RE.finditer(text)]
+
+
+def _windowed(text: str) -> list[tuple[int, int, str]]:
+ return [(m.start(), m.end(), m.group(1)) for m in _iter_func_defs(text)]
+
+
+CORPUS = {
+ "plain_function": "int add(int a, int b) {\n return a + b;\n}\n",
+ "out_of_class_method": (
+ "ReturnType Class::method(int x) const noexcept {\n return {};\n}\n"
+ ),
+ "multiline_prefix_type": (
+ "std::map<std::string,\n std::vector<int>>\nlookup_table(const Key& k) {\n"
+ " return {};\n}\n"
+ ),
+ "trailing_return": "auto make_widget(int id) -> std::unique_ptr<Widget> {\n return nullptr;\n}\n",
+ "keyword_loose_match": "void f() {\n if (cond) {\n run();\n }\n}\n",
+ "match_at_position_zero": "int main() {\n return 0;\n}\n",
+ "no_functions_at_all": "// just a comment\nconstexpr int kAnswer = 42;\n",
+ "empty_text": "",
+ "windows_without_parens": "namespace foo {\nstruct Bar {\n int x;\n};\n}\n",
+ "paren_but_no_match": "int x = (1 + 2);\nint y = 3;\n",
+ "initializer_run_small": (
+ "static inline const std::vector<LetterKey> letters = { LetterKey::VK_0,\n"
+ + "\n".join(
+ f" LetterKey::VK_{c}," for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ )
+ + "\n LetterKey::VK_BACKSLASH, };\n"
+ "void after_the_array() {\n run();\n}\n"
+ ),
+ "brace_init_members": (
+ "struct Settings {\n"
+ " Key activationKey{ Key::Both };\n"
+ " bool gameMode{ true };\n"
+ " std::chrono::milliseconds inputTime{ 300 };\n"
+ "};\n"
+ "void tick() {\n}\n"
+ ),
+ "same_line_after_semicolon": "foo(); int bar() {\n}\n",
+ "two_defs_same_line": "void a() { x(); } void b() { y(); }\n",
+ "noexcept_plain": "bool on_key(Info info) noexcept {\n return false;\n}\n",
+ "noexcept_with_args": (
+ "bool on_key(Info info) noexcept(std::is_nothrow_constructible_v<Info>) {\n"
+ " return false;\n}\n"
+ ),
+ "noexcept_semicolon_after_paren": (
+ # The ';' sits after the first ')' inside the noexcept args, so the
+ # pattern's own [^)]* cannot cross it either — no fallback needed,
+ # behaviour must still be identical.
+ "bool weird(Info info) noexcept(call(1); 2) {\n return false;\n}\n"
+ ),
+ "noexcept_hole_semicolon": (
+ # ';' BEFORE the first ')' after 'noexcept(' — the only shape where
+ # a match could span a delimiter; must trigger the fallback.
+ "bool weird(Info info) noexcept(a; b) {\n return false;\n}\n"
+ ),
+ "noexcept_hole_brace": (
+ "bool weird(Info info) noexcept(T{}) {\n return false;\n}\n"
+ "void normal_after() {\n run();\n}\n"
+ ),
+ "consecutive_braces": "void f() {{\n nested();\n}}\n",
+ "unbalanced_open_paren": "void broken(int x {\n run();\n}\n",
+ "args_span_lines": (
+ "LRESULT CALLBACK LowLevelKeyboardProc(int nCode,\n"
+ " WPARAM wParam,\n"
+ " LPARAM lParam) {\n"
+ " return 0;\n}\n"
+ ),
+ "operator_like_assignment": "auto v = transform(input);\nvoid g() {\n h();\n}\n",
+ "class_with_methods": (
+ "class Widget {\n"
+ "public:\n"
+ " void render(Canvas& c) override {\n c.draw();\n }\n"
+ " int size() const {\n return n_;\n }\n"
+ "private:\n"
+ " int n_;\n"
+ "};\n"
+ ),
+}
+
+
[email protected]("label", sorted(CORPUS))
+def test_windowed_matches_reference(label: str) -> None:
+ text = CORPUS[label]
+ assert _windowed(text) == _reference(text)
+
+
+def test_noexcept_hole_takes_fallback_path() -> None:
+ text = CORPUS["noexcept_hole_semicolon"]
+ assert _NOEXCEPT_HOLE_RE.search(text) is not None
+ assert _windowed(text) == _reference(text)
+
+
+def test_concatenated_corpus_matches_reference() -> None:
+ # One big file mixing every shape; exercises window bookkeeping
+ # across many delimiters and the pos advancement after each match.
+ text = "\n".join(CORPUS[k] for k in sorted(CORPUS))
+ assert _windowed(text) == _reference(text)
+
+
+def test_pathological_run_is_fast_and_equivalent() -> None:
+ # The KeyboardListener.h shape: a multi-KB run of prefix-class chars
+ # (identifiers, ::, commas, whitespace) with no parens. The full-text
+ # scan blows up combinatorially; windowed must stay instant and equal.
+ run = "static inline const std::vector<LetterKey> letters = {\n" + "".join(
+ f" LetterKey::VK_{i},\n" for i in range(250)
+ ) + "};\n"
+ text = run + "void after(int x) {\n run();\n}\n"
+ import time
+
+ t0 = time.monotonic()
+ got = _windowed(text)
+ elapsed = time.monotonic() - t0
+ assert got == _reference(text)
+ assert elapsed < 1.0
diff --git a/tests/unit/ingestion/test_git_deep_commit_index.py b/tests/unit/ingestion/test_git_deep_commit_index.py
new file mode 100644
index 000000000..178099e13
--- /dev/null
+++ b/tests/unit/ingestion/test_git_deep_commit_index.py
@@ -0,0 +1,188 @@
+"""Tests for the deep commit-index walk that replaces per-file log fallbacks.
+
+Files the recent-window commit index never saw used to each spawn a
+per-file ``git log`` subprocess. ``load_deep_commit_index`` walks the
+older history region once (``--skip``) and buckets those files' commits;
+``index_repo`` consults it before falling back per file.
+
+For rename-free linear history the deep bucket must be IDENTICAL to what
+the per-file fallback produces (same shas, order, churn, provenance), so
+the swap is exercised end to end against the fallback as oracle.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.ingestion.git_commit_index import (
+ load_commit_index,
+ load_deep_commit_index,
+)
+from repowise.core.ingestion.git_indexer import GitIndexer
+from repowise.core.ingestion.git_indexer.file_history import _parse_per_file_log
+from repowise.core.ingestion.git_indexer.tiers import GitIndexTier
+
+
+def _build_repo(tmp_path, *, old_files: int = 3, recent_commits: int = 2):
+ """Linear history: each old file gets two commits, then `recent_commits`
+ commits touch only ``recent.py`` — so a window of that size covers only
+ ``recent.py`` and every old file lands beyond the window."""
+ import git as gitpython
+
+ repo = gitpython.Repo.init(tmp_path)
+ with repo.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+
+ for i in range(old_files):
+ p = tmp_path / f"old_{i}.py"
+ p.write_text(f"x{i} = 1\n")
+ repo.index.add([p.name])
+ repo.index.commit(f"feat: add old module {i} with the initial implementation")
+ p.write_text(f"x{i} = 1\ny{i} = 2\n")
+ repo.index.add([p.name])
+ if i == 0:
+ # Agent-attributed commit in the DEEP region — the rollup must
+ # survive the batched path.
+ repo.index.commit(
+ f"fix: adjust old module {i}\n\n"
+ "Co-Authored-By: Claude <[email protected]>"
+ )
+ else:
+ repo.index.commit(f"fix: adjust old module {i} boundary handling")
+
+ recent = tmp_path / "recent.py"
+ for j in range(recent_commits):
+ recent.write_text(f"r = {j}\n")
+ repo.index.add(["recent.py"])
+ repo.index.commit(f"feat: iterate on the recent module step {j}")
+ repo.close()
+
+
+def _rec_tuple(c):
+ return (c.sha, c.ts, c.is_merge, c.added, c.deleted, c.agent, c.agent_tier)
+
+
+def test_deep_bucket_matches_per_file_fallback(tmp_path) -> None:
+ _build_repo(tmp_path)
+ import git as gitpython
+
+ repo = gitpython.Repo(tmp_path)
+ indexable = {"old_0.py", "old_1.py", "old_2.py", "recent.py"}
+ window = load_commit_index(repo, 2, indexable)
+ assert set(window) == {"recent.py"}
+
+ missed = indexable - set(window)
+ deep = load_deep_commit_index(
+ repo, 2, missed, skip=2, deep_limit=20_000
+ )
+ assert set(deep) == missed
+
+ for fp in sorted(missed):
+ oracle, _orig = _parse_per_file_log(
+ repo,
+ fp,
+ commit_limit=2,
+ follow_renames=False,
+ provenance_classifier=None,
+ )
+ # The oracle path classifies provenance only when given a classifier;
+ # compare structure first, then provenance separately below.
+ assert [(c.sha, c.ts, c.is_merge, c.added, c.deleted) for c in deep[fp]] == [
+ (c.sha, c.ts, c.is_merge, c.added, c.deleted) for c in oracle
+ ], fp
+
+ # Agent provenance was classified during the deep walk.
+ agents = [c.agent for c in deep["old_0.py"]]
+ assert any(agents), agents
+ repo.close()
+
+
+def test_deep_bucket_respects_per_file_cap(tmp_path) -> None:
+ _build_repo(tmp_path)
+ import git as gitpython
+
+ repo = gitpython.Repo(tmp_path)
+ deep = load_deep_commit_index(
+ repo, 1, {"old_0.py", "old_1.py"}, skip=2, deep_limit=20_000
+ )
+ assert all(len(v) == 1 for v in deep.values())
+ # Newest first: the cap keeps each file's most recent deep commit.
+ assert all(v[0].subject.startswith("fix:") for v in deep.values())
+ repo.close()
+
+
+def test_deep_bucket_only_contains_wanted_files(tmp_path) -> None:
+ _build_repo(tmp_path)
+ import git as gitpython
+
+ repo = gitpython.Repo(tmp_path)
+ deep = load_deep_commit_index(
+ repo, 2, {"old_1.py"}, skip=2, deep_limit=20_000
+ )
+ assert set(deep) == {"old_1.py"}
+ repo.close()
+
+
+def test_empty_wanted_set_skips_the_walk(tmp_path) -> None:
+ _build_repo(tmp_path)
+ import git as gitpython
+
+ repo = gitpython.Repo(tmp_path)
+ assert load_deep_commit_index(repo, 2, set(), skip=2, deep_limit=20_000) == {}
+ repo.close()
+
+
+async def test_index_repo_uses_deep_index_instead_of_per_file_logs(
+ tmp_path, monkeypatch
+) -> None:
+ """With the deep walk active, files beyond the window must get their
+ metadata WITHOUT any per-file git log, and that metadata must equal the
+ per-file path's output (rename-free history → semantics align)."""
+ _build_repo(tmp_path)
+ # Anchor recency windows to HEAD so the two runs share a reference
+ # "now" (otherwise the decayed temporal score drifts at the 1e-9 level).
+ monkeypatch.setenv("REPOWISE_GIT_WINDOW_ANCHOR", "head")
+
+ # Reference run: deep walk disabled (threshold impossible to reach),
+ # so old files take the per-file fallback exactly as before.
+ monkeypatch.setattr(
+ "repowise.core.ingestion.git_indexer.indexer._DEEP_WALK_MIN_FALLBACK", 10**9
+ )
+ ref = GitIndexer(tmp_path, commit_limit=2, tier=GitIndexTier.ESSENTIAL)
+ _s, ref_meta = await ref.index_repo("r1")
+ ref_by_path = {m["file_path"]: m for m in ref_meta}
+
+ # Deep run: threshold 1 forces the deep walk; per-file fallback must
+ # never fire for files the deep bucket covers.
+ monkeypatch.setattr(
+ "repowise.core.ingestion.git_indexer.indexer._DEEP_WALK_MIN_FALLBACK", 1
+ )
+ calls: list[str] = []
+ import repowise.core.ingestion.git_indexer.file_history as fh
+
+ orig = fh._parse_per_file_log
+
+ def _spy(repo, file_path, **kwargs):
+ calls.append(file_path)
+ return orig(repo, file_path, **kwargs)
+
+ monkeypatch.setattr(fh, "_parse_per_file_log", _spy)
+
+ deep_run = GitIndexer(tmp_path, commit_limit=2, tier=GitIndexTier.ESSENTIAL)
+ _s2, deep_meta = await deep_run.index_repo("r1")
+ deep_by_path = {m["file_path"]: m for m in deep_meta}
+
+ assert calls == [], f"per-file fallback fired for {calls}"
+ assert set(deep_by_path) == set(ref_by_path)
+ skip_fields = {"churn_percentile", "change_entropy_pct"} # repo-relative, identical anyway
+ for fp, ref_m in ref_by_path.items():
+ deep_m = deep_by_path[fp]
+ for field, want in ref_m.items():
+ if field in skip_fields:
+ continue
+ got = deep_m.get(field)
+ if isinstance(want, float):
+ assert got == pytest.approx(want, rel=1e-9), (fp, field)
+ else:
+ assert got == want, (fp, field)
diff --git a/tests/unit/ingestion/test_git_rename_markers.py b/tests/unit/ingestion/test_git_rename_markers.py
new file mode 100644
index 000000000..b6a2b8c95
--- /dev/null
+++ b/tests/unit/ingestion/test_git_rename_markers.py
@@ -0,0 +1,46 @@
+"""Numstat rename-marker expansion, including empty-side directory moves."""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.ingestion.git_indexer.records import _extract_rename_paths
+
+
+ ("stat_path", "want_old", "want_new"),
+ [
+ # Whole-segment rename with shared suffix.
+ ("{old => new}/shared/file.py", "old/shared/file.py", "new/shared/file.py"),
+ # Filename rename inside a stable directory.
+ ("old_dir/{old_name => new_name}.py", "old_dir/old_name.py", "old_dir/new_name.py"),
+ # Directory INSERTED into the path (empty old side) — git emits
+ # ``{ => newdir}``; expanding the empty side must not leave a
+ # doubled slash.
+ (
+ "src/settings-ui/Settings.UI/{ => SettingsXAML}/Views/Page.xaml.cs",
+ "src/settings-ui/Settings.UI/Views/Page.xaml.cs",
+ "src/settings-ui/Settings.UI/SettingsXAML/Views/Page.xaml.cs",
+ ),
+ # Directory REMOVED from the path (empty new side).
+ (
+ "src/{legacy => }/Views/Page.xaml.cs",
+ "src/legacy/Views/Page.xaml.cs",
+ "src/Views/Page.xaml.cs",
+ ),
+ # Mid-path segment rename.
+ ("a/{b => c}/d.py", "a/b/d.py", "a/c/d.py"),
+ ],
+)
+def test_rename_marker_forms(stat_path: str, want_old: str, want_new: str) -> None:
+ known: set[str] = set()
+ old, new = _extract_rename_paths(stat_path, known)
+ assert old == want_old
+ assert new == want_new
+ assert known == {want_old, want_new}
+
+
+def test_plain_path_returns_none() -> None:
+ known: set[str] = set()
+ assert _extract_rename_paths("src/plain/file.py", known) == (None, None)
+ assert known == set()
diff --git a/tests/unit/ingestion/test_xaml_dynamic_hints.py b/tests/unit/ingestion/test_xaml_dynamic_hints.py
index ab51f3903..1417e9af8 100644
--- a/tests/unit/ingestion/test_xaml_dynamic_hints.py
+++ b/tests/unit/ingestion/test_xaml_dynamic_hints.py
@@ -201,3 +201,62 @@ class TestLanguageRegistration:
assert spec is not None
assert spec.is_passthrough is True
assert spec.is_code is False
+
+
+class TestPrebuiltIndexReuse:
+ """The registry attaches the resolver-built DotNetProjectIndex so the
+ XAML extractor skips its duplicate full .cs walk + index build."""
+
+ def _mini_repo(self, tmp_path: Path) -> None:
+ (tmp_path / "App").mkdir()
+ (tmp_path / "App" / "App.csproj").write_text(_CSPROJ)
+ (tmp_path / "App" / "ViewModels").mkdir()
+ (tmp_path / "App" / "ViewModels" / "GeneralViewModel.cs").write_text(
+ "namespace App.ViewModels;\npublic class GeneralViewModel {}\n"
+ )
+ (tmp_path / "App" / "Views").mkdir()
+ (tmp_path / "App" / "Views" / "GeneralPage.xaml").write_text(
+ '<Page xmlns:vm="using:App.ViewModels" x:DataType="vm:GeneralViewModel">\n'
+ "</Page>\n"
+ )
+
+ def test_provided_index_used_without_rebuild(self, tmp_path: Path, monkeypatch) -> None:
+ from repowise.core.ingestion.dynamic_hints import xaml as xaml_mod
+ from repowise.core.ingestion.resolvers.dotnet import build_index
+
+ self._mini_repo(tmp_path)
+ index = build_index(tmp_path)
+ baseline = XamlDynamicHints().extract(tmp_path)
+
+ def _boom(_root):
+ raise AssertionError("xaml must not rebuild the index when one is provided")
+
+ monkeypatch.setattr(xaml_mod, "_load_type_map", _boom)
+ ex = XamlDynamicHints()
+ ex._dotnet_index = index
+ edges = ex.extract(tmp_path)
+ assert sorted((e.source, e.target, e.edge_type, e.hint_source) for e in edges) == sorted(
+ (e.source, e.target, e.edge_type, e.hint_source) for e in baseline
+ )
+ assert edges # the mini repo produces at least the binding edge
+
+ def test_registry_attaches_and_clears_index(self, tmp_path: Path) -> None:
+ from repowise.core.ingestion.dynamic_hints import HintRegistry
+ from repowise.core.ingestion.resolvers.dotnet import build_index
+
+ self._mini_repo(tmp_path)
+ index = build_index(tmp_path)
+
+ seen: list[object] = []
+
+ class _Probe(XamlDynamicHints):
+ def extract(self, repo_root: Path):
+ seen.append(self._dotnet_index)
+ return super().extract(repo_root)
+
+ probe = _Probe()
+ registry = HintRegistry(extractors=[probe])
+ edges = registry.extract_all(tmp_path, dotnet_index=index)
+ assert seen == [index]
+ assert probe._dotnet_index is None # cleared after the run
+ assert edges