Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(core): decompose the shared commit-index walks into small steps (#2606)

Raghav Chamadiya364cdfe992cb
diff --git a/packages/core/src/repowise/core/ingestion/git_commit_index.py b/packages/core/src/repowise/core/ingestion/git_commit_index.py
index 88a9752c6..bbab32a57 100644
--- a/packages/core/src/repowise/core/ingestion/git_commit_index.py
+++ b/packages/core/src/repowise/core/ingestion/git_commit_index.py
@@ -1,25 +1,15 @@
-"""Single-pass repo-wide commit index for git_indexer.
-
-The original per-file path in ``git_indexer._index_file`` spawned one
-``git log --numstat`` subprocess per tracked file. On a 5,000-file repo
-that meant 5,000 process spawns — ~50-100 ms each on Windows — which
-made the git phase dominate the total ``repowise init`` wall-clock.
-
-This module replaces the fan-out with one repo-wide ``git log`` pass
-and an in-memory bucketing step. The shape mirrors what
-``compute_co_changes_and_entropy`` already does — one subprocess, fan-out via
-Python dicts — so any future debugging only has one log format to
-understand.
-
-The batched path is only used when ``follow_renames=False`` (the
-default). It still keeps a file's history across a rename: the walk reads
-git's rename rows newest first and files older commits under the current
-name (see :class:`~.git_indexer.records.RenameTrail`). ``follow_renames``
-switches to the per-file ``--follow`` path instead.
+"""Repo-wide commit index for git_indexer: one ``git log --numstat`` pass, bucketed per file.
+
+Used when ``follow_renames=False`` (the default). The walk reads git's rename
+rows newest first and files older commits under each file's current name (see
+:class:`~.git_indexer.records.RenameTrail`).
"""
from __future__ import annotations
+import functools
+import json
+from collections.abc import Callable, Iterator
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING
@@ -28,6 +18,7 @@ import structlog
if TYPE_CHECKING:
from .git_indexer import RenameTrail, _CommitRec
+ from .git_indexer.agent_provenance import AgentProvenance
logger = structlog.get_logger(__name__)
@@ -62,23 +53,34 @@ def _count_non_merge_commits(repo: object) -> int | None:
def load_git_ai_note_agents(repo: object, commit_limit: int | None) -> dict[str, str]:
"""Map ``commit_sha → agent`` from git-ai authorship notes (``refs/notes/ai``).
- Returns an empty dict when the ref is absent — the common case, gated by a
- single cheap ``for-each-ref`` so 99.9% of repos pay nothing and add no git
- pass. Only repos actually using git-ai incur a ``git log --notes=ai`` walk
- here, whose ``sha → agent`` result the commit walk then reads by-key (no
- re-parse per touched file). The walk is bounded when *commit_limit* is an
- integer; ``None`` covers the complete history for the rare per-file
- fallback lane. Any failure returns ``{}`` — a notes read must never break
- the git index. See the git-ai standard v3.0.0 for the note format.
+ A repo without the ref costs one ``for-each-ref`` and gets ``{}``. The
+ notes walk covers *commit_limit* commits, or the whole history with
+ ``None``. Any failure returns ``{}``: a notes read must never break the git
+ index. See the git-ai standard v3.0.0 for the note format.
"""
from .git_indexer import _FIELD_SEP, _RECORD_SEP
from .git_indexer.agent_provenance import _agent_from_git_ai_note
+ agents: dict[str, str] = {}
+ for chunk in _git_ai_notes_log(repo, commit_limit).split(_RECORD_SEP):
+ if not chunk.strip():
+ continue
+ sha, sep, note = chunk.partition(_FIELD_SEP)
+ if not sep:
+ continue
+ agent = _agent_from_git_ai_note(note)
+ if agent:
+ agents[sha.strip()] = agent
+ return agents
+
+
+def _git_ai_notes_log(repo: object, commit_limit: int | None) -> str:
+ """The raw ``refs/notes/ai`` log, or ``""`` when the ref is absent or git fails."""
try:
if not repo.git.for_each_ref("refs/notes/ai"): # type: ignore[attr-defined]
- return {}
+ return ""
except Exception:
- return {}
+ return ""
try:
# ``%N`` is the note body; the leading ``%x00``/``%x1f`` mirror the main
@@ -86,22 +88,10 @@ def load_git_ai_note_agents(repo: object, commit_limit: int | None) -> dict[str,
args = ["--no-merges", "--notes=ai", "--format=%x00%H%x1f%N"]
if commit_limit is not None:
args.insert(0, f"-{commit_limit}")
- raw = repo.git.log(*args) # type: ignore[attr-defined]
+ return repo.git.log(*args) # type: ignore[attr-defined, no-any-return]
except Exception as exc:
logger.warning("git_ai_notes_load_failed", error=str(exc))
- return {}
-
- agents: dict[str, str] = {}
- for chunk in raw.split(_RECORD_SEP):
- if not chunk.strip():
- continue
- sha, sep, note = chunk.partition(_FIELD_SEP)
- if not sep:
- continue
- agent = _agent_from_git_ai_note(note)
- if agent:
- agents[sha.strip()] = agent
- return agents
+ return ""
#: Cache file name under the repository's ``.repowise`` directory, and the
@@ -138,58 +128,26 @@ def _window_records(repo: object, depth: int, cache_dir: Path | None) -> list[st
walk runs. Every failure on the cache path falls back to the full walk;
the cache can only ever save time, never change the answer.
"""
- import json
-
- from .git_indexer import _LOG_FORMAT, _RECORD_SEP
-
- def _full_walk() -> list[str]:
- raw = repo.git.log( # type: ignore[attr-defined]
- f"-{depth}", "--numstat", "--no-merges", f"--format={_LOG_FORMAT}"
- )
- return [rec for rec in raw.split(_RECORD_SEP) if rec.strip()]
-
if cache_dir is None:
- return _full_walk()
+ return _log_records(repo, f"-{depth}")
cache_path = Path(cache_dir) / _WINDOW_CACHE_NAME
try:
head = repo.head.commit.hexsha # type: ignore[attr-defined]
except Exception:
- return _full_walk()
+ return _log_records(repo, f"-{depth}")
- records: list[str] | None = None
try:
- cached = json.loads(cache_path.read_text(encoding="utf-8"))
- if (
- cached.get("version") == _WINDOW_CACHE_VERSION
- and cached.get("depth") == depth
- and isinstance(cached.get("records"), list)
- ):
- cached_head = str(cached.get("head") or "")
- if cached_head == head:
- records = list(cached["records"])
- elif cached_head:
- # Raises when the cached head is unknown to this repository or
- # not behind HEAD, which is exactly when the cache is stale.
- repo.git.merge_base("--is-ancestor", cached_head, head) # type: ignore[attr-defined]
- raw = repo.git.log( # type: ignore[attr-defined]
- f"{cached_head}..{head}",
- "--numstat",
- "--no-merges",
- f"--format={_LOG_FORMAT}",
- )
- fresh = [rec for rec in raw.split(_RECORD_SEP) if rec.strip()]
- merged = fresh + list(cached["records"])
- # Stable, so a fresh record stays ahead of a cached one at the
- # same second, which is where the full walk puts it too.
- merged.sort(key=_record_ts, reverse=True)
- records = merged[:depth]
+ records = _cached_window_records(repo, cache_path, head, depth)
except Exception:
records = None
-
if records is None:
- records = _full_walk()
+ records = _log_records(repo, f"-{depth}")
+ _write_window_cache(cache_path, head, depth, records)
+ return records
+
+def _write_window_cache(cache_path: Path, head: str, depth: int, records: list[str]) -> None:
try:
from repowise.core.fsutils import atomic_write_text
@@ -206,7 +164,243 @@ def _window_records(repo: object, depth: int, cache_dir: Path | None) -> list[st
)
except Exception as exc:
logger.debug("commit_window_cache_write_failed", error=str(exc))
- return records
+
+
+def _log_records(repo: object, revisions: str) -> list[str]:
+ """The non-empty raw ``--numstat`` log records git returns for *revisions*."""
+ from .git_indexer import _LOG_FORMAT, _RECORD_SEP
+
+ raw = repo.git.log( # type: ignore[attr-defined]
+ revisions, "--numstat", "--no-merges", f"--format={_LOG_FORMAT}"
+ )
+ return [rec for rec in raw.split(_RECORD_SEP) if rec.strip()]
+
+
+def _cached_window_records(
+ repo: object, cache_path: Path, head: str, depth: int
+) -> list[str] | None:
+ """The cached window brought forward to *head*, or ``None`` when it cannot serve.
+
+ Raises on an unreadable cache or a cached head that is not an ancestor of
+ *head*; the caller treats both as a miss.
+ """
+ cached = json.loads(cache_path.read_text(encoding="utf-8"))
+ if (
+ cached.get("version") != _WINDOW_CACHE_VERSION
+ or cached.get("depth") != depth
+ or not isinstance(cached.get("records"), list)
+ ):
+ return None
+ cached_head = str(cached.get("head") or "")
+ if cached_head == head:
+ return list(cached["records"])
+ if not cached_head:
+ return None
+ # Raises when the cached head is unknown to this repository or
+ # not behind HEAD, which is exactly when the cache is stale.
+ repo.git.merge_base("--is-ancestor", cached_head, head) # type: ignore[attr-defined]
+ merged = _log_records(repo, f"{cached_head}..{head}") + list(cached["records"])
+ # Stable, so a fresh record stays ahead of a cached one at the
+ # same second, which is where the full walk puts it too.
+ merged.sort(key=_record_ts, reverse=True)
+ return merged[:depth]
+
+
+def _report(stats_sink: dict[str, int | bool] | None, *, succeeded: bool, commits: int) -> None:
+ if stats_sink is not None:
+ stats_sink.update(succeeded=succeeded, commits=commits)
+
+
+def _depth_newer_than(repo: object, commit_limit: int, since_ts: int) -> int | None:
+ """How deep in the window the commits newer than *since_ts* reach.
+
+ ``None`` when git fails. A ``%ct``-only pass is cheap next to the numstat
+ walk, and both traverse the same revisions in the same order, so every
+ commit past this depth is one the walk's ``ts <= since_ts`` check drops.
+ """
+ try:
+ stamps = repo.git.log( # type: ignore[attr-defined]
+ f"-{commit_limit}", "--no-merges", "--format=%ct"
+ ).split()
+ except Exception as exc:
+ logger.warning("repo_commit_index_failed", error=str(exc))
+ return None
+ return max(
+ (position + 1 for position, stamp in enumerate(stamps) if _stamp_after(stamp, since_ts)),
+ default=0,
+ )
+
+
+def _stamp_after(stamp: str, since_ts: int) -> bool:
+ try:
+ return int(stamp) > since_ts
+ except ValueError: # unparseable stamp: keep it in the window
+ return True
+
+
+def _numstat_counts(cols: list[str]) -> tuple[int, int]:
+ """``(added, deleted)`` of one numstat row; binary (``-``) and garbled rows count 0."""
+ try:
+ added = int(cols[0]) if cols[0] != "-" else 0
+ deleted = int(cols[1]) if cols[1] != "-" else 0
+ except ValueError:
+ return 0, 0
+ return added, deleted
+
+
+def _parse_numstat(
+ numstat_lines: list[str],
+) -> tuple[list[tuple[str, int, int]], list[tuple[str, str]]]:
+ """A commit's ``(path, added, deleted)`` rows and its ``(old, new)`` renames.
+
+ A rename row's churn goes to the new path.
+ """
+ from .git_indexer import _extract_rename_paths
+
+ changes: list[tuple[str, int, int]] = []
+ moved: list[tuple[str, str]] = []
+ for line in numstat_lines:
+ cols = line.split("\t")
+ if len(cols) < 3:
+ continue
+ target = cols[2]
+ if "=>" in target:
+ old_path, new_path = _extract_rename_paths(target, set())
+ if old_path and new_path:
+ moved.append((old_path, new_path))
+ target = new_path or target
+ changes.append((target, *_numstat_counts(cols)))
+ return changes, moved
+
+
+def _resolved(
+ changes: list[tuple[str, int, int]], renames: RenameTrail | None
+) -> list[tuple[str, int, int]]:
+ """*changes* filed under each path's name at HEAD."""
+ if renames is None:
+ return changes
+ return [(renames.resolve(target), added, deleted) for target, added, deleted in changes]
+
+
+def _record_moves(renames: RenameTrail | None, moved: list[tuple[str, str]]) -> None:
+ # Only after the commit's own paths are resolved (see RenameTrail).
+ if renames is not None:
+ for old_path, new_path in moved:
+ renames.record(old_path, new_path)
+
+
+class _CommitLabeler:
+ """Labels each walked commit's agent provenance, once per commit.
+
+ git-ai notes are read to *note_depth* (``{}`` unless the repo uses them);
+ the map is read by sha, so a wider notes pass buys nothing. A caller may
+ share its classifier and trace index so the trace file is read once per
+ index. With *sink*, every labelled commit is also appended as a raw row.
+ """
+
+ def __init__(
+ self,
+ repo: object,
+ note_depth: int,
+ classifier: object | None,
+ trace_index: object | None,
+ sink: list[dict] | None = None,
+ ) -> None:
+ from .git_indexer.agent_provenance import AgentProvenanceClassifier, AgentTraceIndex
+
+ self._note_agents = load_git_ai_note_agents(repo, note_depth)
+ self._trace_index = trace_index if trace_index is not None else AgentTraceIndex.load(repo)
+ self._classifier = classifier if classifier is not None else AgentProvenanceClassifier()
+ self._sink = sink
+
+ def label(self, header: dict, changes: list[tuple[str, int, int]]) -> AgentProvenance:
+ # The agent-trace channel checks overlap with the commit's changed paths.
+ trace_hit = (
+ self._trace_index.resolve( # type: ignore[attr-defined]
+ header["sha"], header["parents"], {t for t, _, _ in changes}
+ )
+ if self._trace_index
+ else None
+ )
+ prov: AgentProvenance = self._classifier.classify( # type: ignore[attr-defined]
+ header["author_name"],
+ header["author_email"],
+ header["committer_name"],
+ header["committer_email"],
+ f"{header['subject']}\n{header['body']}",
+ note_agent=self._note_agents.get(header["sha"]),
+ trace_agent=trace_hit[0] if trace_hit else None,
+ trace_confidence=trace_hit[1] if trace_hit else "high",
+ )
+ if self._sink is not None:
+ self._sink.append(_sink_row(header, changes, prov, trace_hit))
+ return prov
+
+
+def _sink_row(
+ header: dict,
+ changes: list[tuple[str, int, int]],
+ prov: AgentProvenance,
+ trace_hit: tuple[str, str, str | None] | None,
+) -> dict:
+ return {
+ "sha": header["sha"],
+ "author_name": header["author_name"],
+ "author_email": header["author_email"],
+ "ts": header["ts"],
+ "tz_offset_minutes": header.get("tz_offset_minutes"),
+ "subject": header["subject"],
+ "changes": changes,
+ "agent_name": prov.agent,
+ "agent_autonomy_tier": prov.autonomy_tier,
+ "agent_channel": prov.channel,
+ "agent_confidence": prov.confidence,
+ # Model id only when the trace channel actually won: a note
+ # or service-identity match can outrank the trace above, and
+ # only the trace record carries a model.
+ "agent_model_id": (
+ trace_hit[2] if (trace_hit and prov.channel == "agent_trace") else None
+ ),
+ }
+
+
+class _FileBuckets:
+ """Per-file commit records, newest first, for *files* only.
+
+ With *cap*, a file stops collecting once it holds that many records.
+ """
+
+ def __init__(self, files: set[str], cap: int | None = None) -> None:
+ self.files = files
+ self.cap = cap
+ self.records: dict[str, list[_CommitRec]] = {}
+
+ def add(
+ self, header: dict, prov: AgentProvenance, changes: list[tuple[str, int, int]]
+ ) -> None:
+ """File one commit under each of its changed paths that is tracked here."""
+ from .git_indexer import _CommitRec
+
+ for target, added, deleted in changes:
+ if target not in self.files:
+ continue
+ records = self.records.setdefault(target, [])
+ if self.cap is None or len(records) < self.cap:
+ records.append(
+ _CommitRec(
+ sha=header["sha"],
+ author_name=header["author_name"],
+ author_email=header["author_email"],
+ ts=header["ts"],
+ is_merge=header["is_merge"],
+ subject=header["subject"],
+ body=header["body"],
+ added=added,
+ deleted=deleted,
+ agent=prov.agent,
+ agent_tier=prov.autonomy_tier,
+ )
+ )
def load_commit_index(
@@ -242,8 +436,7 @@ def load_commit_index(
files in the commit, not just the indexable subset, so change diffusion is
measured against the real footprint). This rides the same single walk —
no extra git pass — and lets the caller build per-commit rows downstream
- (see :mod:`git_indexer.commit_rows`). The default (``None``) leaves the
- return value and behaviour unchanged.
+ (see :mod:`git_indexer.commit_rows`).
When *since_ts* is supplied (unix seconds), commits at or before it are
skipped — used by the incremental path to capture only commits newer than
@@ -270,47 +463,19 @@ def load_commit_index(
Failures (git unavailable, corrupt log output, etc.) return an
empty dict so the caller can fall back to per-file indexing.
"""
- # Imported here to avoid a circular import — these live in git_indexer's
- # records module and this module is imported from there.
- from .git_indexer import (
- _CommitRec,
- _extract_rename_paths,
- _parse_commit_record,
- )
- from .git_indexer.agent_provenance import AgentProvenanceClassifier, AgentTraceIndex
+ _report(stats_sink, succeeded=False, commits=0)
- if provenance_classifier is None:
- provenance_classifier = AgentProvenanceClassifier()
- if stats_sink is not None:
- stats_sink.update(succeeded=False, commits=0)
-
- # With *since_ts* the walk still asked git for the whole window's numstat
- # and threw almost all of it away in the loop below — 1.5s per update on a
- # 9.3k-commit repo to return nothing. A ``%ct``-only pass over the same
- # walk costs 0.06s and says exactly how deep the surviving prefix goes:
- # both logs traverse the same revisions in the same order under the same
- # filters, so every commit past that depth is one the ``ts <= since_ts``
- # check below would have dropped. The check stays — this only stops git
- # from diffing commits whose fate is already known.
+ # With *since_ts* only the prefix newer than it is asked of git; the
+ # ``ts <= since_ts`` check in the walk stays, this only stops git from
+ # diffing commits whose fate is already known.
depth = commit_limit
if since_ts is not None:
- try:
- stamps = repo.git.log( # type: ignore[attr-defined]
- f"-{commit_limit}", "--no-merges", "--format=%ct"
- ).split()
- except Exception as exc:
- logger.warning("repo_commit_index_failed", error=str(exc))
+ newer = _depth_newer_than(repo, commit_limit, since_ts)
+ if newer is None:
return {}
- depth = 0
- for position, stamp in enumerate(stamps):
- try:
- if int(stamp) > since_ts:
- depth = position + 1
- except ValueError: # unparseable stamp: keep it in the window
- depth = position + 1
+ depth = newer
if depth == 0:
- if stats_sink is not None:
- stats_sink.update(succeeded=True, commits=0)
+ _report(stats_sink, succeeded=True, commits=0)
logger.debug(
"repo_commit_index_built",
commits_parsed=0,
@@ -326,151 +491,43 @@ def load_commit_index(
return {}
if not records:
- if stats_sink is not None:
- stats_sink.update(succeeded=True, commits=0)
+ _report(stats_sink, succeeded=True, commits=0)
return {}
- # git-ai authorship notes for this window (``{}`` unless the repo uses them).
- # Bounded to the same depth as the walk above: the map is read by sha, only
- # for commits that walk parsed, so a wider notes pass buys nothing.
- note_agents = load_git_ai_note_agents(repo, depth)
- # Agent-trace records (empty after one stat call unless the repo has them).
- # The orchestrator may pass a shared instance so the trace file is read once
- # per index rather than once per walk.
- if trace_index is None:
- trace_index = AgentTraceIndex.load(repo)
-
- bucket: dict[str, list[_CommitRec]] = {}
+ labeler = _CommitLabeler(repo, depth, provenance_classifier, trace_index, commit_sink)
+ buckets = _FileBuckets(indexable_files)
commits_parsed = 0
-
- # Split on the NUL record separator rather than newlines: commit bodies
- # (``%b``) are multi-line, so a line-based scan would mistake body lines
- # for numstat rows. The first chunk before the leading separator is empty.
- for record in records:
- parsed = _parse_commit_record(record)
- if parsed is None:
- continue
- header, numstat_lines = parsed
-
- # Incremental bound: drop commits at/older than the newest already
- # persisted (newest-first walk, but author/commit-time can be slightly
- # out of order, so skip rather than break to stay correct).
- if since_ts is not None and header["ts"] <= since_ts:
- continue
+ for header, numstat_lines in _commits_after(records, since_ts):
commits_parsed += 1
-
- # Full change footprint of this commit (every file, not just the
- # indexable subset). Parsed before classification because the
- # agent-trace channel needs the changed-path set for its file-overlap
- # check; the same list feeds the commit sink.
- commit_changes: list[tuple[str, int, int]] = []
- moved: list[tuple[str, str]] = []
-
- for line in numstat_lines:
- cols = line.split("\t")
- if len(cols) < 3:
- continue
-
- stat_path = cols[2]
- # Handle rename markers — ``{old => new}`` resolves to a new
- # path. Without ``--follow`` git still emits these for moves
- # detected via the rename heuristic; we add both names but
- # attribute the churn to the new path.
- if "=>" in stat_path:
- seen: set[str] = set()
- old_path, new_path = _extract_rename_paths(stat_path, seen)
- target = new_path or stat_path
- if old_path and new_path:
- moved.append((old_path, new_path))
- else:
- target = stat_path
-
- try:
- added = int(cols[0]) if cols[0] != "-" else 0
- deleted = int(cols[1]) if cols[1] != "-" else 0
- except ValueError:
- added = 0
- deleted = 0
-
- commit_changes.append((target, added, deleted))
-
- # Agent provenance — classified ONCE per commit here (not per touched
- # file): the per-file records below share the result by reference.
- trace_hit = (
- trace_index.resolve(header["sha"], header["parents"], {t for t, _, _ in commit_changes})
- if trace_index
- else None
- )
- prov = provenance_classifier.classify( # type: ignore[attr-defined]
- header["author_name"],
- header["author_email"],
- header["committer_name"],
- header["committer_email"],
- f"{header['subject']}\n{header['body']}",
- note_agent=note_agents.get(header["sha"]),
- trace_agent=trace_hit[0] if trace_hit else None,
- trace_confidence=trace_hit[1] if trace_hit else "high",
- )
-
- for target, added, deleted in commit_changes:
- if renames is not None:
- target = renames.resolve(target)
- if target not in indexable_files:
- continue
-
- # Each commit becomes one record per file it touched — the
- # per-file analyzer treats this list as the file's own history.
- bucket.setdefault(target, []).append(
- _CommitRec(
- sha=header["sha"],
- author_name=header["author_name"],
- author_email=header["author_email"],
- ts=header["ts"],
- is_merge=header["is_merge"],
- subject=header["subject"],
- body=header["body"],
- added=added,
- deleted=deleted,
- agent=prov.agent,
- agent_tier=prov.autonomy_tier,
- )
- )
-
- if commit_sink is not None:
- commit_sink.append(
- {
- "sha": header["sha"],
- "author_name": header["author_name"],
- "author_email": header["author_email"],
- "ts": header["ts"],
- "tz_offset_minutes": header.get("tz_offset_minutes"),
- "subject": header["subject"],
- "changes": commit_changes,
- "agent_name": prov.agent,
- "agent_autonomy_tier": prov.autonomy_tier,
- "agent_channel": prov.channel,
- "agent_confidence": prov.confidence,
- # Model id only when the trace channel actually won: a note
- # or service-identity match can outrank the trace above, and
- # only the trace record carries a model.
- "agent_model_id": (
- trace_hit[2] if (trace_hit and prov.channel == "agent_trace") else None
- ),
- }
- )
- if renames is not None:
- for old_path, new_path in moved:
- renames.record(old_path, new_path)
+ # The full footprint (every file, not just the indexable subset): the
+ # agent-trace overlap check and the commit sink both need it.
+ commit_changes, moved = _parse_numstat(numstat_lines)
+ prov = labeler.label(header, commit_changes)
+ buckets.add(header, prov, _resolved(commit_changes, renames))
+ _record_moves(renames, moved)
logger.debug(
"repo_commit_index_built",
commits_parsed=commits_parsed,
- files_with_history=len(bucket),
+ files_with_history=len(buckets.records),
indexable_files=len(indexable_files),
)
- if stats_sink is not None:
- stats_sink.update(succeeded=True, commits=commits_parsed)
- return bucket
+ _report(stats_sink, succeeded=True, commits=commits_parsed)
+ return buckets.records
+
+
+def _commits_after(records: list[str], since_ts: int | None) -> Iterator[tuple[dict, list[str]]]:
+ """``(header, numstat_lines)`` of each parseable record newer than *since_ts*.
+
+ Skips rather than stops at an older commit: the walk is newest first, but
+ commit times can be slightly out of order.
+ """
+ from .git_indexer import _parse_commit_record
+
+ for record in records:
+ parsed = _parse_commit_record(record)
+ if parsed is not None and (since_ts is None or parsed[0]["ts"] > since_ts):
+ yield parsed
def load_deep_commit_index(
@@ -497,8 +554,7 @@ def load_deep_commit_index(
*per_file_limit* per file — mirroring the per-file fallback's
``git log -<limit> -- <file>`` cap. One subprocess replaces one
fallback spawn per missed file, which dominates the git phase on
- repos whose history is much deeper than the window (a 9k-commit
- monorepo left 3,295 of 4,857 files to the fallback).
+ repos whose history is much deeper than the window.
Churn comes from the repo-wide diff (rename rows attribute edit churn
through the rename) rather than the pathspec-limited fallback diff (which
@@ -511,23 +567,13 @@ def load_deep_commit_index(
keep the per-file fallback path.
Failures return an empty dict; every missed file then falls back to
- the per-file path exactly as before.
+ the per-file path.
"""
- from .git_indexer import (
- _LOG_FORMAT,
- _RECORD_SEP,
- _CommitRec,
- _extract_rename_paths,
- _parse_commit_record,
- )
- from .git_indexer.agent_provenance import AgentProvenanceClassifier, AgentTraceIndex
+ from .git_indexer import _LOG_FORMAT, _RECORD_SEP
if not wanted_files:
return {}
- if stats_sink is not None:
- stats_sink.update(succeeded=False, commits=0)
- if provenance_classifier is None:
- provenance_classifier = AgentProvenanceClassifier()
+ _report(stats_sink, succeeded=False, commits=0)
try:
raw = repo.git.log( # type: ignore[attr-defined]
@@ -542,120 +588,108 @@ def load_deep_commit_index(
return {}
if not raw:
- if stats_sink is not None:
- stats_sink.update(succeeded=True, commits=0)
+ _report(stats_sink, succeeded=True, commits=0)
return {}
- # git-ai notes across the deep region (``{}`` unless the repo uses them).
- note_agents = load_git_ai_note_agents(repo, skip + deep_limit)
- # Agent-trace records (empty after one stat call unless the repo has them).
- # Reuse the orchestrator's shared instance when supplied (single file read).
- if trace_index is None:
- trace_index = AgentTraceIndex.load(repo)
-
- bucket: dict[str, list[_CommitRec]] = {}
+ labeler = _CommitLabeler(repo, skip + deep_limit, provenance_classifier, trace_index)
+ buckets = _FileBuckets(wanted_files, cap=per_file_limit)
commits_parsed = 0
-
- for record in raw.split(_RECORD_SEP):
- if not record.strip():
- continue
- parsed = _parse_commit_record(record)
- if parsed is None:
- continue
- header, numstat_lines = parsed
+ records = [record for record in raw.split(_RECORD_SEP) if record.strip()]
+ for header, numstat_lines in _commits_after(records, None):
commits_parsed += 1
-
- # Numstat parsed first so classification (below) can see the commit's
- # full changed-path set, which the agent-trace channel's file-overlap
- # check needs. Only commits that actually touch a wanted file pay the
- # provenance work (the window walk classifies every commit because
- # the commit sink needs the labels).
- changes: list[tuple[str, int, int]] = []
- moved: list[tuple[str, str]] = []
- touches_wanted = False
-
- for line in numstat_lines:
- cols = line.split("\t")
- if len(cols) < 3:
- continue
- stat_path = cols[2]
- if "=>" in stat_path:
- seen: set[str] = set()
- old_path, new_path = _extract_rename_paths(stat_path, seen)
- target = new_path or stat_path
- if old_path and new_path:
- moved.append((old_path, new_path))
- else:
- target = stat_path
- if renames is not None:
- target = renames.resolve(target)
-
- try:
- added = int(cols[0]) if cols[0] != "-" else 0
- deleted = int(cols[1]) if cols[1] != "-" else 0
- except ValueError:
- added = 0
- deleted = 0
-
- changes.append((target, added, deleted))
- if target in wanted_files:
- touches_wanted = True
-
- if renames is not None:
- for old_path, new_path in moved:
- renames.record(old_path, new_path)
- if not touches_wanted:
- continue
-
- trace_hit = (
- trace_index.resolve(header["sha"], header["parents"], {t for t, _, _ in changes})
- if trace_index
- else None
- )
- prov = provenance_classifier.classify( # type: ignore[attr-defined]
- header["author_name"],
- header["author_email"],
- header["committer_name"],
- header["committer_email"],
- f"{header['subject']}\n{header['body']}",
- note_agent=note_agents.get(header["sha"]),
- trace_agent=trace_hit[0] if trace_hit else None,
- trace_confidence=trace_hit[1] if trace_hit else "high",
- )
-
- for target, added, deleted in changes:
- if target not in wanted_files:
- continue
- records = bucket.setdefault(target, [])
- if len(records) >= per_file_limit:
- continue
-
- records.append(
- _CommitRec(
- sha=header["sha"],
- author_name=header["author_name"],
- author_email=header["author_email"],
- ts=header["ts"],
- is_merge=header["is_merge"],
- subject=header["subject"],
- body=header["body"],
- added=added,
- deleted=deleted,
- agent=prov.agent,
- agent_tier=prov.autonomy_tier,
- )
- )
+ changes, moved = _parse_numstat(numstat_lines)
+ changes = _resolved(changes, renames)
+ _record_moves(renames, moved)
+ # Only commits touching a wanted file pay for provenance (the window
+ # walk classifies every commit because the commit sink needs labels).
+ if any(target in wanted_files for target, _, _ in changes):
+ buckets.add(header, labeler.label(header, changes), changes)
logger.debug(
"deep_commit_index_built",
commits_parsed=commits_parsed,
- files_bucketed=len(bucket),
+ files_bucketed=len(buckets.records),
wanted_files=len(wanted_files),
skip=skip,
)
- if stats_sink is not None:
- stats_sink.update(succeeded=True, commits=commits_parsed)
- return bucket
+ _report(stats_sink, succeeded=True, commits=commits_parsed)
+ return buckets.records
+
+
+def _underfilled(sample: HistorySample, indexable_files: set[str], per_file_limit: int) -> set[str]:
+ return {path for path in indexable_files if len(sample.commits.get(path, ())) < per_file_limit}
+
+
+class _SharedWalks:
+ """How far the recent and deep repo-wide walks have read into the history.
+
+ Deep walks start below the recent window and step *step* commits at a
+ time. Without a reliable history total one bounded deep walk is the safe
+ limit, and unresolved files keep the per-file fallback.
+ """
+
+ def __init__(
+ self,
+ walk_deep: Callable[..., dict[str, list[_CommitRec]]],
+ *,
+ total_commits: int | None,
+ recent_commits: int,
+ start: int,
+ step: int,
+ ) -> None:
+ self._walk_deep = walk_deep
+ self._total = total_commits
+ self._step = step
+ self.cursor = start
+ self.deep_walked = False
+ self.recent_complete = total_commits is not None and total_commits <= recent_commits
+
+ def has_more(self) -> bool:
+ if self._total is None:
+ return not self.deep_walked
+ return self.cursor < self._total
+
+ def walk_next(self, sample: HistorySample, per_file_limit: int, wanted: set[str]) -> bool:
+ """Top up *sample* from the next deep region; ``False`` when that walk failed."""
+ stats: dict[str, int | bool] = {}
+ deep = self._walk_deep(wanted, skip=self.cursor, stats_sink=stats)
+ for path, records in deep.items():
+ remaining = per_file_limit - len(sample.commits.get(path, ()))
+ if remaining > 0 and records:
+ sample.commits.setdefault(path, []).extend(records[:remaining])
+ sample.deep_files.add(path)
+ if not stats.get("succeeded"):
+ return False
+ self.deep_walked = True
+ sample.deep_commits += int(stats.get("commits", 0))
+ self.cursor += self._step
+ return True
+
+ def reached_root(self) -> bool:
+ if self.recent_complete:
+ return True
+ return self.deep_walked and self._total is not None and self.cursor >= self._total
+
+ def complete_through(self, walked_depth: int) -> int:
+ return walked_depth if self._total is None else min(self._total, walked_depth)
+
+
+def _settle_fallback(
+ sample: HistorySample, indexable_files: set[str], per_file_limit: int, reached_root: bool
+) -> None:
+ """Send files the shared walks left underfilled to the per-file walk.
+
+ Not when those walks reached the repository root: then a short history is
+ the whole history.
+ """
+ if not reached_root:
+ # Ceiling: the per-file fallback log does not follow renames, so a
+ # fallback file's history still starts at its last rename. Passing the
+ # trail's old names as extra pathspecs, cut at the rename time, lifts it.
+ sample.fallback_files = _underfilled(sample, indexable_files, per_file_limit)
+ sample.deep_files.difference_update(sample.fallback_files)
+ for path in indexable_files - sample.fallback_files:
+ sample.commits.setdefault(path, [])
def load_sampled_commit_index(
@@ -703,73 +737,29 @@ def load_sampled_commit_index(
sample.fallback_files = set(indexable_files)
return sample
- total_commits = _count_non_merge_commits(repo)
- underfilled = {
- path for path in indexable_files if len(sample.commits.get(path, ())) < per_file_limit
- }
- deep_stats: dict[str, int | bool] = {}
- deep_walk_succeeded = False
- deep_cursor = per_file_limit
- recent_walk_complete = total_commits is not None and total_commits <= sample.global_commits
- while (
- not recent_walk_complete
- and len(underfilled) >= deep_threshold
- and (total_commits is None or deep_cursor < total_commits)
- ):
- deep_stats = {}
- deep = load_deep_commit_index(
+ walks = _SharedWalks(
+ functools.partial(
+ load_deep_commit_index,
repo,
per_file_limit,
- underfilled,
- skip=deep_cursor,
deep_limit=deep_limit,
provenance_classifier=provenance_classifier,
trace_index=trace_index,
- stats_sink=deep_stats,
renames=renames,
- )
- for path, records in deep.items():
- remaining = per_file_limit - len(sample.commits.get(path, ()))
- if remaining > 0 and records:
- sample.commits.setdefault(path, []).extend(records[:remaining])
- sample.deep_files.add(path)
- if not deep_stats.get("succeeded"):
- break
- deep_walk_succeeded = True
- sample.deep_commits += int(deep_stats.get("commits", 0))
- deep_cursor += deep_limit
- underfilled = {
- path
- for path in indexable_files
- if len(sample.commits.get(path, ())) < per_file_limit
- }
- # Without a reliable history total, one bounded deep walk is the safe
- # limit; unresolved files retain the per-file fallback.
- if total_commits is None:
+ ),
+ total_commits=_count_non_merge_commits(repo),
+ recent_commits=sample.global_commits,
+ start=per_file_limit,
+ step=deep_limit,
+ )
+ underfilled = _underfilled(sample, indexable_files, per_file_limit)
+ while not walks.recent_complete and len(underfilled) >= deep_threshold and walks.has_more():
+ if not walks.walk_next(sample, per_file_limit, underfilled):
break
+ underfilled = _underfilled(sample, indexable_files, per_file_limit)
- shared_walk_complete = bool(
- recent_walk_complete
- or (
- deep_walk_succeeded
- and total_commits is not None
- and deep_cursor >= total_commits
- )
+ sample.history_complete_through_depth = walks.complete_through(
+ sample.global_commits + sample.deep_commits
)
- walked_depth = sample.global_commits + sample.deep_commits
- sample.history_complete_through_depth = (
- min(total_commits, walked_depth) if total_commits is not None else walked_depth
- )
- underfilled = {
- path for path in indexable_files if len(sample.commits.get(path, ())) < per_file_limit
- }
- if not shared_walk_complete:
- # Ceiling: the per-file fallback log does not follow renames, so a
- # fallback file's history still starts at its last rename. Passing the
- # trail's old names as extra pathspecs, cut at the rename time, lifts it.
- sample.fallback_files = underfilled
- sample.deep_files.difference_update(sample.fallback_files)
-
- for path in indexable_files - sample.fallback_files:
- sample.commits.setdefault(path, [])
+ _settle_fallback(sample, indexable_files, per_file_limit, walks.reached_root())
return sample