Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(git): run index_repo setup off the event loop (#4275) (#4395)
Amane64acf691137
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.pyindex 31d20adb1..29e53f3c5 100644--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py@@ -15,7 +15,7 @@ import osimport threadingimport timefrom collections.abc import Callable, Sequence-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, NamedTuple@@ -182,6 +182,29 @@ class _FilePassInputs:trace_index: Any+@dataclass+class _IndexSetup:+ """Result of :meth:`GitIndexer._prepare_full_index`.++ ``outcome`` is ``"ready"`` or the reason ``index_repo`` returns early+ (``"no_repo"``, ``"partial_clone"``, ``"no_tracked"``).+ """++ outcome: str+ repo: Any | None = None+ tracked_files: list[str] = field(default_factory=list)+ indexable_files: list[str] = field(default_factory=list)+ history_files: set[str] = field(default_factory=set)+ commit_sink: list[dict] = field(default_factory=list)+ prov_clf: Any = None+ trace_index: Any = None+ sample: Any = None+ commit_index: Any = None+ fallback_files: set[str] = field(default_factory=set)+ fallback_note_agents: dict[str, Any] = field(default_factory=dict)+ as_of_ts: Any = None++class _ChangedScope(NamedTuple):"""An update's file sets after the allowlist and the history tier."""@@ -285,58 +308,43 @@ class GitIndexer:from repowise.core.pipeline.phase_timing import timedstart = time.monotonic()- repo = self._get_repo()- if repo is None:- return GitIndexSummary(0, 0, 0, 0.0), []- if self._skip_partial_clone_history(repo, on_warning=on_warning):+ # Everything up to the per-file pass is blocking git work (repo open,+ # ``git ls-files``, the repo-wide commit walk, trace and note loads).+ # Run it in a worker thread so it cannot stall the ingestion stage+ # that shares this event loop via ``asyncio.gather``.+ setup = await asyncio.to_thread(self._prepare_full_index, on_warning, timings)++ if setup.outcome == "no_repo":+ return GitIndexSummary(0, 0, 0, 0.0), []+ if setup.outcome == "partial_clone":if on_start is not None:on_start(0)- with contextlib.suppress(Exception):- repo.close()return GitIndexSummary(0, 0, 0, time.monotonic() - start), []-- tracked_files = self._get_tracked_files(repo)- if not tracked_files:+ if setup.outcome == "no_tracked":return GitIndexSummary(0, 0, 0, 0.0), []- # Two tiers. Code files get the full per-file pass (blame, churn- # signals, the per-file fallback walk). Every other tracked file that- # is not vendored, a lockfile or binary gets its history tier (counts,- # span, authors) from the shared repo-wide walk only.- indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]- history_files = _history_tier_files(- self.repo_path, tracked_files, extra_exclude_patterns=self._exclude_patterns- )+ repo = setup.repo+ tracked_files = setup.tracked_files+ indexable_files = setup.indexable_files+ history_files = setup.history_files+ commit_sink = setup.commit_sink+ prov_clf = setup.prov_clf+ trace_index = setup.trace_index+ sample = setup.sample+ commit_index, fallback_files = setup.commit_index, setup.fallback_files+ fallback_note_agents = setup.fallback_note_agents+ as_of_ts = setup.as_of_ts++ # Fired from the event loop, as before the setup moved to a thread.+ if on_start is not None:+ on_start(len(indexable_files) + len(history_files))from concurrent.futures import ThreadPoolExecutorworkers = git_worker_count(len(indexable_files), requested=self.max_workers)loop = asyncio.get_event_loop()- commit_sink: list[dict] = []- prov_clf = self._provenance_classifier()- # Agent-trace records, read once and shared with both commit-index walks- # and the line-share merge, so they load even when no walk runs.- trace_index = self._load_trace_index(repo)- sample = self._full_commit_index(- repo, indexable_files, history_files, commit_sink, prov_clf, trace_index, timings- )- commit_index, fallback_files = sample.commits, sample.fallback_files-- # A non-code file the shared walks could not prove complete gets no row- # (its count stays unknown) rather than a per-file walk of its own.- history_files -= fallback_files- if on_start is not None:- on_start(len(indexable_files) + len(history_files))-- fallback_note_agents: dict[str, Any] = {}- if fallback_files.intersection(indexable_files):- from ..git_commit_index import load_git_ai_note_agents-- fallback_note_agents = load_git_ai_note_agents(repo, None)-- as_of_ts = self._resolve_as_of_ts(repo)executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="git-idx")get_thread_repo, close_thread_repos = self._thread_repo_pool()inputs = _FilePassInputs(@@ -454,6 +462,76 @@ class GitIndexer:)return summary, results+ def _prepare_full_index(+ self,+ on_warning: Callable[[str], None] | None,+ timings: Any | None,+ ) -> _IndexSetup:+ """Blocking setup for :meth:`index_repo`, meant for a worker thread.++ Opens the repo, lists tracked files, splits them into the two tiers,+ runs the repo-wide commit walk and loads the trace index, git-ai notes+ and as-of timestamp. Never calls ``on_start``: that stays on the loop.+ """+ repo = self._get_repo()+ if repo is None:+ return _IndexSetup(outcome="no_repo")++ if self._skip_partial_clone_history(repo, on_warning=on_warning):+ with contextlib.suppress(Exception):+ repo.close()+ return _IndexSetup(outcome="partial_clone")++ tracked_files = self._get_tracked_files(repo)+ if not tracked_files:+ return _IndexSetup(outcome="no_tracked")++ # Two tiers. Code files get the full per-file pass (blame, churn+ # signals, the per-file fallback walk). Every other tracked file that+ # is not vendored, a lockfile or binary gets its history tier (counts,+ # span, authors) from the shared repo-wide walk only.+ indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]+ history_files = _history_tier_files(+ self.repo_path, tracked_files, extra_exclude_patterns=self._exclude_patterns+ )++ commit_sink: list[dict] = []+ prov_clf = self._provenance_classifier()+ # Agent-trace records, read once and shared with both commit-index walks+ # and the line-share merge, so they load even when no walk runs.+ trace_index = self._load_trace_index(repo)+ sample = self._full_commit_index(+ repo, indexable_files, history_files, commit_sink, prov_clf, trace_index, timings+ )+ commit_index, fallback_files = sample.commits, sample.fallback_files++ # A non-code file the shared walks could not prove complete gets no row+ # (its count stays unknown) rather than a per-file walk of its own.+ history_files -= fallback_files++ fallback_note_agents: dict[str, Any] = {}+ if fallback_files.intersection(indexable_files):+ from ..git_commit_index import load_git_ai_note_agents++ fallback_note_agents = load_git_ai_note_agents(repo, None)++ as_of_ts = self._resolve_as_of_ts(repo)+ return _IndexSetup(+ outcome="ready",+ repo=repo,+ tracked_files=tracked_files,+ indexable_files=indexable_files,+ history_files=history_files,+ commit_sink=commit_sink,+ prov_clf=prov_clf,+ trace_index=trace_index,+ sample=sample,+ commit_index=commit_index,+ fallback_files=fallback_files,+ fallback_note_agents=fallback_note_agents,+ as_of_ts=as_of_ts,+ )+def _full_commit_index(self,repo: Any,diff --git a/tests/unit/test_git_indexer.py b/tests/unit/test_git_indexer.pyindex 6a25382e5..075869667 100644--- a/tests/unit/test_git_indexer.py+++ b/tests/unit/test_git_indexer.py@@ -2,6 +2,8 @@from __future__ import annotations+import asyncio+import timefrom datetime import UTC, datetime, timedeltafrom unittest.mock import MagicMock, patch@@ -1484,3 +1486,95 @@ class TestCoChangeCommitWidth:partner = next(p for p in result.partners["a.py"] if p["file_path"] == "b.py")assert partner["frequency"] == 2assert partner["focused_frequency"] == 1+++# ---------------------------------------------------------------------------+# index_repo setup must not block the event loop+# ---------------------------------------------------------------------------+++class TestIndexRepoDoesNotBlockLoop:+ """Git setup shares a loop with ingestion, so it has to run off-loop."""++ @pytest.mark.asyncio+ async def test_blocking_setup_does_not_stall_event_loop(self) -> None:+ # Warm the lazy import index_repo does on the loop; a cold import in an+ # isolated run would otherwise show up as lag unrelated to git setup.+ import repowise.core.pipeline.phase_timing # noqa: F401++ indexer = GitIndexer("/tmp/repo")+ repo = MagicMock()+ repo.git.config.side_effect = ["", ""]++ def slow_tracked_files(_repo: object) -> list[str]:+ time.sleep(0.6) # stands in for ls-files plus the commit walk+ return []++ lags: list[float] = []+ stop = asyncio.Event()++ async def probe() -> None:+ while not stop.is_set():+ t0 = time.monotonic()+ await asyncio.sleep(0.01)+ lags.append(time.monotonic() - t0 - 0.01)++ with (+ patch.object(indexer, "_get_repo", return_value=repo),+ patch.object(indexer, "_skip_partial_clone_history", return_value=False),+ patch.object(indexer, "_get_tracked_files", side_effect=slow_tracked_files),+ ):+ task = asyncio.create_task(probe())+ await asyncio.sleep(0) # let the probe start ticking+ summary, metadata = await indexer.index_repo("test-repo-id")+ stop.set()+ await task++ assert summary.files_indexed == 0+ assert metadata == []+ assert len(lags) > 5, "probe barely ran: the loop was blocked"+ assert max(lags) < 0.2++ @pytest.mark.asyncio+ async def test_on_start_runs_on_event_loop_thread(self) -> None:+ import threading++ indexer = GitIndexer("/tmp/repo")+ repo = MagicMock()+ repo.git.config.side_effect = ["remote.origin.promisor true", "blob:none"]+ seen: list[int] = []++ with patch.object(indexer, "_get_repo", return_value=repo):+ await indexer.index_repo(+ "test-repo-id", on_start=lambda _n: seen.append(threading.get_ident())+ )++ assert seen == [threading.get_ident()]++ @pytest.mark.asyncio+ async def test_full_index_on_real_repo_end_to_end(self, tmp_path) -> None:+ """The whole path (setup thread, per-file pass, summary) on a real repo."""+ import subprocess++ def git(*args: str) -> None:+ subprocess.run(+ ["git", "-c", "user.name=T", "-c", "[email protected]", *args],+ cwd=tmp_path,+ check=True,+ capture_output=True,+ )++ git("init", "-q")+ (tmp_path / "a.py").write_text("print('a')\n")+ (tmp_path / "b.py").write_text("print('b')\n")+ git("add", ".")+ git("commit", "-q", "-m", "init")++ started: list[int] = []+ summary, metadata = await GitIndexer(str(tmp_path)).index_repo(+ "real-repo-id", on_start=started.append+ )++ assert started and started[0] >= 2+ assert summary.files_indexed >= 2+ assert {m["file_path"] for m in metadata} >= {"a.py", "b.py"}