Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(git): run index_repo setup off the event loop (#4275) (#4395)

Amane64acf691137
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
index 31d20adb1..29e53f3c5 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
@@ -15,7 +15,7 @@ import os
import threading
import time
from collections.abc import Callable, Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, NamedTuple
@@ -182,6 +182,29 @@ class _FilePassInputs:
trace_index: Any
+@dataclass
+class _IndexSetup:
+ """Result of :meth:`GitIndexer._prepare_full_index`.
+
+ ``outcome`` is ``"ready"`` or the reason ``index_repo`` returns early
+ (``"no_repo"``, ``"partial_clone"``, ``"no_tracked"``).
+ """
+
+ outcome: str
+ repo: Any | None = None
+ tracked_files: list[str] = field(default_factory=list)
+ indexable_files: list[str] = field(default_factory=list)
+ history_files: set[str] = field(default_factory=set)
+ commit_sink: list[dict] = field(default_factory=list)
+ prov_clf: Any = None
+ trace_index: Any = None
+ sample: Any = None
+ commit_index: Any = None
+ fallback_files: set[str] = field(default_factory=set)
+ fallback_note_agents: dict[str, Any] = field(default_factory=dict)
+ as_of_ts: Any = None
+
+
class _ChangedScope(NamedTuple):
"""An update's file sets after the allowlist and the history tier."""
@@ -285,58 +308,43 @@ class GitIndexer:
from repowise.core.pipeline.phase_timing import timed
start = time.monotonic()
- repo = self._get_repo()
- if repo is None:
- return GitIndexSummary(0, 0, 0, 0.0), []
- if self._skip_partial_clone_history(repo, on_warning=on_warning):
+ # Everything up to the per-file pass is blocking git work (repo open,
+ # ``git ls-files``, the repo-wide commit walk, trace and note loads).
+ # Run it in a worker thread so it cannot stall the ingestion stage
+ # that shares this event loop via ``asyncio.gather``.
+ setup = await asyncio.to_thread(self._prepare_full_index, on_warning, timings)
+
+ if setup.outcome == "no_repo":
+ return GitIndexSummary(0, 0, 0, 0.0), []
+ if setup.outcome == "partial_clone":
if on_start is not None:
on_start(0)
- with contextlib.suppress(Exception):
- repo.close()
return GitIndexSummary(0, 0, 0, time.monotonic() - start), []
-
- tracked_files = self._get_tracked_files(repo)
- if not tracked_files:
+ if setup.outcome == "no_tracked":
return GitIndexSummary(0, 0, 0, 0.0), []
- # Two tiers. Code files get the full per-file pass (blame, churn
- # signals, the per-file fallback walk). Every other tracked file that
- # is not vendored, a lockfile or binary gets its history tier (counts,
- # span, authors) from the shared repo-wide walk only.
- indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]
- history_files = _history_tier_files(
- self.repo_path, tracked_files, extra_exclude_patterns=self._exclude_patterns
- )
+ repo = setup.repo
+ tracked_files = setup.tracked_files
+ indexable_files = setup.indexable_files
+ history_files = setup.history_files
+ commit_sink = setup.commit_sink
+ prov_clf = setup.prov_clf
+ trace_index = setup.trace_index
+ sample = setup.sample
+ commit_index, fallback_files = setup.commit_index, setup.fallback_files
+ fallback_note_agents = setup.fallback_note_agents
+ as_of_ts = setup.as_of_ts
+
+ # Fired from the event loop, as before the setup moved to a thread.
+ if on_start is not None:
+ on_start(len(indexable_files) + len(history_files))
from concurrent.futures import ThreadPoolExecutor
workers = git_worker_count(len(indexable_files), requested=self.max_workers)
loop = asyncio.get_event_loop()
- commit_sink: list[dict] = []
- prov_clf = self._provenance_classifier()
- # Agent-trace records, read once and shared with both commit-index walks
- # and the line-share merge, so they load even when no walk runs.
- trace_index = self._load_trace_index(repo)
- sample = self._full_commit_index(
- repo, indexable_files, history_files, commit_sink, prov_clf, trace_index, timings
- )
- commit_index, fallback_files = sample.commits, sample.fallback_files
-
- # A non-code file the shared walks could not prove complete gets no row
- # (its count stays unknown) rather than a per-file walk of its own.
- history_files -= fallback_files
- if on_start is not None:
- on_start(len(indexable_files) + len(history_files))
-
- fallback_note_agents: dict[str, Any] = {}
- if fallback_files.intersection(indexable_files):
- from ..git_commit_index import load_git_ai_note_agents
-
- fallback_note_agents = load_git_ai_note_agents(repo, None)
-
- as_of_ts = self._resolve_as_of_ts(repo)
executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="git-idx")
get_thread_repo, close_thread_repos = self._thread_repo_pool()
inputs = _FilePassInputs(
@@ -454,6 +462,76 @@ class GitIndexer:
)
return summary, results
+ def _prepare_full_index(
+ self,
+ on_warning: Callable[[str], None] | None,
+ timings: Any | None,
+ ) -> _IndexSetup:
+ """Blocking setup for :meth:`index_repo`, meant for a worker thread.
+
+ Opens the repo, lists tracked files, splits them into the two tiers,
+ runs the repo-wide commit walk and loads the trace index, git-ai notes
+ and as-of timestamp. Never calls ``on_start``: that stays on the loop.
+ """
+ repo = self._get_repo()
+ if repo is None:
+ return _IndexSetup(outcome="no_repo")
+
+ if self._skip_partial_clone_history(repo, on_warning=on_warning):
+ with contextlib.suppress(Exception):
+ repo.close()
+ return _IndexSetup(outcome="partial_clone")
+
+ tracked_files = self._get_tracked_files(repo)
+ if not tracked_files:
+ return _IndexSetup(outcome="no_tracked")
+
+ # Two tiers. Code files get the full per-file pass (blame, churn
+ # signals, the per-file fallback walk). Every other tracked file that
+ # is not vendored, a lockfile or binary gets its history tier (counts,
+ # span, authors) from the shared repo-wide walk only.
+ indexable_files = [fp for fp in tracked_files if not _should_skip_index(fp)]
+ history_files = _history_tier_files(
+ self.repo_path, tracked_files, extra_exclude_patterns=self._exclude_patterns
+ )
+
+ commit_sink: list[dict] = []
+ prov_clf = self._provenance_classifier()
+ # Agent-trace records, read once and shared with both commit-index walks
+ # and the line-share merge, so they load even when no walk runs.
+ trace_index = self._load_trace_index(repo)
+ sample = self._full_commit_index(
+ repo, indexable_files, history_files, commit_sink, prov_clf, trace_index, timings
+ )
+ commit_index, fallback_files = sample.commits, sample.fallback_files
+
+ # A non-code file the shared walks could not prove complete gets no row
+ # (its count stays unknown) rather than a per-file walk of its own.
+ history_files -= fallback_files
+
+ fallback_note_agents: dict[str, Any] = {}
+ if fallback_files.intersection(indexable_files):
+ from ..git_commit_index import load_git_ai_note_agents
+
+ fallback_note_agents = load_git_ai_note_agents(repo, None)
+
+ as_of_ts = self._resolve_as_of_ts(repo)
+ return _IndexSetup(
+ outcome="ready",
+ repo=repo,
+ tracked_files=tracked_files,
+ indexable_files=indexable_files,
+ history_files=history_files,
+ commit_sink=commit_sink,
+ prov_clf=prov_clf,
+ trace_index=trace_index,
+ sample=sample,
+ commit_index=commit_index,
+ fallback_files=fallback_files,
+ fallback_note_agents=fallback_note_agents,
+ as_of_ts=as_of_ts,
+ )
+
def _full_commit_index(
self,
repo: Any,
diff --git a/tests/unit/test_git_indexer.py b/tests/unit/test_git_indexer.py
index 6a25382e5..075869667 100644
--- a/tests/unit/test_git_indexer.py
+++ b/tests/unit/test_git_indexer.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import asyncio
+import time
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
@@ -1484,3 +1486,95 @@ class TestCoChangeCommitWidth:
partner = next(p for p in result.partners["a.py"] if p["file_path"] == "b.py")
assert partner["frequency"] == 2
assert partner["focused_frequency"] == 1
+
+
+# ---------------------------------------------------------------------------
+# index_repo setup must not block the event loop
+# ---------------------------------------------------------------------------
+
+
+class TestIndexRepoDoesNotBlockLoop:
+ """Git setup shares a loop with ingestion, so it has to run off-loop."""
+
+ @pytest.mark.asyncio
+ async def test_blocking_setup_does_not_stall_event_loop(self) -> None:
+ # Warm the lazy import index_repo does on the loop; a cold import in an
+ # isolated run would otherwise show up as lag unrelated to git setup.
+ import repowise.core.pipeline.phase_timing # noqa: F401
+
+ indexer = GitIndexer("/tmp/repo")
+ repo = MagicMock()
+ repo.git.config.side_effect = ["", ""]
+
+ def slow_tracked_files(_repo: object) -> list[str]:
+ time.sleep(0.6) # stands in for ls-files plus the commit walk
+ return []
+
+ lags: list[float] = []
+ stop = asyncio.Event()
+
+ async def probe() -> None:
+ while not stop.is_set():
+ t0 = time.monotonic()
+ await asyncio.sleep(0.01)
+ lags.append(time.monotonic() - t0 - 0.01)
+
+ with (
+ patch.object(indexer, "_get_repo", return_value=repo),
+ patch.object(indexer, "_skip_partial_clone_history", return_value=False),
+ patch.object(indexer, "_get_tracked_files", side_effect=slow_tracked_files),
+ ):
+ task = asyncio.create_task(probe())
+ await asyncio.sleep(0) # let the probe start ticking
+ summary, metadata = await indexer.index_repo("test-repo-id")
+ stop.set()
+ await task
+
+ assert summary.files_indexed == 0
+ assert metadata == []
+ assert len(lags) > 5, "probe barely ran: the loop was blocked"
+ assert max(lags) < 0.2
+
+ @pytest.mark.asyncio
+ async def test_on_start_runs_on_event_loop_thread(self) -> None:
+ import threading
+
+ indexer = GitIndexer("/tmp/repo")
+ repo = MagicMock()
+ repo.git.config.side_effect = ["remote.origin.promisor true", "blob:none"]
+ seen: list[int] = []
+
+ with patch.object(indexer, "_get_repo", return_value=repo):
+ await indexer.index_repo(
+ "test-repo-id", on_start=lambda _n: seen.append(threading.get_ident())
+ )
+
+ assert seen == [threading.get_ident()]
+
+ @pytest.mark.asyncio
+ async def test_full_index_on_real_repo_end_to_end(self, tmp_path) -> None:
+ """The whole path (setup thread, per-file pass, summary) on a real repo."""
+ import subprocess
+
+ def git(*args: str) -> None:
+ subprocess.run(
+ ["git", "-c", "user.name=T", "-c", "[email protected]", *args],
+ cwd=tmp_path,
+ check=True,
+ capture_output=True,
+ )
+
+ git("init", "-q")
+ (tmp_path / "a.py").write_text("print('a')\n")
+ (tmp_path / "b.py").write_text("print('b')\n")
+ git("add", ".")
+ git("commit", "-q", "-m", "init")
+
+ started: list[int] = []
+ summary, metadata = await GitIndexer(str(tmp_path)).index_repo(
+ "real-repo-id", on_start=started.append
+ )
+
+ assert started and started[0] >= 2
+ assert summary.files_indexed >= 2
+ assert {m["file_path"] for m in metadata} >= {"a.py", "b.py"}