Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(git-indexer): share one per-file worker pool between full and incremental indexing (#4003)
Raghav Chamadiyad6487db69dc2
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.pyindex 8e7169b5d..88d2c360a 100644--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py@@ -15,6 +15,7 @@ import osimport threadingimport timefrom collections.abc import Callable, Sequence+from dataclasses import dataclassfrom pathlib import Pathfrom typing import Any@@ -170,6 +171,54 @@ def git_worker_count(return min(resource_ceiling, override) if override is not None else resource_ceiling+@dataclass(frozen=True)+class _FilePassInputs:+ """What every per-file worker of one pass reads."""++ commit_index: dict[str, list[_CommitRec]]+ fallback_files: set[str]+ history_only: set[str]+ include_blame: bool+ as_of_ts: float | None+ provenance_classifier: Any+ note_agents: dict[str, Any]+ trace_index: Any+++def _successful_rows(raw: list[Any], failure_message: str) -> list[dict]:+ """Keep the rows of a per-file pass; log each exception it returned."""+ rows: list[dict] = []+ for r in raw:+ if isinstance(r, Exception):+ logger.warning(failure_message, error=str(r))+ else:+ rows.append(r)+ return rows+++def _apply_co_change(rows: list[dict], walk: CoChangeWalk) -> None:+ """Overlay co-change partners and change entropy from *walk* onto *rows*."""+ for meta in rows:+ fp = meta["file_path"]+ if fp in walk.partners:+ meta["co_change_partners_json"] = json.dumps(walk.partners[fp])+ if fp in walk.entropy:+ meta["change_entropy"] = walk.entropy[fp]+ if fp in walk.partner_count:+ meta["co_change_partner_count"] = walk.partner_count[fp]+ meta["co_change_mass"] = walk.partner_mass.get(fp, 0.0)+++def _apply_prior_defects(rows: list[dict], prior_defects: PriorDefects) -> None:+ """Overlay prior-defect counts onto *rows*."""+ for meta in rows:+ fp = meta["file_path"]+ if fp in prior_defects.counts:+ meta["prior_defect_count"] = prior_defects.counts[fp]+ if fp in prior_defects.raw_counts:+ meta["prior_defect_raw_count"] = prior_defects.raw_counts[fp]++class GitIndexer:"""Mines git history into the git_metadata table.@@ -339,65 +388,19 @@ class GitIndexer:fallback_note_agents = load_git_ai_note_agents(repo, None)- include_blame = self.tier.includes_blameas_of_ts = self._resolve_as_of_ts(repo)executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="git-idx")- semaphore = asyncio.Semaphore(workers)get_thread_repo, close_thread_repos = self._thread_repo_pool()-- def _index_one_sync(file_path: str) -> dict:- """Use a per-thread Repo to avoid shared-handle issues on Windows.-- The Repo is created once per worker thread and reused across files- — constructing a fresh ``gitpython.Repo`` per file costs ~40 ms- each (config + ref resolution), which dominated the git phase on- repos with thousands of files.- """- try:- thread_repo = get_thread_repo()- precomputed = (- None if file_path in fallback_files else commit_index.get(file_path, [])- )- return index_file(- thread_repo,- file_path,- repo_path=self.repo_path,- forge=self.forge,- commit_limit=self.commit_limit,- follow_renames=self.follow_renames,- include_blame=include_blame,- precomputed_commits=precomputed,- as_of_ts=as_of_ts,- provenance_classifier=prov_clf,- note_agents=fallback_note_agents,- trace_index=trace_index,- history_only=file_path in history_files,- )- except Exception:- return {"file_path": file_path}-- async def index_one(file_path: str) -> dict:- async with semaphore:- try:- result = await asyncio.wait_for(- loop.run_in_executor(executor, _index_one_sync, file_path),- timeout=_FILE_INDEX_TIMEOUT_SECS,- )- except TimeoutError:- logger.debug(- "Git indexing timed out for file — using partial data",- path=file_path,- timeout=_FILE_INDEX_TIMEOUT_SECS,- )- result = {"file_path": file_path}- except Exception as exc:- logger.debug("Git indexing failed for file", path=file_path, error=str(exc))- result = {"file_path": file_path}- if on_file_done is not None:- on_file_done()- return result-- file_tasks = [index_one(fp) for fp in [*indexable_files, *sorted(history_files)]]+ inputs = _FilePassInputs(+ commit_index=commit_index,+ fallback_files=fallback_files,+ history_only=history_files,+ include_blame=self.tier.includes_blame,+ as_of_ts=as_of_ts,+ provenance_classifier=prov_clf,+ note_agents=fallback_note_agents,+ trace_index=trace_index,+ )async def _co_change_task() -> CoChangeWalk:# ESSENTIAL tier defers co-change entirely (the expensive repo-wide@@ -426,7 +429,16 @@ class GitIndexer:async def _file_pass() -> list[Any]:with timed(timings, "git.files"):- return await asyncio.gather(*file_tasks, return_exceptions=True)+ return await self._run_per_file_pool(+ [*indexable_files, *sorted(history_files)],+ inputs,+ executor=executor,+ get_thread_repo=get_thread_repo,+ workers=workers,+ failure_event="Git indexing failed for file",+ timeout_event="Git indexing timed out for file — using partial data",+ on_file_done=on_file_done,+ )# The fix-commit walk and the whole-history totals read nothing the# per-file pass produces, so they run beside it instead of after it:@@ -444,12 +456,7 @@ class GitIndexer:executor.shutdown(wait=False, cancel_futures=True)close_thread_repos()- results: list[dict] = []- for r in metadata_list:- if isinstance(r, Exception):- logger.warning("Failed to index file", error=str(r))- else:- results.append(r)+ results = _successful_rows(metadata_list, "Failed to index file")# Per-file fix events ride the fix walk: the diffs are already parsed.# Failure-isolated, so the prior-defect counts stand on their own.@@ -470,20 +477,10 @@ class GitIndexer:# share into metadata.# History-tier rows take none of these; they are code signals.code_results = [meta for meta in results if meta["file_path"] not in history_files]+ _apply_co_change(code_results, walk)+ _apply_prior_defects(code_results, prior_defects)for meta in code_results:- fp = meta["file_path"]- if fp in walk.partners:- meta["co_change_partners_json"] = json.dumps(walk.partners[fp])- if fp in walk.entropy:- meta["change_entropy"] = walk.entropy[fp]- if fp in walk.partner_count:- meta["co_change_partner_count"] = walk.partner_count[fp]- meta["co_change_mass"] = walk.partner_mass.get(fp, 0.0)- if fp in prior_defects.counts:- meta["prior_defect_count"] = prior_defects.counts[fp]- if fp in prior_defects.raw_counts:- meta["prior_defect_raw_count"] = prior_defects.raw_counts[fp]- share = trace_line_shares.get(fp)+ share = trace_line_shares.get(meta["file_path"])if share:meta["agent_line_count"] = share[0]meta["agent_line_model_json"] = json.dumps(share[1])@@ -645,7 +642,6 @@ class GitIndexer:all_files = {fp for fp in all_files if not _should_skip_index(fp)}loop = asyncio.get_event_loop()- include_blame = self.tier.includes_blame# Whether to refresh idle files' decay fields this run. Requires the# repo-wide commit index (rename-tracking mode has none) and the@@ -701,7 +697,6 @@ class GitIndexer:fallback_note_agents = load_git_ai_note_agents(repo, None)as_of_ts = self._resolve_as_of_ts(repo)executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="git-idx")- semaphore = asyncio.Semaphore(workers)get_thread_repo, close_thread_repos = self._thread_repo_pool()executor_closed = False@@ -717,49 +712,26 @@ class GitIndexer:if owner_task is not None:owner_task.add_done_callback(_cleanup_executor)- def _index_one_sync(file_path: str) -> dict:- try:- thread_repo = get_thread_repo()- precomputed = (- None if file_path in fallback_files else commit_index.get(file_path, [])- )- return index_file(- thread_repo,- file_path,- repo_path=self.repo_path,- forge=self.forge,- commit_limit=self.commit_limit,- follow_renames=self.follow_renames,- include_blame=include_blame,- precomputed_commits=precomputed,- as_of_ts=as_of_ts,- provenance_classifier=prov_clf,- note_agents=fallback_note_agents,- trace_index=trace_index,- history_only=file_path in history_changed,- )- except Exception:- return {"file_path": file_path}-- async def index_one(file_path: str) -> dict:- async with semaphore:- try:- return await asyncio.wait_for(- loop.run_in_executor(executor, _index_one_sync, file_path),- timeout=_FILE_INDEX_TIMEOUT_SECS,- )- except (TimeoutError, Exception) as exc:- logger.debug(- "Git indexing failed for changed file",- path=file_path,- error=str(exc),- )- return {"file_path": file_path}-- tasks = [index_one(fp) for fp in [*changed_file_paths, *sorted(history_changed)]]+ inputs = _FilePassInputs(+ commit_index=commit_index,+ fallback_files=fallback_files,+ history_only=history_changed,+ include_blame=self.tier.includes_blame,+ as_of_ts=as_of_ts,+ provenance_classifier=prov_clf,+ note_agents=fallback_note_agents,+ trace_index=trace_index,+ )try:with timed(timings, "rebuild.git.changed_files"):- results_raw = await asyncio.gather(*tasks, return_exceptions=True)+ results_raw = await self._run_per_file_pool(+ [*changed_file_paths, *sorted(history_changed)],+ inputs,+ executor=executor,+ get_thread_repo=get_thread_repo,+ workers=workers,+ failure_event="Git indexing failed for changed file",+ )except BaseException:# Cancellation is a BaseException on supported Python versions.# Release the private pool immediately even when the caller catches@@ -769,12 +741,7 @@ class GitIndexer:repo.close()raise- results: list[dict] = []- for r in results_raw:- if isinstance(r, Exception):- logger.warning("Failed to index changed file", error=str(r))- else:- results.append(r)+ results = _successful_rows(results_raw, "Failed to index changed file")# Idle files (window commits, but not in this change set) whose decay# fields we will refresh — computed here so the prior-defect pass below@@ -805,12 +772,7 @@ class GitIndexer:| set(idle_paths).difference(history_files),as_of_ts=as_of_ts,)- for meta in code_results:- fp = meta["file_path"]- if fp in prior_defects.counts:- meta["prior_defect_count"] = prior_defects.counts[fp]- if fp in prior_defects.raw_counts:- meta["prior_defect_raw_count"] = prior_defects.raw_counts[fp]+ _apply_prior_defects(code_results, prior_defects)except Exception as exc:logger.debug("prior_defect_pass_failed", error=str(exc))prior_defects = PriorDefects()@@ -835,15 +797,7 @@ class GitIndexer:None,as_of_ts,)- for meta in code_results:- fp = meta["file_path"]- if fp in walk.partners:- meta["co_change_partners_json"] = json.dumps(walk.partners[fp])- if fp in walk.entropy:- meta["change_entropy"] = walk.entropy[fp]- if fp in walk.partner_count:- meta["co_change_partner_count"] = walk.partner_count[fp]- meta["co_change_mass"] = walk.partner_mass.get(fp, 0.0)+ _apply_co_change(code_results, walk)if co_change_sink is not None:co_change_sink.update(walk.partners)@@ -883,6 +837,90 @@ class GitIndexer:_release_repo(repo)return results+ def _index_one_file(+ self, get_thread_repo: Callable[[], Any], file_path: str, inputs: _FilePassInputs+ ) -> dict:+ """Index one file on a worker thread; any failure yields a bare row.++ Uses a per-thread Repo to avoid shared-handle issues on Windows. The+ Repo is created once per worker thread and reused across files —+ constructing a fresh ``gitpython.Repo`` per file costs ~40 ms each+ (config + ref resolution), which dominated the git phase on repos+ with thousands of files.+ """+ try:+ thread_repo = get_thread_repo()+ precomputed = (+ None+ if file_path in inputs.fallback_files+ else inputs.commit_index.get(file_path, [])+ )+ return index_file(+ thread_repo,+ file_path,+ repo_path=self.repo_path,+ forge=self.forge,+ commit_limit=self.commit_limit,+ follow_renames=self.follow_renames,+ include_blame=inputs.include_blame,+ precomputed_commits=precomputed,+ as_of_ts=inputs.as_of_ts,+ provenance_classifier=inputs.provenance_classifier,+ note_agents=inputs.note_agents,+ trace_index=inputs.trace_index,+ history_only=file_path in inputs.history_only,+ )+ except Exception:+ return {"file_path": file_path}++ async def _run_per_file_pool(+ self,+ files: list[str],+ inputs: _FilePassInputs,+ *,+ executor: Any,+ get_thread_repo: Callable[[], Any],+ workers: int,+ failure_event: str,+ timeout_event: str | None = None,+ on_file_done: Callable[[], None] | None = None,+ ) -> list[Any]:+ """Index *files* on *executor*, at most *workers* at a time.++ Returns one entry per file in input order (``gather`` with+ ``return_exceptions``). A file that times out or fails yields a bare+ ``{"file_path": ...}`` row; a timeout logs *timeout_event* when given,+ else *failure_event*.+ """+ loop = asyncio.get_event_loop()+ semaphore = asyncio.Semaphore(workers)++ async def index_one(file_path: str) -> dict:+ async with semaphore:+ try:+ result = await asyncio.wait_for(+ loop.run_in_executor(+ executor, self._index_one_file, get_thread_repo, file_path, inputs+ ),+ timeout=_FILE_INDEX_TIMEOUT_SECS,+ )+ except TimeoutError as exc:+ if timeout_event is None:+ logger.debug(failure_event, path=file_path, error=str(exc))+ else:+ logger.debug(+ timeout_event, path=file_path, timeout=_FILE_INDEX_TIMEOUT_SECS+ )+ result = {"file_path": file_path}+ except Exception as exc:+ logger.debug(failure_event, path=file_path, error=str(exc))+ result = {"file_path": file_path}+ if on_file_done is not None:+ on_file_done()+ return result++ return await asyncio.gather(*[index_one(fp) for fp in files], return_exceptions=True)+def _compute_idle_decay(self,repo: Any,