Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(init): time every init stage and take three serial waits off its critical path (#2875)
Raghav Chamadiyaf6fb6116ae2e
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.pyindex 63d9ff616..d6f2c5b26 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py@@ -43,6 +43,11 @@ logger = structlog.get_logger(__name__)# generation finishes, so a wedged DB write can never hang the CLI._DRAIN_TIMEOUT_SECS = 60.0+# Pages written per transaction. The consumer only runs between generation's+# awaits, so pages pile up in the queue; one commit per page left a CPU-bound+# run draining thousands of single-row transactions after generation ended.+_FLUSH_BATCH = 200+async def run_generation_with_persistence(*,@@ -67,7 +72,7 @@ async def run_generation_with_persistence(upsert_repository,)from repowise.core.persistence.crud import upsert_page_from_generated- from repowise.core.pipeline import run_generation+ from repowise.core.pipeline import run_generation, timedurl = get_db_url_for_repo(repo_path)engine = create_engine(url)@@ -95,25 +100,43 @@ async def run_generation_with_persistence(sentinel = object()saved = 0- async def _consumer() -> None:- nonlocal saved- while True:- page = await queue.get()- try:- if page is sentinel:- return+ async def _write(pages: list[Any]) -> int:+ try:+ async with get_session(sf) as session:+ for page in pages:+ await upsert_page_from_generated(session, page, repo_id)+ return len(pages)+ except Exception:+ # One bad page must not cost its batch: retry them one at a time.+ written = 0+ for page in pages:try:async with get_session(sf) as session:await upsert_page_from_generated(session, page, repo_id)- saved += 1+ written += 1except Exception as exc:logger.debug("generation.incremental_persist_failed",page_id=getattr(page, "page_id", "?"),error=str(exc),)+ return written++ async def _consumer() -> None:+ nonlocal saved+ while True:+ batch = [await queue.get()]+ while len(batch) < _FLUSH_BATCH and not queue.empty():+ batch.append(queue.get_nowait())+ try:+ pages = [page for page in batch if page is not sentinel]+ if pages:+ saved += await _write(pages)+ if len(pages) < len(batch):+ returnfinally:- queue.task_done()+ for _ in batch:+ queue.task_done()consumer_task = asyncio.create_task(_consumer())@@ -131,14 +154,16 @@ async def run_generation_with_persistence()finally:await queue.put(sentinel)- try:- await asyncio.wait_for(consumer_task, timeout=_DRAIN_TIMEOUT_SECS)- except TimeoutError:- logger.warning("generation.persist_drain_timeout", saved=saved)- consumer_task.cancel()- except Exception as exc:- logger.debug("generation.persist_consumer_error", error=str(exc))- await engine.dispose()+ progress = generation_kwargs.get("progress")+ with timed(getattr(progress, "table", None), "generation.persist_drain"):+ try:+ await asyncio.wait_for(consumer_task, timeout=_DRAIN_TIMEOUT_SECS)+ except TimeoutError:+ logger.warning("generation.persist_drain_timeout", saved=saved)+ consumer_task.cancel()+ except Exception as exc:+ logger.debug("generation.persist_consumer_error", error=str(exc))+ await engine.dispose()if saved:logger.info("generation.incremental_persist_done", pages_flushed=saved)diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.pyindex 59e673700..b9f2b517b 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py@@ -185,12 +185,14 @@ async def persist_result(# transaction: it heals a best-effort checkpoint failure, and if# this write also fails the ledger below cannot claim ANALYSIS is# complete with parser-default complexity still on disk.- if getattr(result, "health_report", None) is not None:- await persist_symbol_analysis(- session, repo.id, getattr(result, "parsed_files", None)- )- await persist_analysis(result, session, repo.id)- await persist_generation(result, session, repo.id)+ with timed(timings, "persist.analysis"):+ if getattr(result, "health_report", None) is not None:+ await persist_symbol_analysis(+ session, repo.id, getattr(result, "parsed_files", None)+ )+ await persist_analysis(result, session, repo.id)+ with timed(timings, "persist.generation"):+ await persist_generation(result, session, repo.id)# persist_generation has already upserted the current pages, so the# sweep only retires structurally-keyed pages this run did not# reproduce. Without it the incremental-index path (every normaldiff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.pyindex 48fb74464..5ab88da05 100644--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py@@ -36,6 +36,7 @@ from .prior_defects import FixWalk, PriorDefects, collect_fix_commits, compute_pfrom .records import (GitHistoryCoverage,GitIndexSummary,+ RepoTotals,_CommitRec,_history_tier_files,_should_skip_index,@@ -96,6 +97,22 @@ def _available_memory_bytes() -> int | None:return None+def _release_repo(repo: Any) -> None:+ """Release *repo*'s git subprocesses and pack maps, like ``Repo.close()``.++ ``Repo.close()`` also runs two full ``gc.collect()`` passes on Windows so a+ caller can delete the checkout. Nothing here deletes it, and by the end of+ the git stage the heap holds the whole parsed graph, so each pass costs+ seconds: one close per worker thread put ~18 s on PowerToys' critical path.+ """+ import gitdb.util++ with contextlib.suppress(Exception):+ repo.git.clear_cache()+ with contextlib.suppress(Exception):+ gitdb.util.mman.collect()++def git_worker_count(pending: int,*,@@ -184,6 +201,7 @@ class GitIndexer:on_co_change_start: Callable[[int], None] | None = None,on_co_change_done: Callable[[], None] | None = None,on_warning: Callable[[str], None] | None = None,+ timings: Any | None = None,) -> tuple[GitIndexSummary, list[dict]]:"""Full index of all tracked files. Returns summary + list of metadatadicts ready for bulk upsert.@@ -200,7 +218,13 @@ class GitIndexer:completes (BEFORE per-file git indexingfinishes).on_warning(text) — reports a degraded git phase to the caller.++ ``timings`` is the run's phase table; each step below records a+ ``git.*`` row, since the stage overlaps ingestion and its tail is+ otherwise invisible."""+ from repowise.core.pipeline.phase_timing import timed+start = time.monotonic()repo = self._get_repo()if repo is None:@@ -251,20 +275,21 @@ class GitIndexer:if not self.follow_renames:from ..git_commit_index import load_sampled_commit_index- sample = load_sampled_commit_index(- repo,- self.commit_limit,- set(indexable_files) | history_files,- deep_limit=_DEEP_WALK_COMMIT_LIMIT,- # A history-tier file the shared walks leave short has no- # per-file walk to fall back on, so any one of them is worth a- # deep walk.- deep_threshold=1 if history_files else _DEEP_WALK_MIN_FALLBACK,- commit_sink=commit_sink,- provenance_classifier=prov_clf,- trace_index=trace_index,- cache_dir=self._window_cache_dir(),- )+ with timed(timings, "git.commit_index"):+ sample = load_sampled_commit_index(+ repo,+ self.commit_limit,+ set(indexable_files) | history_files,+ deep_limit=_DEEP_WALK_COMMIT_LIMIT,+ # A history-tier file the shared walks leave short has no+ # per-file walk to fall back on, so any one of them is+ # worth a deep walk.+ deep_threshold=1 if history_files else _DEEP_WALK_MIN_FALLBACK,+ commit_sink=commit_sink,+ provenance_classifier=prov_clf,+ trace_index=trace_index,+ cache_dir=self._window_cache_dir(),+ )commit_index = sample.commitsfallback_files = sample.fallback_filesrecent_files = sample.recent_files@@ -369,16 +394,25 @@ class GitIndexer:on_co_change_done()return result+ async def _file_pass() -> list[Any]:+ with timed(timings, "git.files"):+ return await asyncio.gather(*file_tasks, return_exceptions=True)++ # The fix-commit walk and the whole-history totals read nothing the+ # per-file pass produces, so they run beside it instead of after it:+ # on a large history the totals' churn walk alone is seconds of tail.try:- metadata_list, walk = await asyncio.gather(- asyncio.gather(*file_tasks, return_exceptions=True),+ metadata_list, walk, (fix_walk, prior_defects, repo_totals) = await asyncio.gather(+ _file_pass(),_co_change_task(),+ asyncio.to_thread(self._repo_wide_passes, set(indexable_files), as_of_ts, timings),)finally:# Abandon timed-out threads immediately instead of letting# asyncio.run() block for minutes during executor cleanup.- executor.shutdown(wait=False, cancel_futures=True)- close_thread_repos()+ with timed(timings, "git.release"):+ executor.shutdown(wait=False, cancel_futures=True)+ close_thread_repos()results: list[dict] = []for r in metadata_list:@@ -387,30 +421,15 @@ class GitIndexer:else:results.append(r)- # Prior-defect counts: one dedicated windowed git-log pass (NOT the- # depth-capped commit index, which under-counts the busiest files —- # exactly the ones this signal flags). Bounded to the trailing window,- # so it's cheap regardless of total repo age and leakage-free at T0.- prior_defects = PriorDefects()- fix_walk = FixWalk()- try:- fix_walk = collect_fix_commits(repo, set(indexable_files), as_of_ts=as_of_ts)- prior_defects = compute_prior_defects(- repo, set(indexable_files), as_of_ts=as_of_ts, walk=fix_walk- )- except Exception as exc:- logger.debug("prior_defect_pass_failed", error=str(exc))-- # Per-file fix events + SZZ tracing ride the same walk: the diffs are- # already parsed, so this pass only adds blame. Off the event loop, since- # it fans blame subprocesses out across its own pool for long enough that- # progress callbacks would visibly stall. Failure-isolated — the counts- # above stand on their own if tracing breaks.- fix_event_rows, built_ok = await asyncio.to_thread(self._build_fix_events, fix_walk)+ # Per-file fix events ride the fix walk: the diffs are already parsed.+ # Failure-isolated, so the prior-defect counts stand on their own.+ with timed(timings, "git.fix_events"):+ fix_event_rows, built_ok = await asyncio.to_thread(self._build_fix_events, fix_walk)- # Git-tier episodes ride the same walk for the same reason. Off the- # event loop because it writes SQLite.- await asyncio.to_thread(self._record_git_episodes, fix_walk)+ # Git-tier episodes ride the same walk. Off the event loop because it+ # writes SQLite.+ with timed(timings, "git.episodes"):+ await asyncio.to_thread(self._record_git_episodes, fix_walk)# Per-file AI line share from the agent-trace records. Keyed by path# like the aggregates below, so it merges in the same pass and@@ -474,10 +493,10 @@ class GitIndexer:fix_event_rows=fix_event_rows,fix_oldest_ts=fix_walk.oldest_fix_ts,fix_events_built=built_ok,- # Whole-history totals from cheap git calls on the still-open repo —- # true project age / commit / contributor counts for the stats page,- # which must not read them off the depth-capped sample (issue #730).- repo_totals=capture_repo_totals(repo),+ # Whole-history totals: true project age / commit / contributor+ # counts for the stats page, which must not read them off the+ # depth-capped sample (issue #730).+ repo_totals=repo_totals,history_coverage=GitHistoryCoverage(eligible_files=len(indexable_files),files_with_history=sum(@@ -496,7 +515,7 @@ class GitIndexer:workers=workers,),)- repo.close()+ _release_repo(repo)logger.info("Git indexing complete",@@ -822,7 +841,7 @@ class GitIndexer:if owner_task is not None:owner_task.remove_done_callback(_cleanup_executor)_cleanup_executor()- repo.close()+ _release_repo(repo)return resultsdef _compute_idle_decay(@@ -1041,6 +1060,40 @@ class GitIndexer:with contextlib.suppress(Exception):repo.close()+ def _repo_wide_passes(+ self, indexable_files: set[str], as_of_ts: float | None, timings: Any | None+ ) -> tuple[FixWalk, PriorDefects, RepoTotals]:+ """The fix-commit walk, its prior-defect counts, and whole-history totals.++ Runs on a worker thread beside the per-file pass, so it opens its own+ repo handle. Prior-defect counts come from one dedicated windowed+ git-log pass, NOT the depth-capped commit index, which under-counts+ the busiest files (exactly the ones this signal flags). Bounded to the+ trailing window, so it is cheap regardless of total repo age and+ leakage-free at T0.+ """+ from repowise.core.pipeline.phase_timing import timed++ prior_defects = PriorDefects()+ fix_walk = FixWalk()+ repo = self._get_repo()+ if repo is None:+ return fix_walk, prior_defects, RepoTotals()+ try:+ with timed(timings, "git.fix_walk"):+ try:+ fix_walk = collect_fix_commits(repo, indexable_files, as_of_ts=as_of_ts)+ prior_defects = compute_prior_defects(+ repo, indexable_files, as_of_ts=as_of_ts, walk=fix_walk+ )+ except Exception as exc:+ logger.debug("prior_defect_pass_failed", error=str(exc))+ with timed(timings, "git.repo_totals"):+ totals = capture_repo_totals(repo)+ finally:+ _release_repo(repo)+ return fix_walk, prior_defects, totals+def _build_fix_events(self, walk: FixWalk) -> tuple[list[dict], bool]:"""Build *walk*'s fix commits into ``(rows, built_ok)``.@@ -1203,8 +1256,7 @@ class GitIndexer:with lock:repos, created[:] = list(created), []for repo in repos:- with contextlib.suppress(Exception):- repo.close()+ _release_repo(repo)return get_thread_repo, close_alldiff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 08d39cd0b..4e512684f 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -33,6 +33,7 @@ from repowise.core.pipeline.progress import ()from repowise.core.registry import HookProgressCallback+from .phase_timing import timedfrom .phases._common import TEST_RUN_FILE_LIMIT, _phase_done, limit_to_top_pagerankfrom .phases.analysis import (_run_dead_code_analysis,@@ -676,16 +677,17 @@ async def run_pipeline(# the wiki-page backfill (in ``finalize_knowledge_graph``), so# rich page summaries win; FAST mode floors here.will_generate = generate_docs and llm_client is not None- knowledge_graph_result = curate_knowledge_graph(- knowledge_graph_result,- parsed_files=parsed_files,- graph_builder=graph_builder,- repo_structure=repo_structure,- community_info=graph_builder.community_info(),- git_meta_map=git_meta_map,- enabled=curation_enabled(),- defer_summary_floor=will_generate,- )+ with timed(getattr(progress, "table", None), "knowledge_graph.curate"):+ knowledge_graph_result = curate_knowledge_graph(+ knowledge_graph_result,+ parsed_files=parsed_files,+ graph_builder=graph_builder,+ repo_structure=repo_structure,+ community_info=graph_builder.community_info(),+ git_meta_map=git_meta_map,+ enabled=curation_enabled(),+ defer_summary_floor=will_generate,+ )except (ValueError, KeyError, RuntimeError) as cur_err:logger.error("kg_curation_failed", error=str(cur_err), exc_info=True)except (ValueError, KeyError, OSError, RuntimeError) as kg_err:@@ -701,16 +703,19 @@ async def run_pipeline(# written, so it is the only pass that can fold a paraphrase into an# existing one. By the end-of-run persist every group matches on title.if resume_controller is not None and not skip_analysis:- await resume_controller.checkpoint_analysis(- parsed_files=parsed_files,- dead_code_report=dead_code_report,- health_report=health_report,- decision_report=decision_report,- doc_drift_report=doc_drift_report,- git_metadata_list=git_metadata_list,- vector_store=vector_store,- progress=progress,- )+ # No progress phase covers this write either; time it like the INDEX+ # checkpoint so it shows in the totals.+ with timed(getattr(progress, "table", None), "persist.checkpoint_analysis"):+ await resume_controller.checkpoint_analysis(+ parsed_files=parsed_files,+ dead_code_report=dead_code_report,+ health_report=health_report,+ decision_report=decision_report,+ doc_drift_report=doc_drift_report,+ git_metadata_list=git_metadata_list,+ vector_store=vector_store,+ progress=progress,+ )# ---- Phase 3: Generation (optional) ------------------------------------generated_pages: list[Any] | None = Nonediff --git a/packages/core/src/repowise/core/pipeline/phases/git.py b/packages/core/src/repowise/core/pipeline/phases/git.pyindex 322115d4d..13a8bd446 100644--- a/packages/core/src/repowise/core/pipeline/phases/git.py+++ b/packages/core/src/repowise/core/pipeline/phases/git.py@@ -118,6 +118,7 @@ async def _run_git_indexing(on_commit_done=_on_commit_done,on_co_change_done=_on_co_change_done,on_warning=_on_warning,+ timings=getattr(progress, "table", None),)git_meta_map = {m["file_path"]: m for m in git_metadata_list}_phase_done(progress, "git")diff --git a/tests/unit/cli/test_generation_persist.py b/tests/unit/cli/test_generation_persist.pyindex bb19a048f..aff1cf132 100644--- a/tests/unit/cli/test_generation_persist.py+++ b/tests/unit/cli/test_generation_persist.py@@ -136,3 +136,35 @@ async def test_sink_failure_never_breaks_generation(repo_dir, monkeypatch):# The valid page still persisted despite the bad one.stored = await _read_db_page_ids(repo_dir)assert "delta" in stored+++async def test_batched_flush_stores_every_page_and_times_the_drain(repo_dir, monkeypatch):+ """Pages queued faster than they are written go out in batches, none lost,+ and the drain after generation is recorded in the run's phase table."""+ from types import SimpleNamespace++ from repowise.cli.commands.init_cmd import _generation_persist+ from repowise.core.pipeline import PhaseTimings++ monkeypatch.setattr(_generation_persist, "_FLUSH_BATCH", 7)+ emitted = [_page(f"p{i:02d}") for i in range(30)]++ async def fake_run_generation(*, repo_path, on_page_ready=None, prior_pages=None, **_kw):+ # No await between pages: all 30 are queued before the consumer runs.+ for p in emitted:+ on_page_ready(p)+ return emitted++ monkeypatch.setattr(+ "repowise.core.pipeline.run_generation", fake_run_generation, raising=True+ )+ table = PhaseTimings()++ await run_generation_with_persistence(+ repo_path=repo_dir,+ repo_name=repo_dir.name,+ progress=SimpleNamespace(table=table),+ )++ assert await _read_db_page_ids(repo_dir) == {p.page_id for p in emitted}+ assert "generation.persist_drain" in table.totalsdiff --git a/tests/unit/ingestion/test_git_commit_rows_integration.py b/tests/unit/ingestion/test_git_commit_rows_integration.pyindex 7e9a73894..2afa4cc63 100644--- a/tests/unit/ingestion/test_git_commit_rows_integration.py+++ b/tests/unit/ingestion/test_git_commit_rows_integration.py@@ -159,6 +159,38 @@ async def test_index_repo_captures_whole_history_totals(tmp_path) -> None:assert totals.first_commit_at is not None++async def test_index_repo_times_each_git_step(tmp_path) -> None:+ """Every step of the full index lands in the run's phase table, including+ the repo-wide passes that run beside the per-file pass."""+ import git as gitpython++ from repowise.core.pipeline import PhaseTimings++ repo = gitpython.Repo.init(tmp_path)+ _configure_author(repo, "Ada Lovelace", "[email protected]")+ _commit(repo, tmp_path / "a.py", "x = 1\n", "feat: add a")+ _commit(repo, tmp_path / "a.py", "x = 2\n", "fix: correct a")++ table = PhaseTimings()+ summary, _results = await GitIndexer(tmp_path, tier=GitIndexTier.FULL).index_repo(+ "repo1", timings=table+ )++ assert summary.repo_totals is not None+ assert summary.repo_totals.total_commit_count == 2+ for step in (+ "git.commit_index",+ "git.files",+ "git.fix_walk",+ "git.repo_totals",+ "git.fix_events",+ "git.episodes",+ ):+ assert step in table.totals, step++def test_capture_repo_totals_method(tmp_path) -> None:"""The public capture (used by the incremental update path) opens its ownrepo and returns the same whole-history record."""