Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(init): time every init stage and take three serial waits off its critical path (#2875)

Raghav Chamadiyaf6fb6116ae2e
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py
index 63d9ff616..d6f2c5b26 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py
@@ -43,6 +43,11 @@ logger = structlog.get_logger(__name__)
# generation finishes, so a wedged DB write can never hang the CLI.
_DRAIN_TIMEOUT_SECS = 60.0
+# Pages written per transaction. The consumer only runs between generation's
+# awaits, so pages pile up in the queue; one commit per page left a CPU-bound
+# run draining thousands of single-row transactions after generation ended.
+_FLUSH_BATCH = 200
+
async def run_generation_with_persistence(
*,
@@ -67,7 +72,7 @@ async def run_generation_with_persistence(
upsert_repository,
)
from repowise.core.persistence.crud import upsert_page_from_generated
- from repowise.core.pipeline import run_generation
+ from repowise.core.pipeline import run_generation, timed
url = get_db_url_for_repo(repo_path)
engine = create_engine(url)
@@ -95,25 +100,43 @@ async def run_generation_with_persistence(
sentinel = object()
saved = 0
- async def _consumer() -> None:
- nonlocal saved
- while True:
- page = await queue.get()
- try:
- if page is sentinel:
- return
+ async def _write(pages: list[Any]) -> int:
+ try:
+ async with get_session(sf) as session:
+ for page in pages:
+ await upsert_page_from_generated(session, page, repo_id)
+ return len(pages)
+ except Exception:
+ # One bad page must not cost its batch: retry them one at a time.
+ written = 0
+ for page in pages:
try:
async with get_session(sf) as session:
await upsert_page_from_generated(session, page, repo_id)
- saved += 1
+ written += 1
except Exception as exc:
logger.debug(
"generation.incremental_persist_failed",
page_id=getattr(page, "page_id", "?"),
error=str(exc),
)
+ return written
+
+ async def _consumer() -> None:
+ nonlocal saved
+ while True:
+ batch = [await queue.get()]
+ while len(batch) < _FLUSH_BATCH and not queue.empty():
+ batch.append(queue.get_nowait())
+ try:
+ pages = [page for page in batch if page is not sentinel]
+ if pages:
+ saved += await _write(pages)
+ if len(pages) < len(batch):
+ return
finally:
- queue.task_done()
+ for _ in batch:
+ queue.task_done()
consumer_task = asyncio.create_task(_consumer())
@@ -131,14 +154,16 @@ async def run_generation_with_persistence(
)
finally:
await queue.put(sentinel)
- try:
- await asyncio.wait_for(consumer_task, timeout=_DRAIN_TIMEOUT_SECS)
- except TimeoutError:
- logger.warning("generation.persist_drain_timeout", saved=saved)
- consumer_task.cancel()
- except Exception as exc:
- logger.debug("generation.persist_consumer_error", error=str(exc))
- await engine.dispose()
+ progress = generation_kwargs.get("progress")
+ with timed(getattr(progress, "table", None), "generation.persist_drain"):
+ try:
+ await asyncio.wait_for(consumer_task, timeout=_DRAIN_TIMEOUT_SECS)
+ except TimeoutError:
+ logger.warning("generation.persist_drain_timeout", saved=saved)
+ consumer_task.cancel()
+ except Exception as exc:
+ logger.debug("generation.persist_consumer_error", error=str(exc))
+ await engine.dispose()
if saved:
logger.info("generation.incremental_persist_done", pages_flushed=saved)
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index 59e673700..b9f2b517b 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -185,12 +185,14 @@ async def persist_result(
# transaction: it heals a best-effort checkpoint failure, and if
# this write also fails the ledger below cannot claim ANALYSIS is
# complete with parser-default complexity still on disk.
- if getattr(result, "health_report", None) is not None:
- await persist_symbol_analysis(
- session, repo.id, getattr(result, "parsed_files", None)
- )
- await persist_analysis(result, session, repo.id)
- await persist_generation(result, session, repo.id)
+ with timed(timings, "persist.analysis"):
+ if getattr(result, "health_report", None) is not None:
+ await persist_symbol_analysis(
+ session, repo.id, getattr(result, "parsed_files", None)
+ )
+ await persist_analysis(result, session, repo.id)
+ with timed(timings, "persist.generation"):
+ await persist_generation(result, session, repo.id)
# persist_generation has already upserted the current pages, so the
# sweep only retires structurally-keyed pages this run did not
# reproduce. Without it the incremental-index path (every normal
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
index 48fb74464..5ab88da05 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
@@ -36,6 +36,7 @@ from .prior_defects import FixWalk, PriorDefects, collect_fix_commits, compute_p
from .records import (
GitHistoryCoverage,
GitIndexSummary,
+ RepoTotals,
_CommitRec,
_history_tier_files,
_should_skip_index,
@@ -96,6 +97,22 @@ def _available_memory_bytes() -> int | None:
return None
+def _release_repo(repo: Any) -> None:
+ """Release *repo*'s git subprocesses and pack maps, like ``Repo.close()``.
+
+ ``Repo.close()`` also runs two full ``gc.collect()`` passes on Windows so a
+ caller can delete the checkout. Nothing here deletes it, and by the end of
+ the git stage the heap holds the whole parsed graph, so each pass costs
+ seconds: one close per worker thread put ~18 s on PowerToys' critical path.
+ """
+ import gitdb.util
+
+ with contextlib.suppress(Exception):
+ repo.git.clear_cache()
+ with contextlib.suppress(Exception):
+ gitdb.util.mman.collect()
+
+
def git_worker_count(
pending: int,
*,
@@ -184,6 +201,7 @@ class GitIndexer:
on_co_change_start: Callable[[int], None] | None = None,
on_co_change_done: Callable[[], None] | None = None,
on_warning: Callable[[str], None] | None = None,
+ timings: Any | None = None,
) -> tuple[GitIndexSummary, list[dict]]:
"""Full index of all tracked files. Returns summary + list of metadata
dicts ready for bulk upsert.
@@ -200,7 +218,13 @@ class GitIndexer:
completes (BEFORE per-file git indexing
finishes).
on_warning(text) — reports a degraded git phase to the caller.
+
+ ``timings`` is the run's phase table; each step below records a
+ ``git.*`` row, since the stage overlaps ingestion and its tail is
+ otherwise invisible.
"""
+ from repowise.core.pipeline.phase_timing import timed
+
start = time.monotonic()
repo = self._get_repo()
if repo is None:
@@ -251,20 +275,21 @@ class GitIndexer:
if not self.follow_renames:
from ..git_commit_index import load_sampled_commit_index
- sample = load_sampled_commit_index(
- repo,
- self.commit_limit,
- set(indexable_files) | history_files,
- deep_limit=_DEEP_WALK_COMMIT_LIMIT,
- # A history-tier file the shared walks leave short has no
- # per-file walk to fall back on, so any one of them is worth a
- # deep walk.
- deep_threshold=1 if history_files else _DEEP_WALK_MIN_FALLBACK,
- commit_sink=commit_sink,
- provenance_classifier=prov_clf,
- trace_index=trace_index,
- cache_dir=self._window_cache_dir(),
- )
+ with timed(timings, "git.commit_index"):
+ sample = load_sampled_commit_index(
+ repo,
+ self.commit_limit,
+ set(indexable_files) | history_files,
+ deep_limit=_DEEP_WALK_COMMIT_LIMIT,
+ # A history-tier file the shared walks leave short has no
+ # per-file walk to fall back on, so any one of them is
+ # worth a deep walk.
+ deep_threshold=1 if history_files else _DEEP_WALK_MIN_FALLBACK,
+ commit_sink=commit_sink,
+ provenance_classifier=prov_clf,
+ trace_index=trace_index,
+ cache_dir=self._window_cache_dir(),
+ )
commit_index = sample.commits
fallback_files = sample.fallback_files
recent_files = sample.recent_files
@@ -369,16 +394,25 @@ class GitIndexer:
on_co_change_done()
return result
+ async def _file_pass() -> list[Any]:
+ with timed(timings, "git.files"):
+ return await asyncio.gather(*file_tasks, return_exceptions=True)
+
+ # The fix-commit walk and the whole-history totals read nothing the
+ # per-file pass produces, so they run beside it instead of after it:
+ # on a large history the totals' churn walk alone is seconds of tail.
try:
- metadata_list, walk = await asyncio.gather(
- asyncio.gather(*file_tasks, return_exceptions=True),
+ metadata_list, walk, (fix_walk, prior_defects, repo_totals) = await asyncio.gather(
+ _file_pass(),
_co_change_task(),
+ asyncio.to_thread(self._repo_wide_passes, set(indexable_files), as_of_ts, timings),
)
finally:
# Abandon timed-out threads immediately instead of letting
# asyncio.run() block for minutes during executor cleanup.
- executor.shutdown(wait=False, cancel_futures=True)
- close_thread_repos()
+ with timed(timings, "git.release"):
+ executor.shutdown(wait=False, cancel_futures=True)
+ close_thread_repos()
results: list[dict] = []
for r in metadata_list:
@@ -387,30 +421,15 @@ class GitIndexer:
else:
results.append(r)
- # Prior-defect counts: one dedicated windowed git-log pass (NOT the
- # depth-capped commit index, which under-counts the busiest files —
- # exactly the ones this signal flags). Bounded to the trailing window,
- # so it's cheap regardless of total repo age and leakage-free at T0.
- prior_defects = PriorDefects()
- fix_walk = FixWalk()
- try:
- fix_walk = collect_fix_commits(repo, set(indexable_files), as_of_ts=as_of_ts)
- prior_defects = compute_prior_defects(
- repo, set(indexable_files), as_of_ts=as_of_ts, walk=fix_walk
- )
- except Exception as exc:
- logger.debug("prior_defect_pass_failed", error=str(exc))
-
- # Per-file fix events + SZZ tracing ride the same walk: the diffs are
- # already parsed, so this pass only adds blame. Off the event loop, since
- # it fans blame subprocesses out across its own pool for long enough that
- # progress callbacks would visibly stall. Failure-isolated — the counts
- # above stand on their own if tracing breaks.
- fix_event_rows, built_ok = await asyncio.to_thread(self._build_fix_events, fix_walk)
+ # Per-file fix events ride the fix walk: the diffs are already parsed.
+ # Failure-isolated, so the prior-defect counts stand on their own.
+ with timed(timings, "git.fix_events"):
+ fix_event_rows, built_ok = await asyncio.to_thread(self._build_fix_events, fix_walk)
- # Git-tier episodes ride the same walk for the same reason. Off the
- # event loop because it writes SQLite.
- await asyncio.to_thread(self._record_git_episodes, fix_walk)
+ # Git-tier episodes ride the same walk. Off the event loop because it
+ # writes SQLite.
+ with timed(timings, "git.episodes"):
+ await asyncio.to_thread(self._record_git_episodes, fix_walk)
# Per-file AI line share from the agent-trace records. Keyed by path
# like the aggregates below, so it merges in the same pass and
@@ -474,10 +493,10 @@ class GitIndexer:
fix_event_rows=fix_event_rows,
fix_oldest_ts=fix_walk.oldest_fix_ts,
fix_events_built=built_ok,
- # Whole-history totals from cheap git calls on the still-open repo —
- # true project age / commit / contributor counts for the stats page,
- # which must not read them off the depth-capped sample (issue #730).
- repo_totals=capture_repo_totals(repo),
+ # Whole-history totals: true project age / commit / contributor
+ # counts for the stats page, which must not read them off the
+ # depth-capped sample (issue #730).
+ repo_totals=repo_totals,
history_coverage=GitHistoryCoverage(
eligible_files=len(indexable_files),
files_with_history=sum(
@@ -496,7 +515,7 @@ class GitIndexer:
workers=workers,
),
)
- repo.close()
+ _release_repo(repo)
logger.info(
"Git indexing complete",
@@ -822,7 +841,7 @@ class GitIndexer:
if owner_task is not None:
owner_task.remove_done_callback(_cleanup_executor)
_cleanup_executor()
- repo.close()
+ _release_repo(repo)
return results
def _compute_idle_decay(
@@ -1041,6 +1060,40 @@ class GitIndexer:
with contextlib.suppress(Exception):
repo.close()
+ def _repo_wide_passes(
+ self, indexable_files: set[str], as_of_ts: float | None, timings: Any | None
+ ) -> tuple[FixWalk, PriorDefects, RepoTotals]:
+ """The fix-commit walk, its prior-defect counts, and whole-history totals.
+
+ Runs on a worker thread beside the per-file pass, so it opens its own
+ repo handle. Prior-defect counts come from one dedicated windowed
+ git-log pass, NOT the depth-capped commit index, which under-counts
+ the busiest files (exactly the ones this signal flags). Bounded to the
+ trailing window, so it is cheap regardless of total repo age and
+ leakage-free at T0.
+ """
+ from repowise.core.pipeline.phase_timing import timed
+
+ prior_defects = PriorDefects()
+ fix_walk = FixWalk()
+ repo = self._get_repo()
+ if repo is None:
+ return fix_walk, prior_defects, RepoTotals()
+ try:
+ with timed(timings, "git.fix_walk"):
+ try:
+ fix_walk = collect_fix_commits(repo, indexable_files, as_of_ts=as_of_ts)
+ prior_defects = compute_prior_defects(
+ repo, indexable_files, as_of_ts=as_of_ts, walk=fix_walk
+ )
+ except Exception as exc:
+ logger.debug("prior_defect_pass_failed", error=str(exc))
+ with timed(timings, "git.repo_totals"):
+ totals = capture_repo_totals(repo)
+ finally:
+ _release_repo(repo)
+ return fix_walk, prior_defects, totals
+
def _build_fix_events(self, walk: FixWalk) -> tuple[list[dict], bool]:
"""Build *walk*'s fix commits into ``(rows, built_ok)``.
@@ -1203,8 +1256,7 @@ class GitIndexer:
with lock:
repos, created[:] = list(created), []
for repo in repos:
- with contextlib.suppress(Exception):
- repo.close()
+ _release_repo(repo)
return get_thread_repo, close_all
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 08d39cd0b..4e512684f 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -33,6 +33,7 @@ from repowise.core.pipeline.progress import (
)
from repowise.core.registry import HookProgressCallback
+from .phase_timing import timed
from .phases._common import TEST_RUN_FILE_LIMIT, _phase_done, limit_to_top_pagerank
from .phases.analysis import (
_run_dead_code_analysis,
@@ -676,16 +677,17 @@ async def run_pipeline(
# the wiki-page backfill (in ``finalize_knowledge_graph``), so
# rich page summaries win; FAST mode floors here.
will_generate = generate_docs and llm_client is not None
- knowledge_graph_result = curate_knowledge_graph(
- knowledge_graph_result,
- parsed_files=parsed_files,
- graph_builder=graph_builder,
- repo_structure=repo_structure,
- community_info=graph_builder.community_info(),
- git_meta_map=git_meta_map,
- enabled=curation_enabled(),
- defer_summary_floor=will_generate,
- )
+ with timed(getattr(progress, "table", None), "knowledge_graph.curate"):
+ knowledge_graph_result = curate_knowledge_graph(
+ knowledge_graph_result,
+ parsed_files=parsed_files,
+ graph_builder=graph_builder,
+ repo_structure=repo_structure,
+ community_info=graph_builder.community_info(),
+ git_meta_map=git_meta_map,
+ enabled=curation_enabled(),
+ defer_summary_floor=will_generate,
+ )
except (ValueError, KeyError, RuntimeError) as cur_err:
logger.error("kg_curation_failed", error=str(cur_err), exc_info=True)
except (ValueError, KeyError, OSError, RuntimeError) as kg_err:
@@ -701,16 +703,19 @@ async def run_pipeline(
# written, so it is the only pass that can fold a paraphrase into an
# existing one. By the end-of-run persist every group matches on title.
if resume_controller is not None and not skip_analysis:
- await resume_controller.checkpoint_analysis(
- parsed_files=parsed_files,
- dead_code_report=dead_code_report,
- health_report=health_report,
- decision_report=decision_report,
- doc_drift_report=doc_drift_report,
- git_metadata_list=git_metadata_list,
- vector_store=vector_store,
- progress=progress,
- )
+ # No progress phase covers this write either; time it like the INDEX
+ # checkpoint so it shows in the totals.
+ with timed(getattr(progress, "table", None), "persist.checkpoint_analysis"):
+ await resume_controller.checkpoint_analysis(
+ parsed_files=parsed_files,
+ dead_code_report=dead_code_report,
+ health_report=health_report,
+ decision_report=decision_report,
+ doc_drift_report=doc_drift_report,
+ git_metadata_list=git_metadata_list,
+ vector_store=vector_store,
+ progress=progress,
+ )
# ---- Phase 3: Generation (optional) ------------------------------------
generated_pages: list[Any] | None = None
diff --git a/packages/core/src/repowise/core/pipeline/phases/git.py b/packages/core/src/repowise/core/pipeline/phases/git.py
index 322115d4d..13a8bd446 100644
--- a/packages/core/src/repowise/core/pipeline/phases/git.py
+++ b/packages/core/src/repowise/core/pipeline/phases/git.py
@@ -118,6 +118,7 @@ async def _run_git_indexing(
on_commit_done=_on_commit_done,
on_co_change_done=_on_co_change_done,
on_warning=_on_warning,
+ timings=getattr(progress, "table", None),
)
git_meta_map = {m["file_path"]: m for m in git_metadata_list}
_phase_done(progress, "git")
diff --git a/tests/unit/cli/test_generation_persist.py b/tests/unit/cli/test_generation_persist.py
index bb19a048f..aff1cf132 100644
--- a/tests/unit/cli/test_generation_persist.py
+++ b/tests/unit/cli/test_generation_persist.py
@@ -136,3 +136,35 @@ async def test_sink_failure_never_breaks_generation(repo_dir, monkeypatch):
# The valid page still persisted despite the bad one.
stored = await _read_db_page_ids(repo_dir)
assert "delta" in stored
+
+
+async def test_batched_flush_stores_every_page_and_times_the_drain(repo_dir, monkeypatch):
+ """Pages queued faster than they are written go out in batches, none lost,
+ and the drain after generation is recorded in the run's phase table."""
+ from types import SimpleNamespace
+
+ from repowise.cli.commands.init_cmd import _generation_persist
+ from repowise.core.pipeline import PhaseTimings
+
+ monkeypatch.setattr(_generation_persist, "_FLUSH_BATCH", 7)
+ emitted = [_page(f"p{i:02d}") for i in range(30)]
+
+ async def fake_run_generation(*, repo_path, on_page_ready=None, prior_pages=None, **_kw):
+ # No await between pages: all 30 are queued before the consumer runs.
+ for p in emitted:
+ on_page_ready(p)
+ return emitted
+
+ monkeypatch.setattr(
+ "repowise.core.pipeline.run_generation", fake_run_generation, raising=True
+ )
+ table = PhaseTimings()
+
+ await run_generation_with_persistence(
+ repo_path=repo_dir,
+ repo_name=repo_dir.name,
+ progress=SimpleNamespace(table=table),
+ )
+
+ assert await _read_db_page_ids(repo_dir) == {p.page_id for p in emitted}
+ assert "generation.persist_drain" in table.totals
diff --git a/tests/unit/ingestion/test_git_commit_rows_integration.py b/tests/unit/ingestion/test_git_commit_rows_integration.py
index 7e9a73894..2afa4cc63 100644
--- a/tests/unit/ingestion/test_git_commit_rows_integration.py
+++ b/tests/unit/ingestion/test_git_commit_rows_integration.py
@@ -159,6 +159,38 @@ async def test_index_repo_captures_whole_history_totals(tmp_path) -> None:
assert totals.first_commit_at is not None
+
+async def test_index_repo_times_each_git_step(tmp_path) -> None:
+ """Every step of the full index lands in the run's phase table, including
+ the repo-wide passes that run beside the per-file pass."""
+ import git as gitpython
+
+ from repowise.core.pipeline import PhaseTimings
+
+ repo = gitpython.Repo.init(tmp_path)
+ _configure_author(repo, "Ada Lovelace", "[email protected]")
+ _commit(repo, tmp_path / "a.py", "x = 1\n", "feat: add a")
+ _commit(repo, tmp_path / "a.py", "x = 2\n", "fix: correct a")
+
+ table = PhaseTimings()
+ summary, _results = await GitIndexer(tmp_path, tier=GitIndexTier.FULL).index_repo(
+ "repo1", timings=table
+ )
+
+ assert summary.repo_totals is not None
+ assert summary.repo_totals.total_commit_count == 2
+ for step in (
+ "git.commit_index",
+ "git.files",
+ "git.fix_walk",
+ "git.repo_totals",
+ "git.fix_events",
+ "git.episodes",
+ ):
+ assert step in table.totals, step
+
+
def test_capture_repo_totals_method(tmp_path) -> None:
"""The public capture (used by the incremental update path) opens its own
repo and returns the same whole-history record."""