Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(init): time the generation and persistence tail (#2050)
Raghav Chamadiya349689eefa3b
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.pyindex 222b45d24..0fe07132a 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py@@ -169,6 +169,7 @@ def _run_deterministic_generation_phase(embedder_name_resolved: str,embedder_was_requested: bool,resume: bool,+ timings: Any | None = None,) -> str:"""Render the whole wiki from templates, for ``init --index-only``.@@ -244,6 +245,7 @@ def _run_deterministic_generation_phase(embedder_name_resolved=embedder,resume=resume,verbose=True,+ timings=timings,)return embedder@@ -267,6 +269,7 @@ def _run_generation_phase(embedder_name_resolved: str,resume: bool,test_run: bool,+ timings: Any | None = None,) -> tuple[bool, bool]:"""Run the LLM generation phase for a single-repo init.@@ -396,6 +399,7 @@ def _run_generation_phase(resume=resume,verbose=True,test_run=test_run,+ timings=timings,)return False, False@@ -1302,6 +1306,10 @@ def init_command(# durations without changing the pipeline API. Timings get# persisted to state.json below.callback = PhaseTimingRecorder(rich_callback)+ # The denominator. Phases nest (``persist.fts`` inside ``persist``)+ # and some overlap, so the totals can exceed it - without it there is+ # no way to tell an unrecorded stage from an overlapping one.+ callback.table.start("run")# Always run ingestion + analysis first (generate_docs=False).# Generation happens separately after cost confirmation.@@ -1366,12 +1374,9 @@ def init_command()return- # Surface per-phase timing data to the caller — both for the- # state.json persistence below and for any future "profile" tooling- # that wants to introspect a run.- phase_timings: dict[str, float] = callback.timings- # Same idea, for the failures rather than the durations: what the run- # degraded on, in a place an agent can read after the terminal is gone.+ # What the run degraded on, in a place an agent can read after the+ # terminal is gone. Timings are read after persistence instead, so+ # generation and the persist tail are in the table too.run_warnings: list[str] = list(rich_callback.warnings)# ---- Analysis summary (shown between analysis and generation) ----@@ -1412,6 +1417,7 @@ def init_command(embedder_name_resolved=embedder_name_resolved,embedder_was_requested=embedder_was_requested,resume=resume,+ timings=callback.table,)else:gen_stop, cost_declined = _run_generation_phase(@@ -1436,6 +1442,7 @@ def init_command(# say nothing, so a template wiki is never a run to continue.resume=resume and _prior_docs_mode != "deterministic",test_run=test_run,+ timings=callback.table,)if gen_stop:return@@ -1471,6 +1478,7 @@ def init_command(embedder_was_requested=True,embedder_name_resolved=embedder_name_resolved,resume=resume,+ timings=callback.table,)# ---- Persistence ----@@ -1499,15 +1507,19 @@ def init_command(TimeElapsedColumn(),console=console,) as persist_bar:- persist_callback = RichProgressCallback(persist_bar, console)+ persist_callback = callback.rebind(RichProgressCallback(persist_bar, console))# Announced before the work starts and re-announced with a real total# once the page loop knows one, so the stage is never silent.persist_callback.on_phase_start("persist", None)- run_async(persist_result(result, repo_path, persist_callback))+ run_async(persist_result(result, repo_path, persist_callback, callback.table))persist_callback.on_phase_done("persist")# Persistence has its own callback, so its warnings need folding into the# run record explicitly — the state write below is the last chance.run_warnings.extend(persist_callback.warnings)+ # Read now, not before generation: ingestion, analysis, generation and+ # persistence all write into the one table.+ callback.table.stop("run")+ phase_timings: dict[str, float] = callback.timingsconsole.print(f" [{OK}]✓[/] Database updated")# Persist the onboarding choice so subsequent `repowise update` runsdiff --git a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py b/packages/cli/src/repowise/cli/commands/init_cmd/generation.pyindex 486255217..76ad735b3 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/generation.py@@ -259,6 +259,7 @@ def run_repo_generation(resume: bool,verbose: bool,test_run: bool = False,+ timings: Any | None = None,) -> list[Any]:"""Generate wiki pages for one repo and enrich its knowledge graph.@@ -275,7 +276,12 @@ def run_repo_generation(``verbose`` controls only console output: the single-repo flow prints thepage count + KG status; the workspace flow stays quiet and prints its ownper-repo summary.++ ``timings`` is the run's :class:`PhaseTimings` table. Generation owns its+ own progress bar, so without it the phases here land nowhere."""+ from repowise.core.pipeline import PhaseTimingRecorder, timed+from ._generation_persist import run_generation_with_persistenceif verbose:@@ -325,7 +331,9 @@ def run_repo_generation(result.preserved_page_ids = preserved_page_idswith Progress(*columns, console=console) as gen_progress:- gen_callback = RichProgressCallback(gen_progress, console)+ gen_callback: Any = RichProgressCallback(gen_progress, console)+ if timings is not None:+ gen_callback = PhaseTimingRecorder(gen_callback, timings)generated_pages = run_async(run_generation_with_persistence(repo_path=repo_path,@@ -437,13 +445,14 @@ def run_repo_generation(# A deterministic run has no model to ask, and the skeleton's structural# layers stand on their own.if not deterministic:- _enrich_knowledge_graph(- result=result,- provider=provider,- gen_config=gen_config,- generated_pages=generated_pages,- verbose=verbose,- )+ with timed(timings, "generation.kg_enrich"):+ _enrich_knowledge_graph(+ result=result,+ provider=provider,+ gen_config=gen_config,+ generated_pages=generated_pages,+ verbose=verbose,+ )flush_cost_tracker(cost_tracker)# What the run actually spent, for the completion panel. The user was showndiff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.pyindex 58436e4e0..1a370790f 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py@@ -106,7 +106,12 @@ async def _index_preserved_pages(sf: Any, fts: Any, preserved_page_ids: set[str]logger.debug("persist.preserved_fts_backfill_failed", error=str(exc))-async def persist_result(result: Any, repo_path: Path, progress: Any | None = None) -> None:+async def persist_result(+ result: Any,+ repo_path: Path,+ progress: Any | None = None,+ timings: Any | None = None,+) -> None:"""Persist a PipelineResult to the local SQLite database.Handles both index-only (no pages) and full (with pages + FTS) modes.@@ -116,6 +121,9 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = Noto the wiki. Everything after "Generated N pages" used to happen under asingle indeterminate spinner, so on a repo of a few thousand pages the runsat silent for minutes with no way to tell work from a hang.++ *timings* splits that span into its SQL and full-text halves, which the+ single ``persist`` number cannot distinguish."""from datetime import UTC, datetime@@ -126,6 +134,7 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = Nopersist_generation,persist_pipeline_result,sweep_stale_generated_pages,+ timed,tombstone_absent_file_pages,)@@ -240,24 +249,23 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = No# holds a write lock raises "database is locked". The swept-id delete must# therefore stay here (it cannot move ahead of the SQL commit like the# vector delete can); it is idempotent and narrow (orphan FTS rows only).- if fts is not None and swept_page_ids:- await fts.delete_many(swept_page_ids)- if fts is not None and result.generated_pages:- if progress is not None:- progress.on_phase_start("persist", len(result.generated_pages))- for page in result.generated_pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )+ with timed(timings, "persist.fts"):+ if fts is not None and swept_page_ids:+ await fts.delete_many(swept_page_ids)+ if fts is not None and result.generated_pages:if progress is not None:- progress.on_item_done("persist")- if progress is not None:- progress.on_phase_done("persist")- await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))+ progress.on_phase_start("persist", len(result.generated_pages))+ for page in result.generated_pages:+ await fts.index(+ page.page_id,+ page.title,+ page.content,+ summary=page.summary,+ target_path=page.target_path,+ )+ if progress is not None:+ progress.on_item_done("persist")+ await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))# Stamp the analysis (+ generation) phases in the resume ledger now that# they're persisted, so a future resume can skip them too.@@ -274,6 +282,11 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = Noawait ledger.mark_completed(ResumePhase.GENERATION)await engine.dispose()+ # Closed here rather than after the indexing loop: the preserved-page+ # backfill and the ledger stamp are persistence too, and a phase that+ # stopped short of them reported a different span per run mode.+ if progress is not None:+ progress.on_phase_done("persist")# ---------------------------------------------------------------------------diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.pyindex f91a37103..7dcc44b35 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py@@ -94,6 +94,7 @@ def _run_workspace_generation(skip_tests: bool,skip_infra: bool,test_run: bool,+ timings: Any | None = None,reasoning: str = "auto",onboarding: bool = True,wiki_style: str = DEFAULT_STYLE,@@ -166,6 +167,7 @@ def _run_workspace_generation(resume=resume,verbose=False,test_run=test_run,+ timings=timings,)@@ -180,6 +182,7 @@ def _run_workspace_deterministic_generation(onboarding: bool,wiki_style: str,language: str,+ timings: Any | None = None,) -> tuple[list[Any], str]:"""Render one workspace repo's wiki from templates (no model, no cost).@@ -231,6 +234,7 @@ def _run_workspace_deterministic_generation(embedder_name_resolved=embedder,resume=resume,verbose=False,+ timings=timings,)return generated_pages, embedder@@ -344,8 +348,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCtderive_environment_facts=True,))- repo_phase_timings: dict[str, float] = callback.timings- console.print(+ console.print(f" [{OK}]✓[/] {result.file_count:,} files, {result.symbol_count:,} symbols")except Exception as exc:@@ -373,6 +376,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCtresult=result,embedder_name_resolved=ctx.embedder_name_resolved,embedder_was_requested=ctx.embedder_was_requested,+ timings=callback.table,concurrency=ctx.concurrency,resume=ctx.resume,onboarding=ctx.onboarding,@@ -405,6 +409,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCtresult=result,provider=repo_provider,embedder_name_resolved=ctx.embedder_name_resolved,+ timings=callback.table,concurrency=ctx.concurrency,yes=ctx.yes,resume=ctx.resume,@@ -462,7 +467,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt)# Persist to repo-local DB- run_async(persist_result(result, repo.path))+ run_async(persist_result(result, repo.path, timings=callback.table))# Write state.json so `repowise update` knows the base commithead = get_head_commit(repo.path)@@ -478,6 +483,9 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCtif docs_mode == "llm" and provider is not None:state["provider"] = provider.provider_namestate["model"] = provider.model_name+ # Read after generation and persistence, not straight off the pipeline:+ # both write into the same table.+ repo_phase_timings: dict[str, float] = callback.timingsif repo_phase_timings:state["phase_timings"] = repo_phase_timingskg = getattr(result, "knowledge_graph_result", None)diff --git a/packages/core/src/repowise/core/pipeline/__init__.py b/packages/core/src/repowise/core/pipeline/__init__.pyindex 9a35e332d..03cc8616d 100644--- a/packages/core/src/repowise/core/pipeline/__init__.py+++ b/packages/core/src/repowise/core/pipeline/__init__.py@@ -22,7 +22,7 @@ from .persist import (persist_pipeline_result,tombstone_absent_file_pages,)-from .phase_timing import PhaseTimingRecorder+from .phase_timing import PhaseTimingRecorder, PhaseTimings, timedfrom .progress import LoggingProgressCallback, ProgressCallbackfrom .reparse import reparse_repofrom .upgrade import rehydrate_graph_builder@@ -30,6 +30,7 @@ from .upgrade import rehydrate_graph_builder__all__ = ["LoggingProgressCallback","PhaseTimingRecorder",+ "PhaseTimings","PipelineResult","ProgressCallback","persist_analysis",@@ -42,5 +43,6 @@ __all__ = ["run_generation","run_pipeline","sweep_stale_generated_pages",+ "timed","tombstone_absent_file_pages",]diff --git a/packages/core/src/repowise/core/pipeline/phase_timing.py b/packages/core/src/repowise/core/pipeline/phase_timing.pyindex d7d00c20b..330b4a154 100644--- a/packages/core/src/repowise/core/pipeline/phase_timing.py+++ b/packages/core/src/repowise/core/pipeline/phase_timing.py@@ -5,6 +5,10 @@ A ``ProgressCallback`` decorator that observes ``on_phase_start`` /phase. Designed to compose with the real (Rich / Logging) callback sothe timing data is collected without touching pipeline internals.+Stages that build their own progress callback (generation, persistence)+share one :class:`PhaseTimings` table via :meth:`PhaseTimingRecorder.rebind`,+so a run reports one set of totals rather than one per progress bar.+The CLI writes the resulting ``timings`` dict into ``state.json`` sobefore/after perf comparisons across runs become trivial:@@ -14,48 +18,95 @@ before/after perf comparisons across runs become trivial:{"last_sync_commit": "...","phase_timings": {+ "run": 170.54,"traverse": 4.21,"parse": 88.3,"graph.imports": 312.5,...}}++Phases nest (``persist.fts`` inside ``persist``) and some overlap, so the+entries can sum to more than ``run``. Compare each against ``run``, never+against the sum."""from __future__ import annotationsimport time+from collections.abc import Iterator+from contextlib import contextmanagerfrom typing import Any+class PhaseTimings:+ """Accumulated per-phase wall-clock totals for one run.++ Repeated phases accumulate; the total is the sum of every visit. This+ matches how the user perceives the cost - "how much wall-clock time did+ this phase consume". Phases may overlap, so the totals can sum to more+ than the run's wall time.+ """++ def __init__(self) -> None:+ self._starts: dict[str, float] = {}+ self._totals: dict[str, float] = {}++ def start(self, phase: str) -> None:+ """Open ``phase``. Re-announcing an open phase keeps the first start.++ Callers announce a phase before the work begins and again once a real+ item total is known; resetting on the second call would drop whatever+ ran in between.+ """+ self._starts.setdefault(phase, time.monotonic())++ def stop(self, phase: str) -> None:+ """Close ``phase``. A phase that never started is ignored."""+ started = self._starts.pop(phase, None)+ if started is not None:+ self._totals[phase] = self._totals.get(phase, 0.0) + (time.monotonic() - started)++ @property+ def totals(self) -> dict[str, float]:+ """Mapping of phase name -> accumulated seconds (rounded to 0.01s)."""+ return {name: round(secs, 2) for name, secs in self._totals.items()}++class PhaseTimingRecorder:"""Observes pipeline phase events and records wall-clock durations.Wraps another ``ProgressCallback`` (or ``None``) and transparently- delegates every call. Concurrent / nested phases are supported —- each phase name is timed independently from its own- ``on_phase_start`` to its matching ``on_phase_done``.-- Repeated phases (a phase whose ``on_phase_start`` fires more than- once in a single run) accumulate; the total is the sum of every- visit. This matches how the user perceives the cost — "how much- wall-clock time did this phase consume".+ delegates every call. Each phase name is timed independently from its+ own ``on_phase_start`` to its matching ``on_phase_done``."""- def __init__(self, inner: Any | None = None) -> None:+ def __init__(self, inner: Any | None = None, timings: PhaseTimings | None = None) -> None:self._inner = inner- self._starts: dict[str, float] = {}- self._totals: dict[str, float] = {}+ self._timings = timings if timings is not None else PhaseTimings()@propertydef timings(self) -> dict[str, float]:- """Mapping of phase name → accumulated seconds (rounded to 0.01s)."""- return {name: round(secs, 2) for name, secs in self._totals.items()}+ """Mapping of phase name -> accumulated seconds (rounded to 0.01s)."""+ return self._timings.totals++ @property+ def table(self) -> PhaseTimings:+ """The shared totals table, for handing to a later stage."""+ return self._timings++ def rebind(self, inner: Any | None) -> PhaseTimingRecorder:+ """A recorder forwarding to ``inner`` that writes the same totals.++ Generation and persistence each own a progress bar built after the+ pipeline callback is gone; rebinding keeps their phases in one table.+ """+ return PhaseTimingRecorder(inner, self._timings)# ---- ProgressCallback protocol ----------------------------------def on_phase_start(self, phase: str, total: int | None) -> None:- self._starts[phase] = time.monotonic()+ self._timings.start(phase)if self._inner is not None:self._inner.on_phase_start(phase, total)@@ -64,9 +115,7 @@ class PhaseTimingRecorder:self._inner.on_item_done(phase)def on_phase_done(self, phase: str) -> None:- started = self._starts.pop(phase, None)- if started is not None:- self._totals[phase] = self._totals.get(phase, 0.0) + (time.monotonic() - started)+ self._timings.stop(phase)if self._inner is not None:fn = getattr(self._inner, "on_phase_done", None)if callable(fn):@@ -83,3 +132,19 @@ class PhaseTimingRecorder:if self._inner is None:raise AttributeError(name)return getattr(self._inner, name)+++@contextmanager+def timed(timings: PhaseTimings | None, name: str) -> Iterator[None]:+ """Time ``name`` into ``timings``, or do nothing when there is no table.++ For stages that have no progress callback for a recorder to observe.+ """+ if timings is None:+ yield+ return+ timings.start(name)+ try:+ yield+ finally:+ timings.stop(name)diff --git a/tests/unit/pipeline/test_phase_timing.py b/tests/unit/pipeline/test_phase_timing.pyindex 3d8123d00..2d8dced6a 100644--- a/tests/unit/pipeline/test_phase_timing.py+++ b/tests/unit/pipeline/test_phase_timing.py@@ -2,9 +2,10 @@from __future__ import annotations+import contextlibimport time-from repowise.core.pipeline import PhaseTimingRecorder+from repowise.core.pipeline import PhaseTimingRecorder, PhaseTimings, timedclass _CollectingCallback:@@ -85,3 +86,59 @@ def test_unstarted_phase_is_ignored() -> None:rec = PhaseTimingRecorder()rec.on_phase_done("never-started")assert rec.timings == {}+++def test_rebound_recorder_shares_one_table() -> None:+ """Generation and persistence build their own callbacks mid-run."""+ recorder = PhaseTimingRecorder(_CollectingCallback())+ recorder.on_phase_start("ingest", None)+ recorder.on_phase_done("ingest")++ later = recorder.rebind(_CollectingCallback())+ later.on_phase_start("persist", None)+ later.on_phase_done("persist")++ assert set(recorder.timings) == {"ingest", "persist"}+ assert recorder.timings == later.timings+++def test_rebound_recorder_forwards_to_its_own_inner() -> None:+ first, second = _CollectingCallback(), _CollectingCallback()+ recorder = PhaseTimingRecorder(first)+ later = recorder.rebind(second)++ later.on_phase_start("persist", 3)+ later.on_item_done("persist")+ later.on_phase_done("persist")++ assert second.events == [("start", "persist", 3), ("item", "persist", None), ("done", "persist", None)]+ assert first.events == []+++def test_reannounced_phase_keeps_the_first_start() -> None:+ """A phase is announced again once its real item total is known."""+ recorder = PhaseTimingRecorder()+ recorder.on_phase_start("persist", None)+ time.sleep(0.05)+ recorder.on_phase_start("persist", 42)+ recorder.on_phase_done("persist")++ assert recorder.timings["persist"] >= 0.05+++def test_timed_records_a_block_and_tolerates_no_table() -> None:+ timings = PhaseTimings()+ with timed(timings, "kg"):+ time.sleep(0.02)+ with timed(None, "kg"):+ pass++ assert timings.totals["kg"] >= 0.02+++def test_timed_records_even_when_the_block_raises() -> None:+ timings = PhaseTimings()+ with contextlib.suppress(RuntimeError), timed(timings, "kg"):+ raise RuntimeError("boom")++ assert "kg" in timings.totals