Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(init): time the generation and persistence tail (#2050)

Raghav Chamadiya349689eefa3b
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
index 222b45d24..0fe07132a 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
@@ -169,6 +169,7 @@ def _run_deterministic_generation_phase(
embedder_name_resolved: str,
embedder_was_requested: bool,
resume: bool,
+ timings: Any | None = None,
) -> str:
"""Render the whole wiki from templates, for ``init --index-only``.
@@ -244,6 +245,7 @@ def _run_deterministic_generation_phase(
embedder_name_resolved=embedder,
resume=resume,
verbose=True,
+ timings=timings,
)
return embedder
@@ -267,6 +269,7 @@ def _run_generation_phase(
embedder_name_resolved: str,
resume: bool,
test_run: bool,
+ timings: Any | None = None,
) -> tuple[bool, bool]:
"""Run the LLM generation phase for a single-repo init.
@@ -396,6 +399,7 @@ def _run_generation_phase(
resume=resume,
verbose=True,
test_run=test_run,
+ timings=timings,
)
return False, False
@@ -1302,6 +1306,10 @@ def init_command(
# durations without changing the pipeline API. Timings get
# persisted to state.json below.
callback = PhaseTimingRecorder(rich_callback)
+ # The denominator. Phases nest (``persist.fts`` inside ``persist``)
+ # and some overlap, so the totals can exceed it - without it there is
+ # no way to tell an unrecorded stage from an overlapping one.
+ callback.table.start("run")
# Always run ingestion + analysis first (generate_docs=False).
# Generation happens separately after cost confirmation.
@@ -1366,12 +1374,9 @@ def init_command(
)
return
- # Surface per-phase timing data to the caller — both for the
- # state.json persistence below and for any future "profile" tooling
- # that wants to introspect a run.
- phase_timings: dict[str, float] = callback.timings
- # Same idea, for the failures rather than the durations: what the run
- # degraded on, in a place an agent can read after the terminal is gone.
+ # What the run degraded on, in a place an agent can read after the
+ # terminal is gone. Timings are read after persistence instead, so
+ # generation and the persist tail are in the table too.
run_warnings: list[str] = list(rich_callback.warnings)
# ---- Analysis summary (shown between analysis and generation) ----
@@ -1412,6 +1417,7 @@ def init_command(
embedder_name_resolved=embedder_name_resolved,
embedder_was_requested=embedder_was_requested,
resume=resume,
+ timings=callback.table,
)
else:
gen_stop, cost_declined = _run_generation_phase(
@@ -1436,6 +1442,7 @@ def init_command(
# say nothing, so a template wiki is never a run to continue.
resume=resume and _prior_docs_mode != "deterministic",
test_run=test_run,
+ timings=callback.table,
)
if gen_stop:
return
@@ -1471,6 +1478,7 @@ def init_command(
embedder_was_requested=True,
embedder_name_resolved=embedder_name_resolved,
resume=resume,
+ timings=callback.table,
)
# ---- Persistence ----
@@ -1499,15 +1507,19 @@ def init_command(
TimeElapsedColumn(),
console=console,
) as persist_bar:
- persist_callback = RichProgressCallback(persist_bar, console)
+ persist_callback = callback.rebind(RichProgressCallback(persist_bar, console))
# Announced before the work starts and re-announced with a real total
# once the page loop knows one, so the stage is never silent.
persist_callback.on_phase_start("persist", None)
- run_async(persist_result(result, repo_path, persist_callback))
+ run_async(persist_result(result, repo_path, persist_callback, callback.table))
persist_callback.on_phase_done("persist")
# Persistence has its own callback, so its warnings need folding into the
# run record explicitly — the state write below is the last chance.
run_warnings.extend(persist_callback.warnings)
+ # Read now, not before generation: ingestion, analysis, generation and
+ # persistence all write into the one table.
+ callback.table.stop("run")
+ phase_timings: dict[str, float] = callback.timings
console.print(f" [{OK}]✓[/] Database updated")
# Persist the onboarding choice so subsequent `repowise update` runs
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py b/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
index 486255217..76ad735b3 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
@@ -259,6 +259,7 @@ def run_repo_generation(
resume: bool,
verbose: bool,
test_run: bool = False,
+ timings: Any | None = None,
) -> list[Any]:
"""Generate wiki pages for one repo and enrich its knowledge graph.
@@ -275,7 +276,12 @@ def run_repo_generation(
``verbose`` controls only console output: the single-repo flow prints the
page count + KG status; the workspace flow stays quiet and prints its own
per-repo summary.
+
+ ``timings`` is the run's :class:`PhaseTimings` table. Generation owns its
+ own progress bar, so without it the phases here land nowhere.
"""
+ from repowise.core.pipeline import PhaseTimingRecorder, timed
+
from ._generation_persist import run_generation_with_persistence
if verbose:
@@ -325,7 +331,9 @@ def run_repo_generation(
result.preserved_page_ids = preserved_page_ids
with Progress(*columns, console=console) as gen_progress:
- gen_callback = RichProgressCallback(gen_progress, console)
+ gen_callback: Any = RichProgressCallback(gen_progress, console)
+ if timings is not None:
+ gen_callback = PhaseTimingRecorder(gen_callback, timings)
generated_pages = run_async(
run_generation_with_persistence(
repo_path=repo_path,
@@ -437,13 +445,14 @@ def run_repo_generation(
# A deterministic run has no model to ask, and the skeleton's structural
# layers stand on their own.
if not deterministic:
- _enrich_knowledge_graph(
- result=result,
- provider=provider,
- gen_config=gen_config,
- generated_pages=generated_pages,
- verbose=verbose,
- )
+ with timed(timings, "generation.kg_enrich"):
+ _enrich_knowledge_graph(
+ result=result,
+ provider=provider,
+ gen_config=gen_config,
+ generated_pages=generated_pages,
+ verbose=verbose,
+ )
flush_cost_tracker(cost_tracker)
# What the run actually spent, for the completion panel. The user was shown
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index 58436e4e0..1a370790f 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -106,7 +106,12 @@ async def _index_preserved_pages(sf: Any, fts: Any, preserved_page_ids: set[str]
logger.debug("persist.preserved_fts_backfill_failed", error=str(exc))
-async def persist_result(result: Any, repo_path: Path, progress: Any | None = None) -> None:
+async def persist_result(
+ result: Any,
+ repo_path: Path,
+ progress: Any | None = None,
+ timings: Any | None = None,
+) -> None:
"""Persist a PipelineResult to the local SQLite database.
Handles both index-only (no pages) and full (with pages + FTS) modes.
@@ -116,6 +121,9 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = No
to the wiki. Everything after "Generated N pages" used to happen under a
single indeterminate spinner, so on a repo of a few thousand pages the run
sat silent for minutes with no way to tell work from a hang.
+
+ *timings* splits that span into its SQL and full-text halves, which the
+ single ``persist`` number cannot distinguish.
"""
from datetime import UTC, datetime
@@ -126,6 +134,7 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = No
persist_generation,
persist_pipeline_result,
sweep_stale_generated_pages,
+ timed,
tombstone_absent_file_pages,
)
@@ -240,24 +249,23 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = No
# holds a write lock raises "database is locked". The swept-id delete must
# therefore stay here (it cannot move ahead of the SQL commit like the
# vector delete can); it is idempotent and narrow (orphan FTS rows only).
- if fts is not None and swept_page_ids:
- await fts.delete_many(swept_page_ids)
- if fts is not None and result.generated_pages:
- if progress is not None:
- progress.on_phase_start("persist", len(result.generated_pages))
- for page in result.generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ with timed(timings, "persist.fts"):
+ if fts is not None and swept_page_ids:
+ await fts.delete_many(swept_page_ids)
+ if fts is not None and result.generated_pages:
if progress is not None:
- progress.on_item_done("persist")
- if progress is not None:
- progress.on_phase_done("persist")
- await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))
+ progress.on_phase_start("persist", len(result.generated_pages))
+ for page in result.generated_pages:
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
+ if progress is not None:
+ progress.on_item_done("persist")
+ await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))
# Stamp the analysis (+ generation) phases in the resume ledger now that
# they're persisted, so a future resume can skip them too.
@@ -274,6 +282,11 @@ async def persist_result(result: Any, repo_path: Path, progress: Any | None = No
await ledger.mark_completed(ResumePhase.GENERATION)
await engine.dispose()
+ # Closed here rather than after the indexing loop: the preserved-page
+ # backfill and the ledger stamp are persistence too, and a phase that
+ # stopped short of them reported a different span per run mode.
+ if progress is not None:
+ progress.on_phase_done("persist")
# ---------------------------------------------------------------------------
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
index f91a37103..7dcc44b35 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
@@ -94,6 +94,7 @@ def _run_workspace_generation(
skip_tests: bool,
skip_infra: bool,
test_run: bool,
+ timings: Any | None = None,
reasoning: str = "auto",
onboarding: bool = True,
wiki_style: str = DEFAULT_STYLE,
@@ -166,6 +167,7 @@ def _run_workspace_generation(
resume=resume,
verbose=False,
test_run=test_run,
+ timings=timings,
)
@@ -180,6 +182,7 @@ def _run_workspace_deterministic_generation(
onboarding: bool,
wiki_style: str,
language: str,
+ timings: Any | None = None,
) -> tuple[list[Any], str]:
"""Render one workspace repo's wiki from templates (no model, no cost).
@@ -231,6 +234,7 @@ def _run_workspace_deterministic_generation(
embedder_name_resolved=embedder,
resume=resume,
verbose=False,
+ timings=timings,
)
return generated_pages, embedder
@@ -344,8 +348,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
derive_environment_facts=True,
)
)
- repo_phase_timings: dict[str, float] = callback.timings
- console.print(
+ console.print(
f" [{OK}]✓[/] {result.file_count:,} files, {result.symbol_count:,} symbols"
)
except Exception as exc:
@@ -373,6 +376,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
result=result,
embedder_name_resolved=ctx.embedder_name_resolved,
embedder_was_requested=ctx.embedder_was_requested,
+ timings=callback.table,
concurrency=ctx.concurrency,
resume=ctx.resume,
onboarding=ctx.onboarding,
@@ -405,6 +409,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
result=result,
provider=repo_provider,
embedder_name_resolved=ctx.embedder_name_resolved,
+ timings=callback.table,
concurrency=ctx.concurrency,
yes=ctx.yes,
resume=ctx.resume,
@@ -462,7 +467,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
)
# Persist to repo-local DB
- run_async(persist_result(result, repo.path))
+ run_async(persist_result(result, repo.path, timings=callback.table))
# Write state.json so `repowise update` knows the base commit
head = get_head_commit(repo.path)
@@ -478,6 +483,9 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
if docs_mode == "llm" and provider is not None:
state["provider"] = provider.provider_name
state["model"] = provider.model_name
+ # Read after generation and persistence, not straight off the pipeline:
+ # both write into the same table.
+ repo_phase_timings: dict[str, float] = callback.timings
if repo_phase_timings:
state["phase_timings"] = repo_phase_timings
kg = getattr(result, "knowledge_graph_result", None)
diff --git a/packages/core/src/repowise/core/pipeline/__init__.py b/packages/core/src/repowise/core/pipeline/__init__.py
index 9a35e332d..03cc8616d 100644
--- a/packages/core/src/repowise/core/pipeline/__init__.py
+++ b/packages/core/src/repowise/core/pipeline/__init__.py
@@ -22,7 +22,7 @@ from .persist import (
persist_pipeline_result,
tombstone_absent_file_pages,
)
-from .phase_timing import PhaseTimingRecorder
+from .phase_timing import PhaseTimingRecorder, PhaseTimings, timed
from .progress import LoggingProgressCallback, ProgressCallback
from .reparse import reparse_repo
from .upgrade import rehydrate_graph_builder
@@ -30,6 +30,7 @@ from .upgrade import rehydrate_graph_builder
__all__ = [
"LoggingProgressCallback",
"PhaseTimingRecorder",
+ "PhaseTimings",
"PipelineResult",
"ProgressCallback",
"persist_analysis",
@@ -42,5 +43,6 @@ __all__ = [
"run_generation",
"run_pipeline",
"sweep_stale_generated_pages",
+ "timed",
"tombstone_absent_file_pages",
]
diff --git a/packages/core/src/repowise/core/pipeline/phase_timing.py b/packages/core/src/repowise/core/pipeline/phase_timing.py
index d7d00c20b..330b4a154 100644
--- a/packages/core/src/repowise/core/pipeline/phase_timing.py
+++ b/packages/core/src/repowise/core/pipeline/phase_timing.py
@@ -5,6 +5,10 @@ A ``ProgressCallback`` decorator that observes ``on_phase_start`` /
phase. Designed to compose with the real (Rich / Logging) callback so
the timing data is collected without touching pipeline internals.
+Stages that build their own progress callback (generation, persistence)
+share one :class:`PhaseTimings` table via :meth:`PhaseTimingRecorder.rebind`,
+so a run reports one set of totals rather than one per progress bar.
+
The CLI writes the resulting ``timings`` dict into ``state.json`` so
before/after perf comparisons across runs become trivial:
@@ -14,48 +18,95 @@ before/after perf comparisons across runs become trivial:
{
"last_sync_commit": "...",
"phase_timings": {
+ "run": 170.54,
"traverse": 4.21,
"parse": 88.3,
"graph.imports": 312.5,
...
}
}
+
+Phases nest (``persist.fts`` inside ``persist``) and some overlap, so the
+entries can sum to more than ``run``. Compare each against ``run``, never
+against the sum.
"""
from __future__ import annotations
import time
+from collections.abc import Iterator
+from contextlib import contextmanager
from typing import Any
+class PhaseTimings:
+ """Accumulated per-phase wall-clock totals for one run.
+
+ Repeated phases accumulate; the total is the sum of every visit. This
+ matches how the user perceives the cost - "how much wall-clock time did
+ this phase consume". Phases may overlap, so the totals can sum to more
+ than the run's wall time.
+ """
+
+ def __init__(self) -> None:
+ self._starts: dict[str, float] = {}
+ self._totals: dict[str, float] = {}
+
+ def start(self, phase: str) -> None:
+ """Open ``phase``. Re-announcing an open phase keeps the first start.
+
+ Callers announce a phase before the work begins and again once a real
+ item total is known; resetting on the second call would drop whatever
+ ran in between.
+ """
+ self._starts.setdefault(phase, time.monotonic())
+
+ def stop(self, phase: str) -> None:
+ """Close ``phase``. A phase that never started is ignored."""
+ started = self._starts.pop(phase, None)
+ if started is not None:
+ self._totals[phase] = self._totals.get(phase, 0.0) + (time.monotonic() - started)
+
+ @property
+ def totals(self) -> dict[str, float]:
+ """Mapping of phase name -> accumulated seconds (rounded to 0.01s)."""
+ return {name: round(secs, 2) for name, secs in self._totals.items()}
+
+
class PhaseTimingRecorder:
"""Observes pipeline phase events and records wall-clock durations.
Wraps another ``ProgressCallback`` (or ``None``) and transparently
- delegates every call. Concurrent / nested phases are supported —
- each phase name is timed independently from its own
- ``on_phase_start`` to its matching ``on_phase_done``.
-
- Repeated phases (a phase whose ``on_phase_start`` fires more than
- once in a single run) accumulate; the total is the sum of every
- visit. This matches how the user perceives the cost — "how much
- wall-clock time did this phase consume".
+ delegates every call. Each phase name is timed independently from its
+ own ``on_phase_start`` to its matching ``on_phase_done``.
"""
- def __init__(self, inner: Any | None = None) -> None:
+ def __init__(self, inner: Any | None = None, timings: PhaseTimings | None = None) -> None:
self._inner = inner
- self._starts: dict[str, float] = {}
- self._totals: dict[str, float] = {}
+ self._timings = timings if timings is not None else PhaseTimings()
@property
def timings(self) -> dict[str, float]:
- """Mapping of phase name → accumulated seconds (rounded to 0.01s)."""
- return {name: round(secs, 2) for name, secs in self._totals.items()}
+ """Mapping of phase name -> accumulated seconds (rounded to 0.01s)."""
+ return self._timings.totals
+
+ @property
+ def table(self) -> PhaseTimings:
+ """The shared totals table, for handing to a later stage."""
+ return self._timings
+
+ def rebind(self, inner: Any | None) -> PhaseTimingRecorder:
+ """A recorder forwarding to ``inner`` that writes the same totals.
+
+ Generation and persistence each own a progress bar built after the
+ pipeline callback is gone; rebinding keeps their phases in one table.
+ """
+ return PhaseTimingRecorder(inner, self._timings)
# ---- ProgressCallback protocol ----------------------------------
def on_phase_start(self, phase: str, total: int | None) -> None:
- self._starts[phase] = time.monotonic()
+ self._timings.start(phase)
if self._inner is not None:
self._inner.on_phase_start(phase, total)
@@ -64,9 +115,7 @@ class PhaseTimingRecorder:
self._inner.on_item_done(phase)
def on_phase_done(self, phase: str) -> None:
- started = self._starts.pop(phase, None)
- if started is not None:
- self._totals[phase] = self._totals.get(phase, 0.0) + (time.monotonic() - started)
+ self._timings.stop(phase)
if self._inner is not None:
fn = getattr(self._inner, "on_phase_done", None)
if callable(fn):
@@ -83,3 +132,19 @@ class PhaseTimingRecorder:
if self._inner is None:
raise AttributeError(name)
return getattr(self._inner, name)
+
+
+@contextmanager
+def timed(timings: PhaseTimings | None, name: str) -> Iterator[None]:
+ """Time ``name`` into ``timings``, or do nothing when there is no table.
+
+ For stages that have no progress callback for a recorder to observe.
+ """
+ if timings is None:
+ yield
+ return
+ timings.start(name)
+ try:
+ yield
+ finally:
+ timings.stop(name)
diff --git a/tests/unit/pipeline/test_phase_timing.py b/tests/unit/pipeline/test_phase_timing.py
index 3d8123d00..2d8dced6a 100644
--- a/tests/unit/pipeline/test_phase_timing.py
+++ b/tests/unit/pipeline/test_phase_timing.py
@@ -2,9 +2,10 @@
from __future__ import annotations
+import contextlib
import time
-from repowise.core.pipeline import PhaseTimingRecorder
+from repowise.core.pipeline import PhaseTimingRecorder, PhaseTimings, timed
class _CollectingCallback:
@@ -85,3 +86,59 @@ def test_unstarted_phase_is_ignored() -> None:
rec = PhaseTimingRecorder()
rec.on_phase_done("never-started")
assert rec.timings == {}
+
+
+def test_rebound_recorder_shares_one_table() -> None:
+ """Generation and persistence build their own callbacks mid-run."""
+ recorder = PhaseTimingRecorder(_CollectingCallback())
+ recorder.on_phase_start("ingest", None)
+ recorder.on_phase_done("ingest")
+
+ later = recorder.rebind(_CollectingCallback())
+ later.on_phase_start("persist", None)
+ later.on_phase_done("persist")
+
+ assert set(recorder.timings) == {"ingest", "persist"}
+ assert recorder.timings == later.timings
+
+
+def test_rebound_recorder_forwards_to_its_own_inner() -> None:
+ first, second = _CollectingCallback(), _CollectingCallback()
+ recorder = PhaseTimingRecorder(first)
+ later = recorder.rebind(second)
+
+ later.on_phase_start("persist", 3)
+ later.on_item_done("persist")
+ later.on_phase_done("persist")
+
+ assert second.events == [("start", "persist", 3), ("item", "persist", None), ("done", "persist", None)]
+ assert first.events == []
+
+
+def test_reannounced_phase_keeps_the_first_start() -> None:
+ """A phase is announced again once its real item total is known."""
+ recorder = PhaseTimingRecorder()
+ recorder.on_phase_start("persist", None)
+ time.sleep(0.05)
+ recorder.on_phase_start("persist", 42)
+ recorder.on_phase_done("persist")
+
+ assert recorder.timings["persist"] >= 0.05
+
+
+def test_timed_records_a_block_and_tolerates_no_table() -> None:
+ timings = PhaseTimings()
+ with timed(timings, "kg"):
+ time.sleep(0.02)
+ with timed(None, "kg"):
+ pass
+
+ assert timings.totals["kg"] >= 0.02
+
+
+def test_timed_records_even_when_the_block_raises() -> None:
+ timings = PhaseTimings()
+ with contextlib.suppress(RuntimeError), timed(timings, "kg"):
+ raise RuntimeError("boom")
+
+ assert "kg" in timings.totals