Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(pipeline): persist_incremental_index on an ordered step runner (#4005)
Raghav Chamadiya836deee1ef4f
diff --git a/packages/core/src/repowise/core/pipeline/incremental/_common.py b/packages/core/src/repowise/core/pipeline/incremental/_common.pyindex 90af6ba48..7b85f3383 100644--- a/packages/core/src/repowise/core/pipeline/incremental/_common.py+++ b/packages/core/src/repowise/core/pipeline/incremental/_common.py@@ -6,11 +6,11 @@ from collections.abc import Callableimport structlog-# One logger for the whole package, named as before the split.+from repowise.core.pipeline.step_runner import _noop_log++# One logger for the whole package, under the package's name.logger = structlog.get_logger("repowise.core.pipeline.incremental")LogFn = Callable[[str], None]--def _noop_log(message: str) -> None: # pragma: no cover - trivial- return None+__all__ = ["LogFn", "_noop_log", "logger"]diff --git a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py b/packages/core/src/repowise/core/pipeline/incremental/persist_index.pyindex 9433f4011..853eef2fb 100644--- a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py+++ b/packages/core/src/repowise/core/pipeline/incremental/persist_index.py@@ -1,11 +1,20 @@-"""The incremental index writer: every DB step of an update, in order."""+"""The incremental index writer: every DB step of an update, in order.++Each step is a small function of the run's :class:`IncrementalPersist`, listed+once in :data:`INCREMENTAL_STEPS` (inside the session) and+:data:`CLEANUP_STEPS` (after it closes), and run by+:func:`~repowise.core.pipeline.step_runner.run_steps`.+"""from __future__ import annotations+from collections.abc import Callable+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Anyfrom repowise.core.pipeline.phase_timing import PhaseTimings, timed+from repowise.core.pipeline.step_runner import PersistRun, Step, run_stepsfrom ._common import LogFn, _noop_logfrom .commits import persist_incremental_commits@@ -14,6 +23,478 @@ from .health import persist_partial_healthfrom .prune import prune_deleted_files+@dataclass(kw_only=True)+class IncrementalPersist(PersistRun):+ """Inputs of one incremental write, plus the state its steps pass along."""++ repo_path: Any+ graph_builder: Any+ git_meta_map: dict+ dead_code_report: Any = None+ partial_health_report: Any = None+ changed_paths: list[str] = field(default_factory=list)+ doc_drift_report: Any | None = None+ current_graph_file_paths: set[str] | None = None+ file_diffs: list[Any] | None = None+ knowledge_graph_result: Any | None = None+ parsed_files: list[Any] | None = None+ source_map: dict[str, Any] | None = None+ git_decay_map: dict | None = None+ full_git_summary: Any | None = None+ history_window_changed: bool = False+ reconcile_full_scope: bool = False+ full_generation_page_ids: set[str] | None = None+ vector_store: Any | None = None++ session: Any = None+ repo_id: str = ""+ engine: Any = None+ # Filled by the tombstone steps; read after the session closes, so they+ # have to survive a step that was skipped.+ tombstoned_page_ids: list[str] = field(default_factory=list)+ # Same contract, for rows of a page that has been retired outright.+ swept_page_ids: list[str] = field(default_factory=list)+ fts_cleanup_ids: set[str] = field(default_factory=set)+ vector_cleanup_ids: set[str] = field(default_factory=set)+++# --- Steps inside the session -------------------------------------------------+++async def reconcile_scope(ctx: IncrementalPersist) -> None:+ from repowise.core.pipeline.persist import (+ reconcile_full_index_scope,+ )++ current_graph_paths = set(+ getattr(+ ctx.graph_builder,+ "traversed_file_paths",+ {pf.file_info.path for pf in ctx.parsed_files or []},+ )+ )+ current_git_paths = set(ctx.git_meta_map)+ with timed(ctx.timings, "persist.scope_reconcile"):+ ctx.tombstoned_page_ids = await reconcile_full_index_scope(+ ctx.session,+ ctx.repo_id,+ current_graph_paths,+ current_git_paths,+ )+++async def reconcile_generation_scope(ctx: IncrementalPersist) -> None:+ from repowise.core.pipeline.persist import (+ tombstone_pages_outside_generation,+ )++ ctx.tombstoned_page_ids += await tombstone_pages_outside_generation(+ ctx.session, ctx.repo_id, ctx.full_generation_page_ids+ )+++async def sweep_retired(ctx: IncrementalPersist) -> None:+ # Delete rows of pages retired since this index was built. This path never+ # regenerates a repo-wide page, so nothing else here would ever visit one+ # to notice it should be gone, and for a user whose updates all come from+ # the post-commit hook this is the only place a retirement can land.+ from repowise.core.pipeline.persist import (+ sweep_absent_cycle_pages,+ sweep_absent_symbol_spotlights,+ sweep_retired_pages,+ )++ with timed(ctx.timings, "persist.sweeps"):+ ctx.swept_page_ids = await sweep_retired_pages(ctx.session, ctx.repo_id)+ # Same reasoning as the retirement sweep above: this path never+ # regenerates a cycle page, so asking the rebuilt graph whether the+ # cycle still exists is the only way a fixed cycle's page can ever be+ # retired for a user who only runs `update`. Spotlights of removed+ # symbols too.+ ctx.swept_page_ids += await sweep_absent_cycle_pages(+ ctx.session, ctx.repo_id, ctx.graph_builder+ )+ ctx.swept_page_ids += await sweep_absent_symbol_spotlights(+ ctx.session, ctx.repo_id, ctx.graph_builder+ )+++async def mark_tombstones(ctx: IncrementalPersist) -> None:+ # Tombstone pages for deleted/renamed files FIRST — a fresh page for a+ # file that no longer exists misleads every retrieval consumer until the+ # next full regeneration.+ from repowise.core.pipeline.persist import (+ mark_tombstone_pages,+ tombstone_candidates,+ )++ with timed(ctx.timings, "persist.tombstones"):+ ctx.tombstoned_page_ids += await mark_tombstone_pages(+ ctx.session, ctx.repo_id, tombstone_candidates(ctx.file_diffs)+ )+++async def rebuild_tree(ctx: IncrementalPersist) -> None:+ # Placement depends on the whole page set, which on an incremental run+ # lives in the store rather than in the pages just generated.+ from repowise.core.pipeline.page_tree_sync import rebuild_page_tree++ with timed(ctx.timings, "persist.page_tree"):+ await rebuild_page_tree(ctx.session, ctx.repo_id)+++async def persist_git(ctx: IncrementalPersist) -> None:+ with timed(ctx.timings, "persist.git"):+ from repowise.core.pipeline.persist import persist_git_refresh++ await persist_git_refresh(+ ctx.session,+ ctx.repo_id,+ ctx.git_meta_map,+ ctx.git_decay_map,+ ctx.full_git_summary,+ keep_commit_health=not ctx.history_window_changed,+ )+++async def capture_commits(ctx: IncrementalPersist) -> None:+ with timed(ctx.timings, "persist.commits"):+ if ctx.full_git_summary is None:+ await persist_incremental_commits(+ ctx.session, ctx.repo_id, ctx.repo_path, timings=ctx.timings+ )+++async def persist_dead_code(ctx: IncrementalPersist) -> None:+ from repowise.core.persistence.crud import (+ replace_dead_code_findings,+ )++ # The report is repo-wide, and dead code is a cross-file property, so a+ # change-scoped write would drop every verdict the change flipped outside+ # the change set. The scope is the set of files whose confidence was+ # scored on real git metadata; the rest keep what they had.+ with timed(ctx.timings, "persist.dead_code"):+ await replace_dead_code_findings(+ ctx.session,+ ctx.repo_id,+ ctx.dead_code_report.findings,+ scope=ctx.dead_code_report.authoritative_paths,+ )+++async def persist_doc_drift(ctx: IncrementalPersist) -> None:+ from repowise.core.persistence.crud import (+ replace_doc_drift_guarded,+ )++ with timed(ctx.timings, "persist.doc_drift"):+ await replace_doc_drift_guarded(ctx.session, ctx.repo_id, ctx.doc_drift_report)+++async def persist_nodes(ctx: IncrementalPersist) -> None:+ # Re-persist graph_nodes so symbol-level PageRank / betweenness /+ # community ids stay in sync with the current graph build. Without this,+ # ``repowise update`` leaves stale per-symbol metrics from the original+ # init and the UI shows "Not indexed in graph" for every symbol on+ # existing repos. Before health, whose plan annotations read each+ # symbol's ``public_api`` from here; a failed write leaves them reading+ # the last run's flags.+ from repowise.core.pipeline.persist import persist_graph_nodes++ with timed(ctx.timings, "persist.graph_nodes"):+ await persist_graph_nodes(ctx.session, ctx.repo_id, ctx.graph_builder, timings=ctx.timings)+++async def persist_health(ctx: IncrementalPersist) -> None:+ with timed(ctx.timings, "persist.health"):+ await persist_partial_health(+ ctx.session, ctx.repo_id, ctx.partial_health_report, ctx.repo_path+ )+++async def persist_symbols(ctx: IncrementalPersist) -> None:+ # Refresh wiki_symbols for the changed files. Historically the incremental+ # path re-parsed but never persisted symbols, so their start/end bounds+ # fossilized at the last full index and the get_answer hydrator served+ # drifted signatures. Scoped to the changed set for cost.+ from repowise.core.pipeline.persist import persist_incremental_symbols++ with timed(ctx.timings, "persist.symbols"):+ await persist_incremental_symbols(+ ctx.session, ctx.repo_id, ctx.parsed_files, ctx.changed_paths+ )+++async def persist_security(ctx: IncrementalPersist) -> None:+ from repowise.core.pipeline.persist import persist_incremental_security_findings++ with timed(ctx.timings, "persist.security"):+ await persist_incremental_security_findings(+ ctx.session, ctx.repo_id, ctx.parsed_files, ctx.source_map, ctx.changed_paths+ )+++async def persist_edges(ctx: IncrementalPersist) -> None:+ # Refresh graph_edges for the changed files. The full-init path was+ # historically the only writer of edges, so adjacency froze at the last+ # full index: new imports/calls stayed invisible and dropped ones lingered+ # as false paths. Phase E flow-path traversal reads adjacency straight+ # from this table, so it decayed on every update.+ from repowise.core.pipeline.persist import persist_incremental_edges++ with timed(ctx.timings, "persist.edges"):+ await persist_incremental_edges(+ ctx.session, ctx.repo_id, ctx.graph_builder, ctx.parsed_files, ctx.changed_paths+ )+++async def backfill_related(ctx: IncrementalPersist) -> None:+ # Refresh related-pages metadata across the whole wiki. LLM-free, so even+ # index-only updates heal pages generated before the feature shipped (or+ # drifted by new imports) in one run.+ from repowise.core.generation.related_pages import file_import_edges+ from repowise.core.persistence.crud import backfill_related_pages++ with timed(ctx.timings, "persist.related_pages"):+ changed_rel = await backfill_related_pages(+ ctx.session,+ ctx.repo_id,+ import_edges=file_import_edges(ctx.graph_builder),+ git_meta_map=ctx.git_meta_map,+ pagerank=ctx.graph_builder.pagerank(),+ )+ if changed_rel:+ ctx.log(f"Related pages refreshed on {changed_rel} pages")+++async def persist_knowledge_graph(ctx: IncrementalPersist) -> None:+ from repowise.core.pipeline.persist import persist_kg++ with timed(ctx.timings, "persist.kg"):+ await persist_kg(ctx.knowledge_graph_result, ctx.session, ctx.repo_id)+++async def refresh_externals(ctx: IncrementalPersist) -> None:+ # Refresh external systems (C4 L1) when a dependency manifest changed —+ # otherwise the architecture panel serves the init-time dep list forever.+ # Gated + no LLM (see refresh_external_systems).+ with timed(ctx.timings, "persist.external_systems"):+ await refresh_external_systems(+ ctx.session, ctx.repo_id, ctx.repo_path, ctx.file_diffs, log=ctx.log+ )+++async def purge_retired_decisions(ctx: IncrementalPersist) -> None:+ # One-shot drain of proposals from the removed code_comment harvest+ # (#751). Runs on the index-only path too, because the post-commit hook's+ # updates never reach the full decision persist. Confirmed/dismissed rows+ # are kept.+ from repowise.core.persistence.crud import (+ purge_proposed_decisions_by_source,+ )++ await purge_proposed_decisions_by_source(ctx.session, ctx.repo_id, "code_comment")+ # A history-window change re-mines these; a walk that only refreshes rows+ # under new rules does not, so it keeps them.+ if ctx.full_git_summary is not None and ctx.history_window_changed:+ await purge_proposed_decisions_by_source(ctx.session, ctx.repo_id, "git_archaeology")+++async def prune_deleted(ctx: IncrementalPersist) -> None:+ # Last in the session on purpose: everything above upserts, and the git+ # step indexes the changed *paths*, deleted ones included, so the prune+ # has to have the final say on what the store claims.+ await prune_deleted_files(+ ctx.session,+ ctx.repo_id,+ ctx.repo_path,+ ctx.graph_builder,+ (+ ctx.current_graph_file_paths+ if ctx.current_graph_file_paths is not None+ else {pf.file_info.path for pf in ctx.parsed_files or []}+ ),+ health_report=ctx.partial_health_report,+ skip=ctx.skip,+ log=ctx.log,+ degraded=ctx.degraded,+ timings=ctx.timings,+ )+++def _has_git_input(ctx: IncrementalPersist) -> bool:+ return bool(ctx.git_meta_map or ctx.git_decay_map or ctx.full_git_summary is not None)+++# Range-scoped steps are the ones a later run cannot heal on its own; the+# membership test is in persist_incremental_index's docstring.+INCREMENTAL_STEPS: tuple[Step, ...] = (+ Step(+ "Config scope reconciliation",+ reconcile_scope,+ when=lambda ctx: ctx.reconcile_full_scope,+ ),+ Step(+ "Generation scope reconciliation",+ reconcile_generation_scope,+ when=lambda ctx: ctx.full_generation_page_ids is not None,+ ),+ Step("Retired page sweep", sweep_retired),+ Step(+ "Tombstone marking",+ mark_tombstones,+ when=lambda ctx: bool(ctx.file_diffs),+ range_scoped=True,+ ),+ Step("Page tree rebuild", rebuild_tree),+ Step("Git persist", persist_git, when=_has_git_input, range_scoped=True),+ Step("Commit capture", capture_commits, when=_has_git_input),+ Step(+ "Dead-code persist",+ persist_dead_code,+ when=lambda ctx: ctx.dead_code_report is not None,+ range_scoped=True,+ ),+ # Not range_scoped: the pass re-derives every document each run, so a+ # skipped write heals itself on the next update.+ Step(+ "Doc-drift persist",+ persist_doc_drift,+ when=lambda ctx: ctx.doc_drift_report is not None,+ ),+ Step("Graph nodes persist", persist_nodes),+ Step(+ "Health persist",+ persist_health,+ when=lambda ctx: ctx.partial_health_report is not None,+ range_scoped=True,+ ),+ Step("Symbol persist", persist_symbols, range_scoped=True),+ Step("Security persist", persist_security, range_scoped=True),+ Step("Graph edges persist", persist_edges, range_scoped=True),+ Step("Related-pages backfill", backfill_related),+ Step(+ "Knowledge-graph persist",+ persist_knowledge_graph,+ when=lambda ctx: ctx.knowledge_graph_result is not None,+ ),+ Step(+ "External systems refresh",+ refresh_externals,+ when=lambda ctx: bool(ctx.file_diffs),+ range_scoped=True,+ ),+ Step("Decision purge", purge_retired_decisions),+ # Reports its own failures, one sub-step at a time.+ Step("Deleted-file prune", prune_deleted, guarded=False),+)+++# --- Steps after the session closes -------------------------------------------+#+# On SQLite the full-text index shares the database file, so writing to it+# while the session holds a write lock raises "database is locked".+#+# A tombstone can never be an answer — hydration drops it — but retrieval+# fetches a fixed number of rows before that check runs, so every tombstone+# left in the index costs a real candidate its slot.+#+# A swept page's FTS row is worse than a tombstone: search hydrates title and+# snippet from the FTS copy itself, so an orphan answers in full while the+# page it names 404s.+++async def delete_fts_rows(ctx: IncrementalPersist) -> None:+ from repowise.core.persistence.search import FullTextSearch+ from repowise.core.pipeline.cleanup_debt import clear_cleanup_debt++ fts = FullTextSearch(ctx.engine)+ with timed(ctx.timings, "persist.fts"):+ await fts.ensure_index()+ await fts.delete_many(sorted(ctx.fts_cleanup_ids))+ clear_cleanup_debt(Path(ctx.repo_path), "fts", ctx.fts_cleanup_ids)+++async def open_vector_store(ctx: IncrementalPersist) -> None:+ from repowise.core.persistence.vector_store import LanceDBVectorStore+ from repowise.core.providers.embedding.base import MockEmbedder++ # Delete-only access never embeds or checks dimensions.+ ctx.vector_store = LanceDBVectorStore(str(_lance_dir(ctx)), embedder=MockEmbedder())+++async def clear_vector_debt_without_store(ctx: IncrementalPersist) -> None:+ # No store means there are no vector rows left to clean.+ from repowise.core.pipeline.cleanup_debt import clear_cleanup_debt++ clear_cleanup_debt(Path(ctx.repo_path), "vectors", ctx.vector_cleanup_ids)+++async def delete_vector_rows(ctx: IncrementalPersist) -> None:+ from repowise.core.pipeline.cleanup_debt import clear_cleanup_debt++ with timed(ctx.timings, "persist.vectors"):+ await ctx.vector_store.delete_many(sorted(ctx.vector_cleanup_ids))+ clear_cleanup_debt(Path(ctx.repo_path), "vectors", ctx.vector_cleanup_ids)+++def _lance_dir(ctx: IncrementalPersist) -> Path:+ return Path(ctx.repo_path) / ".repowise" / "lancedb"+++def _needs_vector_store(ctx: IncrementalPersist) -> bool:+ return bool(ctx.vector_cleanup_ids) and ctx.vector_store is None+++def _record_debt(kind: str) -> Callable[[IncrementalPersist], None]:+ """Keep the ids a failed cleanup could not remove, for the next run to retry."""++ def record(ctx: IncrementalPersist) -> None:+ from repowise.core.pipeline.cleanup_debt import record_cleanup_debt++ ids = ctx.fts_cleanup_ids if kind == "fts" else ctx.vector_cleanup_ids+ record_cleanup_debt(Path(ctx.repo_path), kind, ids)++ return record+++CLEANUP_STEPS: tuple[Step, ...] = (+ # Range-scoped for the tombstone half: mark_tombstone_pages re-marks and+ # re-returns pages that are already tombstones, so a widened re-run+ # regenerates these ids and retries the delete. The swept half is not+ # repairable this way, because the sweeps deleted those rows and a later+ # run finds nothing to sweep. Tagging still recovers strictly more than+ # not tagging.+ Step(+ "Tombstone full-text removal",+ delete_fts_rows,+ when=lambda ctx: bool(ctx.fts_cleanup_ids),+ range_scoped=True,+ on_error=_record_debt("fts"),+ ),+ Step(+ "Tombstone vector-store open",+ open_vector_store,+ when=lambda ctx: _needs_vector_store(ctx) and _lance_dir(ctx).exists(),+ range_scoped=True,+ on_error=_record_debt("vectors"),+ ),+ Step(+ "Tombstone vector debt clear",+ clear_vector_debt_without_store,+ when=lambda ctx: _needs_vector_store(ctx) and not _lance_dir(ctx).exists(),+ guarded=False,+ ),+ Step(+ "Tombstone vector removal",+ delete_vector_rows,+ when=lambda ctx: bool(ctx.vector_cleanup_ids) and ctx.vector_store is not None,+ on_error=_record_debt("vectors"),+ ),+)++async def persist_incremental_index(repo_path: Any,graph_builder: Any,@@ -69,6 +550,35 @@ async def persist_incremental_index(``persist.*`` row. A step that did not run records nothing, so an absentrow means skipped rather than free."""+ ctx = IncrementalPersist(+ repo_path=repo_path,+ graph_builder=graph_builder,+ git_meta_map=git_meta_map,+ dead_code_report=dead_code_report,+ partial_health_report=partial_health_report,+ changed_paths=changed_paths,+ doc_drift_report=doc_drift_report,+ current_graph_file_paths=current_graph_file_paths,+ file_diffs=file_diffs,+ knowledge_graph_result=knowledge_graph_result,+ parsed_files=parsed_files,+ source_map=source_map,+ git_decay_map=git_decay_map,+ full_git_summary=full_git_summary,+ history_window_changed=history_window_changed,+ reconcile_full_scope=reconcile_full_scope,+ full_generation_page_ids=full_generation_page_ids,+ vector_store=vector_store,+ log=log or _noop_log,+ degraded=degraded,+ failed_steps=failed_steps,+ timings=timings,+ )+ await _write(ctx)+++async def _write(ctx: IncrementalPersist) -> None:+ """Open the store, run every step in one transaction, then the cleanup tail."""from repowise.core.persistence import (create_engine,create_session_factory,@@ -78,330 +588,25 @@ async def persist_incremental_index()from repowise.core.persistence.database import resolve_db_url- log = log or _noop_log-- def _skip(step: str, exc: Exception, *, range_scoped: bool = False) -> None:- log(f"[yellow]{step} skipped: {exc}[/yellow]")- if degraded is not None:- degraded.append(f"{step}: {exc}")- if range_scoped and failed_steps is not None:- failed_steps.append(step)-- url = resolve_db_url(repo_path)- engine = create_engine(url)- # Filled by the tombstone step; read after the session closes, so it has to- # survive a step that was skipped.- tombstoned_page_ids: list[str] = []- # Same contract, for rows of a page that has been retired outright.- swept_page_ids: list[str] = []+ timings = ctx.timings+ ctx.engine = create_engine(resolve_db_url(ctx.repo_path))if timings is not None:timings.start("persist")try:with timed(timings, "persist.open"):- await init_db(engine)- sf = create_session_factory(engine)+ await init_db(ctx.engine)+ sf = create_session_factory(ctx.engine)async with get_session(sf) as session:with timed(timings, "persist.open"):repo = await upsert_repository(- session, name=repo_path.name, local_path=str(repo_path), stamp_head=True- )- repo_id = repo.id-- if reconcile_full_scope:- try:- from repowise.core.pipeline.persist import (- reconcile_full_index_scope,- )-- current_graph_paths = set(- getattr(- graph_builder,- "traversed_file_paths",- {pf.file_info.path for pf in parsed_files or []},- )- )- current_git_paths = set(git_meta_map)- with timed(timings, "persist.scope_reconcile"):- tombstoned_page_ids = await reconcile_full_index_scope(- session,- repo_id,- current_graph_paths,- current_git_paths,- )- except Exception as exc:- _skip("Config scope reconciliation", exc)-- if full_generation_page_ids is not None:- try:- from repowise.core.pipeline.persist import (- tombstone_pages_outside_generation,- )-- tombstoned_page_ids += await tombstone_pages_outside_generation(- session, repo_id, full_generation_page_ids- )- except Exception as exc:- _skip("Generation scope reconciliation", exc)-- # Delete rows of pages retired since this index was built. This- # path never regenerates a repo-wide page, so nothing else here- # would ever visit one to notice it should be gone, and for a user- # whose updates all come from the post-commit hook this is the only- # place a retirement can land.- try:- from repowise.core.pipeline.persist import (- sweep_absent_cycle_pages,- sweep_absent_symbol_spotlights,- sweep_retired_pages,+ session,+ name=ctx.repo_path.name,+ local_path=str(ctx.repo_path),+ stamp_head=True,)-- with timed(timings, "persist.sweeps"):- swept_page_ids = await sweep_retired_pages(session, repo_id)- # Same reasoning as the retirement sweep above: this path- # never regenerates a cycle page, so asking the rebuilt- # graph whether the cycle still exists is the only way a- # fixed cycle's page can ever be retired for a user who- # only runs `update`. Spotlights of removed symbols too.- swept_page_ids += await sweep_absent_cycle_pages(- session, repo_id, graph_builder- )- swept_page_ids += await sweep_absent_symbol_spotlights(- session, repo_id, graph_builder- )- except Exception as exc:- _skip("Retired page sweep", exc)-- # Tombstone pages for deleted/renamed files FIRST — a fresh page- # for a file that no longer exists misleads every retrieval- # consumer until the next full regeneration.- if file_diffs:- try:- from repowise.core.pipeline.persist import (- mark_tombstone_pages,- tombstone_candidates,- )-- with timed(timings, "persist.tombstones"):- tombstoned_page_ids += await mark_tombstone_pages(- session, repo_id, tombstone_candidates(file_diffs)- )- except Exception as exc:- _skip("Tombstone marking", exc, range_scoped=True)-- # Placement depends on the whole page set, which on an incremental- # run lives in the store rather than in the pages just generated.- try:- from repowise.core.pipeline.page_tree_sync import rebuild_page_tree-- with timed(timings, "persist.page_tree"):- await rebuild_page_tree(session, repo_id)- except Exception as exc:- _skip("Page tree rebuild", exc)-- if git_meta_map or git_decay_map or full_git_summary is not None:- try:- with timed(timings, "persist.git"):- from repowise.core.pipeline.persist import persist_git_refresh-- await persist_git_refresh(- session,- repo_id,- git_meta_map,- git_decay_map,- full_git_summary,- keep_commit_health=not history_window_changed,- )- except Exception as exc:- _skip("Git persist", exc, range_scoped=True)-- try:- with timed(timings, "persist.commits"):- if full_git_summary is None:- await persist_incremental_commits(- session, repo_id, repo_path, timings=timings- )- except Exception as exc:- _skip("Commit capture", exc)-- if dead_code_report is not None:- try:- from repowise.core.persistence.crud import (- replace_dead_code_findings,- )-- # The report is repo-wide, and dead code is a cross-file- # property, so a change-scoped write would drop every- # verdict the change flipped outside the change set. The- # scope is the set of files whose confidence was scored on- # real git metadata; the rest keep what they had.- with timed(timings, "persist.dead_code"):- await replace_dead_code_findings(- session,- repo_id,- dead_code_report.findings,- scope=dead_code_report.authoritative_paths,- )- except Exception as exc:- _skip("Dead-code persist", exc, range_scoped=True)-- if doc_drift_report is not None:- try:- from repowise.core.persistence.crud import (- replace_doc_drift_guarded,- )-- with timed(timings, "persist.doc_drift"):- await replace_doc_drift_guarded(- session, repo_id, doc_drift_report- )- except Exception as exc:- # Not range_scoped: the pass re-derives every document each- # run, so a skipped write heals itself on the next update.- _skip("Doc-drift persist", exc)-- # Re-persist graph_nodes so symbol-level PageRank /- # betweenness / community ids stay in sync with the- # current graph build. Without this, ``repowise update``- # leaves stale per-symbol metrics from the original init- # and the UI shows "Not indexed in graph" for every- # symbol on existing repos. Before health, whose plan- # annotations read each symbol's ``public_api`` from here; a- # failed write leaves them reading the last run's flags.- try:- from repowise.core.pipeline.persist import persist_graph_nodes-- with timed(timings, "persist.graph_nodes"):- await persist_graph_nodes(session, repo_id, graph_builder, timings=timings)- except Exception as exc:- _skip("Graph nodes persist", exc)-- if partial_health_report is not None:- try:- with timed(timings, "persist.health"):- await persist_partial_health(- session, repo_id, partial_health_report, repo_path- )- except Exception as exc:- _skip("Health persist", exc, range_scoped=True)-- # Refresh wiki_symbols for the changed files. Historically the- # incremental path re-parsed but never persisted symbols, so their- # start/end bounds fossilized at the last full index and the- # get_answer hydrator served drifted signatures. Scoped to the- # changed set for cost.- try:- from repowise.core.pipeline.persist import persist_incremental_symbols-- with timed(timings, "persist.symbols"):- await persist_incremental_symbols(session, repo_id, parsed_files, changed_paths)- except Exception as exc:- _skip("Symbol persist", exc, range_scoped=True)-- try:- from repowise.core.pipeline.persist import persist_incremental_security_findings-- with timed(timings, "persist.security"):- await persist_incremental_security_findings(- session, repo_id, parsed_files, source_map, changed_paths- )- except Exception as exc:- _skip("Security persist", exc, range_scoped=True)-- # Refresh graph_edges for the changed files. The full-init path was- # historically the only writer of edges, so adjacency froze at the- # last full index: new imports/calls stayed invisible and dropped- # ones lingered as false paths. Phase E flow-path traversal reads- # adjacency straight from this table, so it decayed on every update.- try:- from repowise.core.pipeline.persist import persist_incremental_edges-- with timed(timings, "persist.edges"):- await persist_incremental_edges(- session, repo_id, graph_builder, parsed_files, changed_paths- )- except Exception as exc:- _skip("Graph edges persist", exc, range_scoped=True)-- # Refresh related-pages metadata across the whole wiki. LLM-free,- # so even index-only updates heal pages generated before the- # feature shipped (or drifted by new imports) in one run.- try:- from repowise.core.generation.related_pages import file_import_edges- from repowise.core.persistence.crud import backfill_related_pages-- with timed(timings, "persist.related_pages"):- changed_rel = await backfill_related_pages(- session,- repo_id,- import_edges=file_import_edges(graph_builder),- git_meta_map=git_meta_map,- pagerank=graph_builder.pagerank(),- )- if changed_rel:- log(f"Related pages refreshed on {changed_rel} pages")- except Exception as exc:- _skip("Related-pages backfill", exc)-- if knowledge_graph_result is not None:- try:- from repowise.core.pipeline.persist import persist_kg-- with timed(timings, "persist.kg"):- await persist_kg(knowledge_graph_result, session, repo_id)- except Exception as exc:- _skip("Knowledge-graph persist", exc)-- # Refresh external systems (C4 L1) when a dependency manifest- # changed — otherwise the architecture panel serves the init-time- # dep list forever. Gated + no LLM (see refresh_external_systems).- if file_diffs:- try:- with timed(timings, "persist.external_systems"):- await refresh_external_systems(- session, repo_id, repo_path, file_diffs, log=log- )- except Exception as exc:- _skip("External systems refresh", exc, range_scoped=True)-- # One-shot drain of proposals from the removed code_comment- # harvest (#751). Runs on the index-only path too, because the- # post-commit hook's updates never reach the full decision- # persist. Confirmed/dismissed rows are kept.- try:- from repowise.core.persistence.crud import (- purge_proposed_decisions_by_source,- )-- await purge_proposed_decisions_by_source(session, repo_id, "code_comment")- # A history-window change re-mines these; a walk that only- # refreshes rows under new rules does not, so it keeps them.- if full_git_summary is not None and history_window_changed:- await purge_proposed_decisions_by_source(- session, repo_id, "git_archaeology"- )- except Exception as exc:- _skip("Decision purge", exc)-- # Last in the session on purpose: everything above upserts, and the- # git step indexes the changed *paths*, deleted ones included, so- # the prune has to have the final say on what the store claims.- await prune_deleted_files(- session,- repo_id,- repo_path,- graph_builder,- (- current_graph_file_paths- if current_graph_file_paths is not None- else {pf.file_info.path for pf in parsed_files or []}- ),- health_report=partial_health_report,- skip=_skip,- log=log,- degraded=degraded,- timings=timings,- )+ ctx.session, ctx.repo_id = session, repo.id+ await run_steps(ctx, INCREMENTAL_STEPS)# The transaction commits as the block exits.if timings is not None:timings.start("persist.commit")@@ -413,75 +618,16 @@ async def persist_incremental_index(from repowise.core.pipeline.commit_health import refresh_commit_health_after_updatewith timed(timings, "persist.commit_health"):- await refresh_commit_health_after_update(sf, repo_id, str(repo_path))-- # After the session closes: on SQLite the full-text index shares the- # database file, so writing to it while the session holds a write lock- # raises "database is locked".- #- # A tombstone can never be an answer — hydration drops it — but- # retrieval fetches a fixed number of rows before that check runs, so- # every tombstone left in the index costs a real candidate its slot.- #- # A swept page's FTS row is worse than a tombstone: search hydrates- # title and snippet from the FTS copy itself, so an orphan answers in- # full while the page it names 404s.- from repowise.core.pipeline.cleanup_debt import (- clear_cleanup_debt,- load_cleanup_debt,- record_cleanup_debt,- )+ await refresh_commit_health_after_update(sf, ctx.repo_id, str(ctx.repo_path))++ from repowise.core.pipeline.cleanup_debt import load_cleanup_debt- cleanup_debt = load_cleanup_debt(Path(repo_path))- fts_cleanup_ids = (- set(tombstoned_page_ids) | set(swept_page_ids) | cleanup_debt["fts"]+ cleanup_debt = load_cleanup_debt(Path(ctx.repo_path))+ ctx.fts_cleanup_ids = (+ set(ctx.tombstoned_page_ids) | set(ctx.swept_page_ids) | cleanup_debt["fts"])- if fts_cleanup_ids:- try:- from repowise.core.persistence.search import FullTextSearch-- fts = FullTextSearch(engine)- with timed(timings, "persist.fts"):- await fts.ensure_index()- await fts.delete_many(sorted(fts_cleanup_ids))- clear_cleanup_debt(Path(repo_path), "fts", fts_cleanup_ids)- except Exception as exc:- # Range-scoped for the tombstone half: mark_tombstone_pages- # re-marks and re-returns pages that are already tombstones, so- # a widened re-run regenerates these ids and retries the delete.- # The swept half is not repairable this way, because the sweeps- # deleted those rows and a later run finds nothing to sweep.- # Tagging still recovers strictly more than not tagging.- record_cleanup_debt(Path(repo_path), "fts", fts_cleanup_ids)- _skip("Tombstone full-text removal", exc, range_scoped=True)-- vector_cleanup_ids = set(tombstoned_page_ids) | cleanup_debt["vectors"]- if vector_cleanup_ids and vector_store is None:- lance_dir = Path(repo_path) / ".repowise" / "lancedb"- if lance_dir.exists():- try:- from repowise.core.persistence.vector_store import LanceDBVectorStore- from repowise.core.providers.embedding.base import MockEmbedder-- # Delete-only access never embeds or checks dimensions.- vector_store = LanceDBVectorStore(str(lance_dir), embedder=MockEmbedder())- except Exception as exc:- record_cleanup_debt(- Path(repo_path), "vectors", vector_cleanup_ids- )- _skip("Tombstone vector-store open", exc, range_scoped=True)- else:- # No store means there are no vector rows left to clean.- clear_cleanup_debt(Path(repo_path), "vectors", vector_cleanup_ids)-- if vector_cleanup_ids and vector_store is not None:- try:- with timed(timings, "persist.vectors"):- await vector_store.delete_many(sorted(vector_cleanup_ids))- clear_cleanup_debt(Path(repo_path), "vectors", vector_cleanup_ids)- except Exception as exc:- record_cleanup_debt(Path(repo_path), "vectors", vector_cleanup_ids)- _skip("Tombstone vector removal", exc)+ ctx.vector_cleanup_ids = set(ctx.tombstoned_page_ids) | cleanup_debt["vectors"]+ await run_steps(ctx, CLEANUP_STEPS)# Ceiling: retired non-file pages' *vector* embeddings survive this path.# There is no store here to delete them from, and building one would@@ -492,6 +638,6 @@ async def persist_incremental_index(# can still surface in semantic search until the next docs-mode update# (which does delete it) or a reindex.finally:- await engine.dispose()+ await ctx.engine.dispose()if timings is not None:timings.stop("persist")diff --git a/packages/core/src/repowise/core/pipeline/step_runner.py b/packages/core/src/repowise/core/pipeline/step_runner.pynew file mode 100644index 000000000..686610bb5--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/step_runner.py@@ -0,0 +1,87 @@+"""Ordered, failure-isolated persistence steps.++A writer lists its steps once, in order, and :func:`run_steps` runs them: a+guarded step that raises is reported through :meth:`PersistRun.skip` and the+run carries on. Steps time themselves (``timed``), so each step decides+exactly which ``persist.*`` row it records.+"""++from __future__ import annotations++from collections.abc import Awaitable, Callable, Iterable+from dataclasses import dataclass+from typing import Any++from repowise.core.pipeline.phase_timing import PhaseTimings+++def _noop_log(message: str) -> None: # pragma: no cover - trivial+ return None+++@dataclass(kw_only=True)+class PersistRun:+ """What every step of one write shares: where failures and timings go.++ ``degraded`` collects ``"<step>: <error>"`` for every skipped step.+ ``failed_steps`` collects the names of skipped steps marked+ ``range_scoped``: those whose input was this commit range, which the+ caller records as a repair marker. Writers subclass this with their inputs+ and the state their steps pass along.+ """++ log: Callable[[str], None] = _noop_log+ degraded: list[str] | None = None+ failed_steps: list[str] | None = None+ timings: PhaseTimings | None = None++ def skip(self, step: str, exc: Exception, *, range_scoped: bool = False) -> None:+ self.log(f"[yellow]{step} skipped: {exc}[/yellow]")+ if self.degraded is not None:+ self.degraded.append(f"{step}: {exc}")+ if range_scoped and self.failed_steps is not None:+ self.failed_steps.append(step)+++@dataclass(frozen=True)+class Step:+ """One persistence step.++ ``name`` is the label a failure is reported under. ``when`` gates the step+ on the run's inputs. ``on_error`` runs before the failure is reported (for+ bookkeeping such as cleanup debt), ``reraise`` decides after it whether the+ failure fails the whole write. An unguarded step runs without a net.+ """++ name: str+ run: Callable[[Any], Awaitable[Any]]+ when: Callable[[Any], bool] | None = None+ range_scoped: bool = False+ on_error: Callable[[Any], None] | None = None+ reraise: Callable[[Any], bool] | None = None+ guarded: bool = True+++async def run_step(ctx: PersistRun, step: Step) -> None:+ if step.when is not None and not step.when(ctx):+ return+ if not step.guarded:+ await step.run(ctx)+ return+ try:+ await step.run(ctx)+ except Exception as exc:+ if step.on_error is not None:+ step.on_error(ctx)+ ctx.skip(step.name, exc, range_scoped=step.range_scoped)+ if step.reraise is not None and step.reraise(ctx):+ raise+++async def run_steps(ctx: PersistRun, steps: Iterable[Step]) -> None:+ """Run *steps* in order against *ctx*."""+ for step in steps:+ await run_step(ctx, step)+++__all__ = ["PersistRun", "Step", "run_step", "run_steps"]diff --git a/tests/unit/_persist_step_recorders.py b/tests/unit/_persist_step_recorders.pyindex 0bd5056fe..bd9420a50 100644--- a/tests/unit/_persist_step_recorders.py+++ b/tests/unit/_persist_step_recorders.py@@ -2,8 +2,9 @@Each step is replaced, in the module namespace it is looked up in at calltime, by a stand-in that appends its label to one shared call log and can be-told to raise. The persisters import their steps inside the function body, so-patching the defining module is enough.+told to raise. Most steps are imported inside the function body, so patching+the defining module is enough; a step imported at module level is also patched+at every module that re-exports it (``where=``)."""from __future__ import annotations@@ -36,7 +37,21 @@ class StepRecorder:# A fresh copy: callers extend returned lists in place.return list(ret) if isinstance(ret, list) else ret- def patch(self, module: Any, attr: str, *, ret: Any = None, sync: bool = False) -> None:+ def assert_all_called(self, labels: Any) -> None:+ """Fail on any label that never ran, i.e. a patch that missed its lookup site."""+ missing = [label for label in labels if label not in self.order]+ assert not missing, f"steps never called: {missing}"++ def patch(+ self,+ module: Any,+ attr: str,+ *,+ ret: Any = None,+ sync: bool = False,+ where: tuple[Any, ...] = (),+ ) -> None:+ """Replace ``attr`` on its defining ``module`` and on every module in ``where``."""if sync:def stand_in(*args, **kwargs):@@ -47,7 +62,8 @@ class StepRecorder:async def stand_in(*args, **kwargs):return self._record(attr, args, kwargs, ret)- self.monkeypatch.setattr(module, attr, stand_in)+ for target in (module, *where):+ self.monkeypatch.setattr(target, attr, stand_in)def fts_class(self) -> type:rec = self@@ -90,6 +106,9 @@ def install_shared_steps(rec: StepRecorder, *, prune_ret: tuple[int, list[str]]import repowise.core.persistence.search as searchimport repowise.core.pipeline.commit_health as commit_healthimport repowise.core.pipeline.incremental as incremental+ import repowise.core.pipeline.incremental.commits as commits+ import repowise.core.pipeline.incremental.external_systems as external_systems+ import repowise.core.pipeline.incremental.health as healthimport repowise.core.pipeline.incremental.persist_index as persist_indeximport repowise.core.pipeline.page_tree_sync as page_tree_syncimport repowise.core.pipeline.persist as persist@@ -104,22 +123,20 @@ def install_shared_steps(rec: StepRecorder, *, prune_ret: tuple[int, list[str]]rec.patch(persist, "mark_tombstone_pages", ret=[TOMBSTONED])rec.patch(page_tree_sync, "rebuild_page_tree")rec.patch(persist, "persist_git_refresh")- # The package for the CLI, which imports it at call time; the writer for core.- rec.patch(incremental, "persist_incremental_commits")- rec.patch(persist_index, "persist_incremental_commits")+ # The CLI looks these up on the package at call time, the core writer on its own module.+ reexported = (incremental, persist_index)+ rec.patch(commits, "persist_incremental_commits", where=reexported)rec.patch(crud, "replace_dead_code_findings")rec.patch(crud, "replace_doc_drift_guarded")rec.patch(persist, "persist_graph_nodes")- rec.patch(incremental, "persist_partial_health")- rec.patch(persist_index, "persist_partial_health")+ rec.patch(health, "persist_partial_health", where=reexported)rec.patch(persist, "persist_incremental_symbols")rec.patch(persist, "persist_incremental_security_findings")rec.patch(persist, "persist_incremental_edges")rec.patch(related_pages, "file_import_edges", ret={}, sync=True)rec.patch(crud, "backfill_related_pages", ret=0)rec.patch(persist, "persist_kg")- rec.patch(incremental, "refresh_external_systems")- rec.patch(persist_index, "refresh_external_systems")+ rec.patch(external_systems, "refresh_external_systems", where=reexported)rec.patch(crud, "purge_proposed_decisions_by_source")rec.patch(persist, "prune_deleted_file_rows", ret=prune_ret)rec.patch(persist, "_analyzed_commit", ret="abc123")diff --git a/tests/unit/cli/test_full_update_persist_step_order.py b/tests/unit/cli/test_full_update_persist_step_order.pyindex 84ed6e4c8..30dbda0d6 100644--- a/tests/unit/cli/test_full_update_persist_step_order.py+++ b/tests/unit/cli/test_full_update_persist_step_order.py@@ -178,6 +178,7 @@ async def test_every_step_runs_in_the_pinned_order(repo, monkeypatch):total = await _persist_full(repo, rec, _Inputs(), degraded)+ rec.assert_all_called(HAPPY_ORDER)assert rec.order == HAPPY_ORDERassert degraded == []# The page upsert is a recorder, so the store holds no page to count.diff --git a/tests/unit/pipeline/test_graph_nodes_before_health.py b/tests/unit/pipeline/test_graph_nodes_before_health.pyindex 3700959ca..6d121bdeb 100644--- a/tests/unit/pipeline/test_graph_nodes_before_health.py+++ b/tests/unit/pipeline/test_graph_nodes_before_health.py@@ -12,13 +12,12 @@ import inspectimport pytestfrom repowise.cli.commands.update_cmd import persistence as cli_update-from repowise.core.pipeline import incremental+from repowise.core.pipeline.incremental import persist_index@pytest.mark.parametrize(("function", "health_call"),[- (incremental.persist_incremental_index, "await persist_partial_health("),(cli_update._persist_full_update_async, "await _persist_partial_health("),(cli_update._rescore_health_from_db, "await save_full_health_report("),],@@ -27,3 +26,11 @@ def test_graph_nodes_persist_before_health(function, health_call: str) -> None:source = inspect.getsource(function)assert "await persist_graph_nodes(" in source and health_call in sourceassert source.index("await persist_graph_nodes(") < source.index(health_call)+++def test_incremental_steps_persist_graph_nodes_before_health() -> None:+ names = [step.name for step in persist_index.INCREMENTAL_STEPS]+ assert names.index("Graph nodes persist") < names.index("Health persist")+ by_name = {step.name: step.run for step in persist_index.INCREMENTAL_STEPS}+ assert "await persist_graph_nodes(" in inspect.getsource(by_name["Graph nodes persist"])+ assert "await persist_partial_health(" in inspect.getsource(by_name["Health persist"])diff --git a/tests/unit/pipeline/test_incremental_failure_isolation.py b/tests/unit/pipeline/test_incremental_failure_isolation.pyindex a022d7f8b..8c82db6b5 100644--- a/tests/unit/pipeline/test_incremental_failure_isolation.py+++ b/tests/unit/pipeline/test_incremental_failure_isolation.py@@ -128,13 +128,11 @@ def test_the_range_scoped_set_is_exactly_these_steps():re-walks whatever it skipped. ``External systems refresh`` is the reverse,since it only runs when a dependency manifest is in *this* diff."""- import re- from pathlib import Path-import repowise.core.pipeline.incremental.persist_index as mod- source = Path(mod.__file__).read_text(encoding="utf-8")- tagged = set(re.findall(r'_skip\(\s*"([^"]+)",\s*exc,\s*range_scoped=True', source))+ tagged = {+ step.name for step in (*mod.INCREMENTAL_STEPS, *mod.CLEANUP_STEPS) if step.range_scoped+ }assert tagged == {"Tombstone marking","Git persist",diff --git a/tests/unit/pipeline/test_incremental_persist_step_order.py b/tests/unit/pipeline/test_incremental_persist_step_order.pyindex 6a15b690a..9ea1ae286 100644--- a/tests/unit/pipeline/test_incremental_persist_step_order.py+++ b/tests/unit/pipeline/test_incremental_persist_step_order.py@@ -101,6 +101,7 @@ async def test_every_step_runs_in_the_pinned_order(repo, monkeypatch):degraded, failed_steps = await _persist(repo, rec)+ rec.assert_all_called(HAPPY_ORDER)assert rec.order == HAPPY_ORDERassert degraded == []assert failed_steps == []