Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(update): run the docs-mode persist on the core step runner (#4020)

Raghav Chamadiya8a06fd7bd6ca
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py b/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py
new file mode 100644
index 000000000..03b7c0e67
--- /dev/null
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py
@@ -0,0 +1,500 @@
+"""The docs-mode update writer: every DB step of a full update, in order.
+
+The steps the index-only writer also runs are its own step functions from
+:mod:`repowise.core.pipeline.incremental.persist_index`; the docs-only ones
+(pages, stale decay, decisions, generation job, full-text indexing) live here.
+:data:`FULL_UPDATE_STEPS` runs inside the session, :data:`FULL_UPDATE_CLEANUP`
+after it closes.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from repowise.core.pipeline.incremental.persist_index import (
+ IncrementalPersist,
+ capture_commits,
+ mark_tombstones,
+ persist_doc_drift,
+ persist_edges,
+ persist_git,
+ persist_health,
+ persist_knowledge_graph,
+ persist_security,
+ persist_symbols,
+ prune_deleted,
+ rebuild_tree,
+ reconcile_generation_scope,
+ reconcile_scope,
+ sweep_retired,
+)
+from repowise.core.pipeline.phase_timing import timed
+from repowise.core.pipeline.step_runner import Step, run_steps
+
+# The writer's degraded steps keep logging under the module that calls it.
+log = structlog.get_logger("repowise.cli.commands.update_cmd.persistence")
+
+
+@dataclass(kw_only=True)
+class FullUpdatePersist(IncrementalPersist):
+ """Inputs of one docs-mode write, on top of the index-only writer's."""
+
+ repo_name: str = ""
+ generated_pages: list = field(default_factory=list)
+ new_decision_markers: list = field(default_factory=list)
+ # The shared page/decision vector store, despite the name.
+ decision_vector_store: Any | None = None
+ provider: Any = None
+ decay_paths: list[str] | None = None
+ reconcile_full_generation: bool = False
+ require_config_rebuild_success: bool = False
+ require_decision_persist_success: bool = False
+ degraded_before_persist: int = 0
+ total_pages: int = 0
+
+ def skip(self, step: str, exc: Exception, *, range_scoped: bool = False) -> None:
+ log.warning("update_persist_step_degraded", step=step, error=str(exc))
+ self.degraded.append(f"{step}: {exc}")
+ if range_scoped and self.failed_steps is not None:
+ self.failed_steps.append(step)
+
+
+def _requires_rebuild(ctx: FullUpdatePersist) -> bool:
+ return ctx.require_config_rebuild_success
+
+
+async def _drop_vectors(ctx: FullUpdatePersist, page_ids: list[str]) -> None:
+ if page_ids and ctx.decision_vector_store is not None:
+ await ctx.decision_vector_store.delete_many(page_ids)
+
+
+# --- Steps inside the session -------------------------------------------------
+
+
+async def reconcile_scope_step(ctx: FullUpdatePersist) -> None:
+ await reconcile_scope(ctx)
+ await _drop_vectors(ctx, ctx.tombstoned_page_ids)
+
+
+async def upsert_pages(ctx: FullUpdatePersist) -> None:
+ # Batched (one SELECT + one flush); the checkpointer sink already streamed
+ # each page per-commit for durability.
+ from repowise.core.persistence import upsert_pages_from_generated
+
+ with timed(ctx.timings, "persist.pages"):
+ await upsert_pages_from_generated(ctx.session, ctx.generated_pages, ctx.repo_id)
+
+
+async def reconcile_generation_step(ctx: FullUpdatePersist) -> None:
+ await reconcile_generation_scope(ctx)
+ await _drop_vectors(ctx, ctx.tombstoned_page_ids)
+
+
+async def sweep_retired_step(ctx: FullUpdatePersist) -> None:
+ await sweep_retired(ctx)
+ # Before the SQL session commits, as ``init`` does: a separate engine and
+ # an idempotent delete, so an interrupted run self-heals forwards.
+ await _drop_vectors(ctx, ctx.swept_page_ids)
+
+
+async def backfill_related(ctx: FullUpdatePersist) -> None:
+ # The generation pass only touched this run's pages; this LLM-free
+ # backfill heals every other page in the same transaction.
+ from repowise.core.generation.related_pages import file_import_edges
+ from repowise.core.persistence.crud import backfill_related_pages
+
+ with timed(ctx.timings, "persist.related_pages"):
+ await backfill_related_pages(
+ ctx.session,
+ ctx.repo_id,
+ import_edges=file_import_edges(ctx.graph_builder),
+ git_meta_map=ctx.git_meta_map,
+ pagerank=ctx.graph_builder.pagerank(),
+ # This run's pages carry fresher metadata (including module
+ # siblings) than the recompute could produce.
+ skip_page_ids={p.page_id for p in ctx.generated_pages},
+ )
+
+
+async def decay_stale_pages(ctx: FullUpdatePersist) -> None:
+ # Weakly-affected pages decay to 'stale' so coverage and get_stale_pages see them.
+ from repowise.core.generation import GenerationConfig
+ from repowise.core.generation.cascade import expand_cascade
+ from repowise.core.generation.models import compute_page_id
+ from repowise.core.generation.scope import build_dependencies, load_page_records
+ from repowise.core.pipeline.persist import mark_page_ids_stale
+ from repowise.core.pipeline.scoped_generation import _load_page_rows, load_kg_context
+ from repowise.core.repo_config import load_repo_config
+
+ with timed(ctx.timings, "persist.stale_pages"):
+ if not ctx.decay_paths:
+ return
+ generation_config = GenerationConfig.from_repo_config(load_repo_config(ctx.repo_path))
+ records = load_page_records(await _load_page_rows(ctx.session, ctx.repo_id))
+ deps = build_dependencies(
+ parsed_files=ctx.parsed_files or [],
+ graph_builder=ctx.graph_builder,
+ config=generation_config,
+ kg_ctx=load_kg_context(Path(ctx.repo_path)),
+ records=records,
+ repo_name=ctx.repo_name,
+ )
+ seed_ids = {compute_page_id("file_page", path) for path in ctx.decay_paths}
+ # mode="none": mark dependents stale, never regenerate them; that spend
+ # is opt-in through `generate --stale`.
+ cascade = expand_cascade(seed_ids, "none", deps)
+ # A page this run refreshed is fresh; never decay it again afterwards.
+ generated_ids = {page.page_id for page in ctx.generated_pages}
+ await mark_page_ids_stale(
+ ctx.session, ctx.repo_id, (cascade.stale_ids | seed_ids) - generated_ids
+ )
+
+
+async def repair_decisions(ctx: FullUpdatePersist) -> None:
+ # The store repairs the full-index path runs, each failure its own
+ # degraded step; a required persist fails the update on the first one.
+ from repowise.core.pipeline.persist import repair_decision_store
+
+ def skip(step: str, exc: Exception) -> None:
+ ctx.skip(step, exc)
+ if ctx.require_decision_persist_success:
+ raise exc
+
+ await repair_decision_store(
+ ctx.session, ctx.repo_id, vector_store=ctx.decision_vector_store, skip=skip
+ )
+
+
+async def upsert_decisions(ctx: FullUpdatePersist) -> None:
+ from repowise.core.persistence.crud import (
+ bulk_upsert_decisions,
+ purge_proposed_decisions_by_source,
+ recompute_decision_staleness,
+ )
+
+ session, repo_id = ctx.session, ctx.repo_id
+ if ctx.require_decision_persist_success:
+ await purge_proposed_decisions_by_source(session, repo_id, "git_archaeology")
+ decision_dicts = [dataclasses.asdict(d) for d in ctx.new_decision_markers or []]
+ if decision_dicts:
+ touched_ids = await bulk_upsert_decisions(
+ session, repo_id, decision_dicts, vector_store=ctx.decision_vector_store
+ )
+ if touched_ids and ctx.decision_vector_store is not None:
+ from repowise.core.analysis.decision_evolution import (
+ detect_supersessions_and_conflicts,
+ )
+
+ await detect_supersessions_and_conflicts(
+ session,
+ repo_id,
+ touched_ids=touched_ids,
+ vector_store=ctx.decision_vector_store,
+ provider=ctx.provider,
+ )
+ if ctx.git_meta_map:
+ await recompute_decision_staleness(session, repo_id, ctx.git_meta_map)
+
+
+DECISION_STEPS: tuple[Step, ...] = (
+ # Reports its own failures, one repair at a time.
+ Step("Decision store repairs", repair_decisions, guarded=False),
+ Step(
+ "Decision persist",
+ upsert_decisions,
+ reraise=lambda ctx: ctx.require_decision_persist_success,
+ ),
+)
+
+
+async def persist_decisions(ctx: FullUpdatePersist) -> None:
+ with timed(ctx.timings, "persist.decisions"):
+ await run_steps(ctx, DECISION_STEPS)
+
+
+async def apply_lifecycle(ctx: FullUpdatePersist) -> None:
+ # Every run, not only when this one added records: the revert usually
+ # lands after the decision it retires was stored.
+ from repowise.core.pipeline.persist import apply_decision_lifecycle
+
+ await apply_decision_lifecycle(ctx.session, ctx.repo_id, ctx.repo_path, skip=ctx.skip)
+
+
+async def refresh_governance(ctx: FullUpdatePersist) -> None:
+ from repowise.core.pipeline.persist import refresh_governance_findings
+
+ with timed(ctx.timings, "persist.governance"):
+ # The health step below finalizes the refactoring queue.
+ await refresh_governance_findings(ctx.session, ctx.repo_id, recompose_queue=False)
+
+
+async def persist_nodes(ctx: FullUpdatePersist) -> None:
+ # Before health, whose plan annotations read each symbol's ``public_api``.
+ from repowise.core.pipeline.persist import persist_graph_nodes
+
+ with timed(ctx.timings, "persist.graph_nodes"):
+ await persist_graph_nodes(ctx.session, ctx.repo_id, ctx.graph_builder)
+
+
+async def persist_dead_code(ctx: FullUpdatePersist) -> None:
+ # Repo-wide: dead code is a cross-file property, so a file-scoped write
+ # would keep an unchanged module's stale verdict.
+ from repowise.core.persistence.crud import replace_dead_code_findings
+
+ with timed(ctx.timings, "persist.dead_code"):
+ await replace_dead_code_findings(
+ ctx.session,
+ ctx.repo_id,
+ [dataclasses.asdict(f) for f in ctx.dead_code_report.findings],
+ scope=ctx.dead_code_report.authoritative_paths,
+ )
+
+
+async def refresh_externals(ctx: FullUpdatePersist) -> None:
+ from repowise.core.pipeline.incremental import refresh_external_systems
+
+ with timed(ctx.timings, "persist.external_systems"):
+ await refresh_external_systems(ctx.session, ctx.repo_id, ctx.repo_path, ctx.file_diffs)
+
+
+async def record_generation_job(ctx: FullUpdatePersist) -> None:
+ # Updates the web UI "last synced" timestamp.
+ from datetime import UTC, datetime
+
+ from repowise.core.generation.models import count_stub_fallbacks
+ from repowise.core.persistence.crud import upsert_generation_job
+
+ now = datetime.now(UTC)
+ page_count = len(ctx.generated_pages)
+ # A stub the provider failure put up has a row but no prose, so it is
+ # not a completed page.
+ stub_fallbacks = count_stub_fallbacks(ctx.generated_pages)
+ job = await upsert_generation_job(
+ ctx.session,
+ repository_id=ctx.repo_id,
+ status="completed",
+ total_pages=page_count,
+ config={"mode": "incremental", "source": "cli_update"},
+ )
+ job.completed_pages = page_count - stub_fallbacks
+ job.failed_pages = stub_fallbacks
+ job.started_at = now
+ job.finished_at = now
+
+
+async def count_pages(ctx: FullUpdatePersist) -> None:
+ # The real total for state.json; the upsert count stays if this degrades.
+ from sqlalchemy import func, select
+
+ from repowise.core.persistence.models import Page
+
+ result = await ctx.session.execute(
+ select(func.count()).select_from(Page).where(Page.repository_id == ctx.repo_id)
+ )
+ ctx.total_pages = int(result.scalar_one())
+
+
+async def check_config_rebuild(ctx: FullUpdatePersist) -> None:
+ if ctx.require_config_rebuild_success and len(ctx.degraded) > ctx.degraded_before_persist:
+ raise RuntimeError(
+ "Configuration rebuild did not update every dependent store; "
+ "the previous fingerprint was retained so the next update retries."
+ )
+
+
+def _has_git_input(ctx: FullUpdatePersist) -> bool:
+ return bool(ctx.git_meta_map or ctx.git_decay_map or ctx.full_git_summary is not None)
+
+
+# Pages first and without a net: everything else is derived metadata, but a
+# docs-mode update that cannot write pages failed. Range-scoped as in the
+# index-only writer.
+FULL_UPDATE_STEPS: tuple[Step, ...] = (
+ Step(
+ "Config scope reconciliation",
+ reconcile_scope_step,
+ when=lambda ctx: ctx.reconcile_full_scope,
+ reraise=_requires_rebuild,
+ ),
+ Step("Page upsert", upsert_pages, guarded=False),
+ Step(
+ "Generation scope reconciliation",
+ reconcile_generation_step,
+ when=lambda ctx: ctx.reconcile_full_generation,
+ reraise=_requires_rebuild,
+ ),
+ Step("Retired page sweep", sweep_retired_step),
+ Step("Tombstone marking", mark_tombstones, range_scoped=True),
+ Step("Related-pages backfill", backfill_related),
+ Step("Stale-page decay", decay_stale_pages),
+ Step("Page tree rebuild", rebuild_tree),
+ Step(
+ "Knowledge-graph persist",
+ persist_knowledge_graph,
+ when=lambda ctx: ctx.knowledge_graph_result is not None,
+ ),
+ # A full walk stamps the history version once persisted, so losing its
+ # rows here must fail the run, not stamp it.
+ Step(
+ "Git persist",
+ persist_git,
+ when=_has_git_input,
+ range_scoped=True,
+ reraise=lambda ctx: ctx.require_config_rebuild_success
+ or ctx.full_git_summary is not None,
+ ),
+ Step("Commit capture", capture_commits, when=_has_git_input),
+ Step("Decisions", persist_decisions, guarded=False),
+ Step("Decision lifecycle", apply_lifecycle, guarded=False),
+ Step("Governance findings", refresh_governance),
+ # Before health: the snapshot it takes records the stored drift count.
+ Step(
+ "Doc-drift persist",
+ persist_doc_drift,
+ when=lambda ctx: ctx.doc_drift_report is not None,
+ ),
+ Step("Graph nodes persist", persist_nodes),
+ Step(
+ "Health persist",
+ persist_health,
+ when=lambda ctx: ctx.partial_health_report is not None,
+ range_scoped=True,
+ ),
+ Step(
+ "Dead-code persist",
+ persist_dead_code,
+ when=lambda ctx: ctx.dead_code_report is not None,
+ range_scoped=True,
+ ),
+ Step("Symbol persist", persist_symbols, range_scoped=True),
+ Step("Security persist", persist_security, range_scoped=True),
+ Step("Graph edges persist", persist_edges, range_scoped=True),
+ Step("External systems refresh", refresh_externals, range_scoped=True),
+ # Last of the store writes: the steps above upsert rows for deleted paths
+ # too. Reports its own failures, one sub-step at a time.
+ Step("Deleted-file prune", prune_deleted, guarded=False),
+ Step("Generation job record", record_generation_job),
+ Step("Page count", count_pages),
+ Step("Config rebuild check", check_config_rebuild, guarded=False),
+)
+
+
+# --- Steps after the session closes -------------------------------------------
+
+
+async def index_full_text(ctx: FullUpdatePersist) -> None:
+ # Outside the transaction: rebuildable, and its writer manages its own
+ # connection state.
+ from repowise.core.persistence import FullTextSearch
+ from repowise.core.pipeline.cleanup_debt import clear_cleanup_debt
+
+ with timed(ctx.timings, "persist.fts"):
+ fts = FullTextSearch(ctx.engine)
+ await fts.ensure_index()
+ await fts.index_pages(ctx.generated_pages)
+ if ctx.fts_cleanup_ids:
+ await fts.delete_many(sorted(ctx.fts_cleanup_ids))
+ clear_cleanup_debt(Path(ctx.repo_path), "fts", ctx.fts_cleanup_ids)
+
+
+async def delete_vectors(ctx: FullUpdatePersist) -> None:
+ from repowise.core.pipeline.cleanup_debt import clear_cleanup_debt
+
+ await ctx.decision_vector_store.delete_many(sorted(ctx.vector_cleanup_ids))
+ clear_cleanup_debt(Path(ctx.repo_path), "vectors", ctx.vector_cleanup_ids)
+
+
+def _record_fts_debt(ctx: FullUpdatePersist) -> None:
+ from repowise.core.pipeline.cleanup_debt import record_cleanup_debt
+
+ record_cleanup_debt(Path(ctx.repo_path), "fts", ctx.fts_cleanup_ids)
+
+
+def _record_vector_debt(ctx: FullUpdatePersist) -> None:
+ from repowise.core.pipeline.cleanup_debt import record_cleanup_debt
+
+ record_cleanup_debt(Path(ctx.repo_path), "vectors", ctx.vector_cleanup_ids)
+
+
+FULL_UPDATE_CLEANUP: tuple[Step, ...] = (
+ Step(
+ "Full-text search indexing",
+ index_full_text,
+ range_scoped=True,
+ on_error=_record_fts_debt,
+ reraise=_requires_rebuild,
+ ),
+ Step(
+ "Deferred vector cleanup",
+ delete_vectors,
+ when=lambda ctx: bool(ctx.vector_cleanup_ids)
+ and ctx.decision_vector_store is not None,
+ on_error=_record_vector_debt,
+ reraise=_requires_rebuild,
+ ),
+)
+
+
+async def write_full_update(ctx: FullUpdatePersist) -> int:
+ """Open the store, run every step in one transaction, then the cleanup tail."""
+ from repowise.cli.helpers import get_db_url_for_repo
+ from repowise.core.persistence import (
+ create_engine,
+ create_session_factory,
+ get_session,
+ init_db,
+ upsert_repository,
+ )
+ from repowise.core.pipeline.cleanup_debt import load_cleanup_debt
+
+ timings = ctx.timings
+ ctx.engine = create_engine(get_db_url_for_repo(ctx.repo_path))
+ ctx.degraded_before_persist = len(ctx.degraded)
+ ctx.total_pages = len(ctx.generated_pages)
+ try:
+ with timed(timings, "persist.open"):
+ await init_db(ctx.engine)
+ sf = create_session_factory(ctx.engine)
+
+ async with get_session(sf) as session:
+ with timed(timings, "persist.open"):
+ repo = await upsert_repository(
+ session, name=ctx.repo_name, local_path=str(ctx.repo_path), stamp_head=True
+ )
+ ctx.session, ctx.repo_id = session, repo.id
+ await run_steps(ctx, FULL_UPDATE_STEPS)
+ # The transaction commits as the block exits.
+ if timings is not None:
+ timings.start("persist.commit")
+ if timings is not None:
+ timings.stop("persist.commit")
+
+ debt = load_cleanup_debt(Path(ctx.repo_path))
+ ctx.fts_cleanup_ids = set(ctx.tombstoned_page_ids) | set(ctx.swept_page_ids) | debt["fts"]
+ # The union closes the mark_tombstone_pages path: the scope steps
+ # already drop their ids, but a deleted file's embedding did not.
+ ctx.vector_cleanup_ids = set(ctx.tombstoned_page_ids) | debt["vectors"]
+ await run_steps(ctx, FULL_UPDATE_CLEANUP)
+
+ # After the transaction, so the scan never holds the write lock, and on
+ # every update, so a backlog an earlier budget left drains.
+ from repowise.core.pipeline.commit_health import refresh_commit_health_after_update
+
+ with timed(timings, "persist.commit_health"):
+ await refresh_commit_health_after_update(sf, ctx.repo_id, str(ctx.repo_path))
+ return ctx.total_pages
+ finally:
+ await ctx.engine.dispose()
+
+
+__all__ = [
+ "DECISION_STEPS",
+ "FULL_UPDATE_CLEANUP",
+ "FULL_UPDATE_STEPS",
+ "FullUpdatePersist",
+]
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 3d4fb1b2e..f85ad8a10 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -760,6 +760,7 @@ def _persist_full_update(
require_config_rebuild_success: bool = False,
require_decision_persist_success: bool = False,
history_window_changed: bool = False,
+ failed_steps: list[str] | None = None,
timings: PhaseTimings | None = None,
) -> int:
"""Persist a full (LLM-regenerating) update in one transaction.
@@ -776,6 +777,9 @@ def _persist_full_update(
can stamp ``state["total_pages"]`` with the real DB total (regeneration
upserts existing pages, so accumulating ``len(generated_pages)`` inflates
the count forever).
+
+ ``failed_steps`` (when supplied) collects the range-scoped failed steps,
+ as :func:`~repowise.core.pipeline.incremental.persist_incremental_index` does.
"""
return run_async(
_persist_full_update_async(
@@ -803,6 +807,7 @@ def _persist_full_update(
require_config_rebuild_success=require_config_rebuild_success,
require_decision_persist_success=require_decision_persist_success,
history_window_changed=history_window_changed,
+ failed_steps=failed_steps,
timings=timings,
)
)
@@ -834,685 +839,50 @@ async def _persist_full_update_async(
require_config_rebuild_success: bool = False,
require_decision_persist_success: bool = False,
history_window_changed: bool = False,
+ failed_steps: list[str] | None = None,
timings: PhaseTimings | None = None,
) -> int:
- from repowise.cli.helpers import get_db_url_for_repo
- from repowise.core.persistence import (
- FullTextSearch,
- create_engine,
- create_session_factory,
- get_session,
- init_db,
- upsert_pages_from_generated,
- upsert_repository,
+ from .full_persist import FullUpdatePersist, write_full_update
+
+ ctx = FullUpdatePersist(
+ repo_path=repo_path,
+ repo_name=repo_name,
+ generated_pages=generated_pages,
+ file_diffs=file_diffs,
+ git_meta_map=git_meta_map,
+ new_decision_markers=new_decision_markers,
+ decision_vector_store=decision_vector_store,
+ provider=provider,
+ partial_health_report=partial_health_report,
+ dead_code_report=dead_code_report,
+ graph_builder=graph_builder,
+ knowledge_graph_result=knowledge_graph_result,
+ doc_drift_report=doc_drift_report,
+ decay_paths=decay_paths,
+ parsed_files=parsed_files,
+ source_map=source_map,
+ git_decay_map=git_decay_map,
+ full_git_summary=full_git_summary,
+ reconcile_full_scope=reconcile_full_scope,
+ reconcile_full_generation=reconcile_full_generation,
+ full_generation_page_ids=(
+ {page.page_id for page in generated_pages} if reconcile_full_generation else None
+ ),
+ require_config_rebuild_success=require_config_rebuild_success,
+ require_decision_persist_success=require_decision_persist_success,
+ history_window_changed=history_window_changed,
+ # A config scope change re-covers every parsed file, not just the diff.
+ changed_paths=(
+ [parsed.file_info.path for parsed in parsed_files or []]
+ if reconcile_full_scope
+ else [fd.path for fd in file_diffs]
+ ),
+ log=console.print,
+ degraded=degraded,
+ failed_steps=failed_steps,
+ timings=timings,
)
-
- def _skip(step: str, exc: Exception) -> None:
- log.warning("update_persist_step_degraded", step=step, error=str(exc))
- degraded.append(f"{step}: {exc}")
-
- url = get_db_url_for_repo(repo_path)
- engine = create_engine(url)
- degraded_before_persist = len(degraded)
- # Filled by the tombstone step; read by the full-text block after the
- # session closes, so it has to survive a step that was skipped.
- tombstoned_page_ids: list[str] = []
- # Same contract, for rows of a page that has been retired outright.
- swept_page_ids: list[str] = []
- try:
- with timed(timings, "persist.open"):
- await init_db(engine)
- sf = create_session_factory(engine)
-
- async with get_session(sf) as session:
- with timed(timings, "persist.open"):
- repo = await upsert_repository(
- session, name=repo_name, local_path=str(repo_path), stamp_head=True
- )
- repo_id = repo.id
-
- if reconcile_full_scope:
- try:
- from repowise.core.pipeline.persist import (
- reconcile_full_index_scope,
- )
-
- current_graph_paths = set(
- getattr(
- graph_builder,
- "traversed_file_paths",
- {parsed.file_info.path for parsed in parsed_files or []},
- )
- )
- with timed(timings, "persist.scope_reconcile"):
- tombstoned_page_ids = await reconcile_full_index_scope(
- session,
- repo_id,
- current_graph_paths,
- set(git_meta_map),
- )
- if tombstoned_page_ids and decision_vector_store is not None:
- await decision_vector_store.delete_many(tombstoned_page_ids)
- except Exception as exc:
- _skip("Config scope reconciliation", exc)
- if require_config_rebuild_success:
- raise
-
- # Pages first and without a net: everything else is derived
- # metadata, but a docs-mode update that can't write pages failed.
- # Batched (one SELECT + one flush); the checkpointer sink already
- # streamed each page per-commit for durability.
- with timed(timings, "persist.pages"):
- await upsert_pages_from_generated(session, generated_pages, repo_id)
-
- if reconcile_full_generation:
- try:
- from repowise.core.pipeline.persist import (
- tombstone_pages_outside_generation,
- )
-
- tombstoned_page_ids += await tombstone_pages_outside_generation(
- session,
- repo_id,
- {page.page_id for page in generated_pages},
- )
- if tombstoned_page_ids and decision_vector_store is not None:
- await decision_vector_store.delete_many(tombstoned_page_ids)
- except Exception as exc:
- _skip("Generation scope reconciliation", exc)
- if require_config_rebuild_success:
- raise
-
- # Delete rows of pages that have been retired since this index was
- # built. Nothing else on the update path can reach them: an update
- # runs with ``file_pages_only``, so the ladder returns before the
- # repo-wide levels and never visits an onboarding row to notice it
- # should not exist. Without this the retirement reaches only users
- # who re-index, and everyone else keeps being served a page the
- # product no longer has.
- try:
- from repowise.core.pipeline.persist import (
- sweep_absent_cycle_pages,
- sweep_absent_symbol_spotlights,
- sweep_retired_pages,
- )
-
- with timed(timings, "persist.sweeps"):
- swept_page_ids = await sweep_retired_pages(session, repo_id)
- # Cycle pages are never regenerated on this path (the
- # ladder stops at file pages), so a cycle that no longer
- # exists can only be retired by asking the rebuilt graph
- # directly. A spotlight whose symbol left its file is the
- # same case.
- swept_page_ids += await sweep_absent_cycle_pages(
- session, repo_id, graph_builder
- )
- swept_page_ids += await sweep_absent_symbol_spotlights(
- session, repo_id, graph_builder
- )
-
- # Drop the embeddings before the SQL session commits, the same
- # ordering ``init`` uses: the vector store is a separate engine,
- # so there is no write-lock conflict, the delete is idempotent,
- # and an interrupted run self-heals forwards rather than leaving
- # an embedding whose page is gone. ``decision_vector_store`` is
- # the shared page/decision store despite the name.
- if swept_page_ids and decision_vector_store is not None:
- await decision_vector_store.delete_many(swept_page_ids)
- except Exception as exc:
- _skip("Retired page sweep", exc)
-
- # Tombstone pages for deleted/renamed files — regeneration only
- # rewrites pages for files that still exist.
- try:
- from repowise.core.pipeline.persist import (
- mark_tombstone_pages,
- tombstone_candidates,
- )
-
- with timed(timings, "persist.tombstones"):
- tombstoned_page_ids += await mark_tombstone_pages(
- session, repo_id, tombstone_candidates(file_diffs)
- )
- except Exception as exc:
- _skip("Tombstone marking", exc)
-
- # Refresh related-pages metadata repo-wide. The generation pass
- # only touched this run's regenerated pages; this LLM-free
- # backfill heals every other page (pre-feature pages, import
- # drift) in the same transaction.
- try:
- from repowise.core.generation.related_pages import file_import_edges
- from repowise.core.persistence.crud import backfill_related_pages
-
- with timed(timings, "persist.related_pages"):
- await backfill_related_pages(
- session,
- repo_id,
- import_edges=file_import_edges(graph_builder),
- git_meta_map=git_meta_map,
- pagerank=graph_builder.pagerank(),
- # This run's pages carry fresher metadata (including
- # module siblings) than the recompute could produce.
- skip_page_ids={p.page_id for p in generated_pages},
- )
- except Exception as exc:
- _skip("Related-pages backfill", exc)
-
- # Weakly-affected pages (cascade overflow beyond the budget,
- # co-change partners, 2-hop rename fallout) decay to 'stale' so
- # the coverage view and get_stale_pages reflect them — the
- # detector computed decay_only all along but it was never
- # persisted.
- try:
- from repowise.core.generation import GenerationConfig
- from repowise.core.generation.cascade import expand_cascade
- from repowise.core.generation.models import compute_page_id
- from repowise.core.generation.scope import (
- build_dependencies,
- load_page_records,
- )
- from repowise.core.pipeline.persist import mark_page_ids_stale
- from repowise.core.pipeline.scoped_generation import (
- _load_page_rows,
- load_kg_context,
- )
- from repowise.core.repo_config import load_repo_config
-
- with timed(timings, "persist.stale_pages"):
- if decay_paths:
- repo_cfg = load_repo_config(repo_path)
- generation_config = GenerationConfig.from_repo_config(repo_cfg)
- records = load_page_records(await _load_page_rows(session, repo_id))
- deps = build_dependencies(
- parsed_files=parsed_files or [],
- graph_builder=graph_builder,
- config=generation_config,
- kg_ctx=load_kg_context(Path(repo_path)),
- records=records,
- repo_name=repo_name,
- )
- seed_ids = {compute_page_id("file_page", path) for path in decay_paths}
- # mode="none": mark dependents stale, do not regenerate
- # them. "dependents" would regenerate every module/SCC/
- # repo-wide container touched by this commit, spending
- # model budget on every `update`, the exact cost
- # AUTO_SYNC.md promises sync never incurs. Marking is
- # free; the operator opts into the spend explicitly via
- # `generate --stale`.
- cascade = expand_cascade(seed_ids, "none", deps)
- # A no-model structural recovery can have refreshed one
- # of these dependents in this same run. The page upsert
- # above made it fresh; never decay it again afterwards.
- generated_ids = {page.page_id for page in generated_pages}
- await mark_page_ids_stale(
- session,
- repo_id,
- (cascade.stale_ids | seed_ids) - generated_ids,
- )
- except Exception as exc:
- _skip("Stale-page decay", exc)
-
- # Placement depends on the whole page set, which on an incremental
- # run lives in the store rather than in the pages just generated.
- try:
- from repowise.core.pipeline.page_tree_sync import rebuild_page_tree
-
- with timed(timings, "persist.page_tree"):
- await rebuild_page_tree(session, repo_id)
- except Exception as exc:
- _skip("Page tree rebuild", exc)
-
- # Refreshed knowledge graph — same writers as the init pipeline
- # (full-replace layers/tour/curated meta).
- if knowledge_graph_result is not None:
- try:
- from repowise.core.pipeline.persist import persist_kg
-
- with timed(timings, "persist.kg"):
- await persist_kg(knowledge_graph_result, session, repo_id)
- except Exception as exc:
- _skip("Knowledge-graph persist", exc)
-
- # Updated git metadata + recomputed percentiles + new commit rows.
- if git_meta_map or git_decay_map or full_git_summary is not None:
- try:
- with timed(timings, "persist.git"):
- from repowise.core.pipeline.persist import persist_git_refresh
-
- await persist_git_refresh(
- session,
- repo_id,
- git_meta_map,
- git_decay_map,
- full_git_summary,
- # Set only on a history-window (git config) change,
- # the one walk that invalidates per-commit health.
- keep_commit_health=not history_window_changed,
- )
- except Exception as exc:
- _skip("Git persist", exc)
- # A full walk stamps the history version once persisted,
- # so losing its rows here must fail the run, not stamp it.
- if require_config_rebuild_success or full_git_summary is not None:
- raise
- try:
- with timed(timings, "persist.commits"):
- if full_git_summary is None:
- await _persist_incremental_commits(
- session, repo_id, repo_path, timings=timings
- )
- except Exception as exc:
- _skip("Commit capture", exc)
-
- # Decision records: new markers + harvested decisions, supersession
- # detection, staleness recompute.
- if timings is not None:
- timings.start("persist.decisions")
- try:
- # The same four store repairs the full-index path runs, in
- # the same order (see ``pipeline/persist.py``). They live here
- # too because a user whose workflow is ``repowise update``
- # never takes that path, and every one of them is a repair the
- # store cannot make for itself: ``superseded`` and the
- # retired-source backlog both survive re-extraction,
- # ``source_rank`` is a value copied into rows rather than
- # derived on read, and a stored confidence carries no mark of
- # which formula produced it.
- from repowise.core.analysis.decision_provenance import RETIRED_SOURCES
- from repowise.core.persistence.crud import (
- purge_proposed_decisions_by_source,
- reconcile_decision_confidence,
- reconcile_source_ranks,
- unretire_auto_superseded,
- )
-
- # Before the purge: restoring lands a row at ``proposed``,
- # which is what the purge deletes.
- await unretire_auto_superseded(session)
- for _retired in RETIRED_SOURCES:
- await purge_proposed_decisions_by_source(session, repo_id, _retired)
- await reconcile_source_ranks(session)
- await reconcile_decision_confidence(session)
-
- # Same repairs on the path a ``repowise update`` user takes.
- # Derived ids come first, so everything after this reads a
- # record by the id it will still have after a rebuild.
- from repowise.core.persistence.decision_id_migration import (
- apply_id_migration,
- )
-
- await apply_id_migration(session, repo_id, vector_store=decision_vector_store)
-
- # The entity split is only coherent once legacy rows are
- # classified.
- from repowise.core.persistence.decision_migration import (
- apply_migration,
- backfill_decision_node_links,
- backfill_scope_basis,
- backfill_session_scope_basis,
- prune_unindexed_scope_files,
- )
-
- await apply_migration(session, repo_id)
-
- # Run every index, beside the classification repair and for
- # the same reason: a record written before these rules existed
- # is only reachable from code that runs on an existing store.
- # The prune runs first so the basis repairs judge the file
- # list they will leave behind.
- await prune_unindexed_scope_files(session, repo_id)
- await backfill_scope_basis(session, repo_id)
- await backfill_session_scope_basis(session, repo_id)
- await backfill_decision_node_links(session, repo_id)
-
- if require_decision_persist_success:
- from repowise.core.persistence.crud import (
- purge_proposed_decisions_by_source,
- )
-
- await purge_proposed_decisions_by_source(session, repo_id, "git_archaeology")
-
- decision_dicts: list[dict] = []
- if new_decision_markers:
- import dataclasses as _dc
-
- decision_dicts.extend(_dc.asdict(d) for d in new_decision_markers)
- # A second read folded in ``page.metadata["harvested_decisions"]``
- # here. Nothing ever wrote that key, so it never contributed a
- # record; it is gone along with the generation-time harvest.
-
- if decision_dicts:
- from repowise.core.persistence.crud import bulk_upsert_decisions
-
- touched_ids = await bulk_upsert_decisions(
- session,
- repo_id,
- decision_dicts,
- vector_store=decision_vector_store,
- )
- if touched_ids and decision_vector_store is not None:
- from repowise.core.analysis.decision_evolution import (
- detect_supersessions_and_conflicts,
- )
-
- await detect_supersessions_and_conflicts(
- session,
- repo_id,
- touched_ids=touched_ids,
- vector_store=decision_vector_store,
- provider=provider,
- )
-
- if git_meta_map:
- from repowise.core.persistence.crud import recompute_decision_staleness
-
- await recompute_decision_staleness(session, repo_id, git_meta_map)
- except Exception as exc:
- _skip("Decision persist", exc)
- if require_decision_persist_success:
- raise
- finally:
- if timings is not None:
- timings.stop("persist.decisions")
-
- # Every run, not only when this one added records: the revert
- # usually lands after the decision it retires was stored.
- try:
- from repowise.core.analysis.decisions.reverts import (
- apply_revert_supersession,
- )
-
- await apply_revert_supersession(session, repo_id, repo_path)
- except Exception as exc:
- _skip("Revert supersession", exc)
- try:
- from repowise.core.analysis.decisions.head_artifacts import (
- apply_head_artifact_check,
- )
-
- await apply_head_artifact_check(session, repo_id, repo_path)
- except Exception as exc:
- _skip("Decision HEAD artifact check", exc)
-
- # Governance findings pass: runs after decisions + staleness.
- if timings is not None:
- timings.start("persist.governance")
- try:
- from repowise.core.pipeline.persist import refresh_governance_findings
-
- # The health step below finalizes the refactoring queue.
- await refresh_governance_findings(session, repo_id, recompose_queue=False)
- except Exception as exc:
- _skip("Governance findings", exc)
- finally:
- if timings is not None:
- timings.stop("persist.governance")
-
- # Scoped to the documents the pass actually read, so one this run
- # could not open keeps its rows in both drift tables. Before health:
- # the snapshot it takes records the stored drift count.
- if doc_drift_report is not None:
- try:
- from repowise.core.persistence.crud import (
- replace_doc_drift_guarded,
- )
-
- with timed(timings, "persist.doc_drift"):
- await replace_doc_drift_guarded(
- session, repo_id, doc_drift_report
- )
- except Exception as exc:
- _skip("Doc-drift persist", exc)
-
- # Re-persist graph_nodes so symbol-level PageRank / betweenness /
- # community ids reflect the current build. Before health, whose
- # plan annotations read each symbol's ``public_api`` from here; a
- # failed write leaves them reading the last run's flags.
- try:
- from repowise.core.pipeline.persist import persist_graph_nodes
-
- with timed(timings, "persist.graph_nodes"):
- await persist_graph_nodes(session, repo_id, graph_builder)
- except Exception as exc:
- _skip("Graph nodes persist", exc)
-
- # Code-health findings + metrics (partial — upsert only).
- if partial_health_report is not None:
- try:
- with timed(timings, "persist.health"):
- await _persist_partial_health(
- session, repo_id, partial_health_report, repo_path
- )
- except Exception as exc:
- _skip("Health persist", exc)
-
- # Repo-wide. The old file-scoped write kept unchanged files' rows
- # (#295) at the price of never correcting them: the analysis is
- # repo-wide and dead code is a cross-file property, so dropping an
- # import could make a module dead and that module, being unchanged,
- # kept its stale verdict until the next full re-index.
- if dead_code_report is not None:
- try:
- import dataclasses as _dc_dead
-
- from repowise.core.persistence.crud import replace_dead_code_findings
-
- with timed(timings, "persist.dead_code"):
- await replace_dead_code_findings(
- session,
- repo_id,
- [_dc_dead.asdict(f) for f in dead_code_report.findings],
- scope=dead_code_report.authoritative_paths,
- )
- except Exception as exc:
- _skip("Dead-code persist", exc)
-
- # Refresh wiki_symbols for the changed files (upsert + within-file
- # prune). Without this, symbol bounds fossilize at the last full
- # index and the get_answer hydrator serves drifted signatures.
- try:
- from repowise.core.pipeline.persist import persist_incremental_symbols
-
- with timed(timings, "persist.symbols"):
- await persist_incremental_symbols(
- session,
- repo_id,
- parsed_files,
- (
- [parsed.file_info.path for parsed in parsed_files or []]
- if reconcile_full_scope
- else [fd.path for fd in file_diffs]
- ),
- )
- except Exception as exc:
- _skip("Symbol persist", exc)
-
- try:
- from repowise.core.pipeline.persist import persist_incremental_security_findings
-
- with timed(timings, "persist.security"):
- await persist_incremental_security_findings(
- session,
- repo_id,
- parsed_files,
- source_map,
- (
- [parsed.file_info.path for parsed in parsed_files or []]
- if reconcile_full_scope
- else [fd.path for fd in file_diffs]
- ),
- )
- except Exception as exc:
- _skip("Security persist", exc)
-
- # Refresh graph_edges for the changed files (delete-then-insert of
- # each file's outgoing edges). Without this, adjacency fossilizes at
- # the last full index and Phase E flow-path answers decay on every
- # incremental update.
- try:
- from repowise.core.pipeline.persist import persist_incremental_edges
-
- with timed(timings, "persist.edges"):
- await persist_incremental_edges(
- session,
- repo_id,
- graph_builder,
- parsed_files,
- (
- [parsed.file_info.path for parsed in parsed_files or []]
- if reconcile_full_scope
- else [fd.path for fd in file_diffs]
- ),
- )
- except Exception as exc:
- _skip("Graph edges persist", exc)
-
- # Refresh external systems (C4 L1) when a manifest changed — the
- # docs-mode persister mirrors the incremental one, which does the
- # same. Gated + no LLM inside the shared core helper.
- try:
- from repowise.core.pipeline.incremental import refresh_external_systems
-
- with timed(timings, "persist.external_systems"):
- await refresh_external_systems(session, repo_id, repo_path, file_diffs)
- except Exception as exc:
- _skip("External systems refresh", exc)
-
- # Last of the store writes, for the reason the index-only path
- # gives: the steps above upsert rows for deleted paths too.
- from repowise.core.pipeline.incremental import prune_deleted_files
-
- await prune_deleted_files(
- session,
- repo_id,
- repo_path,
- graph_builder,
- {parsed.file_info.path for parsed in parsed_files or []},
- health_report=partial_health_report,
- skip=_skip,
- log=console.print,
- degraded=degraded,
- timings=timings,
- )
-
- # Record a GenerationJob so the web UI "last synced" timestamp updates.
- try:
- from datetime import UTC as _UTC
- from datetime import datetime
-
- from repowise.core.generation.models import count_stub_fallbacks
- from repowise.core.persistence.crud import upsert_generation_job
-
- now = datetime.now(_UTC)
- page_count = len(generated_pages)
- # See init_cmd/persistence.py: a stub the provider failure put
- # up has a row but no prose, so it is not a completed page.
- stub_fallbacks = count_stub_fallbacks(generated_pages)
- job = await upsert_generation_job(
- session,
- repository_id=repo_id,
- status="completed",
- total_pages=page_count,
- config={"mode": "incremental", "source": "cli_update"},
- )
- job.completed_pages = page_count - stub_fallbacks
- job.failed_pages = stub_fallbacks
- job.started_at = now
- job.finished_at = now
- except Exception as exc:
- _skip("Generation job record", exc)
-
- # Real page total for state.json — see _persist_full_update's
- # docstring. Falls back to the upsert count if the count query
- # itself degrades.
- total_pages = len(generated_pages)
- try:
- from sqlalchemy import func as sa_func
- from sqlalchemy import select as sa_select
-
- from repowise.core.persistence.models import Page
-
- count_result = await session.execute(
- sa_select(sa_func.count())
- .select_from(Page)
- .where(Page.repository_id == repo_id)
- )
- total_pages = int(count_result.scalar_one())
- except Exception as exc:
- _skip("Page count", exc)
-
- if require_config_rebuild_success and len(degraded) > degraded_before_persist:
- raise RuntimeError(
- "Configuration rebuild did not update every dependent store; "
- "the previous fingerprint was retained so the next update retries."
- )
- # The transaction commits as the block exits.
- if timings is not None:
- timings.start("persist.commit")
- if timings is not None:
- timings.stop("persist.commit")
-
- # FTS outside the transaction — rebuildable, and its writer manages
- # its own connection state.
- if timings is not None:
- timings.start("persist.fts")
- from repowise.core.pipeline.cleanup_debt import (
- clear_cleanup_debt,
- load_cleanup_debt,
- record_cleanup_debt,
- )
-
- cleanup_ids = set(tombstoned_page_ids) | set(swept_page_ids)
- cleanup_ids.update(load_cleanup_debt(Path(repo_path))["fts"])
- try:
- fts = FullTextSearch(engine)
- await fts.ensure_index()
- await fts.index_pages(generated_pages)
- # A tombstone can never be an answer — hydration drops it — but
- # retrieval fetches a fixed number of rows before that check runs,
- # so every tombstone left in the index costs a real candidate its
- # slot.
- if cleanup_ids:
- await fts.delete_many(sorted(cleanup_ids))
- # A swept page's FTS row outlives the page row unless it is deleted
- # here, and search hydrates title and snippet from the FTS copy
- # itself — so an orphan keeps answering queries in full, pointing
- # at a page that now 404s. Worse than never having swept it.
- clear_cleanup_debt(Path(repo_path), "fts", cleanup_ids)
- except Exception as exc:
- record_cleanup_debt(Path(repo_path), "fts", cleanup_ids)
- _skip("Full-text search indexing", exc)
- if require_config_rebuild_success:
- raise
- finally:
- if timings is not None:
- timings.stop("persist.fts")
-
- # The union closes the mark_tombstone_pages path: the scope branches
- # above already drop their ids, but a deleted file's embedding did not.
- vector_cleanup_ids = (
- set(tombstoned_page_ids) | load_cleanup_debt(Path(repo_path))["vectors"]
- )
- if vector_cleanup_ids and decision_vector_store is not None:
- try:
- await decision_vector_store.delete_many(sorted(vector_cleanup_ids))
- clear_cleanup_debt(Path(repo_path), "vectors", vector_cleanup_ids)
- except Exception as exc:
- record_cleanup_debt(Path(repo_path), "vectors", vector_cleanup_ids)
- _skip("Deferred vector cleanup", exc)
- if require_config_rebuild_success:
- raise
-
- # After the transaction, so the scan never holds the write lock, and on
- # every update, so a backlog an earlier budget left drains.
- from repowise.core.pipeline.commit_health import refresh_commit_health_after_update
-
- with timed(timings, "persist.commit_health"):
- await refresh_commit_health_after_update(sf, repo_id, str(repo_path))
- return total_pages
- finally:
- await engine.dispose()
+ return await write_full_update(ctx)
async def _rescore_health_from_db(
diff --git a/packages/core/src/repowise/core/pipeline/persist/decisions.py b/packages/core/src/repowise/core/pipeline/persist/decisions.py
index 7d993a579..f3b69ea83 100644
--- a/packages/core/src/repowise/core/pipeline/persist/decisions.py
+++ b/packages/core/src/repowise/core/pipeline/persist/decisions.py
@@ -6,7 +6,7 @@ Shared steps of every path that persists decision records.
from __future__ import annotations
import dataclasses
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
from typing import Any
import structlog
@@ -14,12 +14,26 @@ import structlog
logger = structlog.get_logger("repowise.core.pipeline.persist")
-async def repair_decision_store(session: Any, repo_id: str, *, vector_store: Any = None) -> None:
- """Run the idempotent store repairs, in order, each failure logged and skipped.
+async def repair_decision_store(
+ session: Any,
+ repo_id: str,
+ *,
+ vector_store: Any = None,
+ skip: Callable[[str, Exception], None] | None = None,
+) -> None:
+ """Run the idempotent store repairs, in order, each failure skipped.
Runs whether or not the caller has new decisions: every repair fixes rows
- an earlier run left behind.
+ an earlier run left behind. A failure goes to ``skip(step, exc)`` when
+ given, else to the debug log.
"""
+
+ def _skipped(step: str, event: str, exc: Exception) -> None:
+ if skip is not None:
+ skip(step, exc)
+ else:
+ logger.debug(event, error=str(exc))
+
# Restore records the semantic supersession detector retired before it was
# turned off. ``superseded`` is a protected status, so nothing else will
# ever walk these back, and a store that only stops retiring is still a
@@ -36,7 +50,7 @@ async def repair_decision_store(session: Any, repo_id: str, *, vector_store: Any
await unretire_auto_superseded(session)
except Exception as _unretire_err:
- logger.debug("decision_unretire_skipped", error=str(_unretire_err))
+ _skipped("Decision unretire", "decision_unretire_skipped", _unretire_err)
# One-shot drain of proposals left by retired extraction sources; without
# this, DBs indexed before a removal keep a flooded review queue forever
@@ -48,7 +62,7 @@ async def repair_decision_store(session: Any, repo_id: str, *, vector_store: Any
for _retired in RETIRED_SOURCES:
await purge_proposed_decisions_by_source(session, repo_id, _retired)
except Exception as _purge_err:
- logger.debug("decision_purge_skipped", error=str(_purge_err))
+ _skipped("Retired-source decision purge", "decision_purge_skipped", _purge_err)
# Re-stamp evidence rows left on a previous SOURCE_RANK ladder. Local stores
# are created by ``init_db`` and never see Alembic, so the migration alone
@@ -60,7 +74,7 @@ async def repair_decision_store(session: Any, repo_id: str, *, vector_store: Any
await reconcile_source_ranks(session)
except Exception as _rank_err:
- logger.debug("decision_rank_reconcile_skipped", error=str(_rank_err))
+ _skipped("Decision source-rank repair", "decision_rank_reconcile_skipped", _rank_err)
# The same repair for the scoring formula, which leaves every input valid
# and every stored score stale, so no filter can find it. Its own try, so
@@ -70,7 +84,7 @@ async def repair_decision_store(session: Any, repo_id: str, *, vector_store: Any
await reconcile_decision_confidence(session)
except Exception as _conf_err:
- logger.debug("decision_confidence_reconcile_skipped", error=str(_conf_err))
+ _skipped("Decision confidence repair", "decision_confidence_reconcile_skipped", _conf_err)
# Move legacy records onto ids derived from their own identity, before
# anything else reads or writes one. A random id is re-minted whenever a
@@ -81,7 +95,7 @@ async def repair_decision_store(session: Any, repo_id: str, *, vector_store: Any
await apply_id_migration(session, repo_id, vector_store=vector_store)
except Exception as _id_err:
- logger.debug("decision_id_migration_skipped", error=str(_id_err))
+ _skipped("Decision id migration", "decision_id_migration_skipped", _id_err)
# Classify legacy rows against the entity split. A record promoted by
# recurrence rather than by a person becomes a candidate, which visibly
@@ -107,7 +121,7 @@ async def repair_decision_store(session: Any, repo_id: str, *, vector_store: Any
await backfill_session_scope_basis(session, repo_id)
await backfill_decision_node_links(session, repo_id)
except Exception as _migrate_err:
- logger.debug("decision_entity_migration_skipped", error=str(_migrate_err))
+ _skipped("Decision entity migration", "decision_entity_migration_skipped", _migrate_err)
async def store_decisions(
@@ -180,10 +194,17 @@ async def store_decisions(
return touched_ids
-async def apply_decision_lifecycle(session: Any, repo_id: str, repo_path: Any) -> None:
+async def apply_decision_lifecycle(
+ session: Any,
+ repo_id: str,
+ repo_path: Any,
+ *,
+ skip: Callable[[str, Exception], None] | None = None,
+) -> None:
"""Retire decisions whose commits were reverted or whose HEAD artifacts are gone.
- Every run, whether or not it stored decisions. Each pass is best-effort.
+ Every run, whether or not it stored decisions. Each pass is best-effort: a
+ failure goes to ``skip(step, exc)`` when given, else to the debug log.
"""
# Retire records whose commits were reverted. Not tied to new records: a
# new revert retires a decision stored by an earlier run just as well.
@@ -192,10 +213,16 @@ async def apply_decision_lifecycle(session: Any, repo_id: str, repo_path: Any) -
await apply_revert_supersession(session, repo_id, repo_path)
except Exception as _revert_err:
- logger.debug("revert_supersession_skipped", error=str(_revert_err))
+ if skip is not None:
+ skip("Revert supersession", _revert_err)
+ else:
+ logger.debug("revert_supersession_skipped", error=str(_revert_err))
try:
from repowise.core.analysis.decisions.head_artifacts import apply_head_artifact_check
await apply_head_artifact_check(session, repo_id, repo_path)
except Exception as _artifact_err:
- logger.debug("head_artifact_check_skipped", error=str(_artifact_err))
+ if skip is not None:
+ skip("Decision HEAD artifact check", _artifact_err)
+ else:
+ logger.debug("head_artifact_check_skipped", error=str(_artifact_err))
diff --git a/tests/unit/_persist_step_recorders.py b/tests/unit/_persist_step_recorders.py
index bd9420a50..574fa9288 100644
--- a/tests/unit/_persist_step_recorders.py
+++ b/tests/unit/_persist_step_recorders.py
@@ -123,7 +123,7 @@ def install_shared_steps(rec: StepRecorder, *, prune_ret: tuple[int, list[str]]
rec.patch(persist, "mark_tombstone_pages", ret=[TOMBSTONED])
rec.patch(page_tree_sync, "rebuild_page_tree")
rec.patch(persist, "persist_git_refresh")
- # The CLI looks these up on the package at call time, the core writer on its own module.
+ # Package callers look these up at call time, the core writer on its own module.
reexported = (incremental, persist_index)
rec.patch(commits, "persist_incremental_commits", where=reexported)
rec.patch(crud, "replace_dead_code_findings")
diff --git a/tests/unit/cli/test_full_update_persist_step_order.py b/tests/unit/cli/test_full_update_persist_step_order.py
index 30dbda0d6..a5cb66f62 100644
--- a/tests/unit/cli/test_full_update_persist_step_order.py
+++ b/tests/unit/cli/test_full_update_persist_step_order.py
@@ -148,7 +148,13 @@ class _Inputs:
self.pages = [_page("file_page:src/a.py")]
-async def _persist_full(repo: Path, rec: StepRecorder, inputs: _Inputs, degraded: list[str]) -> int:
+async def _persist_full(
+ repo: Path,
+ rec: StepRecorder,
+ inputs: _Inputs,
+ degraded: list[str],
+ failed_steps: list[str] | None = None,
+) -> int:
return await _persist_full_update_async(
repo_path=repo,
repo_name=repo.name,
@@ -168,6 +174,7 @@ async def _persist_full(repo: Path, rec: StepRecorder, inputs: _Inputs, degraded
parsed_files=inputs.parsed_files,
reconcile_full_scope=True,
reconcile_full_generation=True,
+ failed_steps=failed_steps,
)
@@ -192,23 +199,24 @@ async def test_every_step_runs_in_the_pinned_order(repo, monkeypatch):
assert rec.args_of("fts.delete_many") == [((sorted([SWEPT, TOMBSTONED]),), {})]
+# Steps left working when every other one fails.
+_SURVIVORS = {
+ # Pages have no net: a failure here fails the update.
+ "upsert_pages_from_generated",
+ "tombstone_candidates",
+ "file_import_edges",
+ "prune_deleted_file_rows",
+ "_analyzed_commit",
+ "refresh_commit_health_after_update",
+}
+
+
async def test_every_failing_step_degrades_in_order(repo, monkeypatch):
from repowise.core.pipeline.cleanup_debt import record_cleanup_debt
# Seeded debt is what reaches the vector tail once tombstoning has failed.
record_cleanup_debt(repo, "vectors", {"file_page:src/old.py"})
- survivors = {
- # Pages have no net: a failure here fails the update.
- "upsert_pages_from_generated",
- "tombstone_candidates",
- "file_import_edges",
- "prune_deleted_file_rows",
- "_analyzed_commit",
- "refresh_commit_health_after_update",
- }
- rec = StepRecorder(
- monkeypatch, fail=frozenset(label for label in HAPPY_ORDER if label not in survivors)
- )
+ rec = StepRecorder(monkeypatch, fail=frozenset(HAPPY_ORDER) - _SURVIVORS)
_install(rec, prune_ret=(1, [REFUSAL]))
degraded: list[str] = []
@@ -228,8 +236,14 @@ async def test_every_failing_step_degrades_in_order(repo, monkeypatch):
"persist_kg",
"persist_git_refresh",
"persist_incremental_commits",
- # One try around the whole chain: the first repair ends it.
+ # Each repair fails on its own; the upsert still runs.
"unretire_auto_superseded",
+ "purge_proposed_decisions_by_source",
+ "reconcile_source_ranks",
+ "reconcile_decision_confidence",
+ "apply_id_migration",
+ "apply_migration",
+ "bulk_upsert_decisions",
"apply_revert_supersession",
"apply_head_artifact_check",
"refresh_governance_findings",
@@ -263,6 +277,12 @@ async def test_every_failing_step_degrades_in_order(repo, monkeypatch):
"Knowledge-graph persist",
"Git persist",
"Commit capture",
+ "Decision unretire",
+ "Retired-source decision purge",
+ "Decision source-rank repair",
+ "Decision confidence repair",
+ "Decision id migration",
+ "Decision entity migration",
"Decision persist",
"Revert supersession",
"Decision HEAD artifact check",
@@ -285,6 +305,67 @@ async def test_every_failing_step_degrades_in_order(repo, monkeypatch):
]
+async def test_range_scoped_failures_are_recorded_for_repair(repo, monkeypatch):
+ rec = StepRecorder(monkeypatch, fail=frozenset(HAPPY_ORDER) - _SURVIVORS)
+ _install(rec)
+ failed_steps: list[str] = []
+
+ await _persist_full(repo, rec, _Inputs(), [], failed_steps)
+
+ assert failed_steps == [
+ "Tombstone marking",
+ "Git persist",
+ "Health persist",
+ "Dead-code persist",
+ "Symbol persist",
+ "Security persist",
+ "Graph edges persist",
+ "External systems refresh",
+ "Full-text search indexing",
+ ]
+
+
+async def test_a_failing_decision_repair_skips_only_itself(repo, monkeypatch):
+ rec = StepRecorder(monkeypatch, fail=frozenset({"unretire_auto_superseded"}))
+ _install(rec)
+ degraded: list[str] = []
+
+ await _persist_full(repo, rec, _Inputs(), degraded)
+
+ rec.assert_all_called(HAPPY_ORDER)
+ assert rec.order == HAPPY_ORDER
+ assert degraded == [f"Decision unretire: {FAILURE}"]
+
+
+async def test_a_failing_repair_fails_a_required_decision_persist(repo, monkeypatch):
+ rec = StepRecorder(monkeypatch, fail=frozenset({"unretire_auto_superseded"}))
+ _install(rec)
+ degraded: list[str] = []
+ inputs = _Inputs()
+
+ with pytest.raises(RuntimeError, match=FAILURE):
+ await _persist_full_update_async(
+ repo_path=repo,
+ repo_name=repo.name,
+ generated_pages=inputs.pages,
+ file_diffs=inputs.file_diffs,
+ git_meta_map=dict(GIT_META),
+ new_decision_markers=[],
+ decision_vector_store=rec.vector_store(),
+ provider=None,
+ partial_health_report=None,
+ dead_code_report=None,
+ graph_builder=inputs.graph_builder,
+ knowledge_graph_result=None,
+ degraded=degraded,
+ parsed_files=inputs.parsed_files,
+ require_decision_persist_success=True,
+ )
+
+ assert degraded == [f"Decision unretire: {FAILURE}"]
+ assert "purge_proposed_decisions_by_source" not in rec.order
+
+
# ---------------------------------------------------------------------------
# Parity: the shared middle steps get the same inputs from both persisters
# ---------------------------------------------------------------------------
diff --git a/tests/unit/cli/test_update_working_tree.py b/tests/unit/cli/test_update_working_tree.py
index 7b0989ace..374ebc1f5 100644
--- a/tests/unit/cli/test_update_working_tree.py
+++ b/tests/unit/cli/test_update_working_tree.py
@@ -215,7 +215,7 @@ def test_working_tree_only_update_skips_history_phases_until_the_commit(
analysis = _spy(monkeypatch, update_command_mod, "_run_partial_analysis")
drift = _spy(monkeypatch, update_command_mod, "_run_doc_drift_partial")
- # The CLI path imports it from the package; the core writer holds its own reference.
+ # Both lookup sites: the package and the core writer's own reference.
commits = _spy(monkeypatch, core_incremental, "persist_incremental_commits")
commits_core = _spy(monkeypatch, persist_index, "persist_incremental_commits")
diff --git a/tests/unit/persistence/test_page_tree_wiring.py b/tests/unit/persistence/test_page_tree_wiring.py
index 623e429f6..6b653729f 100644
--- a/tests/unit/persistence/test_page_tree_wiring.py
+++ b/tests/unit/persistence/test_page_tree_wiring.py
@@ -25,7 +25,6 @@ _PERSIST_PATHS = [
("repowise.core.pipeline.persist", "the full index"),
("repowise.core.pipeline.incremental.persist_index", "the incremental index"),
("repowise.core.pipeline.scoped_generation", "a scoped regeneration"),
- ("repowise.cli.commands.update_cmd.persistence", "the docs update"),
("repowise.cli.commands.upgrade_flow", "the fast-to-full upgrade"),
]
@@ -58,9 +57,17 @@ def test_persist_path_rebuilds_the_tree(module_name: str, description: str):
)
+def test_the_docs_update_runs_the_shared_tree_rebuild():
+ """The docs update reuses the incremental index's rebuild step."""
+ from repowise.cli.commands.update_cmd.full_persist import FULL_UPDATE_STEPS
+ from repowise.core.pipeline.incremental.persist_index import rebuild_tree
+
+ assert rebuild_tree in [step.run for step in FULL_UPDATE_STEPS]
+
+
def test_the_writer_list_is_not_silently_empty():
"""Guards the parametrisation itself."""
- assert len(_PERSIST_PATHS) + len(_DEFERRED_PATHS) >= 6
+ assert len(_PERSIST_PATHS) + len(_DEFERRED_PATHS) >= 5
@pytest.mark.parametrize("module_name,description,caller_name", _DEFERRED_PATHS)
diff --git a/tests/unit/pipeline/test_graph_nodes_before_health.py b/tests/unit/pipeline/test_graph_nodes_before_health.py
index 6d121bdeb..75e2f7206 100644
--- a/tests/unit/pipeline/test_graph_nodes_before_health.py
+++ b/tests/unit/pipeline/test_graph_nodes_before_health.py
@@ -11,6 +11,7 @@ import inspect
import pytest
+from repowise.cli.commands.update_cmd import full_persist
from repowise.cli.commands.update_cmd import persistence as cli_update
from repowise.core.pipeline.incremental import persist_index
@@ -18,7 +19,6 @@ from repowise.core.pipeline.incremental import persist_index
@pytest.mark.parametrize(
("function", "health_call"),
[
- (cli_update._persist_full_update_async, "await _persist_partial_health("),
(cli_update._rescore_health_from_db, "await save_full_health_report("),
],
)
@@ -28,9 +28,12 @@ def test_graph_nodes_persist_before_health(function, health_call: str) -> None:
assert source.index("await persist_graph_nodes(") < source.index(health_call)
-def test_incremental_steps_persist_graph_nodes_before_health() -> None:
- names = [step.name for step in persist_index.INCREMENTAL_STEPS]
+ "steps", [persist_index.INCREMENTAL_STEPS, full_persist.FULL_UPDATE_STEPS]
+)
+def test_update_steps_persist_graph_nodes_before_health(steps) -> None:
+ names = [step.name for step in steps]
assert names.index("Graph nodes persist") < names.index("Health persist")
- by_name = {step.name: step.run for step in persist_index.INCREMENTAL_STEPS}
+ by_name = {step.name: step.run for step in steps}
assert "await persist_graph_nodes(" in inspect.getsource(by_name["Graph nodes persist"])
assert "await persist_partial_health(" in inspect.getsource(by_name["Health persist"])
diff --git a/tests/unit/pipeline/test_persist_decision_helpers.py b/tests/unit/pipeline/test_persist_decision_helpers.py
index 2e2ce0c8a..2292698cd 100644
--- a/tests/unit/pipeline/test_persist_decision_helpers.py
+++ b/tests/unit/pipeline/test_persist_decision_helpers.py
@@ -122,3 +122,21 @@ async def test_failing_passes_after_the_upsert_are_swallowed(monkeypatch):
"detect_supersessions_and_conflicts",
"recompute_decision_staleness",
]
+
+
+async def test_repair_failures_go_to_skip_one_repair_at_a_time(monkeypatch):
+ rec = StepRecorder(
+ monkeypatch, fail=frozenset({"unretire_auto_superseded", "apply_migration"})
+ )
+ install_decision_chain(rec)
+ skipped: list[tuple[str, str]] = []
+
+ await repair_decision_store(
+ object(), "repo-1", skip=lambda step, exc: skipped.append((step, str(exc)))
+ )
+
+ assert skipped == [
+ ("Decision unretire", FAILURE),
+ ("Decision entity migration", FAILURE),
+ ]
+ assert "reconcile_source_ranks" in rec.order