Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(indexing): make stale page recovery converge (#2254)

Raghav Chamadiyad6737d2864a6
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
index a2cc51c57..8e2ea4fdb 100644
--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
@@ -81,6 +81,54 @@ async def _page_count(session: object, repo_id: str) -> int:
return int(result.scalar_one())
+async def _stale_page_counts(session: object, repo_id: str) -> dict[str, int]:
+ """Count stale pages by type without hydrating their rendered content."""
+ from sqlalchemy import func, select
+
+ from repowise.core.persistence.models import Page
+
+ result = await session.execute( # type: ignore[attr-defined]
+ select(Page.page_type, func.count())
+ .where(
+ Page.repository_id == repo_id,
+ Page.freshness_status.in_(["stale", "expired"]),
+ )
+ .group_by(Page.page_type)
+ )
+ return {str(page_type): int(count) for page_type, count in result.all()}
+
+
+def _stale_page_guidance(counts: dict[str, int]) -> str:
+ """Describe stale work by owner and name commands that can clear it."""
+ from repowise.core.generation.models import MODEL_WRITTEN_PAGE_TYPES
+
+ total = sum(counts.values())
+ model_written = sum(
+ count for page_type, count in counts.items() if page_type in MODEL_WRITTEN_PAGE_TYPES
+ )
+ structural = total - model_written
+ categories = []
+ if model_written:
+ categories.append(f"{model_written} model-written")
+ if structural:
+ categories.append(f"{structural} structural")
+
+ detail = (
+ f"{total} stale ({', '.join(categories)}) — content no longer matches its "
+ "current inputs or generation selection."
+ )
+ if model_written:
+ detail += (
+ " `repowise generate --stale` is the cheaper refresh for model-written "
+ "pages that are still selected."
+ )
+ detail += (
+ " `repowise update --full` performs the authoritative reconciliation and "
+ "retires pages no longer selected."
+ )
+ return detail + " `--repair` only fixes store drift."
+
+
async def _all_pages_for_reconciliation(session: object, repo_id: str) -> list:
"""Every page this repository has, for reconciling against the indexes.
@@ -327,6 +375,7 @@ def _run_repo_checks(
# 7. Stale page count
stale_count = 0
+ stale_counts: dict[str, int] = {}
if db_ok and page_count > 0:
try:
@@ -336,7 +385,6 @@ def _run_repo_checks(
create_session_factory,
get_repository_by_path,
get_session,
- get_stale_pages,
)
url = get_db_url_for_repo(repo_path)
@@ -346,22 +394,20 @@ def _run_repo_checks(
async with get_session(sf) as session:
repo = await get_repository_by_path(session, str(repo_path))
if repo:
- stale = await get_stale_pages(session, repo.id)
+ counts = await _stale_page_counts(session, repo.id)
await engine.dispose()
- return len(stale)
+ return counts
await engine.dispose()
- return 0
+ return {}
- stale_count = run_async(_check_stale())
+ stale_counts = run_async(_check_stale())
+ stale_count = sum(stale_counts.values())
if stale_count:
checks.append(
_check(
"Stale pages",
False,
- f"{stale_count} stale — pages whose content lags the code "
- "(change cascade exceeded the regeneration budget). "
- "`repowise update --full` regenerates them; "
- "`--repair` cannot, it only fixes store drift.",
+ _stale_page_guidance(stale_counts),
)
)
else:
@@ -863,17 +909,12 @@ def _run_repo_checks(
repaired_count = run_async(_repair())
console.print(f"[bold green]Repaired {repaired_count} entries.[/bold green]")
if stale_count:
- console.print(
- "[yellow]Stale pages are not store drift, so --repair leaves them "
- "alone: they are pages the last docs run could not regenerate "
- "within its budget. Run `repowise update --full` to clear them.[/yellow]"
- )
+ console.print(f"[yellow]{_stale_page_guidance(stale_counts)}[/yellow]")
elif repair and not has_mismatches and not registration_wedged and not agents_need_refresh:
if stale_count:
console.print(
- f"[yellow]No store drift to repair, but {stale_count} stale page(s) "
- "remain — they are content lag, not drift. `repowise update --full` "
- "regenerates them.[/yellow]"
+ f"[yellow]No store drift to repair. "
+ f"{_stale_page_guidance(stale_counts)}[/yellow]"
)
else:
console.print("[green]Nothing to repair.[/green]")
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
index 62bb4b0b0..0df934753 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
@@ -951,12 +951,13 @@ def run_update(
# forever, and the store is asked only once the git checks would otherwise
# have exited, so the common case stays free of a store read.
stale_db_paths: list[str] = []
+ stale_deterministic_ids: set[str] = set()
if git_is_current:
- from repowise.core.persistence import load_stale_structural_file_paths
+ from repowise.core.persistence import load_stale_update_targets
- stale_db_paths = load_stale_structural_file_paths(repo_path)
+ stale_db_paths, stale_deterministic_ids = load_stale_update_targets(repo_path)
- if git_is_current and not stale_db_paths:
+ if git_is_current and not stale_db_paths and not stale_deterministic_ids:
console.print("[green]Already up to date.[/green]")
# D7: on a template (index-only) wiki, "up to date" is true of the code
# but the pages are still unwritten. Point at the command that writes
@@ -1179,6 +1180,15 @@ def run_update(
working_tree_diffs,
)
+ # The up-to-date gate loaded these already. A content-changing run still
+ # has to carry old structural debt forward: otherwise a fresh commit can
+ # repeatedly update file pages while an SCC stranded by an earlier
+ # budget-capped run remains stale forever.
+ if not git_is_current:
+ from repowise.core.persistence import load_stale_update_targets
+
+ stale_db_paths, stale_deterministic_ids = load_stale_update_targets(repo_path)
+
# A stale page whose file is gone is a deletion the diff walk never saw
# (the file left in a range an earlier run did not cover, or was never
# committed). Re-rendering cannot clear it, so hand it to the same
@@ -1203,7 +1213,13 @@ def run_update(
if absent:
console.print(f"Stale pages for deleted files: [cyan]{len(absent)}[/cyan]")
- if not file_diffs and not config_changed and not renderer_changed and not stale_db_paths:
+ if (
+ not file_diffs
+ and not config_changed
+ and not renderer_changed
+ and not stale_db_paths
+ and not stale_deterministic_ids
+ ):
console.print("[green]No changed files detected.[/green]")
# Always advance the sync pointer so the on-disk freshness marker stays
# current on no-op syncs. In docs mode, no changed files means no docs
@@ -1409,6 +1425,10 @@ def run_update(
# until it is retired.
if stale_db_paths:
console.print(f"Reconciling stale structural pages: [cyan]{len(stale_db_paths)}[/cyan]")
+ if stale_deterministic_ids:
+ console.print(
+ f"Reconciling stale repository structure: [cyan]{len(stale_deterministic_ids)}[/cyan]"
+ )
stale_extra = list(dict.fromkeys([*stale_renderer_paths, *stale_db_paths]))
if stale_extra:
affected.regenerate = list(dict.fromkeys([*affected.regenerate, *stale_extra]))
@@ -1515,6 +1535,7 @@ def run_update(
from .deterministic import (
load_prior_page_ids,
persist_deterministic_pages,
+ regenerate_deterministic_page_ids,
regenerate_deterministic_pages,
)
@@ -1544,6 +1565,23 @@ def run_update(
prior_page_ids=prior_ids,
full_scope=generation_config_changed,
)
+ if stale_deterministic_ids and not generation_config_changed:
+ det_pages.extend(
+ regenerate_deterministic_page_ids(
+ repo_path=repo_path,
+ parsed_files=parsed_files,
+ source_map=source_map,
+ graph_builder=graph_builder,
+ repo_structure=repo_structure,
+ git_meta_map=git_meta_map,
+ page_ids=stale_deterministic_ids,
+ cfg=cfg,
+ concurrency=concurrency,
+ degraded=degraded,
+ dead_code_report=dead_code_report,
+ prior_page_ids=prior_ids,
+ )
+ )
if generation_config_changed and len(degraded) > degraded_before_render:
raise RuntimeError(
"Configuration-driven page generation failed; the previous "
@@ -2165,6 +2203,33 @@ def run_update(
if checkpointer.failure:
degraded.append(f"Per-page crash checkpointing: {checkpointer.failure}")
+ # SCC pages are deterministic but describe the complete graph, so the
+ # changed-file generator above cannot safely refresh them. Render only the
+ # stale structural ids from the complete repository view with the template
+ # provider. This performs no model calls and reuses the update's vector
+ # store so semantic search is refreshed with the page.
+ if stale_deterministic_ids and not generation_config_changed:
+ from .deterministic import regenerate_deterministic_page_ids
+
+ with timed(timings, "render.structure"):
+ generated_pages.extend(
+ regenerate_deterministic_page_ids(
+ repo_path=repo_path,
+ parsed_files=parsed_files,
+ source_map=source_map,
+ graph_builder=graph_builder,
+ repo_structure=repo_structure,
+ git_meta_map=git_meta_map,
+ page_ids=stale_deterministic_ids,
+ cfg=cfg,
+ concurrency=concurrency,
+ degraded=degraded,
+ dead_code_report=dead_code_report,
+ prior_page_ids=prior_pages,
+ vector_store=decision_vector_store,
+ )
+ )
+
# Flush the buffered LLM cost rows now that generation is done — a single
# transaction outside the contended generation window (issue #326).
from repowise.cli.providers import flush_cost_tracker
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
index 0c384558f..7878c25d5 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
@@ -122,6 +122,51 @@ def regenerate_deterministic_pages(
)
+def regenerate_deterministic_page_ids(
+ *,
+ repo_path: Path,
+ parsed_files: list,
+ source_map: dict,
+ graph_builder: Any,
+ repo_structure: Any,
+ git_meta_map: dict,
+ page_ids: set[str],
+ cfg: dict,
+ concurrency: int,
+ degraded: list[str],
+ dead_code_report: Any = None,
+ prior_page_ids: dict | None = None,
+ vector_store: Any = None,
+) -> list:
+ """Render exact deterministic page ids from the complete repository view.
+
+ Used for whole-repository structural pages such as SCCs that cannot be
+ rebuilt from the changed-file slice. The TemplateProvider guarantees this
+ recovery path makes no LLM calls; ``only_page_ids`` keeps it from rendering
+ unrelated module/overview/onboarding pages while the level ladder reaches
+ the requested structural level.
+ """
+ if not page_ids:
+ return []
+ return _render_pages(
+ repo_path=repo_path,
+ parsed_files=parsed_files,
+ source_map=source_map,
+ graph_builder=graph_builder,
+ repo_structure=repo_structure,
+ git_meta_map=git_meta_map,
+ regenerate_paths=[],
+ cfg=cfg,
+ concurrency=concurrency,
+ degraded=degraded,
+ dead_code_report=dead_code_report,
+ prior_page_ids=prior_page_ids,
+ degrade_label="Structural page refresh",
+ only_page_ids=page_ids,
+ vector_store=vector_store,
+ )
+
+
def _render_pages(
*,
repo_path: Path,
@@ -138,6 +183,8 @@ def _render_pages(
prior_page_ids: dict | None,
degrade_label: str,
full_scope: bool = False,
+ only_page_ids: set[str] | None = None,
+ vector_store: Any = None,
) -> list:
"""Render the changed files' pages from structure (free, no LLM).
@@ -148,14 +195,15 @@ def _render_pages(
from repowise.core.providers.llm.template import TemplateProvider
regen_set = set(regenerate_paths)
+ complete_context = full_scope or only_page_ids is not None
affected_parsed = (
list(parsed_files)
- if full_scope
+ if complete_context
else [pf for pf in parsed_files if pf.file_info.path in regen_set]
)
affected_source = (
dict(source_map)
- if full_scope
+ if complete_context
else {p: s for p, s in source_map.items() if p in regen_set}
)
if not affected_parsed:
@@ -165,7 +213,7 @@ def _render_pages(
config = GenerationConfig.from_repo_config(
cfg,
deterministic=True,
- file_pages_only=not full_scope,
+ file_pages_only=not complete_context,
max_concurrency=concurrency,
language=cfg.get("language", "en"),
enable_onboarding=bool(cfg.get("enable_onboarding", True)),
@@ -179,13 +227,13 @@ def _render_pages(
# leaves that store untouched; full-text search is unaffected either
# way. It also keeps the lancedb import off the post-commit hook's
# path, which is the one place in this command that avoids it.
- vector_store = None
+ resolved_vector_store = vector_store
embedder_name = deterministic_embedder_name(cfg)
- if embedder_name != "mock":
+ if resolved_vector_store is None and embedder_name != "mock":
from repowise.cli.providers import build_embedder, build_vector_store
try:
- vector_store = build_vector_store(
+ resolved_vector_store = build_vector_store(
repo_path, build_embedder(embedder_name, repo_path)
)
except Exception as exc: # embedding is optional; FTS still indexes
@@ -195,7 +243,7 @@ def _render_pages(
TemplateProvider(),
ContextAssembler(config, repo_path=repo_path),
config,
- vector_store=vector_store,
+ vector_store=resolved_vector_store,
language=config.language,
# Every persisted page id, so interlinking and related-pages can
# resolve references to pages outside this run's slice. Not a reuse
@@ -217,7 +265,7 @@ def _render_pages(
git_meta_map=git_meta_map,
repo_path=repo_path,
dead_code_report=dead_code_report,
- only_page_ids=None,
+ only_page_ids=only_page_ids,
)
)
except Exception as exc:
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 06d34af6e..da61e54c9 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -911,7 +911,15 @@ async def _persist_full_update_async(
# free; the operator opts into the spend explicitly via
# `generate --stale`.
cascade = expand_cascade(seed_ids, "none", deps)
- await mark_page_ids_stale(session, repo_id, cascade.stale_ids | seed_ids)
+ # A no-model structural recovery can have refreshed one
+ # of these dependents in this same run. The page upsert
+ # above made it fresh; never decay it again afterwards.
+ generated_ids = {page.page_id for page in generated_pages}
+ await mark_page_ids_stale(
+ session,
+ repo_id,
+ (cascade.stale_ids | seed_ids) - generated_ids,
+ )
except Exception as exc:
_skip("Stale-page decay", exc)
diff --git a/packages/cli/src/repowise/cli/commands/upgrade_flow.py b/packages/cli/src/repowise/cli/commands/upgrade_flow.py
index 80c7778fb..66c5529ea 100644
--- a/packages/cli/src/repowise/cli/commands/upgrade_flow.py
+++ b/packages/cli/src/repowise/cli/commands/upgrade_flow.py
@@ -183,6 +183,64 @@ async def _backfill_git(
)
+async def _persist_authoritative_pages(
+ *,
+ session: Any,
+ repository_id: str,
+ generated_pages: list[Any],
+ vector_store: Any,
+) -> list[str]:
+ """Persist a full selection and retire every page outside it."""
+ from repowise.core.persistence import upsert_pages_from_generated
+ from repowise.core.pipeline.persist import tombstone_pages_outside_generation
+
+ await upsert_pages_from_generated(session, generated_pages, repository_id)
+ retired_page_ids = await tombstone_pages_outside_generation(
+ session,
+ repository_id,
+ {page.page_id for page in generated_pages},
+ )
+ if retired_page_ids and vector_store is not None:
+ await vector_store.delete_many(retired_page_ids)
+ return retired_page_ids
+
+
+async def _sync_authoritative_fts(
+ *,
+ engine: Any,
+ repo_path: Path,
+ generated_pages: list[Any],
+ retired_page_ids: list[str],
+) -> None:
+ """Replace full-run FTS content and durably retry retired-id cleanup."""
+ from repowise.core.persistence.search import FullTextSearch
+ from repowise.core.pipeline.cleanup_debt import (
+ clear_cleanup_debt,
+ load_cleanup_debt,
+ record_cleanup_debt,
+ )
+
+ cleanup_ids = set(retired_page_ids) | load_cleanup_debt(repo_path)["fts"]
+ try:
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ if cleanup_ids:
+ await fts.delete_many(sorted(cleanup_ids))
+ for page in generated_pages:
+ await fts.index(
+ page.page_id,
+ page.title,
+ page.content,
+ summary=page.summary,
+ target_path=page.target_path,
+ )
+ except BaseException:
+ record_cleanup_debt(repo_path, "fts", cleanup_ids)
+ raise
+ else:
+ clear_cleanup_debt(repo_path, "fts", cleanup_ids)
+
+
async def _run_upgrade(
repo_path: Path,
provider: Any,
@@ -199,12 +257,10 @@ async def _run_upgrade(
from repowise.cli.helpers import get_db_url_for_repo
from repowise.core.generation.cost_tracker import CostTracker
from repowise.core.persistence import (
- FullTextSearch,
create_engine,
create_session_factory,
get_session,
init_db,
- upsert_pages_from_generated,
upsert_repository,
)
from repowise.core.pipeline import rehydrate_graph_builder
@@ -345,7 +401,18 @@ async def _run_upgrade(
# 6. Persist pages + a GenerationJob marker, then build the FTS index.
async with get_session(sf) as session:
- await upsert_pages_from_generated(session, generated_pages, repo_id)
+ # A successful full generation is authoritative for every generated
+ # page type. Retire rows that are absent from its selection even when
+ # their backing file still exists (for example, a once-nontrivial file
+ # that no longer qualifies for a file page). Without this sweep,
+ # `update --full` can report success while doctor keeps reporting those
+ # old rows stale forever.
+ retired_page_ids = await _persist_authoritative_pages(
+ session=session,
+ repository_id=repo_id,
+ generated_pages=generated_pages,
+ vector_store=vector_store,
+ )
try:
from repowise.core.pipeline.page_tree_sync import rebuild_page_tree
@@ -377,16 +444,12 @@ async def _run_upgrade(
pass # job recording is best-effort
try:
- fts = FullTextSearch(engine)
- await fts.ensure_index()
- for page in generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ await _sync_authoritative_fts(
+ engine=engine,
+ repo_path=repo_path,
+ generated_pages=generated_pages,
+ retired_page_ids=retired_page_ids,
+ )
except BaseException:
await engine.dispose()
raise
diff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.py
index 286fe8fb8..6640f1a15 100644
--- a/packages/core/src/repowise/core/persistence/__init__.py
+++ b/packages/core/src/repowise/core/persistence/__init__.py
@@ -63,6 +63,7 @@ from .crud import (
get_stale_file_page_ages,
get_stale_pages,
get_stale_structural_file_paths,
+ get_stale_update_targets,
get_top_entry_points,
link_graph_nodes_to_external_systems,
list_chat_messages,
@@ -74,6 +75,7 @@ from .crud import (
list_pages,
load_prior_pages,
load_stale_structural_file_paths,
+ load_stale_update_targets,
mark_webhook_processed,
recompute_decision_staleness,
save_dead_code_findings,
@@ -261,6 +263,7 @@ __all__ = [
"get_stale_file_page_ages",
"get_stale_pages",
"get_stale_structural_file_paths",
+ "get_stale_update_targets",
"get_top_entry_points",
"init_db",
"link_graph_nodes_to_external_systems",
@@ -274,6 +277,7 @@ __all__ = [
"list_pages",
"load_prior_pages",
"load_stale_structural_file_paths",
+ "load_stale_update_targets",
"mark_webhook_processed",
"recompute_decision_staleness",
"resolve_db_url",
diff --git a/packages/core/src/repowise/core/persistence/crud/pages.py b/packages/core/src/repowise/core/persistence/crud/pages.py
index cb580d6b0..40fb79244 100644
--- a/packages/core/src/repowise/core/persistence/crud/pages.py
+++ b/packages/core/src/repowise/core/persistence/crud/pages.py
@@ -672,38 +672,66 @@ async def get_stale_pages(
#: Page types a scoped ``update`` can re-render for one file. Every other
-#: structural type (cycle, layer, contract, infra) describes the whole
-#: repository and is only written by a full run, so a stale row of those types
-#: is not something an update can clear and must not make it think it can.
+#: Page types the changed-file renderer can refresh from one file path. Whole-
+#: repository deterministic targets are handled separately below by exact id.
_FILE_SCOPED_PAGE_TYPES = frozenset({"file_page", "symbol_spotlight"})
+# Deterministic pages that describe the complete repository rather than one
+# file. A scoped file render cannot refresh these, but update already rebuilds
+# the complete graph and can render an exact id from that view without a model.
+# Keep this deliberately narrow: module/overview/onboarding pages are
+# model-written and belong to ``repowise generate --stale``; layer pages are
+# retired and swept independently.
+_UPDATE_WIDE_DETERMINISTIC_PAGE_TYPES = frozenset({"scc_page"})
-async def get_stale_structural_file_paths(
+
+async def get_stale_update_targets(
session: AsyncSession,
repository_id: str,
-) -> list[str]:
- """File paths whose file-scoped pages are marked ``stale`` or ``expired``.
+) -> tuple[list[str], set[str]]:
+ """Return the stale pages an ordinary update can heal without a model.
- Covers ``file_page`` rows (``target_path`` is the file) and
- ``symbol_spotlight`` rows (``target_path`` is ``<file>::<symbol>``), which
- are the two page kinds a scoped ``update`` re-renders for a file. The
- caller feeds these paths into the same regeneration list the renderer
- staleness path uses, so an already-stale page is reconciled even when HEAD
- has not moved.
+ The first item contains file paths for the existing file-scoped renderer.
+ The second contains exact ids for deterministic whole-repository pages,
+ which must be rendered from the complete parsed/graph view. Returning both
+ from one query keeps the up-to-date fast path to a single store read.
"""
result = await session.execute(
- select(Page.target_path).where(
+ select(Page.id, Page.page_type, Page.target_path).where(
Page.repository_id == repository_id,
- Page.page_type.in_(sorted(_FILE_SCOPED_PAGE_TYPES)),
+ Page.page_type.in_(
+ sorted(_FILE_SCOPED_PAGE_TYPES | _UPDATE_WIDE_DETERMINISTIC_PAGE_TYPES)
+ ),
Page.freshness_status.in_(["stale", "expired"]),
)
)
stale_paths: list[str] = []
- for (target_path,) in result:
+ deterministic_ids: set[str] = set()
+ for page_id, page_type, target_path in result:
+ if page_type in _UPDATE_WIDE_DETERMINISTIC_PAGE_TYPES:
+ deterministic_ids.add(page_id)
+ continue
file_path = (target_path or "").split("::", 1)[0]
if file_path:
stale_paths.append(file_path)
- return list(dict.fromkeys(stale_paths))
+ return list(dict.fromkeys(stale_paths)), deterministic_ids
+
+
+async def get_stale_structural_file_paths(
+ session: AsyncSession,
+ repository_id: str,
+) -> list[str]:
+ """File paths whose file-scoped pages are marked ``stale`` or ``expired``.
+
+ Covers ``file_page`` rows (``target_path`` is the file) and
+ ``symbol_spotlight`` rows (``target_path`` is ``<file>::<symbol>``), which
+ are the two page kinds a scoped ``update`` re-renders for a file. The
+ caller feeds these paths into the same regeneration list the renderer
+ staleness path uses, so an already-stale page is reconciled even when HEAD
+ has not moved.
+ """
+ stale_paths, _ = await get_stale_update_targets(session, repository_id)
+ return stale_paths
def load_stale_structural_file_paths(repo_path: Any) -> list[str]:
@@ -731,6 +759,26 @@ def load_stale_structural_file_paths(repo_path: Any) -> list[str]:
return asyncio.run(_load_stale_structural_file_paths_async(path_obj))
+def load_stale_update_targets(repo_path: Any) -> tuple[list[str], set[str]]:
+ """Sync entry point returning both file paths and whole-repo page ids."""
+ import asyncio
+ import concurrent.futures
+ from pathlib import Path
+
+ path_obj = Path(repo_path)
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = None
+
+ if loop and loop.is_running():
+ with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+ return pool.submit(
+ lambda: asyncio.run(_load_stale_update_targets_async(path_obj))
+ ).result()
+ return asyncio.run(_load_stale_update_targets_async(path_obj))
+
+
async def _load_stale_structural_file_paths_async(repo_path: Any) -> list[str]:
"""Load the stale file-scoped page paths for *repo_path* from its store.
@@ -775,6 +823,42 @@ async def _load_stale_structural_file_paths_async(repo_path: Any) -> list[str]:
await engine.dispose()
+async def _load_stale_update_targets_async(repo_path: Any) -> tuple[list[str], set[str]]:
+ """Load all no-model stale targets for an ordinary update."""
+ from pathlib import Path
+
+ import structlog
+
+ from ..database import (
+ create_engine,
+ create_session_factory,
+ get_configured_db_url,
+ get_repo_db_path,
+ get_session,
+ resolve_db_url,
+ )
+ from .repository import get_repository_by_path
+
+ logger = structlog.get_logger(__name__)
+ path_obj = Path(repo_path)
+
+ if get_configured_db_url() is None and not get_repo_db_path(path_obj).exists():
+ return [], set()
+
+ engine = create_engine(resolve_db_url(path_obj))
+ try:
+ async with get_session(create_session_factory(engine)) as session:
+ repo = await get_repository_by_path(session, str(path_obj))
+ if repo is None:
+ return [], set()
+ return await get_stale_update_targets(session, repo.id)
+ except Exception as exc:
+ logger.warning("load_stale_update_targets_failed", error=str(exc))
+ raise
+ finally:
+ await engine.dispose()
+
+
async def get_stale_file_page_ages(
session: AsyncSession,
repository_id: str,
diff --git a/tests/unit/cli/test_deterministic_update.py b/tests/unit/cli/test_deterministic_update.py
index f75b4f9d7..810611bdf 100644
--- a/tests/unit/cli/test_deterministic_update.py
+++ b/tests/unit/cli/test_deterministic_update.py
@@ -11,7 +11,11 @@ from __future__ import annotations
import pytest
-from repowise.cli.commands.update_cmd.deterministic import deterministic_embedder_name
+from repowise.cli.commands.update_cmd import deterministic
+from repowise.cli.commands.update_cmd.deterministic import (
+ deterministic_embedder_name,
+ regenerate_deterministic_page_ids,
+)
@pytest.fixture(autouse=True)
@@ -52,3 +56,57 @@ class TestDeterministicEmbedderName:
def test_nothing_configured_is_mock(self) -> None:
assert deterministic_embedder_name({}) == "mock"
+
+
+def test_exact_structural_refresh_uses_full_context_without_model(
+ tmp_path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ captured: dict[str, object] = {}
+
+ class _Config:
+ language = "en"
+
+ @classmethod
+ def from_repo_config(cls, _cfg, **kwargs):
+ captured["config"] = kwargs
+ return cls()
+
+ class _Generator:
+ def __init__(self, provider, _assembler, _config, **_kwargs):
+ captured["provider"] = type(provider).__name__
+
+ async def generate_all(self, parsed, source, *_args, only_page_ids=None, **_kwargs):
+ captured["parsed"] = parsed
+ captured["source"] = source
+ captured["only_page_ids"] = only_page_ids
+ return ["scc"]
+
+ import repowise.core.generation as generation
+
+ monkeypatch.setattr(generation, "GenerationConfig", _Config)
+ monkeypatch.setattr(generation, "ContextAssembler", lambda *_a, **_k: object())
+ monkeypatch.setattr(generation, "PageGenerator", _Generator)
+ monkeypatch.setattr(deterministic, "deterministic_embedder_name", lambda _cfg: "mock")
+
+ parsed = [object(), object()]
+ source = {"a.py": b"a", "b.py": b"b"}
+ result = regenerate_deterministic_page_ids(
+ repo_path=tmp_path,
+ parsed_files=parsed,
+ source_map=source,
+ graph_builder=object(),
+ repo_structure=object(),
+ git_meta_map={},
+ page_ids={"scc_page:scc-current"},
+ cfg={},
+ concurrency=2,
+ degraded=[],
+ )
+
+ assert result == ["scc"]
+ assert captured["provider"] == "TemplateProvider"
+ assert captured["config"]["deterministic"] is True
+ assert captured["config"]["file_pages_only"] is False
+ assert captured["parsed"] == parsed
+ assert captured["source"] == source
+ assert captured["only_page_ids"] == {"scc_page:scc-current"}
diff --git a/tests/unit/cli/test_full_upgrade_progress.py b/tests/unit/cli/test_full_upgrade_progress.py
index b65db8675..77e65bc0b 100644
--- a/tests/unit/cli/test_full_upgrade_progress.py
+++ b/tests/unit/cli/test_full_upgrade_progress.py
@@ -15,9 +15,14 @@ buffered ``progress=None`` path.
from __future__ import annotations
+import asyncio
import inspect
+from types import SimpleNamespace
+
+import pytest
from repowise.cli.commands import upgrade_flow
+from repowise.core.pipeline.cleanup_debt import load_cleanup_debt
def _upgrade_source() -> str:
@@ -49,3 +54,76 @@ def test_full_upgrade_estimate_names_structural_pages() -> None:
# The estimate line must call the summary that says most pages never reach
# a model — otherwise "3661 pages" reads as 3661 model calls.
assert "structural_page_summary" in src
+
+
+def test_full_upgrade_reconciles_and_cleans_retired_pages() -> None:
+ src = _upgrade_source()
+ persistence_src = inspect.getsource(upgrade_flow._persist_authoritative_pages)
+
+ assert "_persist_authoritative_pages" in src
+ assert "tombstone_pages_outside_generation" in persistence_src
+ assert "vector_store.delete_many(retired_page_ids)" in persistence_src
+ assert "_sync_authoritative_fts" in src
+
+
+def test_full_upgrade_fts_cleanup_retries_after_failure(
+ tmp_path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ attempts = 0
+ deleted: list[list[str]] = []
+ indexed: list[str] = []
+
+ class _FakeFts:
+ def __init__(self, _engine) -> None:
+ pass
+
+ async def ensure_index(self) -> None:
+ pass
+
+ async def delete_many(self, page_ids: list[str]) -> None:
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ raise RuntimeError("temporary FTS failure")
+ deleted.append(page_ids)
+
+ async def index(self, page_id: str, *_args, **_kwargs) -> None:
+ indexed.append(page_id)
+
+ import repowise.core.persistence.search as search
+
+ monkeypatch.setattr(search, "FullTextSearch", _FakeFts)
+ page = SimpleNamespace(
+ page_id="file_page:current.py",
+ title="Current",
+ content="# Current",
+ summary="Current page",
+ target_path="current.py",
+ )
+
+ with pytest.raises(RuntimeError, match="temporary FTS failure"):
+ asyncio.run(
+ upgrade_flow._sync_authoritative_fts(
+ engine=object(),
+ repo_path=tmp_path,
+ generated_pages=[page],
+ retired_page_ids=["file_page:obsolete.py"],
+ )
+ )
+
+ assert load_cleanup_debt(tmp_path)["fts"] == {"file_page:obsolete.py"}
+
+ # SQL already committed the tombstone, so the retry discovers no newly
+ # retired ids. Durable cleanup debt must replay the missing deletion.
+ asyncio.run(
+ upgrade_flow._sync_authoritative_fts(
+ engine=object(),
+ repo_path=tmp_path,
+ generated_pages=[page],
+ retired_page_ids=[],
+ )
+ )
+
+ assert deleted == [["file_page:obsolete.py"]]
+ assert indexed == ["file_page:current.py"]
+ assert load_cleanup_debt(tmp_path)["fts"] == set()
diff --git a/tests/unit/cli/test_update_persist_reliability.py b/tests/unit/cli/test_update_persist_reliability.py
index d847262fc..4421e4934 100644
--- a/tests/unit/cli/test_update_persist_reliability.py
+++ b/tests/unit/cli/test_update_persist_reliability.py
@@ -451,7 +451,10 @@ def test_update_decay_paths_use_cascade_dependents(
_persist_full_update_async(
repo_path=tmp_path,
repo_name="repo",
- generated_pages=[_page("file_page:a.py")],
+ # The SCC was recovered deterministically in this same update. It
+ # appears in the cascade's stale dependents, but the final decay
+ # must not undo the fresh upsert.
+ generated_pages=[_page("file_page:a.py"), _page("scc_page:cycle")],
file_diffs=[],
git_meta_map={},
new_decision_markers=[],
@@ -471,7 +474,6 @@ def test_update_decay_paths_use_cascade_dependents(
assert recorded["expand_cascade"]["mode"] == "none"
assert recorded["marked"] == {
"module_page:pkg",
- "scc_page:cycle",
"repo_overview:repo",
"file_page:src/a.py",
}
diff --git a/tests/unit/cli/test_update_stale_reconciliation.py b/tests/unit/cli/test_update_stale_reconciliation.py
index 25c55a0a6..79e5f7b4c 100644
--- a/tests/unit/cli/test_update_stale_reconciliation.py
+++ b/tests/unit/cli/test_update_stale_reconciliation.py
@@ -8,7 +8,11 @@ from pathlib import Path
import pytest
from repowise.cli.commands.doctor_cmd import repo_checks
-from repowise.core.persistence import load_stale_structural_file_paths
+from repowise.core.persistence import (
+ get_stale_update_targets,
+ load_stale_structural_file_paths,
+ load_stale_update_targets,
+)
from repowise.core.workspace.update import check_repo_staleness
@@ -96,12 +100,8 @@ def test_load_stale_structural_file_paths(tmp_path: Path) -> None:
assert stale == ["bar.py"]
-def test_repo_wide_structural_pages_are_not_reported(tmp_path: Path) -> None:
- """A stale cycle or layer page is only ever rewritten by a full run.
-
- Reporting it would make every idle update pay a full reparse that cannot
- clear it, on every run, forever.
- """
+def test_stale_cycle_is_an_exact_update_target_not_a_file_path(tmp_path: Path) -> None:
+ """A stale SCC wakes update without being mistaken for a source path."""
from repowise.core.persistence import (
create_engine,
create_session_factory,
@@ -112,30 +112,35 @@ def test_repo_wide_structural_pages_are_not_reported(tmp_path: Path) -> None:
repo_path, _ = asyncio.run(_setup_repo_with_pages(tmp_path, []))
- async def _add_stale_layer_page() -> None:
+ async def _add_stale_cycle_page() -> tuple[list[str], set[str]]:
engine = create_engine(f"sqlite+aiosqlite:///{repo_path / '.repowise' / 'wiki.db'}")
sf = create_session_factory(engine)
async with get_session(sf) as session:
repo = await upsert_repository(session, name="test_repo", local_path=str(repo_path))
await upsert_page(
session,
- page_id="layer_page:core",
+ page_id="scc_page:scc-current",
repository_id=repo.id,
- page_type="layer_page",
- title="Layer: core",
- content="# core\n",
+ page_type="scc_page",
+ title="Cycle: current",
+ content="# cycle\n",
summary="Summary",
- target_path="core",
+ target_path="scc-current",
source_hash="hash",
model_name="mock",
provider_name="mock",
freshness_status="stale",
)
await session.commit()
+ targets = await get_stale_update_targets(session, repo.id)
await engine.dispose()
+ return targets
- asyncio.run(_add_stale_layer_page())
+ paths, page_ids = asyncio.run(_add_stale_cycle_page())
+ assert paths == []
+ assert page_ids == {"scc_page:scc-current"}
assert load_stale_structural_file_paths(repo_path) == []
+ assert load_stale_update_targets(repo_path) == ([], {"scc_page:scc-current"})
def test_load_stale_structural_file_paths_spotlight(tmp_path: Path) -> None:
@@ -183,13 +188,46 @@ def test_doctor_detects_stale_pages_and_clears_after_reconciliation(tmp_path: Pa
stale_check = next(c for c in checks if c.name == "Stale pages")
assert stale_check.ok is False
assert stale_check.detail.startswith("2 stale")
+ assert "2 structural" in stale_check.detail
+ assert "repowise update --full" in stale_check.detail
+ assert "repowise generate --stale" not in stale_check.detail
+ assert "cascade exceeded" not in stale_check.detail
# 2. Verify load_stale_structural_file_paths returns the 2 stale paths
stale_paths = load_stale_structural_file_paths(repo_path)
assert sorted(stale_paths) == ["bar.py", "foo.py"]
-def test_stale_structural_paths_scoped_to_repository(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+def test_doctor_stale_guidance_names_each_owner() -> None:
+ detail = repo_checks._stale_page_guidance(
+ {
+ "module_page": 4,
+ "onboarding": 1,
+ "scc_page": 2,
+ "file_page": 3,
+ }
+ )
+
+ assert detail.startswith("10 stale (5 model-written, 5 structural)")
+ assert "repowise generate --stale" in detail
+ assert "repowise update --full" in detail
+ assert "still selected" in detail
+ assert "retires pages no longer selected" in detail
+ assert "current inputs or generation selection" in detail
+ assert "cascade" not in detail
+
+
+def test_doctor_model_only_guidance_keeps_authoritative_fallback() -> None:
+ detail = repo_checks._stale_page_guidance({"module_page": 2})
+
+ assert "repowise generate --stale" in detail
+ assert "repowise update --full" in detail
+ assert "retires pages no longer selected" in detail
+
+
+def test_stale_structural_paths_scoped_to_repository(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
"""load_stale_structural_file_paths filters by repository_id in a shared database."""
import git as gitpython
diff --git a/tests/unit/generation/test_generation_scope_reconciliation.py b/tests/unit/generation/test_generation_scope_reconciliation.py
new file mode 100644
index 000000000..d806fd805
--- /dev/null
+++ b/tests/unit/generation/test_generation_scope_reconciliation.py
@@ -0,0 +1,65 @@
+"""Authoritative full generation retires pages outside its current selection."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from sqlalchemy import select
+
+from repowise.core.persistence import create_engine, create_session_factory, get_session, init_db
+from repowise.core.persistence.crud import upsert_page, upsert_repository
+from repowise.core.persistence.models import Page
+from repowise.core.pipeline.persist import tombstone_pages_outside_generation
+
+
+async def test_full_generation_tombstones_unselected_pages(tmp_path: Path) -> None:
+ engine = create_engine(f"sqlite+aiosqlite:///{tmp_path / 'wiki.db'}")
+ await init_db(engine)
+ session_factory = create_session_factory(engine)
+ async with get_session(session_factory) as session:
+ repo = await upsert_repository(session, name="demo", local_path="/demo")
+ for page_id, page_type, target_path, freshness in (
+ ("file_page:kept.py", "file_page", "kept.py", "fresh"),
+ ("file_page:old.py", "file_page", "old.py", "stale"),
+ ("module_page:old", "module_page", "old", "stale"),
+ ("scc_page:old", "scc_page", "old", "stale"),
+ ):
+ await upsert_page(
+ session,
+ page_id=page_id,
+ repository_id=repo.id,
+ page_type=page_type,
+ title=page_id,
+ content="# page\n",
+ summary="",
+ target_path=target_path,
+ source_hash="hash",
+ model_name="mock",
+ provider_name="mock",
+ freshness_status=freshness,
+ )
+
+ retired = await tombstone_pages_outside_generation(
+ session,
+ repo.id,
+ {"file_page:kept.py"},
+ )
+
+ assert set(retired) == {
+ "file_page:old.py",
+ "module_page:old",
+ "scc_page:old",
+ }
+ async with get_session(session_factory) as session:
+ rows = (
+ await session.execute(select(Page).where(Page.repository_id == repo.id))
+ ).scalars()
+ statuses = {page.id: page.freshness_status for page in rows}
+
+ assert statuses == {
+ "file_page:kept.py": "fresh",
+ "file_page:old.py": "tombstone",
+ "module_page:old": "tombstone",
+ "scc_page:old": "tombstone",
+ }
+ await engine.dispose()