Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(init): resume regenerates a page whose vector landed but whose row did not (#2883)

Raghav Chamadiya18ffa2e522c1
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py
index d6f2c5b26..01a7e81d3 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py
@@ -3,10 +3,10 @@
The orchestrator's :func:`run_generation` buffers every page in memory and the
CLI writes them to the database only once, at the very end of the run, from
``_persist_result``. That is fine for a clean run but loses everything when a
-long generation phase is interrupted: the pages already embedded into the
-vector store are skipped on the next resume (the store is the resume ground
-truth) yet were never written to the ``pages`` table, so the wiki ends up
-permanently missing them.
+long generation phase is interrupted: pages already embedded into the vector
+store but never written to the ``pages`` table are lost from the wiki. A resume
+counts a page as done only when it has both a vector and a stored row, so such
+a page is regenerated rather than skipped.
:func:`run_generation_with_persistence` closes that gap with two cooperating
mechanisms, both best-effort so neither can fail a generation run:
@@ -59,7 +59,8 @@ async def run_generation_with_persistence(
"""Run :func:`run_generation`, reusing + incrementally persisting pages.
``generation_kwargs`` are forwarded verbatim to ``run_generation``; callers
- must not pass ``prior_pages`` or ``on_page_ready`` (this wrapper owns both).
+ must not pass ``prior_pages``, ``on_page_ready`` or ``persisted_page_ids``
+ (this wrapper owns them).
Returns the generated pages exactly as ``run_generation`` would.
"""
from repowise.cli.helpers import get_db_url_for_repo
@@ -82,16 +83,23 @@ async def run_generation_with_persistence(
async with get_session(sf) as session:
repo_id = (await upsert_repository(session, name=repo_name, local_path=str(repo_path))).id
+ resume = bool(generation_kwargs.get("resume"))
prior_pages: dict[str, Any] = {}
- if reuse_prior_pages:
+ # On resume, the ids with a stored row: a vector without one is a page the
+ # last run embedded but died before saving, so it must be regenerated.
+ # None (load failed) falls back to trusting the vector store.
+ persisted_page_ids: set[str] | None = None
+ if reuse_prior_pages or resume:
try:
async with get_session(sf) as session:
- prior_pages = await load_prior_pages(session, repo_id)
- if prior_pages:
- logger.info("generation.prior_pages_loaded", count=len(prior_pages))
+ loaded = await load_prior_pages(session, repo_id)
+ persisted_page_ids = set(loaded)
+ if reuse_prior_pages:
+ prior_pages = loaded
+ if loaded:
+ logger.info("generation.prior_pages_loaded", count=len(loaded))
except Exception as exc:
logger.debug("generation.prior_pages_load_failed", error=str(exc))
- prior_pages = {}
# A bounded handoff queue decouples the synchronous on_page_ready callback
# (fired inside the generation loop) from the async DB writes, so a slow
@@ -151,6 +159,7 @@ async def run_generation_with_persistence(
**generation_kwargs,
prior_pages=prior_pages,
on_page_ready=_on_page_ready,
+ persisted_page_ids=persisted_page_ids if resume else None,
)
finally:
await queue.put(sentinel)
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index 5f6896eac..e87893285 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -246,6 +246,7 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):
preserved_page_ids: set[str] | None = None,
timings: Any | None = None,
on_warning: Callable[[str], None] | None = None,
+ persisted_page_ids: set[str] | None = None,
) -> list[GeneratedPage]:
"""Generate all wiki pages for a repository.
@@ -267,6 +268,10 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):
stale. Harmless to pass on a non-resume run (nothing is skipped for that
reason, so nothing is added); None when the caller has no use for it.
+ ``persisted_page_ids`` is the set of ids that already have a stored
+ page row. On ``resume`` a page counts as done only when it is in both
+ this set and the vector store; None trusts the vector store alone.
+
``timings`` is the run's shared ``PhaseTimings`` table. Generation
records its per-level and checkpoint spans into it so they report
beside the top-level phases; None disables those spans.
@@ -306,6 +311,7 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):
preserved_page_ids=preserved_page_ids,
timings=timings,
on_warning=on_warning,
+ persisted_page_ids=persisted_page_ids,
)
# ------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 8aa14217e..1c971253c 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -112,6 +112,7 @@ class _GenerationRun:
preserved_page_ids: set[str] | None = None,
timings: Any | None = None,
on_warning: Callable[[str], None] | None = None,
+ persisted_page_ids: set[str] | None = None,
) -> None:
self.gen = gen
self.config = gen._config
@@ -129,6 +130,9 @@ class _GenerationRun:
# wiki the resumed run was there to protect. None when the caller does
# not care (every non-resume path).
self.preserved_page_ids = preserved_page_ids
+ # Resume: the ids that already have a stored page row, or None when the
+ # caller cannot tell. See ``_seed_resume``.
+ self.persisted_page_ids = persisted_page_ids
self.parsed_files = parsed_files
self.source_map = source_map
self.graph_builder = graph_builder
@@ -233,7 +237,8 @@ class _GenerationRun:
if self.repo_path
else str(getattr(self.repo_structure, "root_path", "."))
)
- # On resume, query the vector store directly — it is the ground truth.
+ # On resume, completed ids come from the vector store (narrowed to
+ # pages with a stored row when the caller knows them).
if self.resume and self.vector_store is not None:
# Note: caller drives this synchronously enough; resume seeding is
# awaited in execute() to keep __init__ side-effect free.
@@ -267,6 +272,12 @@ class _GenerationRun:
async def _seed_resume(self) -> None:
if self.job_system is not None and self.resume and self.vector_store is not None:
self.completed_ids = await self.vector_store.list_page_ids()
+ # The vector and the page row are written separately, so a run
+ # killed between the two leaves a vector with no page. Counting it
+ # as done would skip it forever, so a page is done only when both
+ # landed.
+ if self.persisted_page_ids is not None:
+ self.completed_ids &= self.persisted_page_ids
if self.completed_ids:
log.info(
"Resuming generation from vector store",
diff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.py
index 1e2f94579..cbc3e594d 100644
--- a/packages/core/src/repowise/core/pipeline/phases/generation.py
+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py
@@ -54,6 +54,7 @@ async def run_generation(
preserved_page_ids: set[str] | None = None,
test_run: bool = False,
selection_out: dict[str, Any] | None = None,
+ persisted_page_ids: set[str] | None = None,
) -> list[Any]:
"""Run LLM-powered page generation.
@@ -71,6 +72,9 @@ async def run_generation(
the ids it skipped because a prior run already wrote them. Persistence
needs it to keep those pages out of the stale sweep.
+ ``persisted_page_ids`` (ids with a stored page row) narrows what a
+ ``resume`` run treats as already done; see ``PageGenerator.generate_all``.
+
``test_run`` limits generation to the top 10 files by PageRank, so a quick
validation run can exercise the whole generation path without paying for a
full index's worth of pages.
@@ -212,6 +216,7 @@ async def run_generation(
preserved_page_ids=preserved_page_ids,
timings=getattr(progress, "table", None),
on_warning=on_warning,
+ persisted_page_ids=persisted_page_ids,
)
selection = getattr(generator, "selection", None)
if selection_out is not None and selection is not None:
diff --git a/tests/integration/test_generation_pipeline.py b/tests/integration/test_generation_pipeline.py
index 0a2166be2..a3c060a68 100644
--- a/tests/integration/test_generation_pipeline.py
+++ b/tests/integration/test_generation_pipeline.py
@@ -379,6 +379,48 @@ async def test_resume_reports_the_pages_it_skipped(tmp_path):
assert "file_page:pkg/module_1.py" in {page.page_id for page in pages}
+async def test_resume_regenerates_a_page_with_a_vector_but_no_row(tmp_path):
+ """A run killed after embedding a page but before saving its row leaves
+ a vector with no page. Resume must regenerate that page, not skip it."""
+ from repowise.core.pipeline.phases.generation import run_generation
+
+ parsed_files, source_map, repo_structure = _make_concurrency_fixture()
+ builder = GraphBuilder()
+ for parsed in parsed_files:
+ builder.add_file(parsed)
+ builder.build()
+
+ saved = "file_page:pkg/module_0.py"
+ vector_only = "file_page:pkg/module_1.py"
+
+ class _StoreWithPriorRun(_SlowVectorStore):
+ async def list_page_ids(self) -> set[str]:
+ return {saved, vector_only}
+
+ preserved: set[str] = set()
+ pages = await run_generation(
+ repo_path=tmp_path,
+ parsed_files=parsed_files,
+ source_map=source_map,
+ graph_builder=builder,
+ repo_structure=repo_structure,
+ git_meta_map={},
+ llm_client=MockProvider(),
+ embedder=None,
+ vector_store=_StoreWithPriorRun(delay=0),
+ concurrency=2,
+ progress=None,
+ resume=True,
+ generation_config=GenerationConfig(max_file_pages=0, cache_enabled=False),
+ preserved_page_ids=preserved,
+ persisted_page_ids={saved},
+ )
+
+ ids = {page.page_id for page in pages}
+ assert saved in preserved and saved not in ids
+ assert vector_only in ids and vector_only not in preserved
+
+
# ---------------------------------------------------------------------------
# Test class
# ---------------------------------------------------------------------------
diff --git a/tests/unit/cli/test_generation_persist.py b/tests/unit/cli/test_generation_persist.py
index aff1cf132..98255012a 100644
--- a/tests/unit/cli/test_generation_persist.py
+++ b/tests/unit/cli/test_generation_persist.py
@@ -168,3 +168,46 @@ async def test_batched_flush_stores_every_page_and_times_the_drain(repo_dir, mon
assert await _read_db_page_ids(repo_dir) == {p.page_id for p in emitted}
assert "generation.persist_drain" in table.totals
+
+
+async def test_resume_hands_the_persisted_ids_to_generation(repo_dir, monkeypatch):
+ """Resume passes the ids that have a stored row, so a page whose vector
+ landed but whose row did not is regenerated and then persisted."""
+ seen: dict = {}
+
+ async def first_run(*, repo_path, on_page_ready=None, **_kw):
+ on_page_ready(_page("saved"))
+ return [_page("saved")]
+
+ async def resumed_run(*, repo_path, on_page_ready=None, persisted_page_ids=None, **kw):
+ seen["persisted"] = persisted_page_ids
+ seen["resume"] = kw.get("resume")
+ # "vector_only" has a vector from the killed run but no row; the
+ # generator regenerates it because it is not in persisted_page_ids.
+ on_page_ready(_page("vector_only"))
+ return [_page("vector_only")]
+
+ monkeypatch.setattr("repowise.core.pipeline.run_generation", first_run, raising=True)
+ await run_generation_with_persistence(repo_path=repo_dir, repo_name=repo_dir.name)
+
+ monkeypatch.setattr("repowise.core.pipeline.run_generation", resumed_run, raising=True)
+ await run_generation_with_persistence(
+ repo_path=repo_dir, repo_name=repo_dir.name, resume=True, reuse_prior_pages=False
+ )
+
+ assert seen == {"persisted": {"saved"}, "resume": True}
+ assert await _read_db_page_ids(repo_dir) == {"saved", "vector_only"}
+
+
+async def test_non_resume_run_passes_no_persisted_ids(repo_dir, monkeypatch):
+ seen: dict = {}
+
+ async def fake_run_generation(*, repo_path, persisted_page_ids="unset", **_kw):
+ seen["persisted"] = persisted_page_ids
+ return []
+
+ monkeypatch.setattr(
+ "repowise.core.pipeline.run_generation", fake_run_generation, raising=True
+ )
+ await run_generation_with_persistence(repo_path=repo_dir, repo_name=repo_dir.name)
+ assert seen["persisted"] is None
diff --git a/tests/unit/generation/test_job_checkpoint_durability.py b/tests/unit/generation/test_job_checkpoint_durability.py
index 668c5627b..4f7e000f4 100644
--- a/tests/unit/generation/test_job_checkpoint_durability.py
+++ b/tests/unit/generation/test_job_checkpoint_durability.py
@@ -63,6 +63,7 @@ async def test_interrupt_between_page_and_flush_costs_no_resume_coverage(tmp_pat
vector_store=store,
completed_ids=set(),
preserved_page_ids=set(),
+ persisted_page_ids={"file_page:a.py", "file_page:b.py"},
only_page_ids=None,
)
await _GenerationRun._seed_resume(run)
@@ -92,6 +93,7 @@ async def test_a_page_whose_rows_never_landed_is_regenerated(tmp_path):
vector_store=store,
completed_ids=set(),
preserved_page_ids=set(),
+ persisted_page_ids=None,
only_page_ids=None,
)
await _GenerationRun._seed_resume(run)
@@ -99,6 +101,29 @@ async def test_a_page_whose_rows_never_landed_is_regenerated(tmp_path):
assert _GenerationRun._emit(run, "file_page:b.py") is True
+async def test_a_page_whose_vector_landed_but_row_did_not_is_regenerated(tmp_path):
+ """The vector and the page row are written separately. A kill between
+ the two leaves a vector with no row, and resume must regenerate it."""
+ store = InMemoryVectorStore(_StubEmbedder())
+ await store.embed_batch([("file_page:a.py", "a", {}), ("file_page:b.py", "b", {})])
+
+ run = SimpleNamespace(
+ job_system=JobSystem(tmp_path / "jobs"),
+ resume=True,
+ vector_store=store,
+ completed_ids=set(),
+ preserved_page_ids=set(),
+ persisted_page_ids={"file_page:a.py"},
+ only_page_ids=None,
+ )
+ await _GenerationRun._seed_resume(run)
+
+ assert _GenerationRun._emit(run, "file_page:a.py") is False
+ assert _GenerationRun._emit(run, "file_page:b.py") is True
+ assert run.preserved_page_ids == {"file_page:a.py"}
+
+
def test_a_flushed_level_survives_the_same_kill(tmp_path):
"""The bound in reverse: what a level boundary made durable stays durable."""
js, job_id = _job(tmp_path)