Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(init): resume regenerates a page whose vector landed but whose row did not (#2883)
Raghav Chamadiya18ffa2e522c1
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.pyindex d6f2c5b26..01a7e81d3 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/_generation_persist.py@@ -3,10 +3,10 @@The orchestrator's :func:`run_generation` buffers every page in memory and theCLI writes them to the database only once, at the very end of the run, from``_persist_result``. That is fine for a clean run but loses everything when a-long generation phase is interrupted: the pages already embedded into the-vector store are skipped on the next resume (the store is the resume ground-truth) yet were never written to the ``pages`` table, so the wiki ends up-permanently missing them.+long generation phase is interrupted: pages already embedded into the vector+store but never written to the ``pages`` table are lost from the wiki. A resume+counts a page as done only when it has both a vector and a stored row, so such+a page is regenerated rather than skipped.:func:`run_generation_with_persistence` closes that gap with two cooperatingmechanisms, both best-effort so neither can fail a generation run:@@ -59,7 +59,8 @@ async def run_generation_with_persistence("""Run :func:`run_generation`, reusing + incrementally persisting pages.``generation_kwargs`` are forwarded verbatim to ``run_generation``; callers- must not pass ``prior_pages`` or ``on_page_ready`` (this wrapper owns both).+ must not pass ``prior_pages``, ``on_page_ready`` or ``persisted_page_ids``+ (this wrapper owns them).Returns the generated pages exactly as ``run_generation`` would."""from repowise.cli.helpers import get_db_url_for_repo@@ -82,16 +83,23 @@ async def run_generation_with_persistence(async with get_session(sf) as session:repo_id = (await upsert_repository(session, name=repo_name, local_path=str(repo_path))).id+ resume = bool(generation_kwargs.get("resume"))prior_pages: dict[str, Any] = {}- if reuse_prior_pages:+ # On resume, the ids with a stored row: a vector without one is a page the+ # last run embedded but died before saving, so it must be regenerated.+ # None (load failed) falls back to trusting the vector store.+ persisted_page_ids: set[str] | None = None+ if reuse_prior_pages or resume:try:async with get_session(sf) as session:- prior_pages = await load_prior_pages(session, repo_id)- if prior_pages:- logger.info("generation.prior_pages_loaded", count=len(prior_pages))+ loaded = await load_prior_pages(session, repo_id)+ persisted_page_ids = set(loaded)+ if reuse_prior_pages:+ prior_pages = loaded+ if loaded:+ logger.info("generation.prior_pages_loaded", count=len(loaded))except Exception as exc:logger.debug("generation.prior_pages_load_failed", error=str(exc))- prior_pages = {}# A bounded handoff queue decouples the synchronous on_page_ready callback# (fired inside the generation loop) from the async DB writes, so a slow@@ -151,6 +159,7 @@ async def run_generation_with_persistence(**generation_kwargs,prior_pages=prior_pages,on_page_ready=_on_page_ready,+ persisted_page_ids=persisted_page_ids if resume else None,)finally:await queue.put(sentinel)diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.pyindex 5f6896eac..e87893285 100644--- a/packages/core/src/repowise/core/generation/page_generator/core.py+++ b/packages/core/src/repowise/core/generation/page_generator/core.py@@ -246,6 +246,7 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):preserved_page_ids: set[str] | None = None,timings: Any | None = None,on_warning: Callable[[str], None] | None = None,+ persisted_page_ids: set[str] | None = None,) -> list[GeneratedPage]:"""Generate all wiki pages for a repository.@@ -267,6 +268,10 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):stale. Harmless to pass on a non-resume run (nothing is skipped for thatreason, so nothing is added); None when the caller has no use for it.+ ``persisted_page_ids`` is the set of ids that already have a stored+ page row. On ``resume`` a page counts as done only when it is in both+ this set and the vector store; None trusts the vector store alone.+``timings`` is the run's shared ``PhaseTimings`` table. Generationrecords its per-level and checkpoint spans into it so they reportbeside the top-level phases; None disables those spans.@@ -306,6 +311,7 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):preserved_page_ids=preserved_page_ids,timings=timings,on_warning=on_warning,+ persisted_page_ids=persisted_page_ids,)# ------------------------------------------------------------------diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex 8aa14217e..1c971253c 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -112,6 +112,7 @@ class _GenerationRun:preserved_page_ids: set[str] | None = None,timings: Any | None = None,on_warning: Callable[[str], None] | None = None,+ persisted_page_ids: set[str] | None = None,) -> None:self.gen = genself.config = gen._config@@ -129,6 +130,9 @@ class _GenerationRun:# wiki the resumed run was there to protect. None when the caller does# not care (every non-resume path).self.preserved_page_ids = preserved_page_ids+ # Resume: the ids that already have a stored page row, or None when the+ # caller cannot tell. See ``_seed_resume``.+ self.persisted_page_ids = persisted_page_idsself.parsed_files = parsed_filesself.source_map = source_mapself.graph_builder = graph_builder@@ -233,7 +237,8 @@ class _GenerationRun:if self.repo_pathelse str(getattr(self.repo_structure, "root_path", ".")))- # On resume, query the vector store directly — it is the ground truth.+ # On resume, completed ids come from the vector store (narrowed to+ # pages with a stored row when the caller knows them).if self.resume and self.vector_store is not None:# Note: caller drives this synchronously enough; resume seeding is# awaited in execute() to keep __init__ side-effect free.@@ -267,6 +272,12 @@ class _GenerationRun:async def _seed_resume(self) -> None:if self.job_system is not None and self.resume and self.vector_store is not None:self.completed_ids = await self.vector_store.list_page_ids()+ # The vector and the page row are written separately, so a run+ # killed between the two leaves a vector with no page. Counting it+ # as done would skip it forever, so a page is done only when both+ # landed.+ if self.persisted_page_ids is not None:+ self.completed_ids &= self.persisted_page_idsif self.completed_ids:log.info("Resuming generation from vector store",diff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.pyindex 1e2f94579..cbc3e594d 100644--- a/packages/core/src/repowise/core/pipeline/phases/generation.py+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py@@ -54,6 +54,7 @@ async def run_generation(preserved_page_ids: set[str] | None = None,test_run: bool = False,selection_out: dict[str, Any] | None = None,+ persisted_page_ids: set[str] | None = None,) -> list[Any]:"""Run LLM-powered page generation.@@ -71,6 +72,9 @@ async def run_generation(the ids it skipped because a prior run already wrote them. Persistenceneeds it to keep those pages out of the stale sweep.+ ``persisted_page_ids`` (ids with a stored page row) narrows what a+ ``resume`` run treats as already done; see ``PageGenerator.generate_all``.+``test_run`` limits generation to the top 10 files by PageRank, so a quickvalidation run can exercise the whole generation path without paying for afull index's worth of pages.@@ -212,6 +216,7 @@ async def run_generation(preserved_page_ids=preserved_page_ids,timings=getattr(progress, "table", None),on_warning=on_warning,+ persisted_page_ids=persisted_page_ids,)selection = getattr(generator, "selection", None)if selection_out is not None and selection is not None:diff --git a/tests/integration/test_generation_pipeline.py b/tests/integration/test_generation_pipeline.pyindex 0a2166be2..a3c060a68 100644--- a/tests/integration/test_generation_pipeline.py+++ b/tests/integration/test_generation_pipeline.py@@ -379,6 +379,48 @@ async def test_resume_reports_the_pages_it_skipped(tmp_path):assert "file_page:pkg/module_1.py" in {page.page_id for page in pages}+async def test_resume_regenerates_a_page_with_a_vector_but_no_row(tmp_path):+ """A run killed after embedding a page but before saving its row leaves+ a vector with no page. Resume must regenerate that page, not skip it."""+ from repowise.core.pipeline.phases.generation import run_generation++ parsed_files, source_map, repo_structure = _make_concurrency_fixture()+ builder = GraphBuilder()+ for parsed in parsed_files:+ builder.add_file(parsed)+ builder.build()++ saved = "file_page:pkg/module_0.py"+ vector_only = "file_page:pkg/module_1.py"++ class _StoreWithPriorRun(_SlowVectorStore):+ async def list_page_ids(self) -> set[str]:+ return {saved, vector_only}++ preserved: set[str] = set()+ pages = await run_generation(+ repo_path=tmp_path,+ parsed_files=parsed_files,+ source_map=source_map,+ graph_builder=builder,+ repo_structure=repo_structure,+ git_meta_map={},+ llm_client=MockProvider(),+ embedder=None,+ vector_store=_StoreWithPriorRun(delay=0),+ concurrency=2,+ progress=None,+ resume=True,+ generation_config=GenerationConfig(max_file_pages=0, cache_enabled=False),+ preserved_page_ids=preserved,+ persisted_page_ids={saved},+ )++ ids = {page.page_id for page in pages}+ assert saved in preserved and saved not in ids+ assert vector_only in ids and vector_only not in preserved++# ---------------------------------------------------------------------------# Test class# ---------------------------------------------------------------------------diff --git a/tests/unit/cli/test_generation_persist.py b/tests/unit/cli/test_generation_persist.pyindex aff1cf132..98255012a 100644--- a/tests/unit/cli/test_generation_persist.py+++ b/tests/unit/cli/test_generation_persist.py@@ -168,3 +168,46 @@ async def test_batched_flush_stores_every_page_and_times_the_drain(repo_dir, monassert await _read_db_page_ids(repo_dir) == {p.page_id for p in emitted}assert "generation.persist_drain" in table.totals+++async def test_resume_hands_the_persisted_ids_to_generation(repo_dir, monkeypatch):+ """Resume passes the ids that have a stored row, so a page whose vector+ landed but whose row did not is regenerated and then persisted."""+ seen: dict = {}++ async def first_run(*, repo_path, on_page_ready=None, **_kw):+ on_page_ready(_page("saved"))+ return [_page("saved")]++ async def resumed_run(*, repo_path, on_page_ready=None, persisted_page_ids=None, **kw):+ seen["persisted"] = persisted_page_ids+ seen["resume"] = kw.get("resume")+ # "vector_only" has a vector from the killed run but no row; the+ # generator regenerates it because it is not in persisted_page_ids.+ on_page_ready(_page("vector_only"))+ return [_page("vector_only")]++ monkeypatch.setattr("repowise.core.pipeline.run_generation", first_run, raising=True)+ await run_generation_with_persistence(repo_path=repo_dir, repo_name=repo_dir.name)++ monkeypatch.setattr("repowise.core.pipeline.run_generation", resumed_run, raising=True)+ await run_generation_with_persistence(+ repo_path=repo_dir, repo_name=repo_dir.name, resume=True, reuse_prior_pages=False+ )++ assert seen == {"persisted": {"saved"}, "resume": True}+ assert await _read_db_page_ids(repo_dir) == {"saved", "vector_only"}+++async def test_non_resume_run_passes_no_persisted_ids(repo_dir, monkeypatch):+ seen: dict = {}++ async def fake_run_generation(*, repo_path, persisted_page_ids="unset", **_kw):+ seen["persisted"] = persisted_page_ids+ return []++ monkeypatch.setattr(+ "repowise.core.pipeline.run_generation", fake_run_generation, raising=True+ )+ await run_generation_with_persistence(repo_path=repo_dir, repo_name=repo_dir.name)+ assert seen["persisted"] is Nonediff --git a/tests/unit/generation/test_job_checkpoint_durability.py b/tests/unit/generation/test_job_checkpoint_durability.pyindex 668c5627b..4f7e000f4 100644--- a/tests/unit/generation/test_job_checkpoint_durability.py+++ b/tests/unit/generation/test_job_checkpoint_durability.py@@ -63,6 +63,7 @@ async def test_interrupt_between_page_and_flush_costs_no_resume_coverage(tmp_patvector_store=store,completed_ids=set(),preserved_page_ids=set(),+ persisted_page_ids={"file_page:a.py", "file_page:b.py"},only_page_ids=None,)await _GenerationRun._seed_resume(run)@@ -92,6 +93,7 @@ async def test_a_page_whose_rows_never_landed_is_regenerated(tmp_path):vector_store=store,completed_ids=set(),preserved_page_ids=set(),+ persisted_page_ids=None,only_page_ids=None,)await _GenerationRun._seed_resume(run)@@ -99,6 +101,29 @@ async def test_a_page_whose_rows_never_landed_is_regenerated(tmp_path):assert _GenerationRun._emit(run, "file_page:b.py") is True+async def test_a_page_whose_vector_landed_but_row_did_not_is_regenerated(tmp_path):+ """The vector and the page row are written separately. A kill between+ the two leaves a vector with no row, and resume must regenerate it."""+ store = InMemoryVectorStore(_StubEmbedder())+ await store.embed_batch([("file_page:a.py", "a", {}), ("file_page:b.py", "b", {})])++ run = SimpleNamespace(+ job_system=JobSystem(tmp_path / "jobs"),+ resume=True,+ vector_store=store,+ completed_ids=set(),+ preserved_page_ids=set(),+ persisted_page_ids={"file_page:a.py"},+ only_page_ids=None,+ )+ await _GenerationRun._seed_resume(run)++ assert _GenerationRun._emit(run, "file_page:a.py") is False+ assert _GenerationRun._emit(run, "file_page:b.py") is True+ assert run.preserved_page_ids == {"file_page:a.py"}++def test_a_flushed_level_survives_the_same_kill(tmp_path):"""The bound in reverse: what a level boundary made durable stays durable."""js, job_id = _job(tmp_path)