Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): add on_page_ready streaming callback (#328)

Raghav Chamadiya5a776a4b2753
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index ddc5e7c15..67458fb4e 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -170,6 +170,7 @@ class PageGenerator(PerTypeGenerationMixin):
dead_code_report: Any | None = None,
decision_report: Any | None = None,
external_systems: list[dict] | None = None,
+ on_page_ready: Callable[[GeneratedPage], None] | None = None,
) -> list[GeneratedPage]:
"""Generate all wiki pages for a repository.
@@ -196,6 +197,7 @@ class PageGenerator(PerTypeGenerationMixin):
dead_code_report=dead_code_report,
decision_report=decision_report,
external_systems=external_systems,
+ on_page_ready=on_page_ready,
)
# ------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 8127f64c2..264a6f6f4 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -59,6 +59,7 @@ class _GenerationRun:
dead_code_report: Any | None,
decision_report: Any | None,
external_systems: list[dict] | None,
+ on_page_ready: Callable[[GeneratedPage], None] | None = None,
) -> None:
self.gen = gen
self.config = gen._config
@@ -70,6 +71,12 @@ class _GenerationRun:
self.repo_name = repo_name
self.job_system = job_system
self.on_page_done = on_page_done
+ # Fired with the full GeneratedPage the instant it completes (in
+ # addition to on_page_done, which only gets the page_type). Lets a
+ # caller persist/stream pages incrementally — e.g. the hosted indexer
+ # flushing pages.json per page so a budget cutoff yields partial docs
+ # instead of nothing. Optional + best-effort; never blocks generation.
+ self.on_page_ready = on_page_ready
self.on_total_known = on_total_known
self.on_subphase = on_subphase
self.git_meta_map = git_meta_map
@@ -337,6 +344,14 @@ class _GenerationRun:
# Progress tick fires the moment the page is ready.
if self.on_page_done is not None:
self.on_page_done(result.page_type)
+ # Hand the full page to a streaming sink (incremental
+ # persistence). Best-effort: a sink error must not drop the
+ # page or abort the level.
+ if self.on_page_ready is not None:
+ try:
+ self.on_page_ready(result)
+ except Exception as exc: # noqa: BLE001
+ log.debug("on_page_ready.failed", error=str(exc))
if self.vector_store is not None:
embed_items.append(_embed_item(result))
return result
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index e537df2f6..4aee6ce7a 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -157,6 +157,7 @@ async def run_pipeline(
cost_tracker: Any | None = None,
generation_config: Any | None = None,
existing_kg_fingerprint: str | None = None,
+ on_page_ready: Any | None = None,
) -> PipelineResult:
"""Run the repowise indexing/analysis/generation pipeline.
@@ -483,6 +484,7 @@ async def run_pipeline(
dead_code_report=dead_code_report,
decision_report=decision_report,
external_systems=external_systems,
+ on_page_ready=on_page_ready,
)
# ---- Knowledge Graph LLM enrichment (layer naming + tour) -----------------
diff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.py
index 71e76d59d..a6caf5a12 100644
--- a/packages/core/src/repowise/core/pipeline/phases/generation.py
+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py
@@ -38,6 +38,7 @@ async def run_generation(
dead_code_report: Any | None = None,
decision_report: Any | None = None,
external_systems: list[dict] | None = None,
+ on_page_ready: Any | None = None,
) -> list[Any]:
"""Run LLM-powered page generation.
@@ -128,6 +129,7 @@ async def run_generation(
dead_code_report=dead_code_report,
decision_report=decision_report,
external_systems=external_systems,
+ on_page_ready=on_page_ready,
)
# Onboarding summary — count generated slots and surface which ones