Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(pipeline): analysis phase and KG skeleton out of run_pipeline (#4078)

Raghav Chamadiyae881a1b7c348
diff --git a/packages/core/src/repowise/core/pipeline/_run/analysis.py b/packages/core/src/repowise/core/pipeline/_run/analysis.py
new file mode 100644
index 000000000..695d1a331
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/_run/analysis.py
@@ -0,0 +1,277 @@
+"""Analysis phase: the four analyses or their resumed views, the KG skeleton, the ANALYSIS checkpoint."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import structlog
+
+# Phase functions are looked up on the orchestrator at call time, so a patch on
+# ``repowise.core.pipeline.orchestrator`` reaches this module too.
+from repowise.core.pipeline import orchestrator as _orch
+from repowise.core.pipeline.phase_timing import timed
+from repowise.core.pipeline.phases._common import (
+ TEST_RUN_FILE_LIMIT,
+ _phase_done,
+ limit_to_top_pagerank,
+)
+from repowise.core.pipeline.progress import STAGE_ANALYSIS, emit_stage
+from repowise.core.pipeline.resume import ResumePhase
+
+from .state import PipelineRun
+
+logger = structlog.get_logger("repowise.core.pipeline.orchestrator")
+
+
+async def run_analysis_phase(run: PipelineRun) -> None:
+ """Fill the analysis outputs and the KG skeleton of *run*, then checkpoint them."""
+ _limit_test_run(run)
+ emit_stage(run.progress, STAGE_ANALYSIS)
+ run.skip_analysis = await _rehydrate_analysis(run)
+ if not run.skip_analysis:
+ await _compute_analysis(run)
+ _build_knowledge_graph(run)
+ if run.resume_controller is not None and not run.skip_analysis:
+ await _checkpoint_analysis(run)
+
+
+def _limit_test_run(run: PipelineRun) -> None:
+ # Same helper as ``run_generation`` so the two ``--test-run`` paths cannot drift.
+ if not (run.test_run and run.generate_docs):
+ return
+ run.parsed_files = limit_to_top_pagerank(
+ run.parsed_files, run.graph_builder, n=TEST_RUN_FILE_LIMIT
+ )
+ if run.progress:
+ run.progress.on_message("warning", f"Test run: limiting to {len(run.parsed_files)} files")
+
+
+async def _rehydrate_analysis(run: PipelineRun) -> bool:
+ """Reuse the persisted analysis views generation reads; False to recompute.
+
+ Only the thin generation inputs come back (``health_report`` stays None); the
+ persisted rows stay authoritative and are never re-written from these.
+ """
+ controller = run.resume_controller
+ if not (controller and await controller.can_skip(ResumePhase.ANALYSIS)):
+ return False
+ try:
+ (
+ run.gen_dead_code_report,
+ run.gen_decision_report,
+ ) = await controller.rehydrate_analysis()
+ if run.progress:
+ run.progress.on_message("info", " ↳ Resuming — reusing persisted analysis")
+ except Exception as exc:
+ logger.warning("resume_rehydrate_analysis_failed_recomputing", error=str(exc))
+ return False
+ if not await controller.has_completed_decision_extraction():
+ await _backfill_decisions(run)
+ return True
+
+
+async def _backfill_decisions(run: PipelineRun) -> None:
+ """Run decision extraction alone when a resumed analysis never finished it."""
+ if run.progress:
+ run.progress.on_message(
+ "info",
+ " ↳ Resumed analysis missing decision extraction — running extraction",
+ )
+ run.decision_report = await _orch._run_decision_extraction(
+ run.repo_path,
+ llm_client=run.llm_client,
+ graph_builder=run.graph_builder,
+ git_meta_map=run.git_meta_map,
+ parsed_files=run.parsed_files,
+ source_map=run.source_map,
+ progress=run.progress,
+ )
+ if run.decision_report is not None:
+ run.gen_decision_report = run.decision_report
+ await run.resume_controller.checkpoint_decision_backfill(
+ run.decision_report, progress=run.progress
+ )
+
+
+async def _compute_analysis(run: PipelineRun) -> None:
+ """Run dead code, health, decisions and doc drift concurrently.
+
+ They share read-only inputs and no data, so the LLM-bound decision extraction
+ hides behind the CPU-bound dead-code and health work.
+ """
+ progress = run.progress
+ (
+ run.dead_code_report,
+ run.health_report,
+ run.decision_report,
+ run.doc_drift_report,
+ ) = await asyncio.gather(
+ _orch._run_dead_code_analysis(
+ run.graph_builder,
+ run.git_meta_map,
+ source_map=run.source_map,
+ repo_path=run.repo_path,
+ traversal_stats=run.traversal_stats,
+ progress=progress,
+ ),
+ _orch._run_health_analysis(
+ run.graph_builder,
+ run.git_meta_map,
+ run.parsed_files,
+ repo_path=run.repo_path,
+ coverage_report_paths=run.coverage_report_paths,
+ progress=progress,
+ ),
+ _orch._run_decision_extraction(
+ run.repo_path,
+ llm_client=run.llm_client,
+ graph_builder=run.graph_builder,
+ git_meta_map=run.git_meta_map,
+ parsed_files=run.parsed_files,
+ source_map=run.source_map,
+ progress=progress,
+ ),
+ _orch._run_doc_drift_analysis(
+ run.source_map,
+ file_infos=run.file_infos,
+ graph_builder=run.graph_builder,
+ repo_path=run.repo_path,
+ progress=progress,
+ ),
+ )
+ # Health reads the in-memory BlameIndex while it runs; drop it only after the
+ # gather, before it can leak into the result or persistence.
+ _orch.drop_transient_git_signals(run.git_metadata_list)
+ run.gen_dead_code_report = run.dead_code_report
+ run.gen_decision_report = run.decision_report
+
+
+def _build_knowledge_graph(run: PipelineRun) -> None:
+ """Reuse or build the deterministic KG skeleton, then curate it; failures only log."""
+ try:
+ from repowise.core.analysis.knowledge_graph import (
+ KnowledgeGraphResult,
+ build_knowledge_graph_skeleton,
+ compute_kg_fingerprint,
+ should_skip_kg_rebuild,
+ )
+
+ new_fingerprint = compute_kg_fingerprint(run.graph_builder)
+ kg_json_path = run.repo_path / ".repowise" / "knowledge-graph.json"
+ if should_skip_kg_rebuild(run.existing_kg_fingerprint, new_fingerprint, kg_json_path):
+ _reuse_kg_artifact(run, KnowledgeGraphResult, kg_json_path, new_fingerprint)
+
+ run.tech_stack_dicts = [
+ {"name": t.name, "version": t.version, "category": t.category} for t in run.tech_items
+ ]
+ if run.knowledge_graph_result is None:
+ _build_kg_skeleton(run, build_knowledge_graph_skeleton, new_fingerprint)
+ _phase_done(run.progress, "knowledge_graph.skeleton")
+ if run.knowledge_graph_result is not None:
+ _curate_knowledge_graph(run)
+ except (ValueError, KeyError, OSError, RuntimeError) as kg_err:
+ logger.error("kg_skeleton_building_failed", error=str(kg_err), exc_info=True)
+
+
+def _reuse_kg_artifact(
+ run: PipelineRun, result_cls: Any, kg_json_path: Any, new_fingerprint: str
+) -> None:
+ progress = run.progress
+ run.knowledge_graph_result = result_cls.from_file(kg_json_path)
+ if run.knowledge_graph_result is not None:
+ run.knowledge_graph_result.fingerprint = new_fingerprint
+ logger.info(
+ "knowledge_graph.skip",
+ reason="fingerprint_unchanged",
+ fingerprint=new_fingerprint,
+ )
+ if progress:
+ progress.on_message(
+ "info",
+ f" ↳ KG unchanged (fingerprint {new_fingerprint[:8]}…), reusing",
+ )
+ return
+ # The reader refused a file the fingerprint called reusable (unreadable or an
+ # older schema): say so before the rebuild.
+ logger.info(
+ "knowledge_graph.reuse_failed",
+ reason="artifact_unreadable_or_outdated",
+ path=str(kg_json_path),
+ )
+ if progress:
+ progress.on_message(
+ "info",
+ " ↳ knowledge-graph.json could not be reused, rebuilding it",
+ )
+
+
+def _build_kg_skeleton(run: PipelineRun, build_skeleton: Any, new_fingerprint: str) -> None:
+ progress = run.progress
+ if progress:
+ progress.on_phase_start("knowledge_graph.skeleton", None)
+ run.knowledge_graph_result = build_skeleton(
+ parsed_files=run.parsed_files,
+ graph_builder=run.graph_builder,
+ repo_structure=run.repo_structure,
+ tech_stack=run.tech_stack_dicts,
+ external_systems=run.external_systems,
+ git_meta_map=run.git_meta_map,
+ dead_code_report=run.gen_dead_code_report,
+ repo_path=run.repo_path,
+ )
+ run.knowledge_graph_result.fingerprint = new_fingerprint
+ if progress:
+ kg = run.knowledge_graph_result
+ progress.on_message(
+ "info",
+ f" ↳ KG skeleton: {len(kg.nodes)} nodes, "
+ f"{len(kg.edges)} edges, "
+ f"{len(kg.layers)} layers",
+ )
+
+
+def _curate_knowledge_graph(run: PipelineRun) -> None:
+ """Reshape the exported KG only (layers, tour, entry points, summaries).
+
+ Never touches the AST graph, communities or centrality; a no-op when
+ ``REPOWISE_KG_CURATION`` is falsy. Runs in every mode, before generation.
+ """
+ from repowise.core.analysis.kg_curation import curate_knowledge_graph, curation_enabled
+
+ try:
+ # Generate mode defers the summary floor until after the wiki-page
+ # backfill so rich page summaries win; other runs floor here.
+ will_generate = run.generate_docs and run.llm_client is not None
+ with timed(getattr(run.progress, "table", None), "knowledge_graph.curate"):
+ run.knowledge_graph_result = curate_knowledge_graph(
+ run.knowledge_graph_result,
+ parsed_files=run.parsed_files,
+ graph_builder=run.graph_builder,
+ repo_structure=run.repo_structure,
+ community_info=run.graph_builder.community_info(),
+ git_meta_map=run.git_meta_map,
+ enabled=curation_enabled(),
+ defer_summary_floor=will_generate,
+ )
+ except (ValueError, KeyError, RuntimeError) as cur_err:
+ logger.error("kg_curation_failed", error=str(cur_err), exc_info=True)
+
+
+async def _checkpoint_analysis(run: PipelineRun) -> None:
+ """Persist the analysis so a resume skips it; the first write of each decision.
+
+ The vector store goes with it: only this pass can fold a paraphrased decision
+ into an existing one, since by the end-of-run persist every group matches on title.
+ """
+ with timed(getattr(run.progress, "table", None), "persist.checkpoint_analysis"):
+ await run.resume_controller.checkpoint_analysis(
+ parsed_files=run.parsed_files,
+ dead_code_report=run.dead_code_report,
+ health_report=run.health_report,
+ decision_report=run.decision_report,
+ doc_drift_report=run.doc_drift_report,
+ git_metadata_list=run.git_metadata_list,
+ vector_store=run.vector_store,
+ progress=run.progress,
+ )
diff --git a/packages/core/src/repowise/core/pipeline/_run/state.py b/packages/core/src/repowise/core/pipeline/_run/state.py
index 252f4e715..50bd4eca0 100644
--- a/packages/core/src/repowise/core/pipeline/_run/state.py
+++ b/packages/core/src/repowise/core/pipeline/_run/state.py
@@ -57,6 +57,18 @@ class PipelineRun:
git_meta_map: dict[str, dict] = field(default_factory=dict)
external_systems: list[dict] = field(default_factory=list)
+ # Analysis phase outputs. The ``gen_`` reports feed generation and the KG:
+ # rehydrated on a resumed run, the freshly computed ones otherwise.
+ skip_analysis: bool = False
+ dead_code_report: Any = None
+ health_report: Any = None
+ decision_report: Any = None
+ doc_drift_report: Any = None
+ gen_dead_code_report: Any = None
+ gen_decision_report: Any = None
+ tech_stack_dicts: list[dict] = field(default_factory=list)
+ knowledge_graph_result: Any = None
+
def hooked_progress(progress: Any) -> HookProgressCallback:
"""Wrap *progress* so registered pipeline hooks fire around each phase.
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 0dc4ddfa2..1925b99af 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -25,20 +25,16 @@ import structlog
from repowise.core.ingestion.git_indexer._constants import _DEFAULT_COMMIT_LIMIT
from repowise.core.pipeline.modes import OrchestratorMode
-from repowise.core.pipeline.progress import (
- STAGE_ANALYSIS,
- ProgressCallback,
- emit_stage,
- emit_warning,
-)
+from repowise.core.pipeline.progress import ProgressCallback, emit_warning
+from ._run.analysis import run_analysis_phase
from ._run.index import run_index_phase
from ._run.state import PipelineRun, attach_cost_tracker, hooked_progress, start_checkpointer
-from .phase_timing import timed
-from .phases._common import TEST_RUN_FILE_LIMIT, _phase_done, limit_to_top_pagerank
+from .phases._common import _phase_done
# The ``_run`` stage modules call these phase functions through this module
# (``_orch.<name>``), so imports and patches on the orchestrator keep reaching them.
+# ruff: noqa: F401
from .phases.analysis import (
_run_dead_code_analysis,
_run_decision_extraction,
@@ -47,12 +43,11 @@ from .phases.analysis import (
)
from .phases.generation import run_generation
from .phases.git import (
- _run_git_indexing, # noqa: F401
+ _run_git_indexing,
drop_transient_git_signals,
- label_co_change_structure, # noqa: F401
+ label_co_change_structure,
)
-from .phases.ingestion import _run_ingestion, reparse_for_resume # noqa: F401
-from .resume import ResumePhase
+from .phases.ingestion import _run_ingestion, reparse_for_resume
from .resume.controller import ResumeController
logger = structlog.get_logger(__name__)
@@ -322,258 +317,16 @@ async def run_pipeline(
git_meta_map = run.git_meta_map
external_systems = run.external_systems
- # Test-run: limit to top 10 files by PageRank (shared helper with
- # ``run_generation`` so the two paths cannot drift).
- if test_run and generate_docs:
- parsed_files = limit_to_top_pagerank(
- parsed_files, graph_builder, n=TEST_RUN_FILE_LIMIT
- )
- if progress:
- progress.on_message("warning", f"Test run: limiting to {len(parsed_files)} files")
-
- # ---- Phase 2: Analysis --------------------------------------------------
- emit_stage(progress, STAGE_ANALYSIS)
-
- # Resume fast-path: when a prior run already completed (and persisted) the
- # ANALYSIS phase, skip recomputing dead code / health / decisions — the
- # last is the costly one (LLM-backed, minutes on large repos). We rehydrate
- # only the thin views generation reads; the persisted rows stay
- # authoritative and are never re-written from these. ``health_report`` is
- # not a generation input, so it stays None on this path (its persisted rows
- # are untouched). Falls back to a full recompute if rehydration errors.
- skip_analysis = bool(
- resume_controller and await resume_controller.can_skip(ResumePhase.ANALYSIS)
- )
- dead_code_report = None
- health_report = None
- decision_report = None
- doc_drift_report = None
- # Reports actually fed to generation + KG — rehydrated on the skip path,
- # the freshly computed ones otherwise.
- gen_dead_code_report = None
- gen_decision_report = None
- if skip_analysis:
- try:
- (
- gen_dead_code_report,
- gen_decision_report,
- ) = await resume_controller.rehydrate_analysis()
- if progress:
- progress.on_message("info", " ↳ Resuming — reusing persisted analysis")
- except Exception as exc:
- logger.warning("resume_rehydrate_analysis_failed_recomputing", error=str(exc))
- skip_analysis = False
-
- # If analysis was skipped, but decision extraction never completed (e.g. earlier
- # run was interrupted during decision extraction or returned None), re-run decision
- # extraction specifically so that decisions are backfilled without re-running dead code / health.
- if (
- skip_analysis
- and resume_controller
- and not await resume_controller.has_completed_decision_extraction()
- ):
- if progress:
- progress.on_message(
- "info",
- " ↳ Resumed analysis missing decision extraction — running extraction",
- )
- decision_report = await _run_decision_extraction(
- repo_path,
- llm_client=llm_client,
- graph_builder=graph_builder,
- git_meta_map=git_meta_map,
- parsed_files=parsed_files,
- source_map=source_map,
- progress=progress,
- )
- if decision_report is not None:
- gen_decision_report = decision_report
- await resume_controller.checkpoint_decision_backfill(
- decision_report, progress=progress
- )
-
- if not skip_analysis:
- # The four analyses share read-only inputs (graph, git_meta_map,
- # parsed_files; the lazy metric caches were warmed during ingestion)
- # and have no data dependency on each other, so run them concurrently:
- # decision extraction is I/O/LLM-bound and its wall clock hides
- # entirely behind the CPU-bound dead-code + health work.
- (
- dead_code_report,
- health_report,
- decision_report,
- doc_drift_report,
- ) = await asyncio.gather(
- _run_dead_code_analysis(
- graph_builder,
- git_meta_map,
- source_map=source_map,
- repo_path=repo_path,
- traversal_stats=traversal_stats,
- progress=progress,
- ),
- _run_health_analysis(
- graph_builder,
- git_meta_map,
- parsed_files,
- repo_path=repo_path,
- coverage_report_paths=coverage_report_paths,
- progress=progress,
- ),
- _run_decision_extraction(
- repo_path,
- llm_client=llm_client,
- graph_builder=graph_builder,
- git_meta_map=git_meta_map,
- parsed_files=parsed_files,
- source_map=source_map,
- progress=progress,
- ),
- _run_doc_drift_analysis(
- source_map,
- file_infos=file_infos,
- graph_builder=graph_builder,
- repo_path=repo_path,
- progress=progress,
- ),
- )
-
- # Drop the in-memory-only ``BlameIndex`` now that the health biomarkers
- # have consumed it — before it can leak into ``PipelineResult`` and the
- # downstream JSON artifact writers / DB persistence. ``git_meta_map``
- # shares these dict objects, so this cleans both views. Must stay after
- # the gather: health reads the blame index while it runs (decision
- # extraction never does).
- drop_transient_git_signals(git_metadata_list)
- gen_dead_code_report = dead_code_report
- gen_decision_report = decision_report
-
- # ---- Knowledge Graph skeleton (deterministic, no LLM) ----------------
- knowledge_graph_result = None
- try:
- from repowise.core.analysis.knowledge_graph import (
- KnowledgeGraphResult,
- build_knowledge_graph_skeleton,
- compute_kg_fingerprint,
- should_skip_kg_rebuild,
- )
-
- new_fingerprint = compute_kg_fingerprint(graph_builder)
-
- kg_json_path = repo_path / ".repowise" / "knowledge-graph.json"
- if should_skip_kg_rebuild(existing_kg_fingerprint, new_fingerprint, kg_json_path):
- knowledge_graph_result = KnowledgeGraphResult.from_file(kg_json_path)
- if knowledge_graph_result is not None:
- knowledge_graph_result.fingerprint = new_fingerprint
- logger.info(
- "knowledge_graph.skip",
- reason="fingerprint_unchanged",
- fingerprint=new_fingerprint,
- )
- if progress:
- progress.on_message(
- "info",
- f" ↳ KG unchanged (fingerprint {new_fingerprint[:8]}…), reusing",
- )
- else:
- # The fingerprint said the file was reusable but the reader
- # refused it — unreadable, or written by an older schema. The
- # rebuild below is correct either way, but it used to happen
- # in total silence, so a user whose hand-edited file was
- # rejected got no signal at all.
- logger.info(
- "knowledge_graph.reuse_failed",
- reason="artifact_unreadable_or_outdated",
- path=str(kg_json_path),
- )
- if progress:
- progress.on_message(
- "info",
- " ↳ knowledge-graph.json could not be reused, rebuilding it",
- )
-
- tech_stack_dicts = [
- {"name": t.name, "version": t.version, "category": t.category} for t in tech_items
- ]
-
- if knowledge_graph_result is None:
- if progress:
- progress.on_phase_start("knowledge_graph.skeleton", None)
- knowledge_graph_result = build_knowledge_graph_skeleton(
- parsed_files=parsed_files,
- graph_builder=graph_builder,
- repo_structure=repo_structure,
- tech_stack=tech_stack_dicts,
- external_systems=external_systems,
- git_meta_map=git_meta_map,
- dead_code_report=gen_dead_code_report,
- repo_path=repo_path,
- )
- knowledge_graph_result.fingerprint = new_fingerprint
- if progress:
- progress.on_message(
- "info",
- f" ↳ KG skeleton: {len(knowledge_graph_result.nodes)} nodes, "
- f"{len(knowledge_graph_result.edges)} edges, "
- f"{len(knowledge_graph_result.layers)} layers",
- )
- _phase_done(progress, "knowledge_graph.skeleton")
-
- # ---- KG curation/presentation pass (flagged, default on) ---------
- # Reshapes only the exported KG (layers/tour/entry-points/summaries);
- # never touches the AST graph, communities, or centrality. No-op when
- # REPOWISE_KG_CURATION is set to a falsy value (the raw uncurated
- # export). Runs in BOTH FAST and STANDARD (before the generate branch).
- if knowledge_graph_result is not None:
- from repowise.core.analysis.kg_curation import (
- curate_knowledge_graph,
- curation_enabled,
- )
-
- try:
- # In generate mode the summary floor is deferred to run after
- # the wiki-page backfill (in ``finalize_knowledge_graph``), so
- # rich page summaries win; FAST mode floors here.
- will_generate = generate_docs and llm_client is not None
- with timed(getattr(progress, "table", None), "knowledge_graph.curate"):
- knowledge_graph_result = curate_knowledge_graph(
- knowledge_graph_result,
- parsed_files=parsed_files,
- graph_builder=graph_builder,
- repo_structure=repo_structure,
- community_info=graph_builder.community_info(),
- git_meta_map=git_meta_map,
- enabled=curation_enabled(),
- defer_summary_floor=will_generate,
- )
- except (ValueError, KeyError, RuntimeError) as cur_err:
- logger.error("kg_curation_failed", error=str(cur_err), exc_info=True)
- except (ValueError, KeyError, OSError, RuntimeError) as kg_err:
- logger.error("kg_skeleton_building_failed", error=str(kg_err), exc_info=True)
-
- # ---- Checkpoint: ANALYSIS ----------------------------------------------
- # Persist dead code + health + decisions now that the analysis phase is
- # complete, so an interrupt during the long generation phase below can
- # resume past analysis instead of recomputing it. Skipped when we already
- # rehydrated analysis (it's by definition persisted) — best-effort.
- #
- # The store goes with them: this is where a decision record is first
- # written, so it is the only pass that can fold a paraphrase into an
- # existing one. By the end-of-run persist every group matches on title.
- if resume_controller is not None and not skip_analysis:
- # No progress phase covers this write either; time it like the INDEX
- # checkpoint so it shows in the totals.
- with timed(getattr(progress, "table", None), "persist.checkpoint_analysis"):
- await resume_controller.checkpoint_analysis(
- parsed_files=parsed_files,
- dead_code_report=dead_code_report,
- health_report=health_report,
- decision_report=decision_report,
- doc_drift_report=doc_drift_report,
- git_metadata_list=git_metadata_list,
- vector_store=vector_store,
- progress=progress,
- )
+ await run_analysis_phase(run)
+ parsed_files = run.parsed_files
+ dead_code_report = run.dead_code_report
+ health_report = run.health_report
+ decision_report = run.decision_report
+ doc_drift_report = run.doc_drift_report
+ gen_dead_code_report = run.gen_dead_code_report
+ gen_decision_report = run.gen_decision_report
+ tech_stack_dicts = run.tech_stack_dicts
+ knowledge_graph_result = run.knowledge_graph_result
# ---- Phase 3: Generation (optional) ------------------------------------
generated_pages: list[Any] | None = None