Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(pipeline): analysis phase and KG skeleton out of run_pipeline (#4078)
Raghav Chamadiyae881a1b7c348
diff --git a/packages/core/src/repowise/core/pipeline/_run/analysis.py b/packages/core/src/repowise/core/pipeline/_run/analysis.pynew file mode 100644index 000000000..695d1a331--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/_run/analysis.py@@ -0,0 +1,277 @@+"""Analysis phase: the four analyses or their resumed views, the KG skeleton, the ANALYSIS checkpoint."""++from __future__ import annotations++import asyncio+from typing import Any++import structlog++# Phase functions are looked up on the orchestrator at call time, so a patch on+# ``repowise.core.pipeline.orchestrator`` reaches this module too.+from repowise.core.pipeline import orchestrator as _orch+from repowise.core.pipeline.phase_timing import timed+from repowise.core.pipeline.phases._common import (+ TEST_RUN_FILE_LIMIT,+ _phase_done,+ limit_to_top_pagerank,+)+from repowise.core.pipeline.progress import STAGE_ANALYSIS, emit_stage+from repowise.core.pipeline.resume import ResumePhase++from .state import PipelineRun++logger = structlog.get_logger("repowise.core.pipeline.orchestrator")+++async def run_analysis_phase(run: PipelineRun) -> None:+ """Fill the analysis outputs and the KG skeleton of *run*, then checkpoint them."""+ _limit_test_run(run)+ emit_stage(run.progress, STAGE_ANALYSIS)+ run.skip_analysis = await _rehydrate_analysis(run)+ if not run.skip_analysis:+ await _compute_analysis(run)+ _build_knowledge_graph(run)+ if run.resume_controller is not None and not run.skip_analysis:+ await _checkpoint_analysis(run)+++def _limit_test_run(run: PipelineRun) -> None:+ # Same helper as ``run_generation`` so the two ``--test-run`` paths cannot drift.+ if not (run.test_run and run.generate_docs):+ return+ run.parsed_files = limit_to_top_pagerank(+ run.parsed_files, run.graph_builder, n=TEST_RUN_FILE_LIMIT+ )+ if run.progress:+ run.progress.on_message("warning", f"Test run: limiting to {len(run.parsed_files)} files")+++async def _rehydrate_analysis(run: PipelineRun) -> bool:+ """Reuse the persisted analysis views generation reads; False to recompute.++ Only the thin generation inputs come back (``health_report`` stays None); the+ persisted rows stay authoritative and are never re-written from these.+ """+ controller = run.resume_controller+ if not (controller and await controller.can_skip(ResumePhase.ANALYSIS)):+ return False+ try:+ (+ run.gen_dead_code_report,+ run.gen_decision_report,+ ) = await controller.rehydrate_analysis()+ if run.progress:+ run.progress.on_message("info", " ↳ Resuming — reusing persisted analysis")+ except Exception as exc:+ logger.warning("resume_rehydrate_analysis_failed_recomputing", error=str(exc))+ return False+ if not await controller.has_completed_decision_extraction():+ await _backfill_decisions(run)+ return True+++async def _backfill_decisions(run: PipelineRun) -> None:+ """Run decision extraction alone when a resumed analysis never finished it."""+ if run.progress:+ run.progress.on_message(+ "info",+ " ↳ Resumed analysis missing decision extraction — running extraction",+ )+ run.decision_report = await _orch._run_decision_extraction(+ run.repo_path,+ llm_client=run.llm_client,+ graph_builder=run.graph_builder,+ git_meta_map=run.git_meta_map,+ parsed_files=run.parsed_files,+ source_map=run.source_map,+ progress=run.progress,+ )+ if run.decision_report is not None:+ run.gen_decision_report = run.decision_report+ await run.resume_controller.checkpoint_decision_backfill(+ run.decision_report, progress=run.progress+ )+++async def _compute_analysis(run: PipelineRun) -> None:+ """Run dead code, health, decisions and doc drift concurrently.++ They share read-only inputs and no data, so the LLM-bound decision extraction+ hides behind the CPU-bound dead-code and health work.+ """+ progress = run.progress+ (+ run.dead_code_report,+ run.health_report,+ run.decision_report,+ run.doc_drift_report,+ ) = await asyncio.gather(+ _orch._run_dead_code_analysis(+ run.graph_builder,+ run.git_meta_map,+ source_map=run.source_map,+ repo_path=run.repo_path,+ traversal_stats=run.traversal_stats,+ progress=progress,+ ),+ _orch._run_health_analysis(+ run.graph_builder,+ run.git_meta_map,+ run.parsed_files,+ repo_path=run.repo_path,+ coverage_report_paths=run.coverage_report_paths,+ progress=progress,+ ),+ _orch._run_decision_extraction(+ run.repo_path,+ llm_client=run.llm_client,+ graph_builder=run.graph_builder,+ git_meta_map=run.git_meta_map,+ parsed_files=run.parsed_files,+ source_map=run.source_map,+ progress=progress,+ ),+ _orch._run_doc_drift_analysis(+ run.source_map,+ file_infos=run.file_infos,+ graph_builder=run.graph_builder,+ repo_path=run.repo_path,+ progress=progress,+ ),+ )+ # Health reads the in-memory BlameIndex while it runs; drop it only after the+ # gather, before it can leak into the result or persistence.+ _orch.drop_transient_git_signals(run.git_metadata_list)+ run.gen_dead_code_report = run.dead_code_report+ run.gen_decision_report = run.decision_report+++def _build_knowledge_graph(run: PipelineRun) -> None:+ """Reuse or build the deterministic KG skeleton, then curate it; failures only log."""+ try:+ from repowise.core.analysis.knowledge_graph import (+ KnowledgeGraphResult,+ build_knowledge_graph_skeleton,+ compute_kg_fingerprint,+ should_skip_kg_rebuild,+ )++ new_fingerprint = compute_kg_fingerprint(run.graph_builder)+ kg_json_path = run.repo_path / ".repowise" / "knowledge-graph.json"+ if should_skip_kg_rebuild(run.existing_kg_fingerprint, new_fingerprint, kg_json_path):+ _reuse_kg_artifact(run, KnowledgeGraphResult, kg_json_path, new_fingerprint)++ run.tech_stack_dicts = [+ {"name": t.name, "version": t.version, "category": t.category} for t in run.tech_items+ ]+ if run.knowledge_graph_result is None:+ _build_kg_skeleton(run, build_knowledge_graph_skeleton, new_fingerprint)+ _phase_done(run.progress, "knowledge_graph.skeleton")+ if run.knowledge_graph_result is not None:+ _curate_knowledge_graph(run)+ except (ValueError, KeyError, OSError, RuntimeError) as kg_err:+ logger.error("kg_skeleton_building_failed", error=str(kg_err), exc_info=True)+++def _reuse_kg_artifact(+ run: PipelineRun, result_cls: Any, kg_json_path: Any, new_fingerprint: str+) -> None:+ progress = run.progress+ run.knowledge_graph_result = result_cls.from_file(kg_json_path)+ if run.knowledge_graph_result is not None:+ run.knowledge_graph_result.fingerprint = new_fingerprint+ logger.info(+ "knowledge_graph.skip",+ reason="fingerprint_unchanged",+ fingerprint=new_fingerprint,+ )+ if progress:+ progress.on_message(+ "info",+ f" ↳ KG unchanged (fingerprint {new_fingerprint[:8]}…), reusing",+ )+ return+ # The reader refused a file the fingerprint called reusable (unreadable or an+ # older schema): say so before the rebuild.+ logger.info(+ "knowledge_graph.reuse_failed",+ reason="artifact_unreadable_or_outdated",+ path=str(kg_json_path),+ )+ if progress:+ progress.on_message(+ "info",+ " ↳ knowledge-graph.json could not be reused, rebuilding it",+ )+++def _build_kg_skeleton(run: PipelineRun, build_skeleton: Any, new_fingerprint: str) -> None:+ progress = run.progress+ if progress:+ progress.on_phase_start("knowledge_graph.skeleton", None)+ run.knowledge_graph_result = build_skeleton(+ parsed_files=run.parsed_files,+ graph_builder=run.graph_builder,+ repo_structure=run.repo_structure,+ tech_stack=run.tech_stack_dicts,+ external_systems=run.external_systems,+ git_meta_map=run.git_meta_map,+ dead_code_report=run.gen_dead_code_report,+ repo_path=run.repo_path,+ )+ run.knowledge_graph_result.fingerprint = new_fingerprint+ if progress:+ kg = run.knowledge_graph_result+ progress.on_message(+ "info",+ f" ↳ KG skeleton: {len(kg.nodes)} nodes, "+ f"{len(kg.edges)} edges, "+ f"{len(kg.layers)} layers",+ )+++def _curate_knowledge_graph(run: PipelineRun) -> None:+ """Reshape the exported KG only (layers, tour, entry points, summaries).++ Never touches the AST graph, communities or centrality; a no-op when+ ``REPOWISE_KG_CURATION`` is falsy. Runs in every mode, before generation.+ """+ from repowise.core.analysis.kg_curation import curate_knowledge_graph, curation_enabled++ try:+ # Generate mode defers the summary floor until after the wiki-page+ # backfill so rich page summaries win; other runs floor here.+ will_generate = run.generate_docs and run.llm_client is not None+ with timed(getattr(run.progress, "table", None), "knowledge_graph.curate"):+ run.knowledge_graph_result = curate_knowledge_graph(+ run.knowledge_graph_result,+ parsed_files=run.parsed_files,+ graph_builder=run.graph_builder,+ repo_structure=run.repo_structure,+ community_info=run.graph_builder.community_info(),+ git_meta_map=run.git_meta_map,+ enabled=curation_enabled(),+ defer_summary_floor=will_generate,+ )+ except (ValueError, KeyError, RuntimeError) as cur_err:+ logger.error("kg_curation_failed", error=str(cur_err), exc_info=True)+++async def _checkpoint_analysis(run: PipelineRun) -> None:+ """Persist the analysis so a resume skips it; the first write of each decision.++ The vector store goes with it: only this pass can fold a paraphrased decision+ into an existing one, since by the end-of-run persist every group matches on title.+ """+ with timed(getattr(run.progress, "table", None), "persist.checkpoint_analysis"):+ await run.resume_controller.checkpoint_analysis(+ parsed_files=run.parsed_files,+ dead_code_report=run.dead_code_report,+ health_report=run.health_report,+ decision_report=run.decision_report,+ doc_drift_report=run.doc_drift_report,+ git_metadata_list=run.git_metadata_list,+ vector_store=run.vector_store,+ progress=run.progress,+ )diff --git a/packages/core/src/repowise/core/pipeline/_run/state.py b/packages/core/src/repowise/core/pipeline/_run/state.pyindex 252f4e715..50bd4eca0 100644--- a/packages/core/src/repowise/core/pipeline/_run/state.py+++ b/packages/core/src/repowise/core/pipeline/_run/state.py@@ -57,6 +57,18 @@ class PipelineRun:git_meta_map: dict[str, dict] = field(default_factory=dict)external_systems: list[dict] = field(default_factory=list)+ # Analysis phase outputs. The ``gen_`` reports feed generation and the KG:+ # rehydrated on a resumed run, the freshly computed ones otherwise.+ skip_analysis: bool = False+ dead_code_report: Any = None+ health_report: Any = None+ decision_report: Any = None+ doc_drift_report: Any = None+ gen_dead_code_report: Any = None+ gen_decision_report: Any = None+ tech_stack_dicts: list[dict] = field(default_factory=list)+ knowledge_graph_result: Any = None+def hooked_progress(progress: Any) -> HookProgressCallback:"""Wrap *progress* so registered pipeline hooks fire around each phase.diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 0dc4ddfa2..1925b99af 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -25,20 +25,16 @@ import structlogfrom repowise.core.ingestion.git_indexer._constants import _DEFAULT_COMMIT_LIMITfrom repowise.core.pipeline.modes import OrchestratorMode-from repowise.core.pipeline.progress import (- STAGE_ANALYSIS,- ProgressCallback,- emit_stage,- emit_warning,-)+from repowise.core.pipeline.progress import ProgressCallback, emit_warning+from ._run.analysis import run_analysis_phasefrom ._run.index import run_index_phasefrom ._run.state import PipelineRun, attach_cost_tracker, hooked_progress, start_checkpointer-from .phase_timing import timed-from .phases._common import TEST_RUN_FILE_LIMIT, _phase_done, limit_to_top_pagerank+from .phases._common import _phase_done# The ``_run`` stage modules call these phase functions through this module# (``_orch.<name>``), so imports and patches on the orchestrator keep reaching them.+# ruff: noqa: F401from .phases.analysis import (_run_dead_code_analysis,_run_decision_extraction,@@ -47,12 +43,11 @@ from .phases.analysis import ()from .phases.generation import run_generationfrom .phases.git import (- _run_git_indexing, # noqa: F401+ _run_git_indexing,drop_transient_git_signals,- label_co_change_structure, # noqa: F401+ label_co_change_structure,)-from .phases.ingestion import _run_ingestion, reparse_for_resume # noqa: F401-from .resume import ResumePhase+from .phases.ingestion import _run_ingestion, reparse_for_resumefrom .resume.controller import ResumeControllerlogger = structlog.get_logger(__name__)@@ -322,258 +317,16 @@ async def run_pipeline(git_meta_map = run.git_meta_mapexternal_systems = run.external_systems- # Test-run: limit to top 10 files by PageRank (shared helper with- # ``run_generation`` so the two paths cannot drift).- if test_run and generate_docs:- parsed_files = limit_to_top_pagerank(- parsed_files, graph_builder, n=TEST_RUN_FILE_LIMIT- )- if progress:- progress.on_message("warning", f"Test run: limiting to {len(parsed_files)} files")-- # ---- Phase 2: Analysis --------------------------------------------------- emit_stage(progress, STAGE_ANALYSIS)-- # Resume fast-path: when a prior run already completed (and persisted) the- # ANALYSIS phase, skip recomputing dead code / health / decisions — the- # last is the costly one (LLM-backed, minutes on large repos). We rehydrate- # only the thin views generation reads; the persisted rows stay- # authoritative and are never re-written from these. ``health_report`` is- # not a generation input, so it stays None on this path (its persisted rows- # are untouched). Falls back to a full recompute if rehydration errors.- skip_analysis = bool(- resume_controller and await resume_controller.can_skip(ResumePhase.ANALYSIS)- )- dead_code_report = None- health_report = None- decision_report = None- doc_drift_report = None- # Reports actually fed to generation + KG — rehydrated on the skip path,- # the freshly computed ones otherwise.- gen_dead_code_report = None- gen_decision_report = None- if skip_analysis:- try:- (- gen_dead_code_report,- gen_decision_report,- ) = await resume_controller.rehydrate_analysis()- if progress:- progress.on_message("info", " ↳ Resuming — reusing persisted analysis")- except Exception as exc:- logger.warning("resume_rehydrate_analysis_failed_recomputing", error=str(exc))- skip_analysis = False-- # If analysis was skipped, but decision extraction never completed (e.g. earlier- # run was interrupted during decision extraction or returned None), re-run decision- # extraction specifically so that decisions are backfilled without re-running dead code / health.- if (- skip_analysis- and resume_controller- and not await resume_controller.has_completed_decision_extraction()- ):- if progress:- progress.on_message(- "info",- " ↳ Resumed analysis missing decision extraction — running extraction",- )- decision_report = await _run_decision_extraction(- repo_path,- llm_client=llm_client,- graph_builder=graph_builder,- git_meta_map=git_meta_map,- parsed_files=parsed_files,- source_map=source_map,- progress=progress,- )- if decision_report is not None:- gen_decision_report = decision_report- await resume_controller.checkpoint_decision_backfill(- decision_report, progress=progress- )-- if not skip_analysis:- # The four analyses share read-only inputs (graph, git_meta_map,- # parsed_files; the lazy metric caches were warmed during ingestion)- # and have no data dependency on each other, so run them concurrently:- # decision extraction is I/O/LLM-bound and its wall clock hides- # entirely behind the CPU-bound dead-code + health work.- (- dead_code_report,- health_report,- decision_report,- doc_drift_report,- ) = await asyncio.gather(- _run_dead_code_analysis(- graph_builder,- git_meta_map,- source_map=source_map,- repo_path=repo_path,- traversal_stats=traversal_stats,- progress=progress,- ),- _run_health_analysis(- graph_builder,- git_meta_map,- parsed_files,- repo_path=repo_path,- coverage_report_paths=coverage_report_paths,- progress=progress,- ),- _run_decision_extraction(- repo_path,- llm_client=llm_client,- graph_builder=graph_builder,- git_meta_map=git_meta_map,- parsed_files=parsed_files,- source_map=source_map,- progress=progress,- ),- _run_doc_drift_analysis(- source_map,- file_infos=file_infos,- graph_builder=graph_builder,- repo_path=repo_path,- progress=progress,- ),- )-- # Drop the in-memory-only ``BlameIndex`` now that the health biomarkers- # have consumed it — before it can leak into ``PipelineResult`` and the- # downstream JSON artifact writers / DB persistence. ``git_meta_map``- # shares these dict objects, so this cleans both views. Must stay after- # the gather: health reads the blame index while it runs (decision- # extraction never does).- drop_transient_git_signals(git_metadata_list)- gen_dead_code_report = dead_code_report- gen_decision_report = decision_report-- # ---- Knowledge Graph skeleton (deterministic, no LLM) ----------------- knowledge_graph_result = None- try:- from repowise.core.analysis.knowledge_graph import (- KnowledgeGraphResult,- build_knowledge_graph_skeleton,- compute_kg_fingerprint,- should_skip_kg_rebuild,- )-- new_fingerprint = compute_kg_fingerprint(graph_builder)-- kg_json_path = repo_path / ".repowise" / "knowledge-graph.json"- if should_skip_kg_rebuild(existing_kg_fingerprint, new_fingerprint, kg_json_path):- knowledge_graph_result = KnowledgeGraphResult.from_file(kg_json_path)- if knowledge_graph_result is not None:- knowledge_graph_result.fingerprint = new_fingerprint- logger.info(- "knowledge_graph.skip",- reason="fingerprint_unchanged",- fingerprint=new_fingerprint,- )- if progress:- progress.on_message(- "info",- f" ↳ KG unchanged (fingerprint {new_fingerprint[:8]}…), reusing",- )- else:- # The fingerprint said the file was reusable but the reader- # refused it — unreadable, or written by an older schema. The- # rebuild below is correct either way, but it used to happen- # in total silence, so a user whose hand-edited file was- # rejected got no signal at all.- logger.info(- "knowledge_graph.reuse_failed",- reason="artifact_unreadable_or_outdated",- path=str(kg_json_path),- )- if progress:- progress.on_message(- "info",- " ↳ knowledge-graph.json could not be reused, rebuilding it",- )-- tech_stack_dicts = [- {"name": t.name, "version": t.version, "category": t.category} for t in tech_items- ]-- if knowledge_graph_result is None:- if progress:- progress.on_phase_start("knowledge_graph.skeleton", None)- knowledge_graph_result = build_knowledge_graph_skeleton(- parsed_files=parsed_files,- graph_builder=graph_builder,- repo_structure=repo_structure,- tech_stack=tech_stack_dicts,- external_systems=external_systems,- git_meta_map=git_meta_map,- dead_code_report=gen_dead_code_report,- repo_path=repo_path,- )- knowledge_graph_result.fingerprint = new_fingerprint- if progress:- progress.on_message(- "info",- f" ↳ KG skeleton: {len(knowledge_graph_result.nodes)} nodes, "- f"{len(knowledge_graph_result.edges)} edges, "- f"{len(knowledge_graph_result.layers)} layers",- )- _phase_done(progress, "knowledge_graph.skeleton")-- # ---- KG curation/presentation pass (flagged, default on) ---------- # Reshapes only the exported KG (layers/tour/entry-points/summaries);- # never touches the AST graph, communities, or centrality. No-op when- # REPOWISE_KG_CURATION is set to a falsy value (the raw uncurated- # export). Runs in BOTH FAST and STANDARD (before the generate branch).- if knowledge_graph_result is not None:- from repowise.core.analysis.kg_curation import (- curate_knowledge_graph,- curation_enabled,- )-- try:- # In generate mode the summary floor is deferred to run after- # the wiki-page backfill (in ``finalize_knowledge_graph``), so- # rich page summaries win; FAST mode floors here.- will_generate = generate_docs and llm_client is not None- with timed(getattr(progress, "table", None), "knowledge_graph.curate"):- knowledge_graph_result = curate_knowledge_graph(- knowledge_graph_result,- parsed_files=parsed_files,- graph_builder=graph_builder,- repo_structure=repo_structure,- community_info=graph_builder.community_info(),- git_meta_map=git_meta_map,- enabled=curation_enabled(),- defer_summary_floor=will_generate,- )- except (ValueError, KeyError, RuntimeError) as cur_err:- logger.error("kg_curation_failed", error=str(cur_err), exc_info=True)- except (ValueError, KeyError, OSError, RuntimeError) as kg_err:- logger.error("kg_skeleton_building_failed", error=str(kg_err), exc_info=True)-- # ---- Checkpoint: ANALYSIS ----------------------------------------------- # Persist dead code + health + decisions now that the analysis phase is- # complete, so an interrupt during the long generation phase below can- # resume past analysis instead of recomputing it. Skipped when we already- # rehydrated analysis (it's by definition persisted) — best-effort.- #- # The store goes with them: this is where a decision record is first- # written, so it is the only pass that can fold a paraphrase into an- # existing one. By the end-of-run persist every group matches on title.- if resume_controller is not None and not skip_analysis:- # No progress phase covers this write either; time it like the INDEX- # checkpoint so it shows in the totals.- with timed(getattr(progress, "table", None), "persist.checkpoint_analysis"):- await resume_controller.checkpoint_analysis(- parsed_files=parsed_files,- dead_code_report=dead_code_report,- health_report=health_report,- decision_report=decision_report,- doc_drift_report=doc_drift_report,- git_metadata_list=git_metadata_list,- vector_store=vector_store,- progress=progress,- )+ await run_analysis_phase(run)+ parsed_files = run.parsed_files+ dead_code_report = run.dead_code_report+ health_report = run.health_report+ decision_report = run.decision_report+ doc_drift_report = run.doc_drift_report+ gen_dead_code_report = run.gen_dead_code_report+ gen_decision_report = run.gen_decision_report+ tech_stack_dicts = run.tech_stack_dicts+ knowledge_graph_result = run.knowledge_graph_result# ---- Phase 3: Generation (optional) ------------------------------------generated_pages: list[Any] | None = None