Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(pipeline): run state, setup and the index phase out of run_pipeline (#4073)

Raghav Chamadiya478e81fbb995
diff --git a/packages/core/src/repowise/core/pipeline/_run/__init__.py b/packages/core/src/repowise/core/pipeline/_run/__init__.py
new file mode 100644
index 000000000..2a2d9c81f
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/_run/__init__.py
@@ -0,0 +1 @@
+"""Stage modules behind ``run_pipeline``; one shared run state threads through them."""
diff --git a/packages/core/src/repowise/core/pipeline/_run/index.py b/packages/core/src/repowise/core/pipeline/_run/index.py
new file mode 100644
index 000000000..108cf8073
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/_run/index.py
@@ -0,0 +1,191 @@
+"""Index phase: ingestion and git history, external systems, the INDEX checkpoint."""
+
+from __future__ import annotations
+
+import asyncio
+
+import structlog
+
+# Phase functions are looked up on the orchestrator at call time, so a patch on
+# ``repowise.core.pipeline.orchestrator`` reaches this module too.
+from repowise.core.pipeline import orchestrator as _orch
+from repowise.core.pipeline.phases._common import _phase_done
+from repowise.core.pipeline.progress import STAGE_INGESTION, emit_stage, emit_warning
+from repowise.core.pipeline.resume import ResumePhase
+
+from .state import PipelineRun
+
+logger = structlog.get_logger("repowise.core.pipeline.orchestrator")
+
+
+async def run_index_phase(run: PipelineRun) -> None:
+ """Fill the index outputs of *run*, from a resumed checkpoint when one exists."""
+ emit_stage(run.progress, STAGE_INGESTION)
+ run.skip_index = await _rehydrate_index(run)
+ if not run.skip_index:
+ await _compute_index(run)
+ run.external_systems = await _extract_external_systems(run)
+ # Persist now so an interrupted analysis can resume past the index; a
+ # rehydrated index is already persisted.
+ if run.resume_controller is not None and not run.skip_index:
+ await run.resume_controller.checkpoint_index(
+ parsed_files=run.parsed_files,
+ graph_builder=run.graph_builder,
+ git_metadata_list=run.git_metadata_list,
+ git_summary=run.git_summary,
+ external_systems=run.external_systems,
+ source_map=run.source_map,
+ progress=run.progress,
+ )
+ _emit_index_summary(run)
+
+
+async def _rehydrate_index(run: PipelineRun) -> bool:
+ """Reuse a persisted graph + git index and re-parse source only; False to compute.
+
+ Skips the git history walk and the graph centrality kernels, the slow part of a
+ first index. Any rehydration error falls back to a full compute.
+ """
+ controller = run.resume_controller
+ if not (controller and await controller.can_skip(ResumePhase.INDEX)):
+ return False
+ progress = run.progress
+ try:
+ run.graph_builder, run.git_meta_map = await controller.rehydrate_index(run.repo_path)
+ if progress:
+ progress.on_message("info", " ↳ Resuming — reusing persisted graph + git index")
+ (
+ run.parsed_files,
+ run.file_infos,
+ run.repo_structure,
+ run.source_map,
+ run.tech_items,
+ ) = await _orch.reparse_for_resume(
+ run.repo_path,
+ exclude_patterns=run.exclude_patterns,
+ include_submodules=run.include_submodules,
+ include_nested_repos=run.include_nested_repos,
+ skip_tests=run.skip_tests,
+ skip_infra=run.skip_infra,
+ progress=progress,
+ derive_environment_facts=run.derive_environment_facts,
+ )
+ run.traversal_stats = None
+ run.graph_builder.restore_parse_only_attrs(run.parsed_files)
+ # Rehydrated rows can predate the structural label, and nothing
+ # else on this path recomputes it.
+ _orch.label_co_change_structure(run.graph_builder, run.git_meta_map)
+ run.git_metadata_list = list(run.git_meta_map.values())
+ except Exception as exc:
+ logger.warning("resume_rehydrate_failed_recomputing", error=str(exc))
+ return False
+ return True
+
+
+async def _compute_index(run: PipelineRun) -> None:
+ """Parse + build the graph and walk git history concurrently (they share no data).
+
+ Rehydrated graphs already carry co-change edges, so only this path adds them.
+ """
+ (
+ (
+ run.parsed_files,
+ run.file_infos,
+ run.repo_structure,
+ run.source_map,
+ run.graph_builder,
+ run.traversal_stats,
+ run.tech_items,
+ ),
+ (
+ run.git_summary,
+ run.git_metadata_list,
+ run.git_meta_map,
+ ),
+ ) = await asyncio.gather(
+ _orch._run_ingestion(
+ run.repo_path,
+ exclude_patterns=run.exclude_patterns,
+ include_submodules=run.include_submodules,
+ include_nested_repos=run.include_nested_repos,
+ skip_tests=run.skip_tests,
+ skip_infra=run.skip_infra,
+ progress=run.progress,
+ derive_environment_facts=run.derive_environment_facts,
+ ),
+ _orch._run_git_indexing(
+ run.repo_path,
+ commit_depth=run.commit_depth,
+ follow_renames=run.follow_renames,
+ tier=run.git_tier,
+ exclude_patterns=run.exclude_patterns,
+ progress=run.progress,
+ ),
+ )
+ if run.git_meta_map:
+ _orch.label_co_change_structure(run.graph_builder, run.git_meta_map)
+ run.graph_builder.add_co_change_edges(run.git_meta_map)
+
+
+async def _extract_external_systems(run: PipelineRun) -> list[dict]:
+ """Declared third-party dependencies from repo manifests (C4 L1); empty on failure."""
+ progress = run.progress
+ external_systems: list[dict] = []
+ if progress:
+ progress.on_phase_start("external_systems", None)
+ try:
+ from repowise.core.ingestion.external_systems import extract_external_systems
+
+ records = await asyncio.to_thread(extract_external_systems, run.repo_path)
+ external_systems = [
+ {
+ "name": r.name,
+ "display_name": r.display_name,
+ "ecosystem": r.ecosystem,
+ "category": r.category,
+ "io_kind": r.io_kind,
+ "version": r.version,
+ "declared_in": r.declared_in,
+ "is_dev_dep": r.is_dev_dep,
+ }
+ for r in records
+ ]
+ if progress and external_systems:
+ progress.on_message(
+ "info",
+ f"→ External systems: {len(external_systems):,} declared deps across manifests",
+ )
+ except Exception as _ext_err:
+ logger.warning("external_systems_extraction_failed", error=str(_ext_err))
+ emit_warning(
+ progress,
+ f"External dependency manifests not parsed ({_ext_err}); "
+ "pages will not name this repo's third-party systems.",
+ )
+ _phase_done(progress, "external_systems")
+ return external_systems
+
+
+def _emit_index_summary(run: PipelineRun) -> None:
+ progress = run.progress
+ if not progress:
+ return
+ _g = run.graph_builder.graph()
+ progress.on_message(
+ "info",
+ f"→ {len(run.parsed_files):,} files parsed · "
+ f"{sum(len(pf.symbols) for pf in run.parsed_files):,} symbols extracted",
+ )
+ progress.on_message(
+ "info",
+ f"→ Graph: {_g.number_of_nodes():,} nodes · {_g.number_of_edges():,} edges",
+ )
+ git_summary = run.git_summary
+ if git_summary and git_summary.files_indexed:
+ _hotspot_msg = ""
+ if hasattr(git_summary, "hotspots") and git_summary.hotspots:
+ _hotspot_msg = f" · {git_summary.hotspots} hotspots"
+ progress.on_message(
+ "info",
+ f"→ Git: {git_summary.files_indexed:,} files indexed{_hotspot_msg}",
+ )
diff --git a/packages/core/src/repowise/core/pipeline/_run/state.py b/packages/core/src/repowise/core/pipeline/_run/state.py
new file mode 100644
index 000000000..252f4e715
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/_run/state.py
@@ -0,0 +1,83 @@
+"""Run state for one ``run_pipeline`` call and the setup steps that build it."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from repowise.core.pipeline.modes import OrchestratorMode
+from repowise.core.registry import HookProgressCallback
+from repowise.core.registry.plugins import load_plugins
+
+
+@dataclass
+class PipelineRun:
+ """Options of one pipeline run plus the outputs each stage leaves for the next."""
+
+ repo_path: Path
+ start: float
+ mode: OrchestratorMode
+ generate_docs: bool
+ git_tier: Any
+ progress: Any
+ checkpointer: Any
+ commit_depth: int
+ follow_renames: bool
+ skip_tests: bool
+ skip_infra: bool
+ exclude_patterns: list[str] | None
+ include_submodules: bool
+ include_nested_repos: bool
+ derive_environment_facts: bool
+ llm_client: Any
+ embedder: Any
+ vector_store: Any
+ concurrency: int
+ test_run: bool
+ resume: bool
+ generation_config: Any
+ wiki_style: str | None
+ existing_kg_fingerprint: str | None
+ on_page_ready: Any
+ resume_controller: Any
+ coverage_report_paths: list[Path] | None
+
+ # Index phase outputs.
+ skip_index: bool = False
+ parsed_files: list[Any] = field(default_factory=list)
+ file_infos: list[Any] = field(default_factory=list)
+ repo_structure: Any = None
+ source_map: dict[str, bytes] = field(default_factory=dict)
+ graph_builder: Any = None
+ traversal_stats: Any = None
+ tech_items: list[Any] = field(default_factory=list)
+ git_summary: Any = None
+ git_metadata_list: list[dict] = field(default_factory=list)
+ git_meta_map: dict[str, dict] = field(default_factory=dict)
+ external_systems: list[dict] = field(default_factory=list)
+
+
+def hooked_progress(progress: Any) -> HookProgressCallback:
+ """Wrap *progress* so registered pipeline hooks fire around each phase.
+
+ Plugins load first so the hooks they register are among them.
+ """
+ load_plugins()
+ return HookProgressCallback(progress)
+
+
+async def start_checkpointer(job_store: Any, repo_path: Path) -> Any:
+ """Start phase checkpointing for crash-resume when a JobStore is supplied, else None."""
+ if job_store is None:
+ return None
+ from repowise.core.pipeline.checkpoint import PhaseCheckpointer
+
+ checkpointer = PhaseCheckpointer(job_store, str(repo_path))
+ await checkpointer.start()
+ return checkpointer
+
+
+def attach_cost_tracker(llm_client: Any, cost_tracker: Any) -> None:
+ if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):
+ llm_client._cost_tracker = cost_tracker
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 6604be739..0dc4ddfa2 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -27,16 +27,18 @@ from repowise.core.ingestion.git_indexer._constants import _DEFAULT_COMMIT_LIMIT
from repowise.core.pipeline.modes import OrchestratorMode
from repowise.core.pipeline.progress import (
STAGE_ANALYSIS,
- STAGE_INGESTION,
ProgressCallback,
emit_stage,
emit_warning,
)
-from repowise.core.registry import HookProgressCallback
-from repowise.core.registry.plugins import load_plugins
+from ._run.index import run_index_phase
+from ._run.state import PipelineRun, attach_cost_tracker, hooked_progress, start_checkpointer
from .phase_timing import timed
from .phases._common import TEST_RUN_FILE_LIMIT, _phase_done, limit_to_top_pagerank
+
+# The ``_run`` stage modules call these phase functions through this module
+# (``_orch.<name>``), so imports and patches on the orchestrator keep reaching them.
from .phases.analysis import (
_run_dead_code_analysis,
_run_decision_extraction,
@@ -45,11 +47,11 @@ from .phases.analysis import (
)
from .phases.generation import run_generation
from .phases.git import (
- _run_git_indexing,
+ _run_git_indexing, # noqa: F401
drop_transient_git_signals,
- label_co_change_structure,
+ label_co_change_structure, # noqa: F401
)
-from .phases.ingestion import _run_ingestion, reparse_for_resume
+from .phases.ingestion import _run_ingestion, reparse_for_resume # noqa: F401
from .resume import ResumePhase
from .resume.controller import ResumeController
@@ -270,204 +272,55 @@ async def run_pipeline(
"""
repo_path = Path(repo_path).resolve()
start = time.monotonic()
-
- commit_depth = max(1, min(commit_depth, 10000))
-
- # Mode policy: FAST forces ESSENTIAL git indexing and disables doc
- # generation (and therefore all LLM calls). DETERMINISTIC generates every
- # page from a template and supplies its own null provider, so a caller
- # with no API key still gets a wiki. STANDARD preserves the caller's flags
- # exactly. This is the single switch point: the rest of the pipeline
- # reads ``generate_docs`` / the git tier, not ``mode``.
- git_tier = mode.git_tier
- generate_docs = generate_docs and mode.allows_doc_generation
-
- # Wrap the incoming progress callback so registered pipeline hooks fire
- # around each phase transition. Zero-op when no hooks are registered.
- # Plugins load first so the hooks they register are among them.
- load_plugins()
- progress = HookProgressCallback(progress)
-
- # Optional crash-resume checkpointing. When a JobStore is supplied, a
- # checkpointer records each major phase's lifecycle (via the same hook
- # seam) so a re-run can detect completed phases. No-op otherwise — the
- # default (None) preserves existing behaviour exactly.
- _checkpointer = None
- if job_store is not None:
- from repowise.core.pipeline.checkpoint import PhaseCheckpointer
-
- _checkpointer = PhaseCheckpointer(job_store, str(repo_path))
- await _checkpointer.start()
-
- # Attach cost tracker to provider if supplied
- if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):
- llm_client._cost_tracker = cost_tracker
-
- # ---- Phase 1: Ingestion ------------------------------------------------
- emit_stage(progress, STAGE_INGESTION)
-
- # Launch git indexing as a background task immediately — it is independent
- # of parsing and graph-build, so the two stages can run concurrently.
- # _run_ingestion does: traverse → ProcessPool parse → graph build → dynamic hints.
- # _run_git_indexing does: git log → co-change accumulation (I/O bound, own executor).
-
- async def _git_stage() -> tuple:
- return await _run_git_indexing(
- repo_path,
- commit_depth=commit_depth,
- follow_renames=follow_renames,
- tier=git_tier,
- exclude_patterns=exclude_patterns,
- progress=progress,
- )
-
- async def _ingestion_stage() -> tuple:
- return await _run_ingestion(
- repo_path,
- exclude_patterns=exclude_patterns,
- include_submodules=include_submodules,
- include_nested_repos=include_nested_repos,
- skip_tests=skip_tests,
- skip_infra=skip_infra,
- progress=progress,
- derive_environment_facts=derive_environment_facts,
- )
-
- # Resume fast-path: when a prior run already persisted the INDEX phase
- # (graph + git), rehydrate it from the DB and only re-parse source files —
- # skipping the git history walk and the graph centrality kernels, which
- # are the minutes-long work that makes a first index slow. Falls back to a
- # full compute if rehydration yields no graph (nothing was persisted).
- skip_index = bool(resume_controller and await resume_controller.can_skip(ResumePhase.INDEX))
- git_summary = None
- if skip_index:
- try:
- graph_builder, git_meta_map = await resume_controller.rehydrate_index(repo_path)
- if progress:
- progress.on_message("info", " ↳ Resuming — reusing persisted graph + git index")
- (
- parsed_files,
- file_infos,
- repo_structure,
- source_map,
- tech_items,
- ) = await reparse_for_resume(
- repo_path,
- exclude_patterns=exclude_patterns,
- include_submodules=include_submodules,
- include_nested_repos=include_nested_repos,
- skip_tests=skip_tests,
- skip_infra=skip_infra,
- progress=progress,
- derive_environment_facts=derive_environment_facts,
- )
- traversal_stats = None
- graph_builder.restore_parse_only_attrs(parsed_files)
- # Rehydrated rows can predate the structural label, and nothing
- # else on this path recomputes it.
- label_co_change_structure(graph_builder, git_meta_map)
- git_metadata_list = list(git_meta_map.values())
- except Exception as exc:
- logger.warning("resume_rehydrate_failed_recomputing", error=str(exc))
- skip_index = False
-
- if not skip_index:
- (
- (
- parsed_files,
- file_infos,
- repo_structure,
- source_map,
- graph_builder,
- traversal_stats,
- tech_items,
- ),
- (
- git_summary,
- git_metadata_list,
- git_meta_map,
- ),
- ) = await asyncio.gather(_ingestion_stage(), _git_stage())
-
- # Add co-change edges to the graph (rehydrated graphs already carry them)
- if git_meta_map:
- label_co_change_structure(graph_builder, git_meta_map)
- graph_builder.add_co_change_edges(git_meta_map)
-
- # ---- External systems (C4 L1) ------------------------------------------
- # Parse repo manifests for declared third-party dependencies. Failure here
- # must not break the pipeline — log and continue with an empty list.
- external_systems: list[dict] = []
- if progress:
- progress.on_phase_start("external_systems", None)
- try:
- from repowise.core.ingestion.external_systems import extract_external_systems
-
- records = await asyncio.to_thread(extract_external_systems, repo_path)
- external_systems = [
- {
- "name": r.name,
- "display_name": r.display_name,
- "ecosystem": r.ecosystem,
- "category": r.category,
- "io_kind": r.io_kind,
- "version": r.version,
- "declared_in": r.declared_in,
- "is_dev_dep": r.is_dev_dep,
- }
- for r in records
- ]
- if progress and external_systems:
- progress.on_message(
- "info",
- f"→ External systems: {len(external_systems):,} declared deps across manifests",
- )
- except Exception as _ext_err:
- logger.warning("external_systems_extraction_failed", error=str(_ext_err))
- emit_warning(
- progress,
- f"External dependency manifests not parsed ({_ext_err}); "
- "pages will not name this repo's third-party systems.",
- )
- _phase_done(progress, "external_systems")
-
- # ---- Checkpoint: INDEX -------------------------------------------------
- # Persist the freshly-computed graph + git + symbols now so an interrupt
- # during the analysis phase below can resume without redoing the expensive
- # index. Skipped when we rehydrated (already persisted) — best-effort.
- if resume_controller is not None and not skip_index:
- await resume_controller.checkpoint_index(
- parsed_files=parsed_files,
- graph_builder=graph_builder,
- git_metadata_list=git_metadata_list,
- git_summary=git_summary,
- external_systems=external_systems,
- source_map=source_map,
- progress=progress,
- )
-
- # Emit rich insight summary for the ingestion phase
- if progress:
- _g = graph_builder.graph()
- _n_nodes = _g.number_of_nodes()
- _n_edges = _g.number_of_edges()
- progress.on_message(
- "info",
- f"→ {len(parsed_files):,} files parsed · "
- f"{sum(len(pf.symbols) for pf in parsed_files):,} symbols extracted",
- )
- progress.on_message(
- "info",
- f"→ Graph: {_n_nodes:,} nodes · {_n_edges:,} edges",
- )
- if git_summary and git_summary.files_indexed:
- _hotspot_msg = ""
- if hasattr(git_summary, "hotspots") and git_summary.hotspots:
- _hotspot_msg = f" · {git_summary.hotspots} hotspots"
- progress.on_message(
- "info",
- f"→ Git: {git_summary.files_indexed:,} files indexed{_hotspot_msg}",
- )
+ progress = hooked_progress(progress)
+ checkpointer = await start_checkpointer(job_store, repo_path)
+ attach_cost_tracker(llm_client, cost_tracker)
+ run = PipelineRun(
+ repo_path=repo_path,
+ start=start,
+ mode=mode,
+ # Mode policy, the single switch point: FAST forces ESSENTIAL git indexing and
+ # disables doc generation (so every LLM call); DETERMINISTIC renders pages from
+ # templates; STANDARD keeps the caller's flags. Later stages read these, not mode.
+ generate_docs=generate_docs and mode.allows_doc_generation,
+ git_tier=mode.git_tier,
+ progress=progress,
+ checkpointer=checkpointer,
+ commit_depth=max(1, min(commit_depth, 10000)),
+ follow_renames=follow_renames,
+ skip_tests=skip_tests,
+ skip_infra=skip_infra,
+ exclude_patterns=exclude_patterns,
+ include_submodules=include_submodules,
+ include_nested_repos=include_nested_repos,
+ derive_environment_facts=derive_environment_facts,
+ llm_client=llm_client,
+ embedder=embedder,
+ vector_store=vector_store,
+ concurrency=concurrency,
+ test_run=test_run,
+ resume=resume,
+ generation_config=generation_config,
+ wiki_style=wiki_style,
+ existing_kg_fingerprint=existing_kg_fingerprint,
+ on_page_ready=on_page_ready,
+ resume_controller=resume_controller,
+ coverage_report_paths=coverage_report_paths,
+ )
+ generate_docs = run.generate_docs
+
+ await run_index_phase(run)
+ parsed_files = run.parsed_files
+ file_infos = run.file_infos
+ repo_structure = run.repo_structure
+ source_map = run.source_map
+ graph_builder = run.graph_builder
+ traversal_stats = run.traversal_stats
+ tech_items = run.tech_items
+ git_summary = run.git_summary
+ git_metadata_list = run.git_metadata_list
+ git_meta_map = run.git_meta_map
+ external_systems = run.external_systems
# Test-run: limit to top 10 files by PageRank (shared helper with
# ``run_generation`` so the two paths cannot drift).
@@ -963,8 +816,8 @@ async def run_pipeline(
# Flush checkpoints on the success path; on an exception above this point
# the checkpointer is intentionally not closed so interrupted phases stay
# RUNNING for a resumable re-run.
- if _checkpointer is not None:
- await _checkpointer.aclose()
+ if checkpointer is not None:
+ await checkpointer.aclose()
elapsed = time.monotonic() - start
languages = {fi.language for fi in file_infos if hasattr(fi, "language") and fi.language}