Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(pipeline): run state, setup and the index phase out of run_pipeline (#4073)
Raghav Chamadiya478e81fbb995
diff --git a/packages/core/src/repowise/core/pipeline/_run/__init__.py b/packages/core/src/repowise/core/pipeline/_run/__init__.pynew file mode 100644index 000000000..2a2d9c81f--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/_run/__init__.py@@ -0,0 +1 @@+"""Stage modules behind ``run_pipeline``; one shared run state threads through them."""diff --git a/packages/core/src/repowise/core/pipeline/_run/index.py b/packages/core/src/repowise/core/pipeline/_run/index.pynew file mode 100644index 000000000..108cf8073--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/_run/index.py@@ -0,0 +1,191 @@+"""Index phase: ingestion and git history, external systems, the INDEX checkpoint."""++from __future__ import annotations++import asyncio++import structlog++# Phase functions are looked up on the orchestrator at call time, so a patch on+# ``repowise.core.pipeline.orchestrator`` reaches this module too.+from repowise.core.pipeline import orchestrator as _orch+from repowise.core.pipeline.phases._common import _phase_done+from repowise.core.pipeline.progress import STAGE_INGESTION, emit_stage, emit_warning+from repowise.core.pipeline.resume import ResumePhase++from .state import PipelineRun++logger = structlog.get_logger("repowise.core.pipeline.orchestrator")+++async def run_index_phase(run: PipelineRun) -> None:+ """Fill the index outputs of *run*, from a resumed checkpoint when one exists."""+ emit_stage(run.progress, STAGE_INGESTION)+ run.skip_index = await _rehydrate_index(run)+ if not run.skip_index:+ await _compute_index(run)+ run.external_systems = await _extract_external_systems(run)+ # Persist now so an interrupted analysis can resume past the index; a+ # rehydrated index is already persisted.+ if run.resume_controller is not None and not run.skip_index:+ await run.resume_controller.checkpoint_index(+ parsed_files=run.parsed_files,+ graph_builder=run.graph_builder,+ git_metadata_list=run.git_metadata_list,+ git_summary=run.git_summary,+ external_systems=run.external_systems,+ source_map=run.source_map,+ progress=run.progress,+ )+ _emit_index_summary(run)+++async def _rehydrate_index(run: PipelineRun) -> bool:+ """Reuse a persisted graph + git index and re-parse source only; False to compute.++ Skips the git history walk and the graph centrality kernels, the slow part of a+ first index. Any rehydration error falls back to a full compute.+ """+ controller = run.resume_controller+ if not (controller and await controller.can_skip(ResumePhase.INDEX)):+ return False+ progress = run.progress+ try:+ run.graph_builder, run.git_meta_map = await controller.rehydrate_index(run.repo_path)+ if progress:+ progress.on_message("info", " ↳ Resuming — reusing persisted graph + git index")+ (+ run.parsed_files,+ run.file_infos,+ run.repo_structure,+ run.source_map,+ run.tech_items,+ ) = await _orch.reparse_for_resume(+ run.repo_path,+ exclude_patterns=run.exclude_patterns,+ include_submodules=run.include_submodules,+ include_nested_repos=run.include_nested_repos,+ skip_tests=run.skip_tests,+ skip_infra=run.skip_infra,+ progress=progress,+ derive_environment_facts=run.derive_environment_facts,+ )+ run.traversal_stats = None+ run.graph_builder.restore_parse_only_attrs(run.parsed_files)+ # Rehydrated rows can predate the structural label, and nothing+ # else on this path recomputes it.+ _orch.label_co_change_structure(run.graph_builder, run.git_meta_map)+ run.git_metadata_list = list(run.git_meta_map.values())+ except Exception as exc:+ logger.warning("resume_rehydrate_failed_recomputing", error=str(exc))+ return False+ return True+++async def _compute_index(run: PipelineRun) -> None:+ """Parse + build the graph and walk git history concurrently (they share no data).++ Rehydrated graphs already carry co-change edges, so only this path adds them.+ """+ (+ (+ run.parsed_files,+ run.file_infos,+ run.repo_structure,+ run.source_map,+ run.graph_builder,+ run.traversal_stats,+ run.tech_items,+ ),+ (+ run.git_summary,+ run.git_metadata_list,+ run.git_meta_map,+ ),+ ) = await asyncio.gather(+ _orch._run_ingestion(+ run.repo_path,+ exclude_patterns=run.exclude_patterns,+ include_submodules=run.include_submodules,+ include_nested_repos=run.include_nested_repos,+ skip_tests=run.skip_tests,+ skip_infra=run.skip_infra,+ progress=run.progress,+ derive_environment_facts=run.derive_environment_facts,+ ),+ _orch._run_git_indexing(+ run.repo_path,+ commit_depth=run.commit_depth,+ follow_renames=run.follow_renames,+ tier=run.git_tier,+ exclude_patterns=run.exclude_patterns,+ progress=run.progress,+ ),+ )+ if run.git_meta_map:+ _orch.label_co_change_structure(run.graph_builder, run.git_meta_map)+ run.graph_builder.add_co_change_edges(run.git_meta_map)+++async def _extract_external_systems(run: PipelineRun) -> list[dict]:+ """Declared third-party dependencies from repo manifests (C4 L1); empty on failure."""+ progress = run.progress+ external_systems: list[dict] = []+ if progress:+ progress.on_phase_start("external_systems", None)+ try:+ from repowise.core.ingestion.external_systems import extract_external_systems++ records = await asyncio.to_thread(extract_external_systems, run.repo_path)+ external_systems = [+ {+ "name": r.name,+ "display_name": r.display_name,+ "ecosystem": r.ecosystem,+ "category": r.category,+ "io_kind": r.io_kind,+ "version": r.version,+ "declared_in": r.declared_in,+ "is_dev_dep": r.is_dev_dep,+ }+ for r in records+ ]+ if progress and external_systems:+ progress.on_message(+ "info",+ f"→ External systems: {len(external_systems):,} declared deps across manifests",+ )+ except Exception as _ext_err:+ logger.warning("external_systems_extraction_failed", error=str(_ext_err))+ emit_warning(+ progress,+ f"External dependency manifests not parsed ({_ext_err}); "+ "pages will not name this repo's third-party systems.",+ )+ _phase_done(progress, "external_systems")+ return external_systems+++def _emit_index_summary(run: PipelineRun) -> None:+ progress = run.progress+ if not progress:+ return+ _g = run.graph_builder.graph()+ progress.on_message(+ "info",+ f"→ {len(run.parsed_files):,} files parsed · "+ f"{sum(len(pf.symbols) for pf in run.parsed_files):,} symbols extracted",+ )+ progress.on_message(+ "info",+ f"→ Graph: {_g.number_of_nodes():,} nodes · {_g.number_of_edges():,} edges",+ )+ git_summary = run.git_summary+ if git_summary and git_summary.files_indexed:+ _hotspot_msg = ""+ if hasattr(git_summary, "hotspots") and git_summary.hotspots:+ _hotspot_msg = f" · {git_summary.hotspots} hotspots"+ progress.on_message(+ "info",+ f"→ Git: {git_summary.files_indexed:,} files indexed{_hotspot_msg}",+ )diff --git a/packages/core/src/repowise/core/pipeline/_run/state.py b/packages/core/src/repowise/core/pipeline/_run/state.pynew file mode 100644index 000000000..252f4e715--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/_run/state.py@@ -0,0 +1,83 @@+"""Run state for one ``run_pipeline`` call and the setup steps that build it."""++from __future__ import annotations++from dataclasses import dataclass, field+from pathlib import Path+from typing import Any++from repowise.core.pipeline.modes import OrchestratorMode+from repowise.core.registry import HookProgressCallback+from repowise.core.registry.plugins import load_plugins+++@dataclass+class PipelineRun:+ """Options of one pipeline run plus the outputs each stage leaves for the next."""++ repo_path: Path+ start: float+ mode: OrchestratorMode+ generate_docs: bool+ git_tier: Any+ progress: Any+ checkpointer: Any+ commit_depth: int+ follow_renames: bool+ skip_tests: bool+ skip_infra: bool+ exclude_patterns: list[str] | None+ include_submodules: bool+ include_nested_repos: bool+ derive_environment_facts: bool+ llm_client: Any+ embedder: Any+ vector_store: Any+ concurrency: int+ test_run: bool+ resume: bool+ generation_config: Any+ wiki_style: str | None+ existing_kg_fingerprint: str | None+ on_page_ready: Any+ resume_controller: Any+ coverage_report_paths: list[Path] | None++ # Index phase outputs.+ skip_index: bool = False+ parsed_files: list[Any] = field(default_factory=list)+ file_infos: list[Any] = field(default_factory=list)+ repo_structure: Any = None+ source_map: dict[str, bytes] = field(default_factory=dict)+ graph_builder: Any = None+ traversal_stats: Any = None+ tech_items: list[Any] = field(default_factory=list)+ git_summary: Any = None+ git_metadata_list: list[dict] = field(default_factory=list)+ git_meta_map: dict[str, dict] = field(default_factory=dict)+ external_systems: list[dict] = field(default_factory=list)+++def hooked_progress(progress: Any) -> HookProgressCallback:+ """Wrap *progress* so registered pipeline hooks fire around each phase.++ Plugins load first so the hooks they register are among them.+ """+ load_plugins()+ return HookProgressCallback(progress)+++async def start_checkpointer(job_store: Any, repo_path: Path) -> Any:+ """Start phase checkpointing for crash-resume when a JobStore is supplied, else None."""+ if job_store is None:+ return None+ from repowise.core.pipeline.checkpoint import PhaseCheckpointer++ checkpointer = PhaseCheckpointer(job_store, str(repo_path))+ await checkpointer.start()+ return checkpointer+++def attach_cost_tracker(llm_client: Any, cost_tracker: Any) -> None:+ if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):+ llm_client._cost_tracker = cost_trackerdiff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 6604be739..0dc4ddfa2 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -27,16 +27,18 @@ from repowise.core.ingestion.git_indexer._constants import _DEFAULT_COMMIT_LIMITfrom repowise.core.pipeline.modes import OrchestratorModefrom repowise.core.pipeline.progress import (STAGE_ANALYSIS,- STAGE_INGESTION,ProgressCallback,emit_stage,emit_warning,)-from repowise.core.registry import HookProgressCallback-from repowise.core.registry.plugins import load_plugins+from ._run.index import run_index_phase+from ._run.state import PipelineRun, attach_cost_tracker, hooked_progress, start_checkpointerfrom .phase_timing import timedfrom .phases._common import TEST_RUN_FILE_LIMIT, _phase_done, limit_to_top_pagerank++# The ``_run`` stage modules call these phase functions through this module+# (``_orch.<name>``), so imports and patches on the orchestrator keep reaching them.from .phases.analysis import (_run_dead_code_analysis,_run_decision_extraction,@@ -45,11 +47,11 @@ from .phases.analysis import ()from .phases.generation import run_generationfrom .phases.git import (- _run_git_indexing,+ _run_git_indexing, # noqa: F401drop_transient_git_signals,- label_co_change_structure,+ label_co_change_structure, # noqa: F401)-from .phases.ingestion import _run_ingestion, reparse_for_resume+from .phases.ingestion import _run_ingestion, reparse_for_resume # noqa: F401from .resume import ResumePhasefrom .resume.controller import ResumeController@@ -270,204 +272,55 @@ async def run_pipeline("""repo_path = Path(repo_path).resolve()start = time.monotonic()-- commit_depth = max(1, min(commit_depth, 10000))-- # Mode policy: FAST forces ESSENTIAL git indexing and disables doc- # generation (and therefore all LLM calls). DETERMINISTIC generates every- # page from a template and supplies its own null provider, so a caller- # with no API key still gets a wiki. STANDARD preserves the caller's flags- # exactly. This is the single switch point: the rest of the pipeline- # reads ``generate_docs`` / the git tier, not ``mode``.- git_tier = mode.git_tier- generate_docs = generate_docs and mode.allows_doc_generation-- # Wrap the incoming progress callback so registered pipeline hooks fire- # around each phase transition. Zero-op when no hooks are registered.- # Plugins load first so the hooks they register are among them.- load_plugins()- progress = HookProgressCallback(progress)-- # Optional crash-resume checkpointing. When a JobStore is supplied, a- # checkpointer records each major phase's lifecycle (via the same hook- # seam) so a re-run can detect completed phases. No-op otherwise — the- # default (None) preserves existing behaviour exactly.- _checkpointer = None- if job_store is not None:- from repowise.core.pipeline.checkpoint import PhaseCheckpointer-- _checkpointer = PhaseCheckpointer(job_store, str(repo_path))- await _checkpointer.start()-- # Attach cost tracker to provider if supplied- if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):- llm_client._cost_tracker = cost_tracker-- # ---- Phase 1: Ingestion ------------------------------------------------- emit_stage(progress, STAGE_INGESTION)-- # Launch git indexing as a background task immediately — it is independent- # of parsing and graph-build, so the two stages can run concurrently.- # _run_ingestion does: traverse → ProcessPool parse → graph build → dynamic hints.- # _run_git_indexing does: git log → co-change accumulation (I/O bound, own executor).-- async def _git_stage() -> tuple:- return await _run_git_indexing(- repo_path,- commit_depth=commit_depth,- follow_renames=follow_renames,- tier=git_tier,- exclude_patterns=exclude_patterns,- progress=progress,- )-- async def _ingestion_stage() -> tuple:- return await _run_ingestion(- repo_path,- exclude_patterns=exclude_patterns,- include_submodules=include_submodules,- include_nested_repos=include_nested_repos,- skip_tests=skip_tests,- skip_infra=skip_infra,- progress=progress,- derive_environment_facts=derive_environment_facts,- )-- # Resume fast-path: when a prior run already persisted the INDEX phase- # (graph + git), rehydrate it from the DB and only re-parse source files —- # skipping the git history walk and the graph centrality kernels, which- # are the minutes-long work that makes a first index slow. Falls back to a- # full compute if rehydration yields no graph (nothing was persisted).- skip_index = bool(resume_controller and await resume_controller.can_skip(ResumePhase.INDEX))- git_summary = None- if skip_index:- try:- graph_builder, git_meta_map = await resume_controller.rehydrate_index(repo_path)- if progress:- progress.on_message("info", " ↳ Resuming — reusing persisted graph + git index")- (- parsed_files,- file_infos,- repo_structure,- source_map,- tech_items,- ) = await reparse_for_resume(- repo_path,- exclude_patterns=exclude_patterns,- include_submodules=include_submodules,- include_nested_repos=include_nested_repos,- skip_tests=skip_tests,- skip_infra=skip_infra,- progress=progress,- derive_environment_facts=derive_environment_facts,- )- traversal_stats = None- graph_builder.restore_parse_only_attrs(parsed_files)- # Rehydrated rows can predate the structural label, and nothing- # else on this path recomputes it.- label_co_change_structure(graph_builder, git_meta_map)- git_metadata_list = list(git_meta_map.values())- except Exception as exc:- logger.warning("resume_rehydrate_failed_recomputing", error=str(exc))- skip_index = False-- if not skip_index:- (- (- parsed_files,- file_infos,- repo_structure,- source_map,- graph_builder,- traversal_stats,- tech_items,- ),- (- git_summary,- git_metadata_list,- git_meta_map,- ),- ) = await asyncio.gather(_ingestion_stage(), _git_stage())-- # Add co-change edges to the graph (rehydrated graphs already carry them)- if git_meta_map:- label_co_change_structure(graph_builder, git_meta_map)- graph_builder.add_co_change_edges(git_meta_map)-- # ---- External systems (C4 L1) ------------------------------------------- # Parse repo manifests for declared third-party dependencies. Failure here- # must not break the pipeline — log and continue with an empty list.- external_systems: list[dict] = []- if progress:- progress.on_phase_start("external_systems", None)- try:- from repowise.core.ingestion.external_systems import extract_external_systems-- records = await asyncio.to_thread(extract_external_systems, repo_path)- external_systems = [- {- "name": r.name,- "display_name": r.display_name,- "ecosystem": r.ecosystem,- "category": r.category,- "io_kind": r.io_kind,- "version": r.version,- "declared_in": r.declared_in,- "is_dev_dep": r.is_dev_dep,- }- for r in records- ]- if progress and external_systems:- progress.on_message(- "info",- f"→ External systems: {len(external_systems):,} declared deps across manifests",- )- except Exception as _ext_err:- logger.warning("external_systems_extraction_failed", error=str(_ext_err))- emit_warning(- progress,- f"External dependency manifests not parsed ({_ext_err}); "- "pages will not name this repo's third-party systems.",- )- _phase_done(progress, "external_systems")-- # ---- Checkpoint: INDEX -------------------------------------------------- # Persist the freshly-computed graph + git + symbols now so an interrupt- # during the analysis phase below can resume without redoing the expensive- # index. Skipped when we rehydrated (already persisted) — best-effort.- if resume_controller is not None and not skip_index:- await resume_controller.checkpoint_index(- parsed_files=parsed_files,- graph_builder=graph_builder,- git_metadata_list=git_metadata_list,- git_summary=git_summary,- external_systems=external_systems,- source_map=source_map,- progress=progress,- )-- # Emit rich insight summary for the ingestion phase- if progress:- _g = graph_builder.graph()- _n_nodes = _g.number_of_nodes()- _n_edges = _g.number_of_edges()- progress.on_message(- "info",- f"→ {len(parsed_files):,} files parsed · "- f"{sum(len(pf.symbols) for pf in parsed_files):,} symbols extracted",- )- progress.on_message(- "info",- f"→ Graph: {_n_nodes:,} nodes · {_n_edges:,} edges",- )- if git_summary and git_summary.files_indexed:- _hotspot_msg = ""- if hasattr(git_summary, "hotspots") and git_summary.hotspots:- _hotspot_msg = f" · {git_summary.hotspots} hotspots"- progress.on_message(- "info",- f"→ Git: {git_summary.files_indexed:,} files indexed{_hotspot_msg}",- )+ progress = hooked_progress(progress)+ checkpointer = await start_checkpointer(job_store, repo_path)+ attach_cost_tracker(llm_client, cost_tracker)+ run = PipelineRun(+ repo_path=repo_path,+ start=start,+ mode=mode,+ # Mode policy, the single switch point: FAST forces ESSENTIAL git indexing and+ # disables doc generation (so every LLM call); DETERMINISTIC renders pages from+ # templates; STANDARD keeps the caller's flags. Later stages read these, not mode.+ generate_docs=generate_docs and mode.allows_doc_generation,+ git_tier=mode.git_tier,+ progress=progress,+ checkpointer=checkpointer,+ commit_depth=max(1, min(commit_depth, 10000)),+ follow_renames=follow_renames,+ skip_tests=skip_tests,+ skip_infra=skip_infra,+ exclude_patterns=exclude_patterns,+ include_submodules=include_submodules,+ include_nested_repos=include_nested_repos,+ derive_environment_facts=derive_environment_facts,+ llm_client=llm_client,+ embedder=embedder,+ vector_store=vector_store,+ concurrency=concurrency,+ test_run=test_run,+ resume=resume,+ generation_config=generation_config,+ wiki_style=wiki_style,+ existing_kg_fingerprint=existing_kg_fingerprint,+ on_page_ready=on_page_ready,+ resume_controller=resume_controller,+ coverage_report_paths=coverage_report_paths,+ )+ generate_docs = run.generate_docs++ await run_index_phase(run)+ parsed_files = run.parsed_files+ file_infos = run.file_infos+ repo_structure = run.repo_structure+ source_map = run.source_map+ graph_builder = run.graph_builder+ traversal_stats = run.traversal_stats+ tech_items = run.tech_items+ git_summary = run.git_summary+ git_metadata_list = run.git_metadata_list+ git_meta_map = run.git_meta_map+ external_systems = run.external_systems# Test-run: limit to top 10 files by PageRank (shared helper with# ``run_generation`` so the two paths cannot drift).@@ -963,8 +816,8 @@ async def run_pipeline(# Flush checkpoints on the success path; on an exception above this point# the checkpointer is intentionally not closed so interrupted phases stay# RUNNING for a resumable re-run.- if _checkpointer is not None:- await _checkpointer.aclose()+ if checkpointer is not None:+ await checkpointer.aclose()elapsed = time.monotonic() - startlanguages = {fi.language for fi in file_infos if hasattr(fi, "language") and fi.language}