Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(pipeline): extract phase functions into pipeline/phases/ (#257)

Swati Ahuja9d51656b95a2
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index ca62d454b..b1a50866b 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -15,103 +15,28 @@ Callers:
from __future__ import annotations
import asyncio
-import os
import time
-from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
-from dataclasses import dataclass, field, replace
+from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import structlog
-from repowise.core.ingestion.git_indexer import GitIndexTier
from repowise.core.pipeline.modes import OrchestratorMode
from repowise.core.pipeline.progress import ProgressCallback
from repowise.core.registry import HookProgressCallback
-logger = structlog.get_logger(__name__)
-
-
-def _phase_done(progress: ProgressCallback | None, phase: str) -> None:
- """Best-effort call to ``progress.on_phase_done`` — older callbacks may
- not implement it, so fall back to a no-op silently.
- """
- if progress is None:
- return
- fn = getattr(progress, "on_phase_done", None)
- if callable(fn):
- try:
- fn(phase)
- except Exception:
- pass
-
-
-async def _timed_step(
- label: str,
- fn: Any,
- progress: ProgressCallback | None,
-) -> Any:
- """Run *fn* in a worker thread and emit a per-step completion line.
-
- Used to make ``asyncio.gather`` of multiple graph-algorithm calls
- legible: without this, four concurrent thread-bound computations
- (e.g. PageRank + betweenness + symbol PageRank + symbol betweenness)
- appear in the CLI as one opaque several-minute spinner, so the user
- has no signal as to which step is the bottleneck. With it, each algo
- prints ` ↳ <label> ✓ (Xs)` as it finishes — completion order
- surfaces the relative cost without changing the underlying execution.
- """
- t0 = time.monotonic()
- try:
- result = await asyncio.to_thread(fn)
- except Exception as exc:
- if progress is not None:
- progress.on_message(
- "warning",
- f" ↳ {label} failed after {time.monotonic() - t0:.1f}s: {exc}",
- )
- raise
- if progress is not None:
- progress.on_message(
- "info",
- f" ↳ {label} ✓ ({time.monotonic() - t0:.1f}s)",
- )
- return result
+from .phases._common import _phase_done
+from .phases.analysis import (
+ _run_dead_code_analysis,
+ _run_decision_extraction,
+ _run_health_analysis,
+)
+from .phases.generation import run_generation
+from .phases.git import _run_git_indexing
+from .phases.ingestion import _run_ingestion
-
-# ---------------------------------------------------------------------------
-# Process-pool worker (module-level — must be picklable)
-# ---------------------------------------------------------------------------
-
-# Module-level process-local parser cache (one per worker process).
-_WORKER_PARSER: Any = None
-
-
-def _parse_one(path_and_fi_and_bytes: tuple) -> Any:
- """Worker function for ProcessPoolExecutor parsing.
-
- Constructs (or reuses) a process-local ASTParser and parses one file.
- Returns a ParsedFile on success, or (abs_path_str, error_str) on failure.
- The parser is constructed lazily inside the worker — the ASTParser itself
- (which holds compiled tree-sitter Language/Query objects) is never pickled.
- Only FileInfo (input) and ParsedFile (output) cross the process boundary;
- both are plain dataclasses and therefore picklable.
- """
- global _WORKER_PARSER
- fi, source = path_and_fi_and_bytes
- try:
- if _WORKER_PARSER is None:
- from repowise.core.ingestion import ASTParser
-
- _WORKER_PARSER = ASTParser()
- return _WORKER_PARSER.parse_file(fi, source)
- except Exception as exc:
- return (fi.abs_path, str(exc))
-
-
-# Maximum seconds to spend on decision extraction before giving up.
-# Large repos with tens of thousands of files can take arbitrarily long.
-DECISION_EXTRACTION_TIMEOUT_SECS = 300
+logger = structlog.get_logger(__name__)
# ---------------------------------------------------------------------------
@@ -638,670 +563,3 @@ async def run_pipeline(
# ---------------------------------------------------------------------------
# Phase helpers (private)
# ---------------------------------------------------------------------------
-
-
-async def _run_ingestion(
- repo_path: Path,
- *,
- exclude_patterns: list[str] | None,
- include_submodules: bool = False,
- include_nested_repos: bool = False,
- skip_tests: bool,
- skip_infra: bool,
- progress: ProgressCallback | None,
-) -> tuple[list[Any], list[Any], Any, dict[str, bytes], Any, Any]:
- """Traverse, parse, and build the dependency graph.
-
- Returns (parsed_files, file_infos, repo_structure, source_map,
- graph_builder, traversal_stats, tech_items).
- """
- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
-
- traverser = FileTraverser(
- repo_path,
- extra_exclude_patterns=exclude_patterns or None,
- include_submodules=include_submodules,
- include_nested_repos=include_nested_repos,
- )
-
- # Walk directory tree
- all_paths = list(traverser._walk())
- if progress:
- # Use indeterminate progress (spinner) to avoid showing a misleading
- # pre-filter total like "2132/83601".
- progress.on_phase_start("traverse", None)
-
- # Parallel stat + header reads (I/O bound).
- # Use asyncio.wrap_future so the event loop stays responsive while waiting.
- file_infos: list[Any] = []
- io_pool = ThreadPoolExecutor(max_workers=8)
- try:
- aws = [
- asyncio.wrap_future(io_pool.submit(traverser._build_file_info, p)) for p in all_paths
- ]
- for coro in asyncio.as_completed(aws):
- try:
- result = await coro
- except Exception:
- result = None
- if result is not None:
- file_infos.append(result)
- if progress:
- progress.on_item_done("traverse")
- finally:
- # shutdown(wait=True) is blocking — run in a thread to keep the
- # event loop responsive. All submitted futures have already
- # completed by the time we reach here (the for-loop awaited them).
- await asyncio.to_thread(io_pool.shutdown, wait=True)
-
- repo_structure = traverser.get_repo_structure(file_infos)
- _phase_done(progress, "traverse")
-
- # Filter
- if skip_tests:
- file_infos = [fi for fi in file_infos if not fi.is_test]
- if skip_infra:
- file_infos = [
- fi
- for fi in file_infos
- if fi.language not in ("dockerfile", "makefile", "terraform", "shell")
- ]
-
- # ---- Parse phase: CPU-bound, run in ProcessPoolExecutor ----------------
- if progress:
- progress.on_phase_start("parse", len(file_infos))
-
- # Read source bytes up front (I/O, sequential — fast enough; keeps worker
- # args small: FileInfo + bytes, both picklable plain dataclasses/bytes).
- fi_and_bytes: list[tuple] = []
- for fi in file_infos:
- try:
- source = Path(fi.abs_path).read_bytes()
- fi_and_bytes.append((fi, source))
- except Exception:
- if progress:
- progress.on_item_done("parse")
-
- parsed_files: list[Any] = []
- source_map: dict[str, bytes] = {}
- graph_builder = GraphBuilder(repo_path=repo_path)
-
- loop = asyncio.get_running_loop()
- workers = max(1, os.cpu_count() or 4)
-
- _use_process_pool = True
- parse_results: list[Any] = []
-
- try:
- with ProcessPoolExecutor(max_workers=workers) as pool:
- tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]
- # Tick the parse-progress bar as each worker finishes —
- # ``asyncio.gather`` would otherwise hold every event back
- # until the last file is done, which on PowerToys-scale
- # repos looked like a hang at ``0/N`` for many minutes.
- # Per-task done-callbacks fire on the event loop thread and
- # preserve gather's ordered results, so the aggregation
- # loop below still indexes ``fi_and_bytes`` correctly.
- if progress is not None:
- _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
- for fut in tasks:
- fut.add_done_callback(_parse_tick)
- parse_results = await asyncio.gather(*tasks, return_exceptions=True)
- except Exception as pool_exc:
- logger.warning(
- "process_pool_parse_failed_falling_back",
- error=str(pool_exc),
- )
- _use_process_pool = False
- # Fallback: in-process sequential parse
- _fallback_parser = ASTParser()
- for i, (fi, source) in enumerate(fi_and_bytes):
- try:
- result = _fallback_parser.parse_file(fi, source)
- parse_results.append(result)
- except Exception as exc:
- parse_results.append((fi.abs_path, str(exc)))
- if progress:
- progress.on_item_done("parse")
- if i % 50 == 49:
- await asyncio.sleep(0)
-
- # Aggregate results into GraphBuilder on the main loop (not thread-safe).
- for idx, result in enumerate(parse_results):
- fi, source = fi_and_bytes[idx]
- if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):
- # Error tuple: (abs_path_str, error_str)
- logger.debug("parse_error_in_worker", path=result[0], error=result[1])
- elif isinstance(result, Exception):
- logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))
- else:
- parsed_files.append(result)
- source_map[fi.path] = source
- graph_builder.add_file(result)
- # Process-pool path already ticked per-file via the done-callback
- # attached above; only the fallback path ticks here (handled in
- # its own loop). No tick needed in aggregation.
-
- _phase_done(progress, "parse")
-
- # ---- tsconfig path-alias resolver (before graph build) ------------------
- # Only runs when the repo has TS/JS files. On large TS monorepos the
- # resolver indexes hundreds of tsconfig files up-front; without a phase
- # label this shows up as a silent gap right after parsing.
- try:
- from repowise.core.ingestion.tsconfig_resolver import TsconfigResolver
-
- _ts_langs = {"typescript", "javascript"}
- if any(pf.file_info.language in _ts_langs for pf in parsed_files):
- if progress:
- progress.on_phase_start("tsconfig", None)
- _path_set = set(graph_builder._parsed_files.keys())
- _resolver = TsconfigResolver(repo_path=repo_path, path_set=_path_set)
- graph_builder.set_tsconfig_resolver(_resolver)
- _phase_done(progress, "tsconfig")
- except Exception as _resolver_exc:
- logger.warning("tsconfig_resolver_init_failed", error=str(_resolver_exc))
-
- # ---- Graph build phase -------------------------------------------------
- # Sub-phases (graph.imports / graph.heritage / graph.calls) are emitted
- # from inside GraphBuilder.build(); the orchestrator drives metrics/
- # communities/flows below so the longest-running step is no longer an
- # opaque "graph 0/1" spinner.
- if progress:
- progress.on_message(
- "info",
- " (graph build can take several minutes on first run — safe to "
- "Ctrl-C, then run 'repowise init --resume' to continue)",
- )
- await asyncio.to_thread(graph_builder.build, progress)
-
- # Add framework-aware synthetic edges (conftest, Django, FastAPI, Flask)
- tech_items: list = []
- try:
- from repowise.core.generation.editor_files.tech_stack import detect_tech_stack
-
- tech_items = detect_tech_stack(repo_path)
- graph_builder.add_framework_edges([item.name for item in tech_items])
- except Exception:
- pass # framework edge detection is best-effort
-
- # ---- Dynamic hints wiring (after static graph is fully built) ----------
- if progress:
- progress.on_phase_start("dynamic_hints", None)
- try:
- from repowise.core.ingestion.dynamic_hints import HintRegistry
-
- registry = HintRegistry()
- dynamic_edges = await loop.run_in_executor(None, registry.extract_all, repo_path)
- graph_builder.add_dynamic_edges(dynamic_edges)
- logger.info("dynamic_hints_added", count=len(dynamic_edges))
- except Exception as hints_exc:
- logger.warning("dynamic_hints_failed", error=str(hints_exc))
- _phase_done(progress, "dynamic_hints")
-
- # ---- Graph metrics: prime caches with live progress ---------------------
- # pagerank/betweenness/community/symbol_communities/execution_flows are
- # otherwise computed lazily during persist + generation, where they hide
- # behind opaque progress bars. Pre-compute them here so each is its own
- # visible sub-phase, and fan the within-phase work out via
- # asyncio.gather so betweenness (the dominant cost) overlaps with
- # PageRank / community detection rather than running serially.
- #
- # Each algorithm is wrapped in ``_timed_step`` so we emit a per-algo
- # completion line (` ↳ PageRank ✓ (Xs)`) as it finishes. Without these,
- # the whole gather looks like one opaque several-minute void where
- # betweenness centrality dominates — splitting the timing makes it
- # obvious which step is the bottleneck on a given repo.
- if progress:
- progress.on_phase_start("graph.metrics", None)
- await asyncio.gather(
- _timed_step("PageRank", graph_builder.pagerank, progress),
- _timed_step("betweenness centrality", graph_builder.betweenness_centrality, progress),
- _timed_step("symbol PageRank", graph_builder.symbol_pagerank, progress),
- _timed_step("symbol betweenness", graph_builder.symbol_betweenness_centrality, progress),
- )
- _phase_done(progress, "graph.metrics")
-
- if progress:
- progress.on_phase_start("graph.communities", None)
- await asyncio.gather(
- _timed_step("community detection", graph_builder.community_detection, progress),
- _timed_step("symbol communities", graph_builder.symbol_communities, progress),
- )
- _phase_done(progress, "graph.communities")
-
- # Emit filtering summary so users can see what was included/excluded
- stats = traverser.stats
- if progress:
- parts: list[str] = []
- if stats.skipped_gitignore:
- parts.append(f"{stats.skipped_gitignore:,} by .gitignore")
- if stats.skipped_blocked_extension:
- parts.append(f"{stats.skipped_blocked_extension:,} by extension")
- if stats.skipped_blocked_pattern:
- parts.append(f"{stats.skipped_blocked_pattern:,} by filename pattern")
- if stats.skipped_oversized:
- parts.append(f"{stats.skipped_oversized:,} oversized")
- if stats.skipped_binary:
- parts.append(f"{stats.skipped_binary:,} binary")
- if stats.skipped_generated:
- parts.append(f"{stats.skipped_generated:,} generated")
- if stats.skipped_extra_exclude:
- parts.append(f"{stats.skipped_extra_exclude:,} by --exclude")
- if stats.skipped_extra_ignore:
- parts.append(f"{stats.skipped_extra_ignore:,} by .repowiseIgnore")
- if stats.skipped_submodule:
- parts.append(f"{stats.skipped_submodule:,} submodule dirs")
- if stats.skipped_nested_repo:
- parts.append(f"{stats.skipped_nested_repo:,} nested git repos")
- if stats.skipped_unknown_language:
- parts.append(f"{stats.skipped_unknown_language:,} unknown type")
-
- excluded_str = ", ".join(parts) if parts else "none"
- progress.on_message(
- "info",
- f"Scanned {stats.total_paths_walked:,} files, {len(file_infos):,} included",
- )
- if parts:
- progress.on_message("info", f" Excluded: {excluded_str}")
-
- # Language breakdown
- if stats.lang_counts:
- top_langs = sorted(stats.lang_counts.items(), key=lambda x: -x[1])[:6]
- lang_str = ", ".join(f"{lang} {count:,}" for lang, count in top_langs)
- rest_count = sum(
- c for _, c in sorted(stats.lang_counts.items(), key=lambda x: -x[1])[6:]
- )
- if rest_count:
- lang_str += f", other {rest_count:,}"
- progress.on_message("info", f" Languages: {lang_str}")
-
- return parsed_files, file_infos, repo_structure, source_map, graph_builder, stats, tech_items
-
-
-async def _run_git_indexing(
- repo_path: Path,
- *,
- commit_depth: int,
- follow_renames: bool,
- tier: GitIndexTier = GitIndexTier.FULL,
- progress: ProgressCallback | None,
-) -> tuple[Any | None, list[dict], dict[str, dict]]:
- """Run git history indexing.
-
- Returns (git_summary, git_metadata_list, git_meta_map).
- """
- try:
- from repowise.core.ingestion.git_indexer import GitIndexer
-
- git_indexer = GitIndexer(
- repo_path,
- commit_limit=commit_depth,
- follow_renames=follow_renames,
- tier=tier,
- )
-
- def _on_start(total: int) -> None:
- if progress:
- progress.on_phase_start("git", total)
-
- def _on_file_done() -> None:
- if progress:
- progress.on_item_done("git")
-
- def _on_co_change_start(total: int) -> None:
- if progress:
- progress.on_phase_start("co_change", total)
-
- def _on_commit_done() -> None:
- if progress:
- progress.on_item_done("co_change")
-
- def _on_co_change_done() -> None:
- # Stop the co_change timer the moment accumulation finishes;
- # otherwise the recorded duration also includes the parallel
- # per-file git walk that keeps running afterwards (audit #29).
- _phase_done(progress, "co_change")
-
- git_summary, git_metadata_list = await git_indexer.index_repo(
- "",
- on_start=_on_start,
- on_file_done=_on_file_done,
- on_co_change_start=_on_co_change_start,
- on_commit_done=_on_commit_done,
- on_co_change_done=_on_co_change_done,
- )
- git_meta_map = {m["file_path"]: m for m in git_metadata_list}
- _phase_done(progress, "git")
- # co_change phase already closed inside the done-callback above;
- # call again only as a safety-net in case the callback was never
- # invoked (e.g. co-change skipped early). PhaseTimingRecorder
- # ignores done-without-start so this is a no-op in the happy path.
- _phase_done(progress, "co_change")
- return git_summary, git_metadata_list, git_meta_map
- except Exception as exc:
- if progress:
- progress.on_message("warning", f"Git indexing skipped: {exc}")
- _phase_done(progress, "git")
- _phase_done(progress, "co_change")
- return None, [], {}
-
-
-async def _run_dead_code_analysis(
- graph_builder: Any,
- git_meta_map: dict[str, dict],
- *,
- progress: ProgressCallback | None,
-) -> Any | None:
- """Run dead code detection (pure graph traversal, no LLM)."""
- try:
- from repowise.core.analysis.dead_code import DeadCodeAnalyzer
-
- # Four detectors run sequentially inside analyze(); drive a
- # determinate bar so users see progress instead of "0".
- _DEAD_CODE_STEPS = 4
- if progress:
- progress.on_phase_start("dead_code", _DEAD_CODE_STEPS)
-
- analyzer = DeadCodeAnalyzer(
- graph_builder.graph(), git_meta_map, parsed_files=graph_builder._parsed_files
- )
-
- def _step(_stage: str) -> None:
- if progress:
- progress.on_item_done("dead_code")
-
- report = await asyncio.to_thread(analyzer.analyze, None, on_step=_step)
-
- if progress:
- unreachable = sum(1 for f in report.findings if f.kind.value == "unreachable_file")
- unused_exports = sum(1 for f in report.findings if f.kind.value == "unused_export")
- progress.on_message(
- "info",
- f"→ {unreachable} unreachable files · "
- f"{unused_exports} unused exports · ~{report.deletable_lines:,} deletable lines",
- )
-
- _phase_done(progress, "dead_code")
- return report
- except Exception as exc:
- if progress:
- progress.on_message("warning", f"Dead code detection skipped: {exc}")
- _phase_done(progress, "dead_code")
- return None
-
-
-async def _run_health_analysis(
- graph_builder: Any,
- git_meta_map: dict[str, dict],
- parsed_files: list[Any],
- *,
- repo_path: Path | None = None,
- progress: ProgressCallback | None,
-) -> Any | None:
- """Run code-health analysis (complexity + biomarkers + scoring)."""
- try:
- from repowise.core.analysis.health import HealthAnalyzer
- from repowise.core.analysis.health.config import HealthConfig
-
- if progress:
- # Per-file determinate progress: one tick per parsed file.
- progress.on_phase_start("health", len(parsed_files))
-
- # Build a {file_path → community label} map so per-file metrics
- # carry a real module name, not None. Community detection is
- # already computed for the graph view, so this is essentially
- # free.
- module_map: dict[str, str] = {}
- try:
- cd = graph_builder.community_detection()
- ci = graph_builder.community_info()
- for node_id, comm_id in cd.items():
- info = ci.get(comm_id)
- label = getattr(info, "label", None) if info else None
- if label:
- module_map[node_id] = label
- except Exception as exc:
- logger.debug("health_module_map_failed", error=str(exc))
-
- analyzer = HealthAnalyzer(
- graph_builder.graph(),
- git_meta_map=git_meta_map,
- parsed_files=parsed_files,
- module_map=module_map,
- )
-
- # Load per-file override rules from `.repowise/health-rules.json`.
- # Missing or malformed file → empty config (no-op).
- analyzer_config: dict[str, object] | None = None
- if repo_path is not None:
- cfg = HealthConfig.load(repo_path)
- if cfg.disabled_biomarkers or cfg.rules:
- file_paths = [pf.file_info.path for pf in parsed_files]
- analyzer_config = cfg.to_analyzer_config(file_paths)
-
- def _step(_path: str) -> None:
- if progress:
- progress.on_item_done("health")
-
- # Parallel path for repos large enough to benefit (tree-sitter
- # releases the GIL during parsing, so asyncio.gather over a
- # thread pool actually scales). Threshold chosen so small repos
- # avoid the overhead.
- if len(parsed_files) >= 500:
- report = await analyzer.analyze_async(analyzer_config, on_step=_step)
- else:
- report = await asyncio.to_thread(analyzer.analyze, analyzer_config, on_step=_step)
-
- if progress:
- findings_count = len(report.findings)
- avg = report.kpis.get("average_health", 10.0)
- worst = report.kpis.get("worst_performer_score", 10.0)
- progress.on_message(
- "info",
- f"→ {findings_count} health findings · avg {avg}/10 · worst {worst}/10",
- )
-
- _phase_done(progress, "health")
- return report
- except Exception as exc:
- if progress:
- progress.on_message("warning", f"Health analysis skipped: {exc}")
- _phase_done(progress, "health")
- return None
-
-
-async def _run_decision_extraction(
- repo_path: Path,
- *,
- llm_client: Any | None,
- graph_builder: Any,
- git_meta_map: dict[str, dict],
- parsed_files: list[Any],
- progress: ProgressCallback | None,
-) -> Any | None:
- """Extract architectural decisions from source and git history."""
- try:
- from repowise.core.analysis.decision_extractor import DecisionExtractor
-
- # Seven sources run concurrently inside extract_all(); drive a
- # determinate bar so users see live progress.
- _DECISION_STEPS = 7
- if progress:
- progress.on_phase_start("decisions", _DECISION_STEPS)
-
- extractor = DecisionExtractor(
- repo_path=repo_path,
- provider=llm_client,
- graph=graph_builder.graph(),
- git_meta_map=git_meta_map,
- parsed_files=parsed_files,
- )
-
- def _decision_step(_source: str) -> None:
- if progress:
- progress.on_item_done("decisions")
-
- report = await asyncio.wait_for(
- extractor.extract_all(on_step=_decision_step),
- timeout=DECISION_EXTRACTION_TIMEOUT_SECS,
- )
-
- if progress:
- bs = report.by_source
- total_decisions = report.total_found
- progress.on_message(
- "info",
- f"→ {total_decisions} decisions: "
- f"{bs.get('inline_marker', 0)} inline · "
- f"{bs.get('adr', 0)} ADR · "
- f"{bs.get('changelog', 0)} changelog · "
- f"{bs.get('pr', 0)} PR · "
- f"{bs.get('git_archaeology', 0)} git · "
- f"{bs.get('comment', 0)} comments · "
- f"{bs.get('readme_mining', 0)} docs",
- )
-
- _phase_done(progress, "decisions")
- return report
- except Exception as exc:
- if progress:
- progress.on_message("warning", f"Decision extraction skipped: {exc}")
- _phase_done(progress, "decisions")
- return None
-
-
-async def run_generation(
- *,
- repo_path: Path,
- parsed_files: list[Any],
- source_map: dict[str, bytes],
- graph_builder: Any,
- repo_structure: Any,
- git_meta_map: dict[str, dict],
- llm_client: Any,
- embedder: Any | None,
- vector_store: Any | None,
- concurrency: int,
- progress: ProgressCallback | None,
- resume: bool = False,
- cost_tracker: Any | None = None,
- generation_config: Any | None = None,
- dead_code_report: Any | None = None,
- decision_report: Any | None = None,
- external_systems: list[dict] | None = None,
-) -> list[Any]:
- """Run LLM-powered page generation.
-
- Returns a list of ``GeneratedPage`` objects.
- """
- from repowise.core.generation import (
- ContextAssembler,
- JobSystem,
- PageGenerator,
- )
- from repowise.core.persistence.vector_store import InMemoryVectorStore
- from repowise.core.providers.embedding.base import MockEmbedder
-
- # Attach cost tracker to LLM client if available
- if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):
- llm_client._cost_tracker = cost_tracker
-
- from repowise.core.generation import GenerationConfig
-
- # Preserve all caller-supplied GenerationConfig fields (output language, cache flags,
- # token budgets, etc.) and only override max_concurrency to match the resolved value.
- # Falls back to defaults when the pipeline entry point did not thread one through.
- base_config = generation_config if generation_config is not None else GenerationConfig()
- config = replace(base_config, max_concurrency=concurrency)
- assembler = ContextAssembler(config)
-
- # Resolve embedder and vector store
- embedder_impl = embedder if embedder is not None else MockEmbedder()
-
- if vector_store is None:
- vector_store = InMemoryVectorStore(embedder_impl)
-
- # Job system — use a temp-like dir under repo_path for checkpoints
- jobs_dir = repo_path / ".repowise" / "jobs"
- jobs_dir.mkdir(parents=True, exist_ok=True)
- job_system = JobSystem(jobs_dir)
-
- repo_name = repo_path.name
-
- # Track generation progress. Onboarding pages get routed to their own
- # phase so the terminal UI shows them as a distinct, named step rather
- # than blending into the long file_page run.
- _pages_done = 0
-
- def on_page_done(page_type: str) -> None:
- nonlocal _pages_done
- _pages_done += 1
- if progress:
- phase = "onboarding" if page_type == "onboarding" else "generation"
- progress.on_item_done(phase)
- # Push live cost update if the callback supports it
- if cost_tracker is not None and hasattr(progress, "set_cost"):
- progress.set_cost(cost_tracker.session_cost)
-
- if progress:
- progress.on_phase_start("generation", None)
-
- def on_total_known(total: int) -> None:
- if progress:
- progress.on_phase_start("generation", total)
-
- def on_subphase(name: str, total: int | None) -> None:
- """Start a distinct sub-phase (currently used only for onboarding)."""
- if progress:
- progress.on_phase_start(name, total)
-
- generator = PageGenerator(
- llm_client,
- assembler,
- config,
- vector_store=vector_store,
- language=config.language,
- )
-
- generated_pages = await generator.generate_all(
- parsed_files,
- source_map,
- graph_builder,
- repo_structure,
- repo_name,
- job_system=job_system,
- on_page_done=on_page_done,
- on_total_known=on_total_known,
- on_subphase=on_subphase,
- git_meta_map=git_meta_map if git_meta_map else None,
- resume=resume,
- repo_path=repo_path,
- dead_code_report=dead_code_report,
- decision_report=decision_report,
- external_systems=external_systems,
- )
-
- # Onboarding summary — count generated slots and surface which ones
- # were gated out so the user can see the curated collection's state.
- onboarding_generated = [p for p in generated_pages if p.page_type == "onboarding"]
- promoted_present = {
- p.metadata.get("onboarding_slot")
- for p in generated_pages
- if p.metadata.get("onboarding_slot")
- and p.page_type in ("repo_overview", "architecture_diagram")
- }
- if progress:
- if onboarding_generated or promoted_present:
- slots_made = sorted(
- {p.metadata.get("subkind", "?") for p in onboarding_generated} | promoted_present
- )
- progress.on_message(
- "info",
- f"Onboarding: {len(slots_made)}/8 slots — {', '.join(slots_made)}",
- )
- progress.on_message("info", f"Generated {len(generated_pages)} pages")
- _phase_done(progress, "onboarding")
- _phase_done(progress, "generation")
-
- return generated_pages
diff --git a/packages/core/src/repowise/core/pipeline/phases/__init__.py b/packages/core/src/repowise/core/pipeline/phases/__init__.py
new file mode 100644
index 000000000..248fa01d5
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/phases/__init__.py
@@ -0,0 +1 @@
+"""Pipeline phase implementations (ingestion, git, analysis, generation)."""
diff --git a/packages/core/src/repowise/core/pipeline/phases/_common.py b/packages/core/src/repowise/core/pipeline/phases/_common.py
new file mode 100644
index 000000000..8cdfb4dcd
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/phases/_common.py
@@ -0,0 +1,19 @@
+"""Helpers shared across pipeline phases."""
+
+from __future__ import annotations
+
+import contextlib
+
+from repowise.core.pipeline.progress import ProgressCallback
+
+
+def _phase_done(progress: ProgressCallback | None, phase: str) -> None:
+ """Best-effort call to ``progress.on_phase_done`` — older callbacks may
+ not implement it, so fall back to a no-op silently.
+ """
+ if progress is None:
+ return
+ fn = getattr(progress, "on_phase_done", None)
+ if callable(fn):
+ with contextlib.suppress(Exception):
+ fn(phase)
diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.py
new file mode 100644
index 000000000..8a0997665
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py
@@ -0,0 +1,208 @@
+"""Pipeline analysis phase.
+
+Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in
+orchestrator.py) imports these phase functions. No CLI/click/rich imports.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from repowise.core.pipeline.progress import ProgressCallback
+
+from ._common import _phase_done
+
+logger = structlog.get_logger(__name__)
+
+
+# Maximum seconds to spend on decision extraction before giving up.
+# Large repos with tens of thousands of files can take arbitrarily long.
+DECISION_EXTRACTION_TIMEOUT_SECS = 300
+
+
+async def _run_dead_code_analysis(
+ graph_builder: Any,
+ git_meta_map: dict[str, dict],
+ *,
+ progress: ProgressCallback | None,
+) -> Any | None:
+ """Run dead code detection (pure graph traversal, no LLM)."""
+ try:
+ from repowise.core.analysis.dead_code import DeadCodeAnalyzer
+
+ # Four detectors run sequentially inside analyze(); drive a
+ # determinate bar so users see progress instead of "0".
+ dead_code_steps = 4
+ if progress:
+ progress.on_phase_start("dead_code", dead_code_steps)
+
+ analyzer = DeadCodeAnalyzer(
+ graph_builder.graph(), git_meta_map, parsed_files=graph_builder._parsed_files
+ )
+
+ def _step(_stage: str) -> None:
+ if progress:
+ progress.on_item_done("dead_code")
+
+ report = await asyncio.to_thread(analyzer.analyze, None, on_step=_step)
+
+ if progress:
+ unreachable = sum(1 for f in report.findings if f.kind.value == "unreachable_file")
+ unused_exports = sum(1 for f in report.findings if f.kind.value == "unused_export")
+ progress.on_message(
+ "info",
+ f"→ {unreachable} unreachable files · "
+ f"{unused_exports} unused exports · ~{report.deletable_lines:,} deletable lines",
+ )
+
+ _phase_done(progress, "dead_code")
+ return report
+ except Exception as exc:
+ if progress:
+ progress.on_message("warning", f"Dead code detection skipped: {exc}")
+ _phase_done(progress, "dead_code")
+ return None
+
+
+async def _run_health_analysis(
+ graph_builder: Any,
+ git_meta_map: dict[str, dict],
+ parsed_files: list[Any],
+ *,
+ repo_path: Path | None = None,
+ progress: ProgressCallback | None,
+) -> Any | None:
+ """Run code-health analysis (complexity + biomarkers + scoring)."""
+ try:
+ from repowise.core.analysis.health import HealthAnalyzer
+ from repowise.core.analysis.health.config import HealthConfig
+
+ if progress:
+ # Per-file determinate progress: one tick per parsed file.
+ progress.on_phase_start("health", len(parsed_files))
+
+ # Build a {file_path → community label} map so per-file metrics
+ # carry a real module name, not None. Community detection is
+ # already computed for the graph view, so this is essentially
+ # free.
+ module_map: dict[str, str] = {}
+ try:
+ cd = graph_builder.community_detection()
+ ci = graph_builder.community_info()
+ for node_id, comm_id in cd.items():
+ info = ci.get(comm_id)
+ label = getattr(info, "label", None) if info else None
+ if label:
+ module_map[node_id] = label
+ except Exception as exc:
+ logger.debug("health_module_map_failed", error=str(exc))
+
+ analyzer = HealthAnalyzer(
+ graph_builder.graph(),
+ git_meta_map=git_meta_map,
+ parsed_files=parsed_files,
+ module_map=module_map,
+ )
+
+ # Load per-file override rules from `.repowise/health-rules.json`.
+ # Missing or malformed file → empty config (no-op).
+ analyzer_config: dict[str, object] | None = None
+ if repo_path is not None:
+ cfg = HealthConfig.load(repo_path)
+ if cfg.disabled_biomarkers or cfg.rules:
+ file_paths = [pf.file_info.path for pf in parsed_files]
+ analyzer_config = cfg.to_analyzer_config(file_paths)
+
+ def _step(_path: str) -> None:
+ if progress:
+ progress.on_item_done("health")
+
+ # Parallel path for repos large enough to benefit (tree-sitter
+ # releases the GIL during parsing, so asyncio.gather over a
+ # thread pool actually scales). Threshold chosen so small repos
+ # avoid the overhead.
+ if len(parsed_files) >= 500:
+ report = await analyzer.analyze_async(analyzer_config, on_step=_step)
+ else:
+ report = await asyncio.to_thread(analyzer.analyze, analyzer_config, on_step=_step)
+
+ if progress:
+ findings_count = len(report.findings)
+ avg = report.kpis.get("average_health", 10.0)
+ worst = report.kpis.get("worst_performer_score", 10.0)
+ progress.on_message(
+ "info",
+ f"→ {findings_count} health findings · avg {avg}/10 · worst {worst}/10",
+ )
+
+ _phase_done(progress, "health")
+ return report
+ except Exception as exc:
+ if progress:
+ progress.on_message("warning", f"Health analysis skipped: {exc}")
+ _phase_done(progress, "health")
+ return None
+
+
+async def _run_decision_extraction(
+ repo_path: Path,
+ *,
+ llm_client: Any | None,
+ graph_builder: Any,
+ git_meta_map: dict[str, dict],
+ parsed_files: list[Any],
+ progress: ProgressCallback | None,
+) -> Any | None:
+ """Extract architectural decisions from source and git history."""
+ try:
+ from repowise.core.analysis.decision_extractor import DecisionExtractor
+
+ # Seven sources run concurrently inside extract_all(); drive a
+ # determinate bar so users see live progress.
+ decision_steps = 7
+ if progress:
+ progress.on_phase_start("decisions", decision_steps)
+
+ extractor = DecisionExtractor(
+ repo_path=repo_path,
+ provider=llm_client,
+ graph=graph_builder.graph(),
+ git_meta_map=git_meta_map,
+ parsed_files=parsed_files,
+ )
+
+ def _decision_step(_source: str) -> None:
+ if progress:
+ progress.on_item_done("decisions")
+
+ report = await asyncio.wait_for(
+ extractor.extract_all(on_step=_decision_step),
+ timeout=DECISION_EXTRACTION_TIMEOUT_SECS,
+ )
+
+ if progress:
+ bs = report.by_source
+ total_decisions = report.total_found
+ progress.on_message(
+ "info",
+ f"→ {total_decisions} decisions: "
+ f"{bs.get('inline_marker', 0)} inline · "
+ f"{bs.get('adr', 0)} ADR · "
+ f"{bs.get('changelog', 0)} changelog · "
+ f"{bs.get('pr', 0)} PR · "
+ f"{bs.get('git_archaeology', 0)} git · "
+ f"{bs.get('comment', 0)} comments · "
+ f"{bs.get('readme_mining', 0)} docs",
+ )
+
+ _phase_done(progress, "decisions")
+ return report
+ except Exception as exc:
+ if progress:
+ progress.on_message("warning", f"Decision extraction skipped: {exc}")
+ _phase_done(progress, "decisions")
+ return None
diff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.py
new file mode 100644
index 000000000..71e76d59d
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py
@@ -0,0 +1,155 @@
+"""Pipeline generation phase.
+
+Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in
+orchestrator.py) imports these phase functions. No CLI/click/rich imports.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from repowise.core.pipeline.progress import ProgressCallback
+
+from ._common import _phase_done
+
+logger = structlog.get_logger(__name__)
+
+
+async def run_generation(
+ *,
+ repo_path: Path,
+ parsed_files: list[Any],
+ source_map: dict[str, bytes],
+ graph_builder: Any,
+ repo_structure: Any,
+ git_meta_map: dict[str, dict],
+ llm_client: Any,
+ embedder: Any | None,
+ vector_store: Any | None,
+ concurrency: int,
+ progress: ProgressCallback | None,
+ resume: bool = False,
+ cost_tracker: Any | None = None,
+ generation_config: Any | None = None,
+ dead_code_report: Any | None = None,
+ decision_report: Any | None = None,
+ external_systems: list[dict] | None = None,
+) -> list[Any]:
+ """Run LLM-powered page generation.
+
+ Returns a list of ``GeneratedPage`` objects.
+ """
+ from repowise.core.generation import (
+ ContextAssembler,
+ JobSystem,
+ PageGenerator,
+ )
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
+ from repowise.core.providers.embedding.base import MockEmbedder
+
+ # Attach cost tracker to LLM client if available
+ if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):
+ llm_client._cost_tracker = cost_tracker
+
+ from repowise.core.generation import GenerationConfig
+
+ # Preserve all caller-supplied GenerationConfig fields (output language, cache flags,
+ # token budgets, etc.) and only override max_concurrency to match the resolved value.
+ # Falls back to defaults when the pipeline entry point did not thread one through.
+ base_config = generation_config if generation_config is not None else GenerationConfig()
+ config = replace(base_config, max_concurrency=concurrency)
+ assembler = ContextAssembler(config)
+
+ # Resolve embedder and vector store
+ embedder_impl = embedder if embedder is not None else MockEmbedder()
+
+ if vector_store is None:
+ vector_store = InMemoryVectorStore(embedder_impl)
+
+ # Job system — use a temp-like dir under repo_path for checkpoints
+ jobs_dir = repo_path / ".repowise" / "jobs"
+ jobs_dir.mkdir(parents=True, exist_ok=True)
+ job_system = JobSystem(jobs_dir)
+
+ repo_name = repo_path.name
+
+ # Track generation progress. Onboarding pages get routed to their own
+ # phase so the terminal UI shows them as a distinct, named step rather
+ # than blending into the long file_page run.
+ _pages_done = 0
+
+ def on_page_done(page_type: str) -> None:
+ nonlocal _pages_done
+ _pages_done += 1
+ if progress:
+ phase = "onboarding" if page_type == "onboarding" else "generation"
+ progress.on_item_done(phase)
+ # Push live cost update if the callback supports it
+ if cost_tracker is not None and hasattr(progress, "set_cost"):
+ progress.set_cost(cost_tracker.session_cost)
+
+ if progress:
+ progress.on_phase_start("generation", None)
+
+ def on_total_known(total: int) -> None:
+ if progress:
+ progress.on_phase_start("generation", total)
+
+ def on_subphase(name: str, total: int | None) -> None:
+ """Start a distinct sub-phase (currently used only for onboarding)."""
+ if progress:
+ progress.on_phase_start(name, total)
+
+ generator = PageGenerator(
+ llm_client,
+ assembler,
+ config,
+ vector_store=vector_store,
+ language=config.language,
+ )
+
+ generated_pages = await generator.generate_all(
+ parsed_files,
+ source_map,
+ graph_builder,
+ repo_structure,
+ repo_name,
+ job_system=job_system,
+ on_page_done=on_page_done,
+ on_total_known=on_total_known,
+ on_subphase=on_subphase,
+ git_meta_map=git_meta_map if git_meta_map else None,
+ resume=resume,
+ repo_path=repo_path,
+ dead_code_report=dead_code_report,
+ decision_report=decision_report,
+ external_systems=external_systems,
+ )
+
+ # Onboarding summary — count generated slots and surface which ones
+ # were gated out so the user can see the curated collection's state.
+ onboarding_generated = [p for p in generated_pages if p.page_type == "onboarding"]
+ promoted_present = {
+ p.metadata.get("onboarding_slot")
+ for p in generated_pages
+ if p.metadata.get("onboarding_slot")
+ and p.page_type in ("repo_overview", "architecture_diagram")
+ }
+ if progress:
+ if onboarding_generated or promoted_present:
+ slots_made = sorted(
+ {p.metadata.get("subkind", "?") for p in onboarding_generated} | promoted_present
+ )
+ progress.on_message(
+ "info",
+ f"Onboarding: {len(slots_made)}/8 slots — {', '.join(slots_made)}",
+ )
+ progress.on_message("info", f"Generated {len(generated_pages)} pages")
+ _phase_done(progress, "onboarding")
+ _phase_done(progress, "generation")
+
+ return generated_pages
diff --git a/packages/core/src/repowise/core/pipeline/phases/git.py b/packages/core/src/repowise/core/pipeline/phases/git.py
new file mode 100644
index 000000000..06f7c7f2b
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/phases/git.py
@@ -0,0 +1,87 @@
+"""Pipeline git phase.
+
+Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in
+orchestrator.py) imports these phase functions. No CLI/click/rich imports.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from repowise.core.ingestion.git_indexer import GitIndexTier
+from repowise.core.pipeline.progress import ProgressCallback
+
+from ._common import _phase_done
+
+logger = structlog.get_logger(__name__)
+
+
+async def _run_git_indexing(
+ repo_path: Path,
+ *,
+ commit_depth: int,
+ follow_renames: bool,
+ tier: GitIndexTier = GitIndexTier.FULL,
+ progress: ProgressCallback | None,
+) -> tuple[Any | None, list[dict], dict[str, dict]]:
+ """Run git history indexing.
+
+ Returns (git_summary, git_metadata_list, git_meta_map).
+ """
+ try:
+ from repowise.core.ingestion.git_indexer import GitIndexer
+
+ git_indexer = GitIndexer(
+ repo_path,
+ commit_limit=commit_depth,
+ follow_renames=follow_renames,
+ tier=tier,
+ )
+
+ def _on_start(total: int) -> None:
+ if progress:
+ progress.on_phase_start("git", total)
+
+ def _on_file_done() -> None:
+ if progress:
+ progress.on_item_done("git")
+
+ def _on_co_change_start(total: int) -> None:
+ if progress:
+ progress.on_phase_start("co_change", total)
+
+ def _on_commit_done() -> None:
+ if progress:
+ progress.on_item_done("co_change")
+
+ def _on_co_change_done() -> None:
+ # Stop the co_change timer the moment accumulation finishes;
+ # otherwise the recorded duration also includes the parallel
+ # per-file git walk that keeps running afterwards (audit #29).
+ _phase_done(progress, "co_change")
+
+ git_summary, git_metadata_list = await git_indexer.index_repo(
+ "",
+ on_start=_on_start,
+ on_file_done=_on_file_done,
+ on_co_change_start=_on_co_change_start,
+ on_commit_done=_on_commit_done,
+ on_co_change_done=_on_co_change_done,
+ )
+ git_meta_map = {m["file_path"]: m for m in git_metadata_list}
+ _phase_done(progress, "git")
+ # co_change phase already closed inside the done-callback above;
+ # call again only as a safety-net in case the callback was never
+ # invoked (e.g. co-change skipped early). PhaseTimingRecorder
+ # ignores done-without-start so this is a no-op in the happy path.
+ _phase_done(progress, "co_change")
+ return git_summary, git_metadata_list, git_meta_map
+ except Exception as exc:
+ if progress:
+ progress.on_message("warning", f"Git indexing skipped: {exc}")
+ _phase_done(progress, "git")
+ _phase_done(progress, "co_change")
+ return None, [], {}
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
new file mode 100644
index 000000000..a53078a3d
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -0,0 +1,364 @@
+"""Pipeline ingestion phase.
+
+Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in
+orchestrator.py) imports these phase functions. No CLI/click/rich imports.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from repowise.core.pipeline.progress import ProgressCallback
+
+from ._common import _phase_done
+
+logger = structlog.get_logger(__name__)
+
+
+async def _timed_step(
+ label: str,
+ fn: Any,
+ progress: ProgressCallback | None,
+) -> Any:
+ """Run *fn* in a worker thread and emit a per-step completion line.
+
+ Used to make ``asyncio.gather`` of multiple graph-algorithm calls
+ legible: without this, four concurrent thread-bound computations
+ (e.g. PageRank + betweenness + symbol PageRank + symbol betweenness)
+ appear in the CLI as one opaque several-minute spinner, so the user
+ has no signal as to which step is the bottleneck. With it, each algo
+ prints ` ↳ <label> ✓ (Xs)` as it finishes — completion order
+ surfaces the relative cost without changing the underlying execution.
+ """
+ t0 = time.monotonic()
+ try:
+ result = await asyncio.to_thread(fn)
+ except Exception as exc:
+ if progress is not None:
+ progress.on_message(
+ "warning",
+ f" ↳ {label} failed after {time.monotonic() - t0:.1f}s: {exc}",
+ )
+ raise
+ if progress is not None:
+ progress.on_message(
+ "info",
+ f" ↳ {label} ✓ ({time.monotonic() - t0:.1f}s)",
+ )
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Process-pool worker (module-level — must be picklable)
+# ---------------------------------------------------------------------------
+
+# Module-level process-local parser cache (one per worker process).
+_WORKER_PARSER: Any = None
+
+
+def _parse_one(path_and_fi_and_bytes: tuple) -> Any:
+ """Worker function for ProcessPoolExecutor parsing.
+
+ Constructs (or reuses) a process-local ASTParser and parses one file.
+ Returns a ParsedFile on success, or (abs_path_str, error_str) on failure.
+ The parser is constructed lazily inside the worker — the ASTParser itself
+ (which holds compiled tree-sitter Language/Query objects) is never pickled.
+ Only FileInfo (input) and ParsedFile (output) cross the process boundary;
+ both are plain dataclasses and therefore picklable.
+ """
+ global _WORKER_PARSER
+ fi, source = path_and_fi_and_bytes
+ try:
+ if _WORKER_PARSER is None:
+ from repowise.core.ingestion import ASTParser
+
+ _WORKER_PARSER = ASTParser()
+ return _WORKER_PARSER.parse_file(fi, source)
+ except Exception as exc:
+ return (fi.abs_path, str(exc))
+
+
+async def _run_ingestion(
+ repo_path: Path,
+ *,
+ exclude_patterns: list[str] | None,
+ include_submodules: bool = False,
+ include_nested_repos: bool = False,
+ skip_tests: bool,
+ skip_infra: bool,
+ progress: ProgressCallback | None,
+) -> tuple[list[Any], list[Any], Any, dict[str, bytes], Any, Any]:
+ """Traverse, parse, and build the dependency graph.
+
+ Returns (parsed_files, file_infos, repo_structure, source_map,
+ graph_builder, traversal_stats, tech_items).
+ """
+ from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
+
+ traverser = FileTraverser(
+ repo_path,
+ extra_exclude_patterns=exclude_patterns or None,
+ include_submodules=include_submodules,
+ include_nested_repos=include_nested_repos,
+ )
+
+ # Walk directory tree
+ all_paths = list(traverser._walk())
+ if progress:
+ # Use indeterminate progress (spinner) to avoid showing a misleading
+ # pre-filter total like "2132/83601".
+ progress.on_phase_start("traverse", None)
+
+ # Parallel stat + header reads (I/O bound).
+ # Use asyncio.wrap_future so the event loop stays responsive while waiting.
+ file_infos: list[Any] = []
+ io_pool = ThreadPoolExecutor(max_workers=8)
+ try:
+ aws = [
+ asyncio.wrap_future(io_pool.submit(traverser._build_file_info, p)) for p in all_paths
+ ]
+ for coro in asyncio.as_completed(aws):
+ try:
+ result = await coro
+ except Exception:
+ result = None
+ if result is not None:
+ file_infos.append(result)
+ if progress:
+ progress.on_item_done("traverse")
+ finally:
+ # shutdown(wait=True) is blocking — run in a thread to keep the
+ # event loop responsive. All submitted futures have already
+ # completed by the time we reach here (the for-loop awaited them).
+ await asyncio.to_thread(io_pool.shutdown, wait=True)
+
+ repo_structure = traverser.get_repo_structure(file_infos)
+ _phase_done(progress, "traverse")
+
+ # Filter
+ if skip_tests:
+ file_infos = [fi for fi in file_infos if not fi.is_test]
+ if skip_infra:
+ file_infos = [
+ fi
+ for fi in file_infos
+ if fi.language not in ("dockerfile", "makefile", "terraform", "shell")
+ ]
+
+ # ---- Parse phase: CPU-bound, run in ProcessPoolExecutor ----------------
+ if progress:
+ progress.on_phase_start("parse", len(file_infos))
+
+ # Read source bytes up front (I/O, sequential — fast enough; keeps worker
+ # args small: FileInfo + bytes, both picklable plain dataclasses/bytes).
+ fi_and_bytes: list[tuple] = []
+ for fi in file_infos:
+ try:
+ source = Path(fi.abs_path).read_bytes()
+ fi_and_bytes.append((fi, source))
+ except Exception:
+ if progress:
+ progress.on_item_done("parse")
+
+ parsed_files: list[Any] = []
+ source_map: dict[str, bytes] = {}
+ graph_builder = GraphBuilder(repo_path=repo_path)
+
+ loop = asyncio.get_running_loop()
+ workers = max(1, os.cpu_count() or 4)
+
+ _use_process_pool = True
+ parse_results: list[Any] = []
+
+ try:
+ with ProcessPoolExecutor(max_workers=workers) as pool:
+ tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]
+ # Tick the parse-progress bar as each worker finishes —
+ # ``asyncio.gather`` would otherwise hold every event back
+ # until the last file is done, which on PowerToys-scale
+ # repos looked like a hang at ``0/N`` for many minutes.
+ # Per-task done-callbacks fire on the event loop thread and
+ # preserve gather's ordered results, so the aggregation
+ # loop below still indexes ``fi_and_bytes`` correctly.
+ if progress is not None:
+ _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731
+ for fut in tasks:
+ fut.add_done_callback(_parse_tick)
+ parse_results = await asyncio.gather(*tasks, return_exceptions=True)
+ except Exception as pool_exc:
+ logger.warning(
+ "process_pool_parse_failed_falling_back",
+ error=str(pool_exc),
+ )
+ _use_process_pool = False
+ # Fallback: in-process sequential parse
+ _fallback_parser = ASTParser()
+ for i, (fi, source) in enumerate(fi_and_bytes):
+ try:
+ result = _fallback_parser.parse_file(fi, source)
+ parse_results.append(result)
+ except Exception as exc:
+ parse_results.append((fi.abs_path, str(exc)))
+ if progress:
+ progress.on_item_done("parse")
+ if i % 50 == 49:
+ await asyncio.sleep(0)
+
+ # Aggregate results into GraphBuilder on the main loop (not thread-safe).
+ for idx, result in enumerate(parse_results):
+ fi, source = fi_and_bytes[idx]
+ if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):
+ # Error tuple: (abs_path_str, error_str)
+ logger.debug("parse_error_in_worker", path=result[0], error=result[1])
+ elif isinstance(result, Exception):
+ logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))
+ else:
+ parsed_files.append(result)
+ source_map[fi.path] = source
+ graph_builder.add_file(result)
+ # Process-pool path already ticked per-file via the done-callback
+ # attached above; only the fallback path ticks here (handled in
+ # its own loop). No tick needed in aggregation.
+
+ _phase_done(progress, "parse")
+
+ # ---- tsconfig path-alias resolver (before graph build) ------------------
+ # Only runs when the repo has TS/JS files. On large TS monorepos the
+ # resolver indexes hundreds of tsconfig files up-front; without a phase
+ # label this shows up as a silent gap right after parsing.
+ try:
+ from repowise.core.ingestion.tsconfig_resolver import TsconfigResolver
+
+ _ts_langs = {"typescript", "javascript"}
+ if any(pf.file_info.language in _ts_langs for pf in parsed_files):
+ if progress:
+ progress.on_phase_start("tsconfig", None)
+ _path_set = set(graph_builder._parsed_files.keys())
+ _resolver = TsconfigResolver(repo_path=repo_path, path_set=_path_set)
+ graph_builder.set_tsconfig_resolver(_resolver)
+ _phase_done(progress, "tsconfig")
+ except Exception as _resolver_exc:
+ logger.warning("tsconfig_resolver_init_failed", error=str(_resolver_exc))
+
+ # ---- Graph build phase -------------------------------------------------
+ # Sub-phases (graph.imports / graph.heritage / graph.calls) are emitted
+ # from inside GraphBuilder.build(); the orchestrator drives metrics/
+ # communities/flows below so the longest-running step is no longer an
+ # opaque "graph 0/1" spinner.
+ if progress:
+ progress.on_message(
+ "info",
+ " (graph build can take several minutes on first run — safe to "
+ "Ctrl-C, then run 'repowise init --resume' to continue)",
+ )
+ await asyncio.to_thread(graph_builder.build, progress)
+
+ # Add framework-aware synthetic edges (conftest, Django, FastAPI, Flask)
+ tech_items: list = []
+ try:
+ from repowise.core.generation.editor_files.tech_stack import detect_tech_stack
+
+ tech_items = detect_tech_stack(repo_path)
+ graph_builder.add_framework_edges([item.name for item in tech_items])
+ except Exception:
+ pass # framework edge detection is best-effort
+
+ # ---- Dynamic hints wiring (after static graph is fully built) ----------
+ if progress:
+ progress.on_phase_start("dynamic_hints", None)
+ try:
+ from repowise.core.ingestion.dynamic_hints import HintRegistry
+
+ registry = HintRegistry()
+ dynamic_edges = await loop.run_in_executor(None, registry.extract_all, repo_path)
+ graph_builder.add_dynamic_edges(dynamic_edges)
+ logger.info("dynamic_hints_added", count=len(dynamic_edges))
+ except Exception as hints_exc:
+ logger.warning("dynamic_hints_failed", error=str(hints_exc))
+ _phase_done(progress, "dynamic_hints")
+
+ # ---- Graph metrics: prime caches with live progress ---------------------
+ # pagerank/betweenness/community/symbol_communities/execution_flows are
+ # otherwise computed lazily during persist + generation, where they hide
+ # behind opaque progress bars. Pre-compute them here so each is its own
+ # visible sub-phase, and fan the within-phase work out via
+ # asyncio.gather so betweenness (the dominant cost) overlaps with
+ # PageRank / community detection rather than running serially.
+ #
+ # Each algorithm is wrapped in ``_timed_step`` so we emit a per-algo
+ # completion line (` ↳ PageRank ✓ (Xs)`) as it finishes. Without these,
+ # the whole gather looks like one opaque several-minute void where
+ # betweenness centrality dominates — splitting the timing makes it
+ # obvious which step is the bottleneck on a given repo.
+ if progress:
+ progress.on_phase_start("graph.metrics", None)
+ await asyncio.gather(
+ _timed_step("PageRank", graph_builder.pagerank, progress),
+ _timed_step("betweenness centrality", graph_builder.betweenness_centrality, progress),
+ _timed_step("symbol PageRank", graph_builder.symbol_pagerank, progress),
+ _timed_step("symbol betweenness", graph_builder.symbol_betweenness_centrality, progress),
+ )
+ _phase_done(progress, "graph.metrics")
+
+ if progress:
+ progress.on_phase_start("graph.communities", None)
+ await asyncio.gather(
+ _timed_step("community detection", graph_builder.community_detection, progress),
+ _timed_step("symbol communities", graph_builder.symbol_communities, progress),
+ )
+ _phase_done(progress, "graph.communities")
+
+ # Emit filtering summary so users can see what was included/excluded
+ stats = traverser.stats
+ if progress:
+ parts: list[str] = []
+ if stats.skipped_gitignore:
+ parts.append(f"{stats.skipped_gitignore:,} by .gitignore")
+ if stats.skipped_blocked_extension:
+ parts.append(f"{stats.skipped_blocked_extension:,} by extension")
+ if stats.skipped_blocked_pattern:
+ parts.append(f"{stats.skipped_blocked_pattern:,} by filename pattern")
+ if stats.skipped_oversized:
+ parts.append(f"{stats.skipped_oversized:,} oversized")
+ if stats.skipped_binary:
+ parts.append(f"{stats.skipped_binary:,} binary")
+ if stats.skipped_generated:
+ parts.append(f"{stats.skipped_generated:,} generated")
+ if stats.skipped_extra_exclude:
+ parts.append(f"{stats.skipped_extra_exclude:,} by --exclude")
+ if stats.skipped_extra_ignore:
+ parts.append(f"{stats.skipped_extra_ignore:,} by .repowiseIgnore")
+ if stats.skipped_submodule:
+ parts.append(f"{stats.skipped_submodule:,} submodule dirs")
+ if stats.skipped_nested_repo:
+ parts.append(f"{stats.skipped_nested_repo:,} nested git repos")
+ if stats.skipped_unknown_language:
+ parts.append(f"{stats.skipped_unknown_language:,} unknown type")
+
+ excluded_str = ", ".join(parts) if parts else "none"
+ progress.on_message(
+ "info",
+ f"Scanned {stats.total_paths_walked:,} files, {len(file_infos):,} included",
+ )
+ if parts:
+ progress.on_message("info", f" Excluded: {excluded_str}")
+
+ # Language breakdown
+ if stats.lang_counts:
+ top_langs = sorted(stats.lang_counts.items(), key=lambda x: -x[1])[:6]
+ lang_str = ", ".join(f"{lang} {count:,}" for lang, count in top_langs)
+ rest_count = sum(
+ c for _, c in sorted(stats.lang_counts.items(), key=lambda x: -x[1])[6:]
+ )
+ if rest_count:
+ lang_str += f", other {rest_count:,}"
+ progress.on_message("info", f" Languages: {lang_str}")
+
+ return parsed_files, file_infos, repo_structure, source_map, graph_builder, stats, tech_items