Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(pipeline): extract phase functions into pipeline/phases/ (#257)
Swati Ahuja9d51656b95a2
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex ca62d454b..b1a50866b 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -15,103 +15,28 @@ Callers:from __future__ import annotationsimport asyncio-import osimport time-from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor-from dataclasses import dataclass, field, replace+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Anyimport structlog-from repowise.core.ingestion.git_indexer import GitIndexTierfrom repowise.core.pipeline.modes import OrchestratorModefrom repowise.core.pipeline.progress import ProgressCallbackfrom repowise.core.registry import HookProgressCallback-logger = structlog.get_logger(__name__)---def _phase_done(progress: ProgressCallback | None, phase: str) -> None:- """Best-effort call to ``progress.on_phase_done`` — older callbacks may- not implement it, so fall back to a no-op silently.- """- if progress is None:- return- fn = getattr(progress, "on_phase_done", None)- if callable(fn):- try:- fn(phase)- except Exception:- pass---async def _timed_step(- label: str,- fn: Any,- progress: ProgressCallback | None,-) -> Any:- """Run *fn* in a worker thread and emit a per-step completion line.-- Used to make ``asyncio.gather`` of multiple graph-algorithm calls- legible: without this, four concurrent thread-bound computations- (e.g. PageRank + betweenness + symbol PageRank + symbol betweenness)- appear in the CLI as one opaque several-minute spinner, so the user- has no signal as to which step is the bottleneck. With it, each algo- prints ` ↳ <label> ✓ (Xs)` as it finishes — completion order- surfaces the relative cost without changing the underlying execution.- """- t0 = time.monotonic()- try:- result = await asyncio.to_thread(fn)- except Exception as exc:- if progress is not None:- progress.on_message(- "warning",- f" ↳ {label} failed after {time.monotonic() - t0:.1f}s: {exc}",- )- raise- if progress is not None:- progress.on_message(- "info",- f" ↳ {label} ✓ ({time.monotonic() - t0:.1f}s)",- )- return result+from .phases._common import _phase_done+from .phases.analysis import (+ _run_dead_code_analysis,+ _run_decision_extraction,+ _run_health_analysis,+)+from .phases.generation import run_generation+from .phases.git import _run_git_indexing+from .phases.ingestion import _run_ingestion--# ----------------------------------------------------------------------------# Process-pool worker (module-level — must be picklable)-# -----------------------------------------------------------------------------# Module-level process-local parser cache (one per worker process).-_WORKER_PARSER: Any = None---def _parse_one(path_and_fi_and_bytes: tuple) -> Any:- """Worker function for ProcessPoolExecutor parsing.-- Constructs (or reuses) a process-local ASTParser and parses one file.- Returns a ParsedFile on success, or (abs_path_str, error_str) on failure.- The parser is constructed lazily inside the worker — the ASTParser itself- (which holds compiled tree-sitter Language/Query objects) is never pickled.- Only FileInfo (input) and ParsedFile (output) cross the process boundary;- both are plain dataclasses and therefore picklable.- """- global _WORKER_PARSER- fi, source = path_and_fi_and_bytes- try:- if _WORKER_PARSER is None:- from repowise.core.ingestion import ASTParser-- _WORKER_PARSER = ASTParser()- return _WORKER_PARSER.parse_file(fi, source)- except Exception as exc:- return (fi.abs_path, str(exc))---# Maximum seconds to spend on decision extraction before giving up.-# Large repos with tens of thousands of files can take arbitrarily long.-DECISION_EXTRACTION_TIMEOUT_SECS = 300+logger = structlog.get_logger(__name__)# ---------------------------------------------------------------------------@@ -638,670 +563,3 @@ async def run_pipeline(# ---------------------------------------------------------------------------# Phase helpers (private)# ------------------------------------------------------------------------------async def _run_ingestion(- repo_path: Path,- *,- exclude_patterns: list[str] | None,- include_submodules: bool = False,- include_nested_repos: bool = False,- skip_tests: bool,- skip_infra: bool,- progress: ProgressCallback | None,-) -> tuple[list[Any], list[Any], Any, dict[str, bytes], Any, Any]:- """Traverse, parse, and build the dependency graph.-- Returns (parsed_files, file_infos, repo_structure, source_map,- graph_builder, traversal_stats, tech_items).- """- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder-- traverser = FileTraverser(- repo_path,- extra_exclude_patterns=exclude_patterns or None,- include_submodules=include_submodules,- include_nested_repos=include_nested_repos,- )-- # Walk directory tree- all_paths = list(traverser._walk())- if progress:- # Use indeterminate progress (spinner) to avoid showing a misleading- # pre-filter total like "2132/83601".- progress.on_phase_start("traverse", None)-- # Parallel stat + header reads (I/O bound).- # Use asyncio.wrap_future so the event loop stays responsive while waiting.- file_infos: list[Any] = []- io_pool = ThreadPoolExecutor(max_workers=8)- try:- aws = [- asyncio.wrap_future(io_pool.submit(traverser._build_file_info, p)) for p in all_paths- ]- for coro in asyncio.as_completed(aws):- try:- result = await coro- except Exception:- result = None- if result is not None:- file_infos.append(result)- if progress:- progress.on_item_done("traverse")- finally:- # shutdown(wait=True) is blocking — run in a thread to keep the- # event loop responsive. All submitted futures have already- # completed by the time we reach here (the for-loop awaited them).- await asyncio.to_thread(io_pool.shutdown, wait=True)-- repo_structure = traverser.get_repo_structure(file_infos)- _phase_done(progress, "traverse")-- # Filter- if skip_tests:- file_infos = [fi for fi in file_infos if not fi.is_test]- if skip_infra:- file_infos = [- fi- for fi in file_infos- if fi.language not in ("dockerfile", "makefile", "terraform", "shell")- ]-- # ---- Parse phase: CPU-bound, run in ProcessPoolExecutor ----------------- if progress:- progress.on_phase_start("parse", len(file_infos))-- # Read source bytes up front (I/O, sequential — fast enough; keeps worker- # args small: FileInfo + bytes, both picklable plain dataclasses/bytes).- fi_and_bytes: list[tuple] = []- for fi in file_infos:- try:- source = Path(fi.abs_path).read_bytes()- fi_and_bytes.append((fi, source))- except Exception:- if progress:- progress.on_item_done("parse")-- parsed_files: list[Any] = []- source_map: dict[str, bytes] = {}- graph_builder = GraphBuilder(repo_path=repo_path)-- loop = asyncio.get_running_loop()- workers = max(1, os.cpu_count() or 4)-- _use_process_pool = True- parse_results: list[Any] = []-- try:- with ProcessPoolExecutor(max_workers=workers) as pool:- tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]- # Tick the parse-progress bar as each worker finishes —- # ``asyncio.gather`` would otherwise hold every event back- # until the last file is done, which on PowerToys-scale- # repos looked like a hang at ``0/N`` for many minutes.- # Per-task done-callbacks fire on the event loop thread and- # preserve gather's ordered results, so the aggregation- # loop below still indexes ``fi_and_bytes`` correctly.- if progress is not None:- _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731- for fut in tasks:- fut.add_done_callback(_parse_tick)- parse_results = await asyncio.gather(*tasks, return_exceptions=True)- except Exception as pool_exc:- logger.warning(- "process_pool_parse_failed_falling_back",- error=str(pool_exc),- )- _use_process_pool = False- # Fallback: in-process sequential parse- _fallback_parser = ASTParser()- for i, (fi, source) in enumerate(fi_and_bytes):- try:- result = _fallback_parser.parse_file(fi, source)- parse_results.append(result)- except Exception as exc:- parse_results.append((fi.abs_path, str(exc)))- if progress:- progress.on_item_done("parse")- if i % 50 == 49:- await asyncio.sleep(0)-- # Aggregate results into GraphBuilder on the main loop (not thread-safe).- for idx, result in enumerate(parse_results):- fi, source = fi_and_bytes[idx]- if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):- # Error tuple: (abs_path_str, error_str)- logger.debug("parse_error_in_worker", path=result[0], error=result[1])- elif isinstance(result, Exception):- logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))- else:- parsed_files.append(result)- source_map[fi.path] = source- graph_builder.add_file(result)- # Process-pool path already ticked per-file via the done-callback- # attached above; only the fallback path ticks here (handled in- # its own loop). No tick needed in aggregation.-- _phase_done(progress, "parse")-- # ---- tsconfig path-alias resolver (before graph build) ------------------- # Only runs when the repo has TS/JS files. On large TS monorepos the- # resolver indexes hundreds of tsconfig files up-front; without a phase- # label this shows up as a silent gap right after parsing.- try:- from repowise.core.ingestion.tsconfig_resolver import TsconfigResolver-- _ts_langs = {"typescript", "javascript"}- if any(pf.file_info.language in _ts_langs for pf in parsed_files):- if progress:- progress.on_phase_start("tsconfig", None)- _path_set = set(graph_builder._parsed_files.keys())- _resolver = TsconfigResolver(repo_path=repo_path, path_set=_path_set)- graph_builder.set_tsconfig_resolver(_resolver)- _phase_done(progress, "tsconfig")- except Exception as _resolver_exc:- logger.warning("tsconfig_resolver_init_failed", error=str(_resolver_exc))-- # ---- Graph build phase -------------------------------------------------- # Sub-phases (graph.imports / graph.heritage / graph.calls) are emitted- # from inside GraphBuilder.build(); the orchestrator drives metrics/- # communities/flows below so the longest-running step is no longer an- # opaque "graph 0/1" spinner.- if progress:- progress.on_message(- "info",- " (graph build can take several minutes on first run — safe to "- "Ctrl-C, then run 'repowise init --resume' to continue)",- )- await asyncio.to_thread(graph_builder.build, progress)-- # Add framework-aware synthetic edges (conftest, Django, FastAPI, Flask)- tech_items: list = []- try:- from repowise.core.generation.editor_files.tech_stack import detect_tech_stack-- tech_items = detect_tech_stack(repo_path)- graph_builder.add_framework_edges([item.name for item in tech_items])- except Exception:- pass # framework edge detection is best-effort-- # ---- Dynamic hints wiring (after static graph is fully built) ----------- if progress:- progress.on_phase_start("dynamic_hints", None)- try:- from repowise.core.ingestion.dynamic_hints import HintRegistry-- registry = HintRegistry()- dynamic_edges = await loop.run_in_executor(None, registry.extract_all, repo_path)- graph_builder.add_dynamic_edges(dynamic_edges)- logger.info("dynamic_hints_added", count=len(dynamic_edges))- except Exception as hints_exc:- logger.warning("dynamic_hints_failed", error=str(hints_exc))- _phase_done(progress, "dynamic_hints")-- # ---- Graph metrics: prime caches with live progress ---------------------- # pagerank/betweenness/community/symbol_communities/execution_flows are- # otherwise computed lazily during persist + generation, where they hide- # behind opaque progress bars. Pre-compute them here so each is its own- # visible sub-phase, and fan the within-phase work out via- # asyncio.gather so betweenness (the dominant cost) overlaps with- # PageRank / community detection rather than running serially.- #- # Each algorithm is wrapped in ``_timed_step`` so we emit a per-algo- # completion line (` ↳ PageRank ✓ (Xs)`) as it finishes. Without these,- # the whole gather looks like one opaque several-minute void where- # betweenness centrality dominates — splitting the timing makes it- # obvious which step is the bottleneck on a given repo.- if progress:- progress.on_phase_start("graph.metrics", None)- await asyncio.gather(- _timed_step("PageRank", graph_builder.pagerank, progress),- _timed_step("betweenness centrality", graph_builder.betweenness_centrality, progress),- _timed_step("symbol PageRank", graph_builder.symbol_pagerank, progress),- _timed_step("symbol betweenness", graph_builder.symbol_betweenness_centrality, progress),- )- _phase_done(progress, "graph.metrics")-- if progress:- progress.on_phase_start("graph.communities", None)- await asyncio.gather(- _timed_step("community detection", graph_builder.community_detection, progress),- _timed_step("symbol communities", graph_builder.symbol_communities, progress),- )- _phase_done(progress, "graph.communities")-- # Emit filtering summary so users can see what was included/excluded- stats = traverser.stats- if progress:- parts: list[str] = []- if stats.skipped_gitignore:- parts.append(f"{stats.skipped_gitignore:,} by .gitignore")- if stats.skipped_blocked_extension:- parts.append(f"{stats.skipped_blocked_extension:,} by extension")- if stats.skipped_blocked_pattern:- parts.append(f"{stats.skipped_blocked_pattern:,} by filename pattern")- if stats.skipped_oversized:- parts.append(f"{stats.skipped_oversized:,} oversized")- if stats.skipped_binary:- parts.append(f"{stats.skipped_binary:,} binary")- if stats.skipped_generated:- parts.append(f"{stats.skipped_generated:,} generated")- if stats.skipped_extra_exclude:- parts.append(f"{stats.skipped_extra_exclude:,} by --exclude")- if stats.skipped_extra_ignore:- parts.append(f"{stats.skipped_extra_ignore:,} by .repowiseIgnore")- if stats.skipped_submodule:- parts.append(f"{stats.skipped_submodule:,} submodule dirs")- if stats.skipped_nested_repo:- parts.append(f"{stats.skipped_nested_repo:,} nested git repos")- if stats.skipped_unknown_language:- parts.append(f"{stats.skipped_unknown_language:,} unknown type")-- excluded_str = ", ".join(parts) if parts else "none"- progress.on_message(- "info",- f"Scanned {stats.total_paths_walked:,} files, {len(file_infos):,} included",- )- if parts:- progress.on_message("info", f" Excluded: {excluded_str}")-- # Language breakdown- if stats.lang_counts:- top_langs = sorted(stats.lang_counts.items(), key=lambda x: -x[1])[:6]- lang_str = ", ".join(f"{lang} {count:,}" for lang, count in top_langs)- rest_count = sum(- c for _, c in sorted(stats.lang_counts.items(), key=lambda x: -x[1])[6:]- )- if rest_count:- lang_str += f", other {rest_count:,}"- progress.on_message("info", f" Languages: {lang_str}")-- return parsed_files, file_infos, repo_structure, source_map, graph_builder, stats, tech_items---async def _run_git_indexing(- repo_path: Path,- *,- commit_depth: int,- follow_renames: bool,- tier: GitIndexTier = GitIndexTier.FULL,- progress: ProgressCallback | None,-) -> tuple[Any | None, list[dict], dict[str, dict]]:- """Run git history indexing.-- Returns (git_summary, git_metadata_list, git_meta_map).- """- try:- from repowise.core.ingestion.git_indexer import GitIndexer-- git_indexer = GitIndexer(- repo_path,- commit_limit=commit_depth,- follow_renames=follow_renames,- tier=tier,- )-- def _on_start(total: int) -> None:- if progress:- progress.on_phase_start("git", total)-- def _on_file_done() -> None:- if progress:- progress.on_item_done("git")-- def _on_co_change_start(total: int) -> None:- if progress:- progress.on_phase_start("co_change", total)-- def _on_commit_done() -> None:- if progress:- progress.on_item_done("co_change")-- def _on_co_change_done() -> None:- # Stop the co_change timer the moment accumulation finishes;- # otherwise the recorded duration also includes the parallel- # per-file git walk that keeps running afterwards (audit #29).- _phase_done(progress, "co_change")-- git_summary, git_metadata_list = await git_indexer.index_repo(- "",- on_start=_on_start,- on_file_done=_on_file_done,- on_co_change_start=_on_co_change_start,- on_commit_done=_on_commit_done,- on_co_change_done=_on_co_change_done,- )- git_meta_map = {m["file_path"]: m for m in git_metadata_list}- _phase_done(progress, "git")- # co_change phase already closed inside the done-callback above;- # call again only as a safety-net in case the callback was never- # invoked (e.g. co-change skipped early). PhaseTimingRecorder- # ignores done-without-start so this is a no-op in the happy path.- _phase_done(progress, "co_change")- return git_summary, git_metadata_list, git_meta_map- except Exception as exc:- if progress:- progress.on_message("warning", f"Git indexing skipped: {exc}")- _phase_done(progress, "git")- _phase_done(progress, "co_change")- return None, [], {}---async def _run_dead_code_analysis(- graph_builder: Any,- git_meta_map: dict[str, dict],- *,- progress: ProgressCallback | None,-) -> Any | None:- """Run dead code detection (pure graph traversal, no LLM)."""- try:- from repowise.core.analysis.dead_code import DeadCodeAnalyzer-- # Four detectors run sequentially inside analyze(); drive a- # determinate bar so users see progress instead of "0".- _DEAD_CODE_STEPS = 4- if progress:- progress.on_phase_start("dead_code", _DEAD_CODE_STEPS)-- analyzer = DeadCodeAnalyzer(- graph_builder.graph(), git_meta_map, parsed_files=graph_builder._parsed_files- )-- def _step(_stage: str) -> None:- if progress:- progress.on_item_done("dead_code")-- report = await asyncio.to_thread(analyzer.analyze, None, on_step=_step)-- if progress:- unreachable = sum(1 for f in report.findings if f.kind.value == "unreachable_file")- unused_exports = sum(1 for f in report.findings if f.kind.value == "unused_export")- progress.on_message(- "info",- f"→ {unreachable} unreachable files · "- f"{unused_exports} unused exports · ~{report.deletable_lines:,} deletable lines",- )-- _phase_done(progress, "dead_code")- return report- except Exception as exc:- if progress:- progress.on_message("warning", f"Dead code detection skipped: {exc}")- _phase_done(progress, "dead_code")- return None---async def _run_health_analysis(- graph_builder: Any,- git_meta_map: dict[str, dict],- parsed_files: list[Any],- *,- repo_path: Path | None = None,- progress: ProgressCallback | None,-) -> Any | None:- """Run code-health analysis (complexity + biomarkers + scoring)."""- try:- from repowise.core.analysis.health import HealthAnalyzer- from repowise.core.analysis.health.config import HealthConfig-- if progress:- # Per-file determinate progress: one tick per parsed file.- progress.on_phase_start("health", len(parsed_files))-- # Build a {file_path → community label} map so per-file metrics- # carry a real module name, not None. Community detection is- # already computed for the graph view, so this is essentially- # free.- module_map: dict[str, str] = {}- try:- cd = graph_builder.community_detection()- ci = graph_builder.community_info()- for node_id, comm_id in cd.items():- info = ci.get(comm_id)- label = getattr(info, "label", None) if info else None- if label:- module_map[node_id] = label- except Exception as exc:- logger.debug("health_module_map_failed", error=str(exc))-- analyzer = HealthAnalyzer(- graph_builder.graph(),- git_meta_map=git_meta_map,- parsed_files=parsed_files,- module_map=module_map,- )-- # Load per-file override rules from `.repowise/health-rules.json`.- # Missing or malformed file → empty config (no-op).- analyzer_config: dict[str, object] | None = None- if repo_path is not None:- cfg = HealthConfig.load(repo_path)- if cfg.disabled_biomarkers or cfg.rules:- file_paths = [pf.file_info.path for pf in parsed_files]- analyzer_config = cfg.to_analyzer_config(file_paths)-- def _step(_path: str) -> None:- if progress:- progress.on_item_done("health")-- # Parallel path for repos large enough to benefit (tree-sitter- # releases the GIL during parsing, so asyncio.gather over a- # thread pool actually scales). Threshold chosen so small repos- # avoid the overhead.- if len(parsed_files) >= 500:- report = await analyzer.analyze_async(analyzer_config, on_step=_step)- else:- report = await asyncio.to_thread(analyzer.analyze, analyzer_config, on_step=_step)-- if progress:- findings_count = len(report.findings)- avg = report.kpis.get("average_health", 10.0)- worst = report.kpis.get("worst_performer_score", 10.0)- progress.on_message(- "info",- f"→ {findings_count} health findings · avg {avg}/10 · worst {worst}/10",- )-- _phase_done(progress, "health")- return report- except Exception as exc:- if progress:- progress.on_message("warning", f"Health analysis skipped: {exc}")- _phase_done(progress, "health")- return None---async def _run_decision_extraction(- repo_path: Path,- *,- llm_client: Any | None,- graph_builder: Any,- git_meta_map: dict[str, dict],- parsed_files: list[Any],- progress: ProgressCallback | None,-) -> Any | None:- """Extract architectural decisions from source and git history."""- try:- from repowise.core.analysis.decision_extractor import DecisionExtractor-- # Seven sources run concurrently inside extract_all(); drive a- # determinate bar so users see live progress.- _DECISION_STEPS = 7- if progress:- progress.on_phase_start("decisions", _DECISION_STEPS)-- extractor = DecisionExtractor(- repo_path=repo_path,- provider=llm_client,- graph=graph_builder.graph(),- git_meta_map=git_meta_map,- parsed_files=parsed_files,- )-- def _decision_step(_source: str) -> None:- if progress:- progress.on_item_done("decisions")-- report = await asyncio.wait_for(- extractor.extract_all(on_step=_decision_step),- timeout=DECISION_EXTRACTION_TIMEOUT_SECS,- )-- if progress:- bs = report.by_source- total_decisions = report.total_found- progress.on_message(- "info",- f"→ {total_decisions} decisions: "- f"{bs.get('inline_marker', 0)} inline · "- f"{bs.get('adr', 0)} ADR · "- f"{bs.get('changelog', 0)} changelog · "- f"{bs.get('pr', 0)} PR · "- f"{bs.get('git_archaeology', 0)} git · "- f"{bs.get('comment', 0)} comments · "- f"{bs.get('readme_mining', 0)} docs",- )-- _phase_done(progress, "decisions")- return report- except Exception as exc:- if progress:- progress.on_message("warning", f"Decision extraction skipped: {exc}")- _phase_done(progress, "decisions")- return None---async def run_generation(- *,- repo_path: Path,- parsed_files: list[Any],- source_map: dict[str, bytes],- graph_builder: Any,- repo_structure: Any,- git_meta_map: dict[str, dict],- llm_client: Any,- embedder: Any | None,- vector_store: Any | None,- concurrency: int,- progress: ProgressCallback | None,- resume: bool = False,- cost_tracker: Any | None = None,- generation_config: Any | None = None,- dead_code_report: Any | None = None,- decision_report: Any | None = None,- external_systems: list[dict] | None = None,-) -> list[Any]:- """Run LLM-powered page generation.-- Returns a list of ``GeneratedPage`` objects.- """- from repowise.core.generation import (- ContextAssembler,- JobSystem,- PageGenerator,- )- from repowise.core.persistence.vector_store import InMemoryVectorStore- from repowise.core.providers.embedding.base import MockEmbedder-- # Attach cost tracker to LLM client if available- if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):- llm_client._cost_tracker = cost_tracker-- from repowise.core.generation import GenerationConfig-- # Preserve all caller-supplied GenerationConfig fields (output language, cache flags,- # token budgets, etc.) and only override max_concurrency to match the resolved value.- # Falls back to defaults when the pipeline entry point did not thread one through.- base_config = generation_config if generation_config is not None else GenerationConfig()- config = replace(base_config, max_concurrency=concurrency)- assembler = ContextAssembler(config)-- # Resolve embedder and vector store- embedder_impl = embedder if embedder is not None else MockEmbedder()-- if vector_store is None:- vector_store = InMemoryVectorStore(embedder_impl)-- # Job system — use a temp-like dir under repo_path for checkpoints- jobs_dir = repo_path / ".repowise" / "jobs"- jobs_dir.mkdir(parents=True, exist_ok=True)- job_system = JobSystem(jobs_dir)-- repo_name = repo_path.name-- # Track generation progress. Onboarding pages get routed to their own- # phase so the terminal UI shows them as a distinct, named step rather- # than blending into the long file_page run.- _pages_done = 0-- def on_page_done(page_type: str) -> None:- nonlocal _pages_done- _pages_done += 1- if progress:- phase = "onboarding" if page_type == "onboarding" else "generation"- progress.on_item_done(phase)- # Push live cost update if the callback supports it- if cost_tracker is not None and hasattr(progress, "set_cost"):- progress.set_cost(cost_tracker.session_cost)-- if progress:- progress.on_phase_start("generation", None)-- def on_total_known(total: int) -> None:- if progress:- progress.on_phase_start("generation", total)-- def on_subphase(name: str, total: int | None) -> None:- """Start a distinct sub-phase (currently used only for onboarding)."""- if progress:- progress.on_phase_start(name, total)-- generator = PageGenerator(- llm_client,- assembler,- config,- vector_store=vector_store,- language=config.language,- )-- generated_pages = await generator.generate_all(- parsed_files,- source_map,- graph_builder,- repo_structure,- repo_name,- job_system=job_system,- on_page_done=on_page_done,- on_total_known=on_total_known,- on_subphase=on_subphase,- git_meta_map=git_meta_map if git_meta_map else None,- resume=resume,- repo_path=repo_path,- dead_code_report=dead_code_report,- decision_report=decision_report,- external_systems=external_systems,- )-- # Onboarding summary — count generated slots and surface which ones- # were gated out so the user can see the curated collection's state.- onboarding_generated = [p for p in generated_pages if p.page_type == "onboarding"]- promoted_present = {- p.metadata.get("onboarding_slot")- for p in generated_pages- if p.metadata.get("onboarding_slot")- and p.page_type in ("repo_overview", "architecture_diagram")- }- if progress:- if onboarding_generated or promoted_present:- slots_made = sorted(- {p.metadata.get("subkind", "?") for p in onboarding_generated} | promoted_present- )- progress.on_message(- "info",- f"Onboarding: {len(slots_made)}/8 slots — {', '.join(slots_made)}",- )- progress.on_message("info", f"Generated {len(generated_pages)} pages")- _phase_done(progress, "onboarding")- _phase_done(progress, "generation")-- return generated_pagesdiff --git a/packages/core/src/repowise/core/pipeline/phases/__init__.py b/packages/core/src/repowise/core/pipeline/phases/__init__.pynew file mode 100644index 000000000..248fa01d5--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/phases/__init__.py@@ -0,0 +1 @@+"""Pipeline phase implementations (ingestion, git, analysis, generation)."""diff --git a/packages/core/src/repowise/core/pipeline/phases/_common.py b/packages/core/src/repowise/core/pipeline/phases/_common.pynew file mode 100644index 000000000..8cdfb4dcd--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/phases/_common.py@@ -0,0 +1,19 @@+"""Helpers shared across pipeline phases."""++from __future__ import annotations++import contextlib++from repowise.core.pipeline.progress import ProgressCallback+++def _phase_done(progress: ProgressCallback | None, phase: str) -> None:+ """Best-effort call to ``progress.on_phase_done`` — older callbacks may+ not implement it, so fall back to a no-op silently.+ """+ if progress is None:+ return+ fn = getattr(progress, "on_phase_done", None)+ if callable(fn):+ with contextlib.suppress(Exception):+ fn(phase)diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.pynew file mode 100644index 000000000..8a0997665--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py@@ -0,0 +1,208 @@+"""Pipeline analysis phase.++Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in+orchestrator.py) imports these phase functions. No CLI/click/rich imports.+"""++from __future__ import annotations++import asyncio+from pathlib import Path+from typing import Any++import structlog++from repowise.core.pipeline.progress import ProgressCallback++from ._common import _phase_done++logger = structlog.get_logger(__name__)+++# Maximum seconds to spend on decision extraction before giving up.+# Large repos with tens of thousands of files can take arbitrarily long.+DECISION_EXTRACTION_TIMEOUT_SECS = 300+++async def _run_dead_code_analysis(+ graph_builder: Any,+ git_meta_map: dict[str, dict],+ *,+ progress: ProgressCallback | None,+) -> Any | None:+ """Run dead code detection (pure graph traversal, no LLM)."""+ try:+ from repowise.core.analysis.dead_code import DeadCodeAnalyzer++ # Four detectors run sequentially inside analyze(); drive a+ # determinate bar so users see progress instead of "0".+ dead_code_steps = 4+ if progress:+ progress.on_phase_start("dead_code", dead_code_steps)++ analyzer = DeadCodeAnalyzer(+ graph_builder.graph(), git_meta_map, parsed_files=graph_builder._parsed_files+ )++ def _step(_stage: str) -> None:+ if progress:+ progress.on_item_done("dead_code")++ report = await asyncio.to_thread(analyzer.analyze, None, on_step=_step)++ if progress:+ unreachable = sum(1 for f in report.findings if f.kind.value == "unreachable_file")+ unused_exports = sum(1 for f in report.findings if f.kind.value == "unused_export")+ progress.on_message(+ "info",+ f"→ {unreachable} unreachable files · "+ f"{unused_exports} unused exports · ~{report.deletable_lines:,} deletable lines",+ )++ _phase_done(progress, "dead_code")+ return report+ except Exception as exc:+ if progress:+ progress.on_message("warning", f"Dead code detection skipped: {exc}")+ _phase_done(progress, "dead_code")+ return None+++async def _run_health_analysis(+ graph_builder: Any,+ git_meta_map: dict[str, dict],+ parsed_files: list[Any],+ *,+ repo_path: Path | None = None,+ progress: ProgressCallback | None,+) -> Any | None:+ """Run code-health analysis (complexity + biomarkers + scoring)."""+ try:+ from repowise.core.analysis.health import HealthAnalyzer+ from repowise.core.analysis.health.config import HealthConfig++ if progress:+ # Per-file determinate progress: one tick per parsed file.+ progress.on_phase_start("health", len(parsed_files))++ # Build a {file_path → community label} map so per-file metrics+ # carry a real module name, not None. Community detection is+ # already computed for the graph view, so this is essentially+ # free.+ module_map: dict[str, str] = {}+ try:+ cd = graph_builder.community_detection()+ ci = graph_builder.community_info()+ for node_id, comm_id in cd.items():+ info = ci.get(comm_id)+ label = getattr(info, "label", None) if info else None+ if label:+ module_map[node_id] = label+ except Exception as exc:+ logger.debug("health_module_map_failed", error=str(exc))++ analyzer = HealthAnalyzer(+ graph_builder.graph(),+ git_meta_map=git_meta_map,+ parsed_files=parsed_files,+ module_map=module_map,+ )++ # Load per-file override rules from `.repowise/health-rules.json`.+ # Missing or malformed file → empty config (no-op).+ analyzer_config: dict[str, object] | None = None+ if repo_path is not None:+ cfg = HealthConfig.load(repo_path)+ if cfg.disabled_biomarkers or cfg.rules:+ file_paths = [pf.file_info.path for pf in parsed_files]+ analyzer_config = cfg.to_analyzer_config(file_paths)++ def _step(_path: str) -> None:+ if progress:+ progress.on_item_done("health")++ # Parallel path for repos large enough to benefit (tree-sitter+ # releases the GIL during parsing, so asyncio.gather over a+ # thread pool actually scales). Threshold chosen so small repos+ # avoid the overhead.+ if len(parsed_files) >= 500:+ report = await analyzer.analyze_async(analyzer_config, on_step=_step)+ else:+ report = await asyncio.to_thread(analyzer.analyze, analyzer_config, on_step=_step)++ if progress:+ findings_count = len(report.findings)+ avg = report.kpis.get("average_health", 10.0)+ worst = report.kpis.get("worst_performer_score", 10.0)+ progress.on_message(+ "info",+ f"→ {findings_count} health findings · avg {avg}/10 · worst {worst}/10",+ )++ _phase_done(progress, "health")+ return report+ except Exception as exc:+ if progress:+ progress.on_message("warning", f"Health analysis skipped: {exc}")+ _phase_done(progress, "health")+ return None+++async def _run_decision_extraction(+ repo_path: Path,+ *,+ llm_client: Any | None,+ graph_builder: Any,+ git_meta_map: dict[str, dict],+ parsed_files: list[Any],+ progress: ProgressCallback | None,+) -> Any | None:+ """Extract architectural decisions from source and git history."""+ try:+ from repowise.core.analysis.decision_extractor import DecisionExtractor++ # Seven sources run concurrently inside extract_all(); drive a+ # determinate bar so users see live progress.+ decision_steps = 7+ if progress:+ progress.on_phase_start("decisions", decision_steps)++ extractor = DecisionExtractor(+ repo_path=repo_path,+ provider=llm_client,+ graph=graph_builder.graph(),+ git_meta_map=git_meta_map,+ parsed_files=parsed_files,+ )++ def _decision_step(_source: str) -> None:+ if progress:+ progress.on_item_done("decisions")++ report = await asyncio.wait_for(+ extractor.extract_all(on_step=_decision_step),+ timeout=DECISION_EXTRACTION_TIMEOUT_SECS,+ )++ if progress:+ bs = report.by_source+ total_decisions = report.total_found+ progress.on_message(+ "info",+ f"→ {total_decisions} decisions: "+ f"{bs.get('inline_marker', 0)} inline · "+ f"{bs.get('adr', 0)} ADR · "+ f"{bs.get('changelog', 0)} changelog · "+ f"{bs.get('pr', 0)} PR · "+ f"{bs.get('git_archaeology', 0)} git · "+ f"{bs.get('comment', 0)} comments · "+ f"{bs.get('readme_mining', 0)} docs",+ )++ _phase_done(progress, "decisions")+ return report+ except Exception as exc:+ if progress:+ progress.on_message("warning", f"Decision extraction skipped: {exc}")+ _phase_done(progress, "decisions")+ return Nonediff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.pynew file mode 100644index 000000000..71e76d59d--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py@@ -0,0 +1,155 @@+"""Pipeline generation phase.++Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in+orchestrator.py) imports these phase functions. No CLI/click/rich imports.+"""++from __future__ import annotations++from dataclasses import replace+from pathlib import Path+from typing import Any++import structlog++from repowise.core.pipeline.progress import ProgressCallback++from ._common import _phase_done++logger = structlog.get_logger(__name__)+++async def run_generation(+ *,+ repo_path: Path,+ parsed_files: list[Any],+ source_map: dict[str, bytes],+ graph_builder: Any,+ repo_structure: Any,+ git_meta_map: dict[str, dict],+ llm_client: Any,+ embedder: Any | None,+ vector_store: Any | None,+ concurrency: int,+ progress: ProgressCallback | None,+ resume: bool = False,+ cost_tracker: Any | None = None,+ generation_config: Any | None = None,+ dead_code_report: Any | None = None,+ decision_report: Any | None = None,+ external_systems: list[dict] | None = None,+) -> list[Any]:+ """Run LLM-powered page generation.++ Returns a list of ``GeneratedPage`` objects.+ """+ from repowise.core.generation import (+ ContextAssembler,+ JobSystem,+ PageGenerator,+ )+ from repowise.core.persistence.vector_store import InMemoryVectorStore+ from repowise.core.providers.embedding.base import MockEmbedder++ # Attach cost tracker to LLM client if available+ if cost_tracker is not None and llm_client is not None and hasattr(llm_client, "_cost_tracker"):+ llm_client._cost_tracker = cost_tracker++ from repowise.core.generation import GenerationConfig++ # Preserve all caller-supplied GenerationConfig fields (output language, cache flags,+ # token budgets, etc.) and only override max_concurrency to match the resolved value.+ # Falls back to defaults when the pipeline entry point did not thread one through.+ base_config = generation_config if generation_config is not None else GenerationConfig()+ config = replace(base_config, max_concurrency=concurrency)+ assembler = ContextAssembler(config)++ # Resolve embedder and vector store+ embedder_impl = embedder if embedder is not None else MockEmbedder()++ if vector_store is None:+ vector_store = InMemoryVectorStore(embedder_impl)++ # Job system — use a temp-like dir under repo_path for checkpoints+ jobs_dir = repo_path / ".repowise" / "jobs"+ jobs_dir.mkdir(parents=True, exist_ok=True)+ job_system = JobSystem(jobs_dir)++ repo_name = repo_path.name++ # Track generation progress. Onboarding pages get routed to their own+ # phase so the terminal UI shows them as a distinct, named step rather+ # than blending into the long file_page run.+ _pages_done = 0++ def on_page_done(page_type: str) -> None:+ nonlocal _pages_done+ _pages_done += 1+ if progress:+ phase = "onboarding" if page_type == "onboarding" else "generation"+ progress.on_item_done(phase)+ # Push live cost update if the callback supports it+ if cost_tracker is not None and hasattr(progress, "set_cost"):+ progress.set_cost(cost_tracker.session_cost)++ if progress:+ progress.on_phase_start("generation", None)++ def on_total_known(total: int) -> None:+ if progress:+ progress.on_phase_start("generation", total)++ def on_subphase(name: str, total: int | None) -> None:+ """Start a distinct sub-phase (currently used only for onboarding)."""+ if progress:+ progress.on_phase_start(name, total)++ generator = PageGenerator(+ llm_client,+ assembler,+ config,+ vector_store=vector_store,+ language=config.language,+ )++ generated_pages = await generator.generate_all(+ parsed_files,+ source_map,+ graph_builder,+ repo_structure,+ repo_name,+ job_system=job_system,+ on_page_done=on_page_done,+ on_total_known=on_total_known,+ on_subphase=on_subphase,+ git_meta_map=git_meta_map if git_meta_map else None,+ resume=resume,+ repo_path=repo_path,+ dead_code_report=dead_code_report,+ decision_report=decision_report,+ external_systems=external_systems,+ )++ # Onboarding summary — count generated slots and surface which ones+ # were gated out so the user can see the curated collection's state.+ onboarding_generated = [p for p in generated_pages if p.page_type == "onboarding"]+ promoted_present = {+ p.metadata.get("onboarding_slot")+ for p in generated_pages+ if p.metadata.get("onboarding_slot")+ and p.page_type in ("repo_overview", "architecture_diagram")+ }+ if progress:+ if onboarding_generated or promoted_present:+ slots_made = sorted(+ {p.metadata.get("subkind", "?") for p in onboarding_generated} | promoted_present+ )+ progress.on_message(+ "info",+ f"Onboarding: {len(slots_made)}/8 slots — {', '.join(slots_made)}",+ )+ progress.on_message("info", f"Generated {len(generated_pages)} pages")+ _phase_done(progress, "onboarding")+ _phase_done(progress, "generation")++ return generated_pagesdiff --git a/packages/core/src/repowise/core/pipeline/phases/git.py b/packages/core/src/repowise/core/pipeline/phases/git.pynew file mode 100644index 000000000..06f7c7f2b--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/phases/git.py@@ -0,0 +1,87 @@+"""Pipeline git phase.++Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in+orchestrator.py) imports these phase functions. No CLI/click/rich imports.+"""++from __future__ import annotations++from pathlib import Path+from typing import Any++import structlog++from repowise.core.ingestion.git_indexer import GitIndexTier+from repowise.core.pipeline.progress import ProgressCallback++from ._common import _phase_done++logger = structlog.get_logger(__name__)+++async def _run_git_indexing(+ repo_path: Path,+ *,+ commit_depth: int,+ follow_renames: bool,+ tier: GitIndexTier = GitIndexTier.FULL,+ progress: ProgressCallback | None,+) -> tuple[Any | None, list[dict], dict[str, dict]]:+ """Run git history indexing.++ Returns (git_summary, git_metadata_list, git_meta_map).+ """+ try:+ from repowise.core.ingestion.git_indexer import GitIndexer++ git_indexer = GitIndexer(+ repo_path,+ commit_limit=commit_depth,+ follow_renames=follow_renames,+ tier=tier,+ )++ def _on_start(total: int) -> None:+ if progress:+ progress.on_phase_start("git", total)++ def _on_file_done() -> None:+ if progress:+ progress.on_item_done("git")++ def _on_co_change_start(total: int) -> None:+ if progress:+ progress.on_phase_start("co_change", total)++ def _on_commit_done() -> None:+ if progress:+ progress.on_item_done("co_change")++ def _on_co_change_done() -> None:+ # Stop the co_change timer the moment accumulation finishes;+ # otherwise the recorded duration also includes the parallel+ # per-file git walk that keeps running afterwards (audit #29).+ _phase_done(progress, "co_change")++ git_summary, git_metadata_list = await git_indexer.index_repo(+ "",+ on_start=_on_start,+ on_file_done=_on_file_done,+ on_co_change_start=_on_co_change_start,+ on_commit_done=_on_commit_done,+ on_co_change_done=_on_co_change_done,+ )+ git_meta_map = {m["file_path"]: m for m in git_metadata_list}+ _phase_done(progress, "git")+ # co_change phase already closed inside the done-callback above;+ # call again only as a safety-net in case the callback was never+ # invoked (e.g. co-change skipped early). PhaseTimingRecorder+ # ignores done-without-start so this is a no-op in the happy path.+ _phase_done(progress, "co_change")+ return git_summary, git_metadata_list, git_meta_map+ except Exception as exc:+ if progress:+ progress.on_message("warning", f"Git indexing skipped: {exc}")+ _phase_done(progress, "git")+ _phase_done(progress, "co_change")+ return None, [], {}diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.pynew file mode 100644index 000000000..a53078a3d--- /dev/null+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py@@ -0,0 +1,364 @@+"""Pipeline ingestion phase.++Extracted from the former monolithic ``orchestrator.py``; ``run_pipeline`` (in+orchestrator.py) imports these phase functions. No CLI/click/rich imports.+"""++from __future__ import annotations++import asyncio+import os+import time+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor+from pathlib import Path+from typing import Any++import structlog++from repowise.core.pipeline.progress import ProgressCallback++from ._common import _phase_done++logger = structlog.get_logger(__name__)+++async def _timed_step(+ label: str,+ fn: Any,+ progress: ProgressCallback | None,+) -> Any:+ """Run *fn* in a worker thread and emit a per-step completion line.++ Used to make ``asyncio.gather`` of multiple graph-algorithm calls+ legible: without this, four concurrent thread-bound computations+ (e.g. PageRank + betweenness + symbol PageRank + symbol betweenness)+ appear in the CLI as one opaque several-minute spinner, so the user+ has no signal as to which step is the bottleneck. With it, each algo+ prints ` ↳ <label> ✓ (Xs)` as it finishes — completion order+ surfaces the relative cost without changing the underlying execution.+ """+ t0 = time.monotonic()+ try:+ result = await asyncio.to_thread(fn)+ except Exception as exc:+ if progress is not None:+ progress.on_message(+ "warning",+ f" ↳ {label} failed after {time.monotonic() - t0:.1f}s: {exc}",+ )+ raise+ if progress is not None:+ progress.on_message(+ "info",+ f" ↳ {label} ✓ ({time.monotonic() - t0:.1f}s)",+ )+ return result+++# ---------------------------------------------------------------------------+# Process-pool worker (module-level — must be picklable)+# ---------------------------------------------------------------------------++# Module-level process-local parser cache (one per worker process).+_WORKER_PARSER: Any = None+++def _parse_one(path_and_fi_and_bytes: tuple) -> Any:+ """Worker function for ProcessPoolExecutor parsing.++ Constructs (or reuses) a process-local ASTParser and parses one file.+ Returns a ParsedFile on success, or (abs_path_str, error_str) on failure.+ The parser is constructed lazily inside the worker — the ASTParser itself+ (which holds compiled tree-sitter Language/Query objects) is never pickled.+ Only FileInfo (input) and ParsedFile (output) cross the process boundary;+ both are plain dataclasses and therefore picklable.+ """+ global _WORKER_PARSER+ fi, source = path_and_fi_and_bytes+ try:+ if _WORKER_PARSER is None:+ from repowise.core.ingestion import ASTParser++ _WORKER_PARSER = ASTParser()+ return _WORKER_PARSER.parse_file(fi, source)+ except Exception as exc:+ return (fi.abs_path, str(exc))+++async def _run_ingestion(+ repo_path: Path,+ *,+ exclude_patterns: list[str] | None,+ include_submodules: bool = False,+ include_nested_repos: bool = False,+ skip_tests: bool,+ skip_infra: bool,+ progress: ProgressCallback | None,+) -> tuple[list[Any], list[Any], Any, dict[str, bytes], Any, Any]:+ """Traverse, parse, and build the dependency graph.++ Returns (parsed_files, file_infos, repo_structure, source_map,+ graph_builder, traversal_stats, tech_items).+ """+ from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder++ traverser = FileTraverser(+ repo_path,+ extra_exclude_patterns=exclude_patterns or None,+ include_submodules=include_submodules,+ include_nested_repos=include_nested_repos,+ )++ # Walk directory tree+ all_paths = list(traverser._walk())+ if progress:+ # Use indeterminate progress (spinner) to avoid showing a misleading+ # pre-filter total like "2132/83601".+ progress.on_phase_start("traverse", None)++ # Parallel stat + header reads (I/O bound).+ # Use asyncio.wrap_future so the event loop stays responsive while waiting.+ file_infos: list[Any] = []+ io_pool = ThreadPoolExecutor(max_workers=8)+ try:+ aws = [+ asyncio.wrap_future(io_pool.submit(traverser._build_file_info, p)) for p in all_paths+ ]+ for coro in asyncio.as_completed(aws):+ try:+ result = await coro+ except Exception:+ result = None+ if result is not None:+ file_infos.append(result)+ if progress:+ progress.on_item_done("traverse")+ finally:+ # shutdown(wait=True) is blocking — run in a thread to keep the+ # event loop responsive. All submitted futures have already+ # completed by the time we reach here (the for-loop awaited them).+ await asyncio.to_thread(io_pool.shutdown, wait=True)++ repo_structure = traverser.get_repo_structure(file_infos)+ _phase_done(progress, "traverse")++ # Filter+ if skip_tests:+ file_infos = [fi for fi in file_infos if not fi.is_test]+ if skip_infra:+ file_infos = [+ fi+ for fi in file_infos+ if fi.language not in ("dockerfile", "makefile", "terraform", "shell")+ ]++ # ---- Parse phase: CPU-bound, run in ProcessPoolExecutor ----------------+ if progress:+ progress.on_phase_start("parse", len(file_infos))++ # Read source bytes up front (I/O, sequential — fast enough; keeps worker+ # args small: FileInfo + bytes, both picklable plain dataclasses/bytes).+ fi_and_bytes: list[tuple] = []+ for fi in file_infos:+ try:+ source = Path(fi.abs_path).read_bytes()+ fi_and_bytes.append((fi, source))+ except Exception:+ if progress:+ progress.on_item_done("parse")++ parsed_files: list[Any] = []+ source_map: dict[str, bytes] = {}+ graph_builder = GraphBuilder(repo_path=repo_path)++ loop = asyncio.get_running_loop()+ workers = max(1, os.cpu_count() or 4)++ _use_process_pool = True+ parse_results: list[Any] = []++ try:+ with ProcessPoolExecutor(max_workers=workers) as pool:+ tasks = [loop.run_in_executor(pool, _parse_one, item) for item in fi_and_bytes]+ # Tick the parse-progress bar as each worker finishes —+ # ``asyncio.gather`` would otherwise hold every event back+ # until the last file is done, which on PowerToys-scale+ # repos looked like a hang at ``0/N`` for many minutes.+ # Per-task done-callbacks fire on the event loop thread and+ # preserve gather's ordered results, so the aggregation+ # loop below still indexes ``fi_and_bytes`` correctly.+ if progress is not None:+ _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731+ for fut in tasks:+ fut.add_done_callback(_parse_tick)+ parse_results = await asyncio.gather(*tasks, return_exceptions=True)+ except Exception as pool_exc:+ logger.warning(+ "process_pool_parse_failed_falling_back",+ error=str(pool_exc),+ )+ _use_process_pool = False+ # Fallback: in-process sequential parse+ _fallback_parser = ASTParser()+ for i, (fi, source) in enumerate(fi_and_bytes):+ try:+ result = _fallback_parser.parse_file(fi, source)+ parse_results.append(result)+ except Exception as exc:+ parse_results.append((fi.abs_path, str(exc)))+ if progress:+ progress.on_item_done("parse")+ if i % 50 == 49:+ await asyncio.sleep(0)++ # Aggregate results into GraphBuilder on the main loop (not thread-safe).+ for idx, result in enumerate(parse_results):+ fi, source = fi_and_bytes[idx]+ if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):+ # Error tuple: (abs_path_str, error_str)+ logger.debug("parse_error_in_worker", path=result[0], error=result[1])+ elif isinstance(result, Exception):+ logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))+ else:+ parsed_files.append(result)+ source_map[fi.path] = source+ graph_builder.add_file(result)+ # Process-pool path already ticked per-file via the done-callback+ # attached above; only the fallback path ticks here (handled in+ # its own loop). No tick needed in aggregation.++ _phase_done(progress, "parse")++ # ---- tsconfig path-alias resolver (before graph build) ------------------+ # Only runs when the repo has TS/JS files. On large TS monorepos the+ # resolver indexes hundreds of tsconfig files up-front; without a phase+ # label this shows up as a silent gap right after parsing.+ try:+ from repowise.core.ingestion.tsconfig_resolver import TsconfigResolver++ _ts_langs = {"typescript", "javascript"}+ if any(pf.file_info.language in _ts_langs for pf in parsed_files):+ if progress:+ progress.on_phase_start("tsconfig", None)+ _path_set = set(graph_builder._parsed_files.keys())+ _resolver = TsconfigResolver(repo_path=repo_path, path_set=_path_set)+ graph_builder.set_tsconfig_resolver(_resolver)+ _phase_done(progress, "tsconfig")+ except Exception as _resolver_exc:+ logger.warning("tsconfig_resolver_init_failed", error=str(_resolver_exc))++ # ---- Graph build phase -------------------------------------------------+ # Sub-phases (graph.imports / graph.heritage / graph.calls) are emitted+ # from inside GraphBuilder.build(); the orchestrator drives metrics/+ # communities/flows below so the longest-running step is no longer an+ # opaque "graph 0/1" spinner.+ if progress:+ progress.on_message(+ "info",+ " (graph build can take several minutes on first run — safe to "+ "Ctrl-C, then run 'repowise init --resume' to continue)",+ )+ await asyncio.to_thread(graph_builder.build, progress)++ # Add framework-aware synthetic edges (conftest, Django, FastAPI, Flask)+ tech_items: list = []+ try:+ from repowise.core.generation.editor_files.tech_stack import detect_tech_stack++ tech_items = detect_tech_stack(repo_path)+ graph_builder.add_framework_edges([item.name for item in tech_items])+ except Exception:+ pass # framework edge detection is best-effort++ # ---- Dynamic hints wiring (after static graph is fully built) ----------+ if progress:+ progress.on_phase_start("dynamic_hints", None)+ try:+ from repowise.core.ingestion.dynamic_hints import HintRegistry++ registry = HintRegistry()+ dynamic_edges = await loop.run_in_executor(None, registry.extract_all, repo_path)+ graph_builder.add_dynamic_edges(dynamic_edges)+ logger.info("dynamic_hints_added", count=len(dynamic_edges))+ except Exception as hints_exc:+ logger.warning("dynamic_hints_failed", error=str(hints_exc))+ _phase_done(progress, "dynamic_hints")++ # ---- Graph metrics: prime caches with live progress ---------------------+ # pagerank/betweenness/community/symbol_communities/execution_flows are+ # otherwise computed lazily during persist + generation, where they hide+ # behind opaque progress bars. Pre-compute them here so each is its own+ # visible sub-phase, and fan the within-phase work out via+ # asyncio.gather so betweenness (the dominant cost) overlaps with+ # PageRank / community detection rather than running serially.+ #+ # Each algorithm is wrapped in ``_timed_step`` so we emit a per-algo+ # completion line (` ↳ PageRank ✓ (Xs)`) as it finishes. Without these,+ # the whole gather looks like one opaque several-minute void where+ # betweenness centrality dominates — splitting the timing makes it+ # obvious which step is the bottleneck on a given repo.+ if progress:+ progress.on_phase_start("graph.metrics", None)+ await asyncio.gather(+ _timed_step("PageRank", graph_builder.pagerank, progress),+ _timed_step("betweenness centrality", graph_builder.betweenness_centrality, progress),+ _timed_step("symbol PageRank", graph_builder.symbol_pagerank, progress),+ _timed_step("symbol betweenness", graph_builder.symbol_betweenness_centrality, progress),+ )+ _phase_done(progress, "graph.metrics")++ if progress:+ progress.on_phase_start("graph.communities", None)+ await asyncio.gather(+ _timed_step("community detection", graph_builder.community_detection, progress),+ _timed_step("symbol communities", graph_builder.symbol_communities, progress),+ )+ _phase_done(progress, "graph.communities")++ # Emit filtering summary so users can see what was included/excluded+ stats = traverser.stats+ if progress:+ parts: list[str] = []+ if stats.skipped_gitignore:+ parts.append(f"{stats.skipped_gitignore:,} by .gitignore")+ if stats.skipped_blocked_extension:+ parts.append(f"{stats.skipped_blocked_extension:,} by extension")+ if stats.skipped_blocked_pattern:+ parts.append(f"{stats.skipped_blocked_pattern:,} by filename pattern")+ if stats.skipped_oversized:+ parts.append(f"{stats.skipped_oversized:,} oversized")+ if stats.skipped_binary:+ parts.append(f"{stats.skipped_binary:,} binary")+ if stats.skipped_generated:+ parts.append(f"{stats.skipped_generated:,} generated")+ if stats.skipped_extra_exclude:+ parts.append(f"{stats.skipped_extra_exclude:,} by --exclude")+ if stats.skipped_extra_ignore:+ parts.append(f"{stats.skipped_extra_ignore:,} by .repowiseIgnore")+ if stats.skipped_submodule:+ parts.append(f"{stats.skipped_submodule:,} submodule dirs")+ if stats.skipped_nested_repo:+ parts.append(f"{stats.skipped_nested_repo:,} nested git repos")+ if stats.skipped_unknown_language:+ parts.append(f"{stats.skipped_unknown_language:,} unknown type")++ excluded_str = ", ".join(parts) if parts else "none"+ progress.on_message(+ "info",+ f"Scanned {stats.total_paths_walked:,} files, {len(file_infos):,} included",+ )+ if parts:+ progress.on_message("info", f" Excluded: {excluded_str}")++ # Language breakdown+ if stats.lang_counts:+ top_langs = sorted(stats.lang_counts.items(), key=lambda x: -x[1])[:6]+ lang_str = ", ".join(f"{lang} {count:,}" for lang, count in top_langs)+ rest_count = sum(+ c for _, c in sorted(stats.lang_counts.items(), key=lambda x: -x[1])[6:]+ )+ if rest_count:+ lang_str += f", other {rest_count:,}"+ progress.on_message("info", f" Languages: {lang_str}")++ return parsed_files, file_infos, repo_structure, source_map, graph_builder, stats, tech_items