Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(pipeline): surface progress for previously silent ingestion steps (#206)
Raghav Chamadiya1bacd90eb75e
diff --git a/packages/cli/src/repowise/cli/ui.py b/packages/cli/src/repowise/cli/ui.pyindex 7c23288a0..7a7ef2684 100644--- a/packages/cli/src/repowise/cli/ui.py+++ b/packages/cli/src/repowise/cli/ui.py@@ -1145,13 +1145,16 @@ class MaybeCountColumn(ProgressColumn):_PHASE_LABELS: dict[str, str] = {"traverse": "Scanning & filtering files...","parse": "Parsing files...",+ "tsconfig": "Indexing tsconfig path aliases...","graph": "Building dependency graph...","graph.imports": " ↳ Resolving imports","graph.heritage": " ↳ Resolving inheritance","graph.calls": " ↳ Resolving call edges",- "graph.metrics": " ↳ Computing PageRank / centrality",+ "dynamic_hints": " ↳ Wiring dynamic hints",+ "graph.metrics": " ↳ Computing graph metrics (PageRank, betweenness)","graph.communities": " ↳ Detecting communities","graph.flows": " ↳ Tracing execution flows",+ "external_systems": "Parsing external dependency manifests...","git": "Indexing file history...","co_change": "Analyzing co-changes...","dead_code": "Detecting dead code...",diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 05917741e..bef413052 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -43,6 +43,39 @@ def _phase_done(progress: ProgressCallback | None, phase: str) -> None:pass+async def _timed_step(+ label: str,+ fn: Any,+ progress: ProgressCallback | None,+) -> Any:+ """Run *fn* in a worker thread and emit a per-step completion line.++ Used to make ``asyncio.gather`` of multiple graph-algorithm calls+ legible: without this, four concurrent thread-bound computations+ (e.g. PageRank + betweenness + symbol PageRank + symbol betweenness)+ appear in the CLI as one opaque several-minute spinner, so the user+ has no signal as to which step is the bottleneck. With it, each algo+ prints ` ↳ <label> ✓ (Xs)` as it finishes — completion order+ surfaces the relative cost without changing the underlying execution.+ """+ t0 = time.monotonic()+ try:+ result = await asyncio.to_thread(fn)+ except Exception as exc:+ if progress is not None:+ progress.on_message(+ "warning",+ f" ↳ {label} failed after {time.monotonic() - t0:.1f}s: {exc}",+ )+ raise+ if progress is not None:+ progress.on_message(+ "info",+ f" ↳ {label} ✓ ({time.monotonic() - t0:.1f}s)",+ )+ return result++# ---------------------------------------------------------------------------# Process-pool worker (module-level — must be picklable)# ---------------------------------------------------------------------------@@ -286,6 +319,8 @@ async def run_pipeline(# Parse repo manifests for declared third-party dependencies. Failure here# must not break the pipeline — log and continue with an empty list.external_systems: list[dict] = []+ if progress:+ progress.on_phase_start("external_systems", None)try:from repowise.core.ingestion.external_systems import extract_external_systems@@ -309,6 +344,7 @@ async def run_pipeline()except Exception as _ext_err: # noqa: BLE001logger.warning("external_systems_extraction_failed", error=str(_ext_err))+ _phase_done(progress, "external_systems")# Emit rich insight summary for the ingestion phaseif progress:@@ -589,14 +625,20 @@ async def _run_ingestion(_phase_done(progress, "parse")# ---- tsconfig path-alias resolver (before graph build) ------------------+ # Only runs when the repo has TS/JS files. On large TS monorepos the+ # resolver indexes hundreds of tsconfig files up-front; without a phase+ # label this shows up as a silent gap right after parsing.try:from repowise.core.ingestion.tsconfig_resolver import TsconfigResolver_ts_langs = {"typescript", "javascript"}if any(pf.file_info.language in _ts_langs for pf in parsed_files):+ if progress:+ progress.on_phase_start("tsconfig", None)_path_set = set(graph_builder._parsed_files.keys())_resolver = TsconfigResolver(repo_path=repo_path, path_set=_path_set)graph_builder.set_tsconfig_resolver(_resolver)+ _phase_done(progress, "tsconfig")except Exception as _resolver_exc:logger.warning("tsconfig_resolver_init_failed", error=str(_resolver_exc))@@ -624,6 +666,8 @@ async def _run_ingestion(pass # framework edge detection is best-effort# ---- Dynamic hints wiring (after static graph is fully built) ----------+ if progress:+ progress.on_phase_start("dynamic_hints", None)try:from repowise.core.ingestion.dynamic_hints import HintRegistry@@ -633,6 +677,7 @@ async def _run_ingestion(logger.info("dynamic_hints_added", count=len(dynamic_edges))except Exception as hints_exc:logger.warning("dynamic_hints_failed", error=str(hints_exc))+ _phase_done(progress, "dynamic_hints")# ---- Graph metrics: prime caches with live progress ---------------------# pagerank/betweenness/community/symbol_communities/execution_flows are@@ -641,21 +686,27 @@ async def _run_ingestion(# visible sub-phase, and fan the within-phase work out via# asyncio.gather so betweenness (the dominant cost) overlaps with# PageRank / community detection rather than running serially.+ #+ # Each algorithm is wrapped in ``_timed_step`` so we emit a per-algo+ # completion line (` ↳ PageRank ✓ (Xs)`) as it finishes. Without these,+ # the whole gather looks like one opaque several-minute void where+ # betweenness centrality dominates — splitting the timing makes it+ # obvious which step is the bottleneck on a given repo.if progress:progress.on_phase_start("graph.metrics", None)await asyncio.gather(- asyncio.to_thread(graph_builder.pagerank),- asyncio.to_thread(graph_builder.betweenness_centrality),- asyncio.to_thread(graph_builder.symbol_pagerank),- asyncio.to_thread(graph_builder.symbol_betweenness_centrality),+ _timed_step("PageRank", graph_builder.pagerank, progress),+ _timed_step("betweenness centrality", graph_builder.betweenness_centrality, progress),+ _timed_step("symbol PageRank", graph_builder.symbol_pagerank, progress),+ _timed_step("symbol betweenness", graph_builder.symbol_betweenness_centrality, progress),)_phase_done(progress, "graph.metrics")if progress:progress.on_phase_start("graph.communities", None)await asyncio.gather(- asyncio.to_thread(graph_builder.community_detection),- asyncio.to_thread(graph_builder.symbol_communities),+ _timed_step("community detection", graph_builder.community_detection, progress),+ _timed_step("symbol communities", graph_builder.symbol_communities, progress),)_phase_done(progress, "graph.communities")