Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf: concurrent analysis phases and parallel incremental re-ingest (#894)
Raghav Chamadiya56bbeb2e915a
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex f965ad4b9..0570d9f0c 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -64,7 +64,10 @@ def build_repo_graph(Returns ``(parsed_files, source_map, graph_builder, repo_structure,file_count)``."""- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder, compute_content_hash+ import os+ from concurrent.futures import ThreadPoolExecutor++ from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilderlog = log or _noop_log@@ -74,20 +77,28 @@ def build_repo_graph(include_submodules=include_submodules,include_nested_repos=include_nested_repos,)- file_infos = list(traverser.traverse())- repo_structure = traverser.get_repo_structure()-- # Content-hash parse cache: an incremental update re-ingests the whole- # repo, but only the changed files actually need a tree-sitter parse.- # Best-effort — any cache failure falls back to a full parse.- parse_cache = None- try:- from repowise.core.ingestion.parse_cache import ParseCache+ # Parallel stat + header sniffing, mirroring the init ingestion phase.+ # A serial traverse() pays per-file I/O latency sequentially; on a cold+ # OS file cache that was ~50s of every PowerToys-scale update. Passing+ # the file_infos into get_repo_structure also avoids its re-walk.+ all_paths = list(traverser._walk())+ io_workers = min(32, max(4, (os.cpu_count() or 4) * 2))+ with ThreadPoolExecutor(max_workers=io_workers) as io_pool:+ maybe_infos = list(io_pool.map(traverser._build_file_info, all_paths))+ file_infos = [fi for fi in maybe_infos if fi is not None]+ repo_structure = traverser.get_repo_structure(file_infos)++ # Thread-pool source reads + content-hash parse cache split, shared with+ # the init parse phase: only changed files need a tree-sitter parse.+ # Cache failures degrade to all-miss (full parse), as before.+ from repowise.core.pipeline.phases.ingestion import (+ _cache_parsed,+ _read_sources,+ _split_cached,+ )- parse_cache = ParseCache(Path(repo_path) / ".repowise")- parse_cache.load()- except Exception:- parse_cache = None+ fi_and_bytes = _read_sources(file_infos, None)+ parse_cache, cached_hits, to_parse = _split_cached(Path(repo_path), fi_and_bytes, None)parser: Any = None # constructed lazily — every-file-cached updates skip query compilationparsed_files: list = []@@ -100,19 +111,26 @@ def build_repo_graph(include_nested_repos=include_nested_repos,)- skipped = 0- for fi in file_infos:+ # Parse the misses in process and serially: on the update path they are+ # change-sized. (Ceiling: a wiped/stale cache re-parses everything on one+ # core; routing large miss counts through init's process pool would lift+ # it, at the cost of Windows spawn overhead on every routine update.)+ merged: dict[int, Any] = dict(cached_hits)+ for idx, (fi, source), content_hash in to_parse:try:- source = Path(fi.abs_path).read_bytes()- content_hash = compute_content_hash(source)- parsed = parse_cache.get(fi, content_hash) if parse_cache is not None else None- if parsed is None:- if parser is None:- parser = ASTParser()- parsed = parser.parse_file(fi, source)- if parse_cache is not None:- parse_cache.put(parsed, content_hash)+ if parser is None:+ parser = ASTParser()+ parsed = parser.parse_file(fi, source)except Exception:+ continue+ merged[idx] = parsed+ if content_hash:+ _cache_parsed(parse_cache, parsed, content_hash)++ skipped = len(file_infos) - len(fi_and_bytes) # unreadable files+ for idx, (fi, source) in enumerate(fi_and_bytes):+ parsed = merged.get(idx)+ if parsed is None:skipped += 1continueparsed_files.append(parsed)diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 4f7e7a116..f04891eab 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -460,33 +460,38 @@ async def run_pipeline(skip_analysis = Falseif not skip_analysis:- dead_code_report = await _run_dead_code_analysis(- graph_builder, git_meta_map, progress=progress- )-- health_report = await _run_health_analysis(- graph_builder,- git_meta_map,- parsed_files,- repo_path=repo_path,- coverage_report_paths=coverage_report_paths,- progress=progress,+ # The three analyses share read-only inputs (graph, git_meta_map,+ # parsed_files; the lazy metric caches were warmed during ingestion)+ # and have no data dependency on each other, so run them concurrently:+ # decision extraction is I/O/LLM-bound and its wall clock hides+ # entirely behind the CPU-bound dead-code + health work.+ dead_code_report, health_report, decision_report = await asyncio.gather(+ _run_dead_code_analysis(graph_builder, git_meta_map, progress=progress),+ _run_health_analysis(+ graph_builder,+ git_meta_map,+ parsed_files,+ repo_path=repo_path,+ coverage_report_paths=coverage_report_paths,+ progress=progress,+ ),+ _run_decision_extraction(+ repo_path,+ llm_client=llm_client,+ graph_builder=graph_builder,+ git_meta_map=git_meta_map,+ parsed_files=parsed_files,+ progress=progress,+ ),)# Drop the in-memory-only ``BlameIndex`` now that the health biomarkers# have consumed it — before it can leak into ``PipelineResult`` and the# downstream JSON artifact writers / DB persistence. ``git_meta_map``- # shares these dict objects, so this cleans both views.+ # shares these dict objects, so this cleans both views. Must stay after+ # the gather: health reads the blame index while it runs (decision+ # extraction never does).drop_transient_git_signals(git_metadata_list)-- decision_report = await _run_decision_extraction(- repo_path,- llm_client=llm_client,- graph_builder=graph_builder,- git_meta_map=git_meta_map,- parsed_files=parsed_files,- progress=progress,- )gen_dead_code_report = dead_code_reportgen_decision_report = decision_reportdiff --git a/tests/unit/pipeline/test_drop_transient_git_signals.py b/tests/unit/pipeline/test_drop_transient_git_signals.pyindex dc3fd77b8..817a7ee68 100644--- a/tests/unit/pipeline/test_drop_transient_git_signals.py+++ b/tests/unit/pipeline/test_drop_transient_git_signals.py@@ -46,6 +46,26 @@ def test_drop_is_idempotent_and_safe_without_blame_index() -> None:assert meta == {"file_path": "b.py", "commit_count_total": 1}+def test_decision_sources_never_read_blame_index() -> None:+ # The orchestrator runs decision extraction concurrently with health via+ # asyncio.gather, while the transient BlameIndex is still attached to the+ # shared git metadata dicts (health consumes it; the drop happens after+ # the gather). That is only sound while no decision source reads it.+ from pathlib import Path++ import repowise.core.analysis.decisions as decisions_pkg++ pkg_dir = Path(decisions_pkg.__file__).parent+ offenders = [+ py.name for py in pkg_dir.rglob("*.py") if "blame_index" in py.read_text(encoding="utf-8")+ ]+ assert not offenders, (+ f"decision sources reference blame_index ({offenders}); they run "+ "concurrently with health before drop_transient_git_signals, so this "+ "needs a re-think of the analysis-phase gather in orchestrator.py"+ )++def test_drop_cleans_shared_git_meta_map_view() -> None:# git_meta_map is built from the same dict objects as git_metadata_list, so# stripping the list must clean the map view too.