Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf: concurrent analysis phases and parallel incremental re-ingest (#894)

Raghav Chamadiya56bbeb2e915a
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index f965ad4b9..0570d9f0c 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -64,7 +64,10 @@ def build_repo_graph(
Returns ``(parsed_files, source_map, graph_builder, repo_structure,
file_count)``.
"""
- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder, compute_content_hash
+ import os
+ from concurrent.futures import ThreadPoolExecutor
+
+ from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
log = log or _noop_log
@@ -74,20 +77,28 @@ def build_repo_graph(
include_submodules=include_submodules,
include_nested_repos=include_nested_repos,
)
- file_infos = list(traverser.traverse())
- repo_structure = traverser.get_repo_structure()
-
- # Content-hash parse cache: an incremental update re-ingests the whole
- # repo, but only the changed files actually need a tree-sitter parse.
- # Best-effort — any cache failure falls back to a full parse.
- parse_cache = None
- try:
- from repowise.core.ingestion.parse_cache import ParseCache
+ # Parallel stat + header sniffing, mirroring the init ingestion phase.
+ # A serial traverse() pays per-file I/O latency sequentially; on a cold
+ # OS file cache that was ~50s of every PowerToys-scale update. Passing
+ # the file_infos into get_repo_structure also avoids its re-walk.
+ all_paths = list(traverser._walk())
+ io_workers = min(32, max(4, (os.cpu_count() or 4) * 2))
+ with ThreadPoolExecutor(max_workers=io_workers) as io_pool:
+ maybe_infos = list(io_pool.map(traverser._build_file_info, all_paths))
+ file_infos = [fi for fi in maybe_infos if fi is not None]
+ repo_structure = traverser.get_repo_structure(file_infos)
+
+ # Thread-pool source reads + content-hash parse cache split, shared with
+ # the init parse phase: only changed files need a tree-sitter parse.
+ # Cache failures degrade to all-miss (full parse), as before.
+ from repowise.core.pipeline.phases.ingestion import (
+ _cache_parsed,
+ _read_sources,
+ _split_cached,
+ )
- parse_cache = ParseCache(Path(repo_path) / ".repowise")
- parse_cache.load()
- except Exception:
- parse_cache = None
+ fi_and_bytes = _read_sources(file_infos, None)
+ parse_cache, cached_hits, to_parse = _split_cached(Path(repo_path), fi_and_bytes, None)
parser: Any = None # constructed lazily — every-file-cached updates skip query compilation
parsed_files: list = []
@@ -100,19 +111,26 @@ def build_repo_graph(
include_nested_repos=include_nested_repos,
)
- skipped = 0
- for fi in file_infos:
+ # Parse the misses in process and serially: on the update path they are
+ # change-sized. (Ceiling: a wiped/stale cache re-parses everything on one
+ # core; routing large miss counts through init's process pool would lift
+ # it, at the cost of Windows spawn overhead on every routine update.)
+ merged: dict[int, Any] = dict(cached_hits)
+ for idx, (fi, source), content_hash in to_parse:
try:
- source = Path(fi.abs_path).read_bytes()
- content_hash = compute_content_hash(source)
- parsed = parse_cache.get(fi, content_hash) if parse_cache is not None else None
- if parsed is None:
- if parser is None:
- parser = ASTParser()
- parsed = parser.parse_file(fi, source)
- if parse_cache is not None:
- parse_cache.put(parsed, content_hash)
+ if parser is None:
+ parser = ASTParser()
+ parsed = parser.parse_file(fi, source)
except Exception:
+ continue
+ merged[idx] = parsed
+ if content_hash:
+ _cache_parsed(parse_cache, parsed, content_hash)
+
+ skipped = len(file_infos) - len(fi_and_bytes) # unreadable files
+ for idx, (fi, source) in enumerate(fi_and_bytes):
+ parsed = merged.get(idx)
+ if parsed is None:
skipped += 1
continue
parsed_files.append(parsed)
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 4f7e7a116..f04891eab 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -460,33 +460,38 @@ async def run_pipeline(
skip_analysis = False
if not skip_analysis:
- dead_code_report = await _run_dead_code_analysis(
- graph_builder, git_meta_map, progress=progress
- )
-
- health_report = await _run_health_analysis(
- graph_builder,
- git_meta_map,
- parsed_files,
- repo_path=repo_path,
- coverage_report_paths=coverage_report_paths,
- progress=progress,
+ # The three analyses share read-only inputs (graph, git_meta_map,
+ # parsed_files; the lazy metric caches were warmed during ingestion)
+ # and have no data dependency on each other, so run them concurrently:
+ # decision extraction is I/O/LLM-bound and its wall clock hides
+ # entirely behind the CPU-bound dead-code + health work.
+ dead_code_report, health_report, decision_report = await asyncio.gather(
+ _run_dead_code_analysis(graph_builder, git_meta_map, progress=progress),
+ _run_health_analysis(
+ graph_builder,
+ git_meta_map,
+ parsed_files,
+ repo_path=repo_path,
+ coverage_report_paths=coverage_report_paths,
+ progress=progress,
+ ),
+ _run_decision_extraction(
+ repo_path,
+ llm_client=llm_client,
+ graph_builder=graph_builder,
+ git_meta_map=git_meta_map,
+ parsed_files=parsed_files,
+ progress=progress,
+ ),
)
# Drop the in-memory-only ``BlameIndex`` now that the health biomarkers
# have consumed it — before it can leak into ``PipelineResult`` and the
# downstream JSON artifact writers / DB persistence. ``git_meta_map``
- # shares these dict objects, so this cleans both views.
+ # shares these dict objects, so this cleans both views. Must stay after
+ # the gather: health reads the blame index while it runs (decision
+ # extraction never does).
drop_transient_git_signals(git_metadata_list)
-
- decision_report = await _run_decision_extraction(
- repo_path,
- llm_client=llm_client,
- graph_builder=graph_builder,
- git_meta_map=git_meta_map,
- parsed_files=parsed_files,
- progress=progress,
- )
gen_dead_code_report = dead_code_report
gen_decision_report = decision_report
diff --git a/tests/unit/pipeline/test_drop_transient_git_signals.py b/tests/unit/pipeline/test_drop_transient_git_signals.py
index dc3fd77b8..817a7ee68 100644
--- a/tests/unit/pipeline/test_drop_transient_git_signals.py
+++ b/tests/unit/pipeline/test_drop_transient_git_signals.py
@@ -46,6 +46,26 @@ def test_drop_is_idempotent_and_safe_without_blame_index() -> None:
assert meta == {"file_path": "b.py", "commit_count_total": 1}
+def test_decision_sources_never_read_blame_index() -> None:
+ # The orchestrator runs decision extraction concurrently with health via
+ # asyncio.gather, while the transient BlameIndex is still attached to the
+ # shared git metadata dicts (health consumes it; the drop happens after
+ # the gather). That is only sound while no decision source reads it.
+ from pathlib import Path
+
+ import repowise.core.analysis.decisions as decisions_pkg
+
+ pkg_dir = Path(decisions_pkg.__file__).parent
+ offenders = [
+ py.name for py in pkg_dir.rglob("*.py") if "blame_index" in py.read_text(encoding="utf-8")
+ ]
+ assert not offenders, (
+ f"decision sources reference blame_index ({offenders}); they run "
+ "concurrently with health before drop_transient_git_signals, so this "
+ "needs a re-think of the analysis-phase gather in orchestrator.py"
+ )
+
+
def test_drop_cleans_shared_git_meta_map_view() -> None:
# git_meta_map is built from the same dict objects as git_metadata_list, so
# stripping the list must clean the map view too.