Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(update): a one-file update sized by what changed (#2134)
Raghav Chamadiyae681a29ba12b
diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.pyindex e46947bf3..5f4112038 100644--- a/packages/core/src/repowise/core/analysis/health/engine.py+++ b/packages/core/src/repowise/core/analysis/health/engine.py@@ -67,6 +67,7 @@ from .refactoring import (from .refactoring.graph_signals import build_file_scc_index, build_methods_by_filefrom .scoring import attach_impacts, compute_kpis, remap_severities, score_filefrom .source_reader import SourceReader, disk_source_reader+from .walk_cache import HealthWalkCachelog = structlog.get_logger(__name__)@@ -314,6 +315,14 @@ class HealthAnalyzer:# repo's ``.repowise``). None disables caching — the duplication# pass then re-tokenizes everything, exactly as before.self.duplication_cache_dir = duplication_cache_dir+ # The walk result for a file is a function of its bytes, its language+ # and the walker's version, so it is kept beside the index like the+ # parse trees and the clone windows are. Absent without a cache dir.+ self._walk_cache: HealthWalkCache | None = (+ HealthWalkCache(duplication_cache_dir, HEALTH_ANALYZER_VERSION)+ if duplication_cache_dir is not None+ else None+ )# Checkout root, used only to read package boundaries off disk. None# falls back to inferring them from the analyzed file list, which sees# only the manifests the traverser emitted.@@ -385,6 +394,8 @@ class HealthAnalyzer:on_step: Any | None = None,changed_files: set[str] | list[str] | None = None,repo_function_mod_p80: int | None = None,+ timings: Any | None = None,+ duplication_files: set[str] | None = None,) -> HealthReport:"""Analyze the configured parsed files.@@ -403,7 +414,18 @@ class HealthAnalyzer:the percentile computed over the full repo (from the persisted``git_function_blame`` rollup) instead. ``None`` (the default)computes it from the walked set as before.++ *timings* is a phase table; each stage of this pass records an+ ``analysis.health.*`` row so a slow pass names the stage.++ *duplication_files* narrows the clone detector's incremental splice to+ the files whose bytes changed. It defaults to *changed_files*, which on+ an update also carries the performance closure: files re-walked for+ their call paths, whose tokens did not move and whose clone pairs are+ already in the persisted index."""+ from repowise.core.pipeline.phase_timing import timed+cfg = config or {}disabled: list[str] = list(cfg.get("disabled_biomarkers", ()))per_file_disabled: dict[str, set[str]] = cfg.get("per_file_disabled", {}) or {}@@ -431,13 +453,18 @@ class HealthAnalyzer:dup_report = DuplicationReport()else:try:- dup_report = detect_clones(- self.parsed_files,- self.git_meta_map,- cache_dir=self.duplication_cache_dir,- source_reader=self.read_source,- changed_files=changed_set,- )+ with timed(timings, "analysis.health.duplication"):+ dup_report = detect_clones(+ self.parsed_files,+ self.git_meta_map,+ cache_dir=self.duplication_cache_dir,+ source_reader=self.read_source,+ changed_files=(+ set(duplication_files)+ if duplication_files is not None+ else changed_set+ ),+ )_log_duplication_diagnostics(dup_report)except Exception as exc:log.debug("health_duplication_failed", error=str(exc))@@ -458,6 +485,10 @@ class HealthAnalyzer:# per-function modification counts ONCE before any biomarker runs.# The walked list is reused by the per-file biomarker stage below.walked: list[tuple[Any, FileComplexity]] = []+ timings_walk = timed(timings, "analysis.health.walk")+ timings_walk.__enter__()+ if self._walk_cache is not None:+ self._walk_cache.load()for pf in self.parsed_files:if changed_set is not None and pf.file_info.path not in changed_set:continue@@ -471,17 +502,21 @@ class HealthAnalyzer:# evaluate); see analyze_async.if on_step:on_step(pf.file_info.path)-- repo_fn_mod_p80 = (- repo_function_mod_p80- if repo_function_mod_p80 is not None- else _compute_repo_function_mod_p80(walked, self.git_meta_map)- )- repo_dependents_p80 = _compute_repo_dependents_p80(self.parsed_files, self.graph)- repo_active_contributors = _compute_repo_active_contributors(self.git_meta_map)+ self._save_walk_cache()+ timings_walk.__exit__(None, None, None)++ with timed(timings, "analysis.health.repo_stats"):+ repo_fn_mod_p80 = (+ repo_function_mod_p80+ if repo_function_mod_p80 is not None+ else _compute_repo_function_mod_p80(walked, self.git_meta_map)+ )+ repo_dependents_p80 = _compute_repo_dependents_p80(self.parsed_files, self.graph)+ repo_active_contributors = _compute_repo_active_contributors(self.git_meta_map)# Cross-function N+1: augment perf_hits before the biomarker stage.- self._apply_crossfn_perf(walked)+ with timed(timings, "analysis.health.crossfn"):+ self._apply_crossfn_perf(walked)# One shared dataflow service for the whole pass: the promotion pass# and the Extract Method detector below read the same lazily parsed# per-file object, so no file is parsed twice for dataflow.@@ -489,8 +524,11 @@ class HealthAnalyzer:# Dataflow promotion: mark advisory perf hits whose loop is provably# iteration-independent (runs after the graph passes so the# centrality-gated nested-loop hits are present to promote).- apply_perf_promotions(walked, dataflow=dataflow_cache)+ with timed(timings, "analysis.health.promotions"):+ apply_perf_promotions(walked, dataflow=dataflow_cache)+ timings_evaluate = timed(timings, "analysis.health.evaluate")+ timings_evaluate.__enter__()for pf, fcx in walked:# Side-effect: bump Symbol.complexity_estimate when we can# match by enclosing line range. Symbols not matched keep@@ -530,6 +568,7 @@ class HealthAnalyzer:if on_step:on_step(pf.file_info.path)+ timings_evaluate.__exit__(None, None, None)# KPIs are repo-wide; on an incremental run they would be biased# by the changed-files subset. Skip them in that case — the@@ -540,6 +579,8 @@ class HealthAnalyzer:else:kpis = {}+ timings_finalize = timed(timings, "analysis.health.finalize")+ timings_finalize.__enter__()self._mark_perf_entry_reachability(findings)link_performance_findings(findings)opportunities = build_performance_opportunities(findings)@@ -554,6 +595,7 @@ class HealthAnalyzer:suggestions = rank_suggestions(suggestions, centrality=self._refactoring_centrality(suggestions))+ timings_finalize.__exit__(None, None, None)return HealthReport(repo_id="",analyzed_at=datetime.now(UTC),@@ -644,6 +686,8 @@ class HealthAnalyzer:# Pre-walk in worker threads so each task hands a list of# FunctionComplexity entries to the synchronous biomarker stage.# tree-sitter parsing releases the GIL → real parallelism here.+ if self._walk_cache is not None:+ self._walk_cache.load()workers = max(1, int(max_workers or os.cpu_count() or 4))semaphore = asyncio.Semaphore(workers)@@ -875,13 +919,35 @@ class HealthAnalyzer:source = self.read_source(path)if source is None:return FileComplexity(functions=[], classes=[])+ key = None+ if self._walk_cache is not None:+ from repowise.core.ingestion import compute_content_hash++ key = HealthWalkCache.key(language, compute_content_hash(source))+ cached = self._walk_cache.get(key)+ if cached is not None:+ return cachedif language == "sql":# SQL has no tree-sitter grammar here; the sqlglot-backed walker# produces routine CCN + the sql_* smell hits instead.from .sql_complexity import walk_sql_file- return walk_sql_file(pf.file_info, source)- return walk_file(path, language, source)+ fcx = walk_sql_file(pf.file_info, source)+ else:+ fcx = walk_file(path, language, source)+ if key is not None and self._walk_cache is not None:+ self._walk_cache.put(key, fcx)+ return fcx++ def _save_walk_cache(self) -> None:+ """Persist the walk entries this pass used or produced, if any."""+ if self._walk_cache is not None:+ self._walk_cache.save()+ log.debug(+ "health_walk_cache",+ hits=self._walk_cache.hits,+ misses=self._walk_cache.misses,+ )def _extract_method_analyses(self,diff --git a/packages/core/src/repowise/core/analysis/health/walk_cache.py b/packages/core/src/repowise/core/analysis/health/walk_cache.pynew file mode 100644index 000000000..cce8d8b81--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/walk_cache.py@@ -0,0 +1,104 @@+"""Content-keyed cache of the complexity walk, kept beside the index.++The walk over a file is a pure function of its bytes, its language and the+walker's version, and on an update most of the files the health pass walks+did not change: the performance closure re-walks every file whose call path+reaches a changed sink, so the walk was two fifths of a one-file update.+This keeps each file's :class:`FileComplexity` under its content hash the way+the parse cache keeps parse trees, so an unchanged file's walk is a lookup.++Entries are stored as pickled bytes and unpickled on every hit. The pass+mutates the walk result it is handed (cross-function facts, promotions), and+a shared live object would carry one run's annotations into the next; a+fresh copy per hit cannot.+"""++from __future__ import annotations++import pickle+from pathlib import Path+from typing import Any++import structlog++from repowise.core.cache_seal import dump_sealed_pickle, load_sealed_pickle++log = structlog.get_logger(__name__)++_CACHE_FILENAME = "health_walk_cache.pkl"+#: Bump when the walker's output for the same bytes changes.+_CACHE_VERSION = 1+++class HealthWalkCache:+ """``(language, content hash) -> pickled FileComplexity`` for one repository.++ ``load`` reads the previous run's entries; ``get`` and ``put`` serve the+ walk; ``save`` writes back the entries this run used or created, so a file+ that left the repository ages out with it.+ """++ def __init__(self, cache_dir: Path | str, analyzer_version: int) -> None:+ self._path = Path(cache_dir) / _CACHE_FILENAME+ self._analyzer_version = analyzer_version+ self._entries: dict[str, bytes] = {}+ self._fresh: dict[str, bytes] = {}+ self.hits = 0+ self.misses = 0++ @staticmethod+ def key(language: str, content_hash: str) -> str:+ return f"{language}:{content_hash}"++ def load(self) -> None:+ try:+ payload = load_sealed_pickle(self._path, domain=_CACHE_FILENAME)+ if (+ payload.get("version") != _CACHE_VERSION+ or payload.get("analyzer_version") != self._analyzer_version+ ):+ return+ entries = payload.get("files")+ if isinstance(entries, dict):+ self._entries = entries+ except FileNotFoundError:+ return+ except Exception as exc: # corrupt, unsigned or unreadable: walk everything+ log.debug("health_walk_cache_load_failed", error=str(exc))++ def get(self, key: str) -> Any | None:+ blob = self._fresh.get(key) or self._entries.get(key)+ if blob is None:+ self.misses += 1+ return None+ try:+ value = pickle.loads(blob)+ except Exception:+ self.misses += 1+ return None+ self.hits += 1+ self._fresh[key] = blob+ return value++ def put(self, key: str, value: Any) -> None:+ try:+ self._fresh[key] = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)+ except Exception as exc: # an unpicklable walk result is simply not cached+ log.debug("health_walk_cache_put_failed", error=str(exc))++ def save(self) -> None:+ """Atomically persist the entries used or created this run."""+ if not self._fresh:+ return+ try:+ dump_sealed_pickle(+ self._path,+ {+ "version": _CACHE_VERSION,+ "analyzer_version": self._analyzer_version,+ "files": self._fresh,+ },+ domain=_CACHE_FILENAME,+ )+ except Exception as exc:+ log.debug("health_walk_cache_save_failed", error=str(exc))diff --git a/packages/core/src/repowise/core/ingestion/git_commit_index.py b/packages/core/src/repowise/core/ingestion/git_commit_index.pyindex 8db6a6279..90ff23ec6 100644--- a/packages/core/src/repowise/core/ingestion/git_commit_index.py+++ b/packages/core/src/repowise/core/ingestion/git_commit_index.py@@ -19,6 +19,7 @@ input file, not retro-fittable from a repo-wide log.from __future__ import annotations+from pathlib import Pathfrom typing import TYPE_CHECKINGimport structlog@@ -75,6 +76,111 @@ def load_git_ai_note_agents(repo: object, commit_limit: int) -> dict[str, str]:return agents+#: Cache file name under the repository's ``.repowise`` directory, and the+#: shape version it carries. Bump the version when ``_LOG_FORMAT`` or the+#: record layout changes, so an older cache is re-walked rather than misread.+_WINDOW_CACHE_NAME = "commit_window_cache.json"+_WINDOW_CACHE_VERSION = 1+++def _record_ts(record: str) -> int:+ """The committer timestamp of one raw record, or 0 when unparseable."""+ from .git_indexer import _FIELD_SEP++ parts = record.split(_FIELD_SEP, 6)+ try:+ return int(parts[5])+ except (IndexError, ValueError):+ return 0+++def _window_records(repo: object, depth: int, cache_dir: Path | None) -> list[str]:+ """The raw log records of the newest ``depth`` non-merge commits.++ Git computes ``--numstat`` for every commit in the window, and on a+ repository with wide commits that walk costs seconds however few commits+ are new. With *cache_dir* the records from the last walk are kept beside+ the index, keyed by the HEAD they were taken at, and the next call asks git+ only for ``cached_head..HEAD``. The merged list is ordered the way the+ full walk orders it (committer time, newest first) and cut to ``depth``,+ so the parse below sees the same records a fresh walk would produce.++ A cached HEAD that is not an ancestor of the current one (a rebase, a+ reset) or a cache of another depth or version is discarded and the full+ walk runs. Every failure on the cache path falls back to the full walk;+ the cache can only ever save time, never change the answer.+ """+ import json++ from .git_indexer import _LOG_FORMAT, _RECORD_SEP++ def _full_walk() -> list[str]:+ raw = repo.git.log( # type: ignore[attr-defined]+ f"-{depth}", "--numstat", "--no-merges", f"--format={_LOG_FORMAT}"+ )+ return [rec for rec in raw.split(_RECORD_SEP) if rec.strip()]++ if cache_dir is None:+ return _full_walk()++ cache_path = Path(cache_dir) / _WINDOW_CACHE_NAME+ try:+ head = repo.head.commit.hexsha # type: ignore[attr-defined]+ except Exception:+ return _full_walk()++ records: list[str] | None = None+ try:+ cached = json.loads(cache_path.read_text(encoding="utf-8"))+ if (+ cached.get("version") == _WINDOW_CACHE_VERSION+ and cached.get("depth") == depth+ and isinstance(cached.get("records"), list)+ ):+ cached_head = str(cached.get("head") or "")+ if cached_head == head:+ records = list(cached["records"])+ elif cached_head:+ # Raises when the cached head is unknown to this repository or+ # not behind HEAD, which is exactly when the cache is stale.+ repo.git.merge_base("--is-ancestor", cached_head, head) # type: ignore[attr-defined]+ raw = repo.git.log( # type: ignore[attr-defined]+ f"{cached_head}..{head}",+ "--numstat",+ "--no-merges",+ f"--format={_LOG_FORMAT}",+ )+ fresh = [rec for rec in raw.split(_RECORD_SEP) if rec.strip()]+ merged = fresh + list(cached["records"])+ # Stable, so a fresh record stays ahead of a cached one at the+ # same second, which is where the full walk puts it too.+ merged.sort(key=_record_ts, reverse=True)+ records = merged[:depth]+ except Exception:+ records = None++ if records is None:+ records = _full_walk()++ try:+ from repowise.core.fsutils import atomic_write_text++ atomic_write_text(+ cache_path,+ json.dumps(+ {+ "version": _WINDOW_CACHE_VERSION,+ "head": head,+ "depth": depth,+ "records": records,+ }+ ),+ )+ except Exception as exc:+ logger.debug("commit_window_cache_write_failed", error=str(exc))+ return records++def load_commit_index(repo: object,commit_limit: int,@@ -84,6 +190,7 @@ def load_commit_index(since_ts: int | None = None,provenance_classifier: object | None = None,trace_index: object | None = None,+ cache_dir: Path | None = None,) -> dict[str, list[_CommitRec]]:"""Bucket every commit in the recent history by the files it touched.@@ -121,14 +228,17 @@ def load_commit_index(pattern registry; callers with repo-local pattern extensions pass theconfig-aware instance instead.+ *cache_dir*, when given, keeps the window's raw records between runs so+ only the commits since the last walk are asked of git (see+ :func:`_window_records`). Ignored with *since_ts*, whose own bound already+ keeps that walk short.+Failures (git unavailable, corrupt log output, etc.) return anempty dict so the caller can fall back to per-file indexing."""# Imported here to avoid a circular import — these live in git_indexer's# records module and this module is imported from there.from .git_indexer import (- _LOG_FORMAT,- _RECORD_SEP,_CommitRec,_extract_rename_paths,_parse_commit_record,@@ -172,17 +282,12 @@ def load_commit_index(return {}try:- raw = repo.git.log( # type: ignore[attr-defined]- f"-{depth}",- "--numstat",- "--no-merges",- f"--format={_LOG_FORMAT}",- )+ records = _window_records(repo, depth, cache_dir if since_ts is None else None)except Exception as exc:logger.warning("repo_commit_index_failed", error=str(exc))return {}- if not raw:+ if not records:return {}# git-ai authorship notes for this window (``{}`` unless the repo uses them).@@ -201,9 +306,7 @@ def load_commit_index(# Split on the NUL record separator rather than newlines: commit bodies# (``%b``) are multi-line, so a line-based scan would mistake body lines# for numstat rows. The first chunk before the leading separator is empty.- for record in raw.split(_RECORD_SEP):- if not record.strip():- continue+ for record in records:parsed = _parse_commit_record(record)if parsed is None:continuediff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.pyindex b6ed1ae22..aba54d6ae 100644--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py@@ -168,6 +168,7 @@ class GitIndexer:commit_sink=commit_sink,provenance_classifier=prov_clf,trace_index=trace_index,+ cache_dir=self._window_cache_dir(),)# Files the recent window never saw would each spawn a per-file@@ -374,6 +375,15 @@ class GitIndexer:)return summary, results+ def _window_cache_dir(self) -> Path | None:+ """Where the commit-window cache lives: the index directory, once it exists.++ A repository that has no ``.repowise`` yet is being probed, not+ indexed, and gets no cache file written beside it.+ """+ cache_dir = self.repo_path / ".repowise"+ return cache_dir if cache_dir.is_dir() else None+async def index_changed_files(self,changed_file_paths: list[str],@@ -381,6 +391,7 @@ class GitIndexer:co_change_sink: dict[str, list[dict]] | None = None,idle_decay_sink: dict[str, dict] | None = None,on_warning: Callable[[str], None] | None = None,+ timings: Any | None = None,) -> list[dict]:"""Incremental update: re-index only changed files.@@ -413,7 +424,12 @@ class GitIndexer:recomputes just those fields off the walks already loaded here; thepersist path upserts them field-by-field so ownership / age / authorship(correct only from the full init walk) are left intact.++ ``timings`` is the run's phase table; each walk below records a+ ``rebuild.git.*`` row so a slow git step names itself."""+ from repowise.core.pipeline.phase_timing import timed+# The same allowlist the full index applies to the tracked-file set.# Without it an update wrote rows for a changed workflow file, and the# idle refresh minted rows for every tracked config and markup file,@@ -454,12 +470,14 @@ class GitIndexer:# refresh is due, so idle files carry their own precomputed commits.# The git subprocess is identical either way — only the in-memory# bucketing set widens.- commit_index = load_commit_index(- repo,- self.commit_limit,- set(all_files) if refresh_idle else set(changed_file_paths),- provenance_classifier=prov_clf,- )+ with timed(timings, "rebuild.git.commit_index"):+ commit_index = load_commit_index(+ repo,+ self.commit_limit,+ set(all_files) if refresh_idle else set(changed_file_paths),+ provenance_classifier=prov_clf,+ cache_dir=self._window_cache_dir(),+ )as_of_ts = self._resolve_as_of_ts(repo, commit_index)get_thread_repo, close_thread_repos = self._thread_repo_pool()@@ -498,7 +516,8 @@ class GitIndexer:tasks = [index_one(fp) for fp in changed_file_paths]try:- results_raw = await asyncio.gather(*tasks, return_exceptions=True)+ with timed(timings, "rebuild.git.changed_files"):+ results_raw = await asyncio.gather(*tasks, return_exceptions=True)finally:close_thread_repos()@@ -521,9 +540,10 @@ class GitIndexer:# too: the walk is one subprocess regardless, and an idle file whose# only fix aged past the window must drop to 0 (handled below).try:- prior_defects = compute_prior_defects(- repo, {m["file_path"] for m in results} | set(idle_paths), as_of_ts=as_of_ts- )+ with timed(timings, "rebuild.git.prior_defects"):+ prior_defects = compute_prior_defects(+ repo, {m["file_path"] for m in results} | set(idle_paths), as_of_ts=as_of_ts+ )for meta in results:fp = meta["file_path"]if fp in prior_defects.counts:@@ -542,17 +562,18 @@ class GitIndexer:# the full-index path.if self.tier.includes_co_change and all_files:try:- co_changes, change_entropy = await loop.run_in_executor(- None,- compute_co_changes_and_entropy,- repo,- set(all_files),- max(self.commit_limit, _DEFAULT_CO_CHANGE_COMMIT_LIMIT),- _MAX_PARTNERS_PER_FILE,- None,- None,- as_of_ts,- )+ with timed(timings, "rebuild.git.co_change"):+ co_changes, change_entropy = await loop.run_in_executor(+ None,+ compute_co_changes_and_entropy,+ repo,+ set(all_files),+ max(self.commit_limit, _DEFAULT_CO_CHANGE_COMMIT_LIMIT),+ _MAX_PARTNERS_PER_FILE,+ None,+ None,+ as_of_ts,+ )for meta in results:fp = meta["file_path"]if fp in co_changes:@@ -570,19 +591,20 @@ class GitIndexer:# arithmetic; strip to the decay keys so the field-wise upsert# never clobbers full-history columns (ownership, age, authors).if refresh_idle and idle_paths:- idle_decay_sink.update(- await asyncio.to_thread(- self._compute_idle_decay,- repo,- idle_paths,- commit_index,- as_of_ts,- prov_clf,- co_changes,- change_entropy,- prior_defects,+ with timed(timings, "rebuild.git.idle_decay"):+ idle_decay_sink.update(+ await asyncio.to_thread(+ self._compute_idle_decay,+ repo,+ idle_paths,+ commit_index,+ as_of_ts,+ prov_clf,+ co_changes,+ change_entropy,+ prior_defects,+ ))- )except Exception as exc:logger.debug("co_change_pass_failed", error=str(exc))diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex 9d07cdd15..c0bbac27c 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -331,6 +331,7 @@ async def rebuild_graph_and_git(co_change_sink=co_change_full,idle_decay_sink=idle_decay_sink,on_warning=log,+ timings=timings,)git_meta_map = {m["file_path"]: m for m in updated_meta}label_co_change_structure(graph_builder, git_meta_map)@@ -603,6 +604,31 @@ async def load_stored_coverage_map(return {}+def _performance_only_config(config: dict | None, performance_only: set[str]) -> dict | None:+ """Disable every non-performance detector on the closure's files.++ The closure adds files whose call paths reach a changed sink. Only their+ performance findings survive the filter below; every other detector's+ output for them is computed and discarded, and on a central helper the+ closure runs to hundreds of files. The walk still covers them, because the+ performance detectors read it.+ """+ if not performance_only:+ return config+ from repowise.core.analysis.health.biomarkers import registered_biomarkers+ from repowise.core.analysis.health.scoring import dimensions_for++ non_performance = {+ b.name for b in registered_biomarkers() if "performance" not in dimensions_for(b.name)+ }+ out = dict(config or {})+ per_file = {k: set(v) for k, v in (out.get("per_file_disabled") or {}).items()}+ for path in performance_only:+ per_file[path] = per_file.get(path, set()) | non_performance+ out["per_file_disabled"] = per_file+ return out++def run_partial_analysis(repo_path: Any,graph_builder: Any,@@ -699,10 +725,15 @@ def run_partial_analysis(if _hcfg.has_overrides()else None)+ _analyzer_config = _performance_only_config(+ _analyzer_config, _performance_changed - _health_changed+ )partial_health_report = _health_analyzer.analyze(_analyzer_config,changed_files=_health_scope,repo_function_mod_p80=repo_function_mod_p80,+ timings=timings,+ duplication_files=_health_changed,)# The closure exists only to refresh interprocedural performance.# Preserve the historical changed-file scope for every other@@ -1054,7 +1085,9 @@ async def persist_partial_health(session: Any, repo_id: str, report: Any) -> Nonawait snapshot_health_from_store(session, repo_id)-async def persist_incremental_commits(session: Any, repo_id: str, repo_path: Any) -> None:+async def persist_incremental_commits(+ session: Any, repo_id: str, repo_path: Any, *, timings: PhaseTimings | None = None+) -> None:"""Capture + upsert ``git_commits`` rows for commits new since the last index.Foundation 1 only populated the per-commit table on the full orchestrator@@ -1092,14 +1125,17 @@ async def persist_incremental_commits(session: Any, repo_id: str, repo_path: Anydt = newest if newest.tzinfo is not None else newest.replace(tzinfo=UTC)since_ts = int(dt.timestamp())- rows = await asyncio.to_thread(indexer.capture_new_commit_rows, since_ts=since_ts)- if rows:- await upsert_git_commits_bulk(session, repo_id, rows)+ with timed(timings, "persist.commits.capture"):+ rows = await asyncio.to_thread(indexer.capture_new_commit_rows, since_ts=since_ts)+ if rows:+ await upsert_git_commits_bulk(session, repo_id, rows)- await reconcile_commit_experience(session, repo_id, indexer)+ with timed(timings, "persist.commits.experience"):+ await reconcile_commit_experience(session, repo_id, indexer)# Fills the commit-offset column on indexes written before it existed, so a# new capture never needs a re-index to become useful.- await reconcile_commit_offsets(session, repo_id, indexer)+ with timed(timings, "persist.commits.offsets"):+ await reconcile_commit_offsets(session, repo_id, indexer)# Refresh the repo-level whole-history totals so age / commit / contributor# counts keep growing between full re-indexes (#730). Cheap git calls, and@@ -1108,7 +1144,8 @@ async def persist_incremental_commits(session: Any, repo_id: str, repo_path: Any# was stored last time lets it add only the range since, and it re-proves# that range is safe to add before doing so.prior = _churn_prior(await get_repository(session, repo_id))- totals = await asyncio.to_thread(indexer.capture_repo_totals, prior)+ with timed(timings, "persist.commits.totals"):+ totals = await asyncio.to_thread(indexer.capture_repo_totals, prior)await update_repo_git_totals(session,repo_id,@@ -1122,7 +1159,8 @@ async def persist_incremental_commits(session: Any, repo_id: str, repo_path: Anychurn_anchor_sha=totals.churn_anchor_sha,)- await persist_incremental_fix_events(session, repo_id, indexer)+ with timed(timings, "persist.commits.fix_events"):+ await persist_incremental_fix_events(session, repo_id, indexer)def _churn_prior(repo_row: Any) -> Any:@@ -1589,7 +1627,9 @@ async def persist_incremental_index(try:with timed(timings, "persist.commits"):- await persist_incremental_commits(session, repo_id, repo_path)+ await persist_incremental_commits(+ session, repo_id, repo_path, timings=timings+ )except Exception as exc:_skip("Commit capture", exc)@@ -1631,7 +1671,7 @@ async def persist_incremental_index(from repowise.core.pipeline.persist import persist_graph_nodeswith timed(timings, "persist.graph_nodes"):- await persist_graph_nodes(session, repo_id, graph_builder)+ await persist_graph_nodes(session, repo_id, graph_builder, timings=timings)except Exception as exc:_skip("Graph nodes persist", exc)diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex c021a78fc..7785ca973 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -304,6 +304,8 @@ async def persist_graph_nodes(repo_id: str,graph_builder: Any,ep_scores: dict[str, float] | None = None,+ *,+ timings: Any | None = None,) -> None:"""Persist file- and symbol-level graph nodes with full centrality metrics.@@ -316,25 +318,30 @@ async def persist_graph_nodes(batch_upsert_graph_node_membership,batch_upsert_graph_nodes,)+ from repowise.core.pipeline.phase_timing import timedgraph = graph_builder.graph()- pr = graph_builder.pagerank()- bc = graph_builder.betweenness_centrality()- sym_pr = graph_builder.symbol_pagerank()- sym_bc = graph_builder.symbol_betweenness_centrality()- cd = graph_builder.community_detection()- sc = graph_builder.symbol_communities()- ci = graph_builder.community_info()+ with timed(timings, "persist.graph_nodes.metrics"):+ pr = graph_builder.pagerank()+ bc = graph_builder.betweenness_centrality()+ sym_pr = graph_builder.symbol_pagerank()+ sym_bc = graph_builder.symbol_betweenness_centrality()+ cd = graph_builder.community_detection()+ sc = graph_builder.symbol_communities()+ ci = graph_builder.community_info()# ``None`` means "derive scores from the graph" (the incremental update# path passes nothing). An explicit ``{}`` means "no scores" and is left# untouched. Without this, every ``update`` re-upserted symbol nodes with# empty community_meta and wiped the entry_point_scores written at init,# leaving get_execution_flows / the dashboard panel permanently empty.if ep_scores is None:- ep_scores = _derive_entry_point_scores(graph_builder)+ with timed(timings, "persist.graph_nodes.entry_points"):+ ep_scores = _derive_entry_point_scores(graph_builder)bt_commits = _betweenness_commits(graph_builder)+ timings_rows = timed(timings, "persist.graph_nodes.rows")+ timings_rows.__enter__()nodes = []for node_id in graph.nodes:data = graph.nodes[node_id]@@ -400,15 +407,20 @@ async def persist_graph_nodes(})nodes.append(node_dict)+ timings_rows.__exit__(None, None, None)if nodes:- await batch_upsert_graph_nodes(session, repo_id, nodes)+ with timed(timings, "persist.graph_nodes.upsert"):+ await batch_upsert_graph_nodes(session, repo_id, nodes)# Materialize the file-level metrics snapshot (graph_metrics) so large# repos can serve metric reads from SQL without recomputing the NetworkX# centrality kernels. Additive to graph_nodes; never changes node rows.try:- await batch_upsert_graph_metrics(session, repo_id, graph_builder.file_metrics_snapshot())+ with timed(timings, "persist.graph_nodes.snapshots"):+ await batch_upsert_graph_metrics(+ session, repo_id, graph_builder.file_metrics_snapshot()+ )except Exception as exc: # materialization is non-load-bearinglogger.warning("graph_metrics_materialize_skipped", error=str(exc))@@ -416,9 +428,10 @@ async def persist_graph_nodes(# queryable rows (graph_node_membership). Feeds the break-cycle /# move-method refactoring surfaces; non-load-bearing like graph_metrics.try:- await batch_upsert_graph_node_membership(- session, repo_id, graph_builder.node_membership_snapshot()- )+ with timed(timings, "persist.graph_nodes.snapshots"):+ await batch_upsert_graph_node_membership(+ session, repo_id, graph_builder.node_membership_snapshot()+ )except Exception as exc: # materialization is non-load-bearinglogger.warning("graph_node_membership_materialize_skipped", error=str(exc))diff --git a/tests/unit/health/test_walk_cache.py b/tests/unit/health/test_walk_cache.pynew file mode 100644index 000000000..7b0f9b99d--- /dev/null+++ b/tests/unit/health/test_walk_cache.py@@ -0,0 +1,133 @@+"""The walk cache answers exactly what the walk answers, and never leaks a run.++The complexity walk over a file depends on its bytes, its language and the+walker's version. The pass mutates the result it is handed, so a cached entry+has to come back pristine every time or one run's annotations would become+the next run's input.+"""++from __future__ import annotations++from pathlib import Path++from repowise.core.analysis.health.complexity import walk_file+from repowise.core.analysis.health.engine import HEALTH_ANALYZER_VERSION+from repowise.core.analysis.health.walk_cache import _CACHE_FILENAME, HealthWalkCache+from repowise.core.ingestion import compute_content_hash++_SRC = b"""+def outer(items):+ total = 0+ for item in items:+ if item:+ total += item+ return total+++def inner(x):+ return x + 1+"""+++def _walk(tmp_path: Path):+ path = tmp_path / "a.py"+ path.write_bytes(_SRC)+ return walk_file(str(path), "python", _SRC)+++def test_a_hit_equals_the_walk_and_is_a_fresh_object(tmp_path: Path) -> None:+ cache = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION)+ key = HealthWalkCache.key("python", compute_content_hash(_SRC))+ walked = _walk(tmp_path)+ cache.put(key, walked)++ first = cache.get(key)+ assert first == walked+ assert first is not walked+ first.functions.clear()+ second = cache.get(key)+ assert second == walked+ assert cache.hits == 2+++def test_entries_survive_a_save_and_load(tmp_path: Path) -> None:+ key = HealthWalkCache.key("python", compute_content_hash(_SRC))+ walked = _walk(tmp_path)+ cache = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION)+ cache.put(key, walked)+ cache.save()+ assert (tmp_path / _CACHE_FILENAME).exists()++ again = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION)+ again.load()+ assert again.get(key) == walked+ assert again.get(HealthWalkCache.key("python", "0" * 64)) is None+ assert again.misses == 1+++def test_another_analyzer_version_ignores_the_file(tmp_path: Path) -> None:+ key = HealthWalkCache.key("python", compute_content_hash(_SRC))+ cache = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION)+ cache.put(key, _walk(tmp_path))+ cache.save()++ newer = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION + 1)+ newer.load()+ assert newer.get(key) is None+++def test_only_used_entries_are_written_back(tmp_path: Path) -> None:+ """A file that left the repository leaves the cache with it."""+ key = HealthWalkCache.key("python", compute_content_hash(_SRC))+ gone = HealthWalkCache.key("python", "f" * 64)+ cache = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION)+ walked = _walk(tmp_path)+ cache.put(key, walked)+ cache.put(gone, walked)+ cache.save()++ run = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION)+ run.load()+ run.get(key)+ run.save()+ third = HealthWalkCache(tmp_path, HEALTH_ANALYZER_VERSION)+ third.load()+ assert third.get(key) is not None+ assert third.get(gone) is None+++def test_the_analyzer_walks_once_and_serves_the_second_pass_from_the_cache(tmp_path: Path, monkeypatch) -> None:+ import networkx as nx++ from repowise.core.analysis.health import engine as engine_mod+ from repowise.core.ingestion import ASTParser, FileTraverser++ (tmp_path / "a.py").write_bytes(_SRC)+ (tmp_path / ".repowise").mkdir()+ parser = ASTParser()+ parsed = [parser.parse_file(fi, Path(fi.abs_path).read_bytes()) for fi in FileTraverser(tmp_path).traverse()]+ calls: list[str] = []+ real_walk = engine_mod.walk_file++ def counting_walk(path, language, source):+ calls.append(path)+ return real_walk(path, language, source)++ monkeypatch.setattr(engine_mod, "walk_file", counting_walk)++ def analyzer():+ return engine_mod.HealthAnalyzer(+ nx.DiGraph(),+ git_meta_map={},+ parsed_files=parsed,+ duplication_cache_dir=tmp_path / ".repowise",+ repo_root=tmp_path,+ )++ first = analyzer().analyze(None)+ second = analyzer().analyze(None)+ assert len(calls) == 1+ assert [m.score for m in first.metrics] == [m.score for m in second.metrics]+ assert [(f.biomarker_type, f.function_name) for f in first.findings] == [+ (f.biomarker_type, f.function_name) for f in second.findings+ ]diff --git a/tests/unit/ingestion/test_commit_window_cache.py b/tests/unit/ingestion/test_commit_window_cache.pynew file mode 100644index 000000000..a052f2bec--- /dev/null+++ b/tests/unit/ingestion/test_commit_window_cache.py@@ -0,0 +1,149 @@+"""The commit-window cache answers exactly what a fresh walk answers.++``git log --numstat`` over the window costs seconds per update on a repository+with wide commits, whatever the number of new commits. The cache keeps the+last walk's records and asks git only for the commits since. These pin that+the cached answer is the fresh one, in every shape of history the cache has+to survive.+"""++from __future__ import annotations++import json+from pathlib import Path++from repowise.core.ingestion.git_commit_index import _WINDOW_CACHE_NAME, load_commit_index+++def _init(tmp_path: Path):+ import git as gitpython++ repo = gitpython.Repo.init(tmp_path)+ with repo.config_writer() as cw:+ cw.set_value("user", "name", "Alice")+ cw.set_value("user", "email", "[email protected]")+ return repo+++def _commit(repo, path: str, text: str, message: str, ts: int) -> str:+ root = Path(repo.working_dir)+ (root / path).parent.mkdir(parents=True, exist_ok=True)+ (root / path).write_text(text, encoding="utf-8")+ repo.index.add([path])+ stamp = f"{ts} +0000"+ return repo.index.commit(message, author_date=stamp, commit_date=stamp).hexsha+++def _flat(index) -> dict[str, list[tuple]]:+ return {+ path: [(r.sha, r.ts, r.added, r.deleted, r.subject) for r in recs]+ for path, recs in index.items()+ }+++def _walk(repo, files: set[str], cache_dir: Path | None, depth: int = 500):+ return _flat(load_commit_index(repo, depth, files, cache_dir=cache_dir))+++def test_cache_appends_only_the_new_commits_and_matches_a_fresh_walk(tmp_path: Path) -> None:+ repo = _init(tmp_path)+ cache_dir = tmp_path / ".repowise"+ cache_dir.mkdir()+ files = {"a.py", "b.py"}+ base = 1_700_000_000+ _commit(repo, "a.py", "x = 1\n", "c0", base)+ _commit(repo, "b.py", "y = 1\n", "c1", base + 10)++ first = _walk(repo, files, cache_dir)+ assert first == _walk(repo, files, None)+ cached = json.loads((cache_dir / _WINDOW_CACHE_NAME).read_text(encoding="utf-8"))+ assert cached["head"] == repo.head.commit.hexsha+ assert len(cached["records"]) == 2++ _commit(repo, "a.py", "x = 2\n", "c2", base + 20)+ _commit(repo, "b.py", "y = 2\n", "c3", base + 30)++ assert _walk(repo, files, cache_dir) == _walk(repo, files, None)+ cached = json.loads((cache_dir / _WINDOW_CACHE_NAME).read_text(encoding="utf-8"))+ assert cached["head"] == repo.head.commit.hexsha+ assert len(cached["records"]) == 4+++def test_cache_is_cut_to_the_depth_like_the_walk(tmp_path: Path) -> None:+ repo = _init(tmp_path)+ cache_dir = tmp_path / ".repowise"+ cache_dir.mkdir()+ base = 1_700_000_000+ for i in range(3):+ _commit(repo, "a.py", f"x = {i}\n", f"c{i}", base + i * 10)+ _walk(repo, {"a.py"}, cache_dir, depth=3)+ for i in range(3, 6):+ _commit(repo, "a.py", f"x = {i}\n", f"c{i}", base + i * 10)++ assert _walk(repo, {"a.py"}, cache_dir, depth=3) == _walk(repo, {"a.py"}, None, depth=3)+ cached = json.loads((cache_dir / _WINDOW_CACHE_NAME).read_text(encoding="utf-8"))+ assert len(cached["records"]) == 3+++def test_a_merged_branch_with_older_commits_lands_where_the_walk_puts_it(tmp_path: Path) -> None:+ """New commits are not always newer: a merged branch carries its own dates."""+ repo = _init(tmp_path)+ cache_dir = tmp_path / ".repowise"+ cache_dir.mkdir()+ files = {"a.py", "b.py"}+ base = 1_700_000_000+ _commit(repo, "a.py", "x = 1\n", "c0", base)+ main_head = repo.head.commit.hexsha+ _walk(repo, files, cache_dir)++ repo.git.checkout("-b", "side")+ _commit(repo, "b.py", "y = 1\n", "side older", base + 5)+ repo.git.checkout(repo.head.reference.name and "master" if "master" in repo.heads else "main")+ _commit(repo, "a.py", "x = 2\n", "main newer", base + 50)+ repo.git.merge("side", "--no-ff", "-m", "merge side")+ assert repo.head.commit.hexsha != main_head++ assert _walk(repo, files, cache_dir) == _walk(repo, files, None)+++def test_a_rewritten_history_falls_back_to_the_full_walk(tmp_path: Path) -> None:+ repo = _init(tmp_path)+ cache_dir = tmp_path / ".repowise"+ cache_dir.mkdir()+ base = 1_700_000_000+ _commit(repo, "a.py", "x = 1\n", "c0", base)+ _commit(repo, "a.py", "x = 2\n", "c1", base + 10)+ _walk(repo, {"a.py"}, cache_dir)++ repo.git.reset("--hard", "HEAD~1")+ _commit(repo, "a.py", "x = 3\n", "c1 rewritten", base + 20)++ assert _walk(repo, {"a.py"}, cache_dir) == _walk(repo, {"a.py"}, None)+++def test_a_cache_of_another_depth_or_version_is_ignored(tmp_path: Path) -> None:+ repo = _init(tmp_path)+ cache_dir = tmp_path / ".repowise"+ cache_dir.mkdir()+ base = 1_700_000_000+ _commit(repo, "a.py", "x = 1\n", "c0", base)+ _walk(repo, {"a.py"}, cache_dir, depth=5)++ stale = json.loads((cache_dir / _WINDOW_CACHE_NAME).read_text(encoding="utf-8"))+ stale["records"] = ["garbage"]+ (cache_dir / _WINDOW_CACHE_NAME).write_text(json.dumps(stale), encoding="utf-8")+ # Same depth, same head: the cache is trusted, so garbage yields nothing.+ assert _walk(repo, {"a.py"}, cache_dir, depth=5) == {}+ # Another depth: the cache is ignored and the walk answers.+ assert _walk(repo, {"a.py"}, cache_dir, depth=7) == _walk(repo, {"a.py"}, None, depth=7)+ stale["version"] = 0+ stale["depth"] = 7+ (cache_dir / _WINDOW_CACHE_NAME).write_text(json.dumps(stale), encoding="utf-8")+ assert _walk(repo, {"a.py"}, cache_dir, depth=7) == _walk(repo, {"a.py"}, None, depth=7)+++def test_no_cache_dir_writes_nothing(tmp_path: Path) -> None:+ repo = _init(tmp_path)+ _commit(repo, "a.py", "x = 1\n", "c0", 1_700_000_000)+ _walk(repo, {"a.py"}, None)+ assert not list(tmp_path.glob("**/" + _WINDOW_CACHE_NAME))diff --git a/tests/unit/pipeline/test_partial_health_scope.py b/tests/unit/pipeline/test_partial_health_scope.pynew file mode 100644index 000000000..eda49a678--- /dev/null+++ b/tests/unit/pipeline/test_partial_health_scope.py@@ -0,0 +1,64 @@+"""The update's health pass scopes its work to what it keeps.++The performance closure adds files whose call paths reach a changed sink. Only+their performance findings survive the filter in ``run_partial_analysis``, so+running every other detector on them, and re-splicing their clone windows,+was work whose output was discarded: on a central hugo helper the closure ran+to 249 files and cost two thirds of the health pass.+"""++from __future__ import annotations++from repowise.core.analysis.health.biomarkers import registered_biomarkers+from repowise.core.analysis.health.scoring import dimensions_for+from repowise.core.pipeline.incremental import _performance_only_config+++def test_closure_files_keep_only_the_performance_detectors() -> None:+ config = _performance_only_config(None, {"pkg/caller.py"})++ disabled = config["per_file_disabled"]["pkg/caller.py"]+ names = {b.name for b in registered_biomarkers()}+ assert disabled == {n for n in names if "performance" not in dimensions_for(n)}+ assert all("performance" in dimensions_for(n) for n in names - disabled)+ assert "io_in_loop" not in disabled+ assert "dry_violation" in disabled+++def test_an_empty_closure_leaves_the_config_alone() -> None:+ assert _performance_only_config(None, set()) is None+ cfg = {"disabled_biomarkers": ["dry_violation"]}+ assert _performance_only_config(cfg, set()) is cfg+++def test_existing_per_file_rules_are_kept_and_widened() -> None:+ cfg = {"per_file_disabled": {"pkg/caller.py": {"io_in_loop"}, "pkg/other.py": {"god_class"}}}++ out = _performance_only_config(cfg, {"pkg/caller.py"})++ assert "io_in_loop" in out["per_file_disabled"]["pkg/caller.py"]+ assert "dry_violation" in out["per_file_disabled"]["pkg/caller.py"]+ assert out["per_file_disabled"]["pkg/other.py"] == {"god_class"}+ # The caller's dict is not mutated.+ assert cfg["per_file_disabled"]["pkg/caller.py"] == {"io_in_loop"}+++def test_duplication_is_narrowed_to_the_changed_files(monkeypatch, tmp_path) -> None:+ """The clone detector sees the changed files, not the closure."""+ import networkx as nx++ from repowise.core.analysis.health import engine as engine_mod+ from repowise.core.analysis.health.duplication import DuplicationReport++ seen: dict = {}++ def fake_detect_clones(parsed_files, git_meta_map, **kwargs):+ seen["changed_files"] = kwargs.get("changed_files")+ return DuplicationReport()++ monkeypatch.setattr(engine_mod, "detect_clones", fake_detect_clones)+ analyzer = engine_mod.HealthAnalyzer(nx.DiGraph(), git_meta_map={}, parsed_files=[])+ analyzer.analyze(None, changed_files={"a.py", "b.py"}, duplication_files={"a.py"})+ assert seen["changed_files"] == {"a.py"}+ analyzer.analyze(None, changed_files={"a.py", "b.py"})+ assert seen["changed_files"] == {"a.py", "b.py"}