Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(init): time the security scan and change rebuild inside the resume checkpoint, scan off the event loop (#4276) (#4393)
Aman9b394ef2f483
diff --git a/packages/core/src/repowise/core/pipeline/persist/git.py b/packages/core/src/repowise/core/pipeline/persist/git.pyindex 9d8201522..4c8805eaa 100644--- a/packages/core/src/repowise/core/pipeline/persist/git.py+++ b/packages/core/src/repowise/core/pipeline/persist/git.py@@ -9,11 +9,17 @@ import structloglogger = structlog.get_logger("repowise.core.pipeline.persist")-async def persist_git(result: Any, session: Any, repo_id: str) -> None:+async def persist_git(+ result: Any, session: Any, repo_id: str, *, timings: Any | None = None+) -> None:"""Persist git-phase outputs: per-file metadata and per-commit rows.Both writes are idempotent UPSERTs keyed by ``(repo_id, file_path)`` /``(repo_id, sha)`` — safe to call incrementally and on resume.++ ``timings`` is an optional :class:`PhaseTimings` table; when given, the+ change-record rebuild (a first-parent history walk) is timed into it as+ ``persist.changes``."""from repowise.core.persistence.crud import (clear_attribution_if_disabled,@@ -24,6 +30,7 @@ async def persist_git(result: Any, session: Any, repo_id: str) -> None:upsert_git_commits_bulk,upsert_git_metadata_bulk,)+ from repowise.core.pipeline.phase_timing import timedif result.git_metadata_list:await upsert_git_metadata_bulk(session, repo_id, result.git_metadata_list)@@ -40,7 +47,8 @@ async def persist_git(result: Any, session: Any, repo_id: str) -> None:await upsert_git_commit_files_bulk(session, repo_id, commit_file_rows)await clear_attribution_if_disabled(session, repo_id)if summary is not None:- await _rebuild_changes(session, repo_id)+ with timed(timings, "persist.changes"):+ await _rebuild_changes(session, repo_id)# Per fix-commit x file rows (with their SZZ candidates). The prune keeps a# re-index of an already-indexed repo from leaving behind events that havediff --git a/packages/core/src/repowise/core/pipeline/persist/ingestion.py b/packages/core/src/repowise/core/pipeline/persist/ingestion.pyindex b9fce2389..8586ba1f8 100644--- a/packages/core/src/repowise/core/pipeline/persist/ingestion.py+++ b/packages/core/src/repowise/core/pipeline/persist/ingestion.py@@ -15,7 +15,9 @@ from .graph import _edge_row, prune_absent_graph_rows, stamp_edges_parser_fingerlogger = structlog.get_logger("repowise.core.pipeline.persist")-async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:+async def persist_ingestion(+ result: Any, session: Any, repo_id: str, *, timings: Any | None = None+) -> int:"""Persist ingestion-phase outputs: graph nodes/edges, external systems,symbols, and the per-file security scan.@@ -24,6 +26,10 @@ async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:Graph nodes and edges are then pruned to the fresh graph, which is alwaysthe complete graph here (the incremental update path does not call this).+ ``timings`` is an optional :class:`PhaseTimings` table; when given, the+ security scan is timed into it as ``persist.security_scan`` so a caller that+ times the whole write can see how much of it was the scan.+Returns the number of symbols written (for the summary log). Mutates``sym.file_path`` on symbols that lack one — callers should treat theparsed-file symbols as consumed after this call.@@ -34,6 +40,7 @@ async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:bulk_upsert_external_systems,link_graph_nodes_to_external_systems,)+ from repowise.core.pipeline.phase_timing import timed# ---- Graph nodes ---------------------------------------------------------# Prefer the full candidate-score map (all scored entry points), falling@@ -94,7 +101,8 @@ async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:# ---- Security scan -------------------------------------------------------# Best-effort — never breaks the rest of the phase.try:- await _persist.persist_security_findings(result, session, repo_id)+ with timed(timings, "persist.security_scan"):+ await _persist.persist_security_findings(result, session, repo_id)except Exception as _sec_err:logger.warning("security_scan_skipped", error=str(_sec_err))diff --git a/packages/core/src/repowise/core/pipeline/persist/security.py b/packages/core/src/repowise/core/pipeline/persist/security.pyindex c1b050146..f367cb611 100644--- a/packages/core/src/repowise/core/pipeline/persist/security.py+++ b/packages/core/src/repowise/core/pipeline/persist/security.py@@ -2,6 +2,7 @@from __future__ import annotations+import asynciofrom typing import Anyimport structlog@@ -49,5 +50,9 @@ async def _replace_security_findings() -> None:from repowise.core.analysis.security_scan import SecurityScanner, scan_source_map- findings_by_file, scanned_paths = scan_source_map(parsed_files, source_map)+ # The scan is CPU-bound regex work over every file's text; on a large repo+ # it runs for seconds, so keep it off the event loop.+ findings_by_file, scanned_paths = await asyncio.to_thread(+ scan_source_map, parsed_files, source_map+ )await SecurityScanner(session, repo_id).replace_findings(findings_by_file, scanned_paths)diff --git a/packages/core/src/repowise/core/pipeline/resume/controller.py b/packages/core/src/repowise/core/pipeline/resume/controller.pyindex c775913db..a277259ce 100644--- a/packages/core/src/repowise/core/pipeline/resume/controller.py+++ b/packages/core/src/repowise/core/pipeline/resume/controller.py@@ -188,10 +188,14 @@ class ResumeController:try:# On a large repo this write runs for minutes; without its own# timing it is missing from the totals.- with timed(getattr(progress, "table", None), "persist.checkpoint"):+ timings = getattr(progress, "table", None)+ with timed(timings, "persist.checkpoint"):async with get_session(self._sf) as session:- await persist_ingestion(view, session, self._repo_id)- await persist_git(view, session, self._repo_id)+ # The scan and the change rebuild inside these writes time+ # themselves (persist.security_scan, persist.changes) so+ # the checkpoint total is not mistaken for pure writes.+ await persist_ingestion(view, session, self._repo_id, timings=timings)+ await persist_git(view, session, self._repo_id, timings=timings)except Exception as exc:logger.warning("resume_checkpoint_index_failed", error=str(exc))# The CLI tells the user their index was saved and thatdiff --git a/tests/unit/pipeline/test_resume.py b/tests/unit/pipeline/test_resume.pyindex 84c8386be..b272c8a2f 100644--- a/tests/unit/pipeline/test_resume.py+++ b/tests/unit/pipeline/test_resume.py@@ -332,6 +332,80 @@ async def test_index_checkpoint_is_timed(sf, monkeypatch):assert recorder._inner.phases == [("start", "checkpoint.index"), ("done", "checkpoint.index")]+async def test_index_checkpoint_times_the_scan_and_the_change_rebuild(sf, monkeypatch):+ """The security scan and the git walk get their own keys inside the checkpoint.++ They ran under ``persist.checkpoint`` with no name, so a 12 s "saving" row+ was mostly a scan and a history walk. The real persisters run here; only+ the heavy bodies are stubbed.+ """+ from types import SimpleNamespace++ import networkx as nx++ from repowise.core.pipeline.phase_timing import PhaseTimingRecorder++ repo_id = await _make_repo(sf)+ scanned: list[bool] = []+ rebuilt: list[bool] = []++ async def _noop(*_a: object, **_k: object) -> None:+ return None++ async def _scan(*_a: object, **_k: object) -> None:+ scanned.append(True)++ async def _rebuild(*_a: object, **_k: object) -> None:+ rebuilt.append(True)++ monkeypatch.setattr("repowise.core.pipeline.persist.persist_graph_nodes", _noop)+ monkeypatch.setattr("repowise.core.pipeline.persist.persist_security_findings", _scan)+ monkeypatch.setattr("repowise.core.pipeline.persist.git._rebuild_changes", _rebuild)++ recorder = PhaseTimingRecorder(_RecordingProgress())+ await ResumeController(sf, repo_id, resume=False).checkpoint_index(+ parsed_files=[],+ graph_builder=SimpleNamespace(graph=nx.DiGraph),+ git_metadata_list=[],+ git_summary=SimpleNamespace(),+ progress=recorder,+ )++ assert scanned and rebuilt, "the checkpoint no longer ran the scan or the rebuild"+ assert "persist.security_scan" in recorder.timings+ assert "persist.changes" in recorder.timings+ assert "persist.checkpoint" in recorder.timings+++async def test_security_scan_runs_off_the_event_loop(monkeypatch):+ """``scan_source_map`` is CPU-bound; it must not block the loop thread."""+ import threading+ from types import SimpleNamespace++ from repowise.core.pipeline.persist.security import persist_security_findings++ seen: dict[str, int] = {}++ def _scan(parsed_files, source_map):+ seen["thread"] = threading.get_ident()+ return {}, []++ class _Scanner:+ def __init__(self, *_a: object, **_k: object) -> None: ...++ async def replace_findings(self, *_a: object, **_k: object) -> None:+ return None++ monkeypatch.setattr("repowise.core.analysis.security_scan.scan_source_map", _scan)+ monkeypatch.setattr("repowise.core.analysis.security_scan.SecurityScanner", _Scanner)++ await persist_security_findings(+ SimpleNamespace(parsed_files=[], source_map={}), object(), "repo-1"+ )++ assert seen["thread"] != threading.get_ident()++async def test_failed_symbol_analysis_reconciliation_keeps_analysis_retryable(sf, monkeypatch):"""ANALYSIS cannot complete while post-health symbol state is stale."""repo_id = await _make_repo(sf)