Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(init): time the security scan and change rebuild inside the resume checkpoint, scan off the event loop (#4276) (#4393)

Aman9b394ef2f483
diff --git a/packages/core/src/repowise/core/pipeline/persist/git.py b/packages/core/src/repowise/core/pipeline/persist/git.py
index 9d8201522..4c8805eaa 100644
--- a/packages/core/src/repowise/core/pipeline/persist/git.py
+++ b/packages/core/src/repowise/core/pipeline/persist/git.py
@@ -9,11 +9,17 @@ import structlog
logger = structlog.get_logger("repowise.core.pipeline.persist")
-async def persist_git(result: Any, session: Any, repo_id: str) -> None:
+async def persist_git(
+ result: Any, session: Any, repo_id: str, *, timings: Any | None = None
+) -> None:
"""Persist git-phase outputs: per-file metadata and per-commit rows.
Both writes are idempotent UPSERTs keyed by ``(repo_id, file_path)`` /
``(repo_id, sha)`` — safe to call incrementally and on resume.
+
+ ``timings`` is an optional :class:`PhaseTimings` table; when given, the
+ change-record rebuild (a first-parent history walk) is timed into it as
+ ``persist.changes``.
"""
from repowise.core.persistence.crud import (
clear_attribution_if_disabled,
@@ -24,6 +30,7 @@ async def persist_git(result: Any, session: Any, repo_id: str) -> None:
upsert_git_commits_bulk,
upsert_git_metadata_bulk,
)
+ from repowise.core.pipeline.phase_timing import timed
if result.git_metadata_list:
await upsert_git_metadata_bulk(session, repo_id, result.git_metadata_list)
@@ -40,7 +47,8 @@ async def persist_git(result: Any, session: Any, repo_id: str) -> None:
await upsert_git_commit_files_bulk(session, repo_id, commit_file_rows)
await clear_attribution_if_disabled(session, repo_id)
if summary is not None:
- await _rebuild_changes(session, repo_id)
+ with timed(timings, "persist.changes"):
+ await _rebuild_changes(session, repo_id)
# Per fix-commit x file rows (with their SZZ candidates). The prune keeps a
# re-index of an already-indexed repo from leaving behind events that have
diff --git a/packages/core/src/repowise/core/pipeline/persist/ingestion.py b/packages/core/src/repowise/core/pipeline/persist/ingestion.py
index b9fce2389..8586ba1f8 100644
--- a/packages/core/src/repowise/core/pipeline/persist/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/persist/ingestion.py
@@ -15,7 +15,9 @@ from .graph import _edge_row, prune_absent_graph_rows, stamp_edges_parser_finger
logger = structlog.get_logger("repowise.core.pipeline.persist")
-async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:
+async def persist_ingestion(
+ result: Any, session: Any, repo_id: str, *, timings: Any | None = None
+) -> int:
"""Persist ingestion-phase outputs: graph nodes/edges, external systems,
symbols, and the per-file security scan.
@@ -24,6 +26,10 @@ async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:
Graph nodes and edges are then pruned to the fresh graph, which is always
the complete graph here (the incremental update path does not call this).
+ ``timings`` is an optional :class:`PhaseTimings` table; when given, the
+ security scan is timed into it as ``persist.security_scan`` so a caller that
+ times the whole write can see how much of it was the scan.
+
Returns the number of symbols written (for the summary log). Mutates
``sym.file_path`` on symbols that lack one — callers should treat the
parsed-file symbols as consumed after this call.
@@ -34,6 +40,7 @@ async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:
bulk_upsert_external_systems,
link_graph_nodes_to_external_systems,
)
+ from repowise.core.pipeline.phase_timing import timed
# ---- Graph nodes ---------------------------------------------------------
# Prefer the full candidate-score map (all scored entry points), falling
@@ -94,7 +101,8 @@ async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:
# ---- Security scan -------------------------------------------------------
# Best-effort — never breaks the rest of the phase.
try:
- await _persist.persist_security_findings(result, session, repo_id)
+ with timed(timings, "persist.security_scan"):
+ await _persist.persist_security_findings(result, session, repo_id)
except Exception as _sec_err:
logger.warning("security_scan_skipped", error=str(_sec_err))
diff --git a/packages/core/src/repowise/core/pipeline/persist/security.py b/packages/core/src/repowise/core/pipeline/persist/security.py
index c1b050146..f367cb611 100644
--- a/packages/core/src/repowise/core/pipeline/persist/security.py
+++ b/packages/core/src/repowise/core/pipeline/persist/security.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
from typing import Any
import structlog
@@ -49,5 +50,9 @@ async def _replace_security_findings(
) -> None:
from repowise.core.analysis.security_scan import SecurityScanner, scan_source_map
- findings_by_file, scanned_paths = scan_source_map(parsed_files, source_map)
+ # The scan is CPU-bound regex work over every file's text; on a large repo
+ # it runs for seconds, so keep it off the event loop.
+ findings_by_file, scanned_paths = await asyncio.to_thread(
+ scan_source_map, parsed_files, source_map
+ )
await SecurityScanner(session, repo_id).replace_findings(findings_by_file, scanned_paths)
diff --git a/packages/core/src/repowise/core/pipeline/resume/controller.py b/packages/core/src/repowise/core/pipeline/resume/controller.py
index c775913db..a277259ce 100644
--- a/packages/core/src/repowise/core/pipeline/resume/controller.py
+++ b/packages/core/src/repowise/core/pipeline/resume/controller.py
@@ -188,10 +188,14 @@ class ResumeController:
try:
# On a large repo this write runs for minutes; without its own
# timing it is missing from the totals.
- with timed(getattr(progress, "table", None), "persist.checkpoint"):
+ timings = getattr(progress, "table", None)
+ with timed(timings, "persist.checkpoint"):
async with get_session(self._sf) as session:
- await persist_ingestion(view, session, self._repo_id)
- await persist_git(view, session, self._repo_id)
+ # The scan and the change rebuild inside these writes time
+ # themselves (persist.security_scan, persist.changes) so
+ # the checkpoint total is not mistaken for pure writes.
+ await persist_ingestion(view, session, self._repo_id, timings=timings)
+ await persist_git(view, session, self._repo_id, timings=timings)
except Exception as exc:
logger.warning("resume_checkpoint_index_failed", error=str(exc))
# The CLI tells the user their index was saved and that
diff --git a/tests/unit/pipeline/test_resume.py b/tests/unit/pipeline/test_resume.py
index 84c8386be..b272c8a2f 100644
--- a/tests/unit/pipeline/test_resume.py
+++ b/tests/unit/pipeline/test_resume.py
@@ -332,6 +332,80 @@ async def test_index_checkpoint_is_timed(sf, monkeypatch):
assert recorder._inner.phases == [("start", "checkpoint.index"), ("done", "checkpoint.index")]
+async def test_index_checkpoint_times_the_scan_and_the_change_rebuild(sf, monkeypatch):
+ """The security scan and the git walk get their own keys inside the checkpoint.
+
+ They ran under ``persist.checkpoint`` with no name, so a 12 s "saving" row
+ was mostly a scan and a history walk. The real persisters run here; only
+ the heavy bodies are stubbed.
+ """
+ from types import SimpleNamespace
+
+ import networkx as nx
+
+ from repowise.core.pipeline.phase_timing import PhaseTimingRecorder
+
+ repo_id = await _make_repo(sf)
+ scanned: list[bool] = []
+ rebuilt: list[bool] = []
+
+ async def _noop(*_a: object, **_k: object) -> None:
+ return None
+
+ async def _scan(*_a: object, **_k: object) -> None:
+ scanned.append(True)
+
+ async def _rebuild(*_a: object, **_k: object) -> None:
+ rebuilt.append(True)
+
+ monkeypatch.setattr("repowise.core.pipeline.persist.persist_graph_nodes", _noop)
+ monkeypatch.setattr("repowise.core.pipeline.persist.persist_security_findings", _scan)
+ monkeypatch.setattr("repowise.core.pipeline.persist.git._rebuild_changes", _rebuild)
+
+ recorder = PhaseTimingRecorder(_RecordingProgress())
+ await ResumeController(sf, repo_id, resume=False).checkpoint_index(
+ parsed_files=[],
+ graph_builder=SimpleNamespace(graph=nx.DiGraph),
+ git_metadata_list=[],
+ git_summary=SimpleNamespace(),
+ progress=recorder,
+ )
+
+ assert scanned and rebuilt, "the checkpoint no longer ran the scan or the rebuild"
+ assert "persist.security_scan" in recorder.timings
+ assert "persist.changes" in recorder.timings
+ assert "persist.checkpoint" in recorder.timings
+
+
+async def test_security_scan_runs_off_the_event_loop(monkeypatch):
+ """``scan_source_map`` is CPU-bound; it must not block the loop thread."""
+ import threading
+ from types import SimpleNamespace
+
+ from repowise.core.pipeline.persist.security import persist_security_findings
+
+ seen: dict[str, int] = {}
+
+ def _scan(parsed_files, source_map):
+ seen["thread"] = threading.get_ident()
+ return {}, []
+
+ class _Scanner:
+ def __init__(self, *_a: object, **_k: object) -> None: ...
+
+ async def replace_findings(self, *_a: object, **_k: object) -> None:
+ return None
+
+ monkeypatch.setattr("repowise.core.analysis.security_scan.scan_source_map", _scan)
+ monkeypatch.setattr("repowise.core.analysis.security_scan.SecurityScanner", _Scanner)
+
+ await persist_security_findings(
+ SimpleNamespace(parsed_files=[], source_map={}), object(), "repo-1"
+ )
+
+ assert seen["thread"] != threading.get_ident()
+
+
async def test_failed_symbol_analysis_reconciliation_keeps_analysis_retryable(sf, monkeypatch):
"""ANALYSIS cannot complete while post-health symbol state is stale."""
repo_id = await _make_repo(sf)