Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(persist): flush bulk upserts in batches and release the inserted rows (#2864)
Raghav Chamadiyaa26dab8e8bea
diff --git a/packages/core/src/repowise/core/persistence/crud/_shared.py b/packages/core/src/repowise/core/persistence/crud/_shared.pyindex bab93f981..dd4b2801a 100644--- a/packages/core/src/repowise/core/persistence/crud/_shared.py+++ b/packages/core/src/repowise/core/persistence/crud/_shared.py@@ -13,7 +13,7 @@ from dataclasses import dataclass, fieldfrom datetime import UTC, datetimefrom typing import Any-from sqlalchemy import delete, select+from sqlalchemy import delete, inspect, selectfrom sqlalchemy.ext.asyncio import AsyncSessionfrom repowise.core.persistence.models import _new_uuid, _now_utc@@ -149,7 +149,8 @@ async def _batch_upsert_keyed(Within-batch duplicate keys keep the legacy outcome: the first iteminserts, later ones update the pending object (the per-item SELECT used- to see the autoflushed insert).+ to see the autoflushed insert), or the reloaded row once that insert has+ been flushed and released.*gate*, when given, skips items whose stored row already matches (see:class:`UpsertGate`). Only the surviving items are hydrated as ORM rows.@@ -158,8 +159,7 @@ async def _batch_upsert_keyed("""materialized = list(items)if not materialized:- if batch_size is None:- await session.flush()+ await session.flush()returngated_keys: list[Any] | None = None@@ -217,21 +217,30 @@ async def _batch_upsert_keyed()by_key: dict[Any, Any] = {row_key_fn(row): row for row in existing_rows}- if batch_size is None:- chunks: list[list[Any]] = [materialized]- else:- chunks = [materialized[i : i + batch_size] for i in range(0, len(materialized), batch_size)]- for chunk in chunks:- for item in chunk:+ # Rows this call inserted and already flushed, by key -> ORM identity. Only+ # the identity is kept: holding every inserted object until the end pinned+ # the whole table in memory at once (1.8M graph edges took ~5 GiB). A+ # later duplicate key reloads its row through the identity map instead.+ flushed: dict[Any, Any] = {}+ size = batch_size or _BATCH_SIZE+ for start in range(0, len(materialized), size):+ inserted: list[tuple[Any, Any]] = []+ for item in materialized[start : start + size]:key = item_key_fn(item)existing = by_key.get(key)+ if existing is None and key in flushed:+ existing = await session.get(model, flushed[key])if existing is not None:update_fn(existing, item)else:obj = insert_fn(item)session.add(obj)by_key[key] = obj+ inserted.append((key, obj))await session.flush()+ for key, obj in inserted:+ flushed[key] = inspect(obj).identity+ del by_key[key]async def _batch_delete_in(diff --git a/packages/core/src/repowise/core/pipeline/resume/controller.py b/packages/core/src/repowise/core/pipeline/resume/controller.pyindex 969591518..f2141de9b 100644--- a/packages/core/src/repowise/core/pipeline/resume/controller.py+++ b/packages/core/src/repowise/core/pipeline/resume/controller.py@@ -26,6 +26,7 @@ from typing import Anyimport structlogfrom ..persist import persist_analysis, persist_git, persist_ingestion, persist_symbol_analysis+from ..phase_timing import timedfrom ..progress import emit_warningfrom .ledger import ResumeLedgerfrom .phases import RESUME_PHASE_ORDER, ResumePhase@@ -169,9 +170,12 @@ class ResumeController:)await self._ledger.mark_started(ResumePhase.INDEX)try:- async with get_session(self._sf) as session:- await persist_ingestion(view, session, self._repo_id)- await persist_git(view, session, self._repo_id)+ # No progress phase covers this write, and on a large repo it runs+ # for minutes; without its own timing it is missing from the totals.+ with timed(getattr(progress, "table", None), "persist.checkpoint"):+ async with get_session(self._sf) as session:+ await persist_ingestion(view, session, self._repo_id)+ await persist_git(view, session, self._repo_id)except Exception as exc:logger.warning("resume_checkpoint_index_failed", error=str(exc))# The CLI tells the user their index was saved and thatdiff --git a/tests/unit/persistence/test_batch_upsert_keyed.py b/tests/unit/persistence/test_batch_upsert_keyed.pyindex 18b44f6fa..b567ec616 100644--- a/tests/unit/persistence/test_batch_upsert_keyed.py+++ b/tests/unit/persistence/test_batch_upsert_keyed.py@@ -144,3 +144,74 @@ async def test_git_metadata_update_path(async_session):)assert len(rows) == 1assert rows[0].commit_count_30d == 7+++def _edge(source: str, confidence: float = 0.5) -> dict:+ return {+ "source_node_id": source,+ "target_node_id": "t.py",+ "edge_type": "imports",+ "confidence": confidence,+ }+++async def test_duplicate_key_across_a_flush_boundary_updates_one_row(async_session):+ """A key repeated after its insert was flushed still lands one row, last wins."""+ from repowise.core.persistence.crud._shared import _BATCH_SIZE++ repo = await insert_repo(async_session)+ edges = [_edge(f"s{i}.py") for i in range(_BATCH_SIZE + 10)]+ edges.append(_edge("s0.py", confidence=0.9))+ await batch_upsert_graph_edges(async_session, repo.id, edges)+ await async_session.commit()++ rows = (await session_exec(async_session, repo.id)).scalars().all()+ assert len(rows) == _BATCH_SIZE + 10+ assert next(r for r in rows if r.source_node_id == "s0.py").confidence == 0.9+++async def test_inserted_rows_are_released_after_each_flush(async_session):+ """Inserted objects must not all stay alive until the call returns.++ Holding them pinned a whole table in memory at once; on a 1.8M-edge graph+ that was ~5 GiB on top of the run.+ """+ import gc+ import weakref++ from repowise.core.persistence.crud._shared import _BATCH_SIZE, _batch_upsert_keyed+ from repowise.core.persistence.models import _new_uuid++ repo = await insert_repo(async_session)+ total = _BATCH_SIZE * 3+ made: list[weakref.ref] = []+ alive_at_end: list[int] = []++ def insert(e: dict) -> GraphEdge:+ if len(made) == total - 1:+ gc.collect()+ alive_at_end.append(sum(ref() is not None for ref in made))+ obj = GraphEdge(+ id=_new_uuid(),+ repository_id=repo.id,+ source_node_id=e["source_node_id"],+ target_node_id=e["target_node_id"],+ edge_type=e["edge_type"],+ )+ made.append(weakref.ref(obj))+ return obj++ await _batch_upsert_keyed(+ async_session,+ GraphEdge,+ [_edge(f"s{i}.py") for i in range(total)],+ prefilter=(GraphEdge.repository_id == repo.id,),+ item_key_fn=lambda e: (e["source_node_id"], e["target_node_id"], e["edge_type"]),+ row_key_fn=lambda row: (row.source_node_id, row.target_node_id, row.edge_type),+ update_fn=lambda existing, e: None,+ insert_fn=insert,+ )+ await async_session.commit()++ assert alive_at_end and alive_at_end[0] < _BATCH_SIZE+ assert len((await session_exec(async_session, repo.id)).scalars().all()) == totaldiff --git a/tests/unit/pipeline/test_resume.py b/tests/unit/pipeline/test_resume.pyindex a9afd7b16..d3874fb16 100644--- a/tests/unit/pipeline/test_resume.py+++ b/tests/unit/pipeline/test_resume.py@@ -299,6 +299,29 @@ async def test_a_successful_checkpoint_says_nothing(sf, monkeypatch):assert progress.messages == []+async def test_index_checkpoint_is_timed(sf, monkeypatch):+ """The checkpoint write runs outside every progress phase, so it times itself."""+ from repowise.core.pipeline.phase_timing import PhaseTimingRecorder++ repo_id = await _make_repo(sf)++ async def _ok(*_a: object, **_k: object) -> None:+ return None++ monkeypatch.setattr("repowise.core.pipeline.resume.controller.persist_ingestion", _ok)+ monkeypatch.setattr("repowise.core.pipeline.resume.controller.persist_git", _ok)++ recorder = PhaseTimingRecorder(_RecordingProgress())+ await ResumeController(sf, repo_id, resume=False).checkpoint_index(+ parsed_files=[],+ graph_builder=None,+ git_metadata_list=[],+ progress=recorder,+ )++ assert "persist.checkpoint" in recorder.timings++async def test_failed_symbol_analysis_reconciliation_keeps_analysis_retryable(sf, monkeypatch):"""ANALYSIS cannot complete while post-health symbol state is stale."""repo_id = await _make_repo(sf)