Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(persist): flush bulk upserts in batches and release the inserted rows (#2864)

Raghav Chamadiyaa26dab8e8bea
diff --git a/packages/core/src/repowise/core/persistence/crud/_shared.py b/packages/core/src/repowise/core/persistence/crud/_shared.py
index bab93f981..dd4b2801a 100644
--- a/packages/core/src/repowise/core/persistence/crud/_shared.py
+++ b/packages/core/src/repowise/core/persistence/crud/_shared.py
@@ -13,7 +13,7 @@ from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
-from sqlalchemy import delete, select
+from sqlalchemy import delete, inspect, select
from sqlalchemy.ext.asyncio import AsyncSession
from repowise.core.persistence.models import _new_uuid, _now_utc
@@ -149,7 +149,8 @@ async def _batch_upsert_keyed(
Within-batch duplicate keys keep the legacy outcome: the first item
inserts, later ones update the pending object (the per-item SELECT used
- to see the autoflushed insert).
+ to see the autoflushed insert), or the reloaded row once that insert has
+ been flushed and released.
*gate*, when given, skips items whose stored row already matches (see
:class:`UpsertGate`). Only the surviving items are hydrated as ORM rows.
@@ -158,8 +159,7 @@ async def _batch_upsert_keyed(
"""
materialized = list(items)
if not materialized:
- if batch_size is None:
- await session.flush()
+ await session.flush()
return
gated_keys: list[Any] | None = None
@@ -217,21 +217,30 @@ async def _batch_upsert_keyed(
)
by_key: dict[Any, Any] = {row_key_fn(row): row for row in existing_rows}
- if batch_size is None:
- chunks: list[list[Any]] = [materialized]
- else:
- chunks = [materialized[i : i + batch_size] for i in range(0, len(materialized), batch_size)]
- for chunk in chunks:
- for item in chunk:
+ # Rows this call inserted and already flushed, by key -> ORM identity. Only
+ # the identity is kept: holding every inserted object until the end pinned
+ # the whole table in memory at once (1.8M graph edges took ~5 GiB). A
+ # later duplicate key reloads its row through the identity map instead.
+ flushed: dict[Any, Any] = {}
+ size = batch_size or _BATCH_SIZE
+ for start in range(0, len(materialized), size):
+ inserted: list[tuple[Any, Any]] = []
+ for item in materialized[start : start + size]:
key = item_key_fn(item)
existing = by_key.get(key)
+ if existing is None and key in flushed:
+ existing = await session.get(model, flushed[key])
if existing is not None:
update_fn(existing, item)
else:
obj = insert_fn(item)
session.add(obj)
by_key[key] = obj
+ inserted.append((key, obj))
await session.flush()
+ for key, obj in inserted:
+ flushed[key] = inspect(obj).identity
+ del by_key[key]
async def _batch_delete_in(
diff --git a/packages/core/src/repowise/core/pipeline/resume/controller.py b/packages/core/src/repowise/core/pipeline/resume/controller.py
index 969591518..f2141de9b 100644
--- a/packages/core/src/repowise/core/pipeline/resume/controller.py
+++ b/packages/core/src/repowise/core/pipeline/resume/controller.py
@@ -26,6 +26,7 @@ from typing import Any
import structlog
from ..persist import persist_analysis, persist_git, persist_ingestion, persist_symbol_analysis
+from ..phase_timing import timed
from ..progress import emit_warning
from .ledger import ResumeLedger
from .phases import RESUME_PHASE_ORDER, ResumePhase
@@ -169,9 +170,12 @@ class ResumeController:
)
await self._ledger.mark_started(ResumePhase.INDEX)
try:
- async with get_session(self._sf) as session:
- await persist_ingestion(view, session, self._repo_id)
- await persist_git(view, session, self._repo_id)
+ # No progress phase covers this write, and on a large repo it runs
+ # for minutes; without its own timing it is missing from the totals.
+ with timed(getattr(progress, "table", None), "persist.checkpoint"):
+ async with get_session(self._sf) as session:
+ await persist_ingestion(view, session, self._repo_id)
+ await persist_git(view, session, self._repo_id)
except Exception as exc:
logger.warning("resume_checkpoint_index_failed", error=str(exc))
# The CLI tells the user their index was saved and that
diff --git a/tests/unit/persistence/test_batch_upsert_keyed.py b/tests/unit/persistence/test_batch_upsert_keyed.py
index 18b44f6fa..b567ec616 100644
--- a/tests/unit/persistence/test_batch_upsert_keyed.py
+++ b/tests/unit/persistence/test_batch_upsert_keyed.py
@@ -144,3 +144,74 @@ async def test_git_metadata_update_path(async_session):
)
assert len(rows) == 1
assert rows[0].commit_count_30d == 7
+
+
+def _edge(source: str, confidence: float = 0.5) -> dict:
+ return {
+ "source_node_id": source,
+ "target_node_id": "t.py",
+ "edge_type": "imports",
+ "confidence": confidence,
+ }
+
+
+async def test_duplicate_key_across_a_flush_boundary_updates_one_row(async_session):
+ """A key repeated after its insert was flushed still lands one row, last wins."""
+ from repowise.core.persistence.crud._shared import _BATCH_SIZE
+
+ repo = await insert_repo(async_session)
+ edges = [_edge(f"s{i}.py") for i in range(_BATCH_SIZE + 10)]
+ edges.append(_edge("s0.py", confidence=0.9))
+ await batch_upsert_graph_edges(async_session, repo.id, edges)
+ await async_session.commit()
+
+ rows = (await session_exec(async_session, repo.id)).scalars().all()
+ assert len(rows) == _BATCH_SIZE + 10
+ assert next(r for r in rows if r.source_node_id == "s0.py").confidence == 0.9
+
+
+async def test_inserted_rows_are_released_after_each_flush(async_session):
+ """Inserted objects must not all stay alive until the call returns.
+
+ Holding them pinned a whole table in memory at once; on a 1.8M-edge graph
+ that was ~5 GiB on top of the run.
+ """
+ import gc
+ import weakref
+
+ from repowise.core.persistence.crud._shared import _BATCH_SIZE, _batch_upsert_keyed
+ from repowise.core.persistence.models import _new_uuid
+
+ repo = await insert_repo(async_session)
+ total = _BATCH_SIZE * 3
+ made: list[weakref.ref] = []
+ alive_at_end: list[int] = []
+
+ def insert(e: dict) -> GraphEdge:
+ if len(made) == total - 1:
+ gc.collect()
+ alive_at_end.append(sum(ref() is not None for ref in made))
+ obj = GraphEdge(
+ id=_new_uuid(),
+ repository_id=repo.id,
+ source_node_id=e["source_node_id"],
+ target_node_id=e["target_node_id"],
+ edge_type=e["edge_type"],
+ )
+ made.append(weakref.ref(obj))
+ return obj
+
+ await _batch_upsert_keyed(
+ async_session,
+ GraphEdge,
+ [_edge(f"s{i}.py") for i in range(total)],
+ prefilter=(GraphEdge.repository_id == repo.id,),
+ item_key_fn=lambda e: (e["source_node_id"], e["target_node_id"], e["edge_type"]),
+ row_key_fn=lambda row: (row.source_node_id, row.target_node_id, row.edge_type),
+ update_fn=lambda existing, e: None,
+ insert_fn=insert,
+ )
+ await async_session.commit()
+
+ assert alive_at_end and alive_at_end[0] < _BATCH_SIZE
+ assert len((await session_exec(async_session, repo.id)).scalars().all()) == total
diff --git a/tests/unit/pipeline/test_resume.py b/tests/unit/pipeline/test_resume.py
index a9afd7b16..d3874fb16 100644
--- a/tests/unit/pipeline/test_resume.py
+++ b/tests/unit/pipeline/test_resume.py
@@ -299,6 +299,29 @@ async def test_a_successful_checkpoint_says_nothing(sf, monkeypatch):
assert progress.messages == []
+async def test_index_checkpoint_is_timed(sf, monkeypatch):
+ """The checkpoint write runs outside every progress phase, so it times itself."""
+ from repowise.core.pipeline.phase_timing import PhaseTimingRecorder
+
+ repo_id = await _make_repo(sf)
+
+ async def _ok(*_a: object, **_k: object) -> None:
+ return None
+
+ monkeypatch.setattr("repowise.core.pipeline.resume.controller.persist_ingestion", _ok)
+ monkeypatch.setattr("repowise.core.pipeline.resume.controller.persist_git", _ok)
+
+ recorder = PhaseTimingRecorder(_RecordingProgress())
+ await ResumeController(sf, repo_id, resume=False).checkpoint_index(
+ parsed_files=[],
+ graph_builder=None,
+ git_metadata_list=[],
+ progress=recorder,
+ )
+
+ assert "persist.checkpoint" in recorder.timings
+
+
async def test_failed_symbol_analysis_reconciliation_keeps_analysis_retryable(sf, monkeypatch):
"""ANALYSIS cannot complete while post-health symbol state is stale."""
repo_id = await _make_repo(sf)