Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(vector-store): write generated embeddings in a few large batches (#2880)
Raghav Chamadiyabb0df466c18f
diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.pyindex 8dc8e3ea4..ac1db0412 100644--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py@@ -3,6 +3,7 @@from __future__ import annotationsfrom pathlib import Path+from typing import Anyfrom repowise.core.providers.embedding.base import Embedder@@ -23,12 +24,19 @@ __all__ = ["STORED_SNIPPET_CHARS", "LanceDBVectorStore", "read_recorded_vector_d# gigabytes before returning even though the selected result is small._SUMMARY_PATH_BATCH_SIZE = 100-# LanceDB caches each table version's manifest (default cap 1 GiB). Generation-# upserts in embedder-sized chunks, one version each, and every manifest lists-# all fragments so far, so the cache grew quadratically: about 0.55 GiB over-# 22k pages, held until exit. Reads only need the latest version.+# LanceDB caches each table version's manifest (default cap 1 GiB), and every+# manifest lists all fragments so far. Writes are batched below, but update runs+# and long-lived servers still accumulate versions; reads only need the latest._METADATA_CACHE_BYTES = 64 * 1024 * 1024+# Rows per LanceDB write in ``embed_batch``. Every write is a table version+# and a fragment, and ``merge_insert`` joins against the whole table, so+# writing each 16-item embedder chunk on its own made a level's writes grow+# with the square of its size (22k pages: about 1,400 versions). Rows are+# buffered as float32 Arrow data, about 8 KB each at 1,536 dimensions, so a+# full buffer stays near 32 MB.+_UPSERT_BATCH_ROWS = 4096+# ``STORED_SNIPPET_CHARS`` — how much of a page's content each row keeps — is# defined with the embed recipe and re-exported here so the historical import# path keeps working.@@ -109,6 +117,23 @@ def _page_ids_in_filter(page_ids: list[str]) -> str:return f"page_id IN ({quoted})"+def _last_row_per_page(batches: list[Any]) -> Any:+ """One Arrow table from *batches*, keeping each page_id's last row.++ ``merge_insert`` rejects a source holding a key twice once that key is+ in the table, and inserts both rows when it is not. Written chunk by+ chunk, a repeated page simply overwrote itself, so the last occurrence+ is the one that wins here too.+ """+ import pyarrow as pa # type: ignore[import]++ table = pa.Table.from_batches(batches)+ last = {page_id: i for i, page_id in enumerate(table.column("page_id").to_pylist())}+ if len(last) < table.num_rows:+ table = table.take(sorted(last.values()))+ return table++class LanceDBVectorStore(VectorStore):"""Vector store backed by LanceDB (embedded, local file storage).@@ -236,7 +261,8 @@ class LanceDBVectorStore(VectorStore):"content_snippet": content[:STORED_SNIPPET_CHARS],}- async def _upsert_rows(self, rows: list[dict]) -> None:+ async def _upsert_rows(self, rows: list[dict] | Any) -> None:+ """Upsert by ``page_id``; *rows* is a list of row dicts or an Arrow table."""# merge_insert: upsert by page_id (LanceDB 0.12+)try:await (@@ -247,6 +273,8 @@ class LanceDBVectorStore(VectorStore):)except AttributeError:# Fallback for older LanceDB versions: delete + add+ if not isinstance(rows, list):+ rows = rows.to_pylist()for row in rows:safe_id = str(row["page_id"]).replace("'", "''")await self._table.delete(f"page_id = '{safe_id}'") # type: ignore[union-attr]@@ -261,18 +289,38 @@ class LanceDBVectorStore(VectorStore):await self._upsert_rows([self._row(page_id, vector, meta)])async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:- """Embed and upsert in request-sized chunks with failure isolation.+ """Embed in request-sized chunks and write in a few large upserts.One embedder call per :data:`EMBED_BATCH_MAX_ITEMS` items — a wholegeneration level in a single request blew OpenAI's 300k-token cap- and silently lost every file-page embedding. A failed chunk no- longer sinks the rest; the summary error is raised at the end so- callers still see the loss.+ and silently lost every file-page embedding. Writes are buffered up+ to :data:`_UPSERT_BATCH_ROWS` rows (see there for why). A failed+ chunk no longer sinks the rest; the summary error is raised at the+ end so callers still see the loss. A failed write reports every+ chunk it carried, at the ``persistence`` stage."""if not items:returnawait self._ensure_connected()failures: list[BatchChunkFailure] = []+ pending: list[Any] = [] # Arrow record batches, one per embedded chunk+ pending_chunks: list[list[tuple[str, str, dict]]] = []+ pending_rows = 0++ async def flush() -> None:+ nonlocal pending_rows+ if not pending:+ return+ try:+ await self._upsert_rows(_last_row_per_page(pending))+ except Exception as exc: # preserve vectors' failure stage for callers+ failures.extend(+ BatchChunkFailure(tuple(chunk), "persistence", exc) for chunk in pending_chunks+ )+ pending.clear()+ pending_chunks.clear()+ pending_rows = 0+for chunk, texts in iter_embed_chunks(items):try:vectors = await self._embedder.embed(texts)@@ -285,12 +333,25 @@ class LanceDBVectorStore(VectorStore):self._row(page_id, vector, {"content": text, **metadata})for (page_id, text, metadata), vector in zip(chunk, vectors, strict=True)]- await self._upsert_rows(rows)+ pending.append(await self._record_batch(rows))except Exception as exc: # preserve vectors' failure stage for callersfailures.append(BatchChunkFailure(tuple(chunk), "persistence", exc))+ continue+ pending_chunks.append(chunk)+ pending_rows += len(rows)+ if pending_rows >= _UPSERT_BATCH_ROWS:+ await flush()+ await flush()if failures:raise BatchEmbeddingError(failures=failures, total_items=len(items))+ async def _record_batch(self, rows: list[dict]) -> Any:+ """*rows* as Arrow data in the table's schema (float32 vectors)."""+ import pyarrow as pa # type: ignore[import]++ schema = await self._table.schema() # type: ignore[union-attr]+ return pa.RecordBatch.from_pylist(rows, schema=schema)+async def _search_by_vector(self, q_vec: list[float], limit: int, query: str | None = None) -> list[SearchResult]:diff --git a/tests/unit/persistence/test_vector_store.py b/tests/unit/persistence/test_vector_store.pyindex eafc84284..fd6c2ce42 100644--- a/tests/unit/persistence/test_vector_store.py+++ b/tests/unit/persistence/test_vector_store.py@@ -524,3 +524,66 @@ async def test_lancedb_embed_batch_isolates_failed_chunk(tmp_path):assert len(ids) == EMBED_BATCH_MAX_ITEMS * 2finally:await store.close()+++async def test_lancedb_embed_batch_writes_in_few_versions(tmp_path):+ # Each write is a table version and merge_insert joins the whole table,+ # so one write per embedder chunk made a large level quadratic.+ pytest.importorskip("lancedb")+ from repowise.core.persistence.vector_store import LanceDBVectorStore, lancedb_store+ from repowise.core.persistence.vector_store._base import EMBED_BATCH_MAX_ITEMS++ emb = _RecordingEmbedder()+ store = LanceDBVectorStore(str(tmp_path / "lance"), emb)+ try:+ await store.embed_batch(_items(EMBED_BATCH_MAX_ITEMS * 20))+ assert len(emb.calls) == 20 # embedder requests stay request-sized+ assert len(await store.list_page_ids()) == EMBED_BATCH_MAX_ITEMS * 20+ # create_table + one write, not one write per chunk.+ assert await store._table.version() <= 3+ assert lancedb_store._UPSERT_BATCH_ROWS > EMBED_BATCH_MAX_ITEMS+ finally:+ await store.close()+++async def test_lancedb_embed_batch_repeated_page_keeps_last_row(tmp_path):+ pytest.importorskip("lancedb")+ from repowise.core.persistence.vector_store import LanceDBVectorStore+ from repowise.core.persistence.vector_store._base import EMBED_BATCH_MAX_ITEMS++ items = _items(EMBED_BATCH_MAX_ITEMS)+ items.append(("p0", "later", {"target_path": "moved.py"}))+ store = LanceDBVectorStore(str(tmp_path / "lance"), _RecordingEmbedder())+ try:+ await store.embed_batch(items)+ # A second pass over an existing table must not trip LanceDB's+ # ambiguous-merge check either.+ await store.embed_batch(items)+ rows = await store._table.query().select(["page_id", "target_path"]).to_list()+ assert len(rows) == EMBED_BATCH_MAX_ITEMS+ assert {r["target_path"] for r in rows if r["page_id"] == "p0"} == {"moved.py"}+ finally:+ await store.close()+++async def test_lancedb_embed_batch_failed_write_reports_every_chunk(tmp_path, monkeypatch):+ pytest.importorskip("lancedb")+ from repowise.core.persistence.vector_store import BatchEmbeddingError, LanceDBVectorStore+ from repowise.core.persistence.vector_store._base import EMBED_BATCH_MAX_ITEMS++ store = LanceDBVectorStore(str(tmp_path / "lance"), _RecordingEmbedder())++ async def broken_upsert(rows):+ raise OSError("disk full")++ monkeypatch.setattr(store, "_upsert_rows", broken_upsert)+ try:+ with pytest.raises(BatchEmbeddingError) as caught:+ await store.embed_batch(_items(EMBED_BATCH_MAX_ITEMS * 3))+ assert [f.stage for f in caught.value.failures] == ["persistence"] * 3+ assert caught.value.successful_count == 0+ finally:+ await store.close()