Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(update): read the stored vector width from a record, not LanceDB (#2861)
Raghav Chamadiyaeba45b40f17c
diff --git a/packages/cli/src/repowise/cli/providers/vector_store.py b/packages/cli/src/repowise/cli/providers/vector_store.pyindex f7eeb1fb0..bda147b1a 100644--- a/packages/cli/src/repowise/cli/providers/vector_store.py+++ b/packages/cli/src/repowise/cli/providers/vector_store.py@@ -16,9 +16,22 @@ def existing_vector_dim(lance_dir: Path) -> int | None:directory, no lancedb installed, no table yet, or a schema without afixed-width vector column — and every caller treats that as "do not act",never as a difference.++ The width the writer recorded beside the table is read first, so the+ steady state never imports lancedb. A store written before that record+ existed is probed once and the answer recorded; the writer re-stamps it+ on its next write, so a stale record cannot outlive one."""if not lance_dir.exists():return None+ from repowise.core.persistence.vector_store.lancedb_store import (+ read_recorded_vector_dim,+ record_vector_dim,+ )++ recorded = read_recorded_vector_dim(lance_dir, _TABLE_NAME)+ if recorded is not None:+ return recordedtry:import lancedb # type: ignore[import]@@ -30,7 +43,10 @@ def existing_vector_dim(lance_dir: Path) -> int | None:except Exception:return None# pyarrow reports -1 for variable-length lists, which tells us nothing.- return dim if isinstance(dim, int) and dim > 0 else None+ if not (isinstance(dim, int) and dim > 0):+ return None+ record_vector_dim(lance_dir, _TABLE_NAME, dim)+ return dimdef _mock_would_clobber(lance_dir: Path, embedder: Any) -> bool:diff --git a/packages/cli/src/repowise/cli/upgrade.py b/packages/cli/src/repowise/cli/upgrade.pyindex 50f61995e..d0579d77b 100644--- a/packages/cli/src/repowise/cli/upgrade.py+++ b/packages/cli/src/repowise/cli/upgrade.py@@ -127,7 +127,7 @@ def _vector_dims(repo_path: Path) -> tuple[int | None, int | None]:The pin is checked before the table. Reading the stored width costs an``import lancedb`` (1.7s measured, versus 0.09s for the connect and schema- read it exists for), and ``assess_store`` runs on every ``update`` including+ read it exists for) whenever the store has no recorded width yet, and ``assess_store`` runs on every ``update`` includingthe no-op path a post-commit hook fires on. Keyless repos resolve the mockhere and never pay it. A repo that does pin a real embedder now builds itwhere it used to short-circuit, which imports that provider's module, butdiff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.pyindex cfaca4baa..98bc56112 100644--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py@@ -2,6 +2,8 @@from __future__ import annotations+from pathlib import Path+from repowise.core.providers.embedding.base import Embedderfrom ..search import _SNIPPET_LEN, SearchResult, snippet_around@@ -14,7 +16,7 @@ from ._base import (iter_embed_chunks,)-__all__ = ["STORED_SNIPPET_CHARS", "LanceDBVectorStore"]+__all__ = ["STORED_SNIPPET_CHARS", "LanceDBVectorStore", "read_recorded_vector_dim", "record_vector_dim"]# DataFusion expands every literal in a large ``IN`` filter into native query# state. A several-thousand-path generation level can otherwise commit@@ -34,6 +36,38 @@ _SUMMARY_PATH_BATCH_SIZE = 100# holds 200 characters and simply windows to its opener.+def _dim_record(db_path: str | Path, table_name: str) -> Path:+ return Path(db_path) / f"{table_name}.vector_dim"+++def read_recorded_vector_dim(db_path: str | Path, table_name: str) -> int | None:+ """The width last recorded for *table_name*, read without importing lancedb.++ Importing lancedb costs well over a second, and the store-upgrade check+ runs on every ``update``. ``None`` means "no usable record": no file, an+ unparsable one, or a record whose table directory is gone.+ """+ if not (Path(db_path) / f"{table_name}.lance").is_dir():+ return None+ try:+ dim = int(_dim_record(db_path, table_name).read_text(encoding="ascii").strip())+ except (OSError, ValueError):+ return None+ return dim if dim > 0 else None+++def record_vector_dim(db_path: str | Path, table_name: str, dim: int | None) -> None:+ """Record (or with ``None``, forget) *table_name*'s width. Best-effort."""+ path = _dim_record(db_path, table_name)+ try:+ if dim is None:+ path.unlink(missing_ok=True)+ else:+ path.write_text(str(dim), encoding="ascii")+ except OSError:+ pass++def _evidence(stored: str, query: str | None) -> str:"""The served snippet, cut from the wider text a row keeps.@@ -90,6 +124,7 @@ class LanceDBVectorStore(VectorStore):self._table_name = table_name or self._TABLE_NAMEself._db = Noneself._table = None+ self._dim_recorded = Falseasync def _ensure_connected(self) -> None:if self._db is not None:@@ -147,8 +182,15 @@ class LanceDBVectorStore(VectorStore):if self._table is not None:existing_dim = self._existing_vector_dim(await self._table.schema())if existing_dim is None or existing_dim == dim:+ # Re-stamped once per store so a missing or stale record heals+ # on the next write instead of lingering.+ if existing_dim is not None and not self._dim_recorded:+ record_vector_dim(self._db_path, self._table_name, existing_dim)+ self._dim_recorded = Truereturn# Embedder changed dimensions — the old vectors are unusable.+ # Forget the record first so a crash mid-rebuild leaves none.+ record_vector_dim(self._db_path, self._table_name, None)await self._db.drop_table(self._table_name) # type: ignore[union-attr]self._table = None@@ -165,6 +207,8 @@ class LanceDBVectorStore(VectorStore):self._table = await self._db.create_table( # type: ignore[union-attr]self._table_name, schema=schema, exist_ok=True)+ record_vector_dim(self._db_path, self._table_name, dim)+ self._dim_recorded = True@staticmethoddef _row(page_id: str, vector: list[float], metadata: dict) -> dict:diff --git a/tests/unit/cli/test_embedder_resolution.py b/tests/unit/cli/test_embedder_resolution.pyindex 0088d71ee..23e8c092d 100644--- a/tests/unit/cli/test_embedder_resolution.py+++ b/tests/unit/cli/test_embedder_resolution.py@@ -859,3 +859,87 @@ def test_template_run_embedder_downgrades_only_an_unasked_hosted_backend() -> None:"""``init`` predicts this answer to build the run's store before the pipeline."""assert providers.template_run_embedder(resolved, requested) == expected+++# --- the recorded width: the steady state never imports lancedb -------------+++def _lance_dir(repo_path: Path) -> Path:+ return repo_path / ".repowise" / "lancedb"+++def _record(repo_path: Path) -> Path:+ return _lance_dir(repo_path) / "wiki_pages.vector_dim"+++def _forbid_lancedb(monkeypatch: pytest.MonkeyPatch) -> None:+ def _boom(*_a, **_k):+ raise AssertionError("LanceDB was opened on the steady-state path")++ monkeypatch.setattr(lancedb, "connect", _boom)+++async def test_an_unchanged_real_embedder_never_opens_lancedb(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A real-pinned repo's routine update reads the recorded width, not the table."""+ from repowise.cli.upgrade import _vector_dims++ await _seed_store(tmp_path, _WideEmbedder())+ _write_config(tmp_path, embedder="openai")+ monkeypatch.delenv("REPOWISE_EMBEDDER", raising=False)+ monkeypatch.setattr(+ "repowise.cli.providers.embedders.build_embedder", lambda _n, _p=None: _WideEmbedder()+ )+ _forbid_lancedb(monkeypatch)++ assert providers.existing_vector_dim(_lance_dir(tmp_path)) == 1536+ assert _vector_dims(tmp_path) == (None, None)+++async def test_switching_to_a_real_embedder_still_proposes_the_re_embed(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Keyless vectors under a real pin are still caught, from the record alone,+ and the rebuild that resolves them moves the record with the table."""+ from repowise.cli.upgrade import _vector_dims++ await _seed_store(tmp_path, MockEmbedder())+ _write_config(tmp_path, embedder="openai")+ monkeypatch.delenv("REPOWISE_EMBEDDER", raising=False)+ monkeypatch.setattr(+ "repowise.cli.providers.embedders.build_embedder", lambda _n, _p=None: _WideEmbedder()+ )+ with monkeypatch.context() as m:+ _forbid_lancedb(m)+ assert _vector_dims(tmp_path) == (8, 1536)++ await _seed_store(tmp_path, _WideEmbedder())+ assert _record(tmp_path).read_text(encoding="ascii") == "1536"+ assert _vector_dims(tmp_path) == (None, None)+++async def test_a_store_without_a_record_is_probed_once_and_recorded(tmp_path: Path) -> None:+ """Stores written before the record existed still answer, then stop paying."""+ await _seed_store(tmp_path, _WideEmbedder())+ _record(tmp_path).unlink()++ assert providers.existing_vector_dim(_lance_dir(tmp_path)) == 1536+ assert _record(tmp_path).read_text(encoding="ascii") == "1536"+++async def test_a_stale_record_heals_on_the_next_write(tmp_path: Path) -> None:+ await _seed_store(tmp_path, _WideEmbedder())+ _record(tmp_path).write_text("8", encoding="ascii")++ await _seed_store(tmp_path, _WideEmbedder())+ assert _record(tmp_path).read_text(encoding="ascii") == "1536"+++async def test_a_record_without_its_table_is_ignored(tmp_path: Path) -> None:+ import shutil++ await _seed_store(tmp_path, _WideEmbedder())+ shutil.rmtree(_lance_dir(tmp_path) / "wiki_pages.lance")++ assert providers.existing_vector_dim(_lance_dir(tmp_path)) is None