Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

@ (#266)

Raghav Chamadiyae9f3fb22295f
diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
index 86354aec1..84f1f415f 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
@@ -47,11 +47,32 @@ class LanceDBVectorStore(VectorStore):
else:
self._table = None # will be created on first upsert
+ @staticmethod
+ def _existing_vector_dim(schema) -> int | None:
+ """Return the fixed-length dimension of the ``vector`` field, or None.
+
+ Returns None when the field is absent or not a fixed-size list (in
+ which case we can't meaningfully compare dimensions).
+ """
+ try:
+ field = schema.field("vector")
+ except KeyError:
+ return None
+ list_size = getattr(field.type, "list_size", None)
+ # pyarrow uses ``list_size == -1`` for variable-length lists.
+ if isinstance(list_size, int) and list_size > 0:
+ return list_size
+ return None
+
async def _ensure_table(self, sample_vector: list[float]) -> None:
- """Create the LanceDB table if it does not exist yet."""
- if self._table is not None:
- return
+ """Create the LanceDB table if it does not exist yet.
+ If a table already exists but its vector dimension differs from the
+ current embedder's output (e.g. the embedder was switched from
+ ``mock`` (dim 8) to ``openai`` (dim 1536) between reindexes), the stale
+ table is dropped and recreated. Otherwise every write would fail deep
+ inside LanceDB with an opaque IO error that never mentions dimensions.
+ """
try:
import pyarrow as pa # type: ignore[import]
except ImportError as exc:
@@ -61,6 +82,15 @@ class LanceDBVectorStore(VectorStore):
) from exc
dim = len(sample_vector)
+
+ if self._table is not None:
+ existing_dim = self._existing_vector_dim(await self._table.schema())
+ if existing_dim is None or existing_dim == dim:
+ return
+ # Embedder changed dimensions — the old vectors are unusable.
+ await self._db.drop_table(self._table_name) # type: ignore[union-attr]
+ self._table = None
+
schema = pa.schema(
[
pa.field("page_id", pa.string()),
diff --git a/tests/unit/persistence/test_vector_store.py b/tests/unit/persistence/test_vector_store.py
index 17a8620f2..6c42cc147 100644
--- a/tests/unit/persistence/test_vector_store.py
+++ b/tests/unit/persistence/test_vector_store.py
@@ -354,3 +354,56 @@ async def test_lancedb_embed_batch(tmp_path, mock_embedder):
assert ids == {"p1", "p2"}
finally:
await store.close()
+
+
+class _FixedDimEmbedder:
+ """Deterministic embedder with a configurable output dimension."""
+
+ def __init__(self, dimensions: int) -> None:
+ self.dimensions = dimensions
+
+ async def embed(self, texts: list[str]) -> list[list[float]]:
+ out: list[list[float]] = []
+ for i, _ in enumerate(texts):
+ vec = [0.0] * self.dimensions
+ vec[i % self.dimensions] = 1.0 # unit-length
+ out.append(vec)
+ return out
+
+
+async def test_lancedb_reindex_recreates_table_on_dimension_change(tmp_path, mock_embedder):
+ """Switching embedders (mock dim 8 → dim 1536) must recreate the table.
+
+ Regression test for #234: the stale wiki_pages table kept the old vector
+ schema, so every write failed with an opaque LanceDB IO error instead of
+ being transparently rebuilt for the new embedder.
+ """
+ lancedb = pytest.importorskip("lancedb") # noqa: F841
+ from repowise.core.persistence.vector_store import LanceDBVectorStore
+
+ db_path = str(tmp_path / "lance")
+
+ # First index with the 8-dim mock embedder.
+ store = LanceDBVectorStore(db_path, mock_embedder)
+ try:
+ await store.embed_and_upsert(
+ "p1", "old content", {"target_path": "a.py", "content": "old content"}
+ )
+ finally:
+ await store.close()
+
+ # Reindex against the same path with a different-dimension embedder.
+ big_embedder = _FixedDimEmbedder(1536)
+ store = LanceDBVectorStore(db_path, big_embedder)
+ try:
+ await store.embed_and_upsert(
+ "p2", "new content", {"target_path": "b.py", "content": "new content"}
+ )
+ # The old 8-dim row is gone; the table now holds only the new write.
+ ids = await store.list_page_ids()
+ assert ids == {"p2"}
+ results = await store.search("new content", limit=5)
+ assert {r.page_id for r in results} == {"p2"}
+ finally:
+ await store.close()